From ac458ffa7f4eeb25c60394f4f704a8e999da21cf Mon Sep 17 00:00:00 2001 From: Antigravity Date: Sun, 4 Oct 2026 13:37:30 +0000 Subject: [PATCH] feat: publish Dever core with project scaffolding and versioned AI skills --- .gitattributes | 9 + .gitignore | 8 + .gitmodules | 3 + .trellis/.gitignore | 32 + .trellis/.template-hashes.json | 92 + .trellis/.version | 1 + .trellis/agents/check.md | 70 + .trellis/agents/implement.md | 71 + .trellis/config.yaml | 161 + .trellis/scripts/__init__.py | 5 + .trellis/scripts/add_session.py | 1545 +++ .trellis/scripts/common/__init__.py | 94 + .trellis/scripts/common/active_task.py | 857 ++ .trellis/scripts/common/cli_adapter.py | 950 ++ .trellis/scripts/common/config.py | 482 + .trellis/scripts/common/developer.py | 192 + .trellis/scripts/common/git.py | 203 + .trellis/scripts/common/git_context.py | 106 + .trellis/scripts/common/io.py | 160 + .trellis/scripts/common/log.py | 45 + .trellis/scripts/common/packages_context.py | 238 + .trellis/scripts/common/paths.py | 543 + .trellis/scripts/common/safe_commit.py | 322 + .trellis/scripts/common/session_context.py | 892 ++ .trellis/scripts/common/task_context.py | 470 + .trellis/scripts/common/task_queue.py | 188 + .trellis/scripts/common/task_store.py | 1874 +++ .trellis/scripts/common/task_utils.py | 498 + .trellis/scripts/common/tasks.py | 122 + .trellis/scripts/common/trellis_config.py | 239 + .trellis/scripts/common/types.py | 110 + .trellis/scripts/common/workflow_phase.py | 242 + .trellis/scripts/get_context.py | 16 + .trellis/scripts/get_developer.py | 26 + .trellis/scripts/hooks/linear_sync.py | 243 + .trellis/scripts/init_developer.py | 51 + .trellis/scripts/task.py | 801 ++ .trellis/spec/backend/compiler-contracts.md | 194 + .trellis/spec/backend/database-guidelines.md | 235 + .trellis/spec/backend/directory-structure.md | 47 + .trellis/spec/backend/error-handling.md | 88 + .trellis/spec/backend/index.md | 23 + .trellis/spec/backend/logging-guidelines.md | 46 + .trellis/spec/backend/quality-guidelines.md | 120 + .../spec/backend/toolchain-and-library.md | 417 + .../spec/guides/code-reuse-thinking-guide.md | 223 + .../spec/guides/cross-layer-thinking-guide.md | 327 + .trellis/spec/guides/index.md | 97 + .../tasks/09-04-dever-language-mvp/design.md | 793 ++ .../09-04-dever-language-mvp/implement.md | 819 ++ .../tasks/09-04-dever-language-mvp/prd.md | 990 ++ .../research/package-distribution.md | 51 + .../research/runtime-dependencies.md | 55 + .../tasks/09-04-dever-language-mvp/task.json | 38 + .../check.jsonl | 0 .../implement.jsonl | 0 .../prd.md | 72 + .../research/cross-compilation.md | 35 + .../research/distribution-licensing.md | 23 + .../research/native-backend-options.md | 104 + .../task.json | 28 + .../tasks/09-12-http2-performance/check.jsonl | 3 + .../tasks/09-12-http2-performance/design.md | 84 + .../09-12-http2-performance/implement.jsonl | 3 + .../09-12-http2-performance/implement.md | 50 + .trellis/tasks/09-12-http2-performance/prd.md | 40 + .../research/current-benchmark-and-h2-api.md | 31 + .../tasks/09-12-http2-performance/task.json | 26 + .trellis/tasks/09-12-http2/check.jsonl | 4 + .trellis/tasks/09-12-http2/design.md | 21 + .trellis/tasks/09-12-http2/implement.jsonl | 5 + .trellis/tasks/09-12-http2/implement.md | 49 + .trellis/tasks/09-12-http2/prd.md | 16 + .trellis/tasks/09-12-http2/task.json | 26 + .../09-12-runtime-performance-p2/check.jsonl | 3 + .../09-12-runtime-performance-p2/design.md | 11 + .../implement.jsonl | 4 + .../09-12-runtime-performance-p2/implement.md | 21 + .../tasks/09-12-runtime-performance-p2/prd.md | 12 + .../09-12-runtime-performance-p2/task.json | 26 + .../09-12-runtime-performance/check.jsonl | 1 + .../tasks/09-12-runtime-performance/design.md | 9 + .../09-12-runtime-performance/implement.jsonl | 2 + .../09-12-runtime-performance/implement.md | 18 + .../tasks/09-12-runtime-performance/prd.md | 14 + .../tasks/09-12-runtime-performance/task.json | 26 + .trellis/tasks/09-13-typed-orm/check.jsonl | 7 + .trellis/tasks/09-13-typed-orm/design.md | 409 + .../tasks/09-13-typed-orm/implement.jsonl | 10 + .trellis/tasks/09-13-typed-orm/implement.md | 297 + .trellis/tasks/09-13-typed-orm/prd.md | 211 + .../research/architecture-options.md | 76 + .../research/driver-selection.md | 39 + .../09-13-typed-orm/research/field-types.md | 192 + .../research/go-orm-reference.md | 45 + .../research/implicit-suspension.md | 48 + .../research/migration-practices.md | 20 + .../09-13-typed-orm/research/query-api.md | 199 + .../research/schema-change-matrix.md | 90 + .trellis/tasks/09-13-typed-orm/task.json | 26 + .../09-19-application-testing/check.jsonl | 8 + .../tasks/09-19-application-testing/design.md | 143 + .../09-19-application-testing/implement.jsonl | 11 + .../09-19-application-testing/implement.md | 123 + .../tasks/09-19-application-testing/prd.md | 101 + .../tasks/09-19-application-testing/task.json | 28 + .trellis/tasks/09-20-durable-jobs/check.jsonl | 6 + .trellis/tasks/09-20-durable-jobs/design.md | 100 + .../tasks/09-20-durable-jobs/implement.jsonl | 5 + .../tasks/09-20-durable-jobs/implement.md | 127 + .trellis/tasks/09-20-durable-jobs/prd.md | 50 + .../research/implementation-map.md | 109 + .trellis/tasks/09-20-durable-jobs/task.json | 26 + .trellis/tasks/09-20-port-adapter/check.jsonl | 3 + .trellis/tasks/09-20-port-adapter/design.md | 47 + .../tasks/09-20-port-adapter/implement.jsonl | 3 + .../tasks/09-20-port-adapter/implement.md | 68 + .trellis/tasks/09-20-port-adapter/prd.md | 42 + .../research/implementation-map.md | 138 + .trellis/tasks/09-20-port-adapter/task.json | 26 + .../tasks/09-20-production-api/check.jsonl | 3 + .trellis/tasks/09-20-production-api/design.md | 130 + .../09-20-production-api/implement.jsonl | 3 + .../tasks/09-20-production-api/implement.md | 173 + .trellis/tasks/09-20-production-api/prd.md | 86 + .trellis/tasks/09-20-production-api/task.json | 26 + .../check.jsonl | 3 + .../design.md | 113 + .../implement.jsonl | 4 + .../implement.md | 90 + .../prd.md | 102 + .../task.json | 34 + .../09-20-secure-data-contracts/check.jsonl | 3 + .../09-20-secure-data-contracts/design.md | 29 + .../implement.jsonl | 4 + .../09-20-secure-data-contracts/implement.md | 71 + .../tasks/09-20-secure-data-contracts/prd.md | 37 + .../research/secure-contracts-source-map.md | 70 + .../09-20-secure-data-contracts/task.json | 26 + .../tasks/09-20-time-typed-codec/check.jsonl | 3 + .../tasks/09-20-time-typed-codec/design.md | 28 + .../09-20-time-typed-codec/implement.jsonl | 3 + .../tasks/09-20-time-typed-codec/implement.md | 64 + .trellis/tasks/09-20-time-typed-codec/prd.md | 37 + .../research/time-wire-source-map.md | 81 + .../tasks/09-20-time-typed-codec/task.json | 26 + .../tasks/09-28-shared-toolchain/check.jsonl | 2 + .../tasks/09-28-shared-toolchain/design.md | 86 + .../09-28-shared-toolchain/implement.jsonl | 2 + .../tasks/09-28-shared-toolchain/implement.md | 46 + .trellis/tasks/09-28-shared-toolchain/prd.md | 81 + .../tasks/09-28-shared-toolchain/task.json | 26 + .../09-29-build-cache-storage/check.jsonl | 4 + .../tasks/09-29-build-cache-storage/design.md | 76 + .../09-29-build-cache-storage/implement.jsonl | 4 + .../09-29-build-cache-storage/implement.md | 67 + .../tasks/09-29-build-cache-storage/prd.md | 74 + .../research/cache-boundaries.md | 14 + .../tasks/09-29-build-cache-storage/task.json | 26 + .../09-29-external-lib-ecosystems/check.jsonl | 3 + .../09-29-external-lib-ecosystems/design.md | 23 + .../implement.jsonl | 4 + .../implement.md | 42 + .../09-29-external-lib-ecosystems/prd.md | 37 + .../09-29-external-lib-ecosystems/task.json | 26 + .../09-29-external-lib-packaging/check.jsonl | 4 + .../09-29-external-lib-packaging/design.md | 21 + .../implement.jsonl | 4 + .../09-29-external-lib-packaging/implement.md | 49 + .../tasks/09-29-external-lib-packaging/prd.md | 36 + .../09-29-external-lib-packaging/task.json | 26 + .../09-29-external-lib-protocol/check.jsonl | 4 + .../09-29-external-lib-protocol/design.md | 23 + .../implement.jsonl | 4 + .../09-29-external-lib-protocol/implement.md | 40 + .../tasks/09-29-external-lib-protocol/prd.md | 32 + .../09-29-external-lib-protocol/task.json | 26 + .../tasks/09-29-external-libs/check.jsonl | 4 + .trellis/tasks/09-29-external-libs/design.md | 62 + .../tasks/09-29-external-libs/implement.jsonl | 5 + .../tasks/09-29-external-libs/implement.md | 33 + .trellis/tasks/09-29-external-libs/prd.md | 77 + .trellis/tasks/09-29-external-libs/task.json | 30 + .../check.jsonl | 12 + .../design.md | 39 + .../implement.jsonl | 14 + .../implement.md | 890 ++ .../09-29-language-production-closure/prd.md | 93 + .../research/apparmor-nonroot.md | 40 + .../research/ecosystem-closure.md | 125 + .../research/ecosystem-pack-inputs.md | 95 + .../research/linux-ci-scope.md | 147 + .../research/native-pack-production-inputs.md | 71 + .../task.json | 26 + .../10-04-developer-onboarding/check.jsonl | 1 + .../10-04-developer-onboarding/design.md | 9 + .../implement.jsonl | 1 + .../10-04-developer-onboarding/implement.md | 34 + .../tasks/10-04-developer-onboarding/prd.md | 16 + .../10-04-developer-onboarding/task.json | 26 + .../2026-09/00-bootstrap-guidelines/prd.md | 19 + .../2026-09/00-bootstrap-guidelines/task.json | 28 + .../2026-09/09-09-markdown-source/check.jsonl | 1 + .../2026-09/09-09-markdown-source/design.md | 17 + .../09-09-markdown-source/implement.jsonl | 1 + .../09-09-markdown-source/implement.md | 24 + .../2026-09/09-09-markdown-source/prd.md | 25 + .../research/source-flow.md | 8 + .../2026-09/09-09-markdown-source/task.json | 26 + .../09-13-http2-ceiling-soak/check.jsonl | 3 + .../09-13-http2-ceiling-soak/design.md | 70 + .../09-13-http2-ceiling-soak/implement.jsonl | 3 + .../09-13-http2-ceiling-soak/implement.md | 32 + .../2026-09/09-13-http2-ceiling-soak/prd.md | 42 + .../research/client-bottleneck.md | 27 + .../09-13-http2-ceiling-soak/task.json | 26 + .../check.jsonl | 3 + .../09-18-cms-dual-source-contracts/design.md | 26 + .../implement.jsonl | 3 + .../implement.md | 38 + .../09-18-cms-dual-source-contracts/prd.md | 18 + .../09-18-cms-dual-source-contracts/task.json | 26 + .../09-20-large-cms-acceptance/check.jsonl | 3 + .../09-20-large-cms-acceptance/design.md | 24 + .../implement.jsonl | 4 + .../09-20-large-cms-acceptance/implement.md | 40 + .../2026-10/09-20-large-cms-acceptance/prd.md | 25 + .../09-20-large-cms-acceptance/task.json | 26 + .../10-04-external-command/check.jsonl | 3 + .../2026-10/10-04-external-command/design.md | 36 + .../10-04-external-command/implement.jsonl | 4 + .../10-04-external-command/implement.md | 56 + .../2026-10/10-04-external-command/prd.md | 33 + .../2026-10/10-04-external-command/task.json | 26 + .../10-04-linux-arm64-cross-build/check.jsonl | 2 + .../10-04-linux-arm64-cross-build/design.md | 25 + .../implement.jsonl | 4 + .../implement.md | 61 + .../10-04-linux-arm64-cross-build/prd.md | 33 + .../10-04-linux-arm64-cross-build/task.json | 26 + .trellis/workflow.md | 725 ++ AGENTS.md | 31 + Cargo.lock | 2664 ++++ Cargo.toml | 63 + IMPLEMENTATION.md | 623 + LANGUAGE.md | 1519 +++ MARKDOWN-SYNTAX.md | 145 + README.md | 164 + config/setting.example.json | 24 + crates/dever-backend-bridge/Cargo.toml | 125 + crates/dever-backend-bridge/build.rs | 50 + crates/dever-backend-bridge/include/runtime.h | 909 ++ crates/dever-backend-bridge/src/bridge.cpp | 154 + crates/dever-backend-bridge/src/ffi.rs | 10726 ++++++++++++++++ crates/dever-backend-bridge/src/lib.rs | 310 + crates/dever-cli/Cargo.toml | 99 + crates/dever-cli/build.rs | 11 + crates/dever-cli/src/api.rs | 48 + crates/dever-cli/src/bin/dever-launcher.rs | 44 + crates/dever-cli/src/bin/deverd.rs | 34 + crates/dever-cli/src/compile.rs | 213 + crates/dever-cli/src/format.rs | 153 + crates/dever-cli/src/lib.rs | 5 + crates/dever-cli/src/libs.rs | 2620 ++++ crates/dever-cli/src/libs/build.rs | 526 + crates/dever-cli/src/libs/build/npm.js | 103 + crates/dever-cli/src/libs/build/npm.rs | 230 + .../src/libs/build/npm/environment.rs | 326 + crates/dever-cli/src/libs/build/npm/probe.cjs | 9 + crates/dever-cli/src/libs/build/pack.rs | 166 + crates/dever-cli/src/libs/build/process.rs | 186 + .../src/libs/build/process/budget.rs | 68 + crates/dever-cli/src/libs/build/python.py | 119 + crates/dever-cli/src/libs/build/python.rs | 564 + crates/dever-cli/src/libs/mutation.rs | 80 + crates/dever-cli/src/libs/registry.rs | 1619 +++ crates/dever-cli/src/libs/registry/npm.rs | 995 ++ .../src/libs/registry/npm/resolve.rs | 436 + crates/dever-cli/src/libs/registry/sumdb.rs | 524 + .../src/libs/registry/sumdb/merkle.rs | 260 + crates/dever-cli/src/main.rs | 754 ++ crates/dever-cli/src/packages.rs | 1220 ++ crates/dever-cli/src/project.rs | 128 + crates/dever-cli/src/test_runner.rs | 203 + crates/dever-cli/src/toolchain/bootstrap.rs | 434 + .../src/toolchain/bootstrap/sandbox.rs | 138 + .../src/toolchain/bootstrap/service.rs | 174 + .../src/toolchain/bootstrap/transaction.rs | 522 + crates/dever-cli/src/toolchain/cache.rs | 778 ++ crates/dever-cli/src/toolchain/compilation.rs | 326 + .../dever-cli/src/toolchain/core_libraries.rs | 212 + crates/dever-cli/src/toolchain/launcher.rs | 205 + crates/dever-cli/src/toolchain/mod.rs | 25 + crates/dever-cli/src/toolchain/packaging.rs | 520 + .../src/toolchain/packaging/bootstrap.rs | 64 + .../src/toolchain/packaging/builds.rs | 180 + .../src/toolchain/packaging/ecosystems.rs | 152 + .../src/toolchain/packaging/sandbox.rs | 43 + crates/dever-cli/src/toolchain/release.rs | 1194 ++ .../dever-cli/src/toolchain/release_source.rs | 387 + .../dever-cli/src/toolchain/runtime_pack.rs | 464 + crates/dever-cli/src/toolchain/service.rs | 732 ++ .../src/toolchain/service/compile.rs | 273 + .../src/toolchain/service/compile_request.rs | 87 + crates/dever-cli/src/toolchain/skill.rs | 91 + crates/dever-cli/src/toolchain/target.rs | 50 + crates/dever-cli/src/worker.rs | 97 + crates/dever-cli/src/workers.rs | 833 ++ crates/dever-cli/src/workers/command.rs | 239 + crates/dever-cli/src/workers/elf.rs | 176 + crates/dever-cli/src/workers/go_build.rs | 574 + crates/dever-cli/src/workers/python_wheel.rs | 588 + .../src/workers/python_wheel/native.rs | 154 + .../src/workers/python_wheel/scripts.rs | 88 + crates/dever-cli/src/workers/runtime.rs | 183 + crates/dever-core/Cargo.toml | 18 + crates/dever-core/src/api.rs | 304 + crates/dever-core/src/capture.rs | 95 + crates/dever-core/src/check.rs | 850 ++ crates/dever-core/src/check/api.rs | 1005 ++ crates/dever-core/src/check/auth.rs | 554 + crates/dever-core/src/check/body.rs | 648 + crates/dever-core/src/check/cache.rs | 63 + crates/dever-core/src/check/clauses.rs | 628 + crates/dever-core/src/check/collections.rs | 403 + crates/dever-core/src/check/components.rs | 123 + crates/dever-core/src/check/concurrency.rs | 469 + crates/dever-core/src/check/constants.rs | 148 + crates/dever-core/src/check/cycles.rs | 253 + crates/dever-core/src/check/expressions.rs | 890 ++ crates/dever-core/src/check/intrinsics.rs | 640 + crates/dever-core/src/check/jobs.rs | 256 + crates/dever-core/src/check/layout.rs | 131 + crates/dever-core/src/check/model.rs | 2378 ++++ crates/dever-core/src/check/names.rs | 188 + crates/dever-core/src/check/operators.rs | 184 + crates/dever-core/src/check/orm.rs | 944 ++ crates/dever-core/src/check/ports.rs | 1013 ++ crates/dever-core/src/check/sql_read.rs | 173 + crates/dever-core/src/check/symbols.rs | 694 + crates/dever-core/src/check/upload.rs | 147 + crates/dever-core/src/check/values.rs | 313 + crates/dever-core/src/contracts.rs | 89 + .../dever-core/src/contracts/dependencies.rs | 119 + crates/dever-core/src/contracts/effects.rs | 760 ++ .../dever-core/src/contracts/error_effects.rs | 268 + crates/dever-core/src/contracts/facts.rs | 815 ++ .../dever-core/src/contracts/facts/domain.rs | 188 + crates/dever-core/src/contracts/failures.rs | 692 + crates/dever-core/src/contracts/redundancy.rs | 486 + crates/dever-core/src/diagnostic.rs | 69 + crates/dever-core/src/format.rs | 820 ++ crates/dever-core/src/format/document.rs | 126 + crates/dever-core/src/format/markdown.rs | 48 + crates/dever-core/src/hir.rs | 908 ++ crates/dever-core/src/intrinsic.rs | 354 + crates/dever-core/src/lexer.rs | 276 + crates/dever-core/src/lib.rs | 103 + crates/dever-core/src/llvm.rs | 1517 +++ crates/dever-core/src/llvm/abi.rs | 319 + crates/dever-core/src/llvm/api.rs | 570 + crates/dever-core/src/llvm/api_context.rs | 326 + crates/dever-core/src/llvm/application.rs | 798 ++ .../src/llvm/application_commands.rs | 340 + crates/dever-core/src/llvm/asynchronous.rs | 677 + crates/dever-core/src/llvm/collections.rs | 367 + crates/dever-core/src/llvm/concurrency.rs | 292 + crates/dever-core/src/llvm/database.rs | 238 + crates/dever-core/src/llvm/database_rows.rs | 356 + crates/dever-core/src/llvm/database_schema.rs | 548 + crates/dever-core/src/llvm/expressions.rs | 984 ++ crates/dever-core/src/llvm/external.rs | 436 + crates/dever-core/src/llvm/failures.rs | 310 + crates/dever-core/src/llvm/http.rs | 227 + crates/dever-core/src/llvm/http_handlers.rs | 272 + crates/dever-core/src/llvm/http_values.rs | 549 + crates/dever-core/src/llvm/intrinsics.rs | 291 + crates/dever-core/src/llvm/jobs.rs | 390 + crates/dever-core/src/llvm/network.rs | 207 + crates/dever-core/src/llvm/orm.rs | 512 + crates/dever-core/src/llvm/orm_read.rs | 534 + crates/dever-core/src/llvm/orm_relations.rs | 470 + crates/dever-core/src/llvm/orm_stream.rs | 89 + crates/dever-core/src/llvm/ownership.rs | 388 + crates/dever-core/src/llvm/parallel.rs | 99 + crates/dever-core/src/llvm/ports.rs | 266 + crates/dever-core/src/llvm/process_entry.rs | 176 + crates/dever-core/src/llvm/render.rs | 644 + crates/dever-core/src/llvm/resources.rs | 296 + crates/dever-core/src/llvm/rest.rs | 931 ++ crates/dever-core/src/llvm/system.rs | 154 + crates/dever-core/src/llvm/testing.rs | 276 + crates/dever-core/src/llvm/transaction.rs | 190 + crates/dever-core/src/llvm/traversal.rs | 526 + crates/dever-core/src/llvm/wire.rs | 703 + crates/dever-core/src/markdown.rs | 187 + crates/dever-core/src/markdown/contract.rs | 36 + .../src/markdown/contract/metadata.rs | 339 + .../dever-core/src/markdown/contract/model.rs | 84 + .../dever-core/src/markdown/contract/parse.rs | 417 + .../src/markdown/contract/validate.rs | 593 + crates/dever-core/src/model.rs | 641 + crates/dever-core/src/native.rs | 1375 ++ crates/dever-core/src/native/api.rs | 532 + crates/dever-core/src/native/build.rs | 607 + crates/dever-core/src/native/build/cache.rs | 478 + crates/dever-core/src/native/build/cleanup.rs | 343 + crates/dever-core/src/native/collections.rs | 330 + crates/dever-core/src/native/expressions.rs | 494 + crates/dever-core/src/native/http.rs | 314 + crates/dever-core/src/native/intrinsics.rs | 409 + crates/dever-core/src/native/jobs.rs | 457 + crates/dever-core/src/native/liveness.rs | 140 + crates/dever-core/src/native/orm.rs | 2349 ++++ crates/dever-core/src/native/ports.rs | 368 + crates/dever-core/src/native/records.rs | 214 + crates/dever-core/src/native/rest.rs | 491 + crates/dever-core/src/native/sequences.rs | 273 + crates/dever-core/src/native/updates.rs | 250 + crates/dever-core/src/native/wire.rs | 135 + crates/dever-core/src/parser.rs | 1185 ++ crates/dever-core/src/parser/expression.rs | 253 + crates/dever-core/src/reference.rs | 254 + .../dever-core/src/reference/collections.rs | 176 + .../dever-core/src/reference/expressions.rs | 281 + crates/dever-core/src/reference/intrinsics.rs | 345 + crates/dever-core/src/reference/value.rs | 237 + crates/dever-core/src/source.rs | 726 ++ crates/dever-core/src/specialize.rs | 376 + crates/dever-core/src/syntax.rs | 450 + crates/dever-core/src/token.rs | 51 + crates/dever-core/src/types.rs | 417 + crates/dever-core/src/wire.rs | 262 + crates/dever-runtime/Cargo.toml | 49 + crates/dever-runtime/src/abi.rs | 49 + crates/dever-runtime/src/api.rs | 581 + crates/dever-runtime/src/api/input.rs | 264 + crates/dever-runtime/src/api/upload.rs | 580 + crates/dever-runtime/src/application.rs | 212 + crates/dever-runtime/src/async_stream.rs | 104 + crates/dever-runtime/src/auth.rs | 738 ++ crates/dever-runtime/src/auth/store.rs | 646 + crates/dever-runtime/src/bytes.rs | 80 + crates/dever-runtime/src/channel.rs | 117 + crates/dever-runtime/src/collections.rs | 175 + crates/dever-runtime/src/component.rs | 831 ++ crates/dever-runtime/src/component/command.rs | 342 + .../src/component/command/observation.rs | 135 + crates/dever-runtime/src/concurrent.rs | 133 + crates/dever-runtime/src/config.rs | 1467 +++ crates/dever-runtime/src/config/command.rs | 50 + .../dever-runtime/src/config/compilation.rs | 232 + crates/dever-runtime/src/cron.rs | 114 + crates/dever-runtime/src/crypto.rs | 74 + crates/dever-runtime/src/database.rs | 969 ++ crates/dever-runtime/src/endpoint.rs | 51 + crates/dever-runtime/src/external.rs | 701 + crates/dever-runtime/src/http/client.rs | 548 + crates/dever-runtime/src/http/client_body.rs | 167 + .../dever-runtime/src/http/client_driver.rs | 443 + crates/dever-runtime/src/http/executor.rs | 17 + crates/dever-runtime/src/http/live.rs | 289 + crates/dever-runtime/src/http/message.rs | 208 + crates/dever-runtime/src/http/mod.rs | 550 + crates/dever-runtime/src/http/protocol.rs | 97 + crates/dever-runtime/src/http/reply.rs | 456 + crates/dever-runtime/src/http/server_h2.rs | 110 + .../dever-runtime/src/http/write_timeout.rs | 96 + crates/dever-runtime/src/job.rs | 343 + crates/dever-runtime/src/job/resources.rs | 84 + crates/dever-runtime/src/job/store.rs | 556 + crates/dever-runtime/src/job/worker.rs | 470 + crates/dever-runtime/src/lib.rs | 75 + crates/dever-runtime/src/lifecycle.rs | 94 + crates/dever-runtime/src/log.rs | 359 + crates/dever-runtime/src/net.rs | 297 + crates/dever-runtime/src/number.rs | 208 + crates/dever-runtime/src/orm.rs | 695 + crates/dever-runtime/src/pool.rs | 34 + crates/dever-runtime/src/postgres.rs | 922 ++ .../dever-runtime/src/postgres/migration.rs | 660 + crates/dever-runtime/src/process.rs | 14 + crates/dever-runtime/src/render.rs | 165 + crates/dever-runtime/src/resource.rs | 178 + crates/dever-runtime/src/secret.rs | 21 + crates/dever-runtime/src/sqlite.rs | 578 + crates/dever-runtime/src/sqlite/migration.rs | 483 + crates/dever-runtime/src/sse.rs | 108 + crates/dever-runtime/src/task.rs | 1179 ++ crates/dever-runtime/src/task/scoped.rs | 270 + crates/dever-runtime/src/tenant.rs | 483 + crates/dever-runtime/src/text.rs | 58 + crates/dever-runtime/src/time.rs | 188 + crates/dever-runtime/src/tls.rs | 109 + crates/dever-runtime/src/transport.rs | 72 + crates/dever-runtime/src/websocket.rs | 328 + crates/dever-runtime/src/wire.rs | 348 + crates/dever-sandbox/Cargo.toml | 28 + crates/dever-sandbox/src/assets.rs | 182 + crates/dever-sandbox/src/filter.rs | 322 + crates/dever-sandbox/src/lib.rs | 321 + crates/dever-sandbox/src/main.rs | 79 + crates/dever-sandbox/src/mounts.rs | 87 + crates/dever-sandbox/src/observation.rs | 48 + crates/dever-sandbox/src/trusted_helper.rs | 130 + .../plans/2026-09-02-dever-language-mvp.md | 830 -- .../2026-09-02-dever-language-mvp-design.md | 333 - examples/README.md | 63 + examples/cms/dever/config/setting.json | 56 + examples/cms/dever/data/.gitkeep | 1 + .../dever/module/audit/operation/app.dever | 10 + .../dever/module/audit/operation/model.dever | 14 + .../content/category/api/admin/manage.dever | 3 + .../dever/module/content/category/app.dever | 41 + .../module/content/category/domain.dever | 23 + .../dever/module/content/category/model.dever | 10 + .../content/revision/api/admin/manage.dever | 1 + .../dever/module/content/revision/app.dever | 45 + .../dever/module/content/revision/model.dever | 16 + .../dever/module/media/asset/adapter.dever | 3 + .../module/media/asset/api/admin/manage.dever | 1 + .../cms/dever/module/media/asset/app.dever | 43 + .../cms/dever/module/media/asset/model.dever | 12 + .../cms/dever/module/media/asset/port.dever | 1 + .../news/article/api/admin/manage.dever | 9 + .../news/article/api/front/browse.dever | 3 + .../module/news/article/app/editing.dever | 114 + .../module/news/article/app/publication.dever | 95 + .../dever/module/news/article/domain.dever | 49 + .../cms/dever/module/news/article/model.dever | 25 + .../news/article/model/publication.dever | 6 + .../dever/module/platform/tenant/app.dever | 41 + .../dever/module/platform/tenant/domain.dever | 13 + .../dever/module/platform/tenant/model.dever | 11 + .../schedule/api/admin/manage.dever | 1 + .../module/publishing/schedule/app.dever | 76 + .../module/publishing/schedule/domain.dever | 5 + .../module/publishing/schedule/job.dever | 5 + .../module/publishing/schedule/model.dever | 12 + .../cms/dever/module/system/health/api.dever | 1 + .../cms/dever/module/system/health/app.dever | 9 + .../cms/dever/module/user/account/api.dever | 1 + .../user/account/api/admin/session.dever | 3 + .../user/account/api/front/session.dever | 3 + .../cms/dever/module/user/account/app.dever | 157 + .../dever/module/user/account/domain.dever | 49 + .../cms/dever/module/user/account/model.dever | 11 + .../module/user/account/model/principal.dever | 10 + .../user/authorization/api/admin/manage.dever | 9 + .../user/authorization/api/front/manage.dever | 9 + .../dever/module/user/authorization/app.dever | 25 + .../dever/module/user/membership/app.dever | 31 + .../dever/module/user/membership/model.dever | 19 + .../cms/dever/module/user/session/app.dever | 94 + .../cms/dever/module/user/session/job.dever | 7 + .../cms/dever/module/user/session/model.dever | 12 + .../test/content/category/lifecycle.dever | 13 + .../cms/dever/test/news/article/publish.dever | 11 + .../dever/test/user/account/bootstrap.dever | 16 + .../dever/test/user/account/validation.dever | 8 + .../dever/test/user/session/lifecycle.dever | 37 + examples/cms/md/config/setting.json | 56 + .../md/module/audit/operation/app.dever.md | 34 + .../md/module/audit/operation/model.dever.md | 52 + .../category/api/admin/manage.dever.md | 28 + .../md/module/content/category/app.dever.md | 105 + .../module/content/category/domain.dever.md | 89 + .../md/module/content/category/model.dever.md | 44 + .../revision/api/admin/manage.dever.md | 18 + .../md/module/content/revision/app.dever.md | 89 + .../md/module/content/revision/model.dever.md | 62 + .../md/module/media/asset/adapter.dever.md | 24 + .../media/asset/api/admin/manage.dever.md | 18 + .../cms/md/module/media/asset/app.dever.md | 107 + .../cms/md/module/media/asset/model.dever.md | 48 + .../cms/md/module/media/asset/port.dever.md | 23 + .../news/article/api/admin/manage.dever.md | 58 + .../news/article/api/front/browse.dever.md | 28 + .../module/news/article/app/editing.dever.md | 244 + .../news/article/app/publication.dever.md | 203 + .../md/module/news/article/domain.dever.md | 162 + .../cms/md/module/news/article/model.dever.md | 86 + .../news/article/model/publication.dever.md | 29 + .../md/module/platform/tenant/app.dever.md | 106 + .../md/module/platform/tenant/domain.dever.md | 57 + .../md/module/platform/tenant/model.dever.md | 46 + .../schedule/api/admin/manage.dever.md | 18 + .../module/publishing/schedule/app.dever.md | 166 + .../publishing/schedule/domain.dever.md | 25 + .../module/publishing/schedule/job.dever.md | 33 + .../module/publishing/schedule/model.dever.md | 48 + .../cms/md/module/system/health/api.dever.md | 18 + .../cms/md/module/system/health/app.dever.md | 42 + .../cms/md/module/user/account/api.dever.md | 18 + .../user/account/api/admin/session.dever.md | 28 + .../user/account/api/front/session.dever.md | 28 + .../cms/md/module/user/account/app.dever.md | 350 + .../md/module/user/account/domain.dever.md | 141 + .../cms/md/module/user/account/model.dever.md | 46 + .../user/account/model/principal.dever.md | 44 + .../authorization/api/admin/manage.dever.md | 58 + .../authorization/api/front/manage.dever.md | 58 + .../md/module/user/authorization/app.dever.md | 108 + .../md/module/user/membership/app.dever.md | 81 + .../md/module/user/membership/model.dever.md | 75 + .../cms/md/module/user/session/app.dever.md | 213 + .../cms/md/module/user/session/job.dever.md | 42 + .../cms/md/module/user/session/model.dever.md | 47 + .../test/content/category/lifecycle.dever.md | 32 + .../cms/md/test/news/article/publish.dever.md | 30 + .../md/test/user/account/bootstrap.dever.md | 35 + .../md/test/user/account/validation.dever.md | 27 + .../md/test/user/session/lifecycle.dever.md | 56 + examples/old/dever/async/config/setting.json | 10 + examples/old/dever/async/data/.gitkeep | 1 + examples/old/dever/async/module/main.dever | 28 + .../old/dever/async_tcp/config/setting.json | 10 + examples/old/dever/async_tcp/data/.gitkeep | 1 + .../old/dever/async_tcp/module/main.dever | 24 + .../old/dever/backend/config/setting.json | 10 + examples/old/dever/backend/data/.gitkeep | 1 + examples/old/dever/backend/module/main.dever | 69 + .../old/dever/contracts/config/setting.json | 10 + examples/old/dever/contracts/data/.gitkeep | 1 + examples/old/dever/contracts/dever.api | 4 + .../old/dever/contracts/module/domain.dever | 19 + .../old/dever/contracts/module/main.dever | 18 + examples/old/dever/core/config/setting.json | 10 + examples/old/dever/core/data/.gitkeep | 1 + examples/old/dever/core/module/main.dever | 134 + examples/old/dever/hello/config/setting.json | 10 + examples/old/dever/hello/data/.gitkeep | 1 + .../old/dever/hello/module/app/user.dever | 3 + examples/old/dever/hello/module/main.dever | 3 + examples/old/dever/http/config/setting.json | 10 + examples/old/dever/http/data/.gitkeep | 1 + examples/old/dever/http/module/main.dever | 19 + .../old/dever/library/config/setting.json | 10 + examples/old/dever/library/data/.gitkeep | 1 + examples/old/dever/library/module/main.dever | 11 + .../old/dever/live_http/config/setting.json | 10 + examples/old/dever/live_http/data/.gitkeep | 1 + .../old/dever/live_http/module/main.dever | 107 + .../old/dever/parallel/config/setting.json | 10 + examples/old/dever/parallel/data/.gitkeep | 1 + examples/old/dever/parallel/module/main.dever | 10 + .../old/dever/privacy/config/setting.json | 10 + examples/old/dever/privacy/data/.gitkeep | 1 + .../old/dever/privacy/module/counter.dever | 20 + examples/old/dever/privacy/module/main.dever | 7 + .../dever/rejected/coverage/module/main.dever | 9 + .../dever/rejected/failure/module/main.dever | 9 + .../dever/rejected/private/module/main.dever | 5 + .../rejected/private/module/secret.dever | 4 + .../old/dever/rejected/pure/module/main.dever | 5 + .../dever/rejected/range/module/main.dever | 11 + .../old/dever/sequences/config/setting.json | 10 + examples/old/dever/sequences/data/.gitkeep | 1 + .../old/dever/sequences/module/main.dever | 58 + examples/old/dever/stream/config/setting.json | 10 + examples/old/dever/stream/data/.gitkeep | 1 + examples/old/dever/stream/input.txt | 1 + examples/old/dever/stream/module/main.dever | 21 + examples/old/dever/system/config/setting.json | 10 + examples/old/dever/system/data/.gitkeep | 1 + examples/old/dever/system/module/main.dever | 11 + examples/old/dever/text/config/setting.json | 10 + examples/old/dever/text/data/.gitkeep | 1 + examples/old/dever/text/module/main.dever | 29 + examples/old/markdown/config/setting.json | 10 + examples/old/markdown/data/.gitkeep | 1 + examples/old/markdown/module/main.dever.md | 150 + library/decimal.dever | 15 + library/dever/api.dever | 84 + library/dever/auth.dever | 68 + library/dever/bytes.dever | 53 + library/dever/crypto.dever | 62 + library/dever/database.dever | 13 + library/dever/http.dever | 262 + library/dever/id.dever | 7 + library/dever/io.dever | 108 + library/dever/job.dever | 3 + library/dever/json.dever | 76 + library/dever/json/number.dever | 149 + library/dever/json/parse.dever | 388 + library/dever/json/scan.dever | 442 + library/dever/json/value.dever | 29 + library/dever/json/write.dever | 269 + library/dever/log.dever | 15 + library/dever/net.dever | 134 + library/dever/process.dever | 26 + library/dever/site.dever | 3 + library/dever/sse.dever | 21 + library/dever/storage.dever | 16 + library/dever/task.dever | 14 + library/dever/time.dever | 144 + library/dever/tls.dever | 39 + library/dever/websocket.dever | 111 + library/float.dever | 11 + library/int.dever | 7 + library/math.dever | 31 + library/text.dever | 89 + library/uuid.dever | 7 + sdk/README.md | 27 + sdk/apparmor/dever-sandbox | 45 + sdk/go/component.go | 699 + sdk/go/example/main.go | 102 + sdk/go/go.mod | 3 + sdk/javascript/dever_component.mjs | 368 + sdk/javascript/example.mjs | 29 + sdk/native-release.md | 269 + sdk/native-release.rs | 27 + sdk/python/dever_component.py | 404 + sdk/python/example.py | 55 + sdk/release-assets.py | 96 + skills | 1 + test/backend_bridge.rs | 345 + test/ci/README.md | 92 + test/ci/acceptance.py | 298 + test/ci/ignored.json | 1738 +++ test/ci/run.py | 372 + test/ci/test_driver.py | 41 + test/ci/test_gate.py | 317 + test/cms/README.md | 13 + test/cms/acceptance.py | 476 + test/dever-cli-tests/build_arguments.rs | 98 + test/dever-cli-tests/cms_project.rs | 451 + test/dever-cli-tests/external_libs.rs | 1916 +++ test/dever-cli-tests/external_libs/build.rs | 468 + test/dever-cli-tests/external_libs/npm.rs | 691 + .../external_libs/npm_build.rs | 590 + test/dever-cli-tests/external_libs/sumdb.rs | 754 ++ .../external_libs/sumdb_uuid.json | 68 + test/dever-cli-tests/external_libs/wheel.rs | 531 + test/dever-cli-tests/external_workers.rs | 1299 ++ .../external_workers/command.rs | 204 + test/dever-cli-tests/external_workers/npm.rs | 119 + test/dever-cli-tests/llvm_cli.rs | 642 + test/dever-cli-tests/native_release.rs | 542 + .../native_release/acceptance.rs | 567 + .../native_release/bootstrap.rs | 410 + .../native_release/bootstrap_acceptance.rs | 476 + .../native_release/command_acceptance.rs | 205 + .../native_release/cross_acceptance.rs | 291 + .../native_release/cross_ecosystems.rs | 348 + .../native_release/ecosystem_acceptance.rs | 398 + .../native_release/ecosystems.rs | 429 + .../dever-cli-tests/native_release/targets.rs | 178 + test/dever-cli-tests/new_project.rs | 142 + test/dever-cli-tests/packages.rs | 961 ++ .../packages/managed_compile.rs | 456 + test/dever-cli-tests/release_source.rs | 289 + test/dever-cli-tests/shared_toolchain.rs | 1375 ++ .../dever-cli-tests/support/command_library.c | 2 + test/dever-cli-tests/support/command_tool.c | 19 + .../support/compiler_sleeper.c | 18 + test/dever-cli-tests/support/daemon.rs | 42 + test/dever-cli-tests/support/llvm_inputs.rs | 67 + test/dever-cli-tests/support/llvm_pack.rs | 128 + .../support/native_acceptance_init.rs | 41 + test/dever-cli-tests/support/native_elf.rs | 90 + .../dever-cli-tests/support/native_objects.rs | 29 + test/dever-cli-tests/support/wheel.rs | 81 + .../dever-sandbox-tests/apparmor_transition.c | 109 + test/dever-sandbox-tests/isolation.rs | 538 + test/dever-sandbox-tests/syscalls.c | 60 + test/dever-sandbox-tests/trusted_helper.rs | 140 + test/dever-tests/Cargo.toml | 363 + test/dever-tests/examples/async_bench.rs | 83 + test/dever-tests/examples/live_bench.rs | 33 + .../dever-tests/examples/live_bench/config.rs | 125 + .../examples/live_bench/protocol.rs | 325 + test/dever-tests/examples/live_bench/run.rs | 284 + test/dever-tests/examples/network_bench.rs | 58 + .../examples/network_bench/config.rs | 227 + .../examples/network_bench/load.rs | 767 ++ .../examples/network_bench/metrics.rs | 67 + .../examples/network_bench/response.rs | 117 + .../examples/network_bench/service.rs | 369 + .../dever-tests/examples/network_bench/tls.rs | 128 + .../examples/support/benchmark_settings.rs | 80 + test/dever-tests/fixtures/command_fixture.rs | 101 + .../dever-tests/fixtures/component_fixture.rs | 225 + .../benchmark/allocation/app.dever | 95 + .../fixtures/performance/main.dever | 3 + .../fixtures/postgres_api/account_api.dever | 1 + .../fixtures/postgres_api/account_app.dever | 27 + .../postgres_api/membership_app.dever | 13 + test/dever-tests/fixtures/tls/root.pem | 11 + test/dever-tests/fixtures/tls/server-key.pem | 5 + test/dever-tests/fixtures/tls/server.pem | 10 + test/dever-tests/tests/api_context.rs | 250 + test/dever-tests/tests/api_declarations.rs | 933 ++ test/dever-tests/tests/api_routes.rs | 241 + test/dever-tests/tests/application_config.rs | 426 + test/dever-tests/tests/application_testing.rs | 411 + test/dever-tests/tests/async_composition.rs | 201 + test/dever-tests/tests/async_native.rs | 148 + test/dever-tests/tests/async_network.rs | 265 + test/dever-tests/tests/async_runtime.rs | 295 + test/dever-tests/tests/async_stream.rs | 164 + test/dever-tests/tests/auth_runtime.rs | 249 + .../tests/authorization_postgres.rs | 59 + .../tests/authorization_runtime.rs | 52 + test/dever-tests/tests/cms_acceptance.rs | 27 + test/dever-tests/tests/concurrency.rs | 170 + test/dever-tests/tests/contract_api.rs | 226 + test/dever-tests/tests/contract_execution.rs | 291 + test/dever-tests/tests/contract_facts.rs | 249 + test/dever-tests/tests/contract_redundancy.rs | 101 + test/dever-tests/tests/contract_semantics.rs | 236 + test/dever-tests/tests/contract_syntax.rs | 168 + test/dever-tests/tests/core_native.rs | 446 + test/dever-tests/tests/core_semantics.rs | 398 + test/dever-tests/tests/differential.rs | 353 + test/dever-tests/tests/durable_jobs.rs | 1573 +++ test/dever-tests/tests/error_effects.rs | 176 + test/dever-tests/tests/external_command.rs | 452 + test/dever-tests/tests/external_component.rs | 326 + test/dever-tests/tests/external_resources.rs | 494 + test/dever-tests/tests/formatter.rs | 640 + test/dever-tests/tests/hello_native.rs | 356 + test/dever-tests/tests/http2_client.rs | 566 + test/dever-tests/tests/http2_server.rs | 706 + test/dever-tests/tests/http_engine.rs | 621 + test/dever-tests/tests/http_library.rs | 246 + test/dever-tests/tests/json_library.rs | 265 + test/dever-tests/tests/live_library.rs | 103 + test/dever-tests/tests/live_network.rs | 767 ++ test/dever-tests/tests/logging.rs | 14 + test/dever-tests/tests/markdown_source.rs | 938 ++ test/dever-tests/tests/model_syntax.rs | 386 + test/dever-tests/tests/native_cache.rs | 593 + test/dever-tests/tests/native_entry.rs | 793 ++ test/dever-tests/tests/native_ownership.rs | 309 + test/dever-tests/tests/orm_connections.rs | 448 + test/dever-tests/tests/orm_data_migration.rs | 442 + test/dever-tests/tests/orm_migration.rs | 570 + test/dever-tests/tests/orm_native_sql.rs | 432 + .../tests/orm_performance_fixture.rs | 183 + test/dever-tests/tests/orm_query.rs | 549 + test/dever-tests/tests/orm_relations.rs | 380 + test/dever-tests/tests/orm_seed.rs | 212 + test/dever-tests/tests/performance.rs | 288 + .../tests/performance_port_fixture.rs | 99 + test/dever-tests/tests/pooled_library.rs | 260 + test/dever-tests/tests/pooled_network.rs | 691 + test/dever-tests/tests/port_adapter.rs | 984 ++ test/dever-tests/tests/postgres_api.rs | 305 + test/dever-tests/tests/postgres_orm.rs | 260 + test/dever-tests/tests/rest_model.rs | 478 + test/dever-tests/tests/runtime_foundations.rs | 169 + test/dever-tests/tests/secure_contracts.rs | 576 + test/dever-tests/tests/source_architecture.rs | 350 + test/dever-tests/tests/source_visibility.rs | 277 + test/dever-tests/tests/sqlite_orm.rs | 175 + test/dever-tests/tests/sqlite_runtime.rs | 314 + test/dever-tests/tests/standard_library.rs | 124 + .../tests/structured_concurrency.rs | 351 + .../tests/support/authorization.rs | 338 + test/dever-tests/tests/support/llvm.rs | 17 + .../dever-tests/tests/support/llvm_managed.rs | 68 + test/dever-tests/tests/support/llvm_module.rs | 175 + test/dever-tests/tests/support/mod.rs | 73 + test/dever-tests/tests/support/orm_fixture.rs | 352 + .../tests/support/orm_postgres_migration.rs | 163 + test/dever-tests/tests/support/orm_sqlite.rs | 232 + test/dever-tests/tests/support/postgres.rs | 520 + test/dever-tests/tests/support/process.rs | 40 + test/dever-tests/tests/support/sandbox.rs | 40 + .../tests/support/sqlite_inspect.rs | 32 + test/dever-tests/tests/support/temp.rs | 44 + test/dever-tests/tests/support/wire_peer.rs | 115 + test/dever-tests/tests/syntax_and_format.rs | 501 + test/dever-tests/tests/tenant_database.rs | 261 + test/dever-tests/tests/time_codec.rs | 489 + test/dever-tests/tests/upload_boundary.rs | 337 + test/ecosystem-release/arm64/bootstrap.sh | 8 + test/ecosystem-release/arm64/guest.sh | 65 + test/ecosystem-release/arm64/init.sh | 27 + test/ecosystem-release/arm64/probe.go | 49 + test/ecosystem-release/arm64/probe.mjs | 27 + test/ecosystem-release/arm64/probe.py | 31 + test/ecosystem-release/arm64/simulate.py | 88 + test/ecosystem-release/build_inputs.py | 142 + test/ecosystem-release/prepare.py | 243 + test/ecosystem-release/probe.go | 40 + test/ecosystem-release/probe.mjs | 25 + test/ecosystem-release/probe.py | 38 + test/ecosystem-release/test_build_inputs.py | 23 + test/external-sdk/check.py | 282 + test/external-sdk/typed_contract.json | 19 + test/external-sdk/typed_worker.go | 41 + test/external-sdk/typed_worker.mjs | 12 + test/external-sdk/typed_worker.py | 17 + test/go-managed/analyze/main.go | 89 + test/go-managed/make_pack.go | 212 + test/llvm_api_security_cases.rs | 644 + test/llvm_api_source.rs | 895 ++ test/llvm_application_source.rs | 468 + test/llvm_async_native.rs | 280 + test/llvm_async_source.rs | 639 + test/llvm_database_source.rs | 967 ++ test/llvm_external_source.rs | 609 + test/llvm_http_system_source.rs | 702 + test/llvm_job_security_cases.rs | 252 + test/llvm_job_test_source.rs | 621 + test/llvm_lowering.rs | 166 + test/llvm_managed_native.rs | 266 + test/llvm_native.rs | 308 + test/llvm_process_entry_source.rs | 107 + test/llvm_resource_native.rs | 305 + test/llvm_result_native.rs | 388 + test/llvm_stream_network_source.rs | 596 + test/native-runtime-abi/async-driver.c | 23 + .../async-operations-driver.c | 470 + test/native-runtime-abi/database-driver.c | 245 + test/native-runtime-abi/driver-fixture.c | 42 + test/native-runtime-abi/driver.c | 251 + test/native-runtime-abi/log-driver.c | 8 + test/native-runtime-abi/managed-driver.c | 116 + test/native-runtime-abi/protocol-driver.c | 183 + test/native-runtime-abi/resource-driver.c | 204 + test/native-runtime-abi/test-suite-driver.c | 64 + test/native-runtime-abi/wire-driver.c | 232 + test/native_runtime_abi.rs | 89 + test/native_runtime_link.rs | 208 + test/onboarding/test_installer.py | 259 + test/performance/README.md | 349 + test/performance/build.py | 466 + test/performance/cms.py | 675 + test/performance/fixtures/bench.dever | 21 + test/performance/fixtures/http/adapter.dever | 136 + test/performance/fixtures/http/api.dever | 7 + test/performance/fixtures/http/app.dever | 35 + test/performance/fixtures/http/port.dever | 9 + test/performance/fixtures/live/adapter.dever | 213 + test/performance/fixtures/live/api.dever | 5 + test/performance/fixtures/live/app.dever | 18 + test/performance/fixtures/live/port.dever | 3 + test/performance/fixtures/orm/app.dever | 144 + test/performance/fixtures/orm/app/api.dever | 9 + .../fixtures/orm/app/model/item.dever | 13 + .../profile/benchmark/command/api.dever | 1 + .../profile/benchmark/command/app.dever | 4 + .../fixtures/runtime/adapter.dever | 7 + test/performance/fixtures/runtime/api.dever | 11 + test/performance/fixtures/runtime/app.dever | 132 + test/performance/fixtures/runtime/port.dever | 3 + test/performance/live.py | 97 + test/performance/process.py | 231 + test/performance/run.py | 1244 ++ test/performance/settings.py | 59 + test/performance/test_cms.py | 255 + test/performance/test_live_bench.py | 122 + test/performance/test_network_bench.py | 120 + test/performance/test_peer_settings.py | 47 + test/performance/test_runner.py | 803 ++ test/runtime_entry_cleanup.rs | 134 + test/runtime_logging.rs | 170 + 960 files changed, 221319 insertions(+), 1163 deletions(-) create mode 100644 .gitattributes create mode 100644 .gitignore create mode 100644 .gitmodules create mode 100644 .trellis/.gitignore create mode 100644 .trellis/.template-hashes.json create mode 100644 .trellis/.version create mode 100644 .trellis/agents/check.md create mode 100644 .trellis/agents/implement.md create mode 100644 .trellis/config.yaml create mode 100755 .trellis/scripts/__init__.py create mode 100755 .trellis/scripts/add_session.py create mode 100755 .trellis/scripts/common/__init__.py create mode 100755 .trellis/scripts/common/active_task.py create mode 100755 .trellis/scripts/common/cli_adapter.py create mode 100755 .trellis/scripts/common/config.py create mode 100755 .trellis/scripts/common/developer.py create mode 100755 .trellis/scripts/common/git.py create mode 100755 .trellis/scripts/common/git_context.py create mode 100755 .trellis/scripts/common/io.py create mode 100755 .trellis/scripts/common/log.py create mode 100755 .trellis/scripts/common/packages_context.py create mode 100755 .trellis/scripts/common/paths.py create mode 100755 .trellis/scripts/common/safe_commit.py create mode 100755 .trellis/scripts/common/session_context.py create mode 100755 .trellis/scripts/common/task_context.py create mode 100755 .trellis/scripts/common/task_queue.py create mode 100755 .trellis/scripts/common/task_store.py create mode 100755 .trellis/scripts/common/task_utils.py create mode 100755 .trellis/scripts/common/tasks.py create mode 100755 .trellis/scripts/common/trellis_config.py create mode 100755 .trellis/scripts/common/types.py create mode 100755 .trellis/scripts/common/workflow_phase.py create mode 100755 .trellis/scripts/get_context.py create mode 100755 .trellis/scripts/get_developer.py create mode 100755 .trellis/scripts/hooks/linear_sync.py create mode 100755 .trellis/scripts/init_developer.py create mode 100755 .trellis/scripts/task.py create mode 100644 .trellis/spec/backend/compiler-contracts.md create mode 100644 .trellis/spec/backend/database-guidelines.md create mode 100644 .trellis/spec/backend/directory-structure.md create mode 100644 .trellis/spec/backend/error-handling.md create mode 100644 .trellis/spec/backend/index.md create mode 100644 .trellis/spec/backend/logging-guidelines.md create mode 100644 .trellis/spec/backend/quality-guidelines.md create mode 100644 .trellis/spec/backend/toolchain-and-library.md create mode 100644 .trellis/spec/guides/code-reuse-thinking-guide.md create mode 100644 .trellis/spec/guides/cross-layer-thinking-guide.md create mode 100644 .trellis/spec/guides/index.md create mode 100644 .trellis/tasks/09-04-dever-language-mvp/design.md create mode 100644 .trellis/tasks/09-04-dever-language-mvp/implement.md create mode 100644 .trellis/tasks/09-04-dever-language-mvp/prd.md create mode 100644 .trellis/tasks/09-04-dever-language-mvp/research/package-distribution.md create mode 100644 .trellis/tasks/09-04-dever-language-mvp/research/runtime-dependencies.md create mode 100644 .trellis/tasks/09-04-dever-language-mvp/task.json create mode 100644 .trellis/tasks/09-06-private-compiler-distribution/check.jsonl create mode 100644 .trellis/tasks/09-06-private-compiler-distribution/implement.jsonl create mode 100644 .trellis/tasks/09-06-private-compiler-distribution/prd.md create mode 100644 .trellis/tasks/09-06-private-compiler-distribution/research/cross-compilation.md create mode 100644 .trellis/tasks/09-06-private-compiler-distribution/research/distribution-licensing.md create mode 100644 .trellis/tasks/09-06-private-compiler-distribution/research/native-backend-options.md create mode 100644 .trellis/tasks/09-06-private-compiler-distribution/task.json create mode 100644 .trellis/tasks/09-12-http2-performance/check.jsonl create mode 100644 .trellis/tasks/09-12-http2-performance/design.md create mode 100644 .trellis/tasks/09-12-http2-performance/implement.jsonl create mode 100644 .trellis/tasks/09-12-http2-performance/implement.md create mode 100644 .trellis/tasks/09-12-http2-performance/prd.md create mode 100644 .trellis/tasks/09-12-http2-performance/research/current-benchmark-and-h2-api.md create mode 100644 .trellis/tasks/09-12-http2-performance/task.json create mode 100644 .trellis/tasks/09-12-http2/check.jsonl create mode 100644 .trellis/tasks/09-12-http2/design.md create mode 100644 .trellis/tasks/09-12-http2/implement.jsonl create mode 100644 .trellis/tasks/09-12-http2/implement.md create mode 100644 .trellis/tasks/09-12-http2/prd.md create mode 100644 .trellis/tasks/09-12-http2/task.json create mode 100644 .trellis/tasks/09-12-runtime-performance-p2/check.jsonl create mode 100644 .trellis/tasks/09-12-runtime-performance-p2/design.md create mode 100644 .trellis/tasks/09-12-runtime-performance-p2/implement.jsonl create mode 100644 .trellis/tasks/09-12-runtime-performance-p2/implement.md create mode 100644 .trellis/tasks/09-12-runtime-performance-p2/prd.md create mode 100644 .trellis/tasks/09-12-runtime-performance-p2/task.json create mode 100644 .trellis/tasks/09-12-runtime-performance/check.jsonl create mode 100644 .trellis/tasks/09-12-runtime-performance/design.md create mode 100644 .trellis/tasks/09-12-runtime-performance/implement.jsonl create mode 100644 .trellis/tasks/09-12-runtime-performance/implement.md create mode 100644 .trellis/tasks/09-12-runtime-performance/prd.md create mode 100644 .trellis/tasks/09-12-runtime-performance/task.json create mode 100644 .trellis/tasks/09-13-typed-orm/check.jsonl create mode 100644 .trellis/tasks/09-13-typed-orm/design.md create mode 100644 .trellis/tasks/09-13-typed-orm/implement.jsonl create mode 100644 .trellis/tasks/09-13-typed-orm/implement.md create mode 100644 .trellis/tasks/09-13-typed-orm/prd.md create mode 100644 .trellis/tasks/09-13-typed-orm/research/architecture-options.md create mode 100644 .trellis/tasks/09-13-typed-orm/research/driver-selection.md create mode 100644 .trellis/tasks/09-13-typed-orm/research/field-types.md create mode 100644 .trellis/tasks/09-13-typed-orm/research/go-orm-reference.md create mode 100644 .trellis/tasks/09-13-typed-orm/research/implicit-suspension.md create mode 100644 .trellis/tasks/09-13-typed-orm/research/migration-practices.md create mode 100644 .trellis/tasks/09-13-typed-orm/research/query-api.md create mode 100644 .trellis/tasks/09-13-typed-orm/research/schema-change-matrix.md create mode 100644 .trellis/tasks/09-13-typed-orm/task.json create mode 100644 .trellis/tasks/09-19-application-testing/check.jsonl create mode 100644 .trellis/tasks/09-19-application-testing/design.md create mode 100644 .trellis/tasks/09-19-application-testing/implement.jsonl create mode 100644 .trellis/tasks/09-19-application-testing/implement.md create mode 100644 .trellis/tasks/09-19-application-testing/prd.md create mode 100644 .trellis/tasks/09-19-application-testing/task.json create mode 100644 .trellis/tasks/09-20-durable-jobs/check.jsonl create mode 100644 .trellis/tasks/09-20-durable-jobs/design.md create mode 100644 .trellis/tasks/09-20-durable-jobs/implement.jsonl create mode 100644 .trellis/tasks/09-20-durable-jobs/implement.md create mode 100644 .trellis/tasks/09-20-durable-jobs/prd.md create mode 100644 .trellis/tasks/09-20-durable-jobs/research/implementation-map.md create mode 100644 .trellis/tasks/09-20-durable-jobs/task.json create mode 100644 .trellis/tasks/09-20-port-adapter/check.jsonl create mode 100644 .trellis/tasks/09-20-port-adapter/design.md create mode 100644 .trellis/tasks/09-20-port-adapter/implement.jsonl create mode 100644 .trellis/tasks/09-20-port-adapter/implement.md create mode 100644 .trellis/tasks/09-20-port-adapter/prd.md create mode 100644 .trellis/tasks/09-20-port-adapter/research/implementation-map.md create mode 100644 .trellis/tasks/09-20-port-adapter/task.json create mode 100644 .trellis/tasks/09-20-production-api/check.jsonl create mode 100644 .trellis/tasks/09-20-production-api/design.md create mode 100644 .trellis/tasks/09-20-production-api/implement.jsonl create mode 100644 .trellis/tasks/09-20-production-api/implement.md create mode 100644 .trellis/tasks/09-20-production-api/prd.md create mode 100644 .trellis/tasks/09-20-production-api/task.json create mode 100644 .trellis/tasks/09-20-production-application-language/check.jsonl create mode 100644 .trellis/tasks/09-20-production-application-language/design.md create mode 100644 .trellis/tasks/09-20-production-application-language/implement.jsonl create mode 100644 .trellis/tasks/09-20-production-application-language/implement.md create mode 100644 .trellis/tasks/09-20-production-application-language/prd.md create mode 100644 .trellis/tasks/09-20-production-application-language/task.json create mode 100644 .trellis/tasks/09-20-secure-data-contracts/check.jsonl create mode 100644 .trellis/tasks/09-20-secure-data-contracts/design.md create mode 100644 .trellis/tasks/09-20-secure-data-contracts/implement.jsonl create mode 100644 .trellis/tasks/09-20-secure-data-contracts/implement.md create mode 100644 .trellis/tasks/09-20-secure-data-contracts/prd.md create mode 100644 .trellis/tasks/09-20-secure-data-contracts/research/secure-contracts-source-map.md create mode 100644 .trellis/tasks/09-20-secure-data-contracts/task.json create mode 100644 .trellis/tasks/09-20-time-typed-codec/check.jsonl create mode 100644 .trellis/tasks/09-20-time-typed-codec/design.md create mode 100644 .trellis/tasks/09-20-time-typed-codec/implement.jsonl create mode 100644 .trellis/tasks/09-20-time-typed-codec/implement.md create mode 100644 .trellis/tasks/09-20-time-typed-codec/prd.md create mode 100644 .trellis/tasks/09-20-time-typed-codec/research/time-wire-source-map.md create mode 100644 .trellis/tasks/09-20-time-typed-codec/task.json create mode 100644 .trellis/tasks/09-28-shared-toolchain/check.jsonl create mode 100644 .trellis/tasks/09-28-shared-toolchain/design.md create mode 100644 .trellis/tasks/09-28-shared-toolchain/implement.jsonl create mode 100644 .trellis/tasks/09-28-shared-toolchain/implement.md create mode 100644 .trellis/tasks/09-28-shared-toolchain/prd.md create mode 100644 .trellis/tasks/09-28-shared-toolchain/task.json create mode 100644 .trellis/tasks/09-29-build-cache-storage/check.jsonl create mode 100644 .trellis/tasks/09-29-build-cache-storage/design.md create mode 100644 .trellis/tasks/09-29-build-cache-storage/implement.jsonl create mode 100644 .trellis/tasks/09-29-build-cache-storage/implement.md create mode 100644 .trellis/tasks/09-29-build-cache-storage/prd.md create mode 100644 .trellis/tasks/09-29-build-cache-storage/research/cache-boundaries.md create mode 100644 .trellis/tasks/09-29-build-cache-storage/task.json create mode 100644 .trellis/tasks/09-29-external-lib-ecosystems/check.jsonl create mode 100644 .trellis/tasks/09-29-external-lib-ecosystems/design.md create mode 100644 .trellis/tasks/09-29-external-lib-ecosystems/implement.jsonl create mode 100644 .trellis/tasks/09-29-external-lib-ecosystems/implement.md create mode 100644 .trellis/tasks/09-29-external-lib-ecosystems/prd.md create mode 100644 .trellis/tasks/09-29-external-lib-ecosystems/task.json create mode 100644 .trellis/tasks/09-29-external-lib-packaging/check.jsonl create mode 100644 .trellis/tasks/09-29-external-lib-packaging/design.md create mode 100644 .trellis/tasks/09-29-external-lib-packaging/implement.jsonl create mode 100644 .trellis/tasks/09-29-external-lib-packaging/implement.md create mode 100644 .trellis/tasks/09-29-external-lib-packaging/prd.md create mode 100644 .trellis/tasks/09-29-external-lib-packaging/task.json create mode 100644 .trellis/tasks/09-29-external-lib-protocol/check.jsonl create mode 100644 .trellis/tasks/09-29-external-lib-protocol/design.md create mode 100644 .trellis/tasks/09-29-external-lib-protocol/implement.jsonl create mode 100644 .trellis/tasks/09-29-external-lib-protocol/implement.md create mode 100644 .trellis/tasks/09-29-external-lib-protocol/prd.md create mode 100644 .trellis/tasks/09-29-external-lib-protocol/task.json create mode 100644 .trellis/tasks/09-29-external-libs/check.jsonl create mode 100644 .trellis/tasks/09-29-external-libs/design.md create mode 100644 .trellis/tasks/09-29-external-libs/implement.jsonl create mode 100644 .trellis/tasks/09-29-external-libs/implement.md create mode 100644 .trellis/tasks/09-29-external-libs/prd.md create mode 100644 .trellis/tasks/09-29-external-libs/task.json create mode 100644 .trellis/tasks/09-29-language-production-closure/check.jsonl create mode 100644 .trellis/tasks/09-29-language-production-closure/design.md create mode 100644 .trellis/tasks/09-29-language-production-closure/implement.jsonl create mode 100644 .trellis/tasks/09-29-language-production-closure/implement.md create mode 100644 .trellis/tasks/09-29-language-production-closure/prd.md create mode 100644 .trellis/tasks/09-29-language-production-closure/research/apparmor-nonroot.md create mode 100644 .trellis/tasks/09-29-language-production-closure/research/ecosystem-closure.md create mode 100644 .trellis/tasks/09-29-language-production-closure/research/ecosystem-pack-inputs.md create mode 100644 .trellis/tasks/09-29-language-production-closure/research/linux-ci-scope.md create mode 100644 .trellis/tasks/09-29-language-production-closure/research/native-pack-production-inputs.md create mode 100644 .trellis/tasks/09-29-language-production-closure/task.json create mode 100644 .trellis/tasks/10-04-developer-onboarding/check.jsonl create mode 100644 .trellis/tasks/10-04-developer-onboarding/design.md create mode 100644 .trellis/tasks/10-04-developer-onboarding/implement.jsonl create mode 100644 .trellis/tasks/10-04-developer-onboarding/implement.md create mode 100644 .trellis/tasks/10-04-developer-onboarding/prd.md create mode 100644 .trellis/tasks/10-04-developer-onboarding/task.json create mode 100644 .trellis/tasks/archive/2026-09/00-bootstrap-guidelines/prd.md create mode 100644 .trellis/tasks/archive/2026-09/00-bootstrap-guidelines/task.json create mode 100644 .trellis/tasks/archive/2026-09/09-09-markdown-source/check.jsonl create mode 100644 .trellis/tasks/archive/2026-09/09-09-markdown-source/design.md create mode 100644 .trellis/tasks/archive/2026-09/09-09-markdown-source/implement.jsonl create mode 100644 .trellis/tasks/archive/2026-09/09-09-markdown-source/implement.md create mode 100644 .trellis/tasks/archive/2026-09/09-09-markdown-source/prd.md create mode 100644 .trellis/tasks/archive/2026-09/09-09-markdown-source/research/source-flow.md create mode 100644 .trellis/tasks/archive/2026-09/09-09-markdown-source/task.json create mode 100644 .trellis/tasks/archive/2026-09/09-13-http2-ceiling-soak/check.jsonl create mode 100644 .trellis/tasks/archive/2026-09/09-13-http2-ceiling-soak/design.md create mode 100644 .trellis/tasks/archive/2026-09/09-13-http2-ceiling-soak/implement.jsonl create mode 100644 .trellis/tasks/archive/2026-09/09-13-http2-ceiling-soak/implement.md create mode 100644 .trellis/tasks/archive/2026-09/09-13-http2-ceiling-soak/prd.md create mode 100644 .trellis/tasks/archive/2026-09/09-13-http2-ceiling-soak/research/client-bottleneck.md create mode 100644 .trellis/tasks/archive/2026-09/09-13-http2-ceiling-soak/task.json create mode 100644 .trellis/tasks/archive/2026-09/09-18-cms-dual-source-contracts/check.jsonl create mode 100644 .trellis/tasks/archive/2026-09/09-18-cms-dual-source-contracts/design.md create mode 100644 .trellis/tasks/archive/2026-09/09-18-cms-dual-source-contracts/implement.jsonl create mode 100644 .trellis/tasks/archive/2026-09/09-18-cms-dual-source-contracts/implement.md create mode 100644 .trellis/tasks/archive/2026-09/09-18-cms-dual-source-contracts/prd.md create mode 100644 .trellis/tasks/archive/2026-09/09-18-cms-dual-source-contracts/task.json create mode 100644 .trellis/tasks/archive/2026-10/09-20-large-cms-acceptance/check.jsonl create mode 100644 .trellis/tasks/archive/2026-10/09-20-large-cms-acceptance/design.md create mode 100644 .trellis/tasks/archive/2026-10/09-20-large-cms-acceptance/implement.jsonl create mode 100644 .trellis/tasks/archive/2026-10/09-20-large-cms-acceptance/implement.md create mode 100644 .trellis/tasks/archive/2026-10/09-20-large-cms-acceptance/prd.md create mode 100644 .trellis/tasks/archive/2026-10/09-20-large-cms-acceptance/task.json create mode 100644 .trellis/tasks/archive/2026-10/10-04-external-command/check.jsonl create mode 100644 .trellis/tasks/archive/2026-10/10-04-external-command/design.md create mode 100644 .trellis/tasks/archive/2026-10/10-04-external-command/implement.jsonl create mode 100644 .trellis/tasks/archive/2026-10/10-04-external-command/implement.md create mode 100644 .trellis/tasks/archive/2026-10/10-04-external-command/prd.md create mode 100644 .trellis/tasks/archive/2026-10/10-04-external-command/task.json create mode 100644 .trellis/tasks/archive/2026-10/10-04-linux-arm64-cross-build/check.jsonl create mode 100644 .trellis/tasks/archive/2026-10/10-04-linux-arm64-cross-build/design.md create mode 100644 .trellis/tasks/archive/2026-10/10-04-linux-arm64-cross-build/implement.jsonl create mode 100644 .trellis/tasks/archive/2026-10/10-04-linux-arm64-cross-build/implement.md create mode 100644 .trellis/tasks/archive/2026-10/10-04-linux-arm64-cross-build/prd.md create mode 100644 .trellis/tasks/archive/2026-10/10-04-linux-arm64-cross-build/task.json create mode 100644 .trellis/workflow.md create mode 100644 AGENTS.md create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 IMPLEMENTATION.md create mode 100644 LANGUAGE.md create mode 100644 MARKDOWN-SYNTAX.md create mode 100644 README.md create mode 100644 config/setting.example.json create mode 100644 crates/dever-backend-bridge/Cargo.toml create mode 100644 crates/dever-backend-bridge/build.rs create mode 100644 crates/dever-backend-bridge/include/runtime.h create mode 100644 crates/dever-backend-bridge/src/bridge.cpp create mode 100644 crates/dever-backend-bridge/src/ffi.rs create mode 100644 crates/dever-backend-bridge/src/lib.rs create mode 100644 crates/dever-cli/Cargo.toml create mode 100644 crates/dever-cli/build.rs create mode 100644 crates/dever-cli/src/api.rs create mode 100644 crates/dever-cli/src/bin/dever-launcher.rs create mode 100644 crates/dever-cli/src/bin/deverd.rs create mode 100644 crates/dever-cli/src/compile.rs create mode 100644 crates/dever-cli/src/format.rs create mode 100644 crates/dever-cli/src/lib.rs create mode 100644 crates/dever-cli/src/libs.rs create mode 100644 crates/dever-cli/src/libs/build.rs create mode 100644 crates/dever-cli/src/libs/build/npm.js create mode 100644 crates/dever-cli/src/libs/build/npm.rs create mode 100644 crates/dever-cli/src/libs/build/npm/environment.rs create mode 100644 crates/dever-cli/src/libs/build/npm/probe.cjs create mode 100644 crates/dever-cli/src/libs/build/pack.rs create mode 100644 crates/dever-cli/src/libs/build/process.rs create mode 100644 crates/dever-cli/src/libs/build/process/budget.rs create mode 100644 crates/dever-cli/src/libs/build/python.py create mode 100644 crates/dever-cli/src/libs/build/python.rs create mode 100644 crates/dever-cli/src/libs/mutation.rs create mode 100644 crates/dever-cli/src/libs/registry.rs create mode 100644 crates/dever-cli/src/libs/registry/npm.rs create mode 100644 crates/dever-cli/src/libs/registry/npm/resolve.rs create mode 100644 crates/dever-cli/src/libs/registry/sumdb.rs create mode 100644 crates/dever-cli/src/libs/registry/sumdb/merkle.rs create mode 100644 crates/dever-cli/src/main.rs create mode 100644 crates/dever-cli/src/packages.rs create mode 100644 crates/dever-cli/src/project.rs create mode 100644 crates/dever-cli/src/test_runner.rs create mode 100644 crates/dever-cli/src/toolchain/bootstrap.rs create mode 100644 crates/dever-cli/src/toolchain/bootstrap/sandbox.rs create mode 100644 crates/dever-cli/src/toolchain/bootstrap/service.rs create mode 100644 crates/dever-cli/src/toolchain/bootstrap/transaction.rs create mode 100644 crates/dever-cli/src/toolchain/cache.rs create mode 100644 crates/dever-cli/src/toolchain/compilation.rs create mode 100644 crates/dever-cli/src/toolchain/core_libraries.rs create mode 100644 crates/dever-cli/src/toolchain/launcher.rs create mode 100644 crates/dever-cli/src/toolchain/mod.rs create mode 100644 crates/dever-cli/src/toolchain/packaging.rs create mode 100644 crates/dever-cli/src/toolchain/packaging/bootstrap.rs create mode 100644 crates/dever-cli/src/toolchain/packaging/builds.rs create mode 100644 crates/dever-cli/src/toolchain/packaging/ecosystems.rs create mode 100644 crates/dever-cli/src/toolchain/packaging/sandbox.rs create mode 100644 crates/dever-cli/src/toolchain/release.rs create mode 100644 crates/dever-cli/src/toolchain/release_source.rs create mode 100644 crates/dever-cli/src/toolchain/runtime_pack.rs create mode 100644 crates/dever-cli/src/toolchain/service.rs create mode 100644 crates/dever-cli/src/toolchain/service/compile.rs create mode 100644 crates/dever-cli/src/toolchain/service/compile_request.rs create mode 100644 crates/dever-cli/src/toolchain/skill.rs create mode 100644 crates/dever-cli/src/toolchain/target.rs create mode 100644 crates/dever-cli/src/worker.rs create mode 100644 crates/dever-cli/src/workers.rs create mode 100644 crates/dever-cli/src/workers/command.rs create mode 100644 crates/dever-cli/src/workers/elf.rs create mode 100644 crates/dever-cli/src/workers/go_build.rs create mode 100644 crates/dever-cli/src/workers/python_wheel.rs create mode 100644 crates/dever-cli/src/workers/python_wheel/native.rs create mode 100644 crates/dever-cli/src/workers/python_wheel/scripts.rs create mode 100644 crates/dever-cli/src/workers/runtime.rs create mode 100644 crates/dever-core/Cargo.toml create mode 100644 crates/dever-core/src/api.rs create mode 100644 crates/dever-core/src/capture.rs create mode 100644 crates/dever-core/src/check.rs create mode 100644 crates/dever-core/src/check/api.rs create mode 100644 crates/dever-core/src/check/auth.rs create mode 100644 crates/dever-core/src/check/body.rs create mode 100644 crates/dever-core/src/check/cache.rs create mode 100644 crates/dever-core/src/check/clauses.rs create mode 100644 crates/dever-core/src/check/collections.rs create mode 100644 crates/dever-core/src/check/components.rs create mode 100644 crates/dever-core/src/check/concurrency.rs create mode 100644 crates/dever-core/src/check/constants.rs create mode 100644 crates/dever-core/src/check/cycles.rs create mode 100644 crates/dever-core/src/check/expressions.rs create mode 100644 crates/dever-core/src/check/intrinsics.rs create mode 100644 crates/dever-core/src/check/jobs.rs create mode 100644 crates/dever-core/src/check/layout.rs create mode 100644 crates/dever-core/src/check/model.rs create mode 100644 crates/dever-core/src/check/names.rs create mode 100644 crates/dever-core/src/check/operators.rs create mode 100644 crates/dever-core/src/check/orm.rs create mode 100644 crates/dever-core/src/check/ports.rs create mode 100644 crates/dever-core/src/check/sql_read.rs create mode 100644 crates/dever-core/src/check/symbols.rs create mode 100644 crates/dever-core/src/check/upload.rs create mode 100644 crates/dever-core/src/check/values.rs create mode 100644 crates/dever-core/src/contracts.rs create mode 100644 crates/dever-core/src/contracts/dependencies.rs create mode 100644 crates/dever-core/src/contracts/effects.rs create mode 100644 crates/dever-core/src/contracts/error_effects.rs create mode 100644 crates/dever-core/src/contracts/facts.rs create mode 100644 crates/dever-core/src/contracts/facts/domain.rs create mode 100644 crates/dever-core/src/contracts/failures.rs create mode 100644 crates/dever-core/src/contracts/redundancy.rs create mode 100644 crates/dever-core/src/diagnostic.rs create mode 100644 crates/dever-core/src/format.rs create mode 100644 crates/dever-core/src/format/document.rs create mode 100644 crates/dever-core/src/format/markdown.rs create mode 100644 crates/dever-core/src/hir.rs create mode 100644 crates/dever-core/src/intrinsic.rs create mode 100644 crates/dever-core/src/lexer.rs create mode 100644 crates/dever-core/src/lib.rs create mode 100644 crates/dever-core/src/llvm.rs create mode 100644 crates/dever-core/src/llvm/abi.rs create mode 100644 crates/dever-core/src/llvm/api.rs create mode 100644 crates/dever-core/src/llvm/api_context.rs create mode 100644 crates/dever-core/src/llvm/application.rs create mode 100644 crates/dever-core/src/llvm/application_commands.rs create mode 100644 crates/dever-core/src/llvm/asynchronous.rs create mode 100644 crates/dever-core/src/llvm/collections.rs create mode 100644 crates/dever-core/src/llvm/concurrency.rs create mode 100644 crates/dever-core/src/llvm/database.rs create mode 100644 crates/dever-core/src/llvm/database_rows.rs create mode 100644 crates/dever-core/src/llvm/database_schema.rs create mode 100644 crates/dever-core/src/llvm/expressions.rs create mode 100644 crates/dever-core/src/llvm/external.rs create mode 100644 crates/dever-core/src/llvm/failures.rs create mode 100644 crates/dever-core/src/llvm/http.rs create mode 100644 crates/dever-core/src/llvm/http_handlers.rs create mode 100644 crates/dever-core/src/llvm/http_values.rs create mode 100644 crates/dever-core/src/llvm/intrinsics.rs create mode 100644 crates/dever-core/src/llvm/jobs.rs create mode 100644 crates/dever-core/src/llvm/network.rs create mode 100644 crates/dever-core/src/llvm/orm.rs create mode 100644 crates/dever-core/src/llvm/orm_read.rs create mode 100644 crates/dever-core/src/llvm/orm_relations.rs create mode 100644 crates/dever-core/src/llvm/orm_stream.rs create mode 100644 crates/dever-core/src/llvm/ownership.rs create mode 100644 crates/dever-core/src/llvm/parallel.rs create mode 100644 crates/dever-core/src/llvm/ports.rs create mode 100644 crates/dever-core/src/llvm/process_entry.rs create mode 100644 crates/dever-core/src/llvm/render.rs create mode 100644 crates/dever-core/src/llvm/resources.rs create mode 100644 crates/dever-core/src/llvm/rest.rs create mode 100644 crates/dever-core/src/llvm/system.rs create mode 100644 crates/dever-core/src/llvm/testing.rs create mode 100644 crates/dever-core/src/llvm/transaction.rs create mode 100644 crates/dever-core/src/llvm/traversal.rs create mode 100644 crates/dever-core/src/llvm/wire.rs create mode 100644 crates/dever-core/src/markdown.rs create mode 100644 crates/dever-core/src/markdown/contract.rs create mode 100644 crates/dever-core/src/markdown/contract/metadata.rs create mode 100644 crates/dever-core/src/markdown/contract/model.rs create mode 100644 crates/dever-core/src/markdown/contract/parse.rs create mode 100644 crates/dever-core/src/markdown/contract/validate.rs create mode 100644 crates/dever-core/src/model.rs create mode 100644 crates/dever-core/src/native.rs create mode 100644 crates/dever-core/src/native/api.rs create mode 100644 crates/dever-core/src/native/build.rs create mode 100644 crates/dever-core/src/native/build/cache.rs create mode 100644 crates/dever-core/src/native/build/cleanup.rs create mode 100644 crates/dever-core/src/native/collections.rs create mode 100644 crates/dever-core/src/native/expressions.rs create mode 100644 crates/dever-core/src/native/http.rs create mode 100644 crates/dever-core/src/native/intrinsics.rs create mode 100644 crates/dever-core/src/native/jobs.rs create mode 100644 crates/dever-core/src/native/liveness.rs create mode 100644 crates/dever-core/src/native/orm.rs create mode 100644 crates/dever-core/src/native/ports.rs create mode 100644 crates/dever-core/src/native/records.rs create mode 100644 crates/dever-core/src/native/rest.rs create mode 100644 crates/dever-core/src/native/sequences.rs create mode 100644 crates/dever-core/src/native/updates.rs create mode 100644 crates/dever-core/src/native/wire.rs create mode 100644 crates/dever-core/src/parser.rs create mode 100644 crates/dever-core/src/parser/expression.rs create mode 100644 crates/dever-core/src/reference.rs create mode 100644 crates/dever-core/src/reference/collections.rs create mode 100644 crates/dever-core/src/reference/expressions.rs create mode 100644 crates/dever-core/src/reference/intrinsics.rs create mode 100644 crates/dever-core/src/reference/value.rs create mode 100644 crates/dever-core/src/source.rs create mode 100644 crates/dever-core/src/specialize.rs create mode 100644 crates/dever-core/src/syntax.rs create mode 100644 crates/dever-core/src/token.rs create mode 100644 crates/dever-core/src/types.rs create mode 100644 crates/dever-core/src/wire.rs create mode 100644 crates/dever-runtime/Cargo.toml create mode 100644 crates/dever-runtime/src/abi.rs create mode 100644 crates/dever-runtime/src/api.rs create mode 100644 crates/dever-runtime/src/api/input.rs create mode 100644 crates/dever-runtime/src/api/upload.rs create mode 100644 crates/dever-runtime/src/application.rs create mode 100644 crates/dever-runtime/src/async_stream.rs create mode 100644 crates/dever-runtime/src/auth.rs create mode 100644 crates/dever-runtime/src/auth/store.rs create mode 100644 crates/dever-runtime/src/bytes.rs create mode 100644 crates/dever-runtime/src/channel.rs create mode 100644 crates/dever-runtime/src/collections.rs create mode 100644 crates/dever-runtime/src/component.rs create mode 100644 crates/dever-runtime/src/component/command.rs create mode 100644 crates/dever-runtime/src/component/command/observation.rs create mode 100644 crates/dever-runtime/src/concurrent.rs create mode 100644 crates/dever-runtime/src/config.rs create mode 100644 crates/dever-runtime/src/config/command.rs create mode 100644 crates/dever-runtime/src/config/compilation.rs create mode 100644 crates/dever-runtime/src/cron.rs create mode 100644 crates/dever-runtime/src/crypto.rs create mode 100644 crates/dever-runtime/src/database.rs create mode 100644 crates/dever-runtime/src/endpoint.rs create mode 100644 crates/dever-runtime/src/external.rs create mode 100644 crates/dever-runtime/src/http/client.rs create mode 100644 crates/dever-runtime/src/http/client_body.rs create mode 100644 crates/dever-runtime/src/http/client_driver.rs create mode 100644 crates/dever-runtime/src/http/executor.rs create mode 100644 crates/dever-runtime/src/http/live.rs create mode 100644 crates/dever-runtime/src/http/message.rs create mode 100644 crates/dever-runtime/src/http/mod.rs create mode 100644 crates/dever-runtime/src/http/protocol.rs create mode 100644 crates/dever-runtime/src/http/reply.rs create mode 100644 crates/dever-runtime/src/http/server_h2.rs create mode 100644 crates/dever-runtime/src/http/write_timeout.rs create mode 100644 crates/dever-runtime/src/job.rs create mode 100644 crates/dever-runtime/src/job/resources.rs create mode 100644 crates/dever-runtime/src/job/store.rs create mode 100644 crates/dever-runtime/src/job/worker.rs create mode 100644 crates/dever-runtime/src/lib.rs create mode 100644 crates/dever-runtime/src/lifecycle.rs create mode 100644 crates/dever-runtime/src/log.rs create mode 100644 crates/dever-runtime/src/net.rs create mode 100644 crates/dever-runtime/src/number.rs create mode 100644 crates/dever-runtime/src/orm.rs create mode 100644 crates/dever-runtime/src/pool.rs create mode 100644 crates/dever-runtime/src/postgres.rs create mode 100644 crates/dever-runtime/src/postgres/migration.rs create mode 100644 crates/dever-runtime/src/process.rs create mode 100644 crates/dever-runtime/src/render.rs create mode 100644 crates/dever-runtime/src/resource.rs create mode 100644 crates/dever-runtime/src/secret.rs create mode 100644 crates/dever-runtime/src/sqlite.rs create mode 100644 crates/dever-runtime/src/sqlite/migration.rs create mode 100644 crates/dever-runtime/src/sse.rs create mode 100644 crates/dever-runtime/src/task.rs create mode 100644 crates/dever-runtime/src/task/scoped.rs create mode 100644 crates/dever-runtime/src/tenant.rs create mode 100644 crates/dever-runtime/src/text.rs create mode 100644 crates/dever-runtime/src/time.rs create mode 100644 crates/dever-runtime/src/tls.rs create mode 100644 crates/dever-runtime/src/transport.rs create mode 100644 crates/dever-runtime/src/websocket.rs create mode 100644 crates/dever-runtime/src/wire.rs create mode 100644 crates/dever-sandbox/Cargo.toml create mode 100644 crates/dever-sandbox/src/assets.rs create mode 100644 crates/dever-sandbox/src/filter.rs create mode 100644 crates/dever-sandbox/src/lib.rs create mode 100644 crates/dever-sandbox/src/main.rs create mode 100644 crates/dever-sandbox/src/mounts.rs create mode 100644 crates/dever-sandbox/src/observation.rs create mode 100644 crates/dever-sandbox/src/trusted_helper.rs delete mode 100644 docs/superpowers/plans/2026-09-02-dever-language-mvp.md delete mode 100644 docs/superpowers/specs/2026-09-02-dever-language-mvp-design.md create mode 100644 examples/README.md create mode 100644 examples/cms/dever/config/setting.json create mode 100644 examples/cms/dever/data/.gitkeep create mode 100644 examples/cms/dever/module/audit/operation/app.dever create mode 100644 examples/cms/dever/module/audit/operation/model.dever create mode 100644 examples/cms/dever/module/content/category/api/admin/manage.dever create mode 100644 examples/cms/dever/module/content/category/app.dever create mode 100644 examples/cms/dever/module/content/category/domain.dever create mode 100644 examples/cms/dever/module/content/category/model.dever create mode 100644 examples/cms/dever/module/content/revision/api/admin/manage.dever create mode 100644 examples/cms/dever/module/content/revision/app.dever create mode 100644 examples/cms/dever/module/content/revision/model.dever create mode 100644 examples/cms/dever/module/media/asset/adapter.dever create mode 100644 examples/cms/dever/module/media/asset/api/admin/manage.dever create mode 100644 examples/cms/dever/module/media/asset/app.dever create mode 100644 examples/cms/dever/module/media/asset/model.dever create mode 100644 examples/cms/dever/module/media/asset/port.dever create mode 100644 examples/cms/dever/module/news/article/api/admin/manage.dever create mode 100644 examples/cms/dever/module/news/article/api/front/browse.dever create mode 100644 examples/cms/dever/module/news/article/app/editing.dever create mode 100644 examples/cms/dever/module/news/article/app/publication.dever create mode 100644 examples/cms/dever/module/news/article/domain.dever create mode 100644 examples/cms/dever/module/news/article/model.dever create mode 100644 examples/cms/dever/module/news/article/model/publication.dever create mode 100644 examples/cms/dever/module/platform/tenant/app.dever create mode 100644 examples/cms/dever/module/platform/tenant/domain.dever create mode 100644 examples/cms/dever/module/platform/tenant/model.dever create mode 100644 examples/cms/dever/module/publishing/schedule/api/admin/manage.dever create mode 100644 examples/cms/dever/module/publishing/schedule/app.dever create mode 100644 examples/cms/dever/module/publishing/schedule/domain.dever create mode 100644 examples/cms/dever/module/publishing/schedule/job.dever create mode 100644 examples/cms/dever/module/publishing/schedule/model.dever create mode 100644 examples/cms/dever/module/system/health/api.dever create mode 100644 examples/cms/dever/module/system/health/app.dever create mode 100644 examples/cms/dever/module/user/account/api.dever create mode 100644 examples/cms/dever/module/user/account/api/admin/session.dever create mode 100644 examples/cms/dever/module/user/account/api/front/session.dever create mode 100644 examples/cms/dever/module/user/account/app.dever create mode 100644 examples/cms/dever/module/user/account/domain.dever create mode 100644 examples/cms/dever/module/user/account/model.dever create mode 100644 examples/cms/dever/module/user/account/model/principal.dever create mode 100644 examples/cms/dever/module/user/authorization/api/admin/manage.dever create mode 100644 examples/cms/dever/module/user/authorization/api/front/manage.dever create mode 100644 examples/cms/dever/module/user/authorization/app.dever create mode 100644 examples/cms/dever/module/user/membership/app.dever create mode 100644 examples/cms/dever/module/user/membership/model.dever create mode 100644 examples/cms/dever/module/user/session/app.dever create mode 100644 examples/cms/dever/module/user/session/job.dever create mode 100644 examples/cms/dever/module/user/session/model.dever create mode 100644 examples/cms/dever/test/content/category/lifecycle.dever create mode 100644 examples/cms/dever/test/news/article/publish.dever create mode 100644 examples/cms/dever/test/user/account/bootstrap.dever create mode 100644 examples/cms/dever/test/user/account/validation.dever create mode 100644 examples/cms/dever/test/user/session/lifecycle.dever create mode 100644 examples/cms/md/config/setting.json create mode 100644 examples/cms/md/module/audit/operation/app.dever.md create mode 100644 examples/cms/md/module/audit/operation/model.dever.md create mode 100644 examples/cms/md/module/content/category/api/admin/manage.dever.md create mode 100644 examples/cms/md/module/content/category/app.dever.md create mode 100644 examples/cms/md/module/content/category/domain.dever.md create mode 100644 examples/cms/md/module/content/category/model.dever.md create mode 100644 examples/cms/md/module/content/revision/api/admin/manage.dever.md create mode 100644 examples/cms/md/module/content/revision/app.dever.md create mode 100644 examples/cms/md/module/content/revision/model.dever.md create mode 100644 examples/cms/md/module/media/asset/adapter.dever.md create mode 100644 examples/cms/md/module/media/asset/api/admin/manage.dever.md create mode 100644 examples/cms/md/module/media/asset/app.dever.md create mode 100644 examples/cms/md/module/media/asset/model.dever.md create mode 100644 examples/cms/md/module/media/asset/port.dever.md create mode 100644 examples/cms/md/module/news/article/api/admin/manage.dever.md create mode 100644 examples/cms/md/module/news/article/api/front/browse.dever.md create mode 100644 examples/cms/md/module/news/article/app/editing.dever.md create mode 100644 examples/cms/md/module/news/article/app/publication.dever.md create mode 100644 examples/cms/md/module/news/article/domain.dever.md create mode 100644 examples/cms/md/module/news/article/model.dever.md create mode 100644 examples/cms/md/module/news/article/model/publication.dever.md create mode 100644 examples/cms/md/module/platform/tenant/app.dever.md create mode 100644 examples/cms/md/module/platform/tenant/domain.dever.md create mode 100644 examples/cms/md/module/platform/tenant/model.dever.md create mode 100644 examples/cms/md/module/publishing/schedule/api/admin/manage.dever.md create mode 100644 examples/cms/md/module/publishing/schedule/app.dever.md create mode 100644 examples/cms/md/module/publishing/schedule/domain.dever.md create mode 100644 examples/cms/md/module/publishing/schedule/job.dever.md create mode 100644 examples/cms/md/module/publishing/schedule/model.dever.md create mode 100644 examples/cms/md/module/system/health/api.dever.md create mode 100644 examples/cms/md/module/system/health/app.dever.md create mode 100644 examples/cms/md/module/user/account/api.dever.md create mode 100644 examples/cms/md/module/user/account/api/admin/session.dever.md create mode 100644 examples/cms/md/module/user/account/api/front/session.dever.md create mode 100644 examples/cms/md/module/user/account/app.dever.md create mode 100644 examples/cms/md/module/user/account/domain.dever.md create mode 100644 examples/cms/md/module/user/account/model.dever.md create mode 100644 examples/cms/md/module/user/account/model/principal.dever.md create mode 100644 examples/cms/md/module/user/authorization/api/admin/manage.dever.md create mode 100644 examples/cms/md/module/user/authorization/api/front/manage.dever.md create mode 100644 examples/cms/md/module/user/authorization/app.dever.md create mode 100644 examples/cms/md/module/user/membership/app.dever.md create mode 100644 examples/cms/md/module/user/membership/model.dever.md create mode 100644 examples/cms/md/module/user/session/app.dever.md create mode 100644 examples/cms/md/module/user/session/job.dever.md create mode 100644 examples/cms/md/module/user/session/model.dever.md create mode 100644 examples/cms/md/test/content/category/lifecycle.dever.md create mode 100644 examples/cms/md/test/news/article/publish.dever.md create mode 100644 examples/cms/md/test/user/account/bootstrap.dever.md create mode 100644 examples/cms/md/test/user/account/validation.dever.md create mode 100644 examples/cms/md/test/user/session/lifecycle.dever.md create mode 100644 examples/old/dever/async/config/setting.json create mode 100644 examples/old/dever/async/data/.gitkeep create mode 100644 examples/old/dever/async/module/main.dever create mode 100644 examples/old/dever/async_tcp/config/setting.json create mode 100644 examples/old/dever/async_tcp/data/.gitkeep create mode 100644 examples/old/dever/async_tcp/module/main.dever create mode 100644 examples/old/dever/backend/config/setting.json create mode 100644 examples/old/dever/backend/data/.gitkeep create mode 100644 examples/old/dever/backend/module/main.dever create mode 100644 examples/old/dever/contracts/config/setting.json create mode 100644 examples/old/dever/contracts/data/.gitkeep create mode 100644 examples/old/dever/contracts/dever.api create mode 100644 examples/old/dever/contracts/module/domain.dever create mode 100644 examples/old/dever/contracts/module/main.dever create mode 100644 examples/old/dever/core/config/setting.json create mode 100644 examples/old/dever/core/data/.gitkeep create mode 100644 examples/old/dever/core/module/main.dever create mode 100644 examples/old/dever/hello/config/setting.json create mode 100644 examples/old/dever/hello/data/.gitkeep create mode 100644 examples/old/dever/hello/module/app/user.dever create mode 100644 examples/old/dever/hello/module/main.dever create mode 100644 examples/old/dever/http/config/setting.json create mode 100644 examples/old/dever/http/data/.gitkeep create mode 100644 examples/old/dever/http/module/main.dever create mode 100644 examples/old/dever/library/config/setting.json create mode 100644 examples/old/dever/library/data/.gitkeep create mode 100644 examples/old/dever/library/module/main.dever create mode 100644 examples/old/dever/live_http/config/setting.json create mode 100644 examples/old/dever/live_http/data/.gitkeep create mode 100644 examples/old/dever/live_http/module/main.dever create mode 100644 examples/old/dever/parallel/config/setting.json create mode 100644 examples/old/dever/parallel/data/.gitkeep create mode 100644 examples/old/dever/parallel/module/main.dever create mode 100644 examples/old/dever/privacy/config/setting.json create mode 100644 examples/old/dever/privacy/data/.gitkeep create mode 100644 examples/old/dever/privacy/module/counter.dever create mode 100644 examples/old/dever/privacy/module/main.dever create mode 100644 examples/old/dever/rejected/coverage/module/main.dever create mode 100644 examples/old/dever/rejected/failure/module/main.dever create mode 100644 examples/old/dever/rejected/private/module/main.dever create mode 100644 examples/old/dever/rejected/private/module/secret.dever create mode 100644 examples/old/dever/rejected/pure/module/main.dever create mode 100644 examples/old/dever/rejected/range/module/main.dever create mode 100644 examples/old/dever/sequences/config/setting.json create mode 100644 examples/old/dever/sequences/data/.gitkeep create mode 100644 examples/old/dever/sequences/module/main.dever create mode 100644 examples/old/dever/stream/config/setting.json create mode 100644 examples/old/dever/stream/data/.gitkeep create mode 100644 examples/old/dever/stream/input.txt create mode 100644 examples/old/dever/stream/module/main.dever create mode 100644 examples/old/dever/system/config/setting.json create mode 100644 examples/old/dever/system/data/.gitkeep create mode 100644 examples/old/dever/system/module/main.dever create mode 100644 examples/old/dever/text/config/setting.json create mode 100644 examples/old/dever/text/data/.gitkeep create mode 100644 examples/old/dever/text/module/main.dever create mode 100644 examples/old/markdown/config/setting.json create mode 100644 examples/old/markdown/data/.gitkeep create mode 100644 examples/old/markdown/module/main.dever.md create mode 100644 library/decimal.dever create mode 100644 library/dever/api.dever create mode 100644 library/dever/auth.dever create mode 100644 library/dever/bytes.dever create mode 100644 library/dever/crypto.dever create mode 100644 library/dever/database.dever create mode 100644 library/dever/http.dever create mode 100644 library/dever/id.dever create mode 100644 library/dever/io.dever create mode 100644 library/dever/job.dever create mode 100644 library/dever/json.dever create mode 100644 library/dever/json/number.dever create mode 100644 library/dever/json/parse.dever create mode 100644 library/dever/json/scan.dever create mode 100644 library/dever/json/value.dever create mode 100644 library/dever/json/write.dever create mode 100644 library/dever/log.dever create mode 100644 library/dever/net.dever create mode 100644 library/dever/process.dever create mode 100644 library/dever/site.dever create mode 100644 library/dever/sse.dever create mode 100644 library/dever/storage.dever create mode 100644 library/dever/task.dever create mode 100644 library/dever/time.dever create mode 100644 library/dever/tls.dever create mode 100644 library/dever/websocket.dever create mode 100644 library/float.dever create mode 100644 library/int.dever create mode 100644 library/math.dever create mode 100644 library/text.dever create mode 100644 library/uuid.dever create mode 100644 sdk/README.md create mode 100644 sdk/apparmor/dever-sandbox create mode 100644 sdk/go/component.go create mode 100644 sdk/go/example/main.go create mode 100644 sdk/go/go.mod create mode 100644 sdk/javascript/dever_component.mjs create mode 100644 sdk/javascript/example.mjs create mode 100644 sdk/native-release.md create mode 100644 sdk/native-release.rs create mode 100644 sdk/python/dever_component.py create mode 100644 sdk/python/example.py create mode 100644 sdk/release-assets.py create mode 160000 skills create mode 100644 test/backend_bridge.rs create mode 100644 test/ci/README.md create mode 100644 test/ci/acceptance.py create mode 100644 test/ci/ignored.json create mode 100644 test/ci/run.py create mode 100644 test/ci/test_driver.py create mode 100644 test/ci/test_gate.py create mode 100644 test/cms/README.md create mode 100644 test/cms/acceptance.py create mode 100644 test/dever-cli-tests/build_arguments.rs create mode 100644 test/dever-cli-tests/cms_project.rs create mode 100644 test/dever-cli-tests/external_libs.rs create mode 100644 test/dever-cli-tests/external_libs/build.rs create mode 100644 test/dever-cli-tests/external_libs/npm.rs create mode 100644 test/dever-cli-tests/external_libs/npm_build.rs create mode 100644 test/dever-cli-tests/external_libs/sumdb.rs create mode 100644 test/dever-cli-tests/external_libs/sumdb_uuid.json create mode 100644 test/dever-cli-tests/external_libs/wheel.rs create mode 100644 test/dever-cli-tests/external_workers.rs create mode 100644 test/dever-cli-tests/external_workers/command.rs create mode 100644 test/dever-cli-tests/external_workers/npm.rs create mode 100644 test/dever-cli-tests/llvm_cli.rs create mode 100644 test/dever-cli-tests/native_release.rs create mode 100644 test/dever-cli-tests/native_release/acceptance.rs create mode 100644 test/dever-cli-tests/native_release/bootstrap.rs create mode 100644 test/dever-cli-tests/native_release/bootstrap_acceptance.rs create mode 100644 test/dever-cli-tests/native_release/command_acceptance.rs create mode 100644 test/dever-cli-tests/native_release/cross_acceptance.rs create mode 100644 test/dever-cli-tests/native_release/cross_ecosystems.rs create mode 100644 test/dever-cli-tests/native_release/ecosystem_acceptance.rs create mode 100644 test/dever-cli-tests/native_release/ecosystems.rs create mode 100644 test/dever-cli-tests/native_release/targets.rs create mode 100644 test/dever-cli-tests/new_project.rs create mode 100644 test/dever-cli-tests/packages.rs create mode 100644 test/dever-cli-tests/packages/managed_compile.rs create mode 100644 test/dever-cli-tests/release_source.rs create mode 100644 test/dever-cli-tests/shared_toolchain.rs create mode 100644 test/dever-cli-tests/support/command_library.c create mode 100644 test/dever-cli-tests/support/command_tool.c create mode 100644 test/dever-cli-tests/support/compiler_sleeper.c create mode 100644 test/dever-cli-tests/support/daemon.rs create mode 100644 test/dever-cli-tests/support/llvm_inputs.rs create mode 100644 test/dever-cli-tests/support/llvm_pack.rs create mode 100644 test/dever-cli-tests/support/native_acceptance_init.rs create mode 100644 test/dever-cli-tests/support/native_elf.rs create mode 100644 test/dever-cli-tests/support/native_objects.rs create mode 100644 test/dever-cli-tests/support/wheel.rs create mode 100644 test/dever-sandbox-tests/apparmor_transition.c create mode 100644 test/dever-sandbox-tests/isolation.rs create mode 100644 test/dever-sandbox-tests/syscalls.c create mode 100644 test/dever-sandbox-tests/trusted_helper.rs create mode 100644 test/dever-tests/Cargo.toml create mode 100644 test/dever-tests/examples/async_bench.rs create mode 100644 test/dever-tests/examples/live_bench.rs create mode 100644 test/dever-tests/examples/live_bench/config.rs create mode 100644 test/dever-tests/examples/live_bench/protocol.rs create mode 100644 test/dever-tests/examples/live_bench/run.rs create mode 100644 test/dever-tests/examples/network_bench.rs create mode 100644 test/dever-tests/examples/network_bench/config.rs create mode 100644 test/dever-tests/examples/network_bench/load.rs create mode 100644 test/dever-tests/examples/network_bench/metrics.rs create mode 100644 test/dever-tests/examples/network_bench/response.rs create mode 100644 test/dever-tests/examples/network_bench/service.rs create mode 100644 test/dever-tests/examples/network_bench/tls.rs create mode 100644 test/dever-tests/examples/support/benchmark_settings.rs create mode 100644 test/dever-tests/fixtures/command_fixture.rs create mode 100644 test/dever-tests/fixtures/component_fixture.rs create mode 100644 test/dever-tests/fixtures/performance/benchmark/allocation/app.dever create mode 100644 test/dever-tests/fixtures/performance/main.dever create mode 100644 test/dever-tests/fixtures/postgres_api/account_api.dever create mode 100644 test/dever-tests/fixtures/postgres_api/account_app.dever create mode 100644 test/dever-tests/fixtures/postgres_api/membership_app.dever create mode 100644 test/dever-tests/fixtures/tls/root.pem create mode 100644 test/dever-tests/fixtures/tls/server-key.pem create mode 100644 test/dever-tests/fixtures/tls/server.pem create mode 100644 test/dever-tests/tests/api_context.rs create mode 100644 test/dever-tests/tests/api_declarations.rs create mode 100644 test/dever-tests/tests/api_routes.rs create mode 100644 test/dever-tests/tests/application_config.rs create mode 100644 test/dever-tests/tests/application_testing.rs create mode 100644 test/dever-tests/tests/async_composition.rs create mode 100644 test/dever-tests/tests/async_native.rs create mode 100644 test/dever-tests/tests/async_network.rs create mode 100644 test/dever-tests/tests/async_runtime.rs create mode 100644 test/dever-tests/tests/async_stream.rs create mode 100644 test/dever-tests/tests/auth_runtime.rs create mode 100644 test/dever-tests/tests/authorization_postgres.rs create mode 100644 test/dever-tests/tests/authorization_runtime.rs create mode 100644 test/dever-tests/tests/cms_acceptance.rs create mode 100644 test/dever-tests/tests/concurrency.rs create mode 100644 test/dever-tests/tests/contract_api.rs create mode 100644 test/dever-tests/tests/contract_execution.rs create mode 100644 test/dever-tests/tests/contract_facts.rs create mode 100644 test/dever-tests/tests/contract_redundancy.rs create mode 100644 test/dever-tests/tests/contract_semantics.rs create mode 100644 test/dever-tests/tests/contract_syntax.rs create mode 100644 test/dever-tests/tests/core_native.rs create mode 100644 test/dever-tests/tests/core_semantics.rs create mode 100644 test/dever-tests/tests/differential.rs create mode 100644 test/dever-tests/tests/durable_jobs.rs create mode 100644 test/dever-tests/tests/error_effects.rs create mode 100644 test/dever-tests/tests/external_command.rs create mode 100644 test/dever-tests/tests/external_component.rs create mode 100644 test/dever-tests/tests/external_resources.rs create mode 100644 test/dever-tests/tests/formatter.rs create mode 100644 test/dever-tests/tests/hello_native.rs create mode 100644 test/dever-tests/tests/http2_client.rs create mode 100644 test/dever-tests/tests/http2_server.rs create mode 100644 test/dever-tests/tests/http_engine.rs create mode 100644 test/dever-tests/tests/http_library.rs create mode 100644 test/dever-tests/tests/json_library.rs create mode 100644 test/dever-tests/tests/live_library.rs create mode 100644 test/dever-tests/tests/live_network.rs create mode 100644 test/dever-tests/tests/logging.rs create mode 100644 test/dever-tests/tests/markdown_source.rs create mode 100644 test/dever-tests/tests/model_syntax.rs create mode 100644 test/dever-tests/tests/native_cache.rs create mode 100644 test/dever-tests/tests/native_entry.rs create mode 100644 test/dever-tests/tests/native_ownership.rs create mode 100644 test/dever-tests/tests/orm_connections.rs create mode 100644 test/dever-tests/tests/orm_data_migration.rs create mode 100644 test/dever-tests/tests/orm_migration.rs create mode 100644 test/dever-tests/tests/orm_native_sql.rs create mode 100644 test/dever-tests/tests/orm_performance_fixture.rs create mode 100644 test/dever-tests/tests/orm_query.rs create mode 100644 test/dever-tests/tests/orm_relations.rs create mode 100644 test/dever-tests/tests/orm_seed.rs create mode 100644 test/dever-tests/tests/performance.rs create mode 100644 test/dever-tests/tests/performance_port_fixture.rs create mode 100644 test/dever-tests/tests/pooled_library.rs create mode 100644 test/dever-tests/tests/pooled_network.rs create mode 100644 test/dever-tests/tests/port_adapter.rs create mode 100644 test/dever-tests/tests/postgres_api.rs create mode 100644 test/dever-tests/tests/postgres_orm.rs create mode 100644 test/dever-tests/tests/rest_model.rs create mode 100644 test/dever-tests/tests/runtime_foundations.rs create mode 100644 test/dever-tests/tests/secure_contracts.rs create mode 100644 test/dever-tests/tests/source_architecture.rs create mode 100644 test/dever-tests/tests/source_visibility.rs create mode 100644 test/dever-tests/tests/sqlite_orm.rs create mode 100644 test/dever-tests/tests/sqlite_runtime.rs create mode 100644 test/dever-tests/tests/standard_library.rs create mode 100644 test/dever-tests/tests/structured_concurrency.rs create mode 100644 test/dever-tests/tests/support/authorization.rs create mode 100644 test/dever-tests/tests/support/llvm.rs create mode 100644 test/dever-tests/tests/support/llvm_managed.rs create mode 100644 test/dever-tests/tests/support/llvm_module.rs create mode 100644 test/dever-tests/tests/support/mod.rs create mode 100644 test/dever-tests/tests/support/orm_fixture.rs create mode 100644 test/dever-tests/tests/support/orm_postgres_migration.rs create mode 100644 test/dever-tests/tests/support/orm_sqlite.rs create mode 100644 test/dever-tests/tests/support/postgres.rs create mode 100644 test/dever-tests/tests/support/process.rs create mode 100644 test/dever-tests/tests/support/sandbox.rs create mode 100644 test/dever-tests/tests/support/sqlite_inspect.rs create mode 100644 test/dever-tests/tests/support/temp.rs create mode 100644 test/dever-tests/tests/support/wire_peer.rs create mode 100644 test/dever-tests/tests/syntax_and_format.rs create mode 100644 test/dever-tests/tests/tenant_database.rs create mode 100644 test/dever-tests/tests/time_codec.rs create mode 100644 test/dever-tests/tests/upload_boundary.rs create mode 100644 test/ecosystem-release/arm64/bootstrap.sh create mode 100644 test/ecosystem-release/arm64/guest.sh create mode 100644 test/ecosystem-release/arm64/init.sh create mode 100644 test/ecosystem-release/arm64/probe.go create mode 100644 test/ecosystem-release/arm64/probe.mjs create mode 100644 test/ecosystem-release/arm64/probe.py create mode 100644 test/ecosystem-release/arm64/simulate.py create mode 100644 test/ecosystem-release/build_inputs.py create mode 100644 test/ecosystem-release/prepare.py create mode 100644 test/ecosystem-release/probe.go create mode 100644 test/ecosystem-release/probe.mjs create mode 100644 test/ecosystem-release/probe.py create mode 100644 test/ecosystem-release/test_build_inputs.py create mode 100644 test/external-sdk/check.py create mode 100644 test/external-sdk/typed_contract.json create mode 100644 test/external-sdk/typed_worker.go create mode 100644 test/external-sdk/typed_worker.mjs create mode 100644 test/external-sdk/typed_worker.py create mode 100644 test/go-managed/analyze/main.go create mode 100644 test/go-managed/make_pack.go create mode 100644 test/llvm_api_security_cases.rs create mode 100644 test/llvm_api_source.rs create mode 100644 test/llvm_application_source.rs create mode 100644 test/llvm_async_native.rs create mode 100644 test/llvm_async_source.rs create mode 100644 test/llvm_database_source.rs create mode 100644 test/llvm_external_source.rs create mode 100644 test/llvm_http_system_source.rs create mode 100644 test/llvm_job_security_cases.rs create mode 100644 test/llvm_job_test_source.rs create mode 100644 test/llvm_lowering.rs create mode 100644 test/llvm_managed_native.rs create mode 100644 test/llvm_native.rs create mode 100644 test/llvm_process_entry_source.rs create mode 100644 test/llvm_resource_native.rs create mode 100644 test/llvm_result_native.rs create mode 100644 test/llvm_stream_network_source.rs create mode 100644 test/native-runtime-abi/async-driver.c create mode 100644 test/native-runtime-abi/async-operations-driver.c create mode 100644 test/native-runtime-abi/database-driver.c create mode 100644 test/native-runtime-abi/driver-fixture.c create mode 100644 test/native-runtime-abi/driver.c create mode 100644 test/native-runtime-abi/log-driver.c create mode 100644 test/native-runtime-abi/managed-driver.c create mode 100644 test/native-runtime-abi/protocol-driver.c create mode 100644 test/native-runtime-abi/resource-driver.c create mode 100644 test/native-runtime-abi/test-suite-driver.c create mode 100644 test/native-runtime-abi/wire-driver.c create mode 100644 test/native_runtime_abi.rs create mode 100644 test/native_runtime_link.rs create mode 100644 test/onboarding/test_installer.py create mode 100644 test/performance/README.md create mode 100644 test/performance/build.py create mode 100644 test/performance/cms.py create mode 100644 test/performance/fixtures/bench.dever create mode 100644 test/performance/fixtures/http/adapter.dever create mode 100644 test/performance/fixtures/http/api.dever create mode 100644 test/performance/fixtures/http/app.dever create mode 100644 test/performance/fixtures/http/port.dever create mode 100644 test/performance/fixtures/live/adapter.dever create mode 100644 test/performance/fixtures/live/api.dever create mode 100644 test/performance/fixtures/live/app.dever create mode 100644 test/performance/fixtures/live/port.dever create mode 100644 test/performance/fixtures/orm/app.dever create mode 100644 test/performance/fixtures/orm/app/api.dever create mode 100644 test/performance/fixtures/orm/app/model/item.dever create mode 100644 test/performance/fixtures/profile/benchmark/command/api.dever create mode 100644 test/performance/fixtures/profile/benchmark/command/app.dever create mode 100644 test/performance/fixtures/runtime/adapter.dever create mode 100644 test/performance/fixtures/runtime/api.dever create mode 100644 test/performance/fixtures/runtime/app.dever create mode 100644 test/performance/fixtures/runtime/port.dever create mode 100644 test/performance/live.py create mode 100644 test/performance/process.py create mode 100644 test/performance/run.py create mode 100644 test/performance/settings.py create mode 100644 test/performance/test_cms.py create mode 100644 test/performance/test_live_bench.py create mode 100644 test/performance/test_network_bench.py create mode 100644 test/performance/test_peer_settings.py create mode 100644 test/performance/test_runner.py create mode 100644 test/runtime_entry_cleanup.rs create mode 100644 test/runtime_logging.rs diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..bec56db --- /dev/null +++ b/.gitattributes @@ -0,0 +1,9 @@ +# Trellis: append-only developer journals should merge cleanly across +# parallel sessions/worktrees — each session only appends a new block, so +# there is nothing to actually conflict on. +# +# Do NOT add a rule for workspace/*/index.md here — it is fully regenerated +# every session, so a real conflict there is expected and safe to resolve by +# picking either side (task state lives in task.json, not index.md). See +# .trellis/spec/cli/backend/directory-structure.md for details. +.trellis/workspace/*/journal-*.md merge=union diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..632822d --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +/target/ +/config/setting.json +/.release-key.*/ +/.trellis/workspace/ +/.agents/ +/.codex/ +**/__pycache__/ +**/*.pyc diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..5ed696f --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "skills"] + path = skills + url = https://github.com/shemic/dever-main-skills.git diff --git a/.trellis/.gitignore b/.trellis/.gitignore new file mode 100644 index 0000000..5a991ea --- /dev/null +++ b/.trellis/.gitignore @@ -0,0 +1,32 @@ +# Developer identity (local only) +.developer + +# Current task pointer (each dev works on different task) +.current-task + +# Session/window scoped runtime state +.runtime/ + +# Ralph Loop state file +.ralph-state.json + +# Agent runtime files +.agents/ +.agent-log +.session-id + +# Task directory runtime files +.plan-log + +# Atomic update temp files +*.tmp + +# Update backup directories +.backup-* + +# Conflict resolution temp files +*.new + +# Python cache +**/__pycache__/ +**/*.pyc diff --git a/.trellis/.template-hashes.json b/.trellis/.template-hashes.json new file mode 100644 index 0000000..2ea45a6 --- /dev/null +++ b/.trellis/.template-hashes.json @@ -0,0 +1,92 @@ +{ + "__version": 2, + "hashes": { + ".agents/skills/trellis-continue/SKILL.md": "2b948434883636a1f1e2a4ad5b8a3d7dc2edc93ed05da6bca82882e8c024e22b", + ".agents/skills/trellis-finish-work/SKILL.md": "161060fbcd44f787440d3a5c297a9f5223ea7774bb3021a50e376875a9ac5b2d", + ".agents/skills/trellis-start/SKILL.md": "79a5ba7a2aff3c72e06d7f4cd6942dc4f4f4092dd40f9c8e94f1838024a81e4d", + ".agents/skills/trellis-before-dev/SKILL.md": "00c9d1c83bc318e91b27a019bc954c2957cfad907ff20d854b596f16cad2c0f3", + ".agents/skills/trellis-brainstorm/SKILL.md": "17d9bf209730c14f584eb97f38198d0683ad3f8da982d9a6e6d4e84c853f9a94", + ".agents/skills/trellis-break-loop/SKILL.md": "f5a93699832f29dee443b53c135a7459519b371f689af191d15c29e8ee5c7bde", + ".agents/skills/trellis-check/SKILL.md": "dfb0600e95c19c7a83200465b6a92a5514ec9778ba4083efa881bed541cc74a8", + ".agents/skills/trellis-update-spec/SKILL.md": "003ce08a3404aeb50998029392c4d4e57b626edf526d3ebd585032bb92dcbb96", + ".agents/skills/trellis-channel/references/command-reference.md": "c4df3d940d89814310bfdaded20e84dd1d6b96786f6772688de7a123338d3c8e", + ".agents/skills/trellis-channel/references/forum.md": "407db39e8ebde47cea8637e3069bdb3d76ba42b43ed1d08ba97a56f26dbcfcb4", + ".agents/skills/trellis-channel/references/progress-debugging.md": "93b021fd8d5d3d4ee54952c40424ab91ca57c2ab8032bfbfc3b5c5e22f1464cd", + ".agents/skills/trellis-channel/references/workers.md": "d4481b2858bca82050cb8d131916da8a2ccc3857f199cdb9ced8bb41d592e2a7", + ".agents/skills/trellis-channel/references/workflows.md": "0782d602efde6e94a7d7d7950fd605eedc9a1f5a523908284e76b628fd618fd4", + ".agents/skills/trellis-channel/SKILL.md": "ff84801c511a736da018b4b1149348ebc9a16b7fc18d905ec10cfa7acbf269a3", + ".agents/skills/trellis-meta/references/customize-local/add-project-local-conventions.md": "86009ccb5d0373f399582da0bc570c4e5c6053c3c764857424ff93384f0e04e5", + ".agents/skills/trellis-meta/references/customize-local/change-agents.md": "4216ac3cc570038fd8ce3319a85932bb537fea9e0a7f4d11fa315cfa645c9c85", + ".agents/skills/trellis-meta/references/customize-local/change-context-loading.md": "350d319dc1ab99609ddbf52cf8c06c71bd97ba1a29ce2eac8b97d0bb192938bf", + ".agents/skills/trellis-meta/references/customize-local/change-hooks.md": "4c18b134f05d5ba1609c517ec94c97c9442b5f0670aa2211c487c31ed2a4f358", + ".agents/skills/trellis-meta/references/customize-local/change-skills-or-commands.md": "5b942e9f512e049a75e8dd9e8cc4d49786f6da61b22afc82bb447acf808f9fe2", + ".agents/skills/trellis-meta/references/customize-local/change-spec-structure.md": "31eccaad7097d96e66a45c1b4caea1ba4f2e54b7814184c3ebf82c87dafc4841", + ".agents/skills/trellis-meta/references/customize-local/change-task-lifecycle.md": "60ff9efb93604b87a461a4af30322d76750402a51e40f31531a7ff88d309996d", + ".agents/skills/trellis-meta/references/customize-local/change-workflow.md": "43fa780a2ca580de121b10893d49b99f978873deebbf45008c466e5ac6651519", + ".agents/skills/trellis-meta/references/customize-local/overview.md": "ce8f09e9f93ce9a48500763fb3a4db2b3908a5fbf4f985ab71dacebb404cf8f4", + ".agents/skills/trellis-meta/references/local-architecture/bundled-skills.md": "aa6a0bf83060205ee4ea621c467fb900a7db06b4476a4ad472cc4e248c887389", + ".agents/skills/trellis-meta/references/local-architecture/context-injection.md": "b1ebaac467bf0195e2a21f2f8492a8b7c91ff3e0f9312f853468cda0255df1c7", + ".agents/skills/trellis-meta/references/local-architecture/generated-files.md": "7eb2d452eddb4f4226f7578c2ec6d5ee0434ed172ba4c36107cc8bdff7554dc6", + ".agents/skills/trellis-meta/references/local-architecture/multi-agent-channel.md": "56e5070474aeca872e2d70c46feea5aaafecd3d3ec052c3f7b1877358dca62e9", + ".agents/skills/trellis-meta/references/local-architecture/overview.md": "50638fd9eaaba2e0edf2f2a84d920578d5b2fb7934031b174e417d3dc510b2a6", + ".agents/skills/trellis-meta/references/local-architecture/spec-system.md": "b8d8a6a0888b44a232c8f50161b9e20e903cf621ad7be4021715ab6fab226f47", + ".agents/skills/trellis-meta/references/local-architecture/task-system.md": "b154568e74f1c738a51633df9f2503f416b3afa5f309325224422162e0b707e4", + ".agents/skills/trellis-meta/references/local-architecture/workflow.md": "cfcdc6e4468a5d9c816e929fcca01640cd41cfdaaa4824118b40a8e460c927b6", + ".agents/skills/trellis-meta/references/local-architecture/workspace-memory.md": "e6427b46aba744563c2444b30df4043cd856561b7709ec2dece26095416421fd", + ".agents/skills/trellis-meta/references/platform-files/agents.md": "b806b1f0de6dfc74720014aee15ede767a58925b2b3b964f2e557ed39a65bbc0", + ".agents/skills/trellis-meta/references/platform-files/hooks-and-settings.md": "91e9855e418673bcd7770411e24e28390738b123532eef000beba0a29cf154e1", + ".agents/skills/trellis-meta/references/platform-files/overview.md": "1aec9087ccedd56a213af877b5db474131aa4b4789b6ac0e8da73b058aa43bd4", + ".agents/skills/trellis-meta/references/platform-files/platform-map.md": "3c0d4546461c06d1aaf006276e34e250e0ca7bbb5b853fb45c58114cebca4042", + ".agents/skills/trellis-meta/references/platform-files/skills-and-commands.md": "e39831d860bd27a04e7757f7bd941ab83b3c5b11ad4460cec03975b655f26cc3", + ".agents/skills/trellis-meta/SKILL.md": "ee7bcfd0023def62688339c4b63c50f2fbeea12a34eb89e8bb63a992b58cb168", + ".agents/skills/trellis-session-insight/references/cli-quick-reference.md": "c520353fe3fc00b9702ed4f780647c8fbd6d342e66527a03647f533a9fe09779", + ".agents/skills/trellis-session-insight/references/triggering-patterns.md": "121ecd23be83d1567e8ce15c366a81073d7a2b1d3ad616fce235c07ca1f1cc20", + ".agents/skills/trellis-session-insight/SKILL.md": "d20f1d20c6946e26ba6f848b9a016d9a16c803d87501462e5b11a2ced6f56643", + ".agents/skills/trellis-spec-bootstrap/references/mcp-setup.md": "df542fc8f279edd38046d26a7c8151804b708f57b24d4aa2733cea587a88c65e", + ".agents/skills/trellis-spec-bootstrap/references/repository-analysis.md": "0dae98d774f6e34559b9f3442888ac43e3a8af110c37cbefc49ce256986858b6", + ".agents/skills/trellis-spec-bootstrap/references/spec-task-planning.md": "ef493d028c3b0807a8a534bb71fb92a68129f273db763ad27ceb464a522e799d", + ".agents/skills/trellis-spec-bootstrap/references/spec-writing.md": "e9800fe9ed4a4cd87062ea1829cf2caa8d170ec15e141678a6a30e74c497f47d", + ".agents/skills/trellis-spec-bootstrap/SKILL.md": "97bfa68c06cebb558eb4464bc1b81f7d2d56040d75baa8de1ee5ad90cca0196a", + ".codex/agents/trellis-check.toml": "206fd96a8aa17e95ed344cc435661d597093ece571671f298ca64490422fffe0", + ".codex/agents/trellis-implement.toml": "388fb8f39797e0ee6cf4db447c859c79b4ac15f531f7e1e3c68c1d1d71a1c188", + ".codex/agents/trellis-research.toml": "4435ce73197ba1d29d40359a3279b6423f7e4f559a449f934c016808090066c4", + ".codex/hooks/session-start.py": "efdd015c7c4687227ce34a84ac6249d5554c704f737e00d4844c9284ddcf696f", + ".codex/hooks/inject-subagent-context.py": "9b68be8d08f85e824e39d901a29f14d38346dbeeda15880e4cba03076f2a8caa", + ".codex/hooks/inject-workflow-state.py": "71ef1aa1ed1051a697cae3c9ebffa518958a3bbd4d147434f382348829915f8c", + ".codex/hooks.json": "85a58ba7cdf1e19e7f75ddcc64e5680180c487ca266a74bd5005f31abeee2e02", + ".codex/config.toml": "9f2d20e28f0bc9c886312eca3ad3bba41533ef4615aaaafe25e98152302267bb", + "AGENTS.md": "6cacfe99748b435d0660c2463c697bc323d53798aecf3492283ca8eac1b29682", + ".trellis/agents/check.md": "edb4f57361407249a53bf5998ebf91c40d2b969e826a2c5e1b4e813a08bcb175", + ".trellis/agents/implement.md": "66e25ad046c94869442834bc3cdfbd5a9a7412d3ff54561d64d2886552c27e87", + ".trellis/config.yaml": "a966e6d374e9e6ff283cf761ccd99631323ee1754856cef51ad154ca0afb9dfa", + ".trellis/scripts/__init__.py": "1242be5b972094c2e141aecbe81a4efd478f6534e3d5e28306374e6a18fcf46c", + ".trellis/scripts/add_session.py": "f61a452ec5f7eece8d7e0f0dd56ce5851166cbc83ca4822e30acbb36eeb5896f", + ".trellis/scripts/common/__init__.py": "f258cd873681dc5a4d71db0dd71d10fd78656de55bec96418d6702503c16e87d", + ".trellis/scripts/common/active_task.py": "f52715bbcc0677607055903e799674d7d50d6fcbab413a8f453c4e4efc954726", + ".trellis/scripts/common/cli_adapter.py": "5d6bd9d6f5c631e7e792db7dd343351317f9643bc87b73a9a98abd51cefb4307", + ".trellis/scripts/common/config.py": "9a042d6f0e25a33a567510760e87c1e389af14a41faf1daeb1190a00f3df7f5e", + ".trellis/scripts/common/developer.py": "a0f61bb58063e2f4cb357bcb2c22fd5a6f1241286398beaa7130b2a8994a4afa", + ".trellis/scripts/common/git.py": "5605b71662e524b47632030d9db9245121a5cba85b5521eeb524ed464d840ff5", + ".trellis/scripts/common/git_context.py": "fa30ced454f1a91ffc9f8b2abeb32225e3447cbdc90bad783797374eba07265d", + ".trellis/scripts/common/io.py": "5dd8b9e297125ff58543a9bd4790c03ae1aca1d37b9e2ca60c4ffb47601c1cbb", + ".trellis/scripts/common/log.py": "471df6895cfac80f995edebbf9974f6b7440634b7a688f28b8331c868bc0f3cf", + ".trellis/scripts/common/packages_context.py": "efe158d7c99c2268851d0216fbb08de22836e418a8dbeb73575b8cc249eed7b7", + ".trellis/scripts/common/paths.py": "d8586d4d3f9bb2dd46b7f11b0f038b5340d283f41b6fa2c76bb8235c13bbb0b7", + ".trellis/scripts/common/safe_commit.py": "262ad740552279fe6ec6a2a9865b2965ce2448577cf973a4d702dfe2f5f91f04", + ".trellis/scripts/common/session_context.py": "3379ef1766e4e5ca77cbb7c040dbba3883fcca2548580299e3b38dbf22f4f7d5", + ".trellis/scripts/common/task_context.py": "245e654713e6030f9e7b5cee474efc06343ecc0d7f66f6455098874239355ce3", + ".trellis/scripts/common/task_queue.py": "0be61f713462b1fe4574927c82fc4704e678afe72dcb9813543aedf2f9e9e0c5", + ".trellis/scripts/common/task_store.py": "5712b0405e1fd0c2b7be5e5fbd420fa6eec2c49c8e4cf2b8fc69b7d2e4aeb016", + ".trellis/scripts/common/task_utils.py": "35f544317a95e98d05fe089794071482175739e8c1f3b3b0228dae48602ffe90", + ".trellis/scripts/common/tasks.py": "87aca6b73ec4524450cfff8e2ae561b1f2e53bd62a185f036a7a2d484d90c8a2", + ".trellis/scripts/common/trellis_config.py": "f1fc705ec463b0b9c163672dc9e50f10d29da967775a1900429205f7c4fcb51c", + ".trellis/scripts/common/types.py": "9962081cc2608fb9d1deb32c6880e336f62cdca6b338e7ae813304701e155ee9", + ".trellis/scripts/common/workflow_phase.py": "146b69c1ec0e628aeb720d3e34d8333f16050225037fe4233da9bbc4d4ac99c1", + ".trellis/scripts/get_context.py": "ca5bf9e90bdb1d75d3de182b95f820f9d108ab28793d29097b24fd71315adcf5", + ".trellis/scripts/get_developer.py": "84c27076323c3e0f2c9c8ed16e8aa865e225d902a187c37e20ee1a46e7142d8f", + ".trellis/scripts/hooks/linear_sync.py": "e09cc4ce4699aada908808718698f33f705a3edf55c4dcf8f777ad892f80ca79", + ".trellis/scripts/init_developer.py": "f9e6c0d882406e81c8cd6b1c5abb204b0befc0069ff89cf650cd536a80f8c60e", + ".trellis/scripts/task.py": "b415e0a2dbc9e8cba1ada0c7997999aaaaea0eec96d3419db640830e88e84cfe", + ".trellis/workflow.md": "eb4508a4f6792f3d53705c1f9e7d21807dabd774543bd7b01f56372a704f6ad0" + } +} \ No newline at end of file diff --git a/.trellis/.version b/.trellis/.version new file mode 100644 index 0000000..a12cadd --- /dev/null +++ b/.trellis/.version @@ -0,0 +1 @@ +0.6.16 \ No newline at end of file diff --git a/.trellis/agents/check.md b/.trellis/agents/check.md new file mode 100644 index 0000000..6c1bf13 --- /dev/null +++ b/.trellis/agents/check.md @@ -0,0 +1,70 @@ +--- +name: check +description: | + Code quality auditor for the Trellis channel runtime. Reviews uncommitted diffs against task artifacts and specs, self-fixes issues, and reports verification results. +provider: claude +labels: [trellis, check] +--- + +# Check Agent (channel runtime) + +You are the Check Agent spawned by `trellis channel spawn --agent check` inside the Trellis channel runtime. You receive an `Active task: ` line in your inbox; use it to locate task artifacts on disk. + +## Context + +Before reviewing, read in this order: + +1. `/check.jsonl` if present — spec manifest curated for this turn; read every listed file +2. `/prd.md` — requirements +3. `/design.md` if present — technical design +4. `/implement.md` if present — execution plan +5. `.trellis/spec/` — project-wide guidelines (load only what is relevant to the diff under review) + +## Core Responsibilities + +1. **Get the diff** — `git diff` / `git diff --staged` for uncommitted changes +2. **Review against task artifacts** — does the diff satisfy `prd.md` (and `design.md` / `implement.md` if present)? +3. **Review against specs** — naming, structure, type safety, error handling, conventions in `.trellis/spec/` +4. **Self-fix** — when an issue is mechanical and small, fix it directly with the editing tools you have +5. **Run verification** — project lint and typecheck on the changed scope +6. **Report** — concrete findings with `file:line` citations and what was fixed vs. what is open + +## Forbidden Operations + +- `git commit` +- `git push` +- `git merge` + +The supervising main session owns commits. Report the post-fix state; do not commit on its behalf. + +## Workflow + +1. Run `git diff --name-only` and `git diff` to scope the changes +2. Read the task artifacts and relevant spec files +3. For each issue: + - If mechanical (lint nit, missing type, wrong import, dead branch) → fix in-place + - If a design/judgment issue → record and report, do not silently rewrite +4. Run the project's lint and typecheck on the changed scope after self-fixes +5. Report + +## Report Format + +``` +## Self-Check Complete + +### Files Checked +- + +### Issues Found and Fixed +1. `:` — → + +### Issues Not Fixed +- `:` — — + +### Verification Results +- TypeCheck: +- Lint: + +### Summary +Checked files, found issues, fixed , open. +``` diff --git a/.trellis/agents/implement.md b/.trellis/agents/implement.md new file mode 100644 index 0000000..3262f79 --- /dev/null +++ b/.trellis/agents/implement.md @@ -0,0 +1,71 @@ +--- +name: implement +description: | + Code implementation expert for the Trellis channel runtime. Understands specs and task artifacts, then implements features. No git commit allowed. +provider: claude +labels: [trellis, implement] +--- + +# Implement Agent (channel runtime) + +You are the Implement Agent spawned by `trellis channel spawn --agent implement` inside the Trellis channel runtime. You receive an `Active task: ` line in your inbox; use it to locate task artifacts on disk. + +## Context + +Before implementing, read in this order: + +1. `/implement.jsonl` if present — spec manifest curated for this turn; read every listed file +2. `/prd.md` — requirements +3. `/design.md` if present — technical design +4. `/implement.md` if present — execution plan +5. `.trellis/spec/` — project-wide guidelines (load only what is relevant to the diff you are about to write) + +## Core Responsibilities + +1. **Understand specs** — read relevant spec files in `.trellis/spec/` +2. **Understand task artifacts** — read the artifacts listed above +3. **Implement features** — write code that follows specs and existing patterns +4. **Self-check** — run lint and typecheck on the changed scope before reporting + +## Forbidden Operations + +- `git commit` +- `git push` +- `git merge` + +The supervising main session owns commits. Report what changed; do not commit on its behalf. + +## Workflow + +1. Read relevant specs based on task type and the files in `implement.jsonl` if present +2. Read the task's `prd.md`, `design.md` if present, and `implement.md` if present +3. Implement features following specs and existing patterns +4. Run the project's lint and typecheck commands on the changed scope +5. Report files touched, key decisions, and verification results back to the channel + +## Code Standards + +- Follow existing code patterns +- Don't add unnecessary abstractions +- Only do what the PRD asks for; no speculative scope expansion +- Surface uncertainty back to the channel rather than guessing + +## Report Format + +``` +## Implementation Complete + +### Files Modified +- — + +### Implementation Summary +1. +2. + +### Verification Results +- Lint: +- TypeCheck: + +### Open Questions +- +``` diff --git a/.trellis/config.yaml b/.trellis/config.yaml new file mode 100644 index 0000000..dadd0b0 --- /dev/null +++ b/.trellis/config.yaml @@ -0,0 +1,161 @@ +# Trellis Configuration +# Project-level settings for the Trellis workflow system +# +# All values have sensible defaults. Only override what you need. + +#------------------------------------------------------------------------------- +# Session Recording +#------------------------------------------------------------------------------- + +# Commit message used when auto-committing journal/index changes +# after running add_session.py +session_commit_message: "chore: record journal" + +# Maximum lines per journal file before rotating to a new one +max_journal_lines: 2000 + +#------------------------------------------------------------------------------- +# Session Auto-Commit +#------------------------------------------------------------------------------- + +# Auto-commit behavior for session journal + task archive operations. +# - true (default): scripts auto-stage and auto-commit journal / task changes +# after add_session.py / task.py archive runs. +# - false: scripts do not touch git. Files (journal-*.md, task archive moves) +# are still written to disk; you decide whether to git add / commit. +# +# Use `false` if your project's .gitignore intentionally excludes `.trellis/` +# and you want session data kept local-only, or if you prefer to review +# staged changes manually before each commit. +# +# Accepts: true / false / yes / no / 1 / 0 / on / off (case-insensitive). +# +# session_auto_commit: true + +#------------------------------------------------------------------------------- +# Task Lifecycle Hooks +#------------------------------------------------------------------------------- + +# Shell commands to run after task lifecycle events. +# Each hook receives TASK_JSON_PATH environment variable pointing to task.json. +# Hook failures print a warning but do not block the main operation. +# +# hooks: +# after_create: +# - "echo 'Task created'" +# after_start: +# - "echo 'Task started'" +# after_finish: +# - "echo 'Task finished'" +# after_archive: +# - "echo 'Task archived'" + +#------------------------------------------------------------------------------- +# Monorepo / Packages +#------------------------------------------------------------------------------- + +# Declare packages for monorepo projects. +# Trellis auto-detects workspaces during `trellis init`, but you can also +# configure them manually here. +# +# packages: +# frontend: +# path: packages/frontend +# backend: +# path: packages/backend +# docs: +# path: docs-site +# type: submodule +# # For polyrepo / meta-repo layouts (independent .git in each subdir), +# # mark the package with `git: true`. The runtime treats it as an +# # independent repository for things like git-context display. +# webapp: +# path: ./webapp +# git: true + +# Default package used when --package is not specified. +# default_package: frontend + +#------------------------------------------------------------------------------- +# Channel worker OOM guard +#------------------------------------------------------------------------------- +# Default safeguards for `trellis channel spawn` workers. The guard runs +# at spawn time (cleans expired idle workers, then enforces the live-worker +# budget) and inside each supervisor (self-terminates a worker that stays +# continuously idle past `idle_timeout`). +# +# Precedence: CLI flag > env var (TRELLIS_CHANNEL_WORKER_IDLE_TIMEOUT / +# TRELLIS_CHANNEL_MAX_LIVE_WORKERS) > this config > built-in default. +# +# `idle_timeout: 0` disables idle cleanup (workers can sit idle forever +# unless explicitly killed or given `--timeout`). +# `max_live_workers: 0` disables the spawn-time budget check. +# +# `trusted_context_dirs` extends the `--file`/`--jsonl`/`--agent` containment +# check beyond the worker cwd — useful when `.trellis/tasks` or +# `.trellis/workspace` is a symlink to an external directory. Realpaths under +# any listed dir are accepted in addition to cwd. +# `auto_trust_trellis_symlinks: false` disables the narrow auto-trust of +# `.trellis/tasks` / `.trellis/workspace` when either is itself a top-level +# symlink (auto-trust is on by default). +# +channel: + worker_guard: + idle_timeout: 5m + max_live_workers: 6 + # trusted_context_dirs: + # - /work/user/trellis_workspace + # auto_trust_trellis_symlinks: false + +#------------------------------------------------------------------------------- +# Codex (dispatch behavior) +#------------------------------------------------------------------------------- +# Codex-only knob; other platforms ignore it. Default ("auto") dispatches +# trellis-implement / trellis-check / trellis-research sub-agents. This does +# not rely on inherited parent transcripts: `fork_turns` remains +# caller-controlled, while Codex's native SubagentStart hook injects task +# context when trusted and child-side loading remains the fallback when it is +# unavailable. Set to "inline" only to keep implementation and checks in the +# main session. "sub-agent" remains a backwards-compatible alias for "auto". +# Invalid explicit values safely use inline mode. +# +# In "auto" mode, dispatched sub-agents inherit the main session's model +# unless you pin one. To use a cheaper/faster model for implement/check/ +# research sub-agent work, edit `model` / `model_reasoning_effort` directly +# on the generated `.codex/agents/trellis-*.toml` files (see the commented +# hint lines in those files) — there is no config.yaml knob for this, +# `trellis update` preserves your edits across regeneration. +# +# codex: +# dispatch_mode: auto # or "inline"; legacy alias: "sub-agent" + +#------------------------------------------------------------------------------- +# Sub-agent context injection limits +#------------------------------------------------------------------------------- +# Caps how much task context (implement.jsonl / check.jsonl referenced files, +# plus prd.md / design.md / implement.md) gets inlined into a sub-agent's +# first prompt. Oversized files are truncated with a notice; once the total +# payload cap is reached, remaining files degrade to index lines (path + +# reason + size) instead of being inlined. +# +# All values are byte counts. `0` disables the corresponding limit. +# +# context_injection: +# max_file_bytes: 32768 # per implement.jsonl / check.jsonl referenced file +# max_artifact_bytes: 65536 # per task artifact (prd.md / design.md / implement.md) +# max_total_bytes: 131072 # whole injected payload; overflow degrades to index lines + +#------------------------------------------------------------------------------- +# Per-turn prompt injection +#------------------------------------------------------------------------------- +# Escape hatch for the per-turn breadcrumb. When a user +# prompt contains the skip keyword as a standalone word (case-insensitive, +# word-boundary match — "no-trellisfoo" does NOT count), the breadcrumb is +# skipped for that turn only. Does not affect SessionStart or sub-agent +# context injection. +# +# prompt_injection: +# skip_keyword: "no-trellis" # "" disables the escape hatch entirely + +codex: + dispatch_mode: auto diff --git a/.trellis/scripts/__init__.py b/.trellis/scripts/__init__.py new file mode 100755 index 0000000..815a137 --- /dev/null +++ b/.trellis/scripts/__init__.py @@ -0,0 +1,5 @@ +""" +Trellis Python Scripts + +This module provides Python implementations of Trellis workflow scripts. +""" diff --git a/.trellis/scripts/add_session.py b/.trellis/scripts/add_session.py new file mode 100755 index 0000000..40f3680 --- /dev/null +++ b/.trellis/scripts/add_session.py @@ -0,0 +1,1545 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Add a new session to journal file and update index.md. + +Usage: + python3 add_session.py --title "Title" --commit "hash" --summary "Summary" [--package cli] + python3 add_session.py --title "Title" --branch "feat/my-branch" + + # Pipe detailed content via stdin (use --stdin to opt in): + cat << 'EOF' | python3 add_session.py --stdin --title "Title" --summary "Summary" + + EOF + + # Structured content (repeatable; a section with no bullets is omitted): + python3 add_session.py --title "Title" --change "Did X" --test "Ran Y" --next-step "Do Z" + + # Commit evidence that cannot be resolved from the local object database + # (amended, not yet fetched) needs an explicit subject, one per OID: + python3 add_session.py --title "Title" --commit "abc1234" \ + --commit-subject "abc1234=fix(cli): stop truncating the manifest" + +Commit evidence: + Every --commit token is a bounded hex OID (7-40 chars). Each one is + resolved to its real subject from the local object database BEFORE + anything is written. An OID that cannot be resolved fails the command + before the journal or index is touched — no placeholder prose is ever + recorded. Pass "-" (the default) for a planning session with no commits. + +Retry convergence: + The journal append, the index row and the optional auto-commit are one + resumable operation. Every entry carries a fingerprint marker derived from + its own inputs, so re-running an interrupted command repairs the pending + record instead of appending a second one, and a failed auto-commit exits + non-zero with the checkpoint to resume from. A record that is already + committed is never reused: an identical later request is a new session + unless an explicit --idempotency-key says otherwise. + +Branch resolution order: + 1. --branch CLI arg (explicit) + 2. task.json branch field (from active task, if still exists) + 3. git branch --show-current (auto-detect) + 4. None (omitted gracefully) +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import sys +from datetime import datetime +from pathlib import Path + +from common.paths import ( + DIR_TASKS, + DIR_WORKFLOW, + FILE_JOURNAL_PREFIX, + get_repo_root, + get_current_task, + get_developer, + get_workspace_dir, +) +from common.developer import ensure_developer +from common.git import branch_exists_locally, run_git +from common.io import write_text_atomic +from common.log import Colors, colored +from common.safe_commit import ( + print_gitignore_warning, + safe_git_add, + safe_trellis_paths_to_add, +) +from common.tasks import load_task +from common.types import TaskInfo +from common.config import ( + get_packages, + get_session_auto_commit, + get_session_commit_message, + get_max_journal_lines, + is_monorepo, + resolve_package, + validate_package, +) + + +# Bounded, argv-safe input shapes. Anything outside them is rejected during +# preflight, before a single byte is written. +COMMIT_TOKEN_RE = re.compile(r"^[0-9a-fA-F]{7,40}$") +IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._-]{1,64}$") +MAX_SUBJECT_LEN = 500 + +# Machine-readable record identity. An HTML comment renders as nothing, so the +# human evidence in the entry is unchanged by its presence. +MARKER_PREFIX = "$") +ENTRY_DATE_RE = re.compile(r"^\*\*Date\*\*: (\d{4}-\d{2}-\d{2})\s*$") +SESSION_HEADING_RE = re.compile(r"^## Session (\d+):", re.MULTILINE) + +# Recording states, in the order the operation walks them. +STATE_ABSENT = "absent" +STATE_JOURNAL_RECORDED = "journal-recorded" +STATE_INDEX_RECORDED = "index-recorded" +STATE_COMMITTED = "committed" + +# Auto-commit outcomes. +COMMIT_DONE = "committed" +COMMIT_SKIPPED = "skipped" +COMMIT_BLOCKED = "blocked" +COMMIT_FAILED = "failed" + +# Git probes against other refs are best-effort; never let one hang a session. +GIT_PROBE_TIMEOUT = 15.0 +# Upper bound on the refs folded into session numbering. A repo with hundreds +# of stale branches must not turn session recording into a tree walk. +MAX_CONVERGENCE_REFS = 100 + + +# ============================================================================= +# Helper Functions +# ============================================================================= + +def get_latest_journal_info(dev_dir: Path) -> tuple[Path | None, int, int]: + """Get latest journal file info. + + Returns: + Tuple of (file_path, file_number, line_count). + """ + latest_file: Path | None = None + latest_num = -1 + + for f in dev_dir.glob(f"{FILE_JOURNAL_PREFIX}*.md"): + if not f.is_file(): + continue + + match = re.search(r"(\d+)$", f.stem) + if match: + num = int(match.group(1)) + if num > latest_num: + latest_num = num + latest_file = f + + if latest_file: + lines = len(latest_file.read_text(encoding="utf-8").splitlines()) + return latest_file, latest_num, lines + + return None, 0, 0 + + +def get_current_session(index_file: Path) -> int: + """Get current session number from index.md.""" + if not index_file.is_file(): + return 0 + + content = index_file.read_text(encoding="utf-8") + return _max_session_in_index(content) + + +def _max_session_in_index(content: str) -> int: + """Read the `Total Sessions` counter out of an index.md body.""" + for line in content.splitlines(): + if "Total Sessions" in line: + match = re.search(r":\s*(\d+)", line) + if match: + return int(match.group(1)) + return 0 + + +def _max_session_in_journal(content: str) -> int: + """Highest `## Session N:` heading in a journal body (0 when none).""" + numbers = [int(m.group(1)) for m in SESSION_HEADING_RE.finditer(content)] + return max(numbers) if numbers else 0 + + +def _extract_journal_num(filename: str) -> int: + """Extract journal number from filename for sorting.""" + match = re.search(r"(\d+)", filename) + return int(match.group(1)) if match else 0 + + +def count_journal_files(dev_dir: Path, active_num: int) -> str: + """Count journal files and return table rows.""" + active_file = f"{FILE_JOURNAL_PREFIX}{active_num}.md" + result_lines = [] + + files = sorted( + [f for f in dev_dir.glob(f"{FILE_JOURNAL_PREFIX}*.md") if f.is_file()], + key=lambda f: _extract_journal_num(f.stem), + reverse=True + ) + + for f in files: + filename = f.name + lines = len(f.read_text(encoding="utf-8").splitlines()) + status = "Active" if filename == active_file else "Archived" + result_lines.append(f"| `{filename}` | ~{lines} | {status} |") + + return "\n".join(result_lines) + + +def get_current_git_branch(repo_root: Path) -> str | None: + """Return the current checkout branch, or None for detached/non-git states.""" + rc, branch_out, _ = run_git(["branch", "--show-current"], cwd=repo_root) + if rc != 0: + return None + detected = branch_out.strip() + return detected or None + + +def branch_ref_exists(repo_root: Path, branch: str) -> bool: + """Return True when branch exists locally or as the local origin ref.""" + for ref in (f"refs/heads/{branch}", f"refs/remotes/origin/{branch}"): + rc, _, _ = run_git(["show-ref", "--verify", "--quiet", ref], cwd=repo_root) + if rc == 0: + return True + return False + + +def resolve_session_branch( + repo_root: Path, + cli_branch: str | None, + task_data: TaskInfo | None, +) -> str | None: + """Resolve journal branch without trusting stale task.json branch fields.""" + if cli_branch: + return cli_branch + + current_branch = get_current_git_branch(repo_root) + raw_task_branch = task_data.raw.get("branch") if task_data else None + task_branch = raw_task_branch.strip() if isinstance(raw_task_branch, str) else "" + if not task_branch: + return current_branch + + if branch_ref_exists(repo_root, task_branch): + return task_branch + + if current_branch: + print( + f"Warning: task.json branch '{task_branch}' no longer exists locally or as origin/{task_branch}; using current branch '{current_branch}'.", + file=sys.stderr, + ) + return current_branch + + print( + f"Warning: task.json branch '{task_branch}' no longer exists locally or as origin/{task_branch}; omitting branch.", + file=sys.stderr, + ) + return None + + +def is_git_worktree(repo_root: Path) -> bool: + """Return True when repo_root is a linked worktree (not the main working tree). + + Standard test: `git rev-parse --git-dir` (per-worktree) differs from + `git rev-parse --git-common-dir` (shared across all worktrees) once both + are resolved to absolute paths. In the main working tree these are the + same directory. + """ + rc_dir, git_dir, _ = run_git(["rev-parse", "--git-dir"], cwd=repo_root) + rc_common, git_common_dir, _ = run_git( + ["rev-parse", "--git-common-dir"], cwd=repo_root + ) + if rc_dir != 0 or rc_common != 0: + return False + + git_dir_path = (repo_root / git_dir.strip()).resolve() + git_common_dir_path = (repo_root / git_common_dir.strip()).resolve() + return git_dir_path != git_common_dir_path + + +def warn_if_parallel_worktree(repo_root: Path) -> None: + """Non-blocking note: index.md conflicts across parallel worktrees/branches + are expected and safe. Only fires when running in a linked git worktree + (not the main tree) with `session_auto_commit` enabled (#415 quick-fix tier). + """ + if not get_session_auto_commit(repo_root): + return + if not is_git_worktree(repo_root): + return + print( + colored( + "[NOTE] Running in a git worktree with session_auto_commit enabled: " + "journal-*.md files auto-merge via .gitattributes, but index.md " + "conflicts across parallel worktrees/branches are expected and safe " + "to resolve by picking either side (task state lives in task.json, " + "not index.md). See .trellis/spec/cli/backend/directory-structure.md " + '("Workspace Journal Merge Behavior").', + Colors.YELLOW, + ), + file=sys.stderr, + ) + + +def create_new_journal_file( + dev_dir: Path, num: int, developer: str, today: str, max_lines: int = 2000, +) -> Path | None: + """Create a new journal file. Returns None when the write fails.""" + prev_num = num - 1 + new_file = dev_dir / f"{FILE_JOURNAL_PREFIX}{num}.md" + + content = f"""# Journal - {developer} (Part {num}) + +> Continuation from `{FILE_JOURNAL_PREFIX}{prev_num}.md` (archived at ~{max_lines} lines) +> Started: {today} + +--- + +""" + if not write_text_atomic(new_file, content): + return None + return new_file + + +# ============================================================================= +# Commit evidence (R1) — accurate subjects or nothing +# ============================================================================= + +def parse_commit_tokens(commit: str) -> tuple[list[str], str | None]: + """Split --commit into bounded hex OIDs. Returns (tokens, error).""" + raw = (commit or "").strip() + if not raw or raw == "-": + return [], None + + tokens: list[str] = [] + for part in raw.split(","): + token = part.strip() + if not token: + continue + if not COMMIT_TOKEN_RE.match(token): + return [], ( + f"invalid --commit token '{token}': expected a 7-40 character hex " + "commit OID, or '-' for a planning session with no commits" + ) + token = token.lower() + if token not in tokens: + tokens.append(token) + return tokens, None + + +def parse_subject_overrides(values: list[str] | None) -> tuple[dict[str, str], str | None]: + """Parse `--commit-subject =` into a one-to-one mapping.""" + overrides: dict[str, str] = {} + for raw in values or []: + if "=" not in raw: + return {}, ( + f"invalid --commit-subject '{raw}': expected =" + ) + oid_part, subject = raw.split("=", 1) + oid = oid_part.strip().lower() + subject = subject.strip() + if not COMMIT_TOKEN_RE.match(oid): + return {}, ( + f"invalid --commit-subject key '{oid_part.strip()}': expected a " + "7-40 character hex commit OID" + ) + if not subject: + return {}, f"invalid --commit-subject for '{oid}': subject is empty" + if len(subject) > MAX_SUBJECT_LEN: + return {}, ( + f"invalid --commit-subject for '{oid}': subject exceeds " + f"{MAX_SUBJECT_LEN} characters" + ) + if oid in overrides: + return {}, f"duplicate --commit-subject mapping for '{oid}'" + overrides[oid] = subject + return overrides, None + + +def resolve_commit_subject(repo_root: Path, oid: str) -> str | None: + """Resolve one OID to its subject from the local object database. + + Passed as argv (never interpolated into a shell) and peeled with + ``^{commit}`` so a hex-looking ref name cannot resolve to a tree or tag. + Returns None when the object is missing, ambiguous, or has no subject. + """ + rc, out, _ = run_git( + ["show", "-s", "--format=%s", f"{oid}^{{commit}}", "--"], cwd=repo_root + ) + if rc != 0: + return None + for line in out.splitlines(): + subject = line.strip() + if subject: + return subject[:MAX_SUBJECT_LEN] + return None + + +def build_commit_evidence( + repo_root: Path, + tokens: list[str], + overrides: dict[str, str], +) -> tuple[list[tuple[str, str]], str | None]: + """Pair every commit OID with an accurate subject, or fail. + + There is no placeholder path: an OID that neither resolves locally nor + carries an explicit mapping stops the command before any mutation. + """ + evidence: list[tuple[str, str]] = [] + unresolved: list[str] = [] + + for oid in tokens: + if oid in overrides: + evidence.append((oid, overrides[oid])) + continue + subject = resolve_commit_subject(repo_root, oid) + if subject is None: + unresolved.append(oid) + continue + evidence.append((oid, subject)) + + if unresolved: + return [], ( + "cannot resolve commit evidence for: " + + ", ".join(unresolved) + + ". Fetch the objects, correct the OID, or pass the exact subject " + "with --commit-subject =. Nothing was written." + ) + + unused = sorted(oid for oid in overrides if oid not in tokens) + if unused: + return [], ( + "--commit-subject supplied for OIDs that are not in --commit: " + + ", ".join(unused) + + ". The mapping must be one-to-one. Nothing was written." + ) + + return evidence, None + + +def escape_markdown_cell(text: str) -> str: + """Make a commit subject safe inside a Markdown table cell.""" + collapsed = " ".join(text.split()) + return collapsed.replace("\\", "\\\\").replace("|", "\\|") + + +# ============================================================================= +# Record identity (R2) +# ============================================================================= + +def _normalize_text(value: str | None) -> str: + return " ".join((value or "").split()) + + +def _fingerprint_payload( + developer: str, + title: str, + summary: str, + package: str | None, + branch: str | None, + evidence: list[tuple[str, str]], + changes: list[str] | None, + extra_content: str | None, + tests: list[str] | None, + next_steps: list[str] | None, + idempotency_key: str | None, +) -> dict: + """Normalized semantic inputs of one record, shared by both schemes.""" + return { + "developer": developer, + "title": _normalize_text(title), + "summary": _normalize_text(summary), + "package": package or "", + "branch": branch or "", + "commits": [[oid, _normalize_text(subject)] for oid, subject in evidence], + "changes": [_normalize_text(c) for c in changes or []], + "extra": _normalize_text(extra_content), + "tests": [_normalize_text(t) for t in tests or []], + "next_steps": [_normalize_text(n) for n in next_steps or []], + "idempotency_key": idempotency_key or "", + } + + +def _hash_payload(payload: dict) -> str: + encoded = json.dumps(payload, sort_keys=True, ensure_ascii=False).encode("utf-8") + return hashlib.sha256(encoded).hexdigest()[:16] + + +def compute_record_fingerprint(payload: dict) -> str: + """Bounded fingerprint over the normalized semantic inputs of one record. + + This is the retry key while the record is still pending in the worktree — + not a global dedupe key. Two genuinely separate sessions with identical + prose differ only if the caller says so, which is why an already-committed + match is never adopted (see classify_record). + + Deliberately date-free (v2). v1 mixed in the calendar date, which made a + record unfindable by its own retry across a midnight rollover: the journal + and index were already written, the commit had failed, and the recomputed + fingerprint no longer matched the marker sitting in the journal, so the + retry appended a second entry for one session. + + Dropping the date cannot over-collapse two same-day sessions, because the + date was equal for both of them anyway. Across days it only ever separated + records with byte-identical prose, commits, branch and package — and a + *committed* record of that shape is already refused for reuse by + cmd_add_session, which steps to the next `generation` of the payload + rather than reusing the committed marker. What is left + is exactly the question this key should answer: is there an uncommitted + record in this worktree matching these inputs? + """ + return _hash_payload(payload) + + +def compute_legacy_fingerprint(payload: dict, date: str) -> str: + """The v1 fingerprint, for resolving markers written before the change. + + Lookup only — nothing writes this scheme any more. + """ + return _hash_payload({**payload, "date": date}) + + +def render_marker(fingerprint: str) -> str: + """Machine-readable record marker; renders as nothing in Markdown.""" + return f"{MARKER_PREFIX} v={MARKER_VERSION} fp={fingerprint} -->" + + +def render_legacy_marker(fingerprint: str) -> str: + """The v1 marker form: no version tag. Read, never written.""" + return f"{MARKER_PREFIX} fp={fingerprint} -->" + + +# ============================================================================= +# Session numbering (collision-proof across concurrent branches) +# ============================================================================= + +def _repo_relative(repo_root: Path, path: Path) -> str | None: + try: + return path.resolve().relative_to(repo_root.resolve()).as_posix() + except (OSError, ValueError): + return None + + +def _default_branch_local(repo_root: Path) -> str | None: + """Resolve the default branch without ever touching the network. + + `resolve_default_branch()` falls back to `git remote show origin`, which + can block on a fetch. Session recording is a hot path, so it stops at the + local symbolic ref and the two conventional names. + """ + rc, out, _ = run_git( + ["symbolic-ref", "--quiet", "refs/remotes/origin/HEAD"], + cwd=repo_root, + timeout=GIT_PROBE_TIMEOUT, + ) + if rc == 0 and out.strip(): + return out.strip().rsplit("/", 1)[-1] + for candidate in ("main", "master"): + if branch_exists_locally(candidate, repo_root): + return candidate + return None + + +def _convergence_refs(repo_root: Path) -> list[str]: + """Refs whose recorded workspace state must be folded into the counter. + + Ordered by how much they matter, then capped: HEAD and the default branch + first (the merge target every branch eventually meets), then every other + local head — a parallel worktree's branch lives there and is the case that + actually collided — then remote-tracking refs. + """ + refs: list[str] = [] + + def add(ref: str) -> None: + if ref not in refs: + refs.append(ref) + + rc, _, _ = run_git(["rev-parse", "--verify", "--quiet", "HEAD"], cwd=repo_root) + if rc == 0: + add("HEAD") + + default = _default_branch_local(repo_root) + if default: + for ref in (f"refs/heads/{default}", f"refs/remotes/origin/{default}"): + rc, _, _ = run_git( + ["show-ref", "--verify", "--quiet", ref], cwd=repo_root + ) + if rc == 0: + add(ref) + + for pattern in ("refs/heads/", "refs/remotes/"): + rc, out, _ = run_git( + ["for-each-ref", "--format=%(refname)", pattern], + cwd=repo_root, + timeout=GIT_PROBE_TIMEOUT, + ) + if rc != 0: + continue + for line in out.splitlines(): + ref = line.strip() + # A symbolic ref like refs/remotes/origin/HEAD is a duplicate of + # the branch it points at and git grep rejects some of them. + if ref and not ref.endswith("/HEAD"): + add(ref) + + return refs[:MAX_CONVERGENCE_REFS] + + +def _max_session_across_refs(repo_root: Path, refs: list[str], dev_rel: str) -> int: + """Highest session number recorded under `dev_rel` in any of `refs`. + + One `git grep` over every ref rather than an ls-tree + show per ref: the + number of branches in a real repo is unbounded, the per-ref cost is not. + Exit status 1 (no match) is a normal outcome, not an error. + """ + if not refs: + return 0 + + rc, out, _ = run_git( + [ + "grep", + "--no-color", + "-h", + "-E", + "-e", + r"^## Session [0-9]+:", + "-e", + r"Total Sessions", + ] + + refs + + ["--", dev_rel], + cwd=repo_root, + timeout=GIT_PROBE_TIMEOUT, + ) + if rc > 1 or not out: + return 0 + + # `git grep` prefixes each hit with `::`, so the numbers are + # matched anywhere in the line rather than anchored. + numbers = [int(m.group(1)) for m in re.finditer(r"## Session (\d+):", out)] + numbers += [ + int(m.group(1)) for m in re.finditer(r"Total Sessions[^\d]{0,8}(\d+)", out) + ] + return max(numbers) if numbers else 0 + + +def max_local_session(dev_dir: Path, index_file: Path) -> int: + """Highest session number visible in the working tree.""" + highest = get_current_session(index_file) + for f in dev_dir.glob(f"{FILE_JOURNAL_PREFIX}*.md"): + if not f.is_file(): + continue + try: + content = f.read_text(encoding="utf-8") + except OSError: + continue + highest = max(highest, _max_session_in_journal(content)) + return highest + + +def resolve_next_session(repo_root: Path, dev_dir: Path, index_file: Path) -> int: + """Next session number, converged across the local tree and other branches. + + Deriving the number from the working tree alone lets two branches that + each record before merging claim the same number (observed twice on + 2026-08-06). Taking the union with every recorded ref — the default + branch, and the other local heads a parallel worktree records on — makes + the number monotonic across concurrent branches; `journal-*.md` + merge=union then keeps both entries when the branches meet. + """ + highest = max_local_session(dev_dir, index_file) + + dev_rel = _repo_relative(repo_root, dev_dir) + if dev_rel: + highest = max( + highest, + _max_session_across_refs( + repo_root, _convergence_refs(repo_root), dev_rel + ), + ) + + return highest + 1 + + +# ============================================================================= +# Pending-record classifier (R3/R4) +# ============================================================================= + +def find_marker_entries(dev_dir: Path, marker: str) -> list[tuple[Path, int | None]]: + """Locate every journal entry carrying `marker`. + + Returns (journal_file, session_number); the number is None when the marker + is not directly under a `## Session N:` heading, which is malformed + pending evidence and must never be guessed into completion. + """ + hits: list[tuple[Path, int | None]] = [] + for f in sorted(dev_dir.glob(f"{FILE_JOURNAL_PREFIX}*.md")): + if not f.is_file(): + continue + try: + lines = f.read_text(encoding="utf-8").splitlines() + except OSError: + continue + for i, line in enumerate(lines): + if line.strip() != marker: + continue + session_num: int | None = None + for j in range(i - 1, max(-1, i - 4), -1): + match = SESSION_HEADING_RE.match(lines[j]) + if match: + session_num = int(match.group(1)) + break + hits.append((f, session_num)) + return hits + + +def content_at_head(repo_root: Path, path: Path) -> str | None: + """File content as committed at HEAD, or None when it isn't there. + + The `./` prefix makes the path cwd-relative. `HEAD:` alone is + relative to the *git* toplevel, which is not always `repo_root` — the + Trellis root is the nearest directory holding `.trellis/`, which can sit + below the git root. Without it, every lookup in that layout fails and a + committed record looks pending. + """ + rel = _repo_relative(repo_root, path) + if not rel: + return None + rc, out, _ = run_git(["show", f"HEAD:./{rel}"], cwd=repo_root) + return out if rc == 0 else None + + +def index_has_session_row(index_file: Path, session_num: int) -> bool: + """Whether the session-history block already holds this session's row.""" + if not index_file.is_file(): + return False + try: + lines = index_file.read_text(encoding="utf-8").splitlines() + except OSError: + return False + + row_re = re.compile(r"^\|\s*%d\s*\|" % session_num) + in_history = False + for line in lines: + if "@@@auto:session-history" in line: + in_history = True + continue + if "@@@/auto:session-history" in line: + in_history = False + continue + if in_history and row_re.match(line): + return True + return False + + +def _entry_date_at(lines: list[str], marker_index: int) -> str | None: + """The `**Date**:` value belonging to the entry whose marker is at `marker_index`. + + `generate_session_content` renders it two lines below the marker. Scanning + a short window rather than a fixed offset keeps this working if blank-line + spacing around the header ever changes, and stops well before it could + reach the next entry. + """ + for line in lines[marker_index + 1:marker_index + 6]: + match = ENTRY_DATE_RE.match(line.strip()) + if match: + return match.group(1) + return None + + +def resolve_effective_marker( + dev_dir: Path, + payload: dict, + marker: str, +) -> tuple[str, str | None]: + """The marker this record actually carries, across both schemes. + + Returns (marker, error). The v2 marker wins whenever an entry carries it, + which is every record written since the change and costs one scan. + + Otherwise this looks for a pending record written under v1. Those markers + are only a hash, but the entry renders its own `**Date**:` line right + below, so the date that produced the v1 fingerprint is recoverable from + the entry itself: recompute with that date and compare against what is + actually written there. That is exact, and unlike a +/-1 day window it does + not quietly fail a retry resumed after a weekend. + + Returning the v1 marker leaves the entry untouched — it is an in-flight + record about to be committed, and rewriting its marker mid-repair would + move it to a state the machine does not model. + """ + if find_marker_entries(dev_dir, marker): + return marker, None + + matches: set[str] = set() + for journal in sorted(dev_dir.glob(f"{FILE_JOURNAL_PREFIX}*.md")): + if not journal.is_file(): + continue + try: + lines = journal.read_text(encoding="utf-8").splitlines() + except OSError: + continue + for i, line in enumerate(lines): + legacy = LEGACY_MARKER_RE.match(line.strip()) + if not legacy: + continue + date = _entry_date_at(lines, i) + if date is None: + continue + if compute_legacy_fingerprint(payload, date) == legacy.group(1): + matches.add(line.strip()) + + if len(matches) > 1: + return "", ( + f"found {len(matches)} pending journal entries matching this record " + "under the pre-versioning marker scheme. Refusing to guess which one " + "to resume — remove the duplicate entry or pass --idempotency-key to " + "record a new session." + ) + if matches: + return matches.pop(), None + return marker, None + + +def classify_record( + repo_root: Path, + dev_dir: Path, + index_file: Path, + marker: str, +) -> tuple[str, Path | None, int | None, str | None]: + """Classify the current state of this exact record. + + Returns (state, journal_file, session_num, error). Only a unique, exact, + still-uncommitted match is ever adopted; anything ambiguous, malformed or + partially written comes back as an error so the caller fails safely. + """ + hits = find_marker_entries(dev_dir, marker) + + if not hits: + return STATE_ABSENT, None, None, None + + if len(hits) > 1: + files = ", ".join(sorted({p.name for p, _ in hits})) + return "", None, None, ( + f"found {len(hits)} journal entries carrying this record's marker " + f"({files}). Refusing to guess which one to resume — remove the " + "duplicate entry or pass --idempotency-key to record a new session." + ) + + journal_file, session_num = hits[0] + if session_num is None: + return "", None, None, ( + f"{journal_file.name} carries this record's marker without a " + "'## Session N:' heading above it. The pending record is malformed; " + "repair it by hand before retrying." + ) + + head_content = content_at_head(repo_root, journal_file) + if head_content is not None and marker in head_content: + return STATE_COMMITTED, journal_file, session_num, None + + if index_has_session_row(index_file, session_num): + return STATE_INDEX_RECORDED, journal_file, session_num, None + + recorded_total = get_current_session(index_file) + if recorded_total > session_num: + return "", None, None, ( + f"journal entry for session {session_num} has no index row, but " + f"index.md already counts {recorded_total} sessions. Repairing the " + "row would rewrite a later record — resolve index.md by hand." + ) + + return STATE_JOURNAL_RECORDED, journal_file, session_num, None + + +# ============================================================================= +# Rendering +# ============================================================================= + +def _render_bullet_section(header: str, items: list[str], bullet_prefix: str = "- ") -> str: + """Render a Markdown section as bullets, or "" when there is no content. + + A section with zero provided values is omitted entirely from the + rendered entry rather than falling back to a placeholder string. + """ + if not items: + return "" + bullets = "\n".join(f"{bullet_prefix}{item}" for item in items) + return f"\n\n### {header}\n\n{bullets}" + + +def _render_main_changes(changes: list[str], extra_content: str | None) -> str: + """Render the Main Changes section from --change bullets or freeform content.""" + if changes: + return _render_bullet_section("Main Changes", changes) + if extra_content: + return f"\n\n### Main Changes\n\n{extra_content}" + return "" + + +def generate_session_content( + session_num: int, + title: str, + evidence: list[tuple[str, str]], + summary: str, + today: str, + marker: str, + package: str | None = None, + branch: str | None = None, + changes: list[str] | None = None, + extra_content: str | None = None, + tests: list[str] | None = None, + next_steps: list[str] | None = None, +) -> str: + """Generate session content.""" + if evidence: + commit_table = """| Hash | Message | +|------|---------|""" + for oid, subject in evidence: + commit_table += f"\n| `{oid}` | {escape_markdown_cell(subject)} |" + else: + commit_table = "(No commits - planning session)" + + package_line = f"\n**Package**: {package}" if package else "" + branch_line = f"\n**Branch**: `{branch}`" if branch else "" + + main_changes_section = _render_main_changes(changes or [], extra_content) + testing_section = _render_bullet_section("Testing", tests or [], bullet_prefix="- [OK] ") + next_steps_section = _render_bullet_section("Next Steps", next_steps or []) + + return f""" + +## Session {session_num}: {title} +{marker} + +**Date**: {today} +**Task**: {title}{package_line}{branch_line} + +### Summary + +{summary}{main_changes_section} + +### Git Commits + +{commit_table}{testing_section} + +### Status + +[OK] **Completed**{next_steps_section} +""" + + +def format_commit_display(evidence: list[tuple[str, str]]) -> str: + """Index-table rendering of the commit OIDs.""" + if not evidence: + return "-" + return ", ".join(f"`{oid}`" for oid, _ in evidence) + + +def update_index( + index_file: Path, + dev_dir: Path, + title: str, + evidence: list[tuple[str, str]], + new_session: int, + active_file: str, + today: str, + branch: str | None = None, +) -> bool: + """Update index.md with new session info.""" + commit_display = format_commit_display(evidence) + + # Get file number from active_file name + match = re.search(r"(\d+)", active_file) + active_num = int(match.group(1)) if match else 0 + files_table = count_journal_files(dev_dir, active_num) + + print(f"Updating index.md for session {new_session}...") + print(f" Title: {title}") + print(f" Commit: {commit_display}") + print(f" Active File: {active_file}") + print() + + content = index_file.read_text(encoding="utf-8") + + if "@@@auto:current-status" not in content: + print("Error: Markers not found in index.md. Please ensure markers exist.", file=sys.stderr) + return False + + # Process sections + lines = content.splitlines() + new_lines = [] + + in_current_status = False + in_active_documents = False + in_session_history = False + header_written = False + + for line in lines: + if "@@@auto:current-status" in line: + new_lines.append(line) + in_current_status = True + new_lines.append(f"- **Active File**: `{active_file}`") + new_lines.append(f"- **Total Sessions**: {new_session}") + new_lines.append(f"- **Last Active**: {today}") + continue + + if "@@@/auto:current-status" in line: + in_current_status = False + new_lines.append(line) + continue + + if "@@@auto:active-documents" in line: + new_lines.append(line) + in_active_documents = True + new_lines.append("| File | Lines | Status |") + new_lines.append("|------|-------|--------|") + new_lines.append(files_table) + continue + + if "@@@/auto:active-documents" in line: + in_active_documents = False + new_lines.append(line) + continue + + if "@@@auto:session-history" in line: + new_lines.append(line) + in_session_history = True + header_written = False + continue + + if "@@@/auto:session-history" in line: + in_session_history = False + new_lines.append(line) + continue + + if in_current_status: + continue + + if in_active_documents: + continue + + if in_session_history: + # Migrate old 4/6-column headers to 5-column Branch-only history. + if re.match( + r"^\|\s*#\s*\|\s*Date\s*\|\s*Title\s*\|\s*Commits\s*\|\s*Branch\s*\|\s*Base Branch\s*\|\s*$", + line, + ): + new_lines.append("| # | Date | Title | Commits | Branch |") + continue + if re.match(r"^\|\s*#\s*\|\s*Date\s*\|\s*Title\s*\|\s*Commits\s*\|\s*Branch\s*\|\s*$", line): + new_lines.append("| # | Date | Title | Commits | Branch |") + continue + if re.match(r"^\|\s*#\s*\|\s*Date\s*\|\s*Title\s*\|\s*Commits\s*\|\s*$", line): + new_lines.append("| # | Date | Title | Commits | Branch |") + continue + if re.match(r"^\|[-| ]+\|\s*$", line) and not header_written: + new_lines.append("|---|------|-------|---------|--------|") + new_lines.append(f"| {new_session} | {today} | {title} | {commit_display} | `{branch or '-'}` |") + header_written = True + continue + new_lines.append(line) + continue + + new_lines.append(line) + + if not write_text_atomic(index_file, "\n".join(new_lines)): + print(f"Error: failed to write {index_file}", file=sys.stderr) + return False + print("[OK] Updated index.md successfully!") + return True + + +# ============================================================================= +# Main Function +# ============================================================================= + +def _auto_commit_workspace(repo_root: Path) -> str: + """Stage Trellis-owned workspace + current-task paths and commit. + + Path scope is restricted to specific products: the current developer's + journal files + index.md, and ONLY the current task directory (resolved + via ``get_current_task``). We never `git add` the whole `.trellis/` tree + or iterate over all active task dirs (#303: parallel-window dirty task + dirs must not be bundled into the session auto-commit). If `.gitignore` + blocks the specific paths we warn + skip — never retry with ``-f``. + + Honors ``session_auto_commit`` in ``.trellis/config.yaml``: when set to + ``false``, this function returns immediately without touching git + (journal/index files are still written to disk by the caller). + + Returns one of ``COMMIT_DONE`` / ``COMMIT_SKIPPED`` / ``COMMIT_BLOCKED`` / + ``COMMIT_FAILED``. ``COMMIT_BLOCKED`` is the gitignored-`.trellis/` case: + the user has told git to stay out of this tree, so it is a configured skip + like ``session_auto_commit: false``, not a failure to retry. + """ + if not get_session_auto_commit(repo_root): + print( + "[OK] session_auto_commit: false — skipping git stage/commit.", + file=sys.stderr, + ) + return COMMIT_SKIPPED + + # Not a git repository at all: like the gitignored case, this is an + # environment the user configured, not a transient git error — a retry + # can never succeed, so it must not be COMMIT_FAILED's exit-1 loop. + rc, _, _ = run_git(["rev-parse", "--is-inside-work-tree"], cwd=repo_root) + if rc != 0: + print( + "[WARN] Not a git repository — journal/index written, " + "skipping auto-commit.", + file=sys.stderr, + ) + return COMMIT_BLOCKED + + commit_msg = get_session_commit_message(repo_root) + # Resolve the current task so staging is scoped to its dir only. The ref + # is ``.trellis/tasks/`` (or under archive/) — pass the bare name. + current = get_current_task(repo_root) + if current: + task_name = Path(current).name + paths = safe_trellis_paths_to_add(repo_root, task_name=task_name) + else: + # Current task unknown (0 or >=2 parallel sessions — exactly the + # parallel-window case #303 is about). Do NOT fall back to the wide + # `tasks_dir.iterdir()` scan; that would re-leak other tasks' dirty + # dirs into the session commit. Stage only the developer's journal/ + # index and skip every task dir. + paths = [ + p + for p in safe_trellis_paths_to_add(repo_root, task_name=None) + if not p.startswith(f"{DIR_WORKFLOW}/{DIR_TASKS}/") + ] + if not paths: + print("[OK] No workspace changes to commit.", file=sys.stderr) + return COMMIT_SKIPPED + + success, _, err = safe_git_add(paths, repo_root) + if not success: + if err and "ignored by" in err.lower(): + print_gitignore_warning(paths) + return COMMIT_BLOCKED + print( + f"[WARN] git add failed: {err.strip() if err else 'unknown error'}", + file=sys.stderr, + ) + return COMMIT_FAILED + + # Check if there are staged changes for the paths we just staged. + rc, _, _ = run_git( + ["diff", "--cached", "--quiet", "--", *paths], cwd=repo_root + ) + if rc == 0: + print("[OK] No workspace changes to commit.", file=sys.stderr) + return COMMIT_SKIPPED + + # Commit with an explicit pathspec: a bare `git commit` would sweep any + # unrelated entries the developer had staged before this script ran into + # the chore commit (#579). The pathspec keeps their staged work untouched. + rc, _, commit_err = run_git( + ["commit", "-m", commit_msg, "--", *paths], cwd=repo_root + ) + if rc == 0: + print(f"[OK] Auto-committed: {commit_msg}", file=sys.stderr) + return COMMIT_DONE + + print( + f"[WARN] Auto-commit failed: {commit_err.strip()}", + file=sys.stderr, + ) + return COMMIT_FAILED + + +def _print_commit_checkpoint(session_num: int, journal_name: str) -> None: + """Actionable checkpoint for a failed auto-commit (R5).""" + print("", file=sys.stderr) + print( + f"[BLOCKED] Checkpoint: session {session_num} is recorded in " + f"{journal_name} and index.md, but the auto-commit did not complete.", + file=sys.stderr, + ) + print( + "[BLOCKED] The pending record was left exactly as written — nothing was " + "rolled back.", + file=sys.stderr, + ) + print( + "[BLOCKED] Fix the git error above, then re-run the identical " + "add_session.py command: it resumes at the commit step and will not add " + "a second session.", + file=sys.stderr, + ) + + +def add_session( + title: str, + commit: str = "-", + summary: str = "Session summary was not supplied.", + changes: list[str] | None = None, + extra_content: str | None = None, + tests: list[str] | None = None, + next_steps: list[str] | None = None, + auto_commit: bool = True, + package: str | None = None, + branch: str | None = None, + commit_subjects: list[str] | None = None, + idempotency_key: str | None = None, +) -> int: + """Add a new session, resuming an interrupted one instead of duplicating it.""" + repo_root = get_repo_root() + warn_if_parallel_worktree(repo_root) + ensure_developer(repo_root) + + developer = get_developer(repo_root) + if not developer: + print("Error: Developer not initialized", file=sys.stderr) + return 1 + + dev_dir = get_workspace_dir(repo_root) + if not dev_dir: + print("Error: Workspace directory not found", file=sys.stderr) + return 1 + + # ------------------------------------------------------------------- + # Preflight — no writes happen until every one of these succeeds. + # ------------------------------------------------------------------- + if idempotency_key is not None and not IDEMPOTENCY_KEY_RE.match(idempotency_key): + print( + f"Error: invalid --idempotency-key '{idempotency_key}': expected 1-64 " + "characters from [A-Za-z0-9._-]", + file=sys.stderr, + ) + return 1 + + tokens, token_error = parse_commit_tokens(commit) + if token_error: + print(f"Error: {token_error}", file=sys.stderr) + return 1 + + overrides, override_error = parse_subject_overrides(commit_subjects) + if override_error: + print(f"Error: {override_error}", file=sys.stderr) + return 1 + + evidence, evidence_error = build_commit_evidence(repo_root, tokens, overrides) + if evidence_error: + print(f"Error: {evidence_error}", file=sys.stderr) + return 1 + + max_lines = get_max_journal_lines(repo_root) + index_file = dev_dir / "index.md" + today = datetime.now().strftime("%Y-%m-%d") + + payload = _fingerprint_payload( + developer, title, summary, package, branch, evidence, + changes, extra_content, tests, next_steps, idempotency_key, + ) + marker = render_marker(compute_record_fingerprint(payload)) + + # `today` is no longer a fingerprint input; a record has to be findable by + # its own retry after a date rollover. It still dates the rendered entry. + marker, resolve_error = resolve_effective_marker(dev_dir, payload, marker) + if resolve_error: + print(f"Error: {resolve_error}", file=sys.stderr) + return 1 + + state, matched_file, matched_num, classify_error = classify_record( + repo_root, dev_dir, index_file, marker + ) + if classify_error: + print(f"Error: {classify_error}", file=sys.stderr) + return 1 + + if state == STATE_COMMITTED and idempotency_key: + print( + f"[OK] Session {matched_num} with idempotency key " + f"'{idempotency_key}' is already recorded and committed in " + f"{matched_file.name if matched_file else 'the journal'}; " + "nothing to do.", + file=sys.stderr, + ) + return 0 + + # A committed record is finished, so an identical later request is a + # legitimately new session. It must not reuse the committed marker: two + # entries carrying one marker make every later run ambiguous, and + # classify_record refuses to guess between them. Step to the next + # generation of this record instead. The marker stays a pure function of + # (payload, generation), so a retry of the new entry recomputes the same + # marker and still resumes; generation 0 omits the field entirely, leaving + # first-time markers byte-identical to what this scheme already writes. + generation = 0 + while state == STATE_COMMITTED: + generation += 1 + marker = render_marker( + compute_record_fingerprint({**payload, "generation": generation}) + ) + state, matched_file, matched_num, classify_error = classify_record( + repo_root, dev_dir, index_file, marker + ) + if classify_error: + print(f"Error: {classify_error}", file=sys.stderr) + return 1 + + print("========================================", file=sys.stderr) + print("ADD SESSION", file=sys.stderr) + print("========================================", file=sys.stderr) + print("", file=sys.stderr) + + # ------------------------------------------------------------------- + # Absent → journal-recorded + # ------------------------------------------------------------------- + target_file: Path | None + target_num: int + new_session: int + + if state == STATE_ABSENT: + journal_file, current_num, current_lines = get_latest_journal_info(dev_dir) + new_session = resolve_next_session(repo_root, dev_dir, index_file) + + session_content = generate_session_content( + new_session, title, evidence, summary, today, marker, package, branch, + changes=changes, extra_content=extra_content, tests=tests, + next_steps=next_steps, + ) + content_lines = len(session_content.splitlines()) + + print(f"Session: {new_session}", file=sys.stderr) + print(f"Title: {title}", file=sys.stderr) + print(f"Commit: {format_commit_display(evidence)}", file=sys.stderr) + print("", file=sys.stderr) + print(f"Current journal file: {FILE_JOURNAL_PREFIX}{current_num}.md", file=sys.stderr) + print(f"Current lines: {current_lines}", file=sys.stderr) + print(f"New content lines: {content_lines}", file=sys.stderr) + print(f"Total after append: {current_lines + content_lines}", file=sys.stderr) + print("", file=sys.stderr) + + target_file = journal_file + target_num = current_num + + if current_lines + content_lines > max_lines: + target_num = current_num + 1 + print(f"[!] Exceeds {max_lines} lines, creating {FILE_JOURNAL_PREFIX}{target_num}.md", file=sys.stderr) + target_file = create_new_journal_file(dev_dir, target_num, developer, today, max_lines) + if target_file is None: + print( + f"Error: failed to create {FILE_JOURNAL_PREFIX}{target_num}.md; " + "nothing was recorded.", + file=sys.stderr, + ) + return 1 + print(f"Created: {target_file}", file=sys.stderr) + + if target_file is None: + print( + "Error: no journal file to append to. Run init_developer.py first.", + file=sys.stderr, + ) + return 1 + + try: + existing = target_file.read_text(encoding="utf-8") if target_file.is_file() else "" + except OSError as exc: + print(f"Error: cannot read {target_file}: {exc}", file=sys.stderr) + return 1 + + if not write_text_atomic(target_file, existing + session_content): + print( + f"Error: failed to append the session to {target_file.name}; " + "the previous journal content is intact and nothing was recorded.", + file=sys.stderr, + ) + return 1 + print(f"[OK] Appended session to {target_file.name}", file=sys.stderr) + state = STATE_JOURNAL_RECORDED + else: + if matched_file is None or matched_num is None: + print( + f"Error: resumed state '{state}' without a matching journal entry.", + file=sys.stderr, + ) + return 1 + target_file = matched_file + new_session = matched_num + target_num = _extract_journal_num(target_file.stem) + print( + f"[RESUME] Session {new_session} is already in {target_file.name} and " + f"uncommitted; continuing from '{state}' instead of appending again.", + file=sys.stderr, + ) + print("", file=sys.stderr) + + print("", file=sys.stderr) + + # ------------------------------------------------------------------- + # Journal-recorded → index-recorded + # ------------------------------------------------------------------- + active_file = f"{FILE_JOURNAL_PREFIX}{target_num}.md" + if state == STATE_JOURNAL_RECORDED: + if not update_index( + index_file, + dev_dir, + title, + evidence, + new_session, + active_file, + today, + branch, + ): + print( + f"[BLOCKED] Checkpoint: session {new_session} is in " + f"{active_file} but index.md was not updated. Fix index.md, then " + "re-run the identical command to repair the row without adding a " + "second session.", + file=sys.stderr, + ) + return 1 + state = STATE_INDEX_RECORDED + else: + print(f"[OK] index.md already records session {new_session}.", file=sys.stderr) + + print("", file=sys.stderr) + print("========================================", file=sys.stderr) + print(f"[OK] Session {new_session} added successfully!", file=sys.stderr) + print("========================================", file=sys.stderr) + print("", file=sys.stderr) + print("Files updated:", file=sys.stderr) + print(f" - {target_file.name if target_file else 'journal'}", file=sys.stderr) + print(" - index.md", file=sys.stderr) + + # ------------------------------------------------------------------- + # Index-recorded → committed + # ------------------------------------------------------------------- + if not auto_commit: + return 0 + + print("", file=sys.stderr) + outcome = _auto_commit_workspace(repo_root) + + if outcome == COMMIT_FAILED: + _print_commit_checkpoint( + new_session, target_file.name if target_file else "the journal" + ) + return 1 + + if outcome == COMMIT_DONE and target_file is not None: + committed = content_at_head(repo_root, target_file) + if committed is None or marker not in committed: + print( + f"[WARN] Auto-commit reported success but {target_file.name} at " + "HEAD does not contain this session's record.", + file=sys.stderr, + ) + _print_commit_checkpoint(new_session, target_file.name) + return 1 + + return 0 + + +# ============================================================================= +# Main Entry +# ============================================================================= + +def main() -> int: + """CLI entry point.""" + parser = argparse.ArgumentParser( + description="Add a new session to journal file and update index.md" + ) + parser.add_argument("--title", required=True, help="Session title") + parser.add_argument( + "--commit", + default="-", + help=( + "Comma-separated commit OIDs (7-40 hex chars each). Each one is " + "resolved to its real subject before anything is written; an " + "unresolvable OID fails the command. Use '-' for a planning session." + ), + ) + parser.add_argument( + "--commit-subject", + action="append", + metavar="OID=SUBJECT", + help=( + "Explicit subject for a commit that cannot be resolved locally " + "(repeatable). The mapping must be one-to-one with --commit." + ), + ) + parser.add_argument("--summary", default="Session summary was not supplied.", help="Brief summary") + parser.add_argument("--content-file", help="Path to file with detailed content") + parser.add_argument("--package", help="Package name tag (e.g., cli, docs-site)") + parser.add_argument("--branch", help="Branch name (auto-detected if omitted)") + parser.add_argument("--change", action="append", help="Main Changes bullet (repeatable)") + parser.add_argument("--test", action="append", help="Testing bullet (repeatable)") + parser.add_argument("--next-step", action="append", help="Next Steps bullet (repeatable)") + parser.add_argument( + "--idempotency-key", + help=( + "Caller-supplied retry key ([A-Za-z0-9._-], 1-64 chars). Makes an " + "already-committed identical record a no-op instead of a new session." + ), + ) + parser.add_argument("--no-commit", action="store_true", + help="Skip auto-commit of workspace changes") + parser.add_argument("--stdin", action="store_true", + help="Read extra content from stdin (explicit opt-in)") + + args = parser.parse_args() + + extra_content: str | None = None + if args.content_file: + content_path = Path(args.content_file) + if content_path.is_file(): + extra_content = content_path.read_text(encoding="utf-8") + elif args.stdin: + extra_content = sys.stdin.read() + + # Load active task once — shared by package and branch resolution + repo_root = get_repo_root() + current = get_current_task(repo_root) + task_data = load_task(repo_root / current) if current else None + + package = args.package + if package: + # CLI source: fail-fast in monorepo, ignore in single-repo + if not is_monorepo(repo_root): + print("Warning: --package ignored in single-repo project", file=sys.stderr) + package = None + elif not validate_package(package, repo_root): + packages = get_packages(repo_root) + available = ", ".join(sorted(packages.keys())) if packages else "(none)" + print(f"Error: unknown package '{package}'. Available: {available}", file=sys.stderr) + return 1 + else: + # Inferred: active task's task.json.package → default_package → None + task_package = task_data.package if task_data else None + package = resolve_package(task_package, repo_root) + + branch = resolve_session_branch(repo_root, args.branch, task_data) + + return add_session( + args.title, args.commit, args.summary, + changes=args.change, extra_content=extra_content, tests=args.test, + next_steps=args.next_step, + auto_commit=not args.no_commit, + package=package, + branch=branch, + commit_subjects=args.commit_subject, + idempotency_key=args.idempotency_key, + ) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.trellis/scripts/common/__init__.py b/.trellis/scripts/common/__init__.py new file mode 100755 index 0000000..4d8e0fe --- /dev/null +++ b/.trellis/scripts/common/__init__.py @@ -0,0 +1,94 @@ +""" +Common utilities for Trellis workflow scripts. + +This module provides shared functionality used by other Trellis scripts. +""" + +import io +import sys + +# ============================================================================= +# Windows Encoding Fix (MUST be at top, before any other output) +# ============================================================================= +# On Windows, stdout defaults to the system code page (often GBK/CP936). +# This causes UnicodeEncodeError when printing non-ASCII characters. +# +# Any script that imports from common will automatically get this fix. +# ============================================================================= + + +def _configure_stream(stream: object) -> object: + """Configure a stream for UTF-8 encoding on Windows.""" + # Try reconfigure() first (Python 3.7+, more reliable) + if hasattr(stream, "reconfigure"): + stream.reconfigure(encoding="utf-8", errors="replace") # type: ignore[union-attr] + return stream + # Fallback: detach and rewrap with TextIOWrapper + elif hasattr(stream, "detach"): + return io.TextIOWrapper( + stream.detach(), # type: ignore[union-attr] + encoding="utf-8", + errors="replace", + ) + return stream + + +if sys.platform == "win32": + sys.stdout = _configure_stream(sys.stdout) # type: ignore[assignment] + sys.stderr = _configure_stream(sys.stderr) # type: ignore[assignment] + sys.stdin = _configure_stream(sys.stdin) # type: ignore[assignment] + + +def configure_encoding() -> None: + """ + Configure stdout/stderr/stdin for UTF-8 encoding on Windows. + + This is automatically called when importing from common, + but can be called manually for scripts that don't import common. + + Safe to call multiple times. + """ + global sys + if sys.platform == "win32": + sys.stdout = _configure_stream(sys.stdout) # type: ignore[assignment] + sys.stderr = _configure_stream(sys.stderr) # type: ignore[assignment] + sys.stdin = _configure_stream(sys.stdin) # type: ignore[assignment] + + +from .paths import ( + DIR_WORKFLOW, + DIR_WORKSPACE, + DIR_TASKS, + DIR_ARCHIVE, + DIR_SPEC, + DIR_SCRIPTS, + FILE_DEVELOPER, + FILE_CURRENT_TASK, + FILE_TASK_JSON, + FILE_JOURNAL_PREFIX, + ENV_DEVELOPER, + DEVELOPER_HINT, + get_repo_root, + get_developer, + check_developer, + get_tasks_dir, + get_workspace_dir, + get_active_journal_file, + count_lines, + get_current_task, + get_current_task_abs, + normalize_task_ref, + resolve_task_ref, + set_current_task, + clear_current_task, + has_current_task, + generate_task_date_prefix, +) + +from .active_task import ( + ActiveTask, + clear_active_task, + resolve_active_task, + resolve_context_key, + set_active_task, +) diff --git a/.trellis/scripts/common/active_task.py b/.trellis/scripts/common/active_task.py new file mode 100755 index 0000000..b9e4f87 --- /dev/null +++ b/.trellis/scripts/common/active_task.py @@ -0,0 +1,857 @@ +#!/usr/bin/env python3 +"""Session-scoped active task resolution. + +The user-facing concept is a single "active task". Trellis stores that pointer +per AI session/window under `.trellis/.runtime/sessions/`; without a stable +session key there is no active task. +""" + +from __future__ import annotations + +import hashlib +import os +import re +import sys +import time +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from .io import read_json as _io_read_json, write_json as _io_write_json + +DIR_WORKFLOW = ".trellis" +DIR_TASKS = "tasks" +DIR_RUNTIME = ".runtime" +DIR_SESSIONS = "sessions" +DIR_SHELL_TICKETS = "shell-tickets" +# Pre-0.6.13 name, when the bridge was Cursor-only. Still read so a session that +# was mid-command across an upgrade does not silently degrade; never written. +# Tickets are 30-second ephemera, so the old directory ages out by itself — +# there is nothing to migrate, only a glob on a directory that is normally +# absent. The alternative (ignore it) would land its one lost command on the +# platform that works today. +DIR_LEGACY_CURSOR_SHELL_TICKETS = "cursor-shell" +SHELL_TICKET_TTL_SECONDS = 30 +TASK_SESSION_COMMANDS = {"start", "current", "finish"} + +_SESSION_KEYS = ("session_id", "sessionId", "sessionID") +_CONVERSATION_KEYS = ("conversation_id", "conversationId", "conversationID") +_TRANSCRIPT_KEYS = ("transcript_path", "transcriptPath", "transcript") +_NESTED_KEYS = ("input", "properties", "event", "hook_input", "hookInput") +_KNOWN_PLATFORMS = { + "claude", + "codex", + "cursor", + "opencode", + "gemini", + "droid", + "qoder", + "codebuddy", + "kiro", + "copilot", + "pi", + "trae", + "grok", + "kimi", + "zcode", + "snow", + "dsh", +} + +# Every name below records how it was checked. Do NOT add a name by analogy +# with a neighbour: a 2026-08-05 audit of all 21 platforms found 12 of the 21 +# declared names had never existed anywhere — they were pattern-guessed from a +# `_SESSION_ID` shape no vendor agreed to, and the uniformity was the +# only "evidence" behind them. A platform with no verified name belongs in no +# table; it resolves through TRELLIS_CONTEXT_ID or its hook/plugin bridge. +_ENV_SESSION_KEYS: tuple[tuple[str, tuple[str, ...]], ...] = ( + # REAL (reported 2026-08-13 against DSH 0.1.0-rc.6 by @SajoLuo, from a live + # run: DSH exports DSH_SESSION_ID plus DSH_SHELL=1 into its managed shell). + # MUST STAY FIRST. A DSH session can inherit an outer host's identity — a + # DSH launched from Codex still carries CODEX_THREAD_ID — and the untargeted + # lookup below walks this table in order, so any earlier entry would claim + # the session and write a foreign `codex_` pointer for DSH work. + # DSH_SESSION_ID is the only name here no other vendor sets, so first place + # is safe: it cannot mis-claim a non-DSH session. + ("dsh", ("DSH_SESSION_ID",)), + # REAL, undocumented (verified 2026-08-05 in a live Claude Code 2.1.221 bash + # child; absent from code.claude.com/docs/en/env-vars). CLAUDE_SESSION_ID + # was removed here — verified absent from that same live environment. + ("claude", ("CLAUDE_CODE_SESSION_ID",)), + # REAL, undocumented (verified 2026-08-05: injected by codex-cli 0.146.0 + # into shell children, absent from the parent env; openai/codex#19937). + # CODEX_SESSION_ID was removed — absent from a live `codex exec` env. + ("codex", ("CODEX_THREAD_ID",)), + # REAL but HOOK-SCOPE ONLY (verified 2026-08-05): set by Gemini's + # hookRunner.ts. Its shell tool builds the child env in + # shellExecutionService.ts and adds only GEMINI_CLI/TERM/PAGER/GIT_PAGER, so + # this never reaches a bash child — it resolves only inside a hook process. + ("gemini", ("GEMINI_SESSION_ID",)), + # REAL but HOOK-SCOPE ONLY (verified 2026-08-05): docs.qoder.com/zh/ + # extensions/hooks documents it as injected during hook execution by the + # Qoder *IDE plugin*. Absent from the Qoder CLI hook docs and from Lingma. + ("qoder", ("QODER_SESSION_ID",)), + # UNVERIFIED (2026-08-05): absent from kiro.dev/docs/hooks/, but Dynatrace + # dtctl, oh-my-agent and gastown all key agent detection on it and one notes + # it is "set in both interactive and --no-interactive". Kept because that is + # absence of evidence, not evidence of absence. To settle: run + # `env | grep KIRO` from a Kiro shell-tool call on a machine with Kiro. + ("kiro", ("KIRO_SESSION_ID",)), + # UNVERIFIED (2026-08-05): absent from docs.github.com/en/copilot/reference/ + # hooks-reference and from the CLI programmatic reference. To settle: run + # `copilot help environment` (the authoritative list per those docs) — not + # runnable here, the CLI is not installed and copilot-cli ships no source. + ("copilot", ("COPILOT_SESSION_ID", "COPILOT_SESSIONID")), + # REASONED, UNVERIFIED (2026-08-05): ZCode is closed-source and not + # installable here. It mirrors Claude's naming elsewhere (CLAUDE_PLUGIN_ROOT + # / CLAUDE_PLUGIN_DATA compat aliases are in its docs), and the previously + # declared CLAUDE_SESSION_ID does not exist on Claude Code either — so the + # name ZCode would actually reuse is CLAUDE_CODE_SESSION_ID. Try that first, + # keep the historical name as a fallback: if neither exists nothing changes. + # Platform-scoped lookup (_iter_env_keys filters by platform name), so the + # entry only fires once the resolver detected "zcode" — no collision with + # the claude entry above. + ("zcode", ("CLAUDE_CODE_SESSION_ID", "CLAUDE_SESSION_ID")), + # REAL by vendor design (verified 2026-08-05): Snow's sessionIdentityEnv.ts + # exports SNOW_SESSION_ID into hook/terminal/sub-agent children and names + # Trellis in its source header. TRELLIS_CONTEXT_ID stays the preferred + # override — Snow sets that too. + ("snow", ("SNOW_SESSION_ID",)), +) +_ENV_CONVERSATION_KEYS: tuple[tuple[str, tuple[str, ...]], ...] = ( + # REAL in cursor-agent (CLI), undocumented (verified 2026-08-05: the value + # matches ~/.cursor/chats//). The Cursor *IDE* is unverified — a + # 2026-05 forum request for it drew no staff reply. The invented + # CURSOR_SESSION_ID was removed from the session table: empty in a live + # cursor-agent shell. Cursor's other path is the shell ticket below + # (_lookup_shell_ticket_context_key), which is not Cursor-specific. + ("cursor", ("CURSOR_CONVERSATION_ID", "CURSOR_CONVERSATIONID")), +) +_ENV_TRANSCRIPT_KEYS: tuple[tuple[str, tuple[str, ...]], ...] = ( + # REAL but HOOK-SCOPE ONLY (verified 2026-08-05): documented for Cursor hook + # scripts; empty in the agent's own shell env. + ("cursor", ("CURSOR_TRANSCRIPT_PATH",)), + # UNVERIFIED — never researched. The 2026-08-05 audit covered the session + # table only, so do not infer these are real *or* fake from that work + # (CLAUDE_/CODEX_TRANSCRIPT_PATH were removed because those two *were* + # checked: absent from docs and from live envs). To settle each: run + # `env | grep _TRANSCRIPT_PATH` inside a hook and inside a shell-tool call. + ("gemini", ("GEMINI_TRANSCRIPT_PATH",)), + ("droid", ("FACTORY_TRANSCRIPT_PATH", "DROID_TRANSCRIPT_PATH")), + ("qoder", ("QODER_TRANSCRIPT_PATH",)), + ("codebuddy", ("CODEBUDDY_TRANSCRIPT_PATH",)), +) +_ENV_PLATFORM_ALIASES = { + "claude-code": "claude", + "factory": "droid", + "factory-ai": "droid", + "github-copilot": "copilot", +} +# ZCode intentionally reuses Claude's session env var name. Hooks know the host +# is ZCode, while later shell commands see only the shared env name and resolve +# it through the claude entry. Canonicalize both paths to one runtime filename. +_CONTEXT_KEY_PLATFORM_ALIASES = { + "zcode": "claude", + # Factory Droid's config directory is `.factory/`, so a hook that names its + # platform after the directory it was installed in reports "factory". Its + # sibling hooks report "droid". One runtime filename either way. + "factory": "droid", +} + + +@dataclass(frozen=True) +class ActiveTask: + """Resolved active task state.""" + + task_path: str | None + source_type: str + context_key: str | None = None + stale: bool = False + + @property + def source(self) -> str: + """Human-readable source label.""" + if self.source_type == "session" and self.context_key: + return f"session:{self.context_key}" + if self.source_type == "session-fallback" and self.context_key: + return f"session-fallback:{self.context_key}" + return self.source_type + + +def normalize_task_ref(task_ref: str) -> str: + """Normalize a task ref for stable storage and comparison.""" + normalized = task_ref.strip() + if not normalized: + return "" + + path_obj = Path(normalized) + if path_obj.is_absolute(): + return str(path_obj) + + normalized = normalized.replace("\\", "/") + while normalized.startswith("./"): + normalized = normalized[2:] + + if normalized.startswith(f"{DIR_TASKS}/"): + return f"{DIR_WORKFLOW}/{normalized}" + + return normalized + + +def resolve_task_ref(task_ref: str, repo_root: Path) -> Path | None: + """Resolve a task ref to an absolute task directory inside the repo. + + Mirrors `paths.resolve_task_ref` (same containment check). Duplicated + rather than imported because this module is loaded standalone — hooks add + it to `sys.path` directly — so it stays zero-relative-import on purpose. + """ + normalized = normalize_task_ref(task_ref) + if not normalized: + return None + + try: + root = repo_root.resolve() + except OSError: + return None + + path_obj = Path(normalized) + if path_obj.is_absolute(): + candidate = path_obj + elif normalized.startswith(f"{DIR_WORKFLOW}/"): + candidate = root / path_obj + else: + candidate = root / DIR_WORKFLOW / DIR_TASKS / path_obj + + # Both sides are resolved because repo_root itself may sit behind a + # symlink (/tmp on macOS does), and resolve() is what collapses `..` + # instead of leaving it for a lexical relative_to() to wave through. + try: + resolved = candidate.resolve() + workflow_real = (root / DIR_WORKFLOW).resolve() + except OSError: + return None + + try: + resolved.relative_to(root) + return resolved + except ValueError: + pass + + # `.trellis` may itself be a symlink into a store outside the repo (#567). + # The workflow dir's own real location is then a second legitimate + # containment base; a ref that escapes BOTH bases is still refused. Map + # back to the in-repo (lexical) form so callers store a repo-relative ref. + try: + rel = resolved.relative_to(workflow_real) + except ValueError: + return None + + return root / DIR_WORKFLOW / rel + + +def _runtime_sessions_dir(repo_root: Path) -> Path: + return repo_root / DIR_WORKFLOW / DIR_RUNTIME / DIR_SESSIONS + + +def _sanitize_key(raw: str) -> str: + safe = re.sub(r"[^A-Za-z0-9._-]+", "_", raw.strip()) + safe = safe.strip("._-") + return safe[:160] if safe else "" + + +def _hash_value(raw: str) -> str: + return hashlib.sha256(raw.encode("utf-8")).hexdigest()[:24] + + +def _as_dict(value: Any) -> dict[str, Any] | None: + return value if isinstance(value, dict) else None + + +def _string_value(value: Any) -> str | None: + if isinstance(value, str): + stripped = value.strip() + return stripped or None + return None + + +def _lookup_string(data: dict[str, Any], keys: tuple[str, ...]) -> str | None: + for key in keys: + value = _string_value(data.get(key)) + if value: + return value + + for nested_key in _NESTED_KEYS: + nested = _as_dict(data.get(nested_key)) + if not nested: + continue + value = _lookup_string(nested, keys) + if value: + return value + + return None + + +def _detect_platform(platform_input: dict[str, Any] | None, platform: str | None) -> str: + if platform: + return _sanitize_key(platform) or "session" + if platform_input: + for key in ("_trellis_platform", "trellis_platform", "platform", "source"): + value = _string_value(platform_input.get(key)) + if value: + return _sanitize_key(value) or "session" + if _string_value(platform_input.get("cursor_version")): + return "cursor" + return "session" + + +def _context_key(platform_name: str, kind: str, value: str) -> str: + platform_name = _CONTEXT_KEY_PLATFORM_ALIASES.get(platform_name, platform_name) + if kind == "transcript": + return f"{platform_name}_transcript_{_hash_value(value)}" + safe_value = _sanitize_key(value) + if safe_value: + return f"{platform_name}_{safe_value}" + return f"{platform_name}_{_hash_value(value)}" + + +def _iter_env_keys( + env_keys: tuple[tuple[str, tuple[str, ...]], ...], + platform_name: str | None, +) -> tuple[tuple[str, tuple[str, ...]], ...]: + """Narrow an env-key table to one platform, or return all of it. + + A platform with no entry yields an empty tuple, and the caller's `for` loop + simply does not run. That is the normal case, not an error: platforms with + no verified env var name are deliberately absent from these tables. + """ + if not platform_name: + return env_keys + matched = tuple((name, keys) for name, keys in env_keys if name == platform_name) + return matched + + +def _env_platform_name(platform_name: str | None) -> str | None: + if not platform_name or platform_name == "session": + return None + return _ENV_PLATFORM_ALIASES.get(platform_name, platform_name) + + +def _lookup_env_context_key(platform_name: str | None) -> str | None: + """Resolve a context key from platform-provided environment variables. + + Hooks pass `TRELLIS_CONTEXT_ID` to subprocesses they launch, but an AI-run + shell command can only see session identity if the host platform exports it + in the command environment. These names are best-effort adapters; if none + are present, there is no session-scoped active task. + """ + env_platform_name = _env_platform_name(platform_name) + + for name, keys in _iter_env_keys(_ENV_SESSION_KEYS, env_platform_name): + for key in keys: + value = _string_value(os.environ.get(key)) + if value: + return _context_key(name, "session", value) + + for name, keys in _iter_env_keys(_ENV_CONVERSATION_KEYS, env_platform_name): + for key in keys: + value = _string_value(os.environ.get(key)) + if value: + return _context_key(name, "conversation", value) + + for name, keys in _iter_env_keys(_ENV_TRANSCRIPT_KEYS, env_platform_name): + for key in keys: + value = _string_value(os.environ.get(key)) + if value: + return _context_key(name, "transcript", value) + + return None + + +def _find_repo_root_from_cwd() -> Path | None: + current = Path.cwd().resolve() + while True: + if (current / DIR_WORKFLOW).is_dir(): + return current + if current == current.parent: + return None + current = current.parent + + +def _shell_ticket_dirs(repo_root: Path) -> tuple[Path, ...]: + runtime_dir = repo_root / DIR_WORKFLOW / DIR_RUNTIME + return ( + runtime_dir / DIR_SHELL_TICKETS, + runtime_dir / DIR_LEGACY_CURSOR_SHELL_TICKETS, + ) + + +def _remove_file(path: Path) -> bool: + try: + path.unlink() + return True + except OSError: + return False + + +def _task_refs_match(left: str | None, right: str | None, repo_root: Path) -> bool: + if not left or not right: + return False + left_path = resolve_task_ref(left, repo_root) + right_path = resolve_task_ref(right, repo_root) + if left_path is not None and right_path is not None: + return left_path == right_path + return normalize_task_ref(left) == normalize_task_ref(right) + + +def _pending_ticket_matches_args(ticket: dict[str, Any], repo_root: Path) -> bool: + if Path(sys.argv[0]).name != "task.py": + return False + args = tuple(sys.argv[1:]) + if not args: + return False + + command_name = args[0] + if command_name not in TASK_SESSION_COMMANDS: + return False + + subcommands = ticket.get("subcommands") + if not isinstance(subcommands, list): + return False + + for subcommand in subcommands: + if not isinstance(subcommand, dict): + continue + if _string_value(subcommand.get("name")) != command_name: + continue + if command_name != "start": + return True + task_ref = args[1] if len(args) > 1 else None + if _task_refs_match(_string_value(subcommand.get("task_ref")), task_ref, repo_root): + return True + + return False + + +def _ticket_is_fresh(ticket: dict[str, Any], ticket_path: Path, now: float) -> bool: + expires_at = ticket.get("expires_at_epoch") + if isinstance(expires_at, (int, float)) and expires_at < now: + _remove_file(ticket_path) + return False + + created_at = ticket.get("created_at_epoch") + if isinstance(created_at, (int, float)): + if now - created_at <= SHELL_TICKET_TTL_SECONDS: + return True + _remove_file(ticket_path) + return False + return True + + +def _ticket_cwd_matches_repo(ticket: dict[str, Any], repo_root: Path) -> bool: + cwd = _string_value(ticket.get("cwd")) + if not cwd: + return True + try: + Path(cwd).resolve().relative_to(repo_root) + except ValueError: + return False + return True + + +def _matching_ticket_context_key( + ticket_path: Path, + repo_root: Path, + now: float, +) -> str | None: + """Accept a ticket on its merits, never on which platform wrote it. + + The `platform` field a ticket carries is debugging metadata; gating on it + was what kept this bridge invisible to every platform but Cursor. + """ + ticket = _read_json(ticket_path) + if ticket is None: + return None + if not _ticket_is_fresh(ticket, ticket_path, now): + return None + if not _ticket_cwd_matches_repo(ticket, repo_root): + return None + if not _pending_ticket_matches_args(ticket, repo_root): + return None + return _string_value(ticket.get("context_key")) + + +def _lookup_shell_ticket_context_key() -> str | None: + """Resolve session identity from a short-lived shell ticket. + + No researched platform exports its session id into a shell child, but every + hook-capable one hands that id to a hook. So the hook that runs just before + a shell command writes a ticket, and this reads it back. A ticket counts + only when it is fresh, was written for this repo, and matches the `task.py` + subcommand now running — and only when exactly one fresh context key + matches. Two concurrent windows therefore both degrade rather than one + inheriting the other's pointer. + """ + repo_root = _find_repo_root_from_cwd() + if repo_root is None: + return None + + now = time.time() + candidates: set[str] = set() + for ticket_dir in _shell_ticket_dirs(repo_root): + if not ticket_dir.is_dir(): + continue + for ticket_path in ticket_dir.glob("*.json"): + context_key = _matching_ticket_context_key(ticket_path, repo_root, now) + if context_key: + candidates.add(context_key) + + if len(candidates) == 1: + return next(iter(candidates)) + return None + + +def resolve_context_key( + platform_input: dict[str, Any] | None = None, + platform: str | None = None, + *, + allow_environment_context: bool = True, +) -> str | None: + """Resolve a stable session/window context key, if one is available. + + `TRELLIS_CONTEXT_ID` is an explicit context-key override used by CLI + scripts and subprocesses. It does not store the task itself. + """ + if allow_environment_context: + override = _string_value(os.environ.get("TRELLIS_CONTEXT_ID")) + if override: + return _sanitize_key(override) or _hash_value(override) + + data = _as_dict(platform_input) + platform_name = _detect_platform(data, platform) if data or platform else None + + if data: + session_id = _lookup_string(data, _SESSION_KEYS) + if session_id: + return _context_key(platform_name or "session", "session", session_id) + + conversation_id = _lookup_string(data, _CONVERSATION_KEYS) + if conversation_id: + return _context_key(platform_name or "session", "conversation", conversation_id) + + transcript_path = _lookup_string(data, _TRANSCRIPT_KEYS) + if transcript_path: + return _context_key(platform_name or "session", "transcript", transcript_path) + + if allow_environment_context: + env_context_key = _lookup_env_context_key(platform_name) + if env_context_key: + return env_context_key + + # Last in the chain on purpose: a platform that genuinely exports identity + # into the shell outranks a ticket, and no platform name gates the lookup. + if allow_environment_context: + return _lookup_shell_ticket_context_key() + return None + + +def _read_json(path: Path) -> dict[str, Any] | None: + """Tolerant read of a session runtime file, non-objects included.""" + data = _io_read_json(path) + return data if isinstance(data, dict) else None + + +def _write_json(path: Path, data: dict[str, Any]) -> bool: + """Write a session runtime file atomically, creating the runtime dir. + + Routes through io.write_json so session pointers get the same + temp-file-then-rename treatment as task.json (#429). A plain write_text + truncates the target first, so a crash mid-write would leave a session + file that reads back as no active task. + """ + try: + path.parent.mkdir(parents=True, exist_ok=True) + except OSError: + return False + return _io_write_json(path, data) + + +def _canonical_task_ref(task_path: str, repo_root: Path) -> str | None: + normalized = normalize_task_ref(task_path) + if not normalized: + return None + full_path = resolve_task_ref(normalized, repo_root) + if full_path is None or not full_path.is_dir(): + return None + try: + return full_path.relative_to(repo_root.resolve()).as_posix() + except ValueError: + # resolve_task_ref already refused everything outside the repo, so this + # is unreachable. Refuse rather than fall back to an absolute path — + # that fallback is how an out-of-repo ref used to reach the session + # pointer and get replayed on every later turn. + return None + + +def _relative_task_ref(task_path: str, repo_root: Path) -> str: + """Repo-relative posix ref for a task path that need not exist. + + `_canonical_task_ref` resolves through the filesystem and so refuses a task + directory that has been moved away. Rename needs to name both sides of the + move, one of which is always absent. + """ + normalized = normalize_task_ref(task_path) + if not normalized: + return "" + candidate = Path(normalized) + if not candidate.is_absolute(): + return normalized + try: + resolved = candidate.resolve() + root = repo_root.resolve() + workflow_real = (root / DIR_WORKFLOW).resolve() + except OSError: + return "" + try: + return resolved.relative_to(root).as_posix() + except ValueError: + pass + # Same dual-base containment as resolve_task_ref: a path through a + # symlinked `.trellis` (#567) maps back to its in-repo form; anything + # outside both bases is refused rather than stored as an absolute pointer. + try: + rel = resolved.relative_to(workflow_real) + except ValueError: + return "" + return (Path(DIR_WORKFLOW) / rel).as_posix() + + +def _active_from_ref( + task_ref: str | None, + repo_root: Path, + source_type: str, + context_key: str | None = None, +) -> ActiveTask | None: + if not task_ref: + return None + resolved = resolve_task_ref(task_ref, repo_root) + stale = resolved is None or not resolved.is_dir() + return ActiveTask(task_ref, source_type, context_key, stale) + + +def _context_path(repo_root: Path, context_key: str) -> Path: + return _runtime_sessions_dir(repo_root) / f"{context_key}.json" + + +def resolve_active_task( + repo_root: Path, + platform_input: dict[str, Any] | None = None, + platform: str | None = None, + *, + allow_single_session_fallback: bool = True, + allow_environment_context: bool = True, +) -> ActiveTask: + """Resolve the active task from session runtime state only. + + A stale session task is returned as stale. Missing context identity or a + missing/empty session context falls back to single-session inference: if + exactly one session file exists in the runtime, return its task with + source_type="session-fallback" — covers pull-based platform sub-agents + (copilot, gemini, qoder) that don't inherit the parent's session id. ≥2 + files or 0 files yield ActiveTask(None) — refuses to guess across windows. + """ + context_key = resolve_context_key( + platform_input, + platform, + allow_environment_context=allow_environment_context, + ) + if context_key: + context = _read_json(_context_path(repo_root, context_key)) or {} + task_ref = _string_value(context.get("current_task")) + active = _active_from_ref(task_ref, repo_root, "session", context_key) + if active: + return active + + if allow_single_session_fallback: + fallback = _resolve_single_session_fallback(repo_root) + if fallback is not None: + return fallback + + return ActiveTask(None, "none", context_key) + + +def _resolve_single_session_fallback(repo_root: Path) -> ActiveTask | None: + """Return the task pointed at by the sole session file, if exactly one exists. + + Used when context-key resolution fails (typical for class-2 platform + sub-agents). Returns None if 0 or ≥2 session files are present — refuses + to pick across windows so 04-21's multi-session isolation contract holds. + """ + sessions_dir = _runtime_sessions_dir(repo_root) + if not sessions_dir.is_dir(): + return None + + session_files = sorted(sessions_dir.glob("*.json")) + if len(session_files) != 1: + return None + + session_file = session_files[0] + context = _read_json(session_file) or {} + task_ref = _string_value(context.get("current_task")) + if not task_ref: + return None + + fallback_key = session_file.stem + return _active_from_ref(task_ref, repo_root, "session-fallback", fallback_key) + + +def _utc_now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def _context_metadata( + platform_input: dict[str, Any] | None, + platform: str | None, + context_key: str | None = None, +) -> dict[str, Any]: + data = _as_dict(platform_input) or {} + platform_name = _detect_platform(data, platform) + if platform_name == "session" and context_key: + prefix = context_key.split("_", 1)[0] + if prefix in _KNOWN_PLATFORMS: + platform_name = prefix + metadata: dict[str, Any] = { + "platform": platform_name, + "last_seen_at": _utc_now(), + } + for key in (*_SESSION_KEYS, *_CONVERSATION_KEYS, *_TRANSCRIPT_KEYS): + value = _lookup_string(data, (key,)) + if value: + metadata[key] = value + return metadata + + +def set_active_task( + task_path: str, + repo_root: Path, + platform_input: dict[str, Any] | None = None, + platform: str | None = None, +) -> ActiveTask | None: + """Set the active task in session scope. + + Returns None when no context key is available; callers should surface a + user-facing error that explains how to provide session identity. + """ + canonical = _canonical_task_ref(task_path, repo_root) + if canonical is None: + return None + + context_key = resolve_context_key(platform_input, platform) + if not context_key: + return None + + context_path = _context_path(repo_root, context_key) + context = _read_json(context_path) or {} + context.update(_context_metadata(platform_input, platform, context_key)) + context["current_task"] = canonical + context.setdefault("current_run", None) + if not _write_json(context_path, context): + return None + return ActiveTask(canonical, "session", context_key) + + +def clear_active_task( + repo_root: Path, + platform_input: dict[str, Any] | None = None, + platform: str | None = None, +) -> ActiveTask: + """Clear the active task by deleting its resolved session context file.""" + context_key = resolve_context_key(platform_input, platform) + if not context_key: + return ActiveTask(None, "none") + + previous = resolve_active_task(repo_root, platform_input, platform) + if not previous.task_path or not previous.context_key: + return previous + + context_path = _context_path(repo_root, previous.context_key) + if context_path.is_file(): + _remove_file(context_path) + return previous + + +def clear_task_from_sessions(task_path: str, repo_root: Path) -> int: + """Delete all session runtime files that point at a task.""" + target = _canonical_task_ref(task_path, repo_root) or normalize_task_ref(task_path) + if not target: + return 0 + + cleared = 0 + sessions_dir = _runtime_sessions_dir(repo_root) + if not sessions_dir.is_dir(): + return cleared + + for session_path in sessions_dir.glob("*.json"): + context = _read_json(session_path) or {} + current = _string_value(context.get("current_task")) + if not current: + continue + current_ref = _canonical_task_ref(current, repo_root) or normalize_task_ref(current) + if current_ref != target: + continue + if session_path.is_file() and _remove_file(session_path): + cleared += 1 + + return cleared + + +def repoint_task_in_sessions(old_path: str, new_path: str, repo_root: Path) -> int: + """Move every session pointer from `old_path` to `new_path`. + + Rename is the one lifecycle step where the task survives under a different + name, so clearing the pointers (what archive does) would be wrong: the user + would silently lose their active task and have to run `task.py start` + again to get context injection back. Repointing keeps the session valid + across the rename. + """ + # Not `_canonical_task_ref`: the caller repoints *after* moving the + # directory, so `old_path` no longer exists and canonicalization — which + # requires an existing directory — would return None for exactly the ref we + # need to match. + target = _relative_task_ref(old_path, repo_root) + replacement = _relative_task_ref(new_path, repo_root) + if not target or not replacement: + return 0 + + moved = 0 + sessions_dir = _runtime_sessions_dir(repo_root) + if not sessions_dir.is_dir(): + return moved + + for session_path in sorted(sessions_dir.glob("*.json")): + context = _read_json(session_path) + if not context: + continue + current = _string_value(context.get("current_task")) + if not current: + continue + current_ref = _canonical_task_ref(current, repo_root) or _relative_task_ref( + current, repo_root + ) + if current_ref != target: + continue + context["current_task"] = replacement + if _write_json(session_path, context): + moved += 1 + + return moved + + +def get_current_task_source( + repo_root: Path, + platform_input: dict[str, Any] | None = None, + platform: str | None = None, +) -> tuple[str, str | None, str | None]: + """Return (`source_type`, `context_key`, `task_path`) for compatibility.""" + active = resolve_active_task(repo_root, platform_input, platform) + return active.source_type, active.context_key, active.task_path diff --git a/.trellis/scripts/common/cli_adapter.py b/.trellis/scripts/common/cli_adapter.py new file mode 100755 index 0000000..85e1c3a --- /dev/null +++ b/.trellis/scripts/common/cli_adapter.py @@ -0,0 +1,950 @@ +""" +CLI Adapter for Multi-Platform Support. + +Abstracts differences between Claude Code, OpenCode, Cursor, iFlow, Codex, Kilo, Kiro Code, Gemini CLI, Antigravity, Devin, Qoder, CodeBuddy, GitHub Copilot, Factory Droid, and Pi Agent interfaces. + +Supported platforms: +- claude: Claude Code (default) +- opencode: OpenCode +- cursor: Cursor IDE +- iflow: iFlow CLI +- codex: Codex CLI (skills-based) +- kilo: Kilo CLI +- kiro: Kiro Code (skills-based) +- gemini: Gemini CLI +- antigravity: Antigravity (workflow-based) +- devin: Devin (formerly Windsurf; workflow-based) +- qoder: Qoder +- codebuddy: CodeBuddy +- copilot: GitHub Copilot (VS Code) +- droid: Factory Droid (commands-based) +- pi: Pi Agent (extension-backed) +- trae: Trae IDE (IDE-only, hooks-based) +- omp: Oh My Pi +- grok: Grok Build (pull-based skills/agents; no hook context injection) +- kimi: Kimi Code (pull-based skills; commands delivered as skills; no hook context injection) + +Usage: + from common.cli_adapter import CLIAdapter + + adapter = CLIAdapter("opencode") + cmd = adapter.build_run_command( + agent="dispatch", + session_id="abc123", + prompt="Start the pipeline" + ) +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import ClassVar, Literal + +Platform = Literal[ + "claude", + "opencode", + "cursor", + "iflow", + "codex", + "kilo", + "kiro", + "gemini", + "antigravity", + "devin", + "qoder", + "codebuddy", + "copilot", + "droid", + "pi", + "trae", + "omp", + "grok", + "kimi", +] + + +@dataclass +class CLIAdapter: + """Adapter for different AI coding CLI tools.""" + + platform: Platform + + # ========================================================================= + # Agent Name Mapping + # ========================================================================= + + # OpenCode has built-in agents that cannot be overridden + # See: https://github.com/sst/opencode/issues/4271 + # Note: Class-level constant, not a dataclass field + _AGENT_NAME_MAP: ClassVar[dict[Platform, dict[str, str]]] = { + "claude": {}, # No mapping needed + "opencode": { + "plan": "trellis-plan", # 'plan' is built-in in OpenCode + }, + } + + def get_agent_name(self, agent: str) -> str: + """Get platform-specific agent name. + + Args: + agent: Original agent name (e.g., 'plan', 'dispatch') + + Returns: + Platform-specific agent name (e.g., 'trellis-plan' for OpenCode) + """ + mapping = self._AGENT_NAME_MAP.get(self.platform, {}) + return mapping.get(agent, agent) + + # ========================================================================= + # Agent Path + # ========================================================================= + + @property + def config_dir_name(self) -> str: + """Get platform-specific config directory name. + + Returns: + Directory name ('.claude', '.opencode', '.cursor', '.iflow', '.codex', '.kilocode', '.kiro', '.gemini', '.agent', '.devin', '.qoder', '.codebuddy', '.github/copilot', '.factory', '.pi', or '.trae') + """ + if self.platform == "opencode": + return ".opencode" + elif self.platform == "cursor": + return ".cursor" + elif self.platform == "iflow": + return ".iflow" + elif self.platform == "codex": + return ".codex" + elif self.platform == "kilo": + return ".kilocode" + elif self.platform == "kiro": + return ".kiro" + elif self.platform == "gemini": + return ".gemini" + elif self.platform == "antigravity": + return ".agent" + elif self.platform == "devin": + return ".devin" + elif self.platform == "qoder": + return ".qoder" + elif self.platform == "codebuddy": + return ".codebuddy" + elif self.platform == "copilot": + return ".github/copilot" + elif self.platform == "droid": + return ".factory" + elif self.platform == "pi": + return ".pi" + elif self.platform == "trae": + return ".trae" + elif self.platform == "omp": + return ".omp" + elif self.platform == "grok": + return ".grok" + elif self.platform == "kimi": + return ".kimi-code" + else: + return ".claude" + + def get_config_dir(self, project_root: Path) -> Path: + """Get platform-specific config directory. + + Args: + project_root: Project root directory + + Returns: + Path to config directory (.claude, .opencode, .cursor, .iflow, .codex, .kilocode, .kiro, .gemini, .agent, .devin, .qoder, .codebuddy, .github/copilot, .factory, .pi, or .trae) + """ + return project_root / self.config_dir_name + + def get_agent_path(self, agent: str, project_root: Path) -> Path: + """Get path to agent definition file. + + Args: + agent: Agent name (original, before mapping) + project_root: Project root directory + + Returns: + Path to agent definition file (.md for most platforms, .toml for Codex) + """ + mapped_name = self.get_agent_name(agent) + if self.platform == "codex": + return self.get_config_dir(project_root) / "agents" / f"{mapped_name}.toml" + return self.get_config_dir(project_root) / "agents" / f"{mapped_name}.md" + + def get_commands_path(self, project_root: Path, *parts: str) -> Path: + """Get path to commands directory or specific command file. + + Args: + project_root: Project root directory + *parts: Additional path parts (e.g., 'trellis', 'finish-work.md') + + Returns: + Path to commands directory or file + + Note: + Cursor uses prefix naming: .cursor/commands/trellis-.md + Antigravity uses workflow directory: .agent/workflows/.md + Devin uses workflow directory: .devin/workflows/trellis-.md + Copilot uses prompt files: .github/prompts/.prompt.md + Pi uses prompt templates: .pi/prompts/trellis-.md + Claude/OpenCode use subdirectory: .claude/commands/trellis/.md + """ + if self.platform == "pi": + prompts_dir = self.get_config_dir(project_root) / "prompts" + if not parts: + return prompts_dir + if len(parts) >= 2 and parts[0] == "trellis": + filename = parts[-1] + if filename.endswith(".md"): + filename = filename[:-3] + return prompts_dir / f"trellis-{filename}.md" + return prompts_dir / Path(*parts) + # OMP and Grok: flat slash commands under .{platform}/commands/trellis-.md + if self.platform in ("omp", "grok"): + commands_dir = self.get_config_dir(project_root) / "commands" + if not parts: + return commands_dir + if len(parts) >= 2 and parts[0] == "trellis": + filename = parts[-1] + if filename.endswith(".md"): + filename = filename[:-3] + return commands_dir / f"trellis-{filename}.md" + return commands_dir / Path(*parts) + + # Kimi: commands are skills under .kimi-code/skills/trellis-/SKILL.md + if self.platform == "kimi": + skills_dir = self.get_config_dir(project_root) / "skills" + if not parts: + return skills_dir + if len(parts) >= 2 and parts[0] == "trellis": + filename = parts[-1] + if filename.endswith(".md"): + filename = filename[:-3] + return skills_dir / f"trellis-{filename}" / "SKILL.md" + return skills_dir / Path(*parts) + + if self.platform == "devin": + workflow_dir = self.get_config_dir(project_root) / "workflows" + if not parts: + return workflow_dir + if len(parts) >= 2 and parts[0] == "trellis": + filename = parts[-1] + return workflow_dir / f"trellis-{filename}" + return workflow_dir / Path(*parts) + + if self.platform in ("antigravity", "kilo"): + workflow_dir = self.get_config_dir(project_root) / "workflows" + if not parts: + return workflow_dir + if len(parts) >= 2 and parts[0] == "trellis": + filename = parts[-1] + return workflow_dir / filename + return workflow_dir / Path(*parts) + + if self.platform == "copilot": + prompts_dir = project_root / ".github" / "prompts" + if not parts: + return prompts_dir + if len(parts) >= 2 and parts[0] == "trellis": + filename = parts[-1] + if filename.endswith(".md"): + filename = filename[:-3] + return prompts_dir / f"{filename}.prompt.md" + return prompts_dir / Path(*parts) + + if not parts: + return self.get_config_dir(project_root) / "commands" + + # Cursor uses prefix naming instead of subdirectory + if self.platform == "cursor" and len(parts) >= 2 and parts[0] == "trellis": + # Convert trellis/.md to trellis-.md + filename = parts[-1] + return ( + self.get_config_dir(project_root) / "commands" / f"trellis-{filename}" + ) + + return self.get_config_dir(project_root) / "commands" / Path(*parts) + + def get_trellis_command_path(self, name: str) -> str: + """Get relative path to a trellis command file. + + Args: + name: Command name without extension (e.g., 'finish-work', 'check') + + Returns: + Relative path string for use in JSONL entries + + Note: + Cursor: .cursor/commands/trellis-.md + Codex: .agents/skills/trellis-/SKILL.md + Kiro: .kiro/skills/trellis-/SKILL.md + Gemini: .gemini/commands/trellis/.toml + Antigravity: .agent/workflows/.md + Devin: .devin/workflows/trellis-.md + Pi: .pi/prompts/trellis-.md + Others: .{platform}/commands/trellis/.md + """ + if self.platform == "cursor": + return f".cursor/commands/trellis-{name}.md" + elif self.platform == "codex": + # 0.5.0-beta.0 renamed all skill dirs to add the `trellis-` prefix + # (see that release's manifest for the 60+ rename entries). + return f".agents/skills/trellis-{name}/SKILL.md" + elif self.platform == "kiro": + return f".kiro/skills/trellis-{name}/SKILL.md" + elif self.platform == "gemini": + return f".gemini/commands/trellis/{name}.toml" + elif self.platform == "antigravity": + return f".agent/workflows/{name}.md" + elif self.platform == "devin": + return f".devin/workflows/trellis-{name}.md" + elif self.platform == "kilo": + return f".kilocode/workflows/{name}.md" + elif self.platform == "copilot": + return f".github/prompts/{name}.prompt.md" + elif self.platform == "droid": + return f".factory/commands/trellis/{name}.md" + elif self.platform == "pi": + return f".pi/prompts/trellis-{name}.md" + elif self.platform in ("omp", "grok"): + return f"{self.config_dir_name}/commands/trellis-{name}.md" + elif self.platform == "kimi": + return f".kimi-code/skills/trellis-{name}/SKILL.md" + else: + return f"{self.config_dir_name}/commands/trellis/{name}.md" + + # ========================================================================= + # Environment Variables + # ========================================================================= + + def get_non_interactive_env(self) -> dict[str, str]: + """Get environment variables for non-interactive mode. + + Returns: + Dict of environment variables to set + """ + if self.platform == "opencode": + return {"OPENCODE_NON_INTERACTIVE": "1"} + elif self.platform == "iflow": + return {"IFLOW_NON_INTERACTIVE": "1"} + elif self.platform == "codex": + return {"CODEX_NON_INTERACTIVE": "1"} + elif self.platform == "kiro": + return {"KIRO_NON_INTERACTIVE": "1"} + elif self.platform == "gemini": + return {} # Gemini CLI doesn't have a non-interactive env var + elif self.platform == "antigravity": + return {} + elif self.platform == "devin": + return {} + elif self.platform == "qoder": + return {} + elif self.platform == "codebuddy": + return {} + elif self.platform == "copilot": + return {} + elif self.platform == "droid": + return {} + elif self.platform == "pi": + return {} + elif self.platform == "trae": + return {} + elif self.platform == "omp": + return {} + elif self.platform == "grok": + return {} + elif self.platform == "kimi": + return {} + else: + return {"CLAUDE_NON_INTERACTIVE": "1"} + + # ========================================================================= + # CLI Command Building + # ========================================================================= + + def build_run_command( + self, + agent: str, + prompt: str, + session_id: str | None = None, + skip_permissions: bool = True, + verbose: bool = True, + json_output: bool = True, + ) -> list[str]: + """Build CLI command for running an agent. + + Args: + agent: Agent name (will be mapped if needed) + prompt: Prompt to send to the agent + session_id: Optional session ID (Claude Code only for creation) + skip_permissions: Whether to skip permission prompts + verbose: Whether to enable verbose output + json_output: Whether to use JSON output format + + Returns: + List of command arguments + """ + mapped_agent = self.get_agent_name(agent) + + if self.platform == "opencode": + cmd = ["opencode", "run"] + cmd.extend(["--agent", mapped_agent]) + + # Note: OpenCode 'run' mode is non-interactive by default + # No equivalent to Claude Code's --dangerously-skip-permissions + # See: https://github.com/anomalyco/opencode/issues/9070 + + if json_output: + cmd.extend(["--format", "json"]) + + if verbose: + cmd.extend(["--log-level", "DEBUG", "--print-logs"]) + + # Note: OpenCode doesn't support --session-id on creation + # Session ID must be extracted from logs after startup + + cmd.append(prompt) + + elif self.platform == "iflow": + cmd = ["iflow", "-y", "-p"] + cmd.append(f"${mapped_agent} {prompt}") + elif self.platform == "codex": + cmd = ["codex", "exec"] + cmd.append(prompt) + elif self.platform == "kiro": + cmd = ["kiro", "run", prompt] + elif self.platform == "gemini": + cmd = ["gemini"] + cmd.append(prompt) + elif self.platform == "antigravity": + raise ValueError( + "Antigravity workflows are UI slash commands; CLI agent run is not supported." + ) + elif self.platform == "devin": + raise ValueError( + "Devin workflows are UI slash commands; CLI agent run is not supported." + ) + elif self.platform == "qoder": + cmd = ["qodercli", "-p", prompt] + elif self.platform == "codebuddy": + raise ValueError( + "CodeBuddy does not support non-interactive mode (no CLI agent)" + ) + elif self.platform == "copilot": + raise ValueError( + "GitHub Copilot is IDE-only; CLI agent run is not supported." + ) + elif self.platform == "droid": + raise ValueError( + "Factory Droid CLI agent run is not yet supported." + ) + elif self.platform == "pi": + cmd = ["pi", "-p", prompt] + elif self.platform == "trae": + raise ValueError( + "Trae is IDE-only; CLI agent run is not supported." + ) + elif self.platform == "omp": + raise ValueError( + "OMP uses native task tool for agent runs; CLI agent run is not supported." + ) + elif self.platform == "grok": + # Headless single-prompt; sub-agents use in-process spawn_subagent. + cmd = ["grok", "-p", prompt, "--yolo"] + elif self.platform == "kimi": + # Headless single-prompt with auto-approval; sub-agents are the + # built-in coder/explore/plan agents dispatched in-session. + cmd = ["kimi", "-p", prompt, "--yolo"] + + else: # claude + cmd = ["claude", "-p"] + cmd.extend(["--agent", mapped_agent]) + + if session_id: + cmd.extend(["--session-id", session_id]) + + if skip_permissions: + cmd.append("--dangerously-skip-permissions") + + if json_output: + cmd.extend(["--output-format", "stream-json"]) + + if verbose: + cmd.append("--verbose") + + cmd.append(prompt) + + return cmd + + def build_resume_command(self, session_id: str) -> list[str]: + """Build CLI command for resuming a session. + + Args: + session_id: Session ID to resume (ignored for iFlow) + + Returns: + List of command arguments + """ + if self.platform == "opencode": + return ["opencode", "run", "--session", session_id] + elif self.platform == "iflow": + # iFlow uses -c to continue most recent conversation + # session_id is ignored as iFlow doesn't support session IDs + return ["iflow", "-c"] + elif self.platform == "codex": + return ["codex", "resume", session_id] + elif self.platform == "kiro": + return ["kiro", "resume", session_id] + elif self.platform == "gemini": + return ["gemini", "--resume", session_id] + elif self.platform == "antigravity": + raise ValueError( + "Antigravity workflows are UI slash commands; CLI resume is not supported." + ) + elif self.platform == "devin": + raise ValueError( + "Devin workflows are UI slash commands; CLI resume is not supported." + ) + elif self.platform == "qoder": + return ["qodercli", "--resume", session_id] + elif self.platform == "codebuddy": + raise ValueError( + "CodeBuddy does not support non-interactive mode (no CLI agent)" + ) + elif self.platform == "copilot": + raise ValueError( + "GitHub Copilot is IDE-only; CLI resume is not supported." + ) + elif self.platform == "droid": + raise ValueError( + "Factory Droid CLI resume is not yet supported." + ) + elif self.platform == "pi": + return ["pi", "-c", session_id] + elif self.platform == "trae": + raise ValueError( + "Trae is IDE-only; CLI resume is not supported." + ) + elif self.platform == "omp": + raise ValueError( + "OMP uses native task tool for agent runs; CLI resume is not supported." + ) + elif self.platform == "grok": + return ["grok", "-c"] + elif self.platform == "kimi": + return ["kimi", "--session", session_id] + else: + return ["claude", "--resume", session_id] + + def get_resume_command_str(self, session_id: str, cwd: str | None = None) -> str: + """Get human-readable resume command string. + + Args: + session_id: Session ID to resume + cwd: Optional working directory to cd into + + Returns: + Command string for display + """ + cmd = self.build_resume_command(session_id) + cmd_str = " ".join(cmd) + + if cwd: + return f"cd {cwd} && {cmd_str}" + return cmd_str + + # ========================================================================= + # Platform Detection Helpers + # ========================================================================= + + @property + def is_opencode(self) -> bool: + """Check if platform is OpenCode.""" + return self.platform == "opencode" + + @property + def is_claude(self) -> bool: + """Check if platform is Claude Code.""" + return self.platform == "claude" + + @property + def is_cursor(self) -> bool: + """Check if platform is Cursor.""" + return self.platform == "cursor" + + @property + def is_iflow(self) -> bool: + """Check if platform is iFlow CLI.""" + return self.platform == "iflow" + + @property + def cli_name(self) -> str: + """Get CLI executable name. + + Note: Cursor doesn't have a CLI tool, returns None-like value. + """ + if self.is_opencode: + return "opencode" + elif self.is_cursor: + return "cursor" # Note: Cursor is IDE-only, no CLI + elif self.platform == "iflow": + return "iflow" + elif self.platform == "kiro": + return "kiro" + elif self.platform == "gemini": + return "gemini" + elif self.platform == "antigravity": + return "agy" + elif self.platform == "devin": + return "devin" + elif self.platform == "qoder": + return "qodercli" + elif self.platform == "codebuddy": + return "codebuddy" + elif self.platform == "copilot": + return "copilot" + elif self.platform == "droid": + return "droid" + elif self.platform == "pi": + return "pi" + elif self.platform == "trae": + return "trae" + elif self.platform == "omp": + return "omp" + elif self.platform == "grok": + return "grok" + elif self.platform == "kimi": + return "kimi" + else: + return "claude" + + @property + def supports_cli_agents(self) -> bool: + """Check if platform supports running agents via CLI. + + Claude Code, OpenCode, iFlow, and Codex support CLI agent execution. + Cursor is IDE-only and doesn't support CLI agents. + """ + return self.platform in ( + "claude", + "opencode", + "iflow", + "codex", + "pi", + "grok", + "kimi", + ) + + @property + def requires_agent_definition_file(self) -> bool: + """Check if platform requires an agent definition file (.md/.toml) to run. + + Claude Code, OpenCode, iFlow: require agent .md files (--agent flag). + Codex: auto-discovers agents from .codex/agents/*.toml, no --agent flag. + """ + return self.platform in ("claude", "opencode", "iflow") + + # ========================================================================= + # Session ID Handling + # ========================================================================= + + @property + def supports_session_id_on_create(self) -> bool: + """Check if platform supports specifying session ID on creation. + + Claude Code: Yes (--session-id) + OpenCode: No (auto-generated, extract from logs) + iFlow: No (no session ID support) + """ + return self.platform == "claude" + + def extract_session_id_from_log(self, log_content: str) -> str | None: + """Extract session ID from log output (OpenCode only). + + OpenCode generates session IDs in format: ses_xxx + + Args: + log_content: Log file content + + Returns: + Session ID if found, None otherwise + """ + import re + + # OpenCode session ID pattern + match = re.search(r"ses_[a-zA-Z0-9]+", log_content) + if match: + return match.group(0) + return None + + +# ============================================================================= +# Factory Function +# ============================================================================= + + +def get_cli_adapter(platform: str = "claude") -> CLIAdapter: + """Get CLI adapter for the specified platform. + + Args: + platform: Platform name ('claude', 'opencode', 'cursor', 'iflow', 'codex', 'kilo', 'kiro', 'gemini', 'antigravity', 'devin', 'qoder', 'codebuddy', 'copilot', 'droid', 'pi', or 'trae') + + Returns: + CLIAdapter instance + + Raises: + ValueError: If platform is not supported + + Note: + 'windsurf' is accepted as a deprecated alias for 'devin' (Windsurf was + renamed to Devin) and normalized before validation. + """ + # Deprecated alias: Windsurf was renamed to Devin. + if platform == "windsurf": + platform = "devin" + if platform not in ( + "claude", + "opencode", + "cursor", + "iflow", + "codex", + "kilo", + "kiro", + "gemini", + "antigravity", + "devin", + "qoder", + "codebuddy", + "copilot", + "droid", + "pi", + "trae", + "omp", + "grok", + "kimi", + ): + raise ValueError( + f"Unsupported platform: {platform} (must be 'claude', 'opencode', 'cursor', 'iflow', 'codex', 'kilo', 'kiro', 'gemini', 'antigravity', 'devin', 'qoder', 'codebuddy', 'copilot', 'droid', 'pi', 'trae', 'omp', 'grok', or 'kimi')" + ) + + return CLIAdapter(platform=platform) # type: ignore + + +_ALL_PLATFORM_CONFIG_DIRS = ( + ".claude", + ".cursor", + ".iflow", + ".opencode", + ".codex", + ".kilocode", + ".kiro", + ".gemini", + ".agent", + ".devin", + ".windsurf", # deprecated: pre-rename Devin config dir (still a platform signal) + ".qoder", + ".codebuddy", + ".github/copilot", + ".factory", + ".pi", + ".trae", + ".omp", + ".grok", + ".kimi-code", +) +"""Platform-specific config directory names used by detect_platform exclusion +checks. `.agents/skills/` is NOT listed here: it is a shared cross-platform +layer (written by Codex, also consumed by Amp/Cline/Warp/etc. via the +agentskills.io standard), not a single-platform signal. Its presence must not +block detection of Kiro, Antigravity, Devin, or other platforms.""" + + +def _has_other_platform_dir(project_root: Path, exclude: set[str]) -> bool: + """Check if any platform config dir exists besides those in *exclude*.""" + return any( + (project_root / d).is_dir() + for d in _ALL_PLATFORM_CONFIG_DIRS + if d not in exclude + ) + + +def detect_platform(project_root: Path) -> Platform: + """Auto-detect platform based on existing config directories. + + Detection order: + 1. TRELLIS_PLATFORM environment variable (if set) + 2. .opencode directory exists → opencode + 3. .iflow directory exists → iflow + 4. .cursor directory exists (without .claude) → cursor + 5. .gemini directory exists → gemini + 6. .codex exists and no other platform dirs → codex + 7. .kilocode directory exists → kilo + 8. .kiro/skills exists and no other platform dirs → kiro + 9. .agent/workflows exists and no other platform dirs → antigravity + 10. .devin/workflows (or legacy .windsurf/workflows) exists and no other platform dirs → devin + 11. .codebuddy directory exists → codebuddy + 12. .qoder directory exists → qoder + 13. .github/copilot directory exists → copilot + 14. .factory directory exists → droid + 15. .pi directory exists → pi + 16. .trae directory exists → trae + 17. Default → claude + + Args: + project_root: Project root directory + + Returns: + Detected platform ('claude', 'opencode', 'cursor', 'iflow', 'codex', 'kilo', 'kiro', 'gemini', 'antigravity', 'devin', 'qoder', 'codebuddy', 'copilot', 'droid', 'pi', 'trae', or default 'claude') + """ + import os + + # Check environment variable first + env_platform = os.environ.get("TRELLIS_PLATFORM", "").lower() + # Deprecated alias: Windsurf was renamed to Devin. + if env_platform == "windsurf": + env_platform = "devin" + if env_platform in ( + "claude", + "opencode", + "cursor", + "iflow", + "codex", + "kilo", + "kiro", + "gemini", + "antigravity", + "devin", + "qoder", + "codebuddy", + "copilot", + "droid", + "pi", + "trae", + "omp", + "grok", + "kimi", + ): + return env_platform # type: ignore + + # Check for .opencode directory (OpenCode-specific) + if (project_root / ".opencode").is_dir(): + return "opencode" + + # Check for .iflow directory (iFlow-specific) + if (project_root / ".iflow").is_dir(): + return "iflow" + + # Check for .cursor directory (Cursor-specific) + # Only detect as cursor if .claude doesn't exist (to avoid confusion) + if (project_root / ".cursor").is_dir() and not (project_root / ".claude").is_dir(): + return "cursor" + + # Check for .gemini directory (Gemini CLI-specific) + if (project_root / ".gemini").is_dir(): + return "gemini" + + # Check for .codex directory (Codex-specific) + # .agents/skills/ alone does NOT trigger codex detection (it's a shared standard) + if (project_root / ".codex").is_dir() and not _has_other_platform_dir( + project_root, {".codex", ".agents"} + ): + return "codex" + + # Check for .kilocode directory (Kilo-specific) + if (project_root / ".kilocode").is_dir(): + return "kilo" + + # Check for Kiro skills directory only when no other platform config exists + if (project_root / ".kiro" / "skills").is_dir() and not _has_other_platform_dir( + project_root, {".kiro"} + ): + return "kiro" + + # Check for Antigravity workflow directory only when no other platform config exists + if ( + project_root / ".agent" / "workflows" + ).is_dir() and not _has_other_platform_dir( + project_root, {".agent", ".gemini"} + ): + return "antigravity" + + # Check for Devin workflow directory only when no other platform config + # exists. `.windsurf/workflows` is the legacy pre-rename path (still detected + # as devin for back-compat until users migrate via `trellis update --migrate`). + if ( + (project_root / ".devin" / "workflows").is_dir() + or (project_root / ".windsurf" / "workflows").is_dir() + ) and not _has_other_platform_dir( + project_root, {".devin", ".windsurf"} + ): + return "devin" + + # Check for .codebuddy directory (CodeBuddy-specific) + if (project_root / ".codebuddy").is_dir(): + return "codebuddy" + + # Check for .qoder directory (Qoder-specific) + if (project_root / ".qoder").is_dir(): + return "qoder" + + # Check for .github/copilot directory (GitHub Copilot-specific) + if (project_root / ".github" / "copilot").is_dir(): + return "copilot" + + # Check for .factory directory (Factory Droid-specific) + if (project_root / ".factory").is_dir(): + return "droid" + + # Check for .pi directory (Pi Agent-specific) + if (project_root / ".pi").is_dir(): + return "pi" + + # Check for .trae directory (Trae IDE-specific) + if (project_root / ".trae").is_dir(): + return "trae" + + # Check for .omp directory (OMP-specific) + if (project_root / ".omp").is_dir(): + return "omp" + + # Check for .grok directory (Grok Build-specific) + if (project_root / ".grok").is_dir(): + return "grok" + + # Check for .kimi-code directory (Kimi Code-specific) + if (project_root / ".kimi-code").is_dir(): + return "kimi" + + # Fallback: checkout only has the Codex shared-skills layer + # (.agents/skills/trellis-* dirs) and no explicit platform config dir. + # Happens on fresh clones where .codex/ is gitignored/absent but the + # shared skills were committed to git. Must guard against the case + # where .claude/ or any other platform dir also exists — .agents/skills/ + # can legitimately coexist with any platform as a shared consumption + # layer for Amp/Cline/Warp/etc. + agents_skills = project_root / ".agents" / "skills" + if agents_skills.is_dir() and not _has_other_platform_dir( + project_root, set() + ): + try: + for entry in agents_skills.iterdir(): + if entry.is_dir() and entry.name.startswith("trellis-"): + return "codex" + except OSError: + pass + + return "claude" + + +def get_cli_adapter_auto(project_root: Path) -> CLIAdapter: + """Get CLI adapter with auto-detected platform. + + Args: + project_root: Project root directory + + Returns: + CLIAdapter instance for detected platform + """ + platform = detect_platform(project_root) + return CLIAdapter(platform=platform) diff --git a/.trellis/scripts/common/config.py b/.trellis/scripts/common/config.py new file mode 100755 index 0000000..f8fb7f8 --- /dev/null +++ b/.trellis/scripts/common/config.py @@ -0,0 +1,482 @@ +#!/usr/bin/env python3 +""" +Trellis configuration reader. + +Reads settings from .trellis/config.yaml with sensible defaults. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +from .paths import DIR_WORKFLOW, get_repo_root +from .trellis_config import parse_simple_yaml + +# The YAML subset parser lives in trellis_config.py — it imports nothing from +# this package, so hooks can load it as a single standalone file. Two byte- +# equivalent copies is a drift hazard, not a feature. + + +# Defaults +DEFAULT_SESSION_COMMIT_MESSAGE = "chore: record journal" +DEFAULT_MAX_JOURNAL_LINES = 2000 +DEFAULT_SESSION_AUTO_COMMIT = True +DEFAULT_CODEX_DISPATCH_MODE = "auto" + +CONFIG_FILE = "config.yaml" + + +TRUE_CONFIG_VALUES = ("true", "yes", "1", "on") +FALSE_CONFIG_VALUES = ("false", "no", "0", "off") + + +def coerce_config_bool( + value: object, + default: bool, + label: str, +) -> bool: + """Coerce a config value to a bool, warning on anything unrecognized. + + The parser stores every value as a string, so ``git: yes`` arrives as + ``"yes"``. Every boolean config key goes through this one helper: an + accepted-here/rejected-there split means a user writing a perfectly + reasonable YAML boolean silently gets the opposite branch. + + Args: + value: Raw value from the parsed config. + default: Returned when the value is unrecognized. + label: Config key name, used in the warning. + """ + if isinstance(value, bool): + return value + s = str(value).strip().lower() + if s in TRUE_CONFIG_VALUES: + return True + if s in FALSE_CONFIG_VALUES: + return False + print( + f"[WARN] invalid {label} value: {value!r}; using {str(default).lower()} (default)", + file=sys.stderr, + ) + return default + + +def _is_true_config_value(value: object, label: str = "config flag") -> bool: + """Return True when a config value represents an enabled flag.""" + if value is None: + return False + return coerce_config_bool(value, False, label) + + +def _get_config_path(repo_root: Path | None = None) -> Path: + """Get path to config.yaml.""" + root = repo_root or get_repo_root() + return root / DIR_WORKFLOW / CONFIG_FILE + + +def _load_config(repo_root: Path | None = None) -> dict: + """Load and parse config.yaml. Returns empty dict on any error. + + Fail-open, matching ``trellis_config.read_trellis_config``: a malformed + config must not take down ``task.py create``. A parse failure is reported + once on stderr so it is not invisible. + """ + config_file = _get_config_path(repo_root) + try: + content = config_file.read_text(encoding="utf-8") + except (OSError, IOError): + return {} + try: + parsed = parse_simple_yaml(content, source=str(config_file)) + except Exception as e: + print( + f"[WARN] could not parse {config_file}: {type(e).__name__}: {e}; " + "using defaults", + file=sys.stderr, + ) + return {} + return parsed if isinstance(parsed, dict) else {} + + +def get_session_commit_message(repo_root: Path | None = None) -> str: + """Get the commit message for auto-committing session records.""" + config = _load_config(repo_root) + return config.get("session_commit_message", DEFAULT_SESSION_COMMIT_MESSAGE) + + +def get_max_journal_lines(repo_root: Path | None = None) -> int: + """Get the maximum lines per journal file.""" + config = _load_config(repo_root) + value = config.get("max_journal_lines", DEFAULT_MAX_JOURNAL_LINES) + try: + return int(value) + except (ValueError, TypeError): + return DEFAULT_MAX_JOURNAL_LINES + + +def get_session_auto_commit(repo_root: Path | None = None) -> bool: + """Whether scripts should auto-stage + auto-commit session/task changes. + + Governs both ``add_session.py:_auto_commit_workspace`` and + ``task_store.py:_auto_commit_archive``. + + Default: ``True`` (existing behavior — auto-stage + auto-commit). + Set ``session_auto_commit: false`` in ``.trellis/config.yaml`` to skip + auto-staging entirely; the journal/archive files are still written to + disk, but the user manages ``git add`` / ``git commit`` themselves. + + Accepts native YAML booleans (``true`` / ``false``) and the string + aliases ``true / false / yes / no / 1 / 0 / on / off`` (case-insensitive). + Invalid values fall back to ``True`` with a stderr warning. + """ + config = _load_config(repo_root) + raw = config.get("session_auto_commit", DEFAULT_SESSION_AUTO_COMMIT) + return coerce_config_bool( + raw, DEFAULT_SESSION_AUTO_COMMIT, "session_auto_commit" + ) + + +def get_codex_dispatch_mode(repo_root: Path | None = None) -> str: + """Return Codex dispatch mode. + + Default is ``auto``, which dispatches Trellis sub-agents and uses native + context injection with a child-side fallback. ``inline`` is an explicit + opt-out. ``sub-agent`` remains a backwards-compatible alias for ``auto``. + + Invalid explicit configuration falls back to ``inline`` rather than + unexpectedly dispatching a sub-agent. This CLI-facing parser is the only + place that emits a warning for invalid values; hook readers fail safely + without producing per-turn warning noise. + """ + config = _load_config(repo_root) + codex = config.get("codex") + if codex is None: + return DEFAULT_CODEX_DISPATCH_MODE + if not isinstance(codex, dict): + print( + f"[WARN] invalid codex config: {codex!r}; using inline", + file=sys.stderr, + ) + return "inline" + + raw = codex.get("dispatch_mode", DEFAULT_CODEX_DISPATCH_MODE) + mode = str(raw).strip().lower() + if mode in ("auto", "inline"): + return mode + if mode == "sub-agent": + return "auto" + print( + f"[WARN] invalid codex.dispatch_mode value: {raw!r}; using inline", + file=sys.stderr, + ) + return "inline" + + +DEFAULT_CONTEXT_INJECTION_MAX_FILE_BYTES = 32768 +DEFAULT_CONTEXT_INJECTION_MAX_ARTIFACT_BYTES = 65536 +DEFAULT_CONTEXT_INJECTION_MAX_TOTAL_BYTES = 131072 + + +def get_context_injection_limits(repo_root: Path | None = None) -> dict[str, int]: + """Return sub-agent context injection byte limits. + + Reads the ``context_injection:`` section of ``.trellis/config.yaml``: + + context_injection: + max_file_bytes: 32768 + max_artifact_bytes: 65536 + max_total_bytes: 131072 + + ``0`` disables the corresponding limit. Missing keys use their default; + invalid (non-int or negative) values fall back to the default for that + key with a stderr warning. + """ + defaults = { + "max_file_bytes": DEFAULT_CONTEXT_INJECTION_MAX_FILE_BYTES, + "max_artifact_bytes": DEFAULT_CONTEXT_INJECTION_MAX_ARTIFACT_BYTES, + "max_total_bytes": DEFAULT_CONTEXT_INJECTION_MAX_TOTAL_BYTES, + } + + config = _load_config(repo_root) + section = config.get("context_injection") + if not isinstance(section, dict): + return defaults + + result = dict(defaults) + for key, default_value in defaults.items(): + if key not in section: + continue + raw = section[key] + try: + value = int(raw) + except (TypeError, ValueError): + print( + f"[WARN] invalid context_injection.{key} value: {raw!r}; " + f"using default {default_value}", + file=sys.stderr, + ) + continue + if value < 0: + print( + f"[WARN] invalid context_injection.{key} value: {raw!r}; " + f"using default {default_value}", + file=sys.stderr, + ) + continue + result[key] = value + + return result + + +DEFAULT_PROMPT_INJECTION_SKIP_KEYWORD = "no-trellis" + + +def get_prompt_injection_config(repo_root: Path | None = None) -> dict[str, str]: + """Return per-turn prompt injection config. + + Reads the ``prompt_injection:`` section of ``.trellis/config.yaml``: + + prompt_injection: + skip_keyword: "no-trellis" # "" disables the escape hatch entirely + + ``skip_keyword`` is the word-boundary, case-insensitive keyword that, when + present in the user's prompt, makes the per-turn workflow-state injection + emit nothing for that turn. Defaults to ``"no-trellis"``. A non-string + value falls back to the default. + """ + defaults = {"skip_keyword": DEFAULT_PROMPT_INJECTION_SKIP_KEYWORD} + + config = _load_config(repo_root) + section = config.get("prompt_injection") + if not isinstance(section, dict): + return defaults + + result = dict(defaults) + raw = section.get("skip_keyword", DEFAULT_PROMPT_INJECTION_SKIP_KEYWORD) + if isinstance(raw, str): + result["skip_keyword"] = raw + return result + + +def get_hooks(event: str, repo_root: Path | None = None) -> list[str]: + """Get hook commands for a lifecycle event. + + Args: + event: Event name (e.g. "after_create", "after_archive"). + repo_root: Repository root path. + + A hook the user believes is installed and which silently never runs is the + worst outcome for this feature, so a declared-but-unusable shape warns + instead of returning an empty list quietly. + + Returns: + List of shell commands to execute, empty if none configured. + """ + config = _load_config(repo_root) + hooks = config.get("hooks") + if hooks is None: + return [] + if not isinstance(hooks, dict): + print( + f"[WARN] ignoring `hooks` in config.yaml: expected a mapping of " + f"event -> list of commands, got {hooks!r}", + file=sys.stderr, + ) + return [] + commands = hooks.get(event) + if commands is None: + return [] + if isinstance(commands, list): + return [str(c) for c in commands] + # `after_create: echo hi` instead of a `- ` list — parses fine, registers + # nothing. + print( + f"[WARN] ignoring hook `{event}` in config.yaml: expected a list of " + f"commands, got {commands!r}. Write it as:\n" + f" hooks:\n {event}:\n - {commands}", + file=sys.stderr, + ) + return [] + + +# ============================================================================= +# Monorepo / Packages +# ============================================================================= + + +def get_packages(repo_root: Path | None = None) -> dict[str, dict] | None: + """Get monorepo package declarations. + + Returns: + Dict mapping package name to its config (path, type, etc.), + or None if not configured (single-repo mode). + + Example return: + {"cli": {"path": "packages/cli"}, "docs-site": {"path": "docs-site", "type": "submodule"}} + """ + config = _load_config(repo_root) + packages = config.get("packages") + if not isinstance(packages, dict): + return None + # Ensure each value is a dict (filter out scalar entries) + filtered = {k: v for k, v in packages.items() if isinstance(v, dict)} + if not filtered: + return None + return filtered + + +def get_default_package(repo_root: Path | None = None) -> str | None: + """Get the default package name from config. + + Returns: + Package name string, or None if not configured. + """ + config = _load_config(repo_root) + value = config.get("default_package") + return str(value) if value else None + + +def get_submodule_packages(repo_root: Path | None = None) -> dict[str, str]: + """Get packages that are git submodules. + + Returns: + Dict mapping package name to its path for submodule-type packages. + Empty dict if none configured. + + Example return: + {"docs-site": "docs-site"} + """ + packages = get_packages(repo_root) + if packages is None: + return {} + return { + name: cfg.get("path", name) + for name, cfg in packages.items() + if cfg.get("type") == "submodule" + } + + +def get_git_packages(repo_root: Path | None = None) -> dict[str, str]: + """Get packages that have their own independent git repository. + + These are sub-directories with their own .git (not submodules), + marked with ``git: true`` in config.yaml. + + Returns: + Dict mapping package name to its path for git-repo packages. + Empty dict if none configured. + + Example config:: + + packages: + backend: + path: iqs + git: true + + Example return:: + + {"backend": "iqs"} + """ + packages = get_packages(repo_root) + if packages is None: + return {} + return { + name: cfg.get("path", name) + for name, cfg in packages.items() + if _is_true_config_value(cfg.get("git"), f"packages.{name}.git") + } + + +def is_monorepo(repo_root: Path | None = None) -> bool: + """Check if the project is configured as a monorepo (has packages in config).""" + return get_packages(repo_root) is not None + + +def get_spec_base(package: str | None = None, repo_root: Path | None = None) -> str: + """Get the spec directory base path relative to .trellis/. + + Single-repo: returns "spec" + Monorepo with package: returns "spec/" + Monorepo without package: returns "spec" (caller should specify package) + """ + if package and is_monorepo(repo_root): + return f"spec/{package}" + return "spec" + + +def validate_package(package: str, repo_root: Path | None = None) -> bool: + """Check if a package name is valid in this project. + + Single-repo (no packages configured): always returns True. + Monorepo: returns True only if package exists in config.yaml packages. + """ + packages = get_packages(repo_root) + if packages is None: + return True # Single-repo, no validation needed + return package in packages + + +def resolve_package( + task_package: str | None = None, + repo_root: Path | None = None, +) -> str | None: + """Resolve package from inferred sources with validation. + + Checks in order: task_package → default_package. + Invalid inferred values print a warning to stderr and are skipped. + + Returns: + Resolved package name, or None if no valid package found. + + Note: + CLI --package should be validated separately by the caller + (fail-fast with available packages list on error). + """ + packages = get_packages(repo_root) + if packages is None: + return None # Single-repo, no package needed + + # Try task_package (guard against non-string values from malformed JSON) + if task_package and isinstance(task_package, str): + if task_package in packages: + return task_package + print( + f"Warning: task.json package '{task_package}' not found in config, skipping", + file=sys.stderr, + ) + + # Try default_package + default = get_default_package(repo_root) + if default: + if default in packages: + return default + print( + f"Warning: default_package '{default}' not found in config, skipping", + file=sys.stderr, + ) + + return None + + +def get_spec_scope(repo_root: Path | None = None) -> list[str] | str | None: + """Get session.spec_scope configuration. + + Returns: + list[str]: Package names to include in spec scanning. + str: "active_task" to use current task's package. + None: No scope configured (scan all packages). + """ + config = _load_config(repo_root) + session = config.get("session") + if not isinstance(session, dict): + return None + + scope = session.get("spec_scope") + if scope is None: + return None + if isinstance(scope, str): + return scope # e.g. "active_task" + if isinstance(scope, list): + return [str(s) for s in scope] + return None diff --git a/.trellis/scripts/common/developer.py b/.trellis/scripts/common/developer.py new file mode 100755 index 0000000..52cb9e1 --- /dev/null +++ b/.trellis/scripts/common/developer.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +""" +Developer management utilities. + +Provides: + init_developer - Initialize developer + ensure_developer - Ensure developer is initialized (exit if not) + show_developer_info - Show developer information +""" + +from __future__ import annotations + +import sys +from datetime import datetime +from pathlib import Path + +from .paths import ( + DEVELOPER_HINT, + DIR_WORKFLOW, + DIR_WORKSPACE, + DIR_TASKS, + FILE_DEVELOPER, + FILE_JOURNAL_PREFIX, + get_repo_root, + get_developer, + check_developer, +) + + +# ============================================================================= +# Developer Initialization +# ============================================================================= + +def init_developer(name: str, repo_root: Path | None = None) -> bool: + """Initialize developer. + + Creates: + - .trellis/.developer file with developer info + - .trellis/workspace// directory structure + - Initial journal file and index.md + + Args: + name: Developer name. + repo_root: Repository root path. Defaults to auto-detected. + + Returns: + True on success, False on error. + """ + if not name: + print("Error: developer name is required", file=sys.stderr) + return False + + if repo_root is None: + repo_root = get_repo_root() + + dev_file = repo_root / DIR_WORKFLOW / FILE_DEVELOPER + workspace_dir = repo_root / DIR_WORKFLOW / DIR_WORKSPACE / name + + # Create .developer file + initialized_at = datetime.now().isoformat() + try: + dev_file.write_text( + f"name={name}\ninitialized_at={initialized_at}\n", + encoding="utf-8" + ) + except (OSError, IOError) as e: + print(f"Error: Failed to create .developer file: {e}", file=sys.stderr) + return False + + # Create workspace directory structure + try: + workspace_dir.mkdir(parents=True, exist_ok=True) + except (OSError, IOError) as e: + print(f"Error: Failed to create workspace directory: {e}", file=sys.stderr) + return False + + # Create initial journal file + journal_file = workspace_dir / f"{FILE_JOURNAL_PREFIX}1.md" + if not journal_file.exists(): + today = datetime.now().strftime("%Y-%m-%d") + journal_content = f"""# Journal - {name} (Part 1) + +> AI development session journal +> Started: {today} + +--- + +""" + try: + journal_file.write_text(journal_content, encoding="utf-8") + except (OSError, IOError) as e: + print(f"Error: Failed to create journal file: {e}", file=sys.stderr) + return False + + # Create index.md with markers for auto-update + index_file = workspace_dir / "index.md" + if not index_file.exists(): + index_content = f"""# Workspace Index - {name} + +> Journal tracking for AI development sessions. + +--- + +## Current Status + + +- **Active File**: `journal-1.md` +- **Total Sessions**: 0 +- **Last Active**: - + + +--- + +## Active Documents + + +| File | Lines | Status | +|------|-------|--------| +| `journal-1.md` | ~0 | Active | + + +--- + +## Session History + + +| # | Date | Title | Commits | Branch | +|---|------|-------|---------|--------| + + +--- + +## Notes + +- Sessions are appended to journal files +- New journal file created when current exceeds 2000 lines +- Use `add_session.py` to record sessions +""" + try: + index_file.write_text(index_content, encoding="utf-8") + except (OSError, IOError) as e: + print(f"Error: Failed to create index.md: {e}", file=sys.stderr) + return False + + print(f"Developer initialized: {name}") + print(f" .developer file: {dev_file}") + print(f" Workspace dir: {workspace_dir}") + + return True + + +def ensure_developer(repo_root: Path | None = None) -> None: + """Ensure developer is initialized, exit if not. + + Args: + repo_root: Repository root path. Defaults to auto-detected. + """ + if repo_root is None: + repo_root = get_repo_root() + + if not check_developer(repo_root): + print("Error: Developer not initialized.", file=sys.stderr) + print(f"Run: python3 ./{DIR_WORKFLOW}/scripts/init_developer.py ", file=sys.stderr) + print(DEVELOPER_HINT, file=sys.stderr) + sys.exit(1) + + +def show_developer_info(repo_root: Path | None = None) -> None: + """Show developer information. + + Args: + repo_root: Repository root path. Defaults to auto-detected. + """ + if repo_root is None: + repo_root = get_repo_root() + + developer = get_developer(repo_root) + + if not developer: + print("Developer: (not initialized)") + else: + print(f"Developer: {developer}") + print(f"Workspace: {DIR_WORKFLOW}/{DIR_WORKSPACE}/{developer}/") + print(f"Tasks: {DIR_WORKFLOW}/{DIR_TASKS}/") + + +# ============================================================================= +# Main Entry (for testing) +# ============================================================================= + +if __name__ == "__main__": + show_developer_info() diff --git a/.trellis/scripts/common/git.py b/.trellis/scripts/common/git.py new file mode 100755 index 0000000..48e7141 --- /dev/null +++ b/.trellis/scripts/common/git.py @@ -0,0 +1,203 @@ +""" +Git command execution utility. + +Single source of truth for running git commands across all Trellis scripts. +""" + +from __future__ import annotations + +import subprocess +import time +from pathlib import Path + + +# Bounded retry for transient `.git/index.lock` contention. Another process +# (IDE git integration, status daemon, a concurrent Trellis session) can hold +# the lock for a fraction of a second; three attempts spread over ~1.5s ride +# that out without making a genuinely stuck lock hang the command. One sleep +# per retry, so the attempt count follows the backoff tuple. +INDEX_LOCK_RETRY_BACKOFF = (0.5, 1.0) +INDEX_LOCK_RETRY_ATTEMPTS = len(INDEX_LOCK_RETRY_BACKOFF) + 1 + +# Whether a checkout is a linked worktree cannot change while a script runs, and +# the answer costs two subprocesses. Developer-identity resolution asks several +# times per command whenever the local `.developer` file is absent, so memoize. +_CACHE_MISS = object() +_MAIN_WORKTREE_CACHE: dict[Path, Path | None] = {} + + +def run_git( + args: list[str], + cwd: Path | None = None, + timeout: float | None = None, +) -> tuple[int, str, str]: + """Run a git command and return (returncode, stdout, stderr). + + Uses UTF-8 encoding with -c i18n.logOutputEncoding=UTF-8 to ensure + consistent output across all platforms (Windows, macOS, Linux). Callers + may provide a timeout for best-effort probes; normal Git operations remain + unbounded by default. + """ + try: + git_args = ["git", "-c", "i18n.logOutputEncoding=UTF-8"] + args + result = subprocess.run( + git_args, + cwd=cwd, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=timeout, + ) + return result.returncode, result.stdout, result.stderr + except Exception as e: + return 1, "", str(e) + + +def stderr_indicates_index_lock(stderr: str) -> bool: + """git failed because another process holds `.git/index.lock`.""" + if not stderr: + return False + return "index.lock" in stderr.lower() + + +def run_git_retry_index_lock( + args: list[str], + cwd: Path | None = None, + timeout: float | None = None, +) -> tuple[int, str, str]: + """Run a git command, retrying only while `.git/index.lock` is held. + + Any other non-zero exit returns immediately — a retry loop around real + failures (bad path, nothing to commit, hook rejection) just delays the + error. Returns the last (returncode, stdout, stderr). + """ + rc, out, err = run_git(args, cwd=cwd, timeout=timeout) + attempt = 1 + while ( + rc != 0 + and attempt < INDEX_LOCK_RETRY_ATTEMPTS + and stderr_indicates_index_lock(err) + ): + time.sleep(INDEX_LOCK_RETRY_BACKOFF[attempt - 1]) + attempt += 1 + rc, out, err = run_git(args, cwd=cwd, timeout=timeout) + return rc, out, err + + +def index_lock_path(repo_root: Path) -> str: + """Path of the lock file git is contending on, for diagnostics. + + Asks git so worktrees and `GIT_DIR` setups name the real file rather + than a `.git/` guess that does not exist there. + """ + rc, out, _ = run_git(["rev-parse", "--git-path", "index.lock"], cwd=repo_root) + if rc == 0 and out.strip(): + return out.strip() + return ".git/index.lock" + + +def resolve_default_branch(repo_root: Path) -> str | None: + """Resolve the repository's default branch (origin/HEAD target). + + Tries the local `refs/remotes/origin/HEAD` symbolic ref first (no + network access), then falls back to `git remote show origin` (which + may hit the network but also repairs a missing/stale symbolic-ref). + Returns None when neither resolves, so callers can fall back to their + own pre-existing behavior. + """ + rc, out, _ = run_git(["symbolic-ref", "refs/remotes/origin/HEAD"], cwd=repo_root) + if rc == 0 and out.strip(): + return out.strip().rsplit("/", 1)[-1] + + rc, out, _ = run_git(["remote", "show", "origin"], cwd=repo_root) + if rc == 0: + for line in out.splitlines(): + line = line.strip() + if line.startswith("HEAD branch:"): + branch = line.split(":", 1)[1].strip() + if branch and branch != "(unknown)": + return branch + + return None + + +def current_branch_name(repo_root: Path) -> str | None: + """Return the checked-out branch name, or None when there isn't one. + + Empty output covers detached HEAD and "not a git repository" alike, and + callers treat both the same way: there is no branch worth recording. + """ + rc, out, _ = run_git(["branch", "--show-current"], cwd=repo_root) + if rc != 0: + return None + return out.strip() or None + + +def has_git_remote(repo_root: Path) -> bool: + """Whether the repository has at least one configured remote.""" + rc, out, _ = run_git(["remote"], cwd=repo_root) + return rc == 0 and bool(out.strip()) + + +def main_worktree_root(repo_root: Path) -> Path | None: + """Root of the main working tree when `repo_root` is a linked worktree. + + Returns None in the main working tree itself, outside a git repository, and + for a linked worktree of a bare repository (no main checkout to point at). + + `git worktree list --porcelain` reports the main working tree as its first + record, so git identifies it rather than this code deriving it from the + `.git` layout. Deriving it — taking the parent of `--git-common-dir` — is + wrong for a bare repository that happens to sit inside an unrelated + checkout (`~/repos/project.git` under a `~/repos` that is itself a repo): + the parent is a real checkout with a real `.developer`, so the guess is + indistinguishable from a hit and identity leaks across repositories. + """ + cached = _MAIN_WORKTREE_CACHE.get(repo_root, _CACHE_MISS) + if cached is not _CACHE_MISS: + return cached # type: ignore[return-value] + + result = _probe_main_worktree_root(repo_root) + _MAIN_WORKTREE_CACHE[repo_root] = result + return result + + +def _probe_main_worktree_root(repo_root: Path) -> Path | None: + rc_list, listing, _ = run_git(["worktree", "list", "--porcelain"], cwd=repo_root) + rc_top, toplevel, _ = run_git(["rev-parse", "--show-toplevel"], cwd=repo_root) + if rc_list != 0 or rc_top != 0: + return None + + lines = listing.splitlines() + if not lines or not lines[0].startswith("worktree "): + return None + + # Records are blank-line separated; a `bare` attribute on the first one + # means the "main working tree" is a bare repo with nothing to inherit. + for line in lines[1:]: + if not line.strip(): + break + if line.strip() == "bare": + return None + + try: + main_root = Path(lines[0][len("worktree ") :].strip()).resolve() + current_root = Path(toplevel.strip()).resolve() + except (OSError, ValueError): + return None + + if main_root == current_root: + return None + return main_root + + +def branch_exists_locally(branch: str, repo_root: Path) -> bool: + """Check whether a local branch ref exists in the repository.""" + if not branch: + return False + rc, _, _ = run_git( + ["rev-parse", "--verify", "--quiet", f"refs/heads/{branch}"], + cwd=repo_root, + ) + return rc == 0 diff --git a/.trellis/scripts/common/git_context.py b/.trellis/scripts/common/git_context.py new file mode 100755 index 0000000..23fc6ec --- /dev/null +++ b/.trellis/scripts/common/git_context.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Git and Session Context utilities. + +Entry shim — delegates to session_context and packages_context. + +Provides: + output_json - Output context in JSON format + output_text - Output context in text format +""" + +from __future__ import annotations + +import json + +from .git import run_git +from .session_context import ( + get_context_json, + get_context_text, + get_context_record_json, + get_context_text_record, + output_json, + output_text, +) +from .packages_context import ( + get_context_packages_text, + get_context_packages_json, +) +from .trellis_config import read_trellis_config +from .workflow_phase import ( + filter_platform, + get_phase_index, + get_step, + resolve_effective_platform, +) + +# Backward-compatible alias — external modules import this name +_run_git_command = run_git + + +# ============================================================================= +# Main Entry +# ============================================================================= + +def main() -> None: + """CLI entry point.""" + import argparse + + parser = argparse.ArgumentParser(description="Get Session Context for AI Agent") + parser.add_argument( + "--json", + "-j", + action="store_true", + help="Output in JSON format (works with any --mode)", + ) + parser.add_argument( + "--mode", + "-m", + choices=["default", "record", "packages", "phase"], + default="default", + help="Output mode: default (full context), record (for record-session), packages (package info only), phase (workflow step extraction)", + ) + parser.add_argument( + "--step", + help="Step id for --mode phase, e.g. 1.1, 2.2. Omit to get the Phase Index.", + ) + parser.add_argument( + "--platform", + help="Platform name for --mode phase, e.g. cursor, claude-code. Filters platform-tagged blocks.", + ) + + args = parser.parse_args() + + if args.mode == "record": + if args.json: + print(json.dumps(get_context_record_json(), indent=2, ensure_ascii=False)) + else: + print(get_context_text_record()) + elif args.mode == "packages": + if args.json: + print(json.dumps(get_context_packages_json(), indent=2, ensure_ascii=False)) + else: + print(get_context_packages_text()) + elif args.mode == "phase": + content = get_step(args.step) if args.step else get_phase_index() + if not content.strip(): + if args.step: + parser.exit(2, f"Step not found: {args.step}\n") + else: + parser.exit(2, "Phase Index section not found in workflow.md\n") + if args.platform: + effective = resolve_effective_platform( + args.platform, read_trellis_config() + ) + content = filter_platform(content, effective) + print(content, end="") + else: + if args.json: + output_json() + else: + output_text() + + +if __name__ == "__main__": + main() diff --git a/.trellis/scripts/common/io.py b/.trellis/scripts/common/io.py new file mode 100755 index 0000000..f049109 --- /dev/null +++ b/.trellis/scripts/common/io.py @@ -0,0 +1,160 @@ +""" +File I/O utilities. + +Provides read_json / write_json as the single source of truth for JSON file +operations, plus write_text_atomic for the Markdown state files (journal, +index.md) that carry durable session state. +""" + +from __future__ import annotations + +import json +import os +import tempfile +from pathlib import Path + + +JSON_READ_MISSING = "missing" +JSON_READ_INVALID = "invalid" +JSON_READ_UNREADABLE = "unreadable" +JSON_READ_NOT_OBJECT = "not-object" +JSON_READ_EMPTY = "empty" +JSON_READ_UNDECODABLE = "undecodable" + + +def read_json(path: Path) -> dict | None: + """Read and parse a JSON file. + + Returns None if the file doesn't exist, is invalid JSON, or can't be read. + Use this for optional reads only — a caller that is about to overwrite the + file, or that must tell a parse error from a permissions error, wants + read_json_checked instead. + """ + try: + return json.loads(path.read_text(encoding="utf-8")) + except (FileNotFoundError, json.JSONDecodeError, OSError, UnicodeDecodeError): + # UnicodeDecodeError is not an OSError. Without it here a non-UTF-8 + # session file raises out of a tolerant read, so the hook path fails + # instead of degrading to "no active task". + return None + + +def read_json_checked(path: Path) -> tuple[dict | None, str | None]: + """Read a JSON object, keeping the ways it can fail distinguishable. + + Returns ``(data, None)`` on success, or ``(None, reason)`` where reason is + one of the ``JSON_READ_*`` constants. An empty object counts as a failure: + a state file that parses to ``{}`` carries none of the fields callers read, + and treating it as success would silently rebuild it from defaults. + """ + try: + text = path.read_text(encoding="utf-8") + except FileNotFoundError: + return None, JSON_READ_MISSING + except UnicodeDecodeError: + # Not an OSError, so it escaped both handlers and surfaced as a + # traceback. The point of this reader is that every failure mode stays + # nameable, and "not valid UTF-8" is a different repair from + # "not valid JSON". + return None, JSON_READ_UNDECODABLE + except OSError: + return None, JSON_READ_UNREADABLE + + try: + data = json.loads(text) + except json.JSONDecodeError: + return None, JSON_READ_INVALID + + if not isinstance(data, dict): + return None, JSON_READ_NOT_OBJECT + if not data: + return None, JSON_READ_EMPTY + return data, None + + +def describe_json_read_failure(path: Path, reason: str | None) -> tuple[str, str]: + """Return ``(what happened, what to do)`` for a read_json_checked reason.""" + if reason == JSON_READ_MISSING: + return (f"{path}: file not found", "Pass an existing task directory, or create the task first.") + if reason == JSON_READ_UNREADABLE: + return ( + f"{path}: could not be read (permission denied or I/O error)", + "Check the file and directory permissions, then retry.", + ) + if reason == JSON_READ_INVALID: + return ( + f"{path}: not valid JSON", + f"Fix the syntax (e.g. `python3 -m json.tool {path}`), then retry.", + ) + if reason == JSON_READ_NOT_OBJECT: + return ( + f"{path}: top level is not a JSON object", + "Restore the file to a JSON object ({ ... }), then retry.", + ) + if reason == JSON_READ_EMPTY: + return ( + f"{path}: contains an empty JSON object", + "Restore the task fields (or recreate the task), then retry.", + ) + if reason == JSON_READ_UNDECODABLE: + return ( + f"{path}: not valid UTF-8 text", + "Re-save the file as UTF-8 (or restore it from git), then retry.", + ) + return (f"{path}: could not be loaded", "Inspect the file, then retry.") + + +def write_json(path: Path, data: dict) -> bool: + """Write dict to JSON file with pretty formatting. + + The write is atomic: content goes to a temp file in the same directory + and is then renamed over the target. A crash or Ctrl-C mid-write leaves + the existing file intact rather than truncated, so a corrupted task.json + can never make a task silently vanish from `task.py list`. + + Returns True on success, False on error. + """ + return write_text_atomic(path, json.dumps(data, indent=2, ensure_ascii=False)) + + +def write_text_atomic(path: Path, text: str) -> bool: + """Write text to a file atomically (temp in same dir, then replace). + + The same never-truncate-in-place guarantee as :func:`write_json`, for the + Markdown state files that hold durable session state (journal files, + index.md). A crash or Ctrl-C mid-write leaves the previous content intact + instead of a half-written record that no retry can classify. + + Returns True on success, False on error. + """ + try: + fd, tmp = tempfile.mkstemp( + dir=str(path.parent), prefix=f".{path.name}.", suffix=".tmp" + ) + except OSError: + return False + + try: + try: + f = os.fdopen(fd, "w", encoding="utf-8") + except OSError: + # fdopen never took ownership of fd; close it ourselves. + os.close(fd) + raise + with f: + f.write(text) + os.replace(tmp, path) + return True + except OSError: + try: + os.unlink(tmp) + except OSError: + pass + return False + except BaseException: + # Ctrl-C mid-write: drop the temp file, then let the interrupt through. + try: + os.unlink(tmp) + except OSError: + pass + raise diff --git a/.trellis/scripts/common/log.py b/.trellis/scripts/common/log.py new file mode 100755 index 0000000..839c643 --- /dev/null +++ b/.trellis/scripts/common/log.py @@ -0,0 +1,45 @@ +""" +Terminal output utilities: colors and structured logging. + +Single source of truth for Colors and log_* functions +used across all Trellis scripts. +""" + +from __future__ import annotations + + +class Colors: + """ANSI color codes for terminal output.""" + + RED = "\033[0;31m" + GREEN = "\033[0;32m" + YELLOW = "\033[1;33m" + BLUE = "\033[0;34m" + CYAN = "\033[0;36m" + DIM = "\033[2m" + NC = "\033[0m" # No Color / Reset + + +def colored(text: str, color: str) -> str: + """Apply ANSI color to text.""" + return f"{color}{text}{Colors.NC}" + + +def log_info(msg: str) -> None: + """Print info-level message with [INFO] prefix.""" + print(f"{Colors.BLUE}[INFO]{Colors.NC} {msg}") + + +def log_success(msg: str) -> None: + """Print success message with [SUCCESS] prefix.""" + print(f"{Colors.GREEN}[SUCCESS]{Colors.NC} {msg}") + + +def log_warn(msg: str) -> None: + """Print warning message with [WARN] prefix.""" + print(f"{Colors.YELLOW}[WARN]{Colors.NC} {msg}") + + +def log_error(msg: str) -> None: + """Print error message with [ERROR] prefix.""" + print(f"{Colors.RED}[ERROR]{Colors.NC} {msg}") diff --git a/.trellis/scripts/common/packages_context.py b/.trellis/scripts/common/packages_context.py new file mode 100755 index 0000000..e7d4e8c --- /dev/null +++ b/.trellis/scripts/common/packages_context.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 +""" +Package discovery and context output. + +Provides: + get_packages_info - Get structured package info + get_packages_section - Build PACKAGES text section + get_context_packages_text - Full packages text output (--mode packages) + get_context_packages_json - Full packages JSON output (--mode packages --json) +""" + +from __future__ import annotations + +from pathlib import Path + +from .config import _is_true_config_value, get_default_package, get_packages, get_spec_scope +from .paths import ( + DIR_SPEC, + DIR_WORKFLOW, + get_current_task, + get_repo_root, +) +from .tasks import load_task + + +# ============================================================================= +# Internal Helpers +# ============================================================================= + +def _scan_spec_layers(spec_dir: Path, package: str | None = None) -> list[str]: + """Scan spec directory for available layers (subdirectories). + + For monorepo: scans spec// + For single-repo: scans spec/ + """ + target = spec_dir / package if package else spec_dir + if not target.is_dir(): + return [] + return sorted( + d.name for d in target.iterdir() if d.is_dir() and d.name != "guides" + ) + + +def _get_active_task_package(repo_root: Path) -> str | None: + """Get the package field from the active task's task.json.""" + current = get_current_task(repo_root) + if not current: + return None + ct = load_task(repo_root / current) + return ct.package if ct and ct.package else None + + +def _resolve_scope_set( + packages: dict, + spec_scope, + task_pkg: str | None, + default_pkg: str | None, +) -> set | None: + """Resolve spec_scope to a set of allowed package names, or None for full scan.""" + if not packages: + return None + + if spec_scope is None: + return None + + if isinstance(spec_scope, str) and spec_scope == "active_task": + if task_pkg and task_pkg in packages: + return {task_pkg} + if default_pkg and default_pkg in packages: + return {default_pkg} + return None + + if isinstance(spec_scope, list): + valid = {e for e in spec_scope if e in packages} + if valid: + return valid + # All invalid: fallback + if task_pkg and task_pkg in packages: + return {task_pkg} + if default_pkg and default_pkg in packages: + return {default_pkg} + return None + + return None + + +# ============================================================================= +# Public Functions +# ============================================================================= + +def get_packages_info(repo_root: Path) -> list[dict]: + """Get structured package info for monorepo projects. + + Returns list of dicts with keys: name, path, type, default, specLayers, + isSubmodule, isGitRepo. + Returns empty list for single-repo projects. + """ + packages = get_packages(repo_root) + if not packages: + return [] + + default_pkg = get_default_package(repo_root) + spec_dir = repo_root / DIR_WORKFLOW / DIR_SPEC + result = [] + + for pkg_name, pkg_config in packages.items(): + pkg_path = pkg_config.get("path", pkg_name) if isinstance(pkg_config, dict) else str(pkg_config) + pkg_type = pkg_config.get("type", "local") if isinstance(pkg_config, dict) else "local" + pkg_git = pkg_config.get("git", False) if isinstance(pkg_config, dict) else False + layers = _scan_spec_layers(spec_dir, pkg_name) + + result.append({ + "name": pkg_name, + "path": pkg_path, + "type": pkg_type, + "default": pkg_name == default_pkg, + "specLayers": layers, + "isSubmodule": pkg_type == "submodule", + "isGitRepo": _is_true_config_value(pkg_git), + }) + + return result + + +def get_packages_section(repo_root: Path) -> str: + """Build the PACKAGES section for text output.""" + spec_dir = repo_root / DIR_WORKFLOW / DIR_SPEC + pkg_info = get_packages_info(repo_root) + + lines: list[str] = [] + lines.append("## PACKAGES") + + if not pkg_info: + lines.append("(single-repo mode)") + layers = _scan_spec_layers(spec_dir) + if layers: + lines.append(f"Spec layers: {', '.join(layers)}") + return "\n".join(lines) + + default_pkg = get_default_package(repo_root) + + for pkg in pkg_info: + layers_str = f" [{', '.join(pkg['specLayers'])}]" if pkg["specLayers"] else "" + submodule_tag = " (submodule)" if pkg["isSubmodule"] else "" + git_repo_tag = " (git repo)" if pkg["isGitRepo"] else "" + default_tag = " *" if pkg["default"] else "" + lines.append( + f"- {pkg['name']:<16} {pkg['path']:<20}{layers_str}{submodule_tag}{git_repo_tag}{default_tag}" + ) + + if default_pkg: + lines.append(f"Default package: {default_pkg}") + + return "\n".join(lines) + + +def get_context_packages_text(repo_root: Path | None = None) -> str: + """Get packages context as formatted text (for --mode packages).""" + if repo_root is None: + repo_root = get_repo_root() + + pkg_info = get_packages_info(repo_root) + lines: list[str] = [] + + if not pkg_info: + spec_dir = repo_root / DIR_WORKFLOW / DIR_SPEC + lines.append("Single-repo project (no packages configured)") + lines.append("") + layers = _scan_spec_layers(spec_dir) + if layers: + lines.append(f"Spec layers: {', '.join(layers)}") + return "\n".join(lines) + + # Resolve scope for annotations + packages_dict = get_packages(repo_root) or {} + default_pkg = get_default_package(repo_root) + spec_scope = get_spec_scope(repo_root) + task_pkg = _get_active_task_package(repo_root) + scope_set = _resolve_scope_set(packages_dict, spec_scope, task_pkg, default_pkg) + + lines.append("## PACKAGES") + lines.append("") + for pkg in pkg_info: + default_tag = " (default)" if pkg["default"] else "" + type_tag = f" [{pkg['type']}]" if pkg["type"] != "local" else "" + git_tag = " [git repo]" if pkg["isGitRepo"] else "" + + # Scope annotation + scope_tag = "" + if scope_set is not None and pkg["name"] not in scope_set: + scope_tag = " (out of scope)" + + lines.append(f"### {pkg['name']}{default_tag}{type_tag}{git_tag}{scope_tag}") + lines.append(f"Path: {pkg['path']}") + if pkg["specLayers"]: + lines.append(f"Spec layers: {', '.join(pkg['specLayers'])}") + for layer in pkg["specLayers"]: + lines.append(f" - .trellis/spec/{pkg['name']}/{layer}/index.md") + else: + lines.append("Spec: not configured") + lines.append("") + + # Also show shared guides + guides_dir = repo_root / DIR_WORKFLOW / DIR_SPEC / "guides" + if guides_dir.is_dir(): + lines.append("### Shared Guides (always included)") + lines.append("Path: .trellis/spec/guides/index.md") + lines.append("") + + return "\n".join(lines) + + +def get_context_packages_json(repo_root: Path | None = None) -> dict: + """Get packages context as a dictionary (for --mode packages --json).""" + if repo_root is None: + repo_root = get_repo_root() + + pkg_info = get_packages_info(repo_root) + + if not pkg_info: + spec_dir = repo_root / DIR_WORKFLOW / DIR_SPEC + layers = _scan_spec_layers(spec_dir) + return { + "mode": "single-repo", + "specLayers": layers, + } + + default_pkg = get_default_package(repo_root) + spec_scope = get_spec_scope(repo_root) + task_pkg = _get_active_task_package(repo_root) + + return { + "mode": "monorepo", + "packages": pkg_info, + "defaultPackage": default_pkg, + "specScope": spec_scope, + "activeTaskPackage": task_pkg, + } diff --git a/.trellis/scripts/common/paths.py b/.trellis/scripts/common/paths.py new file mode 100755 index 0000000..64d2168 --- /dev/null +++ b/.trellis/scripts/common/paths.py @@ -0,0 +1,543 @@ +#!/usr/bin/env python3 +""" +Common path utilities for Trellis workflow. + +Provides: + get_repo_root - Get repository root directory + get_developer - Get developer name + get_workspace_dir - Get developer workspace directory + get_tasks_dir - Get tasks directory + get_active_journal_file - Get current journal file +""" + +from __future__ import annotations + +import os +import re +from datetime import datetime +from pathlib import Path + +from .git import main_worktree_root + + +# ============================================================================= +# Path Constants (change here to rename directories) +# ============================================================================= + +# Directory names +DIR_WORKFLOW = ".trellis" +DIR_WORKSPACE = "workspace" +DIR_TASKS = "tasks" +DIR_ARCHIVE = "archive" +DIR_SPEC = "spec" +DIR_SCRIPTS = "scripts" + +# File names +FILE_DEVELOPER = ".developer" +FILE_CURRENT_TASK = ".current-task" +FILE_TASK_JSON = "task.json" +FILE_JOURNAL_PREFIX = "journal-" + +# Environment override for the developer identity, ahead of the .developer file. +ENV_DEVELOPER = "TRELLIS_DEVELOPER" + +# Appended to every "no developer set" error so the two non-obvious sources are +# discoverable from the failure itself. +DEVELOPER_HINT = ( + f" Or set {ENV_DEVELOPER}= in the environment.\n" + f" A linked git worktree inherits {DIR_WORKFLOW}/{FILE_DEVELOPER} from its " + f"main checkout — run init_developer.py there to cover every worktree." +) + + +# ============================================================================= +# Repository Root +# ============================================================================= + +def get_repo_root(start_path: Path | None = None) -> Path: + """Find the nearest directory containing .trellis/ folder. + + This handles nested git repos correctly (e.g., test project inside another repo). + + Args: + start_path: Starting directory to search from. Defaults to current directory. + + Returns: + Path to repository root, or current directory if no .trellis/ found. + """ + current = (start_path or Path.cwd()).resolve() + + while current != current.parent: + if (current / DIR_WORKFLOW).is_dir(): + return current + current = current.parent + + # Fallback to current directory if no .trellis/ found + return Path.cwd().resolve() + + +# ============================================================================= +# Developer +# ============================================================================= + +def _read_developer_file(dev_file: Path) -> str | None: + """Read the `name=` field out of a .developer file, or None.""" + if not dev_file.is_file(): + return None + + try: + content = dev_file.read_text(encoding="utf-8") + except (OSError, IOError): + return None + + for line in content.splitlines(): + if line.startswith("name="): + return line.split("=", 1)[1].strip() or None + + return None + + +def get_developer(repo_root: Path | None = None) -> str | None: + """Get the developer name for this checkout. + + Resolution order, first hit wins (a CLI `--assignee` flag overrides all of + it, before this function is ever called): + + 1. The ``TRELLIS_DEVELOPER`` environment variable. + 2. ``.trellis/.developer`` in this checkout. + 3. ``.trellis/.developer`` in the main checkout, when this checkout is a + linked git worktree. + + Step 3 exists because `.developer` is gitignored on purpose — it carries a + personal identity and no tracked file should. A fresh `git worktree add` + therefore starts with no identity file of its own, which used to make every + task.py command fail until init_developer.py was re-run per worktree. The + main checkout's file is read, never copied: a copy would go stale and shadow + later changes made in the main checkout. + + Args: + repo_root: Repository root path. Defaults to auto-detected. + + Returns: + Developer name or None if not initialized. + """ + env_name = os.environ.get(ENV_DEVELOPER, "").strip() + if env_name: + return env_name + + if repo_root is None: + repo_root = get_repo_root() + + local = _read_developer_file(repo_root / DIR_WORKFLOW / FILE_DEVELOPER) + if local: + return local + + main_root = main_worktree_root(repo_root) + if main_root is None: + return None + + return _read_developer_file(main_root / DIR_WORKFLOW / FILE_DEVELOPER) + + +def check_developer(repo_root: Path | None = None) -> bool: + """Check if developer is initialized. + + Args: + repo_root: Repository root path. Defaults to auto-detected. + + Returns: + True if developer is initialized. + """ + return get_developer(repo_root) is not None + + +# ============================================================================= +# Tasks Directory +# ============================================================================= + +def get_tasks_dir(repo_root: Path | None = None) -> Path: + """Get tasks directory path. + + Args: + repo_root: Repository root path. Defaults to auto-detected. + + Returns: + Path to tasks directory. + """ + if repo_root is None: + repo_root = get_repo_root() + return repo_root / DIR_WORKFLOW / DIR_TASKS + + +# ============================================================================= +# Workspace Directory +# ============================================================================= + +def get_workspace_dir(repo_root: Path | None = None) -> Path | None: + """Get developer workspace directory. + + Args: + repo_root: Repository root path. Defaults to auto-detected. + + Returns: + Path to workspace directory or None if developer not set. + """ + if repo_root is None: + repo_root = get_repo_root() + + developer = get_developer(repo_root) + if developer: + return repo_root / DIR_WORKFLOW / DIR_WORKSPACE / developer + return None + + +# ============================================================================= +# Journal File +# ============================================================================= + +def get_active_journal_file(repo_root: Path | None = None) -> Path | None: + """Get the current active journal file. + + Args: + repo_root: Repository root path. Defaults to auto-detected. + + Returns: + Path to active journal file or None if not found. + """ + if repo_root is None: + repo_root = get_repo_root() + + workspace_dir = get_workspace_dir(repo_root) + if workspace_dir is None or not workspace_dir.is_dir(): + return None + + latest: Path | None = None + highest = 0 + + for f in workspace_dir.glob(f"{FILE_JOURNAL_PREFIX}*.md"): + if not f.is_file(): + continue + + # Extract number from filename + name = f.stem # e.g., "journal-1" + match = re.search(r"(\d+)$", name) + if match: + num = int(match.group(1)) + if num > highest: + highest = num + latest = f + + return latest + + +def count_lines(file_path: Path) -> int: + """Count lines in a file. + + Args: + file_path: Path to file. + + Returns: + Number of lines, or 0 if file doesn't exist. + """ + if not file_path.is_file(): + return 0 + + try: + return len(file_path.read_text(encoding="utf-8").splitlines()) + except (OSError, IOError): + return 0 + + +# ============================================================================= +# Current Task Management +# ============================================================================= + +def normalize_task_ref(task_ref: str) -> str: + """Normalize a task ref for stable runtime storage. + + Stored refs should prefer repo-relative POSIX paths like + `.trellis/tasks/03-27-my-task`, even on Windows. Absolute paths are preserved + unless they can later be converted back to repo-relative form by callers. + """ + normalized = task_ref.strip() + if not normalized: + return "" + + path_obj = Path(normalized) + if path_obj.is_absolute(): + return str(path_obj) + + normalized = normalized.replace("\\", "/") + while normalized.startswith("./"): + normalized = normalized[2:] + + if normalized.startswith(f"{DIR_TASKS}/"): + return f"{DIR_WORKFLOW}/{normalized}" + + return normalized + + +def resolve_task_ref(task_ref: str, repo_root: Path | None = None) -> Path | None: + """Resolve a task ref to an absolute task directory path inside the repo. + + Returns None when the ref resolves outside `repo_root`. Every reader of the + active task — `task.py`, the shared hooks, the platform extensions — comes + through here, so containment is enforced at this one point rather than at + each call site. + + It matters because a ref is not always something the user typed. It round + trips through the session pointer under `.trellis/.runtime/sessions/`, and + `..` segments used to survive that trip intact: `_canonical_task_ref` + compares lexically, and a lexical `relative_to` accepts + `/.trellis/tasks/../../../elsewhere` because the string does start + with the root. The ref was then stored verbatim and replayed on every later + turn, so `task.py start .trellis/tasks/../../../elsewhere` both rewrote that + directory's `task.json` and fed its files to the model. + + Resolving here also normalises the path, so callers get a ref without `..` + to store. + """ + if repo_root is None: + repo_root = get_repo_root() + + normalized = normalize_task_ref(task_ref) + if not normalized: + return None + + try: + root = repo_root.resolve() + except OSError: + return None + + path_obj = Path(normalized) + if path_obj.is_absolute(): + candidate = path_obj + elif normalized.startswith(f"{DIR_WORKFLOW}/"): + candidate = root / path_obj + else: + candidate = root / DIR_WORKFLOW / DIR_TASKS / path_obj + + # resolve() collapses `..` and follows symlinks, so a task directory that + # links outside the repo is refused too. Both sides are resolved because + # repo_root itself may sit behind a symlink (/tmp on macOS does). + try: + resolved = candidate.resolve() + workflow_real = (root / DIR_WORKFLOW).resolve() + except OSError: + return None + + try: + resolved.relative_to(root) + return resolved + except ValueError: + pass + + # `.trellis` may itself be a symlink into a store outside the repo (#567). + # The workflow dir's own real location is then a second legitimate + # containment base: a ref through that link never left the workflow tree. + # A ref that escapes BOTH bases (traversal, absolute path elsewhere, a + # task dir symlinked out of the tree) is still refused. + try: + rel = resolved.relative_to(workflow_real) + except ValueError: + return None + + # Map back to the in-repo (lexical) form so callers store the same + # repo-relative ref as in the non-symlinked layout. + return root / DIR_WORKFLOW / rel + + +def get_current_task( + repo_root: Path | None = None, + platform_input: dict | None = None, + platform: str | None = None, +) -> str | None: + """Get current task directory path (relative to repo_root). + + Args: + repo_root: Repository root path. Defaults to auto-detected. + + Returns: + Relative path to current task directory or None. + """ + if repo_root is None: + repo_root = get_repo_root() + + from .active_task import resolve_active_task + + return resolve_active_task(repo_root, platform_input, platform).task_path + + +def get_current_task_abs( + repo_root: Path | None = None, + platform_input: dict | None = None, + platform: str | None = None, +) -> Path | None: + """Get current task directory absolute path. + + Args: + repo_root: Repository root path. Defaults to auto-detected. + + Returns: + Absolute path to current task directory or None. + """ + if repo_root is None: + repo_root = get_repo_root() + + relative = get_current_task(repo_root, platform_input, platform) + if relative: + return resolve_task_ref(relative, repo_root) + return None + + +def get_current_task_source( + repo_root: Path | None = None, + platform_input: dict | None = None, + platform: str | None = None, +) -> tuple[str, str | None, str | None]: + """Get active task source as (`source`, `context_key`, `task_path`).""" + if repo_root is None: + repo_root = get_repo_root() + + from .active_task import get_current_task_source as _get_source + + return _get_source(repo_root, platform_input, platform) + + +def set_current_task( + task_path: str, + repo_root: Path | None = None, + platform_input: dict | None = None, + platform: str | None = None, +) -> bool: + """Set current task in session scope. + + Args: + task_path: Task directory path (relative to repo_root). + repo_root: Repository root path. Defaults to auto-detected. + + Returns: + True on success, False on error. + """ + if repo_root is None: + repo_root = get_repo_root() + + from .active_task import set_active_task + + return set_active_task( + task_path, + repo_root, + platform_input=platform_input, + platform=platform, + ) is not None + + +def clear_current_task( + repo_root: Path | None = None, + platform_input: dict | None = None, + platform: str | None = None, +) -> bool: + """Clear current task in session scope. + + Args: + repo_root: Repository root path. Defaults to auto-detected. + + Returns: + True on success. + """ + if repo_root is None: + repo_root = get_repo_root() + + from .active_task import clear_active_task + + clear_active_task( + repo_root, + platform_input=platform_input, + platform=platform, + ) + return True + + +def has_current_task(repo_root: Path | None = None) -> bool: + """Check if has current task. + + Args: + repo_root: Repository root path. Defaults to auto-detected. + + Returns: + True if current task is set. + """ + return get_current_task(repo_root) is not None + + +# ============================================================================= +# Task ID Generation +# ============================================================================= + +def generate_task_date_prefix() -> str: + """Generate task ID based on date (MM-DD format). + + Returns: + Date prefix string (e.g., "01-21"). + """ + return datetime.now().strftime("%m-%d") + + +# ============================================================================= +# Monorepo / Package Paths +# ============================================================================= + + +def get_spec_dir(package: str | None = None, repo_root: Path | None = None) -> Path: + """Get the spec directory path. + + Single-repo: .trellis/spec + Monorepo with package: .trellis/spec/ + + Uses lazy import to avoid circular dependency with config.py. + """ + if repo_root is None: + repo_root = get_repo_root() + + from .config import get_spec_base + + base = get_spec_base(package, repo_root) + return repo_root / DIR_WORKFLOW / base + + +def get_package_path(package: str, repo_root: Path | None = None) -> Path | None: + """Get a package's source directory absolute path from config. + + Returns: + Absolute path to the package directory, or None if not found. + """ + if repo_root is None: + repo_root = get_repo_root() + + from .config import get_packages + + packages = get_packages(repo_root) + if not packages or package not in packages: + return None + + info = packages[package] + if isinstance(info, dict): + rel_path = info.get("path", package) + else: + rel_path = str(info) + + return repo_root / rel_path + + +# ============================================================================= +# Main Entry (for testing) +# ============================================================================= + +if __name__ == "__main__": + repo = get_repo_root() + print(f"Repository root: {repo}") + print(f"Developer: {get_developer(repo)}") + print(f"Tasks dir: {get_tasks_dir(repo)}") + print(f"Workspace dir: {get_workspace_dir(repo)}") + print(f"Journal file: {get_active_journal_file(repo)}") + print(f"Current task: {get_current_task(repo)}") diff --git a/.trellis/scripts/common/safe_commit.py b/.trellis/scripts/common/safe_commit.py new file mode 100755 index 0000000..19afa4d --- /dev/null +++ b/.trellis/scripts/common/safe_commit.py @@ -0,0 +1,322 @@ +""" +Safe git-add helpers for Trellis-owned paths. + +Why this module exists +---------------------- +A real user incident: a project's `.gitignore` listed `.trellis/` (company-wide +template / personal habit). When `add_session.py` and `task.py archive` ran +their auto-commit and `git add` failed with `ignored by .gitignore`, the AI +agent driving the workflow "fixed" it by retrying with +`git add -f .trellis/` — which fan-out-included every ignored subtree +(`.trellis/.backup-*/`, `.trellis/worktrees/`, `.trellis/.template-hashes.json`, +`.trellis/.runtime/`), committing 548 files / 83474 lines of caches/backups. + +Design +------ +- Scripts only stage SPECIFIC product paths (journal files, index.md, the + current task dir, the archive dir). Never the whole `.trellis/` tree. +- If plain `git add ` fails with "ignored by", DO NOT retry with + ``-f``. The presence of `.trellis/` in `.gitignore` is treated as user + intent ("keep .trellis/ local-only"). The script warns and skips the + auto-commit; users who want auto-staging can either fix their `.gitignore` + or set ``session_auto_commit: false`` and manage git themselves. +- The warning includes a negative example: ``Do NOT use `git add -f .trellis/` ...`` + so any AI rereading the log doesn't reinvent the bug. + +History note: 0.5.10 introduced an automatic ``git add -f`` retry on the +specific paths. That was reverted in 0.5.11 — auto-forcing into a tree the +user had gitignored violates user intent even when the path list is narrow. +The wider-grain forbidden command stays forbidden, and the narrow-grain auto +``-f`` is gone too. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +from .git import run_git, run_git_retry_index_lock +from .paths import ( + DIR_ARCHIVE, + DIR_TASKS, + DIR_WORKFLOW, + DIR_WORKSPACE, + FILE_JOURNAL_PREFIX, + get_developer, +) + + +# Paths under .trellis/ that must NEVER be auto-staged. Listed here so the +# warning to the user can show concrete subpaths to ignore individually +# instead of ignoring the whole `.trellis/` tree. +TRELLIS_IGNORED_SUBPATHS = ( + ".trellis/.backup-*", + ".trellis/worktrees/", + ".trellis/.template-hashes.json", + ".trellis/.runtime/", + ".trellis/.cache/", +) + + +def safe_trellis_paths_to_add( + repo_root: Path, + task_name: str | None = None, +) -> list[str]: + """Return the list of repo-relative paths the auto-commit should stage. + + Only includes paths that exist on disk so callers don't pass non-existent + arguments to git. The caller is responsible for `git diff --cached` + checking afterwards. + + Included: + - .trellis/workspace//journal-*.md + - .trellis/workspace//index.md + - .trellis/tasks// (ONLY the current task dir when + ``task_name`` is passed; plus its archive location if the task + already lives under archive/) + + Excluded (intentionally — these must not be staged): + - .trellis/.backup-*, .trellis/worktrees/, + .trellis/.template-hashes.json, .trellis/.runtime/, .trellis/.cache/ + + Scope contract (see #303 / break-loop analysis): when ``task_name`` is + passed, the task segment stages ONLY that task directory — it never walks + ``tasks_dir.iterdir()`` over all active tasks. This mirrors + :func:`safe_archive_paths_to_add` and prevents dirty changes in OTHER + parallel-window task dirs from being bundled into the session auto-commit. + + Backwards-compat: with no ``task_name``, the function walks every active + task directory (+ the archive subtree) the old wide way. New callers + should always pass ``task_name``. + """ + paths: list[str] = [] + + # Workspace journal files + index.md + developer = get_developer(repo_root) + if developer: + ws = repo_root / DIR_WORKFLOW / DIR_WORKSPACE / developer + if ws.is_dir(): + for f in sorted(ws.glob(f"{FILE_JOURNAL_PREFIX}*.md")): + if f.is_file(): + paths.append( + f"{DIR_WORKFLOW}/{DIR_WORKSPACE}/{developer}/{f.name}" + ) + index_md = ws / "index.md" + if index_md.is_file(): + paths.append( + f"{DIR_WORKFLOW}/{DIR_WORKSPACE}/{developer}/index.md" + ) + + tasks_dir = repo_root / DIR_WORKFLOW / DIR_TASKS + if not tasks_dir.is_dir(): + return paths + + if task_name is not None: + # Narrow scope — ONLY the current task directory (active or archived). + # Never iterdir() all tasks: parallel-window dirty task dirs must not + # leak into the session auto-commit. + active_task = tasks_dir / task_name + if active_task.is_dir(): + paths.append(f"{DIR_WORKFLOW}/{DIR_TASKS}/{task_name}") + archived_task = tasks_dir / DIR_ARCHIVE / task_name + if archived_task.is_dir(): + paths.append( + f"{DIR_WORKFLOW}/{DIR_TASKS}/{DIR_ARCHIVE}/{task_name}" + ) + return paths + + # Legacy wide scope (no task_name): each direct child of tasks/ that is a + # directory and not the archive root, plus the whole archive subtree. + for child in sorted(tasks_dir.iterdir()): + if not child.is_dir(): + continue + if child.name == DIR_ARCHIVE: + continue + paths.append(f"{DIR_WORKFLOW}/{DIR_TASKS}/{child.name}") + + archive_dir = tasks_dir / DIR_ARCHIVE + if archive_dir.is_dir(): + paths.append(f"{DIR_WORKFLOW}/{DIR_TASKS}/{DIR_ARCHIVE}") + + return paths + + +def safe_archive_paths_to_add( + repo_root: Path, + task_name: str | None = None, + modified_children: list[str] | None = None, +) -> list[str]: + """Return paths to stage after `task.py archive`. + + Scoped to ONLY the paths the archive operation actually touched: + + - the archive subtree (where the freshly-moved task lives) + - the source task directory (for source-side deletes; caller pairs + this with `git rm --cached` since `git add` won't stage deletes + for a path that no longer exists in the working tree) + - any child task directories whose `task.json` was edited to drop + the archived parent (parent-children relationship update) + + This narrow scope avoids "scope creep" — dirty changes in OTHER + active task dirs (parallel-window edits) are NOT bundled into the + archive commit. Callers handle each kind of change in its own + commit boundary. + + Backwards-compat: with no arguments, the function walks the whole + `.trellis/tasks/` subtree the old way (active tasks + archive). New + callers should always pass `task_name`. + """ + paths: list[str] = [] + tasks_dir = repo_root / DIR_WORKFLOW / DIR_TASKS + if not tasks_dir.is_dir(): + return paths + + archive_dir = tasks_dir / DIR_ARCHIVE + + if task_name is not None: + # Narrow scope — only paths that still exist on disk (so + # `git add` doesn't choke on the moved-away source). The caller + # handles the source-side deletes via `git rm --cached` + # explicitly. + if archive_dir.is_dir(): + paths.append( + f"{DIR_WORKFLOW}/{DIR_TASKS}/{DIR_ARCHIVE}" + ) + for child_name in modified_children or []: + paths.append(f"{DIR_WORKFLOW}/{DIR_TASKS}/{child_name}") + return paths + + # Legacy wide scope (no task_name): preserve old behavior so callers + # that have not been updated keep working. + if archive_dir.is_dir(): + paths.append(f"{DIR_WORKFLOW}/{DIR_TASKS}/{DIR_ARCHIVE}") + for child in sorted(tasks_dir.iterdir()): + if not child.is_dir(): + continue + if child.name == DIR_ARCHIVE: + continue + paths.append(f"{DIR_WORKFLOW}/{DIR_TASKS}/{child.name}") + return paths + + +def _stderr_indicates_ignored(stderr: str) -> bool: + """git add error indicates the path is excluded by .gitignore.""" + if not stderr: + return False + lowered = stderr.lower() + return "ignored by" in lowered + + +def safe_git_add( + paths: list[str], repo_root: Path, retry_on_index_lock: bool = False +) -> tuple[bool, bool, str]: + """Run `git add` on specific paths; never retry with -f. + + Returns ``(success, used_force, stderr)``. The ``used_force`` field is + kept for signature compatibility with the 0.5.10 implementation but is + always ``False`` — we never auto-force. + + Behavior: + - No paths passed → success, no force, empty stderr. + - Plain ``git add -- `` succeeds → return success. + - Plain fails (any reason — ignored or otherwise) → return failure with + the stderr. Callers should inspect the stderr (see + :func:`print_gitignore_warning`) and skip the auto-commit. + + ``retry_on_index_lock`` opts into the bounded backoff-retry for a held + ``.git/index.lock`` (see :func:`~.git.run_git_retry_index_lock`). It is + off by default: only the archive path, which has already moved the task + directory on disk by the time it stages, needs to wait out a transient + lock rather than fail. + """ + if not paths: + return True, False, "" + + runner = run_git_retry_index_lock if retry_on_index_lock else run_git + rc, _, err = runner(["add", "--", *paths], cwd=repo_root) + if rc == 0: + return True, False, "" + return False, False, err + + +def print_gitignore_warning(paths: list[str]) -> None: + """Explain to the user (and any AI reading the log) what to do. + + CRITICAL: includes the negative example + ``Do NOT use `git add -f .trellis/``` — agents reading the warning are + known to invent that command, which fans out to ignored caches/backups. + """ + print( + "[WARN] git add failed because .trellis/ paths are ignored by your .gitignore.", + file=sys.stderr, + ) + print( + "[WARN] Skipping auto-commit. The journal/task files were still written to disk;", + file=sys.stderr, + ) + print( + "[WARN] git was not touched.", + file=sys.stderr, + ) + print("[WARN]", file=sys.stderr) + print( + "[WARN] Trellis manages these specific paths and they should be tracked:", + file=sys.stderr, + ) + if paths: + for p in paths: + print(f"[WARN] {p}", file=sys.stderr) + else: + print( + "[WARN] .trellis/workspace//{journal-*.md,index.md}", + file=sys.stderr, + ) + print( + "[WARN] .trellis/tasks//", + file=sys.stderr, + ) + print( + "[WARN] .trellis/tasks/archive/", + file=sys.stderr, + ) + print("[WARN]", file=sys.stderr) + print( + "[WARN] Recommended: change your .gitignore from `.trellis/` to specific", + file=sys.stderr, + ) + print( + "[WARN] subpaths that should remain ignored, e.g.:", + file=sys.stderr, + ) + for sub in TRELLIS_IGNORED_SUBPATHS: + print(f"[WARN] {sub}", file=sys.stderr) + print("[WARN]", file=sys.stderr) + print( + "[WARN] Or, if you intentionally keep .trellis/ local-only, set in", + file=sys.stderr, + ) + print( + "[WARN] .trellis/config.yaml:", + file=sys.stderr, + ) + print( + "[WARN] session_auto_commit: false", + file=sys.stderr, + ) + print( + "[WARN] so the scripts skip git entirely and you can review / commit", + file=sys.stderr, + ) + print( + "[WARN] manually with `git status` / `git add` / `git commit`.", + file=sys.stderr, + ) + print("[WARN]", file=sys.stderr) + print( + "[WARN] Do NOT use `git add -f .trellis/` — it pulls in backups, worktrees,", + file=sys.stderr, + ) + print( + "[WARN] and runtime caches that should never be committed.", + file=sys.stderr, + ) diff --git a/.trellis/scripts/common/session_context.py b/.trellis/scripts/common/session_context.py new file mode 100755 index 0000000..1e3fe1f --- /dev/null +++ b/.trellis/scripts/common/session_context.py @@ -0,0 +1,892 @@ +#!/usr/bin/env python3 +""" +Session context generation (default + record modes). + +Provides: + get_context_json - JSON output for default mode + get_context_text - Text output for default mode + get_context_record_json - JSON for record mode + get_context_text_record - Text for record mode + output_json - Print JSON + output_text - Print text + get_update_hint - Once-per-session "update available" line +""" + +from __future__ import annotations + +import json +import os +import re +import subprocess +import sys +from pathlib import Path + +from .active_task import resolve_context_key +from .config import get_git_packages +from .git import run_git +from .packages_context import get_packages_section +from .tasks import iter_active_tasks, load_task, get_all_statuses, children_progress +from .paths import ( + DIR_SCRIPTS, + DIR_SPEC, + DIR_TASKS, + DIR_WORKFLOW, + DIR_WORKSPACE, + count_lines, + get_active_journal_file, + get_current_task, + get_current_task_source, + get_developer, + get_repo_root, + get_tasks_dir, +) + + +# ============================================================================= +# Helpers +# ============================================================================= + +_PACKAGE_NAME = "@mindfoldhq/trellis" +_UPDATE_CHECK_TIMEOUT_SECONDS = 1.0 +_VERSION_RE = re.compile( + r"^\s*(\d+)(?:\.(\d+))?(?:\.(\d+))?(?:-([0-9A-Za-z.-]+))?\s*$" +) +_VERSION_TOKEN_RE = re.compile(r"\b\d+(?:\.\d+){1,2}(?:-[0-9A-Za-z.-]+)?\b") +_POLYREPO_IGNORED_DIRS = { + "node_modules", + "target", + "dist", + "build", + "out", + "bin", + "obj", + "vendor", + "coverage", + "tmp", + "__pycache__", +} +_POLYREPO_SCAN_MAX_DEPTH = 2 +_POLYREPO_SCAN_MAX_REPOS = 8 +_GIT_PROBE_TIMEOUT_SECONDS = 2.0 + + +def _is_git_worktree(path: Path) -> bool: + """Return True when path is inside a Git worktree.""" + rc, out, _ = run_git( + ["rev-parse", "--is-inside-work-tree"], + cwd=path, + timeout=_GIT_PROBE_TIMEOUT_SECONDS, + ) + return rc == 0 and out.strip().lower() == "true" + + +def _parse_recent_commits(log_output: str) -> list[dict]: + """Parse `git log --oneline` output into structured commit entries.""" + commits = [] + for line in log_output.splitlines(): + if not line.strip(): + continue + parts = line.split(" ", 1) + if len(parts) >= 2: + commits.append({"hash": parts[0], "message": parts[1]}) + elif len(parts) == 1: + commits.append({"hash": parts[0], "message": ""}) + return commits + + +def _collect_git_repo_info(name: str, rel_path: str, repo_dir: Path) -> dict | None: + """Collect Git status for one known repository directory.""" + if not (repo_dir / ".git").exists(): + return None + + status_rc, status_out, _ = run_git( + ["status", "--porcelain"], + cwd=repo_dir, + timeout=_GIT_PROBE_TIMEOUT_SECONDS, + ) + if status_rc != 0: + return None + changes = len([line for line in status_out.splitlines() if line.strip()]) + + _, branch_out, _ = run_git( + ["branch", "--show-current"], + cwd=repo_dir, + timeout=_GIT_PROBE_TIMEOUT_SECONDS, + ) + branch = branch_out.strip() or "unknown" + + _, log_out, _ = run_git( + ["log", "--oneline", "-5"], + cwd=repo_dir, + timeout=_GIT_PROBE_TIMEOUT_SECONDS, + ) + + return { + "name": name, + "path": rel_path, + "branch": branch, + "isClean": changes == 0, + "uncommittedChanges": changes, + "recentCommits": _parse_recent_commits(log_out), + } + + +def _collect_root_git_info(repo_root: Path) -> dict: + """Collect root Git info without pretending a non-Git root is clean.""" + if not _is_git_worktree(repo_root): + return { + "isRepo": False, + "branch": "", + "isClean": False, + "uncommittedChanges": 0, + "recentCommits": [], + } + + _, branch_out, _ = run_git( + ["branch", "--show-current"], + cwd=repo_root, + timeout=_GIT_PROBE_TIMEOUT_SECONDS, + ) + branch = branch_out.strip() or "unknown" + + status_rc, status_out, _ = run_git( + ["status", "--porcelain"], + cwd=repo_root, + timeout=_GIT_PROBE_TIMEOUT_SECONDS, + ) + status_lines = [line for line in status_out.splitlines() if line.strip()] + + _, short_out, _ = run_git( + ["status", "--short"], + cwd=repo_root, + timeout=_GIT_PROBE_TIMEOUT_SECONDS, + ) + + _, log_out, _ = run_git( + ["log", "--oneline", "-5"], + cwd=repo_root, + timeout=_GIT_PROBE_TIMEOUT_SECONDS, + ) + + return { + "isRepo": True, + "branch": branch, + "isClean": status_rc == 0 and len(status_lines) == 0, + "uncommittedChanges": len(status_lines), + "statusShort": short_out.splitlines(), + "recentCommits": _parse_recent_commits(log_out), + } + + +def _discover_child_git_repos(repo_root: Path) -> list[tuple[str, str]]: + """Discover child Git repositories using the init-time polyrepo heuristic.""" + found: list[str] = [] + overflow = False + + def is_candidate_dir(path: Path) -> bool: + name = path.name + return not name.startswith(".") and name not in _POLYREPO_IGNORED_DIRS + + def scan(rel_dir: Path, depth: int) -> None: + nonlocal overflow + if overflow: + return + if depth >= _POLYREPO_SCAN_MAX_DEPTH: + return + abs_dir = repo_root / rel_dir + try: + children = sorted(abs_dir.iterdir(), key=lambda p: p.name) + except OSError: + return + + for child in children: + if not child.is_dir() or not is_candidate_dir(child): + continue + + child_rel = ( + rel_dir / child.name if rel_dir != Path(".") else Path(child.name) + ) + if (child / ".git").exists(): + if len(found) >= _POLYREPO_SCAN_MAX_REPOS: + overflow = True + return + found.append(child_rel.as_posix()) + continue + scan(child_rel, depth + 1) + + scan(Path("."), 0) + if overflow: + print( + "warning: found more than " + f"{_POLYREPO_SCAN_MAX_REPOS} child Git repositories; " + "skipping automatic Git status collection. Configure explicit " + "packages entries with path and git: true in .trellis/config.yaml.", + file=sys.stderr, + ) + return [] + if len(found) < 2: + return [] + return [(path.replace("/", "_"), path) for path in sorted(found)] + + +def _collect_package_git_info( + repo_root: Path, + discover_unconfigured: bool = False, +) -> list[dict]: + """Collect Git status for independent package repositories. + + Packages marked with ``git: true`` in config.yaml are authoritative. + When the Trellis root is not a Git repo and no configured package repos are + available, optionally fall back to the bounded polyrepo child scan. + + Returns: + List of dicts with keys: name, path, branch, isClean, + uncommittedChanges, recentCommits. + Empty list if no git-repo packages are configured. + """ + git_pkgs = get_git_packages(repo_root) + result = [] + for pkg_name, pkg_path in git_pkgs.items(): + pkg_dir = repo_root / pkg_path + info = _collect_git_repo_info(pkg_name, pkg_path, pkg_dir) + if info is not None: + result.append(info) + + if result or not discover_unconfigured: + return result + + discovered = [] + for pkg_name, pkg_path in _discover_child_git_repos(repo_root): + info = _collect_git_repo_info(pkg_name, pkg_path, repo_root / pkg_path) + if info is not None: + discovered.append(info) + return discovered + + +def _append_root_git_context(lines: list[str], root_git_info: dict) -> None: + """Append root Git status without misleading non-Git roots.""" + lines.append("## GIT STATUS") + if not root_git_info["isRepo"]: + lines.append("Root is not a Git repository.") + lines.append("Run Git commands from the package repository paths listed below.") + else: + lines.append(f"Branch: {root_git_info['branch']}") + if root_git_info["isClean"]: + lines.append("Working directory: Clean") + else: + lines.append( + f"Working directory: {root_git_info['uncommittedChanges']} " + "uncommitted change(s)" + ) + lines.append("") + lines.append("Changes:") + for line in root_git_info.get("statusShort", [])[:10]: + lines.append(line) + lines.append("") + + lines.append("## RECENT COMMITS") + if not root_git_info["isRepo"]: + lines.append( + "Root has no Git commit history because it is not a Git repository." + ) + elif root_git_info["recentCommits"]: + for commit in root_git_info["recentCommits"]: + lines.append(f"{commit['hash']} {commit['message']}") + else: + lines.append("(no commits)") + lines.append("") + + +def _append_package_git_context(lines: list[str], package_git_info: list[dict]) -> None: + """Append Git status and recent commits for package repositories.""" + for pkg in package_git_info: + lines.append(f"## GIT STATUS ({pkg['name']}: {pkg['path']})") + lines.append(f"Branch: {pkg['branch']}") + if pkg["isClean"]: + lines.append("Working directory: Clean") + else: + lines.append( + f"Working directory: {pkg['uncommittedChanges']} uncommitted change(s)" + ) + lines.append("") + lines.append(f"## RECENT COMMITS ({pkg['name']}: {pkg['path']})") + if pkg["recentCommits"]: + for commit in pkg["recentCommits"]: + lines.append(f"{commit['hash']} {commit['message']}") + else: + lines.append("(no commits)") + lines.append("") + + +def _read_project_version(repo_root: Path) -> str | None: + try: + version = (repo_root / DIR_WORKFLOW / ".version").read_text( + encoding="utf-8" + ).strip() + except OSError: + return None + return version or None + + +def _fetch_trellis_version_output() -> str | None: + try: + result = subprocess.run( + ["trellis", "--version"], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=_UPDATE_CHECK_TIMEOUT_SECONDS, + ) + except (OSError, subprocess.SubprocessError, TimeoutError): + return None + + if result.returncode != 0: + return None + output = f"{result.stdout}\n{result.stderr}".strip() + return output or None + + +def _extract_available_update_version(output: str) -> str | None: + update_match = re.search( + r"Trellis update available:\s*" + r"(?P\S+)\s*(?:→|->)\s*(?P\S+)", + output, + ) + if update_match: + return update_match.group("latest").strip() + candidates = _VERSION_TOKEN_RE.findall(output) + return candidates[-1] if candidates else None + + +def _resolve_available_update_version() -> str | None: + output = _fetch_trellis_version_output() + if not output: + return None + return _extract_available_update_version(output) + + +def _parse_version(version: str) -> tuple[tuple[int, int, int], tuple[str, ...] | None] | None: + match = _VERSION_RE.match(version) + if not match: + return None + major, minor, patch, prerelease = match.groups() + numbers = (int(major), int(minor or "0"), int(patch or "0")) + prerelease_parts = tuple(prerelease.split(".")) if prerelease else None + return numbers, prerelease_parts + + +def _compare_prerelease( + left: tuple[str, ...] | None, + right: tuple[str, ...] | None, +) -> int: + if left is None and right is None: + return 0 + if left is None: + return 1 + if right is None: + return -1 + + for left_part, right_part in zip(left, right): + if left_part == right_part: + continue + left_numeric = left_part.isdigit() + right_numeric = right_part.isdigit() + if left_numeric and right_numeric: + left_int = int(left_part) + right_int = int(right_part) + return (left_int > right_int) - (left_int < right_int) + if left_numeric: + return -1 + if right_numeric: + return 1 + return (left_part > right_part) - (left_part < right_part) + + return (len(left) > len(right)) - (len(left) < len(right)) + + +def _compare_versions(left: str, right: str) -> int | None: + parsed_left = _parse_version(left) + parsed_right = _parse_version(right) + if parsed_left is None or parsed_right is None: + return None + + left_numbers, left_prerelease = parsed_left + right_numbers, right_prerelease = parsed_right + if left_numbers != right_numbers: + return (left_numbers > right_numbers) - (left_numbers < right_numbers) + return _compare_prerelease(left_prerelease, right_prerelease) + + +def _update_marker_path(repo_root: Path, context_key: str | None = None) -> Path: + """Path of the once-per-session marker that throttles the update check. + + `context_key` lets a caller that already resolved session identity pass it + in — the SessionStart hook reads the session id from hook stdin, which is + more reliable than this function's environment-only fallback chain. Shell + entry points leave it None and keep the previous behavior. + """ + if not context_key: + context_key = resolve_context_key() + if not context_key: + terminal_key = os.environ.get("TERM_SESSION_ID", "").strip() + context_key = terminal_key or f"ppid-{os.getppid()}" + safe_key = re.sub(r"[^A-Za-z0-9._-]+", "_", context_key).strip("._-") + if not safe_key: + safe_key = "session" + return ( + repo_root + / DIR_WORKFLOW + / ".runtime" + / f"update-check-{safe_key[:160]}.marker" + ) + + +def _mark_update_check_attempted( + repo_root: Path, + context_key: str | None = None, +) -> bool: + marker_path = _update_marker_path(repo_root, context_key) + if marker_path.exists(): + return False + try: + marker_path.parent.mkdir(parents=True, exist_ok=True) + marker_path.write_text("checked\n", encoding="utf-8") + except OSError: + pass + return True + + +def get_update_hint(repo_root: Path, context_key: str | None = None) -> str | None: + """Return the "update available" line for this session, at most once. + + Public because the SessionStart hook imports it: the text-mode CLI path + (`get_context.py`) used to be the only caller, so hook-driven platforms — + Claude Code included — never saw the reminder at all. + """ + marker_path = _update_marker_path(repo_root, context_key) + if marker_path.exists(): + return None + + current_version = _read_project_version(repo_root) + if not current_version: + return None + + latest_version = _resolve_available_update_version() + if not latest_version: + return None + + _mark_update_check_attempted(repo_root, context_key) + comparison = _compare_versions(current_version, latest_version) + if comparison is None or comparison >= 0: + return None + + return ( + f"Trellis update available: {current_version} -> {latest_version}, " + "run trellis update" + ) + + +# ============================================================================= +# JSON Output +# ============================================================================= + +def get_context_json(repo_root: Path | None = None) -> dict: + """Get context as a dictionary. + + Args: + repo_root: Repository root path. Defaults to auto-detected. + + Returns: + Context dictionary. + """ + if repo_root is None: + repo_root = get_repo_root() + + developer = get_developer(repo_root) + tasks_dir = get_tasks_dir(repo_root) + journal_file = get_active_journal_file(repo_root) + + journal_lines = 0 + journal_relative = "" + if journal_file and developer: + journal_lines = count_lines(journal_file) + journal_relative = ( + f"{DIR_WORKFLOW}/{DIR_WORKSPACE}/{developer}/{journal_file.name}" + ) + + root_git_info = _collect_root_git_info(repo_root) + + # Tasks + tasks = [ + { + "dir": t.dir_name, + "name": t.name, + "status": t.status, + "children": list(t.children), + "parent": t.parent, + } + for t in iter_active_tasks(tasks_dir) + ] + + # Package git repos (independent sub-repositories) + pkg_git_info = _collect_package_git_info( + repo_root, + discover_unconfigured=not root_git_info["isRepo"], + ) + + result = { + "developer": developer or "", + "git": { + "isRepo": root_git_info["isRepo"], + "branch": root_git_info["branch"], + "isClean": root_git_info["isClean"], + "uncommittedChanges": root_git_info["uncommittedChanges"], + "recentCommits": root_git_info["recentCommits"], + }, + "tasks": { + "active": tasks, + "directory": f"{DIR_WORKFLOW}/{DIR_TASKS}", + }, + "journal": { + "file": journal_relative, + "lines": journal_lines, + "nearLimit": journal_lines > 1800, + }, + } + + if pkg_git_info: + result["packageGit"] = pkg_git_info + + return result + + +def output_json(repo_root: Path | None = None) -> None: + """Output context in JSON format. + + Args: + repo_root: Repository root path. Defaults to auto-detected. + """ + context = get_context_json(repo_root) + print(json.dumps(context, indent=2, ensure_ascii=False)) + + +# ============================================================================= +# Text Output +# ============================================================================= + +def get_context_text(repo_root: Path | None = None) -> str: + """Get context as formatted text. + + Args: + repo_root: Repository root path. Defaults to auto-detected. + + Returns: + Formatted text output. + """ + if repo_root is None: + repo_root = get_repo_root() + + lines = [] + lines.append("========================================") + lines.append("SESSION CONTEXT") + lines.append("========================================") + lines.append("") + + developer = get_developer(repo_root) + + # Developer section + lines.append("## DEVELOPER") + if not developer: + lines.append( + f"ERROR: Not initialized. Run: python3 ./{DIR_WORKFLOW}/{DIR_SCRIPTS}/init_developer.py " + ) + return "\n".join(lines) + + lines.append(f"Name: {developer}") + lines.append("") + + root_git_info = _collect_root_git_info(repo_root) + _append_root_git_context(lines, root_git_info) + + # Package git repos — independent sub-repositories + _append_package_git_context( + lines, + _collect_package_git_info( + repo_root, + discover_unconfigured=not root_git_info["isRepo"], + ), + ) + + # Current task + lines.append("## CURRENT TASK") + current_task = get_current_task(repo_root) + if current_task: + current_task_dir = repo_root / current_task + source_type, context_key, _ = get_current_task_source(repo_root) + lines.append(f"Path: {current_task}") + lines.append( + f"Source: {source_type}" + (f":{context_key}" if context_key else "") + ) + + ct = load_task(current_task_dir) + if ct: + lines.append(f"Name: {ct.name}") + lines.append(f"Status: {ct.status}") + lines.append(f"Created: {ct.raw.get('createdAt', 'unknown')}") + if ct.description: + lines.append(f"Description: {ct.description}") + + # Check for prd.md + prd_file = current_task_dir / "prd.md" + if prd_file.is_file(): + lines.append("") + lines.append("[!] This task has prd.md - read it for task details") + else: + lines.append("(none)") + lines.append("") + + # Active tasks + lines.append("## ACTIVE TASKS") + tasks_dir = get_tasks_dir(repo_root) + task_count = 0 + + # Collect all task data for hierarchy display + all_tasks = {t.dir_name: t for t in iter_active_tasks(tasks_dir)} + all_statuses = {name: t.status for name, t in all_tasks.items()} + + def _print_task_tree(name: str, indent: int = 0) -> None: + nonlocal task_count + t = all_tasks[name] + progress = children_progress(t.children, all_statuses) + prefix = " " * indent + lines.append(f"{prefix}- {name}/ ({t.status}){progress} @{t.assignee or '-'}") + task_count += 1 + for child in t.children: + if child in all_tasks: + _print_task_tree(child, indent + 1) + + for dir_name in sorted(all_tasks.keys()): + if not all_tasks[dir_name].parent: + _print_task_tree(dir_name) + + if task_count == 0: + lines.append("(no active tasks)") + lines.append(f"Total: {task_count} active task(s)") + lines.append("") + + # My tasks + lines.append("## MY TASKS (Assigned to me)") + my_task_count = 0 + + for t in all_tasks.values(): + if t.assignee == developer and t.status != "done": + progress = children_progress(t.children, all_statuses) + lines.append(f"- [{t.priority}] {t.title} ({t.status}){progress}") + my_task_count += 1 + + if my_task_count == 0: + lines.append("(no tasks assigned to you)") + lines.append("") + + # Journal file + lines.append("## JOURNAL FILE") + journal_file = get_active_journal_file(repo_root) + if journal_file: + journal_lines = count_lines(journal_file) + relative = f"{DIR_WORKFLOW}/{DIR_WORKSPACE}/{developer}/{journal_file.name}" + lines.append(f"Active file: {relative}") + lines.append(f"Line count: {journal_lines} / 2000") + if journal_lines > 1800: + lines.append("[!] WARNING: Approaching 2000 line limit!") + else: + lines.append("No journal file found") + lines.append("") + + # Packages + packages_text = get_packages_section(repo_root) + if packages_text: + lines.append(packages_text) + lines.append("") + + # Paths + lines.append("## PATHS") + lines.append(f"Workspace: {DIR_WORKFLOW}/{DIR_WORKSPACE}/{developer}/") + lines.append(f"Tasks: {DIR_WORKFLOW}/{DIR_TASKS}/") + lines.append(f"Spec: {DIR_WORKFLOW}/{DIR_SPEC}/") + lines.append("") + + lines.append("========================================") + + return "\n".join(lines) + + +# ============================================================================= +# Record Mode +# ============================================================================= + +def get_context_record_json(repo_root: Path | None = None) -> dict: + """Get record-mode context as a dictionary. + + Focused on: my active tasks, git status, current task. + """ + if repo_root is None: + repo_root = get_repo_root() + + developer = get_developer(repo_root) + tasks_dir = get_tasks_dir(repo_root) + + root_git_info = _collect_root_git_info(repo_root) + + # My tasks (single pass — collect statuses and filter by assignee) + all_tasks_list = list(iter_active_tasks(tasks_dir)) + all_statuses = {t.dir_name: t.status for t in all_tasks_list} + + my_tasks = [] + for t in all_tasks_list: + if t.assignee == developer: + done = sum( + 1 for c in t.children + if all_statuses.get(c) in ("completed", "done") + ) + my_tasks.append({ + "dir": t.dir_name, + "title": t.title, + "status": t.status, + "priority": t.priority, + "children": list(t.children), + "childrenDone": done, + "parent": t.parent, + "meta": t.meta, + }) + + # Current task + current_task_info = None + current_task = get_current_task(repo_root) + if current_task: + source_type, context_key, _ = get_current_task_source(repo_root) + ct = load_task(repo_root / current_task) + if ct: + current_task_info = { + "path": current_task, + "name": ct.name, + "status": ct.status, + "source": source_type, + "contextKey": context_key, + } + + # Package git repos + pkg_git_info = _collect_package_git_info( + repo_root, + discover_unconfigured=not root_git_info["isRepo"], + ) + + result = { + "developer": developer or "", + "git": { + "isRepo": root_git_info["isRepo"], + "branch": root_git_info["branch"], + "isClean": root_git_info["isClean"], + "uncommittedChanges": root_git_info["uncommittedChanges"], + "recentCommits": root_git_info["recentCommits"], + }, + "myTasks": my_tasks, + "currentTask": current_task_info, + } + + if pkg_git_info: + result["packageGit"] = pkg_git_info + + return result + + +def get_context_text_record(repo_root: Path | None = None) -> str: + """Get context as formatted text for record-session mode. + + Focused output: MY ACTIVE TASKS first (with [!!!] emphasis), + then GIT STATUS, RECENT COMMITS, CURRENT TASK. + """ + if repo_root is None: + repo_root = get_repo_root() + + lines: list[str] = [] + lines.append("========================================") + lines.append("SESSION CONTEXT (RECORD MODE)") + lines.append("========================================") + lines.append("") + + developer = get_developer(repo_root) + if not developer: + lines.append( + f"ERROR: Not initialized. Run: python3 ./{DIR_WORKFLOW}/{DIR_SCRIPTS}/init_developer.py " + ) + return "\n".join(lines) + + # MY ACTIVE TASKS — first and prominent + lines.append(f"## [!!!] MY ACTIVE TASKS (Assigned to {developer})") + lines.append("[!] Review whether any should be archived before recording this session.") + lines.append("") + + tasks_dir = get_tasks_dir(repo_root) + my_task_count = 0 + + # Single pass — collect all tasks and filter by assignee + all_statuses = get_all_statuses(tasks_dir) + + for t in iter_active_tasks(tasks_dir): + if t.assignee == developer: + progress = children_progress(t.children, all_statuses) + lines.append(f"- [{t.priority}] {t.title} ({t.status}){progress} — {t.dir_name}") + my_task_count += 1 + + if my_task_count == 0: + lines.append("(no active tasks assigned to you)") + lines.append("") + + root_git_info = _collect_root_git_info(repo_root) + _append_root_git_context(lines, root_git_info) + + # Package git repos — independent sub-repositories + _append_package_git_context( + lines, + _collect_package_git_info( + repo_root, + discover_unconfigured=not root_git_info["isRepo"], + ), + ) + + # CURRENT TASK + lines.append("## CURRENT TASK") + current_task = get_current_task(repo_root) + if current_task: + source_type, context_key, _ = get_current_task_source(repo_root) + lines.append(f"Path: {current_task}") + lines.append( + f"Source: {source_type}" + (f":{context_key}" if context_key else "") + ) + ct = load_task(repo_root / current_task) + if ct: + lines.append(f"Name: {ct.name}") + lines.append(f"Status: {ct.status}") + else: + lines.append("(none)") + lines.append("") + + lines.append("========================================") + + return "\n".join(lines) + + +def output_text(repo_root: Path | None = None) -> None: + """Output context in text format. + + Args: + repo_root: Repository root path. Defaults to auto-detected. + """ + if repo_root is None: + repo_root = get_repo_root() + update_hint = get_update_hint(repo_root) + if update_hint: + print(update_hint) + print("") + print(get_context_text(repo_root)) diff --git a/.trellis/scripts/common/task_context.py b/.trellis/scripts/common/task_context.py new file mode 100755 index 0000000..5465b22 --- /dev/null +++ b/.trellis/scripts/common/task_context.py @@ -0,0 +1,470 @@ +#!/usr/bin/env python3 +""" +Task JSONL context management. + +Provides: + cmd_add_context - Add entry to JSONL context file + cmd_validate - Validate JSONL context files + cmd_list_context - List JSONL context entries + +Note: + ``cmd_init_context`` was removed in v0.5.0-beta.12. JSONL context files + are created empty at ``task.py create`` time; the AI agent curates real + entries during planning when the task needs sub-agent/spec context. See + ``.trellis/workflow.md`` for the current planning artifact contract. + + Older Trellis versions seeded those files with a ``{"_example": ...}`` + placeholder row. ``cmd_validate`` now rejects that row so a task cannot + validate locally and then fail PR preflight, which treats it as + unresolved scaffolding. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from .config import get_context_injection_limits +from .git import branch_exists_locally +from .io import read_json +from .log import Colors, colored +from .paths import DIR_ARCHIVE, DIR_TASKS, DIR_WORKFLOW, FILE_TASK_JSON, get_repo_root +from .task_utils import resolve_task_dir + +# Extensions that look like code rather than spec/research docs. Entries with +# one of these extensions outside .trellis/spec/, docs/docs-site, or the +# task's own directory get a hygiene warning in `task.py validate` — the +# reader is a sub-agent, not a human, so code paths belong in the diff the +# agent reads itself, not in implement.jsonl / check.jsonl. +_CODE_FILE_EXTENSIONS = { + ".ts", + ".tsx", + ".js", + ".jsx", + ".mjs", + ".cjs", + ".py", + ".go", + ".rs", + ".java", + ".rb", + ".c", + ".cc", + ".cpp", + ".h", +} + + +# ============================================================================= +# Command: add-context +# ============================================================================= + +def cmd_add_context(args: argparse.Namespace) -> int: + """Add entry to JSONL context file.""" + repo_root = get_repo_root() + target_dir = resolve_task_dir(args.dir, repo_root) + if target_dir is None: + return 1 + + jsonl_name = args.file + path = args.path + reason = args.reason or "Added manually" + + if not target_dir or not target_dir.is_dir(): + print(colored(f"Error: Directory not found: {target_dir}", Colors.RED)) + return 1 + + # The JSONL name is user input joined onto the task dir — keep it a plain + # filename so it cannot create files elsewhere. + if "/" in jsonl_name or "\\" in jsonl_name or jsonl_name in (".", ".."): + print(colored( + f"Error: context file must be a plain name (e.g. implement, check): {jsonl_name}", + Colors.RED, + )) + return 1 + + # Support shorthand + if not jsonl_name.endswith(".jsonl"): + jsonl_name = f"{jsonl_name}.jsonl" + + jsonl_file = target_dir / jsonl_name + full_path = repo_root / path + + entry_type = "file" + if full_path.is_dir(): + entry_type = "directory" + if not path.endswith("/"): + path = f"{path}/" + elif not full_path.is_file(): + print(colored(f"Error: Path not found: {path}", Colors.RED)) + return 1 + + # Check if already exists + if jsonl_file.is_file(): + content = jsonl_file.read_text(encoding="utf-8") + if f'"{path}"' in content: + print(colored(f"Warning: Entry already exists for {path}", Colors.YELLOW)) + return 0 + + # Add entry + entry: dict + if entry_type == "directory": + entry = {"file": path, "type": "directory", "reason": reason} + else: + entry = {"file": path, "reason": reason} + + with jsonl_file.open("a", encoding="utf-8") as f: + f.write(json.dumps(entry, ensure_ascii=False) + "\n") + + print(colored(f"Added {entry_type}: {path}", Colors.GREEN)) + return 0 + + +# ============================================================================= +# Command: validate +# ============================================================================= + +def curated_entry_count(jsonl_file: Path) -> int | None: + """Count curated entries in a jsonl context manifest. + + Returns None when the file does not exist — `task.py create` seeds the + manifests only on sub-agent-capable platforms, so an absent file means no + sub-agent will ever read it and callers should not gate on it. A curated + entry is a JSON object row carrying a truthy ``file`` (or legacy ``path``) + value: the same rows the sub-agent injection hook materializes. + """ + if not jsonl_file.is_file(): + return None + try: + lines = jsonl_file.read_text(encoding="utf-8").splitlines() + except OSError: + return 0 + count = 0 + for line in lines: + line = line.strip() + if not line: + continue + try: + data = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(data, dict) and (data.get("file") or data.get("path")): + count += 1 + return count + + +def cmd_validate(args: argparse.Namespace) -> int: + """Validate JSONL context files.""" + repo_root = get_repo_root() + target_dir = resolve_task_dir(args.dir, repo_root) + + if target_dir is None or not target_dir.is_dir(): + print(colored("Error: task directory required", Colors.RED)) + return 1 + + print(colored("=== Validating Context Files ===", Colors.BLUE)) + print(f"Target dir: {target_dir}") + print() + + # Warn (don't fail validation) when the recorded branch is stale — it + # was likely already merged and deleted (#399 item 2). + task_json_path = target_dir / FILE_TASK_JSON + if task_json_path.is_file(): + task_data = read_json(task_json_path) + stored_branch = task_data.get("branch") if task_data else None + if stored_branch and not branch_exists_locally(stored_branch, repo_root): + print( + colored( + f"Warning: recorded branch '{stored_branch}' no longer exists locally " + "(likely merged and deleted).", + Colors.YELLOW, + ) + ) + print() + + total_errors = 0 + for jsonl_name in ["implement.jsonl", "check.jsonl"]: + jsonl_file = target_dir / jsonl_name + errors = _validate_jsonl(jsonl_file, repo_root, target_dir) + total_errors += errors + + print() + if total_errors == 0: + print(colored("✓ All validations passed", Colors.GREEN)) + return 0 + else: + print(colored(f"✗ Validation failed ({total_errors} errors)", Colors.RED)) + return 1 + + +def _is_exempt_from_code_file_warning(file_path: str, task_rel: str) -> bool: + """Whether a jsonl entry path is exempt from the code-file hygiene warning. + + Exempt: spec docs (``.trellis/spec/``), documentation (``docs``, + ``docs-site``), and the task's own directory (execution plans, generated + artifacts, etc. legitimately live there). + """ + posix_path = file_path.replace("\\", "/").lstrip("/") + exempt_prefixes = (".trellis/spec/", "docs/", "docs-site/") + if posix_path.startswith(exempt_prefixes): + return True + if task_rel and (posix_path == task_rel or posix_path.startswith(f"{task_rel}/")): + return True + return False + + +def _resolve_context_entry_path( + file_path: str, repo_root: Path, task_dir: Path | None +) -> Path | None: + """Resolve a JSONL entry, binding archived self-references to the archive copy. + + Exact historical self-references are remapped only for archived tasks. + ``None`` means the remapped path traversed or resolved outside that archive. + """ + repo_path = repo_root / file_path + if task_dir is None: + return repo_path + + try: + task_parts = task_dir.resolve().relative_to(repo_root.resolve()).parts + except ValueError: + return repo_path + + archive_prefix = (DIR_WORKFLOW, DIR_TASKS, DIR_ARCHIVE) + if len(task_parts) != 5 or task_parts[:3] != archive_prefix: + return repo_path + + year_month = task_parts[3] + if ( + len(year_month) != 7 + or year_month[4] != "-" + or not year_month[:4].isdigit() + or not year_month[5:].isdigit() + ): + return repo_path + + historical_root = f"{DIR_WORKFLOW}/{DIR_TASKS}/{task_dir.name}" + posix_path = file_path.replace("\\", "/") + if posix_path == historical_root: + relative_parts: tuple[str, ...] = () + elif posix_path.startswith(f"{historical_root}/"): + relative_path = posix_path[len(historical_root) + 1 :] + if relative_path.endswith("/"): + relative_path = relative_path[:-1] + relative_parts = tuple(relative_path.split("/")) if relative_path else () + if any(part in ("", ".", "..") for part in relative_parts): + return None + else: + return repo_path + + try: + archive_root = task_dir.resolve() + resolved_path = task_dir.joinpath(*relative_parts).resolve() + resolved_path.relative_to(archive_root) + except (OSError, RuntimeError, ValueError): + return None + return resolved_path + + +def _validate_jsonl(jsonl_file: Path, repo_root: Path, task_dir: Path | None = None) -> int: + """Validate a single JSONL file. + + ``{"_example": ...}`` placeholder rows written by older Trellis versions + are hard errors: PR preflight rejects them as unresolved scaffolding, so + accepting them here would pass locally and fail later. Other rows without + a ``file`` field are skipped silently, matching what consumers do. + + Beyond hard errors (missing file/dir, invalid JSON), this also prints + non-blocking hygiene warnings (never counted in ``errors``, never change + the exit code): entries that look like code files rather than + spec/research docs, and entries whose file size exceeds the configured + sub-agent context injection cap (``context_injection.max_file_bytes``). + """ + file_name = jsonl_file.name + errors = 0 + + if not jsonl_file.is_file(): + print(f" {colored(f'{file_name}: not found (skipped)', Colors.YELLOW)}") + return 0 + + task_rel = "" + if task_dir is not None: + try: + task_rel = task_dir.resolve().relative_to(repo_root.resolve()).as_posix() + except ValueError: + task_rel = "" + + max_file_bytes = get_context_injection_limits(repo_root).get("max_file_bytes", 0) + + line_num = 0 + real_entries = 0 + for line in jsonl_file.read_text(encoding="utf-8").splitlines(): + line_num += 1 + if not line.strip(): + continue + + try: + data = json.loads(line) + except json.JSONDecodeError: + print(f" {colored(f'{file_name}:{line_num}: Invalid JSON', Colors.RED)}") + errors += 1 + continue + + if not isinstance(data, dict): + print( + f" {colored(f'{file_name}:{line_num}: Expected a JSON object', Colors.RED)}" + ) + errors += 1 + continue + + if "_example" in data: + error_message = ( + f"{file_name}:{line_num}: Placeholder `_example` row left by an older " + "task.py create — delete this line, or replace it with " + '{"file": "", "reason": ""}' + ) + print(f" {colored(error_message, Colors.RED)}") + errors += 1 + continue + + file_path = data.get("file") + entry_type = data.get("type", "file") + + if not file_path: + # Comment / unknown row without a path — skip silently + continue + + if not isinstance(file_path, str): + # A truthy non-string (e.g. {"file": 1}) reached path joining and + # raised TypeError, so validation crashed on the row it exists to + # report. + print( + f" {colored(f'{file_name}:{line_num}: `file` must be a string path', Colors.RED)}" + ) + errors += 1 + continue + + real_entries += 1 + full_path = _resolve_context_entry_path(file_path, repo_root, task_dir) + if entry_type == "directory": + if full_path is None or not full_path.is_dir(): + print(f" {colored(f'{file_name}:{line_num}: Directory not found: {file_path}', Colors.RED)}") + errors += 1 + continue + + if full_path is None or not full_path.is_file(): + print(f" {colored(f'{file_name}:{line_num}: File not found: {file_path}', Colors.RED)}") + errors += 1 + continue + + extension = Path(file_path).suffix.lower() + if extension in _CODE_FILE_EXTENSIONS and not _is_exempt_from_code_file_warning( + file_path, task_rel + ): + warning_message = ( + f"{file_name}:{line_num}: Warning: {file_path} looks like a code file — " + "implement/check.jsonl should reference spec/research docs; " + "agents read code themselves" + ) + print(f" {colored(warning_message, Colors.YELLOW)}") + + if max_file_bytes: + # Advisory hygiene warning, so it must never be what fails + # `validate`. `stat()` can still raise after the `is_file()` check + # above — a permission change, or the file removed in between. + try: + size: int | None = full_path.stat().st_size + except OSError: + size = None + if size is not None and size > max_file_bytes: + warning_message = ( + f"{file_name}:{line_num}: Warning: {file_path} is {size} bytes, " + f"exceeds context_injection.max_file_bytes ({max_file_bytes}); " + "injection will truncate it" + ) + print(f" {colored(warning_message, Colors.YELLOW)}") + + if errors == 0 and real_entries == 0: + # Seed-only / empty manifest: sub-agents dispatched for this task + # would run with zero spec context (#573). Silent-green here is how + # more than half the tasks in the report ended up uncurated. + action = file_name.split(".", 1)[0] + print( + f" {colored(f'{file_name}: ✗ (0 curated entries — sub-agents would get zero spec context)', Colors.RED)}" + ) + print( + f" Curate it: python3 .trellis/scripts/task.py add-context {action} \"\"" + ) + print( + " Intentionally empty? Bypass at start: task.py start --allow-empty-context" + ) + return 1 + + if errors == 0: + print(f" {colored(f'{file_name}: ✓ ({real_entries} entries)', Colors.GREEN)}") + else: + print(f" {colored(f'{file_name}: ✗ ({errors} errors)', Colors.RED)}") + + return errors + + +# ============================================================================= +# Command: list-context +# ============================================================================= + +def cmd_list_context(args: argparse.Namespace) -> int: + """List JSONL context entries.""" + repo_root = get_repo_root() + target_dir = resolve_task_dir(args.dir, repo_root) + + if target_dir is None or not target_dir.is_dir(): + print(colored("Error: task directory required", Colors.RED)) + return 1 + + print(colored("=== Context Files ===", Colors.BLUE)) + print() + + for jsonl_name in ["implement.jsonl", "check.jsonl"]: + jsonl_file = target_dir / jsonl_name + if not jsonl_file.is_file(): + continue + + print(colored(f"[{jsonl_name}]", Colors.CYAN)) + + count = 0 + curated = False + for line in jsonl_file.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + + try: + data = json.loads(line) + except json.JSONDecodeError: + continue + + if not isinstance(data, dict): + continue + + file_path = data.get("file") + if not file_path: + # Placeholder / comment row — don't count as a real entry + continue + curated = True + + count += 1 + entry_type = data.get("type", "file") + reason = data.get("reason", "-") + + if entry_type == "directory": + print(f" {colored(f'{count}.', Colors.GREEN)} [DIR] {file_path}") + else: + print(f" {colored(f'{count}.', Colors.GREEN)} {file_path}") + print(f" {colored('→', Colors.YELLOW)} {reason}") + + if not curated: + print(f" {colored('(no curated entries yet)', Colors.YELLOW)}") + + print() + + return 0 diff --git a/.trellis/scripts/common/task_queue.py b/.trellis/scripts/common/task_queue.py new file mode 100755 index 0000000..f7485e2 --- /dev/null +++ b/.trellis/scripts/common/task_queue.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +""" +Task queue utility functions. + +Provides: + list_tasks_by_status - List tasks by status + list_pending_tasks - List tasks with pending status + list_tasks_by_assignee - List tasks by assignee + list_my_tasks - List tasks assigned to current developer + get_task_stats - Get P0/P1/P2/P3 counts +""" + +from __future__ import annotations + +from pathlib import Path + +from .paths import ( + get_repo_root, + get_developer, + get_tasks_dir, +) +from .tasks import iter_active_tasks + + +# ============================================================================= +# Internal helper +# ============================================================================= + +def _task_to_dict(t) -> dict: + """Convert TaskInfo to the dict format callers expect.""" + return { + "priority": t.priority, + "id": t.raw.get("id", ""), + "title": t.title, + "status": t.status, + "assignee": t.assignee or "-", + "dir": t.dir_name, + "children": list(t.children), + "parent": t.parent, + } + + +# ============================================================================= +# Public Functions +# ============================================================================= + +def list_tasks_by_status( + filter_status: str | None = None, + repo_root: Path | None = None +) -> list[dict]: + """List tasks by status. + + Args: + filter_status: Optional status filter. + repo_root: Repository root path. Defaults to auto-detected. + + Returns: + List of task info dicts with keys: priority, id, title, status, assignee. + """ + if repo_root is None: + repo_root = get_repo_root() + + tasks_dir = get_tasks_dir(repo_root) + results = [] + + for t in iter_active_tasks(tasks_dir): + if filter_status and t.status != filter_status: + continue + results.append(_task_to_dict(t)) + + return results + + +def list_pending_tasks(repo_root: Path | None = None) -> list[dict]: + """List pending tasks. + + Args: + repo_root: Repository root path. Defaults to auto-detected. + + Returns: + List of task info dicts. + """ + return list_tasks_by_status("planning", repo_root) + + +def list_tasks_by_assignee( + assignee: str, + filter_status: str | None = None, + repo_root: Path | None = None +) -> list[dict]: + """List tasks assigned to a specific developer. + + Args: + assignee: Developer name. + filter_status: Optional status filter. + repo_root: Repository root path. Defaults to auto-detected. + + Returns: + List of task info dicts. + """ + if repo_root is None: + repo_root = get_repo_root() + + tasks_dir = get_tasks_dir(repo_root) + results = [] + + for t in iter_active_tasks(tasks_dir): + if (t.assignee or "-") != assignee: + continue + if filter_status and t.status != filter_status: + continue + results.append(_task_to_dict(t)) + + return results + + +def list_my_tasks( + filter_status: str | None = None, + repo_root: Path | None = None +) -> list[dict]: + """List tasks assigned to current developer. + + Args: + filter_status: Optional status filter. + repo_root: Repository root path. Defaults to auto-detected. + + Returns: + List of task info dicts. + + Raises: + ValueError: If developer not set. + """ + if repo_root is None: + repo_root = get_repo_root() + + developer = get_developer(repo_root) + if not developer: + raise ValueError("Developer not set") + + return list_tasks_by_assignee(developer, filter_status, repo_root) + + +def get_task_stats(repo_root: Path | None = None) -> dict[str, int]: + """Get task statistics. + + Args: + repo_root: Repository root path. Defaults to auto-detected. + + Returns: + Dict with keys: P0, P1, P2, P3, Total. + """ + if repo_root is None: + repo_root = get_repo_root() + + tasks_dir = get_tasks_dir(repo_root) + stats = {"P0": 0, "P1": 0, "P2": 0, "P3": 0, "Total": 0} + + for t in iter_active_tasks(tasks_dir): + if t.priority in stats: + stats[t.priority] += 1 + stats["Total"] += 1 + + return stats + + +def format_task_stats(stats: dict[str, int]) -> str: + """Format task stats as string. + + Args: + stats: Stats dict from get_task_stats. + + Returns: + Formatted string like "P0:0 P1:1 P2:2 P3:0 Total:3". + """ + return f"P0:{stats['P0']} P1:{stats['P1']} P2:{stats['P2']} P3:{stats['P3']} Total:{stats['Total']}" + + +# ============================================================================= +# Main Entry (for testing) +# ============================================================================= + +if __name__ == "__main__": + stats = get_task_stats() + print(format_task_stats(stats)) + print() + print("Pending tasks:") + for task in list_pending_tasks(): + print(f" {task['priority']}|{task['id']}|{task['title']}|{task['status']}|{task['assignee']}") diff --git a/.trellis/scripts/common/task_store.py b/.trellis/scripts/common/task_store.py new file mode 100755 index 0000000..6ded01c --- /dev/null +++ b/.trellis/scripts/common/task_store.py @@ -0,0 +1,1874 @@ +#!/usr/bin/env python3 +""" +Task CRUD operations. + +Provides: + ensure_tasks_dir - Ensure tasks directory exists + cmd_create - Create a new task + cmd_rename - Rename a task and every reference to it + cmd_archive - Archive completed task + cmd_set_branch - Set git branch for task + cmd_set_base_branch - Set PR target branch + cmd_set_scope - Set scope for PR title + cmd_set_meta - Set/overwrite a task metadata key + cmd_add_subtask - Link child task to parent + cmd_remove_subtask - Unlink child task from parent +""" + +from __future__ import annotations + +import argparse +import re +import sys +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path + +from .config import ( + get_codex_dispatch_mode, + get_packages, + get_session_auto_commit, + is_monorepo, + resolve_package, + validate_package, +) +from .git import ( + INDEX_LOCK_RETRY_ATTEMPTS, + branch_exists_locally, + has_git_remote, + index_lock_path, + resolve_default_branch, + run_git, + run_git_retry_index_lock, + stderr_indicates_index_lock, +) +from .io import describe_json_read_failure, read_json_checked, write_json +from .log import Colors, colored +from .paths import ( + DEVELOPER_HINT, + DIR_ARCHIVE, + DIR_TASKS, + DIR_WORKFLOW, + FILE_TASK_JSON, + generate_task_date_prefix, + get_developer, + get_repo_root, + get_tasks_dir, +) +from .safe_commit import ( + print_gitignore_warning, + safe_archive_paths_to_add, + safe_git_add, +) +from .task_utils import ( + archive_destination_for, + archive_task_complete, + find_task_by_name, + is_within_tasks_dir, + resolve_task_dir, + run_task_hooks, +) + + +# ============================================================================= +# Helper Functions +# ============================================================================= + +# Characters stripped before deciding whether a title / description is empty. +# This is deliberately the *union* of Python's str.strip() default set and +# ECMAScript's String.trim() set, because the two disagree: Python strips +# U+0085 (NEL) and U+001C-U+001F, which JS keeps; JS strips U+FEFF (BOM), +# which Python keeps. Stripping the union means anything create accepts is +# still non-empty after either side trims it, so a record cannot pass here and +# then be refused as empty by a JS pre-archive validator. +BLANK_CHARS = ( + "\t\n\v\f\r" # U+0009-U+000D + "\x1c\x1d\x1e\x1f" # U+001C-U+001F separators (Python only) + " " # U+0020 space + "\x85" # U+0085 next line (Python only) + "\xa0" # U+00A0 no-break space + "\u1680" # ogham space mark + "\u2000\u2001\u2002\u2003\u2004\u2005" # en/em quad, en/em/three/four-per-em + "\u2006\u2007\u2008\u2009\u200a" # six-per-em .. hair space + "\u2028\u2029" # line / paragraph separator + "\u202f\u205f\u3000" # narrow nbsp, math space, ideographic space + "\ufeff" # zero-width no-break space / BOM (JS only) +) + + +def strip_blank(value: str | None) -> str: + """Return ``value`` with :data:`BLANK_CHARS` trimmed from both ends.""" + return (value or "").strip(BLANK_CHARS) + + +def _slugify(title: str) -> str: + """Convert title to slug (only works with ASCII).""" + result = title.lower() + result = re.sub(r"[^a-z0-9]", "-", result) + result = re.sub(r"-+", "-", result) + result = result.strip("-") + return result + + +def ensure_tasks_dir(repo_root: Path) -> Path: + """Ensure tasks directory exists.""" + tasks_dir = get_tasks_dir(repo_root) + archive_dir = tasks_dir / "archive" + + if not tasks_dir.exists(): + tasks_dir.mkdir(parents=True) + print(colored(f"Created tasks directory: {tasks_dir}", Colors.GREEN), file=sys.stderr) + + if not archive_dir.exists(): + archive_dir.mkdir(parents=True) + + return tasks_dir + + +def _find_archived_task_by_dir_name(tasks_dir: Path, dir_name: str) -> Path | None: + """Find an archived task directory with the exact active-task dir name.""" + archive_dir = tasks_dir / DIR_ARCHIVE + if not archive_dir.is_dir(): + return None + + for month_dir in sorted(archive_dir.iterdir()): + if not month_dir.is_dir(): + continue + candidate = month_dir / dir_name + if candidate.is_dir(): + return candidate + + return None + + +def _repo_relative_path(path: Path, repo_root: Path) -> str: + """Format a path relative to the repo root when possible.""" + try: + return path.relative_to(repo_root).as_posix() + except ValueError: + return str(path) + + +def _report_read_failure(path: Path, reason: str | None) -> None: + """Print why a JSON file could not be loaded, and what to do about it. + + These commands are about to rewrite the file they just failed to read, so + this message is the last warning the user gets. Exiting non-zero with no + output at all leaves a parse error, a permissions error and an empty file + indistinguishable. + """ + problem, hint = describe_json_read_failure(path, reason) + print(colored(f"Error: {problem}", Colors.RED), file=sys.stderr) + print(hint, file=sys.stderr) + + +def _ensure_children_list(data: dict) -> list: + """The task's `children` as a list, repaired in place if it is not one. + + `.get(key, default)` returns the default only when the key is *absent*. A + task.json carrying `"children": null` — older format, or hand-edited — + yields None, and every caller below goes on to use the result as a list. + In `cmd_create` that raise lands *after* the new task.json is written, + leaving a task on disk its parent does not reference. + + The repair is written into `data` rather than only returned, because the + unlink path removes a name it may not find and would otherwise persist the + malformed value untouched — no crash, but the next caller inherits it. + + Anything that is not a list is discarded rather than coerced: a string + would iterate per character and a dict per key, each producing a plausible + child set that is not one. + """ + children = data.get("children") + if not isinstance(children, list): + children = [] + data["children"] = children + return children + + +def _report_write_failure(path: Path) -> None: + """Print that a JSON write failed. Writes are atomic, so nothing changed.""" + print(colored(f"Error: Failed to write {path}", Colors.RED), file=sys.stderr) + print( + "The existing file is unchanged (writes are atomic). " + "Check permissions and free disk space, then retry.", + file=sys.stderr, + ) + + +# ============================================================================= +# Sub-agent platform detection + JSONL context files +# ============================================================================= + +# Config directories of platforms that consume implement.jsonl / check.jsonl. +# Keep in sync with src/types/ai-tools.ts AI_TOOLS entries — these are the +# platforms listed in workflow.md's "agent-capable" Skill Routing block. +# Codex is checked separately because explicit inline mode does not consume +# JSONL. Kilo / Antigravity / Devin are NOT in this list either: they load +# specs through skills instead of JSONL. +_SUBAGENT_CONFIG_DIRS: tuple[str, ...] = ( + ".claude", + ".cursor", + ".kiro", + ".gemini", + ".opencode", + ".qoder", + ".codebuddy", + ".factory", # Factory Droid + ".github/copilot", + ".pi", # Pi Agent + ".trae", # Trae IDE + ".omp", # Oh My Pi + ".zcode", # ZCode + ".grok", # Grok Build + ".kimi-code", # Kimi Code +) +_CODEX_CONFIG_DIR = ".codex" + + +def _has_subagent_platform(repo_root: Path) -> bool: + """Return True if any sub-agent-capable platform is configured. + + Detected by probing well-known config directories at the repo root. Codex + counts by default through ``codex.dispatch_mode: auto`` (including the + legacy ``sub-agent`` alias); explicit inline mode loads context through + skills, not JSONL. + """ + for config_dir in _SUBAGENT_CONFIG_DIRS: + if (repo_root / config_dir).is_dir(): + return True + if (repo_root / _CODEX_CONFIG_DIR).is_dir(): + return get_codex_dispatch_mode(repo_root) == "auto" + return False + + +def _parse_meta_pairs(pairs: list[str] | None) -> dict[str, str] | None: + """Parse repeatable ``--meta key=value`` pairs into a dict. + + Returns ``None`` (after printing an error naming the bad value) on the + first malformed pair: missing ``=`` or an empty key. Values are stored + as-is (strings, no nesting, no type coercion). + """ + meta: dict[str, str] = {} + for pair in pairs or []: + key, sep, value = pair.partition("=") + if not sep or not key: + print( + colored(f"Error: malformed --meta value '{pair}' (expected key=value)", Colors.RED), + file=sys.stderr, + ) + return None + meta[key] = value + return meta + + +def _default_prd_content(title: str, description: str | None = None) -> str: + """Return the default PRD skeleton created with every task.""" + goal = (description or "").strip() or "TBD." + heading = title.strip() or "Untitled task" + return f"""# {heading} + +## Goal + +{goal} + +## Requirements + +- TBD + +## Acceptance Criteria + +- [ ] TBD + +## Notes + +- Keep `prd.md` focused on requirements, constraints, and acceptance criteria. +- Lightweight tasks can remain PRD-only. +- For complex tasks, add `design.md` for technical design and `implement.md` for execution planning before `task.py start`. +""" + + +# ============================================================================= +# Command: create +# ============================================================================= + +def cmd_create(args: argparse.Namespace) -> int: + """Create a new task.""" + repo_root = get_repo_root() + + # Title and description are checked first, before ensure_tasks_dir and any + # other write, so a rejected create leaves the tree untouched. Both are + # required non-empty by pre-archive validation; catching that here costs + # one retyped command instead of blocking a PR hours later. + if not strip_blank(args.title): + print(colored("Error: title is required", Colors.RED), file=sys.stderr) + print( + 'Pass a non-empty (whitespace only does not count): a task ' + "with an empty title is refused at archive.", + file=sys.stderr, + ) + return 1 + + description = strip_blank(getattr(args, "description", None)) + if not description: + print(colored("Error: --description is required", Colors.RED), file=sys.stderr) + print( + 'Pass --description "<what this task delivers>" (whitespace only does ' + "not count): a task with an empty description is refused at archive.", + file=sys.stderr, + ) + return 1 + + # Validate --meta (CLI source: fail-fast, before any directory is created) + meta = _parse_meta_pairs(getattr(args, "meta", None)) + if meta is None: + return 1 + + # Validate --package (CLI source: fail-fast) + package: str | None = getattr(args, "package", None) + if not is_monorepo(repo_root): + # Single-repo: ignore --package, no package prefix + if package: + print(colored(f"Warning: --package ignored in single-repo project", Colors.YELLOW), file=sys.stderr) + package = None + elif package: + if not validate_package(package, repo_root): + packages = get_packages(repo_root) + available = ", ".join(sorted(packages.keys())) if packages else "(none)" + print(colored(f"Error: unknown package '{package}'. Available: {available}", Colors.RED), file=sys.stderr) + return 1 + else: + # Inferred: default_package → None (no task.json yet for create) + package = resolve_package(repo_root=repo_root) + + # Default assignee to current developer + assignee = args.assignee + if not assignee: + assignee = get_developer(repo_root) + if not assignee: + print(colored("Error: No developer set. Run init_developer.py first or use --assignee", Colors.RED), file=sys.stderr) + print(DEVELOPER_HINT, file=sys.stderr) + return 1 + + ensure_tasks_dir(repo_root) + + # Get current developer as creator + creator = get_developer(repo_root) or assignee + + # Generate slug if not provided. A title-derived slug is sanitized by + # _slugify; an explicit --slug is not, so reject the characters that would + # let it escape the tasks directory once joined into the dir name. + slug = args.slug or _slugify(args.title) + if not slug: + print(colored("Error: could not generate slug from title", Colors.RED), file=sys.stderr) + return 1 + + if args.slug and ("/" in slug or "\\" in slug or ".." in slug): + print( + colored( + f"Error: --slug must be a plain name without path separators or '..': {slug}", + Colors.RED, + ), + file=sys.stderr, + ) + return 1 + + # Create task directory with MM-DD-slug format + tasks_dir = get_tasks_dir(repo_root) + date_prefix = generate_task_date_prefix() + + # Guard against date-prefixed --slug (e.g. a full task dir name pasted in), + # which would otherwise produce MM-DD-MM-DD-slug (issue #377). Only an + # explicit --slug is guarded; title-derived slugs are left untouched. + if args.slug: + m = re.match(r"^(\d{2})-(\d{2})-(.+)$", slug) + if m and 1 <= int(m.group(1)) <= 12 and 1 <= int(m.group(2)) <= 31: + slug_prefix = f"{m.group(1)}-{m.group(2)}" + if slug_prefix == date_prefix: + slug = m.group(3) + print( + colored( + f'warning: --slug should not include the MM-DD prefix; normalized to "{slug}"', + Colors.YELLOW, + ), + file=sys.stderr, + ) + else: + print( + colored( + f"Error: --slug starts with a date prefix ({slug_prefix}-), but task.py create always uses today's date ({date_prefix}).", + Colors.RED, + ), + file=sys.stderr, + ) + print(f"Pass only the slug body, e.g. --slug {m.group(3)}", file=sys.stderr) + return 1 + + # Resolve --parent before anything is created. An explicit --parent that + # cannot be linked is a failed request, not a warning: creating the task + # anyway leaves a half-specified relationship that reads as success. + parent_dir: Path | None = None + parent_data: dict | None = None + if args.parent: + parent_dir = resolve_task_dir(args.parent, repo_root) + if parent_dir is None: + print(colored(f"Error: Parent task not resolved: {args.parent}", Colors.RED), file=sys.stderr) + print("No task was created. Pass an existing task directory to --parent.", file=sys.stderr) + return 1 + parent_json_path = parent_dir / FILE_TASK_JSON + if not parent_json_path.is_file(): + print(colored(f"Error: Parent task.json not found: {args.parent}", Colors.RED), file=sys.stderr) + print("No task was created. Pass an existing task directory to --parent.", file=sys.stderr) + return 1 + parent_data, parent_reason = read_json_checked(parent_json_path) + if parent_data is None: + _report_read_failure(parent_json_path, parent_reason) + print("No task was created. Fix the parent task.json, then retry.", file=sys.stderr) + return 1 + + dir_name = f"{date_prefix}-{slug}" + task_dir = tasks_dir / dir_name + task_json_path = task_dir / FILE_TASK_JSON + + archived_task_dir = _find_archived_task_by_dir_name(tasks_dir, dir_name) + if archived_task_dir: + print(colored(f"Error: Task already archived: {dir_name}", Colors.RED), file=sys.stderr) + print(f"Archived at: {_repo_relative_path(archived_task_dir, repo_root)}", file=sys.stderr) + print("Use a new slug if you intend to create a new task.", file=sys.stderr) + return 1 + + # Reusing a slug on the same day is an ordinary accident, and continuing + # would rewrite the existing task.json below — resetting status, children, + # parent, branch and meta, which nothing else can reconstruct. Fail like + # the archived-name collision above unless the caller asks for it. + if task_dir.exists(): + if not getattr(args, "force", False): + print(colored(f"Error: Task already exists: {dir_name}", Colors.RED), file=sys.stderr) + print(f"Existing task at: {_repo_relative_path(task_dir, repo_root)}", file=sys.stderr) + print( + "Use a different --slug, or pass --force to overwrite its task.json.", + file=sys.stderr, + ) + return 1 + print( + colored( + f"Warning: --force: overwriting task.json in existing task: {dir_name}", + Colors.YELLOW, + ), + file=sys.stderr, + ) + created_dir = False + else: + task_dir.mkdir(parents=True) + created_dir = True + + today = datetime.now().strftime("%Y-%m-%d") + + # Record the PR target branch. Prefer the repo's actual default branch + # (origin/HEAD) so creating a task from a feature branch doesn't + # mis-stamp that feature branch as the PR target (#399 item 1). Falls + # back to the checked-out branch when the default can't be resolved + # (no remote configured, offline, etc.) — the pre-existing behavior. + # --base-branch lets the caller override both when neither is correct. + _, branch_out, _ = run_git(["branch", "--show-current"], cwd=repo_root) + current_branch = branch_out.strip() or "main" + explicit_base_branch: str | None = getattr(args, "base_branch", None) + if explicit_base_branch: + base_branch = explicit_base_branch + else: + resolved_base_branch = resolve_default_branch(repo_root) + if resolved_base_branch: + base_branch = resolved_base_branch + else: + base_branch = current_branch + print( + colored( + f"warning: could not resolve the repository's default branch " + f"(no remote configured, offline, etc.); stamping base_branch as " + f"the checked-out branch '{base_branch}'. Pass --base-branch to override.", + Colors.YELLOW, + ), + file=sys.stderr, + ) + + task_data = { + "id": slug, + "name": slug, + "title": args.title, + "description": description, + "status": "planning", + "dev_type": None, + "scope": None, + "package": package, + "priority": args.priority, + "creator": creator, + "assignee": assignee, + "createdAt": today, + "completedAt": None, + "branch": None, + "base_branch": base_branch, + "worktree_path": None, + "commit": None, + "pr_url": None, + "subtasks": [], + "children": [], + "parent": None, + "relatedFiles": [], + "notes": "", + "meta": meta, + } + + # A directory without task.json is not a task: `list` hides it and every + # lifecycle command refuses it. Fail here rather than printing "Created + # task" and emitting the path for script chaining. + if not write_json(task_json_path, task_data): + _report_write_failure(task_json_path) + print(colored(f"No task was created: {dir_name}", Colors.RED), file=sys.stderr) + if created_dir: + try: + task_dir.rmdir() + except OSError: + print( + f"Leftover empty directory: {_repo_relative_path(task_dir, repo_root)}", + file=sys.stderr, + ) + return 1 + + prd_path = task_dir / "prd.md" + if not prd_path.exists(): + prd_path.write_text( + _default_prd_content(args.title, description), + encoding="utf-8", + ) + + # Create empty implement.jsonl / check.jsonl for sub-agent-capable + # platforms. They stay empty until the agent curates real entries during + # planning — a placeholder row would read as unresolved scaffolding to + # `task.py validate` and to PR preflight, so the curation instructions go + # to the console below instead of into the files. Agent-less platforms + # (Kilo / Antigravity / Devin) skip this — they load specs via the + # trellis-before-dev skill instead of JSONL. + created_jsonl = False + if _has_subagent_platform(repo_root): + for jsonl_name in ("implement.jsonl", "check.jsonl"): + jsonl_path = task_dir / jsonl_name + if not jsonl_path.exists(): + jsonl_path.write_text("", encoding="utf-8") + created_jsonl = True + + # Establish the bidirectional link. Both sides were validated above, so a + # failure here is a write failure, not a bad argument. + if parent_dir is not None and parent_data is not None: + parent_json_path = parent_dir / FILE_TASK_JSON + + # Add child to parent's children list + parent_children = _ensure_children_list(parent_data) + if dir_name not in parent_children: + parent_children.append(dir_name) + parent_data["children"] = parent_children + if not write_json(parent_json_path, parent_data): + _report_write_failure(parent_json_path) + print( + f"The task exists at {_repo_relative_path(task_dir, repo_root)} but is NOT " + f"linked to {parent_dir.name}. Re-link with: task.py add-subtask " + f"{parent_dir.name} {dir_name}", + file=sys.stderr, + ) + return 1 + + # Set parent in child's task.json + task_data["parent"] = parent_dir.name + if not write_json(task_json_path, task_data): + _report_write_failure(task_json_path) + print( + f"Link is half-written: {parent_dir.name} now lists '{dir_name}' as a child, " + f"but the new task does not record its parent.", + file=sys.stderr, + ) + return 1 + + print(colored(f"Linked as child of: {parent_dir.name}", Colors.GREEN), file=sys.stderr) + + # Auto-activate the new task so the per-turn breadcrumb fires planning + # state. Best-effort: gracefully degrade if no session identity (CLI run + # outside an AI session) — the task is still created, the user can run + # task.py start later. Pointer is session-scoped so this never affects + # other AI sessions. + if getattr(args, "no_start", False): + print( + colored( + "Skipped session activation (--no-start); run task.py start when ready.", + Colors.YELLOW, + ), + file=sys.stderr, + ) + else: + try: + from .active_task import resolve_context_key, set_active_task + except Exception as exc: + print( + colored(f"Warning: session activation unavailable (import failed: {exc})", Colors.YELLOW), + file=sys.stderr, + ) + else: + try: + context_key = resolve_context_key() + except Exception as exc: + print( + colored(f"Warning: session activation failed (context resolution: {exc})", Colors.YELLOW), + file=sys.stderr, + ) + else: + # No session identity is the normal CLI-outside-an-AI-session + # case (see comment above) — stay silent, not a failure. + if context_key: + try: + rel_dir = task_dir.relative_to(repo_root).as_posix() + except ValueError: + rel_dir = str(task_dir) + try: + active = set_active_task(rel_dir, repo_root) + except Exception as exc: + print( + colored(f"Warning: session activation failed (pointer persistence: {exc})", Colors.YELLOW), + file=sys.stderr, + ) + else: + if active: + print( + colored(f"Activated task for this session: {active.task_path}", Colors.GREEN), + file=sys.stderr, + ) + print(f"Source: {active.source}", file=sys.stderr) + else: + print( + colored("Warning: session activation failed (no pointer returned)", Colors.YELLOW), + file=sys.stderr, + ) + + print(colored(f"Created task: {dir_name}", Colors.GREEN), file=sys.stderr) + print("", file=sys.stderr) + print(colored("Next steps:", Colors.BLUE), file=sys.stderr) + print(" - Fill prd.md with requirements and acceptance criteria", file=sys.stderr) + print(" - Lightweight task: PRD-only is valid", file=sys.stderr) + print(" - Complex task: add design.md and implement.md before task.py start", file=sys.stderr) + if created_jsonl: + print( + " - Curate implement.jsonl / check.jsonl (created empty) as spec/research " + "manifests before task.py start when sub-agents need context:", + file=sys.stderr, + ) + print( + ' one JSON object per line — {"file": "<path>", "reason": "<why>"}; ' + "spec/research docs only, no code paths", + file=sys.stderr, + ) + print( + " list available specs: python3 .trellis/scripts/get_context.py --mode packages", + file=sys.stderr, + ) + print(" - Use /trellis:continue or phase context to decide the next step", file=sys.stderr) + print("", file=sys.stderr) + + # Output relative path for script chaining + print(f"{DIR_WORKFLOW}/{DIR_TASKS}/{dir_name}") + + run_task_hooks("after_create", task_json_path, repo_root) + return 0 + + +# ============================================================================= +# Command: rename +# ============================================================================= + +# task.json fields that carry the task's own slug. The directory name carries +# it too, prefixed with the creation date; everything else that names the task +# (parent / children / subtasks in *other* tasks, jsonl paths) stores the full +# directory name instead. +RENAME_IDENTITY_FIELDS: tuple[str, ...] = ("id", "name") + +_JSONL_NAMES: tuple[str, ...] = ("implement.jsonl", "check.jsonl") + + +@dataclass +class _RenamePlan: + """Every change a rename would make, computed before anything is written. + + ``--dry-run`` and the real run render their output from this one structure, + so the printed change set cannot drift from the applied one. + """ + + task_dir: Path + new_dir: Path + old_name: str + new_name: str + old_rel: str + new_rel: str + task_json_path: Path + task_data: dict + # (field, current value, new value) for each identity field that changes. + identity: list[tuple[str, object, str]] = field(default_factory=list) + # (jsonl path, changed line numbers, full new file text) + jsonl: list[tuple[Path, list[int], str]] = field(default_factory=list) + # (other task's task.json, rewritten data, labels of the changed refs) + backrefs: list[tuple[Path, dict, list[str]]] = field(default_factory=list) + # (file, line number) of old-name mentions elsewhere under .trellis/ + reported: list[tuple[Path, int]] = field(default_factory=list) + # Task dirs whose task.json could not be read, so back-refs in them are + # neither rewritten nor ruled out. + unreadable: list[Path] = field(default_factory=list) + + +def _split_date_prefix(dir_name: str) -> tuple[str, str]: + """Split ``MM-DD-slug`` into ``("MM-DD", "slug")``. + + Returns ``("", dir_name)`` when there is no plausible date prefix, so a + hand-made directory name without one is renamed to the bare slug. + """ + m = re.match(r"^(\d{2})-(\d{2})-(.+)$", dir_name) + if m and 1 <= int(m.group(1)) <= 12 and 1 <= int(m.group(2)) <= 31: + return f"{m.group(1)}-{m.group(2)}", m.group(3) + return "", dir_name + + +def _validate_rename_slug(slug: str, date_prefix: str) -> str | None: + """Return the slug body to rename to, or None after reporting the refusal. + + Mirrors ``create``'s ``--slug`` handling: no path separators or ``..`` (the + slug is joined into a directory name), and a pasted-in date prefix is + normalized away rather than doubled — but a rename keeps the task's + *original* creation date, not today's. + """ + if not slug: + print(colored("Error: new slug is required", Colors.RED), file=sys.stderr) + return None + + if "/" in slug or "\\" in slug or ".." in slug: + print( + colored( + f"Error: <new-slug> must be a plain name without path separators or '..': {slug}", + Colors.RED, + ), + file=sys.stderr, + ) + return None + + slug_prefix, body = _split_date_prefix(slug) + if not slug_prefix: + return slug + if slug_prefix == date_prefix: + print( + colored( + f'warning: <new-slug> should not include the MM-DD prefix; normalized to "{body}"', + Colors.YELLOW, + ), + file=sys.stderr, + ) + return body + print( + colored( + f"Error: <new-slug> starts with a date prefix ({slug_prefix}-), but rename keeps " + f"the task's own creation date ({date_prefix}).", + Colors.RED, + ), + file=sys.stderr, + ) + print(f"Pass only the slug body, e.g. {body}", file=sys.stderr) + return None + + +def _plan_jsonl_rewrites( + task_dir: Path, old_rel: str, new_rel: str +) -> list[tuple[Path, list[int], str]]: + """Plan rewrites of context entries pointing under the old task directory. + + Matching on ``<old path>/`` rather than the bare path keeps a sibling task + whose name merely starts with this one's out of the rewrite. + """ + rewrites: list[tuple[Path, list[int], str]] = [] + needle = f"{old_rel}/" + replacement = f"{new_rel}/" + + for jsonl_name in _JSONL_NAMES: + jsonl_path = task_dir / jsonl_name + if not jsonl_path.is_file(): + continue + try: + lines = jsonl_path.read_text(encoding="utf-8").splitlines(keepends=True) + except (OSError, UnicodeDecodeError): + continue + + changed: list[int] = [] + for index, line in enumerate(lines): + if needle in line: + lines[index] = line.replace(needle, replacement) + changed.append(index + 1) + if changed: + rewrites.append((jsonl_path, changed, "".join(lines))) + + return rewrites + + +def _plan_backrefs( + tasks_dir: Path, task_dir: Path, old_name: str, new_name: str +) -> tuple[list[tuple[Path, dict, list[str]]], list[Path]]: + """Plan back-reference rewrites in the other active tasks. + + Returns ``(changes, unreadable)``. ``subtasks`` is the legacy spelling of + ``children`` and is still carried in older task.json files, so both lists + are rewritten. + """ + changes: list[tuple[Path, dict, list[str]]] = [] + unreadable: list[Path] = [] + + for candidate in sorted(tasks_dir.iterdir()): + if not candidate.is_dir() or candidate.name == DIR_ARCHIVE: + continue + if candidate == task_dir: + continue + json_path = candidate / FILE_TASK_JSON + if not json_path.is_file(): + continue + + data, _reason = read_json_checked(json_path) + if data is None: + unreadable.append(json_path) + continue + + labels: list[str] = [] + if data.get("parent") == old_name: + data["parent"] = new_name + labels.append("parent") + for list_field in ("children", "subtasks"): + values = data.get(list_field) + if not isinstance(values, list): + continue + for index, value in enumerate(values): + if value == old_name: + values[index] = new_name + labels.append(f"{list_field}[{index}]") + + if labels: + changes.append((json_path, data, labels)) + + return changes, unreadable + + +def _plan_reported_refs( + repo_root: Path, task_dir: Path, rewritten: set[Path], old_name: str +) -> list[tuple[Path, int]]: + """Find remaining mentions of the old task name elsewhere under .trellis/. + + These are reported, never rewritten: journal entries and workflow prose + cite tasks in free text, and a blind substitution there is how a rename + turns into a diff nobody asked for. The boundary-anchored pattern keeps a + longer task name that merely contains this one out. + + Runtime session pointers are excluded because they are not prose and they + *are* rewritten — see ``repoint_task_in_sessions``. Listing them here as + "not rewritten" would be a false statement about the one file whose + staleness actually breaks the next command. + """ + from .active_task import _runtime_sessions_dir + + pattern = re.compile( + r"(?<![0-9A-Za-z_-])" + re.escape(old_name) + r"(?![0-9A-Za-z_-])" + ) + hits: list[tuple[Path, int]] = [] + trellis_dir = repo_root / DIR_WORKFLOW + sessions_dir = _runtime_sessions_dir(repo_root) + if not trellis_dir.is_dir(): + return hits + + for path in sorted(trellis_dir.rglob("*")): + if not path.is_file() or path in rewritten: + continue + if path == task_dir or task_dir in path.parents: + continue + if path.parent == sessions_dir: + continue + if any( + part == "__pycache__" or part.startswith(".backup-") + for part in path.relative_to(trellis_dir).parts[:-1] + ): + continue + try: + text = path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + continue + for lineno, line in enumerate(text.splitlines(), 1): + if pattern.search(line): + hits.append((path, lineno)) + + return hits + + +def _render_rename_plan(plan: _RenamePlan, repo_root: Path) -> list[str]: + """Render the change set. Identical for --dry-run and the real run.""" + lines = [ + f"rename: {plan.old_name} -> {plan.new_name}", + f" dir: {plan.old_rel} -> {plan.new_rel}", + ] + for field_name, old_value, new_value in plan.identity: + lines.append(f" task.json: {field_name}: {old_value} -> {new_value}") + for json_path, _data, labels in plan.backrefs: + rel = _repo_relative_path(json_path, repo_root) + for label in labels: + lines.append( + f" backref: {rel}: {label}: {plan.old_name} -> {plan.new_name}" + ) + for jsonl_path, linenos, _text in plan.jsonl: + rel = _repo_relative_path(jsonl_path, repo_root) + for lineno in linenos: + lines.append( + f" jsonl: {rel}:{lineno}: {plan.old_rel}/ -> {plan.new_rel}/" + ) + for path, lineno in plan.reported: + lines.append( + f" reported (not rewritten): {_repo_relative_path(path, repo_root)}:{lineno}" + ) + lines.append( + f" sessions: any active-task pointer at {plan.old_rel} is repointed to {plan.new_rel}" + ) + return lines + + +def _rename_interrupted(plan: _RenamePlan) -> None: + """Explain how to finish a rename that stopped part-way through.""" + print( + f"The task is still at {plan.old_rel} and was NOT moved. Every step up to " + f"this one is idempotent: fix the write failure, then run the same rename " + f"again to finish it.", + file=sys.stderr, + ) + + +def _apply_rename(plan: _RenamePlan, repo_root: Path) -> int: + """Write the planned change set. + + The directory move goes last on purpose. Everything before it is an + in-place edit that re-running the identical command recomputes as + already-done, so an interruption leaves a tree that the same command + finishes; a move-first ordering would instead leave the old name + unresolvable and the remaining edits to be made by hand. + """ + if plan.identity: + data = dict(plan.task_data) + for field_name, _old_value, new_value in plan.identity: + data[field_name] = new_value + if not write_json(plan.task_json_path, data): + _report_write_failure(plan.task_json_path) + print("Nothing was renamed.", file=sys.stderr) + return 1 + + for jsonl_path, _linenos, text in plan.jsonl: + try: + jsonl_path.write_text(text, encoding="utf-8") + except OSError as exc: + print( + colored(f"Error: Failed to write {jsonl_path}: {exc}", Colors.RED), + file=sys.stderr, + ) + _rename_interrupted(plan) + return 1 + + for json_path, data, _labels in plan.backrefs: + if not write_json(json_path, data): + _report_write_failure(json_path) + _rename_interrupted(plan) + return 1 + + try: + plan.task_dir.rename(plan.new_dir) + except OSError as exc: + print( + colored(f"Error: Failed to move {plan.old_rel} -> {plan.new_rel}: {exc}", Colors.RED), + file=sys.stderr, + ) + _rename_interrupted(plan) + return 1 + + # Last, and after the move: a session pointing at the old path would now + # resolve to a missing directory, so `current` would report the task stale + # and the context hook would inject nothing until the user ran `start` + # again. Archive clears these pointers because the task is leaving; rename + # moves them, because the task is still the one being worked on. + from .active_task import repoint_task_in_sessions + + repoint_task_in_sessions(str(plan.task_dir), str(plan.new_dir), repo_root) + + return 0 + + +def cmd_rename(args: argparse.Namespace) -> int: + """Rename a task and every reference to it.""" + repo_root = get_repo_root() + tasks_dir = get_tasks_dir(repo_root) + + task_dir = resolve_task_dir(args.name, repo_root) + if task_dir is None or not task_dir.is_dir(): + if task_dir is not None: + print(colored(f"Error: Task not found: {args.name}", Colors.RED), file=sys.stderr) + return 1 + + # Narrower than resolve_task_dir's containment: an archived task has left + # the active set, so its back-references are no longer maintained here. + if not is_within_tasks_dir(task_dir, repo_root): + print( + colored( + f"Error: refusing to rename '{args.name}': " + f"{task_dir} is not an active task under {tasks_dir}", + Colors.RED, + ), + file=sys.stderr, + ) + return 1 + + old_name = task_dir.name + date_prefix, _old_slug = _split_date_prefix(old_name) + + slug = _validate_rename_slug(args.new_slug, date_prefix) + if slug is None: + return 1 + + new_name = f"{date_prefix}-{slug}" if date_prefix else slug + if new_name == old_name: + print( + colored(f"Error: '{new_name}' is the task's current name", Colors.RED), + file=sys.stderr, + ) + return 1 + + new_dir = task_dir.parent / new_name + if new_dir.exists(): + print( + colored(f"Error: Task already exists: {new_name}", Colors.RED), + file=sys.stderr, + ) + print(f"Existing task at: {_repo_relative_path(new_dir, repo_root)}", file=sys.stderr) + print("Pick a different slug.", file=sys.stderr) + return 1 + + archived_task_dir = _find_archived_task_by_dir_name(tasks_dir, new_name) + if archived_task_dir: + print( + colored(f"Error: Task already archived: {new_name}", Colors.RED), + file=sys.stderr, + ) + print(f"Archived at: {_repo_relative_path(archived_task_dir, repo_root)}", file=sys.stderr) + print("Pick a different slug.", file=sys.stderr) + return 1 + + task_json_path = task_dir / FILE_TASK_JSON + if not task_json_path.is_file(): + print( + colored(f"Error: task.json not found at {task_dir}", Colors.RED), + file=sys.stderr, + ) + return 1 + + task_data, reason = read_json_checked(task_json_path) + if task_data is None: + _report_read_failure(task_json_path, reason) + print("Nothing was renamed.", file=sys.stderr) + return 1 + + plan = _RenamePlan( + task_dir=task_dir, + new_dir=new_dir, + old_name=old_name, + new_name=new_name, + old_rel=_repo_relative_path(task_dir, repo_root), + new_rel=_repo_relative_path(new_dir, repo_root), + task_json_path=task_json_path, + task_data=task_data, + ) + plan.identity = [ + (name, task_data.get(name), slug) + for name in RENAME_IDENTITY_FIELDS + if task_data.get(name) != slug + ] + plan.jsonl = _plan_jsonl_rewrites(task_dir, plan.old_rel, plan.new_rel) + plan.backrefs, plan.unreadable = _plan_backrefs( + tasks_dir, task_dir, old_name, new_name + ) + rewritten = {path for path, _linenos, _text in plan.jsonl} + rewritten.update(path for path, _data, _labels in plan.backrefs) + plan.reported = _plan_reported_refs(repo_root, task_dir, rewritten, old_name) + + for line in _render_rename_plan(plan, repo_root): + print(line) + + for json_path in plan.unreadable: + print( + colored( + f"Warning: {_repo_relative_path(json_path, repo_root)} could not be read; " + "any back-reference in it is left as-is.", + Colors.YELLOW, + ), + file=sys.stderr, + ) + + if getattr(args, "dry_run", False): + print(colored("Dry run: nothing was written.", Colors.YELLOW), file=sys.stderr) + return 0 + + rc = _apply_rename(plan, repo_root) + if rc != 0: + return rc + + print(colored(f"Renamed: {old_name} -> {new_name}", Colors.GREEN), file=sys.stderr) + if plan.reported: + print( + colored( + f"{len(plan.reported)} reference(s) elsewhere under {DIR_WORKFLOW}/ " + "still name the old task; they are listed above and were not rewritten.", + Colors.YELLOW, + ), + file=sys.stderr, + ) + return 0 + + +# ============================================================================= +# Command: archive +# ============================================================================= + +def _task_branch_field(data: dict, key: str) -> str: + """Read a branch field as a trimmed string ("" when unset or not a string).""" + value = data.get(key) + return strip_blank(value) if isinstance(value, str) else "" + + +def _validate_branch_metadata( + data: dict, + task_name: str, + repo_root: Path, + skip: bool, +) -> bool: + """Check branch metadata before the task leaves the active tree. + + Returns False when archiving must stop. Archive is the last gate that sees + a task, so metadata nobody can reconstruct afterwards is refused here + rather than repaired by hand later (#399 follow-up). + + "PR-backed" is deliberately pragmatic: a task carrying a base_branch in a + repo that has a remote was created expecting a PR, so a missing `branch` + means the metadata was never recorded — not that the work had no branch. + Local-only repos and tasks without a base_branch are left alone. + + A recorded branch that no longer exists locally stays a warning: after a + merge the feature branch is normally deleted, and refusing to archive then + would be backwards. + """ + branch = _task_branch_field(data, "branch") + base_branch = _task_branch_field(data, "base_branch") + task_py = f"python3 {DIR_WORKFLOW}/scripts/task.py" + + if branch and not branch_exists_locally(branch, repo_root): + print( + colored( + f"Warning: recorded branch '{branch}' no longer exists locally " + "(likely merged and deleted).", + Colors.YELLOW, + ), + file=sys.stderr, + ) + + if skip: + return True + + if branch and base_branch and branch == base_branch: + print( + colored( + f"Error: refusing to archive '{task_name}': branch and base_branch " + f"are both '{branch}'. A PR cannot target its own branch, so this " + "metadata cannot describe the work that was merged.", + Colors.RED, + ), + file=sys.stderr, + ) + print("Repair whichever field is wrong:", file=sys.stderr) + print(f" {task_py} set-branch {task_name} <feature-branch>", file=sys.stderr) + print(f" {task_py} set-base-branch {task_name} <target-branch>", file=sys.stderr) + print( + f" {task_py} archive {task_name} --skip-branch-validation" + " # only if this task was never PR-backed", + file=sys.stderr, + ) + return False + + if not branch and base_branch and has_git_remote(repo_root): + print( + colored( + f"Error: refusing to archive '{task_name}': no branch is recorded, " + f"but the task targets base_branch '{base_branch}' in a repo with a " + "remote — the branch it was built on was never written down.", + Colors.RED, + ), + file=sys.stderr, + ) + print("Repair with:", file=sys.stderr) + print(f" {task_py} set-branch {task_name} <branch>", file=sys.stderr) + print( + f" {task_py} archive {task_name} --skip-branch-validation" + " # only if the work landed without a branch of its own", + file=sys.stderr, + ) + return False + + return True + + +def cmd_archive(args: argparse.Namespace) -> int: + """Archive completed task.""" + repo_root = get_repo_root() + task_name = args.name + + if not task_name: + print(colored("Error: Task name is required", Colors.RED), file=sys.stderr) + return 1 + + tasks_dir = get_tasks_dir(repo_root) + + # Resolve task directory (supports task name, relative path, or absolute path) + task_dir = resolve_task_dir(task_name, repo_root) + + if task_dir is None or not task_dir.is_dir(): + if task_dir is None: + # resolve_task_dir already reported why; keep the archive-specific + # refusal so it reads the same as the is_within_tasks_dir guard. + print(colored( + f"Error: refusing to archive '{task_name}': " + f"it does not resolve to a task under {tasks_dir}", + Colors.RED), file=sys.stderr) + else: + print(colored(f"Error: Task not found: {task_name}", Colors.RED), file=sys.stderr) + print("Active tasks:", file=sys.stderr) + # Import lazily to avoid circular dependency + from .tasks import iter_active_tasks + for t in iter_active_tasks(tasks_dir): + print(f" - {t.dir_name}/", file=sys.stderr) + return 1 + + # Refuse to archive anything that isn't a real task directly under + # .trellis/tasks/. resolve_task_dir keeps the target inside the tasks dir; + # this narrows it further to a direct child, so an already-archived task + # (.trellis/tasks/archive/<month>/<task>) is not archived a second time. + if not is_within_tasks_dir(task_dir, repo_root): + print(colored( + f"Error: refusing to archive '{task_name}': " + f"{task_dir} is not a task under {tasks_dir}", + Colors.RED), file=sys.stderr) + return 1 + + # Check the destination before anything below mutates task state. The + # mover refuses a collision too, but by then this command has already + # marked the task completed, re-parented its children and cleared the + # sessions pointing at it — all of which would have to be undone by hand. + archive_dest_check = archive_destination_for(task_dir) + if archive_dest_check.exists(): + print(colored( + f"Error: refusing to archive '{task_name}': " + f"archive destination already exists: " + f"{_repo_relative_path(archive_dest_check, repo_root)}", + Colors.RED), file=sys.stderr) + print(f"Task remains at: {_repo_relative_path(task_dir, repo_root)}", file=sys.stderr) + print("Move or rename the existing archived task, then retry.", file=sys.stderr) + return 1 + + dir_name = task_dir.name + task_json_path = task_dir / FILE_TASK_JSON + + # Update status before archiving + today = datetime.now().strftime("%Y-%m-%d") + # Names of child task dirs whose task.json gets modified below; passed + # into safe_archive_paths_to_add so they're staged in this commit. + modified_children: list[str] = [] + if task_json_path.is_file(): + data, read_reason = read_json_checked(task_json_path) + if data is None: + # Archiving is still the right outcome for a task whose task.json + # is broken — but say so, or the missing "completed" status looks + # like the archive silently did half its job. + problem, _ = describe_json_read_failure(task_json_path, read_reason) + print( + colored( + f"Warning: {problem}; archiving without updating status/children.", + Colors.YELLOW, + ), + file=sys.stderr, + ) + else: + # Before any mutation: branch metadata is unrecoverable once the + # task leaves the active tree. Stale branches only warn. + if not _validate_branch_metadata( + data, + task_name, + repo_root, + getattr(args, "skip_branch_validation", False), + ): + print( + f"Not archived: {_repo_relative_path(task_dir, repo_root)} is unchanged.", + file=sys.stderr, + ) + return 1 + + data["status"] = "completed" + data["completedAt"] = today + if not write_json(task_json_path, data): + _report_write_failure(task_json_path) + print( + f"Not archived: {_repo_relative_path(task_dir, repo_root)} is unchanged. " + "Archiving a task still marked in progress would hide it from `list` " + "with the wrong status.", + file=sys.stderr, + ) + return 1 + + # Handle subtask relationships on archive. + # Keep this task in its parent's children list so progress + # counters (children_progress) stay consistent — children + # missing from the active set are treated as completed. + task_children = data.get("children", []) + + # If this is a parent, clear parent field in all children + if task_children: + for child_name in task_children: + child_dir_path = find_task_by_name(child_name, tasks_dir) + if child_dir_path: + child_json = child_dir_path / FILE_TASK_JSON + if child_json.is_file(): + child_data, child_reason = read_json_checked(child_json) + if child_data is None: + problem, _ = describe_json_read_failure(child_json, child_reason) + print( + colored( + f"Warning: {problem}; child '{child_dir_path.name}' " + "keeps its parent reference.", + Colors.YELLOW, + ), + file=sys.stderr, + ) + continue + child_data["parent"] = None + if not write_json(child_json, child_data): + # Stop before the move: a child pointing at a + # parent that has left .trellis/tasks/ is a + # dangling reference nothing repairs later. + # Retrying is safe — every step so far is + # idempotent. + _report_write_failure(child_json) + print( + f"Not archived: {_repo_relative_path(task_dir, repo_root)} is " + f"marked completed but stays in place because child " + f"'{child_dir_path.name}' could not be unlinked. " + "Fix the child, then run archive again.", + file=sys.stderr, + ) + return 1 + modified_children.append(child_dir_path.name) + + # Clear any session that still points at this task before the path moves. + from .active_task import clear_task_from_sessions + clear_task_from_sessions(str(task_dir), repo_root) + + # Archive + result = archive_task_complete(task_dir, repo_root) + if "archived_to" in result: + archive_dest = Path(result["archived_to"]) + year_month = archive_dest.parent.name + print(colored(f"Archived: {dir_name} -> archive/{year_month}/", Colors.GREEN), file=sys.stderr) + + # Auto-commit unless --no-commit + if not getattr(args, "no_commit", False): + if not _auto_commit_archive(dir_name, repo_root, modified_children): + print( + colored( + "Archive moved on disk, but git auto-commit did not complete. " + "Resolve `git status` before continuing.", + Colors.RED, + ), + file=sys.stderr, + ) + return 1 + + # Return the archive path + print(f"{DIR_WORKFLOW}/{DIR_TASKS}/{DIR_ARCHIVE}/{year_month}/{dir_name}") + + # Run hooks with the archived path + archived_json = archive_dest / FILE_TASK_JSON + run_task_hooks("after_archive", archived_json, repo_root) + return 0 + + return 1 + + +def _auto_commit_archive( + task_name: str, + repo_root: Path, + modified_children: list[str] | None = None, +) -> bool: + """Stage Trellis-owned task paths and commit after archive. + + Scoped narrowly to the archived task's source + destination paths + plus any child task dirs whose ``task.json`` was edited (parent → + children relationship update). Dirty changes in OTHER active task + dirs are NOT bundled into the archive commit. + + If ``.gitignore`` blocks the paths, we warn + skip — we do NOT + retry with ``git add -f``. The warning explicitly forbids + ``git add -f .trellis/`` (which would fan out to caches/backups) + and points users at ``session_auto_commit: false``. + + Honors ``session_auto_commit`` in ``.trellis/config.yaml``: when + set to ``false``, this function returns immediately without + touching git (the archive directory move on disk is unaffected). + """ + if not get_session_auto_commit(repo_root): + print( + "[OK] session_auto_commit: false — skipping git stage/commit.", + file=sys.stderr, + ) + return True + + source_rel = f"{DIR_WORKFLOW}/{DIR_TASKS}/{task_name}" + rc, tracked_out, _ = run_git( + ["ls-files", "--", source_rel], + cwd=repo_root, + ) + source_was_tracked = rc == 0 and bool(tracked_out.strip()) + + paths = safe_archive_paths_to_add( + repo_root, task_name=task_name, modified_children=modified_children + ) + if not paths: + print("[OK] No task changes to commit.", file=sys.stderr) + return True + + success, _, err = safe_git_add(paths, repo_root, retry_on_index_lock=True) + if not success: + if err and "ignored by" in err.lower(): + print_gitignore_warning(paths) + elif stderr_indicates_index_lock(err): + _print_index_lock_warning( + "git add", task_name, repo_root, [*paths, source_rel] + ) + else: + print( + f"[WARN] git add failed: {err.strip() if err else 'unknown error'}", + file=sys.stderr, + ) + return not source_was_tracked + + # Belt-and-suspenders for the phantom-delete bug: `safe_git_add` uses + # `git add` (no -A) which only stages additions/modifications. The + # source task directory was moved away by `shutil.move`, so its files + # need an explicit `git rm --cached` to stage the deletions in this + # same commit — otherwise they sit as uncommitted "phantom deletes" + # against HEAD until something later picks them up. + # + # `--ignore-unmatch` makes this a no-op when the task was never tracked + # (e.g. archiving a task that lived only in working tree). + rc, _, err = run_git_retry_index_lock( + ["rm", "-r", "--cached", "--ignore-unmatch", "--", source_rel], + cwd=repo_root, + ) + if rc != 0 and stderr_indicates_index_lock(err): + # Committing now would record the archived copy without the + # source-side deletes — a half-archived tree in history. + _print_index_lock_warning( + "git rm --cached", task_name, repo_root, [*paths, source_rel] + ) + return not source_was_tracked + + rc, _, _ = run_git( + ["diff", "--cached", "--quiet", "--", *paths, source_rel], + cwd=repo_root, + ) + if rc == 0: + print("[OK] No task changes to commit.", file=sys.stderr) + return True + + commit_msg = f"chore(task): archive {task_name}" + # Commit with an explicit pathspec: a bare `git commit` would sweep any + # unrelated entries the developer had staged before archiving into the + # chore commit (#579). `source_rel` is included so the source-side + # deletions staged above land in the same commit. + rc, _, err = run_git_retry_index_lock( + ["commit", "-m", commit_msg, "--", *paths, source_rel], cwd=repo_root + ) + if rc == 0: + print(f"[OK] Auto-committed: {commit_msg}", file=sys.stderr) + return True + elif stderr_indicates_index_lock(err): + _print_index_lock_warning( + "git commit", task_name, repo_root, [*paths, source_rel] + ) + return not source_was_tracked + else: + print(f"[WARN] Auto-commit failed: {err.strip()}", file=sys.stderr) + return not source_was_tracked + + +def _print_index_lock_warning( + action: str, task_name: str, repo_root: Path, paths: list[str] +) -> None: + """Report an archive auto-commit that gave up on a held index.lock. + + The move itself already succeeded, so the state is consistent — the task + lives in archive/ and its changes are staged-or-not but never half of + both in a commit. Only the commit is outstanding, which the user (or an + agent reading the log) has to finish by hand. + + ``paths`` are the paths the auto-commit would have staged, so the manual + command keeps the same narrow scope — a blanket ``git add -A -- .trellis/`` + would sweep dirty changes from other active tasks into the archive commit. + """ + lock = index_lock_path(repo_root) + print( + f"[WARN] {action} gave up after {INDEX_LOCK_RETRY_ATTEMPTS} attempts: " + f"another process is holding {lock}", + file=sys.stderr, + ) + print( + "[WARN] The task was moved into archive/ on disk; only the commit is pending.", + file=sys.stderr, + ) + print( + "[WARN] Close whatever holds the lock (an IDE git integration, a status", + file=sys.stderr, + ) + print( + f"[WARN] daemon, another session), or delete {lock} if it is stale, then", + file=sys.stderr, + ) + print( + f'[WARN] commit manually: git add -A -- {" ".join(paths)} && ' + f'git commit -m "chore(task): archive {task_name}"', + file=sys.stderr, + ) + + +# ============================================================================= +# Command: add-subtask +# ============================================================================= + +def cmd_add_subtask(args: argparse.Namespace) -> int: + """Link a child task to a parent task.""" + repo_root = get_repo_root() + + parent_dir = resolve_task_dir(args.parent_dir, repo_root) + child_dir = resolve_task_dir(args.child_dir, repo_root) + if parent_dir is None or child_dir is None: + return 1 + + if not parent_dir: + print(colored(f"Error: Parent task.json not found: {args.parent_dir}", Colors.RED), file=sys.stderr) + return 1 + + if not child_dir: + print(colored(f"Error: Child task.json not found: {args.child_dir}", Colors.RED), file=sys.stderr) + return 1 + + parent_json_path = parent_dir / FILE_TASK_JSON + child_json_path = child_dir / FILE_TASK_JSON + + if not parent_json_path.is_file(): + print(colored(f"Error: Parent task.json not found: {args.parent_dir}", Colors.RED), file=sys.stderr) + return 1 + + if not child_json_path.is_file(): + print(colored(f"Error: Child task.json not found: {args.child_dir}", Colors.RED), file=sys.stderr) + return 1 + + parent_data, parent_reason = read_json_checked(parent_json_path) + if parent_data is None: + _report_read_failure(parent_json_path, parent_reason) + return 1 + child_data, child_reason = read_json_checked(child_json_path) + if child_data is None: + _report_read_failure(child_json_path, child_reason) + return 1 + + # Check if child already has a parent + existing_parent = child_data.get("parent") + if existing_parent: + print(colored(f"Error: Child task already has a parent: {existing_parent}", Colors.RED), file=sys.stderr) + return 1 + + # Add child to parent's children list + parent_children = _ensure_children_list(parent_data) + child_dir_name = child_dir.name + if child_dir_name not in parent_children: + parent_children.append(child_dir_name) + parent_data["children"] = parent_children + + # Set parent in child's task.json + child_data["parent"] = parent_dir.name + + # Write both. A link is two files; if the second write fails silently the + # two sides disagree with no error, so check each and name the side that + # did land so the user knows what to undo. + if not write_json(parent_json_path, parent_data): + print(colored(f"Error: Failed to write parent task.json: {parent_json_path}", Colors.RED), file=sys.stderr) + print("No link was written.", file=sys.stderr) + return 1 + if not write_json(child_json_path, child_data): + print(colored(f"Error: Failed to write child task.json: {child_json_path}", Colors.RED), file=sys.stderr) + print( + f"Link is half-written: {parent_json_path} now lists " + f"'{child_dir.name}' as a child, but the child does not record the parent.", + file=sys.stderr, + ) + return 1 + + print(colored(f"Linked: {child_dir.name} -> {parent_dir.name}", Colors.GREEN), file=sys.stderr) + return 0 + + +# ============================================================================= +# Command: remove-subtask +# ============================================================================= + +def cmd_remove_subtask(args: argparse.Namespace) -> int: + """Unlink a child task from a parent task.""" + repo_root = get_repo_root() + + parent_dir = resolve_task_dir(args.parent_dir, repo_root) + child_dir = resolve_task_dir(args.child_dir, repo_root) + if parent_dir is None or child_dir is None: + return 1 + + if not parent_dir: + print(colored(f"Error: Parent task.json not found: {args.parent_dir}", Colors.RED), file=sys.stderr) + return 1 + + if not child_dir: + print(colored(f"Error: Child task.json not found: {args.child_dir}", Colors.RED), file=sys.stderr) + return 1 + + parent_json_path = parent_dir / FILE_TASK_JSON + child_json_path = child_dir / FILE_TASK_JSON + + if not parent_json_path.is_file(): + print(colored(f"Error: Parent task.json not found: {args.parent_dir}", Colors.RED), file=sys.stderr) + return 1 + + if not child_json_path.is_file(): + print(colored(f"Error: Child task.json not found: {args.child_dir}", Colors.RED), file=sys.stderr) + return 1 + + parent_data, parent_reason = read_json_checked(parent_json_path) + if parent_data is None: + _report_read_failure(parent_json_path, parent_reason) + return 1 + child_data, child_reason = read_json_checked(child_json_path) + if child_data is None: + _report_read_failure(child_json_path, child_reason) + return 1 + + # Remove child from parent's children list + parent_children = _ensure_children_list(parent_data) + child_dir_name = child_dir.name + if child_dir_name in parent_children: + parent_children.remove(child_dir_name) + parent_data["children"] = parent_children + + # Clear parent in child's task.json + child_data["parent"] = None + + # Write both — see cmd_add_subtask: an unchecked second write leaves the + # two sides disagreeing with no error. + if not write_json(parent_json_path, parent_data): + print(colored(f"Error: Failed to write parent task.json: {parent_json_path}", Colors.RED), file=sys.stderr) + print("Nothing was unlinked.", file=sys.stderr) + return 1 + if not write_json(child_json_path, child_data): + print(colored(f"Error: Failed to write child task.json: {child_json_path}", Colors.RED), file=sys.stderr) + print( + f"Unlink is half-written: {parent_json_path} no longer lists " + f"'{child_dir.name}', but the child still records the parent.", + file=sys.stderr, + ) + return 1 + + print(colored(f"Unlinked: {child_dir.name} from {parent_dir.name}", Colors.GREEN), file=sys.stderr) + return 0 + + +# ============================================================================= +# Command: set-branch +# ============================================================================= + +def cmd_set_branch(args: argparse.Namespace) -> int: + """Set git branch for task.""" + repo_root = get_repo_root() + target_dir = resolve_task_dir(args.dir, repo_root) + if target_dir is None: + return 1 + branch = args.branch + + if not branch: + print(colored("Error: Missing arguments", Colors.RED)) + print("Usage: python3 task.py set-branch <task-dir> <branch-name>") + return 1 + + if not target_dir: + # target_dir is None here, so it must not appear in the message. This + # is also the branch a ref pointing outside the repo lands in. + print(colored(f"Error: Task not found: {args.dir}", Colors.RED)) + return 1 + + task_json = target_dir / FILE_TASK_JSON + if not task_json.is_file(): + print(colored(f"Error: task.json not found at {target_dir}", Colors.RED)) + return 1 + + data, reason = read_json_checked(task_json) + if data is None: + _report_read_failure(task_json, reason) + return 1 + + data["branch"] = branch + if not write_json(task_json, data): + _report_write_failure(task_json) + return 1 + + print(colored(f"✓ Branch set to: {branch}", Colors.GREEN)) + return 0 + + +# ============================================================================= +# Command: set-base-branch +# ============================================================================= + +def cmd_set_base_branch(args: argparse.Namespace) -> int: + """Set the base branch (PR target) for task.""" + repo_root = get_repo_root() + target_dir = resolve_task_dir(args.dir, repo_root) + if target_dir is None: + return 1 + base_branch = args.base_branch + + if not base_branch: + print(colored("Error: Missing arguments", Colors.RED)) + print("Usage: python3 task.py set-base-branch <task-dir> <base-branch>") + print("Example: python3 task.py set-base-branch <dir> develop") + print() + print("This sets the target branch for PR (the branch your feature will merge into).") + return 1 + + if not target_dir: + # target_dir is None here, so it must not appear in the message. This + # is also the branch a ref pointing outside the repo lands in. + print(colored(f"Error: Task not found: {args.dir}", Colors.RED)) + return 1 + + task_json = target_dir / FILE_TASK_JSON + if not task_json.is_file(): + print(colored(f"Error: task.json not found at {target_dir}", Colors.RED)) + return 1 + + data, reason = read_json_checked(task_json) + if data is None: + _report_read_failure(task_json, reason) + return 1 + + data["base_branch"] = base_branch + if not write_json(task_json, data): + _report_write_failure(task_json) + return 1 + + print(colored(f"✓ Base branch set to: {base_branch}", Colors.GREEN)) + print(f" PR will target: {base_branch}") + return 0 + + +# ============================================================================= +# Command: set-scope +# ============================================================================= + +def cmd_set_scope(args: argparse.Namespace) -> int: + """Set scope for PR title.""" + repo_root = get_repo_root() + target_dir = resolve_task_dir(args.dir, repo_root) + if target_dir is None: + return 1 + scope = args.scope + + if not scope: + print(colored("Error: Missing arguments", Colors.RED)) + print("Usage: python3 task.py set-scope <task-dir> <scope>") + return 1 + + if not target_dir: + # target_dir is None here, so it must not appear in the message. This + # is also the branch a ref pointing outside the repo lands in. + print(colored(f"Error: Task not found: {args.dir}", Colors.RED)) + return 1 + + task_json = target_dir / FILE_TASK_JSON + if not task_json.is_file(): + print(colored(f"Error: task.json not found at {target_dir}", Colors.RED)) + return 1 + + data, reason = read_json_checked(task_json) + if data is None: + _report_read_failure(task_json, reason) + return 1 + + data["scope"] = scope + if not write_json(task_json, data): + _report_write_failure(task_json) + return 1 + + print(colored(f"✓ Scope set to: {scope}", Colors.GREEN)) + return 0 + + +# ============================================================================= +# Command: set-meta +# ============================================================================= + +def cmd_set_meta(args: argparse.Namespace) -> int: + """Set/overwrite one metadata key on an existing task.""" + repo_root = get_repo_root() + target_dir = resolve_task_dir(args.dir, repo_root) + if target_dir is None: + return 1 + key = args.key + value = args.value + + if not key: + print(colored("Error: Missing arguments", Colors.RED)) + print("Usage: python3 task.py set-meta <task-dir> <key> <value>") + return 1 + + if not target_dir: + # target_dir is None here, so it must not appear in the message. This + # is also the branch a ref pointing outside the repo lands in. + print(colored(f"Error: Task not found: {args.dir}", Colors.RED)) + return 1 + + task_json = target_dir / FILE_TASK_JSON + if not task_json.is_file(): + print(colored(f"Error: task.json not found at {target_dir}", Colors.RED)) + return 1 + + data, reason = read_json_checked(task_json) + if data is None: + _report_read_failure(task_json, reason) + return 1 + + meta = data.get("meta") + if not isinstance(meta, dict): + meta = {} + meta[key] = value + data["meta"] = meta + if not write_json(task_json, data): + _report_write_failure(task_json) + return 1 + + print(colored(f"✓ Meta set: {key} = {value}", Colors.GREEN)) + return 0 diff --git a/.trellis/scripts/common/task_utils.py b/.trellis/scripts/common/task_utils.py new file mode 100755 index 0000000..a6e3470 --- /dev/null +++ b/.trellis/scripts/common/task_utils.py @@ -0,0 +1,498 @@ +#!/usr/bin/env python3 +""" +Task utility functions. + +Provides: + is_within_tasks_dir - Check a resolved path is a task directly under tasks/ + find_task_by_name - Find task directory by name + resolve_task_dir - Resolve task directory from name, relative, or absolute path + archive_destination_for - Path a task would be archived to + archive_task_dir - Archive task to monthly directory + run_task_hooks - Run lifecycle hooks for task events +""" + +from __future__ import annotations + +import shutil +import sys +from datetime import datetime +from pathlib import Path +from typing import TYPE_CHECKING + +from .paths import get_repo_root, get_tasks_dir + +if TYPE_CHECKING: + import subprocess + + +# ============================================================================= +# Path Safety +# ============================================================================= + +def is_within_tasks_dir(task_dir_abs: Path, repo_root: Path | None = None) -> bool: + """Check that a resolved task directory really is a task under the tasks dir. + + A real task lives directly at ``.trellis/tasks/<name>``. This returns True + only when ``task_dir_abs`` is an immediate child of the tasks directory. + + Narrows ``resolve_task_dir``'s containment for archive: that chokepoint + accepts anything under the tasks dir, including a task already in + ``archive/<YYYY-MM>/``. This rejects those, plus the tasks dir itself, so + ``shutil.move`` never re-archives an archived task into a nested copy. + """ + if repo_root is None: + repo_root = get_repo_root() + try: + resolved = task_dir_abs.resolve() + tasks_resolved = get_tasks_dir(repo_root).resolve() + except (OSError, RuntimeError): + return False + if resolved.parent != tasks_resolved: + return False + return resolved.name != "archive" + + +# ============================================================================= +# Task Lookup +# ============================================================================= + +def find_task_by_name(task_name: str, tasks_dir: Path) -> Path | None: + """Find task directory by name (exact or suffix match). + + A task name is a single directory name under ``tasks_dir``, never a path: + names carrying a separator or a dot segment are rejected before the join, + so ``".."`` cannot hand back the tasks dir's own parent. + + An ambiguous suffix (two tasks created on different days with the same + slug) is a failure, not a coin flip — ``iterdir()`` order is filesystem + order, so silently picking the first match picks a different task on a + different machine. + + Args: + task_name: Task name to find. + tasks_dir: Tasks directory path. + + Returns: + Absolute path to task directory, or None if not found or ambiguous. + """ + if not task_name or not tasks_dir or not tasks_dir.is_dir(): + return None + + if "/" in task_name or "\\" in task_name or task_name in (".", ".."): + print(f"Error: invalid task name: {task_name}", file=sys.stderr) + return None + + # Try exact match first + exact_match = tasks_dir / task_name + if exact_match.is_dir(): + return exact_match + + # Try suffix match (e.g., "my-task" matches "01-21-my-task") + matches = sorted( + d for d in tasks_dir.iterdir() + if d.is_dir() and d.name.endswith(f"-{task_name}") + ) + if len(matches) == 1: + return matches[0] + if matches: + print(f"Error: ambiguous task name '{task_name}' matches:", file=sys.stderr) + for match in matches: + print(f" - {match.name}", file=sys.stderr) + print("Pass the full task directory name.", file=sys.stderr) + + return None + + +# ============================================================================= +# Archive Operations +# ============================================================================= + +def archive_destination_for(task_dir_abs: Path) -> Path: + """Path a task would be archived to: <tasks>/archive/<YYYY-MM>/<name>.""" + tasks_dir = task_dir_abs.parent + year_month = datetime.now().strftime("%Y-%m") + return tasks_dir / "archive" / year_month / task_dir_abs.name + + +def archive_task_dir(task_dir_abs: Path, repo_root: Path | None = None) -> Path | None: + """Archive a task directory to archive/{YYYY-MM}/. + + Args: + task_dir_abs: Absolute path to task directory. + repo_root: Repository root path. Defaults to auto-detected. + + Returns: + Path to archived directory, or None on error. + """ + if not task_dir_abs.is_dir(): + print(f"Error: task directory not found: {task_dir_abs}", file=sys.stderr) + return None + + dest = archive_destination_for(task_dir_abs) + month_dir = dest.parent + + # Create archive directory + try: + month_dir.mkdir(parents=True, exist_ok=True) + except (OSError, IOError) as e: + print(f"Error: Failed to create archive directory: {e}", file=sys.stderr) + return None + + # shutil.move into an existing directory moves the source *inside* it, + # producing archive/<month>/<task>/<task>/ and returning a path that is not + # where the task actually landed. That wrong path then flows into the + # printed result, the after_archive hook's TASK_JSON_PATH, and auto-commit + # staging, so refuse before the move instead. + if dest.exists(): + print( + f"Error: refusing to archive {task_dir_abs}: " + f"archive destination already exists: {dest}", + file=sys.stderr, + ) + print( + "Move or rename the existing archived task, then retry.", + file=sys.stderr, + ) + return None + + try: + shutil.move(str(task_dir_abs), str(dest)) + except (OSError, IOError, shutil.Error) as e: + print(f"Error: Failed to move task to archive: {e}", file=sys.stderr) + return None + + return dest + + +def archive_task_complete( + task_dir_abs: Path, + repo_root: Path | None = None +) -> dict[str, str]: + """Complete archive workflow: archive directory. + + Args: + task_dir_abs: Absolute path to task directory. + repo_root: Repository root path. Defaults to auto-detected. + + Returns: + Dict with archive result info. + """ + if not task_dir_abs.is_dir(): + print(f"Error: task directory not found: {task_dir_abs}", file=sys.stderr) + return {} + + archive_dest = archive_task_dir(task_dir_abs, repo_root) + if archive_dest: + return {"archived_to": str(archive_dest)} + + return {} + + +# ============================================================================= +# Task Directory Resolution +# ============================================================================= + +def resolve_task_dir(target_dir: str, repo_root: Path) -> Path | None: + """Resolve task directory to an absolute path inside the tasks directory. + + Supports: + - Absolute path: /path/to/task + - Relative path: .trellis/tasks/01-31-my-task + - Task name: my-task (uses find_task_by_name for lookup) + + This is the containment chokepoint for every command that accepts a task + directory argument. The candidate is resolved (following symlinks) and must + land strictly under ``.trellis/tasks/``; archived tasks under + ``archive/<YYYY-MM>/`` qualify. Traversal (``../victim``), absolute paths + outside the repo, a task dir symlinked out of the tasks tree, and the + tasks directory itself are all rejected here so no caller has to re-check. + The containment base is the tasks dir's own real location, so a + ``.trellis`` that is itself a symlink into a shared store (#567) works. + + Args: + target_dir: Task directory specification. + repo_root: Repository root path. + + Returns: + Absolute path spelled through the repo's own tasks dir (in-repo + lexical form), or None when it is not a location inside the tasks + directory (an error naming the path is printed to stderr). + """ + if not target_dir: + print("Error: task directory is required", file=sys.stderr) + return None + + normalized = target_dir.replace("\\", "/") + while normalized.startswith("./"): + normalized = normalized[2:] + + tasks_dir = get_tasks_dir(repo_root) + + if Path(target_dir).is_absolute(): + candidate = Path(target_dir) + elif "/" in normalized or normalized.startswith(".trellis"): + # Relative path (contains path separator or starts with .trellis) + candidate = repo_root / Path(normalized) + else: + # Task name - must resolve inside the tasks directory. The historical + # fallback to repo_root/<name> only ever produced a path the check + # below rejects, so a miss ends here instead. + candidate = find_task_by_name(target_dir, tasks_dir) + if candidate is None: + # find_task_by_name reports invalid names and ambiguity itself. + print( + f"Error: could not resolve task '{target_dir}' under {tasks_dir}", + file=sys.stderr, + ) + return None + + try: + resolved = candidate.resolve() + tasks_lexical = get_tasks_dir(repo_root.resolve()) + tasks_resolved = tasks_lexical.resolve() + except (OSError, RuntimeError) as e: + print(f"Error: could not resolve task directory '{target_dir}': {e}", file=sys.stderr) + return None + + if resolved == tasks_resolved: + print( + f"Error: refusing to use '{target_dir}': {tasks_resolved} is the tasks " + "directory itself, not a task", + file=sys.stderr, + ) + return None + + if tasks_resolved not in resolved.parents: + print( + f"Error: refusing to use '{target_dir}': {resolved} is outside {tasks_resolved}", + file=sys.stderr, + ) + return None + + # Hand back the path spelled through the repo's own tasks dir rather than + # `resolved`: `.trellis` may be a symlink into a store outside the repo + # (#567), in which case `resolved` sits outside the repo even though the + # task is legitimate, and callers converting to a repo-relative ref for + # storage would refuse it. + return tasks_lexical / resolved.relative_to(tasks_resolved) + + +# ============================================================================= +# Lifecycle Hooks +# ============================================================================= + +HOOK_TIMEOUT_SECONDS = 60 +HOOK_KILL_GRACE_SECONDS = 5 +HOOK_OUTPUT_LIMIT = 2000 + + +def _kill_hook_tree(proc: subprocess.Popen[str]) -> None: + """Kill a timed-out hook's whole process tree, not just the shell. + + With ``shell=True`` the direct child is the shell; the actual work runs in + its children. Killing only the shell leaves grandchildren alive holding the + inherited stdout/stderr pipes, and collecting output afterwards then blocks + until those orphans exit — the exact "command that never returns" the + timeout exists to prevent. + + POSIX: the hook is started with ``start_new_session=True``, so the shell and + every descendant share one fresh process group; SIGKILL the group. + Windows: ``taskkill /F /T`` walks the tree (best effort). + Either way ``proc.kill()`` is the fallback. + + Limitation: a hook that calls ``setsid`` itself leaves the group and + survives this. Accepted and out of scope — a hook is arbitrary code, and the + timeout is a liveness guarantee, not a containment boundary. + """ + import os + import signal + import subprocess + + if os.name == "posix": + try: + os.killpg(os.getpgid(proc.pid), signal.SIGKILL) + return + except (ProcessLookupError, PermissionError, OSError): + pass + else: + try: + subprocess.run( + ["taskkill", "/F", "/T", "/PID", str(proc.pid)], + capture_output=True, + ) + except (OSError, ValueError): + pass + + try: + proc.kill() + except OSError: + pass + + +def _release_hook_process(proc: subprocess.Popen[str]) -> None: + """Close our pipe ends and reap a finished hook under a bound. + + ``Popen`` used as a context manager calls ``wait()`` with no timeout when + the block exits. That is the one place the liveness guarantee could still + be lost: if ``_kill_hook_tree`` failed outright — both the process-group + kill and the direct kill raising — the lifecycle command would block there + forever, which is the exact hang the timeout exists to prevent. + """ + import subprocess + + for stream in (proc.stdout, proc.stderr, proc.stdin): + if stream is not None: + try: + stream.close() + except OSError: + pass + try: + proc.wait(timeout=HOOK_KILL_GRACE_SECONDS) + except (subprocess.TimeoutExpired, OSError, ValueError): + # Leaves a zombie until this process exits. Fail-open and bounded beats + # a task command that never returns. + pass + + +def _decode_hook_output(raw: object) -> str: + """TimeoutExpired carries bytes or str depending on the platform.""" + if raw is None: + return "" + if isinstance(raw, bytes): + return raw.decode("utf-8", errors="replace") + return str(raw) + + +def _print_hook_stream(name: str, text: str) -> None: + """Print a captured hook stream to stderr, truncated.""" + text = text.strip() + if not text: + return + if len(text) > HOOK_OUTPUT_LIMIT: + text = ( + text[:HOOK_OUTPUT_LIMIT] + + f"\n… ({len(text) - HOOK_OUTPUT_LIMIT} more characters truncated)" + ) + print(f" {name}:", file=sys.stderr) + for line in text.splitlines(): + print(f" {line}", file=sys.stderr) + + +def run_task_hooks(event: str, task_json_path: Path, repo_root: Path) -> None: + """Run lifecycle hooks for a task event. + + Hooks are shell commands read from ``.trellis/config.yaml`` and executed + with ``shell=True`` from the repo root — see the trust boundary note in + ``.trellis/spec/cli/backend/script-conventions.md``. + + Fail-open by design: a broken hook warns and the lifecycle command + continues. The warning names the event, command, exit status, and both + captured streams, because a hook whose only symptom is "nothing happened" + is undebuggable. A hook that hangs is bounded by + ``HOOK_TIMEOUT_SECONDS``; output is captured, so without the timeout the + user sees a task command that never returns and prints nothing. On timeout + the hook's entire process tree is killed (see ``_kill_hook_tree``) and + output is collected under a bounded grace, so a surviving grandchild + holding the pipes cannot re-create that same hang. Cleanup is bounded for + the same reason — see ``_release_hook_process``. + + Args: + event: Event name (e.g. "after_create"). + task_json_path: Absolute path to the task's task.json. + repo_root: Repository root for cwd and config lookup. + """ + import os + import subprocess + + from .config import get_hooks + from .log import Colors, colored + + commands = get_hooks(event, repo_root) + if not commands: + return + + env = {**os.environ, "TASK_JSON_PATH": str(task_json_path)} + + # POSIX only: a fresh session puts the shell and every descendant in one + # process group, which is what makes the timeout kill the whole tree. + popen_kwargs: dict = {} + if os.name == "posix": + popen_kwargs["start_new_session"] = True + + for cmd in commands: + proc = None + try: + proc = subprocess.Popen( + cmd, + shell=True, + cwd=repo_root, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding="utf-8", + errors="replace", + **popen_kwargs, + ) + try: + stdout, stderr = proc.communicate(timeout=HOOK_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired as e: + _kill_hook_tree(proc) + stdout = _decode_hook_output(e.stdout) + stderr = _decode_hook_output(e.stderr) + try: + # Bounded: an orphan that escaped the kill still holds + # the pipes, and waiting on it forever here would be + # the very hang the timeout prevents. + rest_out, rest_err = proc.communicate( + timeout=HOOK_KILL_GRACE_SECONDS + ) + stdout = rest_out or stdout + stderr = rest_err or stderr + except (subprocess.TimeoutExpired, OSError, ValueError): + pass + print( + colored( + f"[WARN] Hook timed out ({event}) after " + f"{HOOK_TIMEOUT_SECONDS}s: {cmd}", + Colors.YELLOW, + ), + file=sys.stderr, + ) + print(f" cwd: {repo_root}", file=sys.stderr) + _print_hook_stream("stdout", stdout) + _print_hook_stream("stderr", stderr) + continue + + if proc.returncode != 0: + print( + colored( + f"[WARN] Hook failed ({event}): exit {proc.returncode}: {cmd}", + Colors.YELLOW, + ), + file=sys.stderr, + ) + print(f" cwd: {repo_root}", file=sys.stderr) + _print_hook_stream("stdout", stdout or "") + _print_hook_stream("stderr", stderr or "") + except Exception as e: + print( + colored( + f"[WARN] Hook error ({event}): {cmd} — {type(e).__name__}: {e}", + Colors.YELLOW, + ), + file=sys.stderr, + ) + + +# ============================================================================= +# Main Entry (for testing) +# ============================================================================= + +if __name__ == "__main__": + repo = get_repo_root() + tasks = get_tasks_dir(repo) + + print(f"Tasks dir: {tasks}") + print(f"resolve_task_dir('.trellis/tasks/test'): {resolve_task_dir('.trellis/tasks/test', repo)}") + print(f"resolve_task_dir('../test'): {resolve_task_dir('../test', repo)}") diff --git a/.trellis/scripts/common/tasks.py b/.trellis/scripts/common/tasks.py new file mode 100755 index 0000000..ae7f46f --- /dev/null +++ b/.trellis/scripts/common/tasks.py @@ -0,0 +1,122 @@ +""" +Task data access layer. + +Single source of truth for loading and iterating task directories. +Replaces scattered task.json parsing across 9+ files. + +Provides: + load_task — Load a single task by directory path + iter_active_tasks — Iterate all non-archived tasks (sorted) + get_all_statuses — Get {dir_name: status} map for children progress +""" + +from __future__ import annotations + +import sys +from collections.abc import Iterator +from pathlib import Path + +from .io import describe_json_read_failure, read_json_checked +from .paths import FILE_TASK_JSON +from .types import TaskInfo + + +def load_task(task_dir: Path) -> TaskInfo | None: + """Load task from a directory containing task.json. + + Args: + task_dir: Absolute path to the task directory. + + Returns: + TaskInfo if task.json exists and is valid, None otherwise. + + A directory without task.json is not a task, so it is skipped silently. + A task.json that exists but cannot be loaded is different: the task + disappears from `task.py list` and from every context the iterator feeds. + Callers stay tolerant, but the skip is announced on stderr so a task + cannot vanish from the workflow with no diagnostic anywhere. + """ + task_json = task_dir / FILE_TASK_JSON + if not task_json.is_file(): + return None + + data, reason = read_json_checked(task_json) + if data is None: + problem, hint = describe_json_read_failure(task_json, reason) + print(f"[WARN] Skipping task '{task_dir.name}': {problem}", file=sys.stderr) + print(f" {hint}", file=sys.stderr) + return None + + return TaskInfo( + dir_name=task_dir.name, + directory=task_dir, + title=data.get("title") or data.get("name") or "unknown", + status=data.get("status", "unknown"), + assignee=data.get("assignee", ""), + priority=data.get("priority", "P2"), + children=tuple(data.get("children", [])), + parent=data.get("parent"), + package=data.get("package"), + raw=data, + ) + + +def iter_active_tasks(tasks_dir: Path) -> Iterator[TaskInfo]: + """Iterate all active (non-archived) tasks, sorted by directory name. + + Skips the "archive" directory and directories without valid task.json. + + Args: + tasks_dir: Path to the tasks directory. + + Yields: + TaskInfo for each valid task. + """ + if not tasks_dir.is_dir(): + return + + for d in sorted(tasks_dir.iterdir()): + if not d.is_dir() or d.name == "archive": + continue + info = load_task(d) + if info is not None: + yield info + + +def get_all_statuses(tasks_dir: Path) -> dict[str, str]: + """Get a {dir_name: status} mapping for all active tasks. + + Useful for computing children progress without loading full TaskInfo. + + Args: + tasks_dir: Path to the tasks directory. + + Returns: + Dict mapping directory names to status strings. + """ + return {t.dir_name: t.status for t in iter_active_tasks(tasks_dir)} + + +def children_progress( + children: tuple[str, ...] | list[str], + all_statuses: dict[str, str], +) -> str: + """Format children progress string like " [2/3 done]". + + Args: + children: List of child directory names. + all_statuses: Status map from get_all_statuses(). + + Returns: + Formatted string, or "" if no children. + """ + if not children: + return "" + # A child missing from active statuses has been archived (cmd_archive + # sets status=completed before moving the dir). Count it as done so + # parent progress doesn't regress when children are archived. + done = sum( + 1 for c in children + if c not in all_statuses or all_statuses.get(c) in ("completed", "done") + ) + return f" [{done}/{len(children)} done]" diff --git a/.trellis/scripts/common/trellis_config.py b/.trellis/scripts/common/trellis_config.py new file mode 100755 index 0000000..800a591 --- /dev/null +++ b/.trellis/scripts/common/trellis_config.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +""" +Standalone reader for .trellis/config.yaml. + +Owns the minimal YAML parser used across Trellis. ``common.config`` imports +``parse_simple_yaml`` from here rather than keeping its own copy: this module +imports nothing from the package, so hooks can load it as a single file, and +one parser cannot drift from another. Returns an empty dict on +missing/malformed files so callers stay simple. + +Supported subset: ``key: value`` scalars (everything is a string), nested +mappings by indentation, ``- `` lists of scalars, ``#`` comments (whole-line +and inline outside quotes), and one layer of matching surrounding quotes. +Constructs outside that subset — block scalars, anchors, aliases, merge keys, +flow collections, and mappings nested inside a list — are reported on stderr +and skipped rather than parsed into a plausible-looking wrong value. +""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Optional + + +CONFIG_REL_PATH = ".trellis/config.yaml" + + +def _unquote(value: str) -> str: + if len(value) >= 2 and value[0] == value[-1] and value[0] in ('"', "'"): + return value[1:-1] + return value + + +def _strip_inline_comment(value: str) -> str: + """Strip ` # …` inline comments while preserving `#` inside quoted strings. + + YAML treats ` #` (space-hash) as a comment opener; bare `#` inside a token + is part of the value. Quoted strings are immune. + """ + in_quote: str | None = None + for idx, ch in enumerate(value): + if in_quote: + if ch == in_quote: + in_quote = None + continue + if ch in ('"', "'"): + in_quote = ch + continue + if ch == "#" and (idx == 0 or value[idx - 1].isspace()): + return value[:idx] + return value + + +def _next_content_line(lines: list[str], start: int) -> tuple[int, str]: + i = start + while i < len(lines): + stripped = lines[i].strip() + if stripped and not stripped.startswith("#"): + return i, lines[i] + i += 1 + return i, "" + + +def _warn_unsupported(source: str, lineno: int, line: str, reason: str) -> None: + """Report a YAML construct this parser cannot represent, and move on.""" + print( + f"[WARN] {source}:{lineno}: {reason}; ignoring: {line.strip()}", + file=sys.stderr, + ) + + +def _is_block_scalar(value: str) -> bool: + """True for ``|``, ``>`` and their chomping/indent indicators (``|-``, ``>2``).""" + if not value or value[0] not in ("|", ">"): + return False + return all(ch in "+-0123456789" for ch in value[1:]) + + +def _unsupported_value(key: str, value: str) -> str | None: + """Name the unsupported construct in an unquoted scalar value, else None. + + Only unquoted values are inspected: ``cmd: "[a] | b"`` is a string the user + wrote deliberately, while a bare ``notes: |`` or ``base: *anchor`` would + otherwise be stored as the literal marker with the real content dropped. + """ + if key == "<<": + return "YAML merge keys are not supported" + if _is_block_scalar(value): + return "block scalars are not supported" + if value.startswith("&"): + return "YAML anchors are not supported" + if value.startswith("*"): + return "YAML aliases are not supported" + if value.startswith("["): + return "flow sequences are not supported (use `- ` list items)" + if value.startswith("{"): + return "flow mappings are not supported (use an indented mapping)" + return None + + +def _skip_indented_body(lines: list[str], start: int, indent: int) -> int: + """Skip the continuation lines of a rejected key (block scalar body etc.).""" + i = start + while i < len(lines): + stripped = lines[i].strip() + if stripped and len(lines[i]) - len(lines[i].lstrip()) <= indent: + break + i += 1 + return i + + +def _parse_yaml_block( + lines: list[str], start: int, min_indent: int, target: dict, source: str +) -> int: + i = start + current_list: list | None = None + # Indent of the key that opened current_list, so a deeper `key: value` + # can be recognized as a mapping inside that list. + list_owner_indent = 0 + + while i < len(lines): + line = lines[i] + stripped = line.strip() + + if not stripped or stripped.startswith("#"): + i += 1 + continue + + indent = len(line) - len(line.lstrip()) + if indent < min_indent: + break + + if stripped.startswith("- "): + if current_list is not None: + current_list.append(_unquote(stripped[2:].strip())) + i += 1 + elif ":" in stripped: + if current_list is not None and indent > list_owner_indent: + # `- name: cli` / ` path: x`: the second key belongs to a + # mapping inside the list. Storing it would hoist it into the + # parent dict as a sibling of the list — a nested key silently + # becoming a root key. + _warn_unsupported( + source, + i + 1, + line, + "mappings inside a list are not supported", + ) + i += 1 + continue + + key, _, value = stripped.partition(":") + key = key.strip() + value = _strip_inline_comment(value).strip() + was_quoted = len(value) >= 2 and value[0] == value[-1] and value[0] in ('"', "'") + + if not was_quoted: + reason = _unsupported_value(key, value) + if reason is not None: + _warn_unsupported(source, i + 1, line, reason) + current_list = None + i = _skip_indented_body(lines, i + 1, indent) + continue + + value = _unquote(value) + current_list = None + + if value or was_quoted: + target[key] = value + i += 1 + else: + next_i, next_line = _next_content_line(lines, i + 1) + if next_i >= len(lines): + target[key] = {} + i = next_i + elif next_line.strip().startswith("- "): + current_list = [] + list_owner_indent = indent + target[key] = current_list + i += 1 + else: + next_indent = len(next_line) - len(next_line.lstrip()) + if next_indent > indent: + nested: dict = {} + target[key] = nested + i = _parse_yaml_block(lines, i + 1, next_indent, nested, source) + else: + target[key] = {} + i += 1 + else: + i += 1 + + return i + + +def parse_simple_yaml(content: str, source: str = "config.yaml") -> dict: + """Parse simple YAML with nested dict support (no dependencies). + + Supports: + - key: value (string) + - key: (followed by list items) + - item1 + - item2 + - key: (followed by nested dict) + nested_key: value + nested_key2: + - item + + Uses indentation to detect nesting (2+ spaces deeper = child). Every value + is a string; consumers coerce. Unsupported constructs are reported on + stderr against ``source`` and skipped — see the module docstring. + + Args: + content: YAML content string. + source: Label used in warnings, normally the config file path. + + Returns: + Parsed dict (values can be str, list[str], or dict). + """ + lines = content.splitlines() + result: dict = {} + _parse_yaml_block(lines, 0, 0, result, source) + return result + + +def read_trellis_config(repo_root: Optional[Path] = None) -> dict: + """Read .trellis/config.yaml. Returns {} on missing or malformed file.""" + root = repo_root or Path.cwd() + config_file = root / CONFIG_REL_PATH + try: + content = config_file.read_text(encoding="utf-8") + except (FileNotFoundError, OSError): + return {} + try: + parsed = parse_simple_yaml(content, source=str(config_file)) + except Exception: + return {} + return parsed if isinstance(parsed, dict) else {} diff --git a/.trellis/scripts/common/types.py b/.trellis/scripts/common/types.py new file mode 100755 index 0000000..5802e10 --- /dev/null +++ b/.trellis/scripts/common/types.py @@ -0,0 +1,110 @@ +""" +Core type definitions for Trellis task data. + +Provides: + TaskData — TypedDict for task.json shape (read-path type hints only) + TaskInfo — Frozen dataclass for loaded task (the public API type) + AgentRecord — TypedDict for registry.json agent entries +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import TypedDict + + +# ============================================================================= +# task.json shape (TypedDict — used only for read-path type hints) +# ============================================================================= + +class TaskData(TypedDict, total=False): + """Shape of task.json on disk. + + Used only for type annotations when reading task.json. + Writes must use the original dict to avoid losing unknown fields. + """ + + id: str + name: str + title: str + description: str + status: str + dev_type: str + scope: str | None + package: str | None + priority: str + creator: str + assignee: str + createdAt: str + completedAt: str | None + branch: str | None + base_branch: str | None + worktree_path: str | None + commit: str | None + pr_url: str | None + subtasks: list[str] + children: list[str] + parent: str | None + relatedFiles: list[str] + notes: str + meta: dict + + +# ============================================================================= +# Loaded task object (frozen dataclass — the public API type) +# ============================================================================= + +@dataclass(frozen=True) +class TaskInfo: + """Immutable view of a loaded task. + + Created by load_task() / iter_active_tasks(). + Contains the commonly accessed fields; the original dict + is preserved in `raw` for write-back and uncommon field access. + """ + + dir_name: str + directory: Path + title: str + status: str + assignee: str + priority: str + children: tuple[str, ...] + parent: str | None + package: str | None + raw: dict # original dict — use for writes and uncommon fields + + @property + def name(self) -> str: + """Task name (id or name field).""" + return self.raw.get("name") or self.raw.get("id") or self.dir_name + + @property + def description(self) -> str: + return self.raw.get("description", "") + + @property + def branch(self) -> str | None: + return self.raw.get("branch") + + @property + def meta(self) -> dict: + return self.raw.get("meta", {}) + + +# ============================================================================= +# registry.json agent entry +# ============================================================================= + +class AgentRecord(TypedDict, total=False): + """Shape of an agent entry in registry.json.""" + + id: str + pid: int + task_dir: str + worktree_path: str + branch: str + platform: str + started_at: str + status: str diff --git a/.trellis/scripts/common/workflow_phase.py b/.trellis/scripts/common/workflow_phase.py new file mode 100755 index 0000000..b268578 --- /dev/null +++ b/.trellis/scripts/common/workflow_phase.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Workflow Phase Extraction. + +Extracts step-level content from .trellis/workflow.md and optionally filters +platform-specific blocks. + +Platform marker syntax in workflow.md: + + [Claude Code, Cursor, ...] + agent-capable content + [/Claude Code, Cursor, ...] + +Provides: + get_phase_index - Extract the Phase Index section (no --step) + get_step - Extract a single step (#### X.X) section + filter_platform - Strip platform blocks that don't include the given name +""" + +from __future__ import annotations + +import re + +from .paths import DIR_WORKFLOW, get_repo_root + + +def _workflow_md_path(): + return get_repo_root() / DIR_WORKFLOW / "workflow.md" + +# Match a line that *is* a platform marker: "[A, B, C]" or "[/A, B, C]" +_MARKER_RE = re.compile(r"^\[(/?)([A-Za-z][^\[\]]*)\]\s*$") + +# Step heading: "#### 1.0 Title" or "#### 1.0 ..." +_STEP_HEADING_RE = re.compile(r"^####\s+(\d+\.\d+)\b.*$") + +# Phase Index starts here; Phase 1/2/3 step bodies follow; ends at Breadcrumbs. +_PHASE_INDEX_HEADING = "## Phase Index" + + +def _read_workflow() -> str: + path = _workflow_md_path() + if not path.exists(): + raise FileNotFoundError(f"workflow.md not found: {path}") + return path.read_text(encoding="utf-8") + + +def _parse_marker(line: str) -> tuple[bool, list[str]] | None: + """Parse a platform marker line. + + Returns: + (is_closing, [platform_names]) if line is a marker, else None. + """ + m = _MARKER_RE.match(line) + if not m: + return None + is_closing = m.group(1) == "/" + names = [p.strip() for p in m.group(2).split(",") if p.strip()] + return is_closing, names + + +def get_phase_index() -> str: + """Return the compact Phase Index summary from workflow.md. + + SessionStart and no-step phase context use this small summary as their + orientation payload. Detailed Phase 1/2/3 instructions are loaded with + ``get_step`` on demand. ``[workflow-state:STATUS]`` tag blocks are + consumed by the per-turn hook, so they're stripped from this output. + """ + text = _read_workflow() + lines = text.splitlines() + + start: int | None = None + end: int | None = None + for i, line in enumerate(lines): + stripped = line.strip() + if start is None and stripped == _PHASE_INDEX_HEADING: + start = i + continue + if start is not None and stripped == "## Phase 1: Plan": + end = i + break + + if start is None: + return "" + if end is None: + end = len(lines) + + section = "\n".join(lines[start:end]).rstrip() + # Strip [workflow-state:STATUS]...[/workflow-state:STATUS] blocks since + # they're injected separately by inject-workflow-state.py per-turn. + import re as _re + tag_re = _re.compile( + r"\[workflow-state:([A-Za-z0-9_-]+)\]\s*\n.*?\n\s*\[/workflow-state:\1\]\n?", + _re.DOTALL, + ) + return tag_re.sub("", section).rstrip() + "\n" + + +def get_step(step_id: str) -> str: + """Return the `#### X.X` section matching step_id (header + body). + + Body ends at the next `####` or `---` or `##` heading (whichever comes first). + """ + text = _read_workflow() + lines = text.splitlines() + + start: int | None = None + for i, line in enumerate(lines): + m = _STEP_HEADING_RE.match(line) + if m and m.group(1) == step_id: + start = i + break + if start is None: + return "" + + end: int = len(lines) + for j in range(start + 1, len(lines)): + line = lines[j] + if line.startswith("#### "): + end = j + break + if line.startswith("## "): + end = j + break + # Horizontal rule at column 0 + if line.strip() == "---": + end = j + break + + return "\n".join(lines[start:end]).rstrip() + "\n" + + +def _platform_matches(platform: str, block_names: list[str]) -> bool: + """Case-insensitive fuzzy match: accept 'cursor', 'Cursor', 'claude-code', 'Claude Code'.""" + needle = platform.lower().replace("-", "").replace("_", "").replace(" ", "") + for name in block_names: + hay = name.lower().replace("-", "").replace("_", "").replace(" ", "") + if needle == hay: + return True + return False + + +_PLATFORM_MARKER_LABELS: dict[str, str] = { + # workflow.md marker blocks label platforms with their product names, but + # every caller passes the stable id instead (`--platform {{CLI_FLAG}}` in + # the start / continue commands). `_platform_matches` only strips + # punctuation, so an id that is not its label-minus-spaces never matches and + # `filter_platform` drops the block WITHOUT error — the section just comes + # back empty. Four platforms shipped that way before this table existed. + # + # Add an entry whenever a platform's id is not its marker label with the + # separators removed. `test/registry-invariants.test.ts` asserts every + # registry id keeps a non-empty routing section, so a missing entry fails + # there rather than silently blanking that platform's routing. + "claude": "Claude Code", + "kimi": "Kimi Code", + "omp": "Oh My Pi", + "dsh": "DeepSeek Harness", +} + + +def resolve_effective_platform(platform: str, config: dict) -> str: + """Map ``codex`` to a dispatch-mode-namespaced virtual platform name. + + When ``--platform codex`` is passed, return ``"codex-sub-agent"`` by + default or ``"codex-inline"`` when explicitly configured in + ``.trellis/config.yaml``. ``sub-agent`` remains an alias for ``auto``. + ``filter_platform`` then surfaces blocks whose marker lists include the + namespaced name (e.g. ``[codex-sub-agent, ...]`` or ``[codex-inline, Kilo, + Antigravity, Devin]``). + + Native Codex context injection supports the ``auto`` default. Invalid + explicit values fall back to ``inline`` safely; this renderer deliberately + does not warn because it can run in normal CLI output flows. + + Platforms whose marker label differs from their id resolve through + ``_PLATFORM_MARKER_LABELS``. Everything else is returned unchanged. + """ + label = _PLATFORM_MARKER_LABELS.get(platform.strip().lower()) + if label: + return label + if platform == "codex": + mode = "auto" + codex_cfg = config.get("codex") if isinstance(config, dict) else None + if codex_cfg is not None: + if not isinstance(codex_cfg, dict): + mode = "inline" + else: + cfg_mode = str(codex_cfg.get("dispatch_mode", mode)).strip().lower() + if cfg_mode == "inline": + mode = "inline" + elif cfg_mode in ("auto", "sub-agent"): + mode = "auto" + else: + mode = "inline" + return "codex-sub-agent" if mode == "auto" else "codex-inline" + return platform + + +def filter_platform(content: str, platform: str) -> str: + """Keep lines outside any `[...]` block + lines inside blocks that include platform. + + Marker lines themselves are dropped from the output. + """ + lines = content.splitlines() + out: list[str] = [] + + in_block = False + keep_block = False + + for line in lines: + marker = _parse_marker(line) + if marker is not None: + is_closing, names = marker + if not is_closing: + in_block = True + keep_block = _platform_matches(platform, names) + else: + in_block = False + keep_block = False + continue # drop the marker line itself + + if in_block: + if keep_block: + out.append(line) + continue + out.append(line) + + # Collapse runs of 3+ blank lines that may arise from dropped markers + collapsed: list[str] = [] + blank_run = 0 + for line in out: + if line.strip() == "": + blank_run += 1 + if blank_run <= 2: + collapsed.append(line) + else: + blank_run = 0 + collapsed.append(line) + + return "\n".join(collapsed).rstrip() + "\n" diff --git a/.trellis/scripts/get_context.py b/.trellis/scripts/get_context.py new file mode 100755 index 0000000..bc63463 --- /dev/null +++ b/.trellis/scripts/get_context.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +""" +Get Session Context for AI Agent. + +Usage: + python3 get_context.py Output context in text format + python3 get_context.py --json Output context in JSON format +""" + +from __future__ import annotations + +from common.git_context import main + + +if __name__ == "__main__": + main() diff --git a/.trellis/scripts/get_developer.py b/.trellis/scripts/get_developer.py new file mode 100755 index 0000000..f8a89eb --- /dev/null +++ b/.trellis/scripts/get_developer.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +""" +Get current developer name. + +This is a wrapper that uses common/paths.py +""" + +from __future__ import annotations + +import sys + +from common.paths import get_developer + + +def main() -> None: + """CLI entry point.""" + developer = get_developer() + if developer: + print(developer) + else: + print("Developer not initialized", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/.trellis/scripts/hooks/linear_sync.py b/.trellis/scripts/hooks/linear_sync.py new file mode 100755 index 0000000..5659fde --- /dev/null +++ b/.trellis/scripts/hooks/linear_sync.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 +"""Linear sync hook for Trellis task lifecycle. + +Syncs task events to Linear via the `linearis` CLI. + +Usage (called automatically by task.py hooks): + python3 .trellis/scripts/hooks/linear_sync.py create + python3 .trellis/scripts/hooks/linear_sync.py start + python3 .trellis/scripts/hooks/linear_sync.py archive + +Manual usage: + TASK_JSON_PATH=.trellis/tasks/<name>/task.json python3 .trellis/scripts/hooks/linear_sync.py sync + +Environment: + TASK_JSON_PATH - Absolute path to task.json (set by task.py) + +Configuration: + .trellis/hooks.local.json - Local config (gitignored), example: + { + "linear": { + "team": "TEAM_KEY", + "project": "Project Name", + "assignees": { + "dev-name": "linear-user-id" + } + } + } +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +# ─── Configuration ──────────────────────────────────────────────────────────── + +# Trellis priority → Linear priority (1=Urgent, 2=High, 3=Medium, 4=Low) +PRIORITY_MAP = {"P0": 1, "P1": 2, "P2": 3, "P3": 4} + +# Linear status names (must match your team's workflow) +STATUS_IN_PROGRESS = "In Progress" +STATUS_DONE = "Done" + + +def _load_config() -> dict: + """Load local hook config from .trellis/hooks.local.json.""" + task_json_path = os.environ.get("TASK_JSON_PATH", "") + if task_json_path: + # Walk up from task.json to find .trellis/ + trellis_dir = Path(task_json_path).parent.parent.parent + else: + trellis_dir = Path(".trellis") + + config_path = trellis_dir / "hooks.local.json" + try: + with open(config_path, encoding="utf-8") as f: + return json.load(f) + except (OSError, json.JSONDecodeError): + return {} + + +CONFIG = _load_config() +LINEAR_CFG = CONFIG.get("linear", {}) + +TEAM = LINEAR_CFG.get("team", "") +PROJECT = LINEAR_CFG.get("project", "") +ASSIGNEE_MAP = LINEAR_CFG.get("assignees", {}) + +# ─── Helpers ────────────────────────────────────────────────────────────────── + + +def _read_task() -> tuple[dict, str]: + path = os.environ.get("TASK_JSON_PATH", "") + if not path: + print("TASK_JSON_PATH not set", file=sys.stderr) + sys.exit(1) + with open(path, encoding="utf-8") as f: + return json.load(f), path + + +def _write_task(data: dict, path: str) -> None: + with open(path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + f.write("\n") + + +def _linearis(*args: str) -> dict | None: + result = subprocess.run( + ["linearis", *args], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + ) + if result.returncode != 0: + print(f"linearis error: {result.stderr.strip()}", file=sys.stderr) + sys.exit(1) + stdout = result.stdout.strip() + if stdout: + return json.loads(stdout) + return None + + +def _get_linear_issue(task: dict) -> str | None: + meta = task.get("meta") + if isinstance(meta, dict): + return meta.get("linear_issue") + return None + + +# ─── Actions ────────────────────────────────────────────────────────────────── + + +def cmd_create() -> None: + if not TEAM: + print("No linear.team configured in hooks.local.json", file=sys.stderr) + sys.exit(1) + + task, path = _read_task() + + # Skip if already linked + if _get_linear_issue(task): + print(f"Already linked: {_get_linear_issue(task)}") + return + + title = task.get("title") or task.get("name") or "Untitled" + args = ["issues", "create", title, "--team", TEAM] + + # Map priority + priority = PRIORITY_MAP.get(task.get("priority", ""), 0) + if priority: + args.extend(["-p", str(priority)]) + + # Set project + if PROJECT: + args.extend(["--project", PROJECT]) + + # Assign to Linear user + assignee = task.get("assignee", "") + linear_user_id = ASSIGNEE_MAP.get(assignee) + if linear_user_id: + args.extend(["--assignee", linear_user_id]) + + # Link to parent's Linear issue if available + parent_issue = _resolve_parent_linear_issue(task) + if parent_issue: + args.extend(["--parent-ticket", parent_issue]) + + result = _linearis(*args) + if result and "identifier" in result: + if not isinstance(task.get("meta"), dict): + task["meta"] = {} + task["meta"]["linear_issue"] = result["identifier"] + _write_task(task, path) + print(f"Created Linear issue: {result['identifier']}") + + +def cmd_start() -> None: + task, _ = _read_task() + issue = _get_linear_issue(task) + if not issue: + return + _linearis("issues", "update", issue, "-s", STATUS_IN_PROGRESS) + print(f"Updated {issue} -> {STATUS_IN_PROGRESS}") + cmd_sync() + + +def cmd_archive() -> None: + task, _ = _read_task() + issue = _get_linear_issue(task) + if not issue: + return + _linearis("issues", "update", issue, "-s", STATUS_DONE) + print(f"Updated {issue} -> {STATUS_DONE}") + + +def cmd_sync() -> None: + """Sync prd.md content to Linear issue description.""" + task, _ = _read_task() + issue = _get_linear_issue(task) + if not issue: + print("No linear_issue in meta, run create first", file=sys.stderr) + sys.exit(1) + + # Find prd.md next to task.json + task_json_path = os.environ.get("TASK_JSON_PATH", "") + prd_path = Path(task_json_path).parent / "prd.md" + if not prd_path.is_file(): + print(f"No prd.md found at {prd_path}", file=sys.stderr) + sys.exit(1) + + description = prd_path.read_text(encoding="utf-8").strip() + _linearis("issues", "update", issue, "-d", description) + print(f"Synced prd.md to {issue} description") + + +# ─── Parent Issue Resolution ───────────────────────────────────────────────── + + +def _resolve_parent_linear_issue(task: dict) -> str | None: + """Find parent task's Linear issue identifier.""" + parent_name = task.get("parent") + if not parent_name: + return None + + task_json_path = os.environ.get("TASK_JSON_PATH", "") + if not task_json_path: + return None + + current_task_dir = Path(task_json_path).parent + tasks_dir = current_task_dir.parent + parent_json = tasks_dir / parent_name / "task.json" + + if parent_json.exists(): + try: + with open(parent_json, encoding="utf-8") as f: + parent_task = json.load(f) + return _get_linear_issue(parent_task) + except (json.JSONDecodeError, OSError): + pass + return None + + +# ─── Main ───────────────────────────────────────────────────────────────────── + +if __name__ == "__main__": + action = sys.argv[1] if len(sys.argv) > 1 else "" + actions = { + "create": cmd_create, + "start": cmd_start, + "archive": cmd_archive, + "sync": cmd_sync, + } + fn = actions.get(action) + if fn: + fn() + else: + print(f"Unknown action: {action}", file=sys.stderr) + print(f"Valid actions: {', '.join(actions)}", file=sys.stderr) + sys.exit(1) diff --git a/.trellis/scripts/init_developer.py b/.trellis/scripts/init_developer.py new file mode 100755 index 0000000..9fb53f5 --- /dev/null +++ b/.trellis/scripts/init_developer.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +""" +Initialize developer for workflow. + +Usage: + python3 init_developer.py <developer-name> + +This creates: + - .trellis/.developer file with developer info + - .trellis/workspace/<name>/ directory structure +""" + +from __future__ import annotations + +import sys + +from common.paths import ( + DIR_WORKFLOW, + FILE_DEVELOPER, + get_developer, +) +from common.developer import init_developer + + +def main() -> None: + """CLI entry point.""" + if len(sys.argv) < 2: + print(f"Usage: {sys.argv[0]} <developer-name>") + print() + print("Example:") + print(f" {sys.argv[0]} john") + sys.exit(1) + + name = sys.argv[1] + + # Check if already initialized + existing = get_developer() + if existing: + print(f"Developer already initialized: {existing}") + print() + print(f"To reinitialize, remove {DIR_WORKFLOW}/{FILE_DEVELOPER} first") + sys.exit(0) + + if init_developer(name): + sys.exit(0) + else: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/.trellis/scripts/task.py b/.trellis/scripts/task.py new file mode 100755 index 0000000..dd7e71f --- /dev/null +++ b/.trellis/scripts/task.py @@ -0,0 +1,801 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Task Management Script. + +Usage: + python3 task.py create "<title>" --description "<desc>" [--slug <name>] [--assignee <dev>] [--priority P0|P1|P2|P3] [--parent <dir>] [--package <pkg>] [--no-start] [--force] + python3 task.py add-context <dir> <file> <path> [reason] # Add jsonl entry + python3 task.py validate <dir> # Validate jsonl files + python3 task.py list-context <dir> # List jsonl entries + python3 task.py start <dir> # Set active task, record current branch + python3 task.py current [--source] [--json] # Show active task + python3 task.py finish # Clear active task + python3 task.py set-branch <dir> <branch> # Set git branch + python3 task.py set-base-branch <dir> <branch> # Set PR target branch + python3 task.py set-scope <dir> <scope> # Set scope for PR title + python3 task.py set-meta <dir> <key> <value> # Set a task metadata key + python3 task.py rename <dir> <new-slug> [--dry-run] # Rename task + references + python3 task.py archive <task-dir> [--skip-branch-validation] # Archive completed task + python3 task.py list # List active tasks + python3 task.py list-archive [month] # List archived tasks + python3 task.py add-subtask <parent-dir> <child-dir> # Link child to parent + python3 task.py remove-subtask <parent-dir> <child-dir> # Unlink child from parent +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +from common.log import Colors, colored +from common.paths import ( + DEVELOPER_HINT, + DIR_WORKFLOW, + DIR_TASKS, + FILE_TASK_JSON, + get_repo_root, + get_developer, + get_tasks_dir, + get_current_task, +) +from common.active_task import ( + clear_active_task, + resolve_active_task, + resolve_context_key, + set_active_task, +) +from common.git import current_branch_name +from common.io import ( + describe_json_read_failure, + read_json_checked, + write_json, +) +from common.task_utils import resolve_task_dir, run_task_hooks +from common.tasks import iter_active_tasks, children_progress + +# Import command handlers from split modules (also re-exports for plan.py compatibility) +from common.task_store import ( + cmd_create, + cmd_rename, + cmd_archive, + cmd_set_branch, + cmd_set_base_branch, + cmd_set_scope, + cmd_set_meta, + cmd_add_subtask, + cmd_remove_subtask, +) +from common.task_context import ( + cmd_add_context, + cmd_validate, + cmd_list_context, + curated_entry_count, +) + + +# ============================================================================= +# Command: start / finish +# ============================================================================= + +def _record_start_state( + task_json_path: Path, + repo_root: Path, + label: str = "", +) -> None: + """Move a freshly started task to in_progress and record its branch. + + Both updates share one read/write: the status flip from planning, and the + checked-out branch when `branch` is still empty. Recording at start is what + keeps `branch` trustworthy at archive time — a task whose branch is only + ever set by hand tends to reach archive with `branch: null`. + + Tolerant on purpose — a broken task.json does not fail `start`, because the + session pointer is the point of the command. But the read overwrites the + file it just read, so no failure may be silent: without a message the + absent status line looks like the task simply was not in planning. + """ + data, reason = read_json_checked(task_json_path) + if data is None: + problem, hint = describe_json_read_failure(task_json_path, reason) + print( + colored(f"Warning: {problem}; task.json not updated.", Colors.YELLOW), + file=sys.stderr, + ) + print(hint, file=sys.stderr) + return + + applied: list[str] = [] + + if data.get("status") == "planning": + data["status"] = "in_progress" + applied.append(f"✓ Status: planning → in_progress{label}") + + # Only fill an empty field: an explicit `set-branch` must survive a later + # `start` (re-starting a task after a checkout is a normal thing to do). + base_branch_conflict: str | None = None + if not data.get("branch"): + branch = current_branch_name(repo_root) + if branch: + data["branch"] = branch + applied.append(f"✓ Branch recorded: {branch}{label}") + if branch == data.get("base_branch"): + base_branch_conflict = branch + else: + print( + colored( + "Note: no checked-out branch (detached HEAD, or not a git " + "repository); task branch not recorded.", + Colors.YELLOW, + ), + file=sys.stderr, + ) + + if not applied: + return + + if not write_json(task_json_path, data): + print( + colored( + f"Warning: Failed to write {task_json_path}; " + "status and branch are unchanged.", + Colors.YELLOW, + ), + file=sys.stderr, + ) + return + + for line in applied: + print(colored(line, Colors.GREEN)) + + if base_branch_conflict: + # Recorded anyway — the value is true, it just cannot describe a PR. + # Archive refuses this shape, so say so now rather than at the gate. + print( + colored( + f"Warning: '{base_branch_conflict}' is also this task's base_branch; " + "a PR cannot target its own branch, and archive will refuse it.", + Colors.YELLOW, + ), + file=sys.stderr, + ) + print( + f"Once you branch off, run: python3 {DIR_WORKFLOW}/scripts/task.py " + "set-branch <task> <feature-branch>", + file=sys.stderr, + ) + + +def cmd_start(args: argparse.Namespace) -> int: + """Set active task.""" + repo_root = get_repo_root() + task_input = args.dir + + if not task_input: + print(colored("Error: task directory or name required", Colors.RED)) + return 1 + + # Resolve task directory (supports task name, relative path, or absolute path) + full_path = resolve_task_dir(task_input, repo_root) + + if full_path is None: + # resolve_task_dir already named the exact reason on stderr. A second, + # generic line on stdout would split one diagnosis across two streams + # and bury the specific message. + return 1 + + if not full_path.is_dir(): + print(colored(f"Error: Task not found: {task_input}", Colors.RED)) + print("Hint: Use task name (e.g., 'my-task') or full path (e.g., '.trellis/tasks/01-31-my-task')") + return 1 + + # Context-manifest gate (#573): a seeded-but-uncurated implement/check + # manifest means every sub-agent dispatched for this task runs with zero + # spec context, and nothing downstream surfaces that to the main session. + # An absent manifest is not gated — create seeds the files only on + # sub-agent-capable platforms, so absence means no sub-agent reads them. + if not getattr(args, "allow_empty_context", False): + empty_manifests = [ + name + for name in ("implement.jsonl", "check.jsonl") + if curated_entry_count(full_path / name) == 0 + ] + if empty_manifests: + print(colored( + f"Error: {' and '.join(empty_manifests)} " + f"{'has' if len(empty_manifests) == 1 else 'have'} no curated entries", + Colors.RED, + )) + print("Sub-agents (implement/check) would run with zero spec context.") + print(f" Curate: python3 .trellis/scripts/task.py add-context {task_input} implement <path> \"<why>\"") + print(f" Verify: python3 .trellis/scripts/task.py validate {task_input}") + print(" Intentionally empty? Re-run start with --allow-empty-context") + return 1 + + # Convert to relative path for storage. repo_root is resolved because + # full_path already is (resolve_task_dir only returns paths inside the + # resolved root), so an unresolved repo_root would mismatch under a + # symlink (e.g. /tmp on macOS) and reject a perfectly normal task. + try: + task_dir = full_path.relative_to(repo_root.resolve()).as_posix() + except ValueError: + # resolve_task_dir already refused everything outside the repo, so + # this is unreachable in practice. Refuse rather than fall back to + # str(full_path) — that fallback (a lexical relative_to() paired with + # an absolute-path fallback) is exactly the pattern that let a `..` + # ref escape into storage before this fix. + print(colored(f"Error: Task not found: {task_input}", Colors.RED)) + print("Hint: Use task name (e.g., 'my-task') or full path (e.g., '.trellis/tasks/01-31-my-task')") + return 1 + + task_json_path = full_path / FILE_TASK_JSON + + if not resolve_context_key(): + # Degraded mode: no session identity available. + # Hook didn't inject TRELLIS_CONTEXT_ID (common on Windows + Claude Code, + # --continue resume path, fork distribution, hooks disabled, etc.). Skip + # per-session pointer write; AI continues based on conversation context. + print(colored( + "ℹ Session identity not available; active-task pointer not persisted " + "this session (degraded mode). AI continues based on conversation context.", + Colors.YELLOW, + )) + print(colored( + "Hint: run inside an AI IDE/session that exposes session identity, " + "or set TRELLIS_CONTEXT_ID before running task.py start.", + Colors.YELLOW, + )) + + # Still flip task.json status: planning → in_progress so downstream phases proceed. + if task_json_path.is_file(): + _record_start_state(task_json_path, repo_root, " (degraded)") + run_task_hooks("after_start", task_json_path, repo_root) + return 0 + + active = set_active_task(task_dir, repo_root) + if active: + print(colored(f"✓ Current task set to: {task_dir}", Colors.GREEN)) + print(f"Source: {active.source}") + + if task_json_path.is_file(): + _record_start_state(task_json_path, repo_root) + + print() + print(colored("The hook will now inject context from this task's jsonl files.", Colors.BLUE)) + + run_task_hooks("after_start", task_json_path, repo_root) + return 0 + else: + print(colored("Error: Failed to set current task", Colors.RED)) + return 1 + + +def cmd_finish(args: argparse.Namespace) -> int: + """Clear active task.""" + repo_root = get_repo_root() + active = clear_active_task(repo_root) + current = active.task_path + + if not current: + print(colored("No current task set", Colors.YELLOW)) + return 0 + + # Resolve task.json path before clearing + task_json_path = repo_root / current / FILE_TASK_JSON + + print(colored(f"✓ Cleared current task (was: {current})", Colors.GREEN)) + print(f"Source: {active.source}") + + if task_json_path.is_file(): + run_task_hooks("after_finish", task_json_path, repo_root) + return 0 + + +def cmd_current(args: argparse.Namespace) -> int: + """Show active task.""" + repo_root = get_repo_root() + active = resolve_active_task(repo_root) + + if getattr(args, "json", False): + task_obj = None + read_error = None + if active.task_path: + task_json_path = repo_root / active.task_path / FILE_TASK_JSON + data, reason = read_json_checked(task_json_path) + if data is None: + # Without this, a corrupt task.json emits null for every field + # — indistinguishable from a task whose fields really are null. + problem, hint = describe_json_read_failure(task_json_path, reason) + read_error = { + "file": str(task_json_path), + "reason": reason, + "message": f"{problem}. {hint}", + } + data = {} + task_obj = { + "dir": active.task_path, + "id": data.get("id") or data.get("name"), + "title": data.get("title"), + "status": data.get("status"), + "parent": data.get("parent"), + "children": data.get("children", []), + "branch": data.get("branch"), + "base_branch": data.get("base_branch"), + } + payload = { + "current_task": task_obj, + "source": active.source, + "stale": active.stale, + } + # Only present when the read failed, so the healthy shape is unchanged. + if read_error: + payload["error"] = read_error + print(json.dumps(payload, ensure_ascii=False)) + return 0 if active.task_path else 1 + + if args.source: + print(f"Current task: {active.task_path or '(none)'}") + print(f"Source: {active.source}") + if active.stale: + print("State: stale") + return 0 if active.task_path else 1 + + if active.task_path: + print(active.task_path) + return 0 + + return 1 + + +# ============================================================================= +# Command: list +# ============================================================================= + +def _display_status(t, all_statuses: dict) -> str: + """Return the status label to show for a task in `list` output. + + A parent task's stored status stays "planning" until someone runs + `task.py start` on the parent directly, even while its children are + actively being worked — a misleading label for anyone scanning the + list (#399 item 3). Show "active" instead when at least one child is + past planning; the stored status.json value is left untouched. + """ + if t.status == "planning" and t.children: + child_in_flight = any( + all_statuses.get(c) not in (None, "planning") for c in t.children + ) + if child_in_flight: + return "active" + return t.status + + +def cmd_list(args: argparse.Namespace) -> int: + """List active tasks.""" + repo_root = get_repo_root() + tasks_dir = get_tasks_dir(repo_root) + current_task = get_current_task(repo_root) + developer = get_developer(repo_root) + filter_mine = args.mine + filter_status = args.status + as_json = getattr(args, "json", False) + + # Single pass: collect all tasks via shared iterator + all_tasks = {t.dir_name: t for t in iter_active_tasks(tasks_dir)} + all_statuses = {name: t.status for name, t in all_tasks.items()} + + if as_json: + if filter_mine and not developer: + print( + json.dumps({"error": "No developer set", "hint": DEVELOPER_HINT}), + file=sys.stderr, + ) + return 1 + + items = [] + for dir_name in sorted(all_tasks.keys()): + t = all_tasks[dir_name] + if filter_mine and (t.assignee or "-") != developer: + continue + if filter_status and t.status != filter_status: + continue + items.append({ + "dir": f"{DIR_WORKFLOW}/{DIR_TASKS}/{dir_name}", + "id": t.raw.get("id") or dir_name, + "title": t.title, + "status": t.status, + "display_status": _display_status(t, all_statuses), + "priority": t.priority, + "assignee": t.assignee or None, + "parent": t.parent, + "children": list(t.children), + "package": t.package, + }) + print(json.dumps({"tasks": items}, ensure_ascii=False)) + return 0 + + if filter_mine: + if not developer: + print(colored("Error: No developer set. Run init_developer.py first", Colors.RED), file=sys.stderr) + print(DEVELOPER_HINT, file=sys.stderr) + return 1 + print(colored(f"My tasks (assignee: {developer}):", Colors.BLUE)) + else: + print(colored("All active tasks:", Colors.BLUE)) + print() + + # Display tasks hierarchically + count = 0 + + def _print_task(dir_name: str, indent: int = 0) -> None: + nonlocal count + t = all_tasks[dir_name] + + # Apply --mine filter + if filter_mine and (t.assignee or "-") != developer: + return + + # Apply --status filter + if filter_status and t.status != filter_status: + return + + relative_path = f"{DIR_WORKFLOW}/{DIR_TASKS}/{dir_name}" + marker = "" + if relative_path == current_task: + marker = f" {colored('<- current', Colors.GREEN)}" + + # Children progress + progress = children_progress(t.children, all_statuses) + status_label = _display_status(t, all_statuses) + + # Package tag + pkg_tag = f" @{t.package}" if t.package else "" + + prefix = " " * indent + " - " + + if filter_mine: + print(f"{prefix}{dir_name}/ ({status_label}){pkg_tag}{progress}{marker}") + else: + print(f"{prefix}{dir_name}/ ({status_label}){pkg_tag}{progress} [{colored(t.assignee or '-', Colors.CYAN)}]{marker}") + count += 1 + + # Print children indented + for child_name in t.children: + if child_name in all_tasks: + _print_task(child_name, indent + 1) + + # Display only top-level tasks: those without a parent, plus orphans + # whose recorded parent is not (or no longer) in the active set — a + # dangling parent ref must still render flat instead of disappearing. + for dir_name in sorted(all_tasks.keys()): + parent = all_tasks[dir_name].parent + if not parent or parent not in all_tasks: + _print_task(dir_name) + + if count == 0: + if filter_mine: + print(" (no tasks assigned to you)") + else: + print(" (no active tasks)") + + print() + print(f"Total: {count} task(s)") + return 0 + + +# ============================================================================= +# Command: list-archive +# ============================================================================= + +def cmd_list_archive(args: argparse.Namespace) -> int: + """List archived tasks.""" + repo_root = get_repo_root() + tasks_dir = get_tasks_dir(repo_root) + archive_dir = tasks_dir / "archive" + month = args.month + + print(colored("Archived tasks:", Colors.BLUE)) + print() + + if month: + month_dir = archive_dir / month + if month_dir.is_dir(): + print(f"[{month}]") + for d in sorted(month_dir.iterdir()): + if d.is_dir(): + print(f" - {d.name}/") + else: + print(f" No archives for {month}") + else: + if archive_dir.is_dir(): + for month_dir in sorted(archive_dir.iterdir()): + if month_dir.is_dir(): + month_name = month_dir.name + count = sum(1 for d in month_dir.iterdir() if d.is_dir()) + print(f"[{month_name}] - {count} task(s)") + + return 0 + + +# ============================================================================= +# Help +# ============================================================================= + +def show_usage() -> None: + """Show usage help.""" + print("""Task Management Script + +Usage: + python3 task.py create <title> --description <desc> Create new task directory (both required, non-empty) + python3 task.py create <title> --description <desc> --package <pkg> Create task for a specific package + python3 task.py create <title> --description <desc> --parent <dir> Create task as child of parent + python3 task.py create <title> --description <desc> --no-start Create without making it active in this session + python3 task.py add-context <dir> <jsonl> <path> [reason] Add entry to jsonl + python3 task.py validate <dir> Validate jsonl files + python3 task.py list-context <dir> List jsonl entries + python3 task.py start <dir> Set active task; records the checked-out branch when unset + python3 task.py current [--source] Show active task + python3 task.py finish Clear active task + python3 task.py set-branch <dir> <branch> Set git branch + python3 task.py set-base-branch <dir> <branch> Set PR target branch + python3 task.py set-scope <dir> <scope> Set scope for PR title + python3 task.py set-meta <dir> <key> <value> Set/overwrite a task metadata key + python3 task.py rename <dir> <new-slug> Rename task, identity fields and references + python3 task.py archive <task-dir> Archive completed task + python3 task.py add-subtask <parent> <child> Link child task to parent + python3 task.py remove-subtask <parent> <child> Unlink child from parent + python3 task.py list [--mine] [--status <status>] [--json] List tasks + python3 task.py list-archive [YYYY-MM] List archived tasks + +Monorepo options: + --package <pkg> Package name (validated against config.yaml packages) + +Rename options: + --dry-run Print the change set without writing anything + +Archive options: + --no-commit Skip the auto git commit after archiving + --skip-branch-validation Archive despite missing or self-referential branch metadata. + Archive normally refuses a task with no `branch` when it has a + `base_branch` and the repo has a remote, or with + `branch == base_branch`; repair those with `set-branch` / + `set-base-branch` instead. Use this flag only for tasks that + were never PR-backed. A recorded branch that was merged and + deleted is only a warning and needs no flag. + +List options: + --mine, -m Show only tasks assigned to current developer + --status, -s <s> Filter by status (planning, in_progress, review, completed) + --json Output machine-readable JSON (also available on `current`) + +Examples: + python3 task.py create "Add login feature" --description "Email + password sign-in" --slug add-login + python3 task.py create "Add login feature" --description "Email + password sign-in" --slug add-login --package cli + python3 task.py create "Add login feature" --description "Email + password sign-in" --meta linear=ENG-123 --meta epic=auth + python3 task.py create "Child task" --description "Session cookie handling" --slug child --parent .trellis/tasks/01-21-parent + python3 task.py add-context <dir> implement .trellis/spec/cli/backend/auth.md "Auth guidelines" + python3 task.py set-branch <dir> task/add-login + python3 task.py start .trellis/tasks/01-21-add-login + python3 task.py current --source + python3 task.py finish + python3 task.py rename add-login add-sso --dry-run # Preview the change set + python3 task.py rename add-login add-sso + python3 task.py archive add-login + python3 task.py archive add-login --skip-branch-validation # Task never had a branch of its own + python3 task.py add-subtask parent-task child-task # Link existing tasks + python3 task.py remove-subtask parent-task child-task + python3 task.py list # List all active tasks + python3 task.py list --mine # List my tasks only + python3 task.py list --mine --status in_progress # List my in-progress tasks +""") + + +# ============================================================================= +# Main Entry +# ============================================================================= + +def main() -> int: + """CLI entry point.""" + # Deprecation guard: `init-context` was removed in v0.5.0-beta.12. + # Detect early so argparse doesn't mask the real reason with a generic + # "invalid choice" error. + if len(sys.argv) >= 2 and sys.argv[1] == "init-context": + print( + colored( + "Error: `task.py init-context` was removed in v0.5.0-beta.12.", + Colors.RED, + ), + file=sys.stderr, + ) + print( + "implement.jsonl / check.jsonl are now seeded on `task.py create` for", + file=sys.stderr, + ) + print( + "sub-agent-capable platforms and curated by the AI during planning when needed.", + file=sys.stderr, + ) + print("See .trellis/workflow.md planning artifact guidance or run:", file=sys.stderr) + print( + " python3 ./.trellis/scripts/get_context.py --mode phase --step 1", + file=sys.stderr, + ) + print( + "Use `task.py add-context <dir> implement|check <path> <reason>` to append entries.", + file=sys.stderr, + ) + return 2 + + parser = argparse.ArgumentParser( + description="Task Management Script", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + subparsers = parser.add_subparsers(dest="command", help="Commands") + + # create + p_create = subparsers.add_parser("create", help="Create new task") + p_create.add_argument("title", help="Task title (required, non-empty)") + p_create.add_argument("--slug", "-s", help="Task slug without the MM-DD date prefix") + p_create.add_argument("--assignee", "-a", help="Assignee developer") + p_create.add_argument("--priority", "-p", default="P2", help="Priority (P0-P3)") + p_create.add_argument( + "--description", + "-d", + help="Task description (required, non-empty — an empty one is refused at archive)", + ) + p_create.add_argument("--parent", help="Parent task directory (establishes subtask link)") + p_create.add_argument("--package", help="Package name for monorepo projects") + p_create.add_argument( + "--base-branch", + help="PR target branch (overrides origin/HEAD detection and the checked-out-branch fallback)", + ) + p_create.add_argument( + "--meta", + action="append", + help="Task metadata key=value (repeatable)", + ) + p_create.add_argument( + "--no-start", + action="store_true", + help="Create the task without making it active in this session", + ) + p_create.add_argument( + "--force", + action="store_true", + help="Overwrite task.json when the task directory already exists", + ) + + # add-context + p_add = subparsers.add_parser("add-context", help="Add context entry") + p_add.add_argument("dir", help="Task directory") + p_add.add_argument("file", help="JSONL file (implement|check)") + p_add.add_argument("path", help="File path to add") + p_add.add_argument("reason", nargs="?", help="Reason for adding") + + # validate + p_validate = subparsers.add_parser("validate", help="Validate context files") + p_validate.add_argument("dir", help="Task directory") + + # list-context + p_listctx = subparsers.add_parser("list-context", help="List context entries") + p_listctx.add_argument("dir", help="Task directory") + + # start + p_start = subparsers.add_parser("start", help="Set active task") + p_start.add_argument("dir", help="Task directory") + p_start.add_argument( + "--allow-empty-context", + action="store_true", + help="Start even when implement.jsonl / check.jsonl have no curated entries", + ) + + # current + p_current = subparsers.add_parser("current", help="Show active task") + p_current.add_argument("--source", action="store_true", + help="Show active task source") + p_current.add_argument("--json", action="store_true", + help="Output machine-readable JSON") + + # finish + subparsers.add_parser("finish", help="Clear active task") + + # set-branch + p_branch = subparsers.add_parser("set-branch", help="Set git branch") + p_branch.add_argument("dir", help="Task directory") + p_branch.add_argument("branch", help="Branch name") + + # set-base-branch + p_base = subparsers.add_parser("set-base-branch", help="Set PR target branch") + p_base.add_argument("dir", help="Task directory") + p_base.add_argument("base_branch", help="Base branch name (PR target)") + + # set-scope + p_scope = subparsers.add_parser("set-scope", help="Set scope") + p_scope.add_argument("dir", help="Task directory") + p_scope.add_argument("scope", help="Scope name") + + # set-meta + p_setmeta = subparsers.add_parser("set-meta", help="Set/overwrite a task metadata key") + p_setmeta.add_argument("dir", help="Task directory") + p_setmeta.add_argument("key", help="Metadata key") + p_setmeta.add_argument("value", help="Metadata value") + + # rename + p_rename = subparsers.add_parser("rename", help="Rename task and its references") + p_rename.add_argument("name", help="Task directory or name") + p_rename.add_argument("new_slug", help="New slug without the MM-DD date prefix") + p_rename.add_argument( + "--dry-run", + action="store_true", + help="Print the change set without writing anything", + ) + + # archive + p_archive = subparsers.add_parser("archive", help="Archive task") + p_archive.add_argument("name", help="Task directory or name") + p_archive.add_argument("--no-commit", action="store_true", help="Skip auto git commit after archive") + p_archive.add_argument( + "--skip-branch-validation", + action="store_true", + help=( + "Archive even when branch metadata is missing or self-referential " + "(for tasks that were never PR-backed)" + ), + ) + + # list + p_list = subparsers.add_parser("list", help="List tasks") + p_list.add_argument("--mine", "-m", action="store_true", help="My tasks only") + p_list.add_argument("--status", "-s", help="Filter by status") + p_list.add_argument("--json", action="store_true", help="Output machine-readable JSON") + + # add-subtask + p_addsub = subparsers.add_parser("add-subtask", help="Link child task to parent") + p_addsub.add_argument("parent_dir", help="Parent task directory") + p_addsub.add_argument("child_dir", help="Child task directory") + + # remove-subtask + p_rmsub = subparsers.add_parser("remove-subtask", help="Unlink child task from parent") + p_rmsub.add_argument("parent_dir", help="Parent task directory") + p_rmsub.add_argument("child_dir", help="Child task directory") + + # list-archive + p_listarch = subparsers.add_parser("list-archive", help="List archived tasks") + p_listarch.add_argument("month", nargs="?", help="Month (YYYY-MM)") + + args = parser.parse_args() + + if not args.command: + show_usage() + return 1 + + commands = { + "create": cmd_create, + "add-context": cmd_add_context, + "validate": cmd_validate, + "list-context": cmd_list_context, + "start": cmd_start, + "current": cmd_current, + "finish": cmd_finish, + "set-branch": cmd_set_branch, + "set-base-branch": cmd_set_base_branch, + "set-scope": cmd_set_scope, + "set-meta": cmd_set_meta, + "rename": cmd_rename, + "archive": cmd_archive, + "add-subtask": cmd_add_subtask, + "remove-subtask": cmd_remove_subtask, + "list": cmd_list, + "list-archive": cmd_list_archive, + } + + if args.command in commands: + return commands[args.command](args) + else: + show_usage() + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.trellis/spec/backend/compiler-contracts.md b/.trellis/spec/backend/compiler-contracts.md new file mode 100644 index 0000000..b5f373f --- /dev/null +++ b/.trellis/spec/backend/compiler-contracts.md @@ -0,0 +1,194 @@ +# Compiler Contracts + +## 1. Scope / Trigger + +Applies to AST/HIR contract metadata, typed LLVM/resource/coroutine lowering, private fields, range/failure/effect analysis, structural warnings and public API baseline enforcement. Keep clause-only control flow and ordinary value semantics. Official sources receive the same hard checks; no blanket recovery annotation or bundled exception. + +## 2. Signatures + +- `llvm::emit_executable[_with_resources]` and `emit_test_executable` wrap the existing callable roots in C `main`. Process startup owns SIGPIPE initialization; callable roots preserve host policy. Success/failure paths release outputs, typed faults, rendered Text and diagnostic Buffer, then flush the existing logger. Test main validates one decimal index and uses the same index for execution and all fault/output owners. Private CLI defaults to these entries and never invokes a host compiler or Rust fallback. +- `llvm::emit_kernel(&Program, &SourceMap, entry) -> Result<String, String>` lowers typed ordinary function kernels and inferred-suspending coroutines, not the default application backend. It consumes only checked HIR and existing specialization/Domain metadata; no Rust source, generic Value or interpreter dispatch is emitted. Keep direct typed arguments, explicit output/fault pointers, clause projections, value-copy mutation, static handlers, source evaluation order, short-circuiting and checked numeric semantics. Fault origin and each caller reference the emitted source-span table; frame capacity is reachable specialization count plus one, so acyclic calls never truncate a trace. Collect nominal types from nested reachable expressions as well as locals/signatures. Validate and collect every reachable `specialize::failures` type and its recursive owned fields, including Port-declared errors unused in implementation bodies: fault layout/release still references them. Reject unsupported reachable types/effects/application roots at their source location before returning IR; never silently substitute a value or fallback backend. +- `llvm::emit_application(&Program, &SourceMap) -> Result<String, String>` emits callable compiler-owned CMD, HTTP/REST and Job roots. `dever_application_entry(Unit*, fault*) -> i32` uses existing output/fault release exports. CMD reads a qualified name and one JSON object, decodes checked inputs, invokes App/selected Dever or external Adapter and prints the existing success envelope; a mixed HTTP/CMD application must not bind HTTP when executing CMD. HTTP/REST uses static route/auth/permission descriptors and concrete callbacks, reusing the existing API, JWT/business verify, site, tenant, RBAC, Upload and log owners. No dynamic Value enters business code. Model/SQL/Related/migration/Seed/DbError, Job resources and hidden transactions share one invocation-owned application Session. `emit_application_with_resources` additionally consumes the packager's verified `EmbeddedResource` table; private CLI adds an executable process entry; managed Linux compilation rechecks portable sources in a trusted daemon worker. Formal target packs remain unfinished. +- LLVM external lowering reuses checked Port names, declared failures and wire field codecs. Decode selected Setting before starting a Worker. Named result/error objects require every top-level field, including nullable fields; unknown/missing fields, invalid types and undeclared error identities remain source-located runtime faults. Move fully decoded outputs or exact nominal Choice payloads into their owner once. JSON exists only at the process boundary. Embed immutable bytes and metadata with the native backend's shared digest/dedupe helpers. Production resources/startup never enter a fake-only Test graph. +- `llvm::emit_test_suite(&Program, &SourceMap) -> Result<String, String>` emits one module/executable with stable case indices from case-local checked `test_program` metadata. It exports `dever_test_count`, `dever_test_entry`, `dever_test_fault_size`, `dever_test_fault_release`, `dever_test_fault_render` and `dever_test_text_release`. Allocate the selected fault size with eight-byte alignment; render/release with the same index. Cases retain their own effects, fakes, faults, private symbols, database and virtual clock. Only declared internal LLVM symbol/type tokens are qualified; never rewrite strings, comments or shared ABI declarations. Production routes/auth are removed and unreachable Jobs pruned before lowering; reachable Job payloads still use the shared typed wire contract. No deployment config, real Lib or service startup is allowed in ordinary tests. +- LLVM application Dever Port dispatch reuses checked production Adapter candidates and `Settings` selection/validation. Decode only the selected typed Setting; a single unconfigured Adapter without Model needs no settings file. Concrete selection/Setting/database binding slots are module-private. A nonblocking atomic entry gate rejects overlapping/reentrant roots that own Port, database, application or clock slots. The root Scope drains before same-runtime Session close, selected Setting drop, slot reset and gate release, including partial initialization and faults. Child Tasks read immutable selected slots but detach transaction context; they neither reload config nor own independent application roots. Writing CMD begins after input decode and commits only after output encoding, before stdout. Nested/sequential calls share one transaction without savepoints; cancelled unconfirmed operations discard the connection. Database ABI/storage/error contracts and live acceptance are owned by `database-guidelines.md` §9, not a second ORM implementation. +- LLVM inferred-suspending functions are `ptr @f(...typed inputs, ptr out, ptr fault, ptr state) presplitcoroutine`; synchronous functions remain `i32 @f(...typed inputs, ptr out, ptr fault)`. Both share clause/binding/statement emission, TypeOwner guards and specialized typed invocation. Coroutine/runtime descriptors and fixed ABI signatures live in `toolchain-and-library.md` and the canonical C header. +- Application source identity is `module/<component>/<domain>/<role>.dever[.md]` or a cohesive role topic file; role is app/domain/model/port/adapter/api/job. Generated API/CMD/Job roots replace application `main.dever`. Model/API root files may coexist with topic directories; other roles keep root/directory exclusivity and non-API topics are flat. Reject generic common/shared/utils/helper/base/service buckets. Application source has no package/exposes/import declarations; `public` is limited to an individual HTTP declaration. +- App functions use logical `<component>.<domain>.<function>` names; role/topic never enter the call. Main calls App; API calls its own App; App calls App or same-domain Domain/Port; Domain calls same-domain Domain; Adapter calls only same-file helpers and standard capabilities. Model operations remain owned by their domain App. API handlers and qualified Adapter/fake implementations are not source-callable. +- Port operations are one bodyless ordinary signature with typed value inputs, unbounded outputs and required `fails app.ErrorChoice` (or public standard error choice). The choice contains an explicit error variant and belongs to the same-domain public App contract. Port/Adapter may reference the App types used by this contract, never call App. Each production Adapter completely implements one Port identity; a test file can fake multiple Ports. Input names/types and ordered output names/types agree exactly. Every implementation failure set is a subset of the declaration. +- An external Adapter is `external <exec|pip|npm|go> "relative/entry" { ... }` in an Adapter-role file. Body clauses are newline-separated `lib "name@version"` and `allow <capability>`; `lib` derives its ecosystem from the declaration, while exec uses explicit ecosystem-prefixed Lib specs. It may coexist with one `setting` record, but not with Dever functions. The compiler synthesizes every operation of the domain's single Port identity; App code still calls the Port and never receives Worker, process or JSON values. `ExternalWorkerContract::sdk_manifest()` exposes the same checked setting/input/output/error wire DAG and fixed identity for SDK generation, never a separately maintained type registry. +- App parameter, output and inferred failure types are public contract. Reject an App capability that leaks a private Domain/Adapter type; define externally visible business error choices in App and let same-domain private roles refer to them through `app.<Type>`. +- Fields: `private name: Type` for records and Models; ordinary record privacy is package-owned, Model privacy belongs only to the owning domain App. `name: Int >= low and <= high` (also Decimal) applies to record/payload/output/handler fields. +- Choice: `error Failed(message: Text)`; markers are explicit, never inferred from variant names. +- `fail(Error.Variant(...))` terminates a clause; ordinary calls propagate inferred errors. `result(named_call(...))` requires a choice with one success variant matching the call outputs and an exact recoverable error set. +- Functions: `name(inputs) (outputs) pure recover("reason")`; modifiers optional, identical across all clauses of one signature. Source has no `async` modifier or `await(...)` form. +- Transaction functions use `transaction name(inputs) (outputs)` and infer a single physical database effect through their complete call graph; source never receives a transaction handle. Sequential `blocking` is allowed only through its existing synchronous/no-database/no-concurrency contract; detached, parallel, stream and channel work remain forbidden. +- Model files synthesize opaque `model.id`, generated `id/created_at`, a record Page type and static short CRUD symbols. Anonymous Model blocks are contextual syntax, not general Map values. +- `Program::api_snapshot() -> String`, `check_api(&str) -> Result<(), String>`, `application_warnings(entry) -> Result<Vec<Diagnostic>, String>`. +- Application tests use `test/<component>/<domain>/<topic>.dever[.md]`; the filename selects the one ordinary zero-input, zero-output `<topic>() ()` entry. Test helpers/types remain private to that file. Tests call same-domain App/Domain and cross-domain App only; they cannot call Model operations, Port, Adapter, API or another test, and production cannot call tests. +- Test assertions are compiler-owned `assert(Bool)` and `assert_eq(actual, expected)`. Equality requires the same comparable type; only `null`/empty expressions that already require context receive the left type, so numeric promotion does not make unequal source types valid. Business failures continue through `result(call)` and explicit recovery. +- CLI: `check <root>`, `test <root>`, `run <root> [--tenant <id>] [-- <cmd> <json>]`, `build <root> --output <new-file>`, `tenant migrate <root> <id>`, `tenant owner <root> <id> <site> <user-id>` and `tenant component enable|disable <root> <id> <component>`. Run/build collect checked API/CMD/Job declarations into one generated entry and reject application `main.dever`. Test checks `module/ + test/` before running stable-sorted cases; check/run/build load only module/. Deployment and database settings come only from `config/setting.json`. +- Module API files contain declarations such as `get profile = app.profile`, `post publish = app.publish`, `cmd bootstrap = app.bootstrap` and `rest model`; they do not contain handler function bodies. HTTP routes inherit the App signature and return the fixed envelope. `public` is valid only on one HTTP declaration. Recursive `api/<site-path>/<topic>.dever` supplies configured site ownership without source `site` statements. +- Authentication providers, sites, external HTTP origins, Cookie names/lifetimes and tenant routing come only from `config/setting.json`. Verify is one read-only App function from `Claims` to an App-owned record with exactly `id: Text`, `user_id: ModelId?`, and `tenant_id: ModelId?`; it cannot return permissions. A protected site's generated entry verifies its configured provider, enters trusted tenant scope, checks every reachable tenant component and authorizes the exact permission before reading business input or dispatching App. A `public` declaration ignores an ambient browser Cookie, but an explicitly supplied invalid `Authorization` credential remains an authentication error. + +- Trusted `auth.user_id()` / `tenant_id()` keep their default `Int?` result. An explicit nullable ModelId context selects the same identity intrinsic with that nominal type; absence remains nullable and arbitrary Int casts remain forbidden. Configured validation walks every protected API's reachable functions and each generated REST field binding, matching the selected ID against its actual verify identity field; `owns_user(ModelId)` follows the same user-ID rule. Source-only checking defers provider matching, never configured builds. Native wraps the existing nullable i64; LLVM retains its nullable ModelId ABI. Public/CMD/Job/Test capability rules still apply. Ordinary tests must reject reachable identity effects instead of synthesizing a trusted user. + +## 3. Contracts + +The current LLVM kernel accepts ordinary checked functions with supported value types, including Decimal, Text/Id, Bytes standard operations, List/Map and nested Record/Choice/Nullable, File/Stream, Task/Group/Channel, AsyncStream, TCP Socket/Listener and HTTP/TLS/WebSocket resources. Time/calendar, process arguments, stdout/logging, Uuid and Secret/crypto use the same typed runtime bridge. List/Bytes each/filter/find/reduce/reduce_until/sum and Stream/AsyncStream traversal use static typed handlers, including the existing checked suspending-handler contracts, never a generic Value interpreter. `dever_entry(ptr out, ptr fault) -> i32` initializes both slots, returns 0 on success and 1 on checked runtime or business failure; outputs are read only after success. Fault is `{i32 code, i32 origin, i32 depth, [capacity x i32] caller_locations, {ptr, i64} message, i32 business_type, i32 business_variant, [words x i64] business_payload}` with separate source-span metadata. Keep fields 0..4 stable; the aligned payload holds the complete concrete Choice and is sized for the largest reachable error type. Named fault codes (overflow=1, division-by-zero=2, RuntimeAbi=3, Business=4) are not the runtime C ABI status. RuntimeAbi retains original owned diagnostic bytes; Business retains nominal identity (`type_id+1`, 0 means absent), variant and typed payload. Callers release root outputs and both fault payloads through `dever_outputs_release` and `dever_fault_release`, including zero-initialized outputs after failure. Application entries and unsupported application intrinsics remain source-located rejections. + +File create uses create_new and never truncates an existing path; clone/close preserve shared owner semantics. File chunks yield a shared synchronous Stream cursor, with one terminal Failed before End. Source traversal uses only its existing each/reduce/reduce_until contracts; stream_pull is internal ABI, not a new source pull/first(Stream). Preserve aliases and unvisited rows on early stop, and release resources on both typed business and runtime faults. + +AsyncStream constructors use List/Bytes/Channel/ticks or the existing TCP producer; aliases share progress and closure. Register its consumption-close guard only after source, initial state and context expressions finish evaluating. Early stop, normal return, fault and cancellation close the shared producer; unlike synchronous Stream, another alias cannot continue. Release pending operations before closing the producer. Terminal ReadEvent/ConnectResult Failed remains one actual typed row before End. TCP operations reuse runtime net deadlines, direction locks and partial-write cancellation rather than blocking wrappers. Map async statuses explicitly before decoding standard resource choices. + +HTTP values cross the ABI as fixed named fields plus opaque resource handles; clone/register every owned field before fallible header conversion. Static sync/suspending handlers and optional context share the existing coroutine descriptors; borrowed request/context inputs become owned before initial suspend. Handler faults retain typed ownership until terminal request-boundary rendering, log and produce 500/abort as the existing runtime requires, then permit later requests. Do not turn a handler fault into a server-wide fault or serialize intermediate values through JSON. Uuid is an actual UUID owner, not Text; Secret stays non-observable. Password operations preserve checked blocking boundaries. Reachable Log/HTTP-server roots flush after scope drain on success and failure; pure kernels do not initialize a logger. + +Timeout consumes a Task and uses an independent presence flag with the full output row: successful null and zero-output success are not timeouts. Only an actual timeout invokes its statically specialized fallback after runtime cleanup; numeric, business and RuntimeAbi faults propagate with their original kind and spans. Race uses canonical output descriptors shared by same checked signatures, consumes tasks only after non-null operation creation and drains all losers on every completion path. ParallelEach uses the existing scoped concurrent, bounded blocking and async stream owners for each checked sequence/handler combination; a plain Group loop is not equivalent to capacity-before-pull and failure observation while a producer is idle. + +Coroutines clone borrowed inputs before initial suspend; all live guards, operation owners, values, output slots and typed faults survive real Pending. Initial/pending destroy runs reverse cleanup once; normal completion runs cleanup then complete/final suspend, and final destroy frees only the frame. Generated wrappers reuse the existing single task Scope/supervisor and bounded blocking pool, never per-function block_on. Validate cross-thread Send before returning IR against checked inputs/outputs, inferred error-variant fields and all reachable suspending frames including expression temporaries. Task/Group affine locals are frame-safe, not source-transferable. Non-suspending blocking functions may create/drop a Stream locally without transferring it. Internal Unit/Outputs ABI rows derive transferability field-wise (Unit=true); never copy their source `Type::transferable=false` property or change the language property to fix ABI metadata. + +LLVM `fail` transfers the concrete Choice into the business tail and disarms its temporary owner; normal propagation appends the call's source frame. `result(named_call)` shares ordinary invocation and uses only `specialize::failures` plus `capture::targets`, including handler specialization. Success packs zero/one/multiple ordered outputs into the checked success variant. A matching business failure transfers directly or wraps one error-only Choice according to that mapping, then clears the entire fault; explicitly raised outer variants must not be unwrapped. Numeric and RuntimeAbi faults remain uncaptured and propagate with their original origin and caller frames. Root fault release statically dispatches nominal error types through the existing typed drop owner, including nonzero variants. Do not add runtime generic Value, parser-based error matching or a parallel failure registry. + +BytesToText/BytesFromInts/BytesSlice use the existing runtime handle ABI: status 0 transfers a success value; only status 1 creates the standard Failed(Text) from the original diagnostic Buffer, freeing that Buffer after Text construction; status 2 stays RuntimeAbi. Standard Bytes wrappers then use the same `fail`/capture channel, not a separate error-return shortcut. Never inspect a success output on a failing status. + +LLVM ordinary function inputs are borrowed; clause bindings, locals and temporaries own typed copies. Track ownership with entry-initialized guards, recursively retain/drop only present Nullable payloads and active Choice variants, and release on normal and fault exits. Success transfers outputs and disarms the corresponding guards. Reuse native last-use/field-update analyses rather than duplicating them; a final field move clears that field while preserving its containing record's owner. An ABI `_take` consumes its receiver even on failure, so disarm before calling it. First/Get allocate output storage for the actual List element or Map value type, not just the inner type of the nullable result. The ABI's outer presence flag only determines whether this row may be read: an already-nullable row is transferred intact, not wrapped again or overwritten with Some. + +Reuse clause interval normalization and facts/domain containment at every constrained construction, write, output and handler boundary. Concrete handlers may establish outputs by inference; forwarded signatures require input contravariance and output covariance. Unknown operations establish no new bounds. + +Facts preserve value snapshots and invalidate the modified path. Use shared immutable field graphs and bounded proof work, not recursive copying of a type/value DAG. `contracts::MAX_PROOF_WORK` is 16,384; exceeding proof work reports C015 and never grants a guarantee. Before analysis, nominal type depths are computed iteratively and must not exceed the shared 128-level syntax depth limit. Failure-free nominal type graphs are memoized and skipped by failure expansion. + +Failure origins survive aliases, aggregates and projections. An unknown List may be empty and Nullable may be null, so possible error payloads do not establish guaranteed failure. A short-circuited handler or empty traversal cannot consume a pre-existing obligation. Empty reduce returns the original state's obligation unchanged. Replacing a Map entry cannot retain the replaced failure's identity in the new Map. Partial selection is conservative about the unselected remainder. + +A capture error variant may contain one error-only choice payload, e.g. `error Database(error: dever.database.Error)`. It wraps that choice's inferred failures without renaming them or parsing messages. Explicitly raised outer variants retain their nominal identity. Success shapes, complete error sets and non-overlapping wrappers are checked before emission; captured failures still carry the ordinary handling obligation. Native, LLVM and reference execution share the same capture mapping, including specialized handler error sets. + +Effects and effective recovery reasons propagate through the existing call/handler substitution flow. An unused handler argument adds no effective recovery. Recovery is not an external effect and is allowed in pure functions, but its reason belongs in reachable public API summaries even when declared in a private helper. Keep private helper names and private record representation out of snapshots. + +Port remains a Function ID with compiler-owned successors. Shared dependencies, effect worklists and specialization include every production implementation; failure inference seeds the declared error set, independent of selection. Resolve the complete signature type graph before type-shape access checks, including App DTOs nested in Port-local DTOs; unrelated App types remain inaccessible to Port/Adapter. Unmediated file/network I/O propagates through functions and static handlers and is cleared only at the Port boundary. App/Domain reject it; only the compiler-generated application root may start API/Job service loops. Other effects retain existing policy. Production summaries exclude Test bodies. Checking and suite emission reuse one case-specific closed graph with that case's fakes, including failure-subset checks for unused fake operations and declaration checks for uninstantiated handler helpers. Each case gets an independent native module and sync/async entry; one rustc invocation still produces one suite binary. Never union different cases' fake effects or run a synchronous case under another case's async runtime. Production settings and fallback implementations are absent from the reachable suite. + +An Adapter-role external declaration synthesizes exactly one complete implementation of the domain's single Port identity. Its schema hash is derived deterministically from the optional setting wire schema and every operation's ordered input/output wire shape, including top-level field names and declared business error payload fields. The synthetic functions enter the existing dependency, effect, failure and specialization graphs; do not add a parallel external-call registry. Only fixed `network`, `file`, `process` and `gpu` capabilities are accepted, and they seed conservative effects. An external declaration may coexist only with one setting record and never with Dever implementation/helper functions. Test graphs replace production implementations with case-local fakes and therefore never start an external Worker. Ecological declarations do not weaken these checks or imply their target packager is implemented. + +Database effects carry a typed `DatabaseOwner::Model` or `DatabaseOwner::Job`, resolved centrally by Program to connection plus global/tenant scope. A transaction must reach at least one database operation (including enqueue), may resolve to only one physical database and cannot mix global/tenant scopes. It cannot start detached, parallel, streaming or channel work; a validated sequential `BlockingCall` does not escape the transaction. Ordinary database helpers accept a generated hidden transaction context so nested transaction calls and enqueue reuse the outer physical transaction. Never synthesize a Model to represent a queue. + +Job declares an explicit database and a private `job name(payload?) () retry(n) timeout(ms)` entry. One input must be a same-domain App record; no outputs, overloading, multiple clauses, pure/recover/fails or direct non-App calls. App enqueue uses a checked static same-domain target and lowers to a dedicated HIR node. Enqueue propagates only its queue connection/failures, not handler effects, failure or synchronous recursion. Native execution reachability adds handlers as independent roots; application test closures apply the case's Port fakes to those roots too. UTC cron checking and runtime scheduling share one bounded parser. + +Every Job enqueue requires a compiler/runtime-established execution identity. Generated CMD, cron and Test roots establish `System`; a protected API establishes `User` only after provider verification, component checks and automatic authorization. Missing identity never falls back to `System`, and a `public` API call graph that can enqueue a Job is rejected. User enqueue persists a re-verifiable provider/site/subject/session/tenant reference plus the one exact permission key of its HTTP entry; it never persists credentials, roles or a permission collection. Every attempt invokes verify again, validates the restored tenant, confirms the key still belongs to the current compiled permission catalog, checks reachable component state and performs the current role query in that order before opening the business transaction. Chained enqueue inherits the restored identity and permission key. + +Protected custom API permissions are `component.domain.site.action`; REST reserves `read/create/replace/delete`. The HTTP method is catalog metadata, not part of the key. Two declarations cannot own the same key even if their method or URL differs. `public` is the only anonymous form and emits no permission; its complete reachable graph cannot touch a tenant component. The service atomically synchronizes compiled permissions into the global control catalog and marks removed rows inactive. Site and tenant are hidden context for role management; source never writes `auth.require`, an `access` clause or a permission list in Identity. + +Suspension is an internal monotone effect inferred from async runtime intrinsics, ordinary calls, bound static handlers and collection handlers. HIR records `suspends`; API snapshots do not expose it. Direct source calls preserve sequential completion, while `run` is the explicit concurrency boundary and `wait` consumes Task/Group values. A fully non-suspending entry emits no Tokio runtime. + +Facts, effects and failures share `contracts/dependencies.rs` for function/static-handler dependencies and initial order. Effects/failures enqueue reverse callers only when a summary changes; conservative dependency cycles still converge without discarding obligations. Type comparison/move/transfer properties are computed once per nominal definition after type-cycle validation, avoiding repeated expansion of shared type graphs. + +Domain dependency edges come only from resolved type, function and handler references. Physical role files in one domain share one dependency boundary; cross-domain cycles remain invalid and function recursion remains independently invalid. Source code does not duplicate edges in a header, and dependency inference does not install or download packages. + +API snapshots sort semantic names and contracts, not source paths/IDs. A baseline is optional, explicit and create-new; checks never update it. Protect its edits outside the language. CLI suppresses bundled-source suggestions only; Program retains diagnostics and bundled hard errors remain visible. Entry-specific export advice must not infer unknown library consumers. + +Test authority comes from compiler-assigned source origin, never a user-controlled path string. A checked `TestCase` holds the private function ID and database-effect metadata; native compilation accepts only that checked identity and does not relax normal entry visibility. Each case runs serially in its own native process and temporary project. Successful child output stays captured; failed stdout/stderr is reported in labelled sections, later tests continue, and any failed case makes the command fail. Tests cannot reach `dever.api.serve()`. + +For `module/<component>/<domain>/api.dever`, the route is `/<component>/<domain>/<action>`; recursive `api/<scopes>/<file>.dever` inserts scopes and file before action. `get/post/put/delete` bind same-domain App functions and inherit their wire-safe input/output shapes; GET/DELETE read query, POST/PUT read a unique-key JSON object. `rest` generates Model CRUD and can use checked owner/create/replace/search field bindings. Each successful request receives JSON `{code:0,message:"ok",data:<value>}` with HTTP 200; standard API errors map to fixed 400/401/403/404/409/429 envelopes and unknown failures become a redacted 500. A top-level POST `Upload` switches the statically known route to bounded multipart parsing and affine ownership. No environment-variable config or implicit database fallback is introduced. + +Response Header/Cookie mutations are staged in request-local pending metadata. Generated dispatch serializes the response value, commits the database transaction, and only then promotes pending metadata for the server response; serialization, business, commit or rollback failure discards it. Cookie issue/clear uses the configured name, lifetime and attributes. Credential-changing requests require Origin and Host to exactly match the site's configured external HTTP/HTTPS scheme, host and effective port; a missing or cross-origin value is rejected without trusting forwarding headers. A write API call graph containing password verification or hashing does not receive an entry-wide transaction. Every reachable write must instead cross an explicit short `transaction` function, and password crypto is forbidden inside that transaction. Other write entries retain their generated transaction. + +## 4. Validation & Error Matrix + +`wire.rs` is the sole Type-to-wire eligibility owner; API checking and `native/wire.rs` consume its interned DAG. Policy (Output, ApiInput, SettingInput, Job) participates in the versioned fingerprint and specialization cache. Records reject private or bounded fields until concrete bound validation exists; Choice, Map, Related and resources are rejected. ModelId encodes as i64 while retaining nominal Model name in the fingerprint, never unstable compiler IDs. Secret is decode-only for ApiInput/SettingInput. `Program::wire_schema` exposes checked nominal schemas; no consumer may add a parallel Type allow-list. Concrete helpers retain field order and map missing nullable fields to None, while unknown/missing required fields fail. Persisted Model fingerprints never include wire schemas. + +Sensitivity belongs to the cached type graph in `types.rs`: Secret and private Model data propagate through nominal records/choices and every container, including Related association cycles via a monotone reverse-edge worklist. Ordinary record private fields retain their existing package access contract. App inputs reject private Models; App/entry outputs and error payloads reject both sensitive categories. API JSON and assertion rendering reuse the same observable property. Generate Render only for observable nominal types; Debug for runtime Secret is always redacted. Test-only `secret(Text)` is resolved from compiler-assigned source origin and reserved against user declarations. + +| Condition | Result | +| --- | --- | +| Reachable LLVM kernel type/effect/application root is not implemented | Source-located Err before IR is returned; no Rust fallback or successful stub | +| Dynamic LLVM Int add/sub/mul/neg overflow or div/rem invalid operation | Entry status 1 with original source-span index and every caller frame; never unchecked poison/trap arithmetic | +| Runtime value error during a managed LLVM call other than a recoverable Bytes operation | RuntimeAbi fault with the original diagnostic Buffer and source/caller spans; clean all guards and release message through the caller boundary | +| Explicit business fail or its ordinary-call propagation | Business fault with concrete nominal type/variant/payload and original source/caller spans; release through the caller boundary unless captured | +| Result matches a declared business failure | Transfer direct/wrapped typed payload and clear fault before any following call; do not swallow numeric/RuntimeAbi faults | +| Bytes conversion/range value error; invalid ABI | Value error becomes the standard Failed(Text); invalid ABI remains an uncaptured RuntimeAbi fault | +| File I/O value error; negative time/task sleep | Standard original Failed(Text); only the operation's recoverable status is captured | +| Pending, optional receive End, or Channel element null | Pending initializes no row/fault; receive reads a row only when present; preserve the element's full Nullable state | +| Async initial/pending/final destroy | Drop live guards/operation owners once at initial/pending; final destroy frees frame only after normal cleanup | +| Non-Send task/blocking frame or inferred failure payload | Source-located lowering Err before IR; no unsafe Send shortcut or erased fault | +| First/Get has no row, or finds a row whose value is null | No row: return zero Nullable without reading untouched output; nullable row: preserve its own null flag and full concrete layout | +| Invalid role layout, generic bucket, file/directory conflict or one-file role directory | C003 | +| `package`/`exposes`/`public` application syntax or forbidden role call | P001 / C006 | +| App signature or propagated failure exposes a private type | C006 | +| Duplicate API method/path | C002 | +| API non-prefixed/unsupported input or output | C005 | +| API missing/duplicate/extra or malformed request inputs | JSON 400, code=400 | +| Unknown route / unsupported method on known route / handler failure | JSON 404 / 405 (Allow header) / 500 | +| Unproved record/payload/write/output/handler bound | C011 | +| Discarded, overwritten or merely logged failure | C012 | +| Capture error set mismatch or overlapping nested error wrappers | C012 | +| Effect in pure function or inconsistent clause contract | C013 | +| Non-transferable value live across an inferred suspension point | C005 | +| Blocking or parallel effect executed inline on a suspending path, including a concretely bound handler | C005 | +| Invalid Model or transaction database contract | C014 | +| Contract analysis exceeds bounded work | C015, no executable Program | +| Proven redundant check / duplicate private implementation / repeated pure call / trivial private forwarding | W002 / W003 / W004 / W005 | +| Unused or overwritten write | Existing W001, no duplicate W006 | +| Unneeded exposure for a selected application | W007, not a hard error | +| API drift or unreadable baseline | CLI failure before native compilation | +| Existing snapshot output | CLI failure, existing bytes preserved | +| Invalid test path, missing/overloaded/non-ordinary entry or duplicate `.dever`/`.dever.md` identity | C002/C003/C005 before any test runs | +| Test crosses a private role, calls Model/API/another test, or production calls a test | C006 before native compilation | +| `assert` receives non-Bool or `assert_eq` receives unequal/non-comparable types | C005 at the assertion | +| Test assertion, uncaptured business error or runtime fault | Current case fails; later cases run; final status is nonzero | +| `external` outside Adapter, mixed with a Dever function, or unable to select exactly one domain Port identity | P001/C005 before native generation | +| External entry is absolute, escapes with `.`/`..`, contains empty/Windows-only segments, or names an unknown/duplicate capability | C002/C005 before native generation | +| External input/output/error payload is not eligible for the shared wire policy | C005 before native generation | +| `public` API can reach Job enqueue, or enqueue has no trusted execution identity | C006 / runtime rejection before persistence; never synthesize System | +| `public` API reaches a tenant component | C014 before native generation | +| `public` API reaches authorization administration, or GET reaches role mutation | C006 / C013 before native generation | +| Duplicate `component.domain.site.action` across methods or paths | C002 before native generation | +| Protected API has no current role grant | JSON 403 before business input/App dispatch | +| User Job provider/site/reference, compiled permission, component or current role fails | Block with a redacted identity/component category; do not enter the business transaction | +| Credential-changing request has missing or non-matching Origin | JSON authentication error; no Cookie/Header is committed | +| Response serialization, business transaction or commit fails after metadata was staged | Discard staged response metadata | +| Password crypto API reaches a write outside an explicit transaction, or a transaction reaches password crypto | C014 before native generation | + +## 5. Good / Base / Bad Cases + +LLVM base: lower checked Int 40+2, emit and link real objects, execute on the native host. Good: a nested nominal value that exists only inside a comparison still declares all required LLVM types; a short-circuited failing RHS never runs; managed aliases survive COW updates and fault cleanup. First/Get/Find on List<Text?> or Map<Text,Text?> preserve null and present values without double wrapping. List/Map Float values containing NaN remain non-reflexive under equality. Resource/async good: a second create_new fails without overwriting distinct content; a large owned multi-output survives Pending; stopping pending tasks drains descendants and releases frames; internal Unit and Outputs rows transfer through typed sync Task/blocking descriptors. Bad: consider IR substring or hand-written coroutine smoke sufficient checked-source evidence, clone parameters only after initial resume, omit temporary nominal/Send types, shrink a nullable row output slot or treat foreign-target object generation as successful execution. + +Base: direct `Port { value = 8080 }` satisfies `Int >= 1 and <= 65535`; a project without `test/` reports zero tests. Good: `test/user/account/register.dever` calls `app.register`, asserts returned values and uses `result(call)` for expected business failures. A protected `post publish = app.publish` automatically owns `news.article.admin.publish`; an enqueued User Job stores that one key and rechecks the current catalog and role on every attempt. A login reads credentials and verifies the password outside a transaction, then calls one short transaction that rechecks mutable account/tenant/membership state and creates the session. An external Adapter declares one owned entry and the minimum fixed capability set while its tests replace the production Worker with case-local Port fakes. Bad: call `model.create` or an API handler from a test, share test helpers across files, expose test entries, configure databases through an environment variable, enqueue from `public`, put permissions in Identity, write `auth.require`, copy a token/role/permission snapshot into Job storage, let password work overlap a database transaction, guess that every public POST NotFound means Unauthorized, write response Cookie/Header before commit, or let an external declaration coexist with helper functions. + +## 6. Required Tests + +LLVM resource/async changes select `llvm_resource_native`, `llvm_async_native` and `llvm_async_source` plus existing managed/result/kernel regressions. Use explicitly built runtime-only archives, actual checked source -> object -> archive execution and an atomic allocator ledger (one warm-up + 64 repeated calls). Cover create_new's second creation failing within the same run and verify distinct existing content; aliases, EOF/terminal error, early shared cursor stop, each business failure and runtime unwind. Async cases assert real Pending, initial/final destroy, large typed Text/List/Map/record and multi-output transfer, async static handlers, sync Task and Group, bounded backpressure, nullable Channel row/End, wait/stop/nested descendants, typed business failure/capture/refail, division code2 with exact origin and runtime ABI code3 with original bytes. Six-target object generation is not six-OS execution; bounded allocation balance is not RSS/performance acceptance. + +AsyncStream/TCP/composition changes additionally select `llvm_stream_network_source`: nullable rows, Channel backpressure, ticks and alias closure, async/sync handlers and all ParallelEach dispatch categories, owned multi-output/nullable/Unit timeout and race, original business/numeric/runtime faults, TCP short reads/EOF/shared close, exact creation errors and one terminal failure. An owned loopback cancellation fixture observes peer EOF after timeout returns to prove descendant Socket release. Keep the hand-written C smoke's invalid status assertion separate: async invalid input is 4, not typed-fault 2. Repeat actual native allocations through the same atomic ledger; do not claim performance or six-platform execution. + +LLVM business/Bytes changes select `llvm_lowering` plus opt-in `llvm_result_native`, reusing the runtime-linked runner in `test/dever-tests/tests/support/llvm_managed.rs`. Assert independent expected success/failure branches (especially zero-output capture), ordered multi-output payloads, direct/wrapped/explicit-outer identity, specialized each/reduce failures, capture-success-recapture, nested owned COW, root ownership transfer and nonzero-variant root release. Root faults must retain exact original source/caller spans. Dynamic division and duplicate Map-key diagnostics must remain uncaptured. Bytes cases cover Unicode, empty buffers, aliases, invalid UTF-8, -1/256 byte values and invalid/reversed ranges with original messages. Repeat native ownership checks through the same bounded archive/allocator fixture as managed tests; no recursive archive build or default database/service checks. + +LLVM changes select root `llvm_lowering` and embedded-bridge `llvm_native`: real checked sources to IR/object/LLD/native execution, static handlers and aligned nested Record/Choice/Nullable, multiple outputs, short-circuit faulting RHS, checked Int extrema, IEEE Float and original fault/caller spans. Managed changes also select `llvm_managed_native` with embedded/runtime-abi: Unicode and Decimal, collection COW/order/key types, nullable First/Get/Find, Unit handlers, owned reduce state/early stop, nested field replacement, original runtime diagnostic and failure/root-output cleanup. Its opt-in native runs link the explicitly prepared runtime-only archive, initialize outputs through the real entry and check allocation balance across 64 repetitions after warmup; this is ownership regression evidence, not throughput/RSS/64-or-128-MiB acceptance. Simple kernels link on all six targets; complex/managed aggregates emit on all six but only native host execution is evidence of execution. Windows CRT/stack-probe dependencies require a real target runtime pack, not fixture symbols. Runtime ABI changes select root `native_runtime_abi` plus the explicitly prepared `native_runtime_link` C/LLVM acceptance fixture. Prepare only an offline `runtime-abi` staticlib; author C/Rust tools are not product runtime dependencies or formal release assets. Tests remain under root `test/` and run no database or service by default. + +Select `structured_concurrency`, `async_native`, `contract_syntax`, `contract_facts`, `contract_semantics`, `contract_api`, `contract_redundancy`, `contract_execution`, `error_effects` according to changes. Assert suspension propagation through ordinary calls and concrete handler substitution, non-transferable liveness, worker boundaries, API omission, alias mutation, handler variance, short-circuit/empty traversal, overwrite/projection obligations, bounded shared graphs and unchanged native results. Error capture tests cover native/reference wrapping, explicit outer identity, ambiguity and discarded captures. Parser tests must reject the removed package-header dependency clause; formatter round trips must prove it cannot be emitted. Do not run the entire suite or service tests by default. + +For this source/API contract select `source_architecture`, `source_visibility`, `contract_execution`, `markdown_source`, `api_declarations`, `rest_model`, `native_entry`, `api_routes`, `auth_runtime`, `authorization_runtime`, `secure_contracts`, `tenant_database`, `durable_jobs` and `upload_boundary` as needed. Assert App aliases, topic hiding, role call direction, Model ownership, layout rejection, generated roots, strict inputs, JSON envelopes, Markdown parity, site/auth ownership, exact permission keys, duplicate-key rejection, public-to-tenant rejection, REST row scope and Upload cleanup. Authentication coverage includes public Cookie isolation, explicit invalid Authorization, exact Origin, staged metadata ordering, generated System roots, site-scoped multi-role union, Owner isolation and User Job current-role checks. Actual loopback HTTP verification requires an explicitly scoped service check. + +Application-test changes select `application_testing`, `formatter` and `cms_project`. Assert stable discovery, plain/Markdown duplicates, filename entry validation, both visibility directions, assertion typing/rendered source locations, existing result-choice recovery, failure continuation/output capture, zero-test success, per-case SQLite Seed isolation, ignored deployment settings, production commands ignoring test sources and module/test formatting preflight. + +External Adapter changes select `port_adapter`, `external_component` and `application_testing`. Assert Adapter-only contextual syntax and formatting, setting/top-level wire names in the schema hash, complete Port effect/failure propagation, reference rejection, case-local fakes, strict framed messages and request IDs, business-error decoding, bounded cancel/timeout/crash/restart/shutdown, queued-call admission closure, explicit child reap and native end-to-end dispatch. + +`external command` reuses the external Port boundary but not the Worker protocol. Require exactly one operation with named `args: List<Text>` and optional second `stdin: Bytes?`, one named `output: dever.process.Output`, and `fails dever.process.Error`; reject setting/lib declarations. The checked `Command` wire policy alone admits Bytes as strict padded base64; API, Job and protocol Worker policies retain their previous allowed types. HIR Worker contract/entry queries exclude commands, while `external_command_contracts()` shares their contract construction. Both typed backends use the same external async ABI and Bytes wire bridge. Test plain/Markdown formatting, wrong signatures, fake isolation and six-target object emission separately from actual runtime execution. + +## 7. Wrong vs Correct + +LLVM application wrong: emit an unbound auth descriptor, accept a nonpublic route without its exact permission, clone a borrowed handle but return the original SSA value, or give DELETE's encoded `null` two owners. Correct: validate auth metadata at source and again at the private ABI; return the actual guard-held clone before releasing its source; register one REST response owner through encoding/commit and transfer it once. Channel/Task/Group wrapper clones can change address even though Text Arc retain does not. + +LLVM request scopes must receive one boxed compiled-handler Future. Inline embedding the same large Future in every auth/tenant/job task-local branch amplifies debug poll stacks and caused a real nonpublic HTTP SIGSEGV. Keep one request-level indirection with the original scope/cancellation ownership; do not enlarge thread stacks, relax the allocation oracle or box every database operation. Permanent probes are `llvm_api_source` (private REST including DELETE, concurrent identities, rollback and 64 repeated roots) and `llvm_async_source::cloned_channel_fields_survive_their_source_scope`. + +LLVM wrong: collect types only from signatures/locals or assume every object needs no platform runtime. Correct: visit all reachable nested expressions, verify actual typed IR and native behavior, and distinguish cross-target object/link evidence from formal target runtime and OS execution acceptance. + +Managed LLVM wrong: bit-copy an owned handle, overwrite an owned field without dropping it, disarm a whole record when moving one field, or use the nullable result payload as First/Get's physical row slot. Correct: use typed lifecycle callbacks/guards, reuse final-read analyses, transfer only the selected owner, and allocate the full concrete row before interpreting outer presence. Preserve internal null rather than constructing Some(null). + +LLVM Result wrong: stringify business payloads, remap errors by variant name/message, catch numeric faults, leave a captured fault live, or accept either success or failure in every test expectation. Correct: share checked capture targets, preserve typed nominal payload and exact source frames, transfer then clear business ownership, propagate non-business faults, and assert the expected branch for each input. + +LLVM async wrong: use source Unit/Outputs transferability for internal ABI rows, clone arguments only on first resume, mark Send from parameters alone, drop PreparedSync input inside invoke, or use a RuntimeAbi-string oracle for division. Correct: field-wise ABI metadata, initial-frame ownership, transitive suspend-frame checks including temporaries, runtime-owned synchronous input cleanup, and exact fault-kind/source oracles. Resource tests must make erroneous overwrite observable with different contents and a mutually exclusive result. + +LLVM protocol wrong: round-trip source records through JSON, render a sensitive failure variant, make one failed request stop the server, or call all six targets runtime-tested after generating objects. Correct: fixed field ABI, typed resource/handler ownership, checked observable terminal errors, original request-boundary behavior, independent wire peers and explicit host-execution evidence. + +Wrong: preserve application package/import/main syntax, accept arbitrary run/build entries, call private roles across domains, make topic filenames part of App calls, turn API declarations into handler functions, trust client tenant/user fields, infer System from an absent identity, put permissions in verify Identity, write manual permission strings, persist a credential/role/permission snapshot, publish pending response metadata, or count a possibly-empty List as proof of returned failure. + +Correct: derive one source-role identity from each path, publish only logical App capabilities, keep API declarations and private roles narrow, generate application roots and exact permissions from checked API/CMD/Job contracts, establish System only at compiler-owned roots, and re-verify persisted User references plus the one entry permission on every Job attempt. Preserve conservative HIR facts and require explicit recovery where the program intentionally degrades. Derive Test identity from a compiler-owned source root, retain private checked entries and reuse the normal HIR/native/error pipeline; expose response metadata only after serialization and transaction commit. + +External wrong: expose JSON/process handles to App, hand-write a second operation registry, omit setting or top-level wire names from schema identity, or start the production Worker from a Test. External correct: synthesize the complete implementation from one checked Port, reuse shared wire/effect/failure/specialization owners, and bind tests only to case-local Dever fakes. diff --git a/.trellis/spec/backend/database-guidelines.md b/.trellis/spec/backend/database-guidelines.md new file mode 100644 index 0000000..0c4bb52 --- /dev/null +++ b/.trellis/spec/backend/database-guidelines.md @@ -0,0 +1,235 @@ +# Database Guidelines + +## 1. Scope + +Dever owns a static ORM. Application code declares persistence in `module/<component>/<domain>/model.dever` or, for multiple cohesive models, `model/<topic>.dever`; Markdown uses the same identity and schema rules. It does not add CRUD App/API wrappers or pass database/transaction handles. SQLite and PostgreSQL consume the same checked `ModelSchema`, `QueryPlan`, bind values, row shapes and error propagation. + +## 2. Model Contract + +- A role-file Model has one automatically visible record matching the domain in UpperCamelCase; a topic Model record matches its topic. Source does not write `public`. +- `id` and `created_at` are synthesized, generated and cannot be assigned by source code. +- Persistent fields use lower_snake_case and the checked types `Bool/Int/Float/Decimal/Text/Bytes/Uuid/DateTime/Date/Time/Duration/Json`, Model choices or `<model-package>.id`. +- Persistent metadata includes `generated`, constant `default`, `index`, `unique` and `from <old_field>`. REST-only `owner`, `create = expr`, `replace = expr` and `search = expr` are checked API contracts and do not alter ordinary ORM semantics or storage fingerprints. +- `private` is language visibility metadata: only the owning domain App accesses that field, including generated create/update blocks. It does not enter ModelSchema, SQL column order or migration fingerprints. Private Model snapshots and containers cannot cross App contracts; expose a View. Secret is never a persistent type; password hashes use private Text with enough room for PHC encoding. +- Default and Seed literals share storage validation: integer/date values fit i64 without a fractional part, Float is finite, Decimal satisfies exact precision/scale and Text lengths count Unicode scalars. Invalid values fail checking rather than Rust emission or database startup. +- Composite `index(...)`, `unique(...)`, reverse `relation`, one `seed` block and stable named `migrate` blocks live in the Model file. +- Model IDs are opaque nominal values. A foreign Model ID is not interchangeable with `Int` or another Model ID. + +## 3. Runtime And Query Contract + +Use compiler-synthesized short Model functions: `create/create_many/get/first/list/cursor/count/exists/stream/update/delete/upsert`. Only the owning domain App may call them, as `model.<operation>` or `model.<topic>.<operation>`. Main, API and other domains call the owning App instead. Anonymous create/update/query blocks are contextual syntax and never become runtime maps or reusable dynamic query objects. + +`where` builds a static condition tree; values are always bind parameters. Equality on nullable values is null-safe (`IS`/`IS NOT` on SQLite, `IS [NOT] DISTINCT FROM` on PostgreSQL). `order` is static: an omitted order defaults to `id DESC`, while an explicit order receives a stable `id` tie-breaker when needed. `list` defaults to page 1 and size 20, and runtime rejects a size above `database.<name>.max_page_size`. Do not add an Active Record type, chainable query object, reflection decoder or duplicated backend CRUD implementation. + +`cursor` uses the same ordered Model fields as a static composite keyset. Its nullable `next` value is the last returned Model snapshot and is passed back as nullable `after`; the compiler binds the ordered field values in declaration order and rejects nullable cursor-order fields. SQLite uses `IS` and PostgreSQL uses `IS NOT DISTINCT FROM` for equal key prefixes. The nullable guard must cast the first PostgreSQL placeholder to the ordered field's SQL type before `IS NULL`; an untyped `$n IS NULL` leaves PostgreSQL unable to infer the parameter type. `stream` remains the existing `AsyncStream<Model>` source surface and uses a bounded runtime producer. Its database worker is registered in the current structured task scope; full consumption, `close`, drop, cancellation and decoder/driver failure all release or discard the owned connection according to its known state. + +Connection selection order is explicit `database <name>`, root package name, then required `database.default`. `config/setting.json` is strict and external to the binary. SQLite paths remain below the executable directory; invalid or missing settings fail startup. + +Repository PostgreSQL acceptance tests and performance tools use the repository-root `config/setting.json` entry `database.postgres_test`. They do not accept connection URLs or TLS modes from environment variables or command-line flags. The local file is ignored by Git; tests stage only the selected connection as `database.default` beside their owned executable. + +`test/dever-tests/tests/support/postgres.rs` owns both schema-level and physical-tenant-database fixtures. The latter requires an explicitly isolated test connection with `{case}` in the database name and permission to create/drop test databases. It creates one random control database and a fresh tenant prefix, rejects a pre-existing namespace, and only drops that exact control name plus the generated prefix with canonical positive tenant IDs. Cleanup failures fail acceptance, including when the test itself panics. No configured URL means no connection; these real-service tests remain ignored by default. + +`postgres_api` runs stock CMS from a test-owned project copy and validates Owner HTTP requests plus two physical tenant databases. Its explicit test-only member fixture also exercises real non-Owner sessions, two admin roles, a same-named front role and immediate grant/revoke effects; never change reserved Owner mutability or production CMS to manufacture this evidence. Permission catalogs are site-filtered: look up each site's keys through that site's authenticated API. GET/DELETE arguments use query, not JSON bodies. Its `cms.py postgres-case` helper stages its own temporary copy: private ports, secrets, config changes and compiled artifacts must never modify the supplied source project, and persistent reports must omit credentials. A standalone Owner-only run without `--rbac-fixture` still does not prove non-Owner authorization. + +The PostgreSQL driver receives named `ConnectionOptions`, shared by initial and tenant connections. These are internal constructor arguments, not an additional configuration layer or environment-variable path. + +Application tests never read or copy the project's deployment database settings. A test with reachable Model effects receives a runner-generated `config/setting.json` in its per-case temporary project, containing SQLite `database.default` plus SQLite entries for every explicit Model connection. It uses the normal settings validation, complete schema initialization, migrations, Seed, transactions and shutdown. A suite containing only tests without Model effects uses the base runtime. A mixed suite links the SQLite-capable runtime once, but a selected non-database case receives no generated setting and does not load settings or initialize Models. Neither path accepts database environment variables, PostgreSQL URLs or writes project `data/`. + +Global Models resolve their connection once during startup into a `OnceLock<Database>`. When top-level `tenant.database` is configured, ordinary Models resolve through the hidden trusted tenant scope to a bounded, idle-evicted physical pool; there is no mode switch and they never fall back to the global connection. SQLite and PostgreSQL are independent optional runtime features; native build profiles contain only configured drivers. + +`global type` marks platform data. With top-level tenant settings, every other persistent Model and its Job queue is tenant scoped. SQLite derives a private database file below `tenant_directory`; PostgreSQL derives a database name from `tenant_database_prefix` and the positive internal tenant id. Only explicit `dever tenant migrate <project-root> <id>` creates/migrates storage and records matching local/control readiness markers. For PostgreSQL that command connects through the configured control URL, creates the derived database when absent, treats SQLSTATE `42P04` as the idempotent already-exists case, and then migrates it; ordinary requests/workers never create or fall back to another database. Requests, Jobs and tenant CMD execution require a ready marker and current schema fingerprint. Table/field modes are not accepted until their entire query, relation, unique, migration and native-SQL contracts exist. + +`tenant` accepts only `database`, `max_pools` and `idle_timeout_ms`; `database` selects the control connection. The control database owns `_dever_tenant_storage`, `_dever_tenant_component` and the global `_dever_permission` catalog. Tenant components are the sorted component names owning a tenant Model or tenant Job; platform/global-only and unknown names cannot be toggled. A mixed component is wholly tenant scoped: every Job it owns uses tenant storage even when that Job's own body touches only global data. After migration, `dever tenant component disable|enable <root> <id> <component>` writes only the control table. API, CMD and Job check all tenant components in their reachable execution graph before business work. A `public` API reaching any tenant component is rejected statically. + +Authorization role storage is framework-private: `_dever_auth_role`, `_dever_auth_role_permission` and `_dever_auth_user_role`, versioned by `_dever_auth_version`. Platform identities use the control database; tenant identities use their tenant database. Role identity and both relation tables use the composite `(site, role_id)` boundary, so sites may reuse role names without collision and grants cannot cross sites. The v1 single-ID schema migrates transactionally by copying each relation's site from its role before replacing the three tables; an incomplete or unknown private schema fails startup. Disabled roles do not authorize, multiple roles form a union and an `all_permissions` role covers only its site. `_dever_owner:<site>` is reserved and can be provisioned only by `dever tenant owner` after tenant migration; ordinary save, grant, revoke and disable operations must reject it. Applications may expose thin App/API adapters over the core role operations but must not create duplicate business permission tables. + +SQLite uses `deadpool-sqlite 0.13` with bundled SQLite, WAL, foreign keys, normal synchronous mode, a busy timeout and a bounded pool. PostgreSQL uses `deadpool-postgres 0.14.2` with `tokio-postgres 0.7.18`, cached static statements and rustls system roots; `min_connections` is prepared before Model initialization. Pool create/recycle and PostgreSQL socket connection have fixed runtime bounds. `wait_timeout_ms` bounds pool wait, while independent `io_timeout_ms` bounds SQL, transaction and migration I/O; both accept 1–300000 milliseconds. + +PostgreSQL parameter types must match the runtime's binary bind codec. Dever `Int` binds as i64 and therefore requires a `BIGINT` context in typed native SQL; `Uuid` binds as the native PostgreSQL UUID type and generated placeholders use `$n::uuid`, never `$n::text::uuid`. Casts resolve PostgreSQL parameter types and must not change the runtime value representation. + +Model operations, row-stream pulls and transaction begin/commit propagate `dever.database.Error` categories (Pool, PoolExhausted, Connection, Timeout, Cancelled, Database, Constraint, NotFound, InvalidData, Migration). Generated code converts runtime ErrorKind directly to source variants with message payloads; it never flattens recoverable errors into Fault or guesses their category from text. A service capture can use `type ReadResult { Found(value: User) error Failed(error: dever.database.Error) }`. The inferred set conservatively includes every database category. Startup failures remain process failures before application code begins. + +## 4. Validation And Error Matrix + +| Condition | Required result | +| --- | --- | +| Missing/invalid `setting.json`, unresolved database, conflicting tables or cross-database FK | Startup or compile failure before serving work | +| Transaction has no database effect, spans physical connections/scopes, or starts escaping concurrent work | Compile failure; validated sequential `blocking` remains allowed | +| Bound handler specializes to database work | Specialized effect set and hidden transaction context are retained | +| `run`/thread/parallel/HTTP callback performs database work | Detached context; it cannot inherit an ambient transaction | +| Operation, `BEGIN`, migration or transaction finish is cancelled before confirmed completion | Detach the pool object; never recycle an unknown connection state | +| Actual catalog differs from recorded catalog | Migration failure before schema mutation | +| Constraint tightening conflicts with stored rows | Database error and full transaction rollback | +| Business body fails and rollback also fails | Preserve the business error and append rollback failure as a cause | +| Missing get row or duplicate unique key | Recoverable NotFound / Constraint, including through task wait and transaction rollback | +| PostgreSQL cursor nullable guard leaves `$n` untyped | Generate `CAST($n AS <field SQL type>) IS NULL` | +| PostgreSQL reports admin/crash shutdown, cannot-connect-now or database-dropped SQLSTATE | Classify as a connection error so pool recovery semantics apply | +| Typed native SQL uses an `Int` parameter where PostgreSQL infers int4 | Give the placeholder a `BIGINT` context; do not change the runtime codec | +| Job claim/renew/complete waits for a database write lock | Read the injected Clock only after the lock is owned; never extend or complete an already expired lease with a cached timestamp | +| SQLite `job.lease_ms <= Job timeout_ms` | Fail startup before listen/claim; a persisted incompatible timeout becomes `blocked/invalid_policy` before App dispatch | +| An expired Job already consumed `max_attempts` | Move it to dead with bounded cleanup; it must not occupy an execution slot or hide a later eligible Job | +| Concurrent enqueue repeats an active `(target, key)` | Return the same Job Id from the atomic upsert; never create a second pending/running row | +| Job enqueue has no trusted execution identity | Reject before persistence; absence never means System | +| User Job identity cannot be re-verified or changes tenant | Block with a redacted identity category before business dispatch | +| User Job permission is absent from the current compiled catalog or current role grant | Block as `identity_rejected` before business dispatch | +| A reachable tenant component is disabled | API 403, CMD failure or Job `blocked/component_disabled`; no business dispatch | +| Component name is global-only or absent from the compiled tenant manifest | Reject the control operation; do not create an arbitrary row | +| Legacy active Job lacks identity or crosses the v4 permission-contract upgrade | Block it as `missing_execution_identity` / `authorization_contract_changed`; never upgrade it to System | +| PostgreSQL tenant database is absent during explicit tenant migration | Create the derived database through the configured control connection, then migrate; concurrent already-exists is success | + +## 5. Required Patterns + +Declare a business function with `transaction name(...) (...)`. The compiler propagates its database effects through ordinary calls and specialized handler bindings, then validates the concrete call graph. The generated hidden context owns one pool object at the outer boundary; nested transaction calls reuse it without savepoints. + +Success commits. Error/fault rolls back before propagation. A rollback failure is appended as a cause and does not replace the original error. Commit and rollback consume the transaction, immediately returning a one-connection SQLite pool object after completion. SQLite write transactions use `BEGIN IMMEDIATE`. An API call graph containing password verification or hashing receives no entry-wide transaction; every write in that graph must be owned by an explicit short transaction reached after the expensive work, and that transaction must recheck the mutable state on which the write depends. + +Use one checked query planner and shared value model for both database executors. Backend-specific code is limited to SQL dialect, bind/row codecs, pool operations and catalog migration. Keep business transactions in the owning domain App; Model CRUD remains compiler-generated. + +Typed Model SQL may return its private result record or an explicit owning-domain App View (`app.ArticleView`, including App topic files), as one row, optional row or list. Reuse ordinary App alias resolution and existing scalar row decoding. App Views must contain only visible supported storage scalar fields; Secret, nested Model/record values and cross-domain or Domain-owned result records are rejected. All regular SQL result records reject field bounds: row decoding does not prove source range refinements. This supports bounded list projections without exporting private Model snapshots or duplicating View types; existing App/API sensitivity checks remain mandatory. + +GET may reach typed SQL only when both dialects are proved to be a complete single-own-table field projection: SELECT fields FROM own_table, optional WHERE field = numbered-bind joined by AND, optional ORDER BY own fields, optional LIMIT numbered-bind. `check/sql_read.rs` owns this conservative proof. Unknown syntax, functions, CTEs, subqueries, other tables or extra statements retain the existing write effect; a SELECT prefix alone never establishes safety. Test both dialects independently and preserve POST SQL write behavior. + +## 6. Schema, Migration, Seed And Tests + +Every database has `_dever_history` rows for schema, named migrations and Seed revisions. Initialization takes a backend lock and updates schema, indexes, Seed and history in one transaction. + +- A missing table is created directly at the final schema; old `from` and drop history are not replayed. +- The normalized actual catalog must match recorded history before any update. SQLite fingerprints table DDL and managed index DDL. PostgreSQL fingerprints columns, types, nullability, identity/default metadata, constraints and managed indexes. +- `from` maps one old field to one new field. Name similarity is never inferred. +- Removed fields require a new named `migrate ... { drop field }`; applied migrations cannot be removed or modified. +- Named migrations may contain repeated `before` / `after` blocks with explicit `sqlite`, `postgres` and literal `parameters`. Reuse checked placeholder scanning and existing bind codecs; accept one INSERT/UPDATE/DELETE only, reject reserved internal identifiers, DDL and transaction control. Unclosed quoted regions are diagnostics, including Unicode input. Context-free parameters support Bool/Int/Text/null and finite exponent-form Float; do not silently reinterpret Decimal as Float. +- PostgreSQL dollar-quote recognition must respect identifier boundaries and validate both delimiter and body termination. Do not eagerly construct an EOF slice inside `then_some`; malformed SQL must produce C014, never panic. The same scanner serves typed SQL and migration SQL. +- For an existing table, validate history/order/catalog first, run pending before blocks in source order, apply automatic DDL, then pending after blocks, Seed and history in the same Model transaction. Fresh databases skip old data transforms and record the final migrations as applied. SQL revisions cover phase, source order, both dialects and parameter types/values. Keep drop-only revision calculation stable. Applied migration order is immutable. +- The migration boundary is one Model on one database. Data transforms may reference tables already present at that point; do not promise cross-Model atomicity or arbitrary table initialization order. +- Text/Bytes constraint changes and nullable/unique/FK/choice tightening rely on database validation and roll back without truncating or inventing values. +- Cross-family or lossy type changes are rejected. The currently automatic widening is `Int -> Float`; Text and Bytes bound changes stay within their storage family. +- SQLite rebuilds the table transactionally when fields change and synchronizes managed indexes directly for index-only changes. +- PostgreSQL uses transactional `ALTER TABLE`, named `_dever_` CHECK/FK constraints, advisory locks and managed index replacement. Fresh schemas create tables first, then add foreign keys in a second phase so source order and cyclic references do not break initialization. +- Seed requires a covered unique key and uses parameterized `ON CONFLICT DO NOTHING`. It inserts missing initialization rows and never overwrites existing business data. + +Date/time values use integer milliseconds: DateTime is Unix epoch milliseconds, Date is UTC midnight epoch milliseconds, Time is milliseconds since midnight in [0, 86400000), and Duration is signed milliseconds. External wire strings do not change the storage codec, logical Model type or schema fingerprint. JSON uses checked text. Decimal uses a fixed-width sortable text key on SQLite and native `NUMERIC(P,S)` on PostgreSQL. UUID uses a 16-byte SQLite BLOB and native PostgreSQL `UUID`; generated UUID fields receive UUIDv7 values from the runtime. + +Long-term tests live under repository-root `test/`. Focused verification must cover database-effect propagation through helpers, nested transactions and handler specialization; rejection of concurrent transaction work; nullable equality with both present and absent values; typed cursor placeholder casts; Decimal/UUID round trips; cancellation isolation; migration catalog drift; settings and Model syntax. PostgreSQL catalog, migration, TLS, native codecs, disconnect classification and cancellation require an explicitly supplied isolated test database; never probe or mutate a local/development PostgreSQL service by assumption. + +Durable Job changes select `durable_jobs`. Assert atomic business-write/enqueue commit and rollback, active-key concurrency, claim-time attempt, fresh fencing tokens, old-token CAS rejection, expired-lease recovery, bounded exhausted-row cleanup, blocked target/schema/payload/policy/identity/component, cursor rollback/restart and completed-occurrence dedupe. Cover System/User persistence, missing-identity rejection, v2 identity migration, v4 User authorization-contract blocking, physical/execution tenant dedupe isolation, chained inheritance and generated User re-verification/exact-permission checks. The SQLite regression must hold `BEGIN IMMEDIATE` across a lease renewal window and advance the injected Clock while claim/renew/complete waits, proving timestamps are sampled after lock ownership. Mode/signal tests use only test-owned processes, loopback listeners and temporary SQLite databases. + +`model_syntax` covers invalid default/Seed values; `markdown_source` covers Model metadata and schema parity; `sqlite_orm` verifies captured error identity and zero persisted rows after rollback. `orm_data_migration` covers bound SQL, revisions, catalog checks, source order, once-only execution, fresh databases, rollback and native upgrades. `contract_execution` compares both CMS source declarations, Model/API snapshots and settings; `cms_project` checks both source formats and isolated packaged executables. Native row decoding must return its ORM Result inside its own closure; a bare generated block lets `?`/`return` escape into the outer application error type, breaking `first()`. + +## Durable Job Store + +`job/store.rs` owns versioned private `_dever_jobs`/cursor schema, separate from Model history. Use the existing Database/Executor/Transaction and pool. Private schema v5 stores physical `tenant_id`, execution `scope_tenant_id`, `execution_kind`, provider/site/subject/session/tenant references and one `auth_permission` key inherited from the protected HTTP entry. It never stores a token, Cookie, Header, role or permission collection. SQLite initialization/claims use the ordinary write transaction; PostgreSQL schema installation uses a transaction advisory lock and claims use bounded `FOR UPDATE SKIP LOCKED` rows. + +An active partial unique index is authoritative for physical tenant + execution tenant + target + business key. Atomic upsert with RETURNING retains the active row lock until commit and avoids an insert/select completion race. Ambient enqueue shares the caller's physical transaction; standalone enqueue owns a short transaction. Every claim increments attempt and creates a fresh token. Renew and every completion require id/token/running/unexpired lease. Unknown target, wire fingerprint mismatch and decode failure become blocked with fixed redacted categories; retry and timeout use bounded backoff, then dead. + +CMD, cron and compiler-owned Test entries explicitly establish System identity. Protected API and restored User Jobs establish User identity from verified Claims; nested enqueue inherits it. On every User attempt generated dispatch selects the configured provider/site verify App, reconstructs the trusted Claims reference, validates the returned subject and tenant, confirms the persisted key still belongs to the compiled site catalog, checks current tenant components and performs the latest role query, and only then begins the business transaction. Invalid identity is blocked; transient verify database failures retain normal retry behavior. A missing task-local identity is always an enqueue error. + +Private migration v2/v3 preserves the existing fail-closed identity behavior. Migration v4 to v5 adds `auth_permission`, blocks every active User row as `authorization_contract_changed`, and drops the obsolete `auth_capabilities` column because a previous capability list cannot be guessed into one entry permission. Active System rows and terminal history remain unchanged. No migration guesses System or reconstructs authorization from payload data. + +Scheduler identity/fingerprint/cursor are durable. Occurrence insert and cursor advance share a transaction; first enable records the current minute, each pass considers at most 256 elapsed minutes. Completed occurrences cannot be recreated. Job database effects participate in settings-aware transaction validation and per-case SQLite test isolation without manufacturing Model metadata or a second storage stack. + +Read the injected Clock after obtaining the write lock: claims establish their deadline after row selection, and renew/completion lock the claim row before checking expiry. A timestamp captured before pool/lock wait cannot prove a lease is still valid. SQLite worker preflight requires `lease_ms > Job timeout_ms`, because a business write transaction blocks heartbeat writes even with a spare connection; PostgreSQL retains shorter renewable leases. + +Validate persisted claim policy before dispatch too: older timeout policies incompatible with the current SQLite lease become blocked with `invalid_policy`, without running App. Clean exhausted rows in a separate bounded statement within the claim transaction; they must not occupy execution slots or make test drain mistake an eligible queue for an empty one. + +## 7. Good / Base / Bad + +Base: a System CMD enqueues a tenant Job and persists its controlled execution tenant. Good: protected `post publish` automatically authorizes and binds `news.article.admin.publish`; each attempt re-verifies the session/membership, compiled permission, component state and current role. Bad: derive actor/tenant from payload fields, treat a missing identity as System, persist JWT/Cookie/roles/current permissions, write manual authorization strings, allow two execution tenants to share a global-queue dedupe key, or create a tenant database during an ordinary request. + +## 8. Wrong Versus Correct + +- Wrong: hand-written CRUD Service/API wrappers, dynamic query maps, reflection decoders or database handles in source code. Correct: Model methods lower to a checked `QueryPlan` and thin backend executors. +- Wrong: infer a renamed column from similarity or silently accept catalog drift. Correct: require `from <old_field>` and compare recorded actual catalog before migration. +- Wrong: carry an ambient transaction into `run` or another concurrent boundary. Correct: concurrent callbacks use detached database context; only sequential calls inherit the transaction. +- Wrong: release a connection whose async/blocking operation did not report completion. Correct: detach it so the pool replaces it. +- Wrong: make PostgreSQL accept an i64/UUID bind by converting the runtime value to text. Correct: keep the native binary codec and emit the exact `BIGINT`/`UUID` placeholder context. +- Wrong: claim cursor/stream, bulk create/upsert, relation loading or live PostgreSQL behavior from AOT compilation. Correct: claim each only after its dedicated runtime tests exist. +- Wrong: assume an extra SQLite pool connection can renew a lease while a handler holds `BEGIN IMMEDIATE`, or capture `now` before waiting for that lock. Correct: require the SQLite lease to exceed the bounded handler timeout and read Clock after acquiring the write/claim lock. +- Wrong: let exhausted rows consume `LIMIT` and return an empty claim batch while eligible work remains. Correct: clean exhausted rows with a separate bounded statement and query execution slots only from `attempt < max_attempts` rows. +- Wrong: snapshot authorization into payload/queue columns or let legacy rows become System. Correct: persist only a trusted re-verifiable identity reference and the protected entry's exact permission key, then re-verify the current catalog, components and role before each attempt. +- Wrong: let password hashing hold SQLite's writer reservation or switch the whole flow to a deferred snapshot that later upgrades to a write. Correct: perform password work outside a transaction, then enter an explicit short `BEGIN IMMEDIATE` transaction that rechecks mutable state and writes. +- Wrong: silently connect to the control/global database when a tenant database is missing. Correct: create/migrate it only through explicit `tenant migrate`, require readiness/fingerprint markers, and fail ordinary work closed. + +AOT compilation proves feature and generated-code compatibility, not live PostgreSQL semantics. + +## 9. LLVM Model Application Boundary + +### Scope / Trigger + +Checked Model/SQL/transaction lowering serves LLVM CMD, API, Job and case-local Test roots. The API Tenant Session section below defines identity, authorization and tenant-pool ownership. Missing tenant identity/readiness fails closed; it never silently selects the platform database. External Worker and default CLI integration remain separate work. + +### Signatures + +- Core `llvm::emit_application(&Program, &SourceMap)` emits typed Model functions and the owned database application root. Pure SQL/DDL builders remain owned by `native/orm.rs` and shared by both backends. +- Runtime `database::Session::load(profile, bindings, transactions)`, `prepare()`, `database(explicit, root)` and `close()` own only this invocation's Settings and database pools. +- `task::run_entry_with_typed_cleanup(config, future, cleanup)` returns the original typed result after descendants drain and cleanup completes inside the same runtime. +- Canonical `dever_rt_v1_db_*` declarations live only in `crates/dever-backend-bridge/include/runtime.h`; `runtime-sqlite` / `runtime-postgres` add actual drivers independently of `embedded`. + +### Contracts + +Deployment settings still come only from executable-local `config/setting.json`; no connection environment variable or PATH language runtime is introduced. Ordinary resolution is explicit connection → component root → required default. Persistence ABI accepts only fixed storage scalar rows; business Model records, nullable fields, Related and DbError are concrete typed owners, not dynamic business values. Related ToMany groups at most 998 parent keys per SQL batch and retains the existing per-parent row limit. Migrations group by resolved database, create all tables before foreign keys and preserve existing history/Seed validation. + +Writing CMD order is input decode → begin → sequential App/handler calls → output encode → commit → stdout. Nested transaction shares the outer owner, without savepoints; Task and parallel boundaries detach. Failed or cancelled unconfirmed operations discard the transaction connection. Rollback must move the original fault, retain its type/source/payload and only append cleanup cause; commit failure must release the already-initialized output. Drain children, stop every pool, await pool drivers, then destroy the runtime. Never reuse the process-global Settings/pools for repeated embedded roots. + +### Validation & Error Matrix + +| Condition | Required result | +|---|---| +| Invalid/misaligned output or typed callback slot | ABI fault before I/O, receiver advancement or fault consumption | +| Missing/invalid settings or missing tenant identity/readiness | Explicit root failure, no default connection fallback | +| Database error | Exact one of the ten checked DbError variants with owned Text payload | +| Output encoding or numeric fault after a write | Original runtime fault; rollback; no stdout | +| Captured PostgreSQL SQL constraint error | Current transaction remains aborted; no implicit savepoint/continued-query promise | +| Partial decode, stream stop/cancel or root exit | Paired typed cleanup and connection release/discard in its known state | + +### Good / Base / Bad Cases + +Base: a CMD creates, reads and deletes a row in its owned SQLite database. Good: real PostgreSQL round-trips storage scalars, captures NotFound without losing identity, and rolls back an uncaught unique violation before the next command inspects persisted state. Bad: use SQLite's ability to continue after a SQL constraint error as a PostgreSQL contract, close a pool after its runtime disappears or reuse a closed OnceLock pool on the next entry. + +### Tests Required + +Select root `llvm_database_source`, `runtime_entry_cleanup` and the canonical C database driver in `native_runtime_link`. Static cases must pass the existing checker and emit objects for all six targets. Opt-in native cases link the explicitly prepared runtime-only archive, execute with a cleared environment, and check output/fault plus allocation balance over warmup and 64 repeats. Cover CRUD/bulk/upsert, nullable/codec values, cursor/offset, stream cancellation with one connection, ToOne/ToMany, typed SQL, nested/sequential transactions, uncaught constraint/numeric/encoding rollback and successive schema/Seed revisions. Live PostgreSQL requires only the shared `database.postgres_test` isolated fixture, not an environment URL. These checks do not prove target runtime packs, throughput/RSS, tenant/API equivalence or injected commit/close failures. + +### Wrong Versus Correct + +Wrong: publish Session settings into global OnceLock, run pool cleanup in a second runtime, copy SQL/driver code for LLVM or add savepoints to make a SQLite test pass on PostgreSQL. Correct: invocation-owned Session, original shared SQL/drivers, same-runtime drain/close ordering and tests respecting each driver's existing transaction semantics. + +### LLVM API Tenant Session + +Job bindings and clock belong to the same invocation Session. Enqueue uses the caller's physical database and ambient transaction; handler transactions and timeout rollback reuse the existing store/worker and transaction owners. Tenant migrations prepare the queue in each selected tenant database before readiness. Component membership alone never enables tenant isolation: Job component checks apply only when deployment settings configure tenant storage. Without tenant configuration, ordinary SQLite/System Jobs must run without a tenant ID. Protected API authorization must bind its exact permission to the trusted Job execution context before enqueue; workers recheck the current subject/session, tenant, catalog, component and role rather than trusting a stored grant. + +HTTP/REST and log-enabled CMD use `application::Session` as the single config, +lifecycle, database, tenant and migration-pool owner. The fixed route descriptors +only reference checked Models and auth contracts; SQL/DDL/role storage remain +shared with the original backend. Configured database-only tenant storage is +supported, not a reason to reject every root. Missing tenant identity/readiness +still fails closed; `global` Models use control storage, other Models require the +tenant physical database. A disabled component also denies Tenant Owner. + +Tenant pool creation registers a manager entry with `ready=false` before awaiting +schema preparation/marker validation. Interrupted preparation never becomes a +usable pool, and retirement remains owned through `close_pool().await`. Migration +pools register with the Session before their first prepare await. After child +drain, close migration pools, tenant manager and platform pools on the same +runtime; preserve the original fault and append cleanup causes rather than +replacing it. Do not cache a failed or incomplete pool as ready. + +`llvm_api_source` uses 65 SQLite roots with two tenant files and the unchanged +allocation ledger; it asserts exact site roles, component denial, owner row +filtering, concurrent identity isolation and POST rollback. Its opt-in PostgreSQL +case shares `support/postgres::run_in_isolated_databases`, supplies only +`config/setting.json.database.postgres_test`, and cleans one random control and +two tenant databases. Neither cross-target objects nor prior Model-only PG +success proves API/tenant execution. Commit/close failure injection and +throughput/RSS remain separate acceptance work. + +REST SQL snapshots may inspect HIR without deployment settings, but executable +REST fixtures must use `check_with_settings` with a real site/provider binding +and a checked identity App. Reuse one owned fixture for this setup, as in +`rest_model`; missing auth metadata or generated authorization helpers must not +be worked around by disabling permissions. + +### Dependency Lock-Order Regression + +The old bundled SQLite 3.51.1 had an ABBA VFS lock inversion: replacement connection creation (`findReusableFd`) acquired global → inode, while WAL connection close (`unixLock` → `unixIsSharingShmNode`) acquired inode → global. A cancelled stream correctly discarded its unknown connection; concurrent background close/replacement then hung runtime shutdown. Native stdout already contained a successful CMD response, so checking only SQL success or Scope completion missed this dependency defect. Repeated opt-in process-bounded stream/root execution and both blocking-worker stacks exposed the exact cycle; no timeout extension, reusable-unknown connection or checkpoint suppression is an acceptable workaround. + +Keep the fixed dependency chain (`deadpool-sqlite` 0.14.0, locked `rusqlite` 0.40.2 / `libsqlite3-sys` 0.38.2, bundled SQLite 3.53.2), without editing registry cache or shipping a private SQLite fork. The author workspace requires Rust 1.95. `sqlite::value(column, ValueRef)` receives the real index in ordinary, bounded, owned-SQL and stream decoders; malformed Text retains `ErrorKind::Database` and the column diagnostic. Select the native early-close/repeated-root case and `sqlite_runtime` invalid-Text/replacement tests on dependency changes. An earlier successful run is not sufficient evidence against an observed intermittent shutdown hang. diff --git a/.trellis/spec/backend/directory-structure.md b/.trellis/spec/backend/directory-structure.md new file mode 100644 index 0000000..cc7d1f0 --- /dev/null +++ b/.trellis/spec/backend/directory-structure.md @@ -0,0 +1,47 @@ +# Directory Structure + +- `crates/dever-core/src/lib.rs`: separate syntax `parse` and semantic `check` entry points. +- `source.rs`: UTF-8 original-source ownership, sorted `.dever`/`.dever.md` loading, the single main/component/domain/role/topic classifier and byte-to-line/column conversion. +- `markdown.rs`: CommonMark top-level literal `dever` and `typescript dever` fence extraction, explicit closure and complete-declaration validation. +- `markdown/contract/`: `.dever.md` heading/list model, one-section-per-declaration mapping and checked package/type/function documentation contracts. +- `diagnostic.rs`: source-located errors, related labels and rendering. +- `lexer.rs` and `token.rs`: lexical rules and preserved numeric spellings. +- `syntax.rs`: spanned syntax structures and operator precedence. +- `parser.rs`: declarations, types, patterns and linear statements. +- `parser/expression.rs`: expressions, postfix operations and collection construction, sharing the same parser cursor. +- `check.rs` and `check/`: register all declarations before bodies; `check/layout.rs` owns application layout validation, `check/symbols.rs` owns logical App aliases and role access, and `check/api.rs` validates HTTP routes after signatures resolve. Names, types, linear flow, expressions/operators/value construction, collection handlers and symbolic clause coverage retain separate owners. Cycle checks include nested expressions, handler calls and type references, collapsing physical files in one domain for dependency-cycle purposes. +- `types.rs` and `hir.rs`: nominal types, opaque Stream values, static handler signatures/targets, concrete typed expressions, normalized value-only clause domains and local slots. Only checking constructs executable HIR. +- `wire.rs`: shared checked wire schema, policy identity and interned concrete type graph. `native/wire.rs` and `llvm/wire.rs` emit concrete backend-specific encoders/decoders; runtime `wire.rs` owns bounded duplicate-aware JSON parsing and exact raw JSON output. API dispatch, Adapter settings and Job codecs consume this owner, never maintain independent allowed-type registries. +- `specialize.rs`: the single function-plus-handler specialization graph used by checking and native emission. It resolves forwarded handlers and rejects recursion after substitution. +- `llvm.rs` and `llvm/`: typed HIR function/coroutine kernels with shared ownership/fault helpers. `network.rs`, `concurrency.rs` and `parallel.rs` own streams/TCP, task composition and worker packs. `http*.rs` lower protocol capabilities; `system.rs` owns system intrinsics and `render.rs` terminal faults. `application.rs` owns compiler-generated CMD/HTTP/Job roots; `api.rs` static route/auth/permission callbacks; `api_context.rs` auth/request/Cookie/Upload capabilities; `rest.rs` generated Model REST; `application_commands.rs` typed tenant management/CMD normalization. `jobs.rs` owns typed payload dispatch, enqueue, binding and clock adapters; `testing.rs` links case-local checked programs into one suite with indexed fault ownership. `wire.rs` owns concrete checked codecs, `ports.rs` selected Dever Adapter dispatch and Setting lifetime. `database*.rs`, `orm*.rs` and `transaction.rs` own persistence codecs, shared SQL/DDL and hidden transactions. Runtime `application::Session` owns config/lifecycle/database/tenant/migration/Job resources and closes after Scope drain within the same runtime. `external.rs` owns checked Worker calls and embedded resources; `process_entry.rs` wraps callable roots in process main. Private and managed Linux CLI compilation are connected; formal six-target delivery remains unfinished. +- `contracts.rs` and `contracts/`: post-HIR facts/domain proofs, failure obligations, transitive effects/dependencies and structural suggestions. `api.rs` owns public snapshots and application exposure analysis; CLI `api.rs` owns explicit baseline file I/O. +- `intrinsic.rs`: the single name-to-operation mapping for system primitives; checker and emitter exhaustively match this enum. Official result choice shapes are checked before emission and variants resolve by name, not declaration index. +- `native.rs` and `native/`: concrete record/choice generation, direct functions and collection loops, source-located failures, generated API/CMD/Job application roots and owned temporary build artifacts. `native/build.rs` owns native compilation and the single shared Cargo runtime target; `native/build/cache.rs` owns exact immutable inputs and executable cache identity; `native/build/cleanup.rs` owns marker-protected temporary directories and bounded local cleanup. CLI code must not reproduce those filesystem rules. `native/api.rs` emits checked static route dispatch, auth scope and response conversion. No generic runtime Value or interpreter dispatch. +- `check/jobs.rs` owns Job declarations, App payload contracts and static enqueue checking. `native/jobs.rs` owns concrete payload dispatch and queue initialization, reusing the shared wire and database owners. `specialize::execution_reachable` adds Job handlers as independent execution roots without adding synchronous call edges. +- `crates/dever-runtime/src/`: shared Decimal128/checked Int rules, ordered copy-on-write collections, Bytes, resource handles, pull-based Stream ownership, rendering and stdout. Native programs link this same runtime library; generated Rust is not a duplicate runtime implementation. +- `crates/dever-backend-bridge/`: private LLVM/LLD object/link owner. Its safe API uses explicit six-target selection; only private `ffi.rs` may contain authorized unsafe calls. LLD calls are serialized; failed crash recovery terminates the private compiler worker before any cleanup/re-entry. Outputs are exclusively claimed before linking. The opt-in `embedded` authoring build requires a private LLVM 18 SDK; independent `runtime-abi` exports the canonical C runtime boundary without that SDK. HIR lowering remains in core, and neither build mode is a formal released compiler/target pack. +- Runtime `async_stream.rs` owns the shared async producer/cursor and closure; `net.rs` owns Tokio Socket/Listener, duplex I/O, deadlines and resource cancellation. `task.rs` owns the single runtime, structured tasks/Group and bounded async stream dispatch. File resources remain in `resource.rs`. +- Runtime `job.rs` owns transactional enqueue and trusted execution/component checks, `job/store.rs` owns private queue/cursor schema and fenced state changes, and `job/worker.rs` owns bounded production/test dispatch. `job/resources.rs` owns invocation-local bindings, clock, initialization and tenant queue migrations. `auth::authorize_permission` binds the checked permission after authorization for both backends. `cron.rs` supplies the shared UTC parser. `lifecycle.rs` owns service-mode gating and signal-driven stop/drain; all work remains under the existing structured task owner. +- Runtime `component.rs` owns external Adapter resolution, bounded admission, Worker protocol, restart and shutdown. `component/command.rs` reuses that owner for one ordinary process per call, with binary I/O and caller-scope completion; `dever.process.Output` is its checked result, never an OS process handle. `check/ports.rs` synthesizes external implementations from the existing Port owner; `native/ports.rs` emits concrete codecs and dispatch. Do not add per-language runtime supervisors or expose protocol JSON to App code. CLI `workers/command.rs` packages domain-owned ELF/resources and reuses `workers/elf.rs` with native Python library validation. +- Runtime `external.rs` owns verified embedded external-resource extraction below the executable's private `data/cache/lib/<digest>` root. It is a packager primitive, not a registry client or shared-cache writer. +- Runtime `http/` owns Hyper HTTP/1, bounded messages, streaming reply/session ownership and connection deadlines. Runtime `api.rs` adapts typed inputs/JSON envelopes and reuses HTTP serving. `sse.rs` owns event encoding; `websocket.rs` reuses tokio-tungstenite. TCP and WebSocket share private `endpoint.rs` for close/wakeup ownership. Native `http.rs` converts checked source records and statically bound handlers; no application business code belongs in the runtime. +- Runtime `http/client.rs` owns fixed-origin pooled requests and leases; `client_driver.rs` owns physical/request capacity, idle expiry and the one structured pool driver; `client_body.rs` owns pull-driven upload/download. `tls.rs` owns shared rustls configurations; the private `transport.rs` enum delegates TCP/TLS I/O for HTTP and WebSocket. +- `library/**/*.dever`: bundled official packages processed by the same parser/checker as user source. Compiler-assigned source origins, not filenames, authorize reserved namespaces; `SourceMap::files()` remains user-only while `get()` also resolves bundled diagnostic spans. +- `format.rs` and `format/`: shared AST-driven declaration layout, preserved per-block comment attachment and syntax-only canonical formatting. Markdown replaces only program-block interiors. +- `reference.rs` and `reference/`: opt-in HIR semantic evaluator and differential output/fault representation. Never link into production native programs. +- `native/liveness.rs` and `native/updates.rs`: clause-local last-use analysis, safe record alias coalescing and scalar snapshots before aggregate consumption. `types.rs` caches nominal comparison/move/transfer properties after cycle validation. +- `native/sequences.rs`: conservative total-handler proof and eligible collection fusion; collection emission shares lazy Text splitting with the runtime. +- `contracts/dependencies.rs`: shared dependency order and reverse-caller work scheduling for contract summaries. +- `check/cache.rs`: per-thread last-result cache with complete source identity; bundled AST caching belongs to `lib.rs`/`source.rs`. +- `native/build/cache.rs`: native artifact identity, integrity validation and atomic publication. `native/build.rs` owns current runtime freshness, snapshots and isolated executable lifetimes. +- `crates/dever-cli/src/main.rs` and `format.rs`: argument parsing, source loading, direct execution of the owned native program, local clean command adaptation and prepared/atomic source formatting writes. `test_runner.rs` is the single owner of application-test ordering, isolated native processes, captured output and temporary project cleanup. +- CLI binary-private `compile.rs` owns LLVM/object/LLD orchestration; `worker.rs` is the service-owned source compiler. Shared `toolchain/runtime_pack.rs` validates packs/snapshots without bridge imports; `compilation.rs` owns canonical input and staging validation, while `service/` owns authenticated transport and child supervision. Reuse core `native::compile_artifact` for private builds and `NativeProgram` for caller-owned received executables. `toolchain/packaging.rs` owns deterministic native release assembly/signing from explicit author inputs; `sdk/native-release.rs` only adapts its private author CLI. Root `runtime-pack` profile and `sdk/native-release.md` own the four-archive build recipe. `test/dever-cli-tests/support/llvm_inputs.rs` holds explicit fixture SDK inputs, and `support/daemon.rs` holds the shared owned-daemon lifecycle; neither is a production input discovery mechanism. +- `library/dever/json/`: ordinary Dever JSON algorithm. `library/dever/http.dever`, `sse.dever`, `websocket.dever` and `tls.dever`: public protocol types/helpers and the native bridge; no duplicate source protocol stack. `task.dever` exposes runtime configuration, sleep and periodic streams. +- `examples/cms/dever/` and `examples/cms/md/`: maintained CMS projects with equal declarations, schema, settings, business behavior and application tests. Generated API/CMD roots replace `module/main.dever`; `api/admin/**` and `api/front/**` are assigned by `sites.<key>.path`. Both projects keep matching `test/<component>/<domain>/<topic>` cases and `config/setting.json`. +- `examples/old/`: archived pre-role-layout language and regression material. It is not a current application root and must not be presented as a directory or visibility template. Formal tests remain under root `test/`. +- `examples/README.md`: current CMS commands and the archive boundary. Historical rejection fixtures keep their original diagnostics only when invoked by dedicated compiler-level tests, not as current application projects. +- `test/dever-tests/tests/`: permanent focused tests. The canonical full-language syntax example is read directly from the PRD so a copied fixture cannot drift. + +Core foundations, formatting, optional reference evaluation, official library follow-through and the SQLite/PostgreSQL static ORM have concrete implementations. Compiler distribution and self-hosting remain unshipped; validation evidence belongs to the active task, not this directory map. Compiler phase APIs are bootstrap internals, not an alternative program source format. + +Use one owning implementation for each language rule. Do not introduce a generic parser framework, service layer or backend registry for a single concrete implementation. diff --git a/.trellis/spec/backend/error-handling.md b/.trellis/spec/backend/error-handling.md new file mode 100644 index 0000000..55d5d0b --- /dev/null +++ b/.trellis/spec/backend/error-handling.md @@ -0,0 +1,88 @@ +# Error Handling + +Source loading returns path-specific `LoadError` values because unavailable or invalid UTF-8 files do not have a valid source span. Parsed-source errors use `Diagnostic` with a stable code, primary span and optional related labels. + +- `Span` is a half-open UTF-8 byte range owned by one `SourceId`. +- Display positions are one-based lines and Unicode scalar columns. LF, CRLF and CR line endings use the same source model. +- Render the original source line with its explicit line/column location. Do not guess terminal display width with one ASCII space per Unicode scalar. +- Preserve source literal spelling through parsing; numeric representability belongs to checking. +- Reject invalid source with a diagnostic. Never turn an unsupported construct into a successful default AST. +- Parser recovery synchronizes only at balanced body boundaries or top-level newlines. +- Both parser call depth and resulting expression tree depth are bounded by `MAX_SYNTAX_DEPTH`. Long iterative operator/postfix chains must not bypass the limit and overflow during recursive AST destruction. +- I/O and syntax errors are explicit results. `expect` is reserved for compiler-owned invariants or infallible String formatting, not malformed source. +- Contract structural-depth rejection is fatal to that analysis pass: `contracts::check` returns `Err(C015)` before producing summaries. Both the main checker and Port fake reanalysis must stop before consumers read those summaries; ordinary contract diagnostics still aggregate after summaries are populated. The 256-layer nominal-type regression must report C015 rather than panic in a later phase. + +Current codes: `L001` unexpected character, `L003` excluded keyword, `L004` malformed number, `L005` unterminated Text, `L006` invalid escape, `P001` expected syntax, `P002` invalid statement/assignment, `P009` excessive syntax nesting. + +Markdown source codes: M001 no top-level program block, M002 unclosed program fence, M003 declaration spans multiple blocks, M004 invalid package documentation structure, M005 invalid declaration documentation structure, M006 documented package/API/declaration name mismatch, M007 function input mismatch, M008 function output or type-shape mismatch. Semantic documentation mismatches use the Markdown value as the primary span and add the corresponding Dever declaration as a related label. Tokens and diagnostics refer to original `.dever.md` byte positions, including Unicode columns and source newline conventions. Markdown cannot introduce a synthetic filename or bypass normal language diagnostics. + +Semantic codes: `C002` duplicate declarations/bindings/known Map keys, `C003` package path or reserved namespace, `C004` unresolved names/signatures or uninitialized locals, `C005` type/arity/construction/primitive-contract errors, `C006` private cross-package access, `C007` function/type/package cycles, `C008` statically known numeric faults, `C009` clause overlap/coverage/reachability, `C010` naming format. `W001` reports unused/overwritten writes without rejecting the program. Diagnostics sort by source ID and byte span; check unused declarations before native reachability pruning. + +Standard output errors propagate to the native entry as a failure containing the `.dever` call location. The process exits nonzero even if stderr itself cannot be written. CLI/toolchain failures are explicit command errors; `check` never invokes rustc. Native temporary directories have exclusive ownership and are cleaned on compilation failure or after use. Cleanup failure cannot mask the primary result; explicit build output never overwrites an existing file. + +External Adapter business errors are decoded only when their exact identity and payload match a Port-declared error variant. Startup, schema/handshake mismatch, invalid or out-of-order protocol messages, timeout, child crash and undeclared error identities are runtime faults at the Port call site, never synthetic business variants. Protocol diagnostics contain metadata only and must not echo setting secrets or call payloads. A protocol fault terminates the Worker channel and closes pending calls. Generated entry cleanup runs after success, business failure and runtime fault; shutdown/database cleanup augments rather than replaces the primary error. + +Embedded external resources are never executed before their relative path, regular-file type and SHA-256 digest verify. Extraction rejects symlinks, path escape, duplicate entries, truncated staging and invalid bundle identity; an existing or concurrently published directory is reusable only after a full recheck. Missing artifacts remain an explicit offline preparation error with the corresponding `dever lib add/update` command. + +## Core Execution Contract + +Compiler contract diagnostics add C011 (unproved range), C012 (unhandled failure), C013 (pure/clause contract), C014 (Model/transaction contract) and C015 (bounded proof work). W002–W005 and W007 are suggestions; W001 remains the owner of unused/overwritten writes. Details and counterexamples are in [Compiler Contracts](./compiler-contracts.md). + +### 1. Scope / Trigger + +Applies when changing checker-to-HIR-to-native contracts, bundled packages or runtime operations. + +### 2. Signatures + +- `check(&SourceMap) -> Result<Program, Vec<Diagnostic>>`; successful programs expose `warnings()` separately. +- `native::compile_project(&Program, &SourceMap, rustc, profile) -> Result<NativeProgram, String>` generates the application entry from API/CMD/Job declarations; compiler-internal library fixtures may still use `native::compile` with an explicitly selected zero-input entry. +- `dever.io.read(file, limit) -> ReadState`; `dever.net.read(socket, limit)` returns the same state. ReadResult is the capture choice. +- `dever.io.chunks(file, limit) -> Stream<ReadEvent>`; `dever.net.chunks(socket, limit) -> AsyncStream<ReadEvent>`, and `dever.net.connections(listener) -> AsyncStream<ConnectResult>`. ReadStreamResult choices capture creation failures. +- `dever.http.send -> Response`; ResponseResult captures recoverable errors. `serve` has zero outputs and propagates configuration/accept faults. Handler faults are logged at the request boundary and return 500 or abort an already-started response; later requests remain available. Peer protocol/I/O errors close only the affected connection; body/handler timeouts return 408/504 before closing, incomplete-header timeout closes directly. + +### 3. Contracts + +`ReadState` is `Read(bytes: Bytes)` or `End`; `ReadResult` is `Done(state: ReadState)` or `Failed(message: Text)`. Ordinary calls return success values; explicit result captures convert the inferred failure set into choice values. Stream reads yield source-defined `ReadEvent` values and produce at most one terminal `Failed`; normal EOF produces no synthetic item. File creation uses create-new; resources share identity and close state across aliases, unlike ordinary value-semantic records/collections. Read limits and socket timeouts are positive Ints; reads may be short. Stream creation rejects a non-positive limit or already-closed source before returning a Stream. Core `close(stream)` is idempotent. + +Handler parameters are compile-time bindings, never runtime values or clause axes. Every value type nested in a handler signature uses the same generic arity and Map-key validation as an ordinary function boundary. Handler signature, output-name and specialization-cycle errors are source diagnostics before build. `reduce_until` applies the current item, then stops without pulling another item. + +`result(call)` changes business-failure representation, never scheduling. Captured calls share ordinary-call suspension/liveness and inline-blocking checks. A synchronous capture must remain legal with synchronous resources; capturing a suspending call must diagnose non-transferable live locals at the capture span, and capturing a blocking call must not bypass an async caller's worker boundary. Keep both negative paths and the synchronous positive control in `structured_concurrency`. + +Build/run use the local bootstrap toolchain (`RUSTC`, optional `CARGO`) and locked cached dependencies with `--offline`; missing tools/dependencies fail explicitly. Generated binaries link the runtime, not the checker or a generic evaluator, and execute without the build toolchain. + +All runtime feature combinations build through one Cargo target. Only artifacts reported for the current Cargo graph enter an immutable input view; a missing, symbolic-link, changed or incomplete input fails before generated code is linked. Native temporary directories carry a versioned owner marker and are removed immediately by RAII. Recovery cleanup deletes only current-user, real directories with an exact Dever name/marker older than 24 hours; unknown, fresh, active or symbolic-link paths are skipped. Cleanup failure may be returned by explicit `clean`, but automatic cleanup never replaces the current compile or run result. + +### 4. Validation & Error Matrix + +| Condition | Result | +| --- | --- | +| Incomplete record/output, type drift, illegal key | Source diagnostic before build | +| Invalid generic or Map key nested in a handler signature | C005 before build | +| Clause overlap, gap or empty complement | C009; overlap labels both clauses | +| Known Int/Decimal overflow or zero divisor | C008 before build | +| Dynamic Int/Decimal fault or stdout failure | Source-located native failure and nonzero exit | +| Invalid bytes, file/socket error, closed resource | Explicit Failed choice | +| Invalid stream limit or closed source at creation | Explicit Failed choice; no Stream returned | +| Stream producer terminal error | One Failed item, then end; producer released immediately | +| timeout/race receives a program fault | Propagate after structured cleanup; never invoke timeout fallback for a fault | +| timeout expires | Stop and drain the task before its signature-matched fallback runs | +| Pool closed, request/stream deadline or TLS verification failure | Explicit Failed; failed/incomplete connections are not reused | +| Stream(Channel/List/Bytes) contains error choices | Preserve their failure obligations; constructing or closing the stream does not handle them | +| HTTP listener explicitly closed | Stop accepting, drain active responses; application timeout may stop remaining work | +| Invalid/reused build output path | Command error; existing file preserved | +| Runtime input view is incomplete or changed | Build error; never scan unrelated historical artifacts | +| Clean candidate has no valid owner marker, is fresh, active or a symlink | Preserve it unchanged | + +### 5. Good / Base / Bad Cases + +Good: `Text` plus `null` clauses infer `Text?`; `first([null])` in an `Int?` output context remains `Int?`. Base: hello prints one UTF-8 line. Bad: `Bool` plus `true` clauses overlap; repeated constant or zero-payload choice Map keys are rejected. + +### 6. Required Tests + +Use `core_semantics` for checking/rejection, `core_native` for exact outputs/evaluation order/native failure locations, `runtime_foundations` for decimal precision/COW/resource closure, and `hello_native` for CLI-backend regressions. Stream tests must assert that mapping preserves a terminal item as terminal, releases its producer on that pull, and that official wrappers map invalid creation to `ReadStreamResult.Failed`. Run only affected targets and explicitly announce owned temporary-file/loopback checks. + +### 7. Wrong vs Correct + +Wrong: select the first matching source clause, append anonymous standard files to a cloned SourceMap, emit an untyped Value interpreter, reorder record expressions by field name, or implement Stream mapping through an API that erases `Pull::Last`. + +Correct: prove disjoint exhaustive domains before emission, keep bundled source origins retrievable by the caller's SourceMap, emit concrete types/direct handlers, evaluate constructor fields in source order, and preserve `Item`/`Last`/`End` through Stream adapters. diff --git a/.trellis/spec/backend/index.md b/.trellis/spec/backend/index.md new file mode 100644 index 0000000..c0f8a40 --- /dev/null +++ b/.trellis/spec/backend/index.md @@ -0,0 +1,23 @@ +# Compiler Development Guidelines + +This repository implements the Dever language compiler in Rust. It is not a Go Dever application or a database-backed service. + +## Pre-Development Checklist + +1. Read the current task PRD, design and implementation progress. +2. Read [Directory Structure](./directory-structure.md), [Error Handling](./error-handling.md) and [Quality Guidelines](./quality-guidelines.md). + For CLI, formatter, native optimization or official packages, also read [Toolchain and Library Contracts](./toolchain-and-library.md), including signed ecosystem packs and bounded binary resource attachments. + For language contracts and API boundaries, read [Compiler Contracts](./compiler-contracts.md). + For LLVM typed lowering or runtime C ABI work, read both contracts; distinguish kernel/object evidence from complete application semantics and formal target runtime packs. + For Model, ORM, database configuration or runtime connections, also read [Database Guidelines](./database-guidelines.md). + For Dever logging or HTTP request-boundary records, read [Logging Guidelines](./logging-guidelines.md). +3. Inspect affected compiler phases and existing tests before adding another rule or helper. + +## Quality Check + +- Check the implemented milestone only; syntax success must not imply semantic validity. +- Run the focused checks listed in the quality guide. +- Keep all tests under repository-root `test/`. +- Preserve unrelated user changes and do not create Git commits without authorization. + +Database behavior is a compiler/runtime contract only within the static ORM boundary documented in [Database Guidelines](./database-guidelines.md). Shared logging and native exit-flush contracts are documented in [Logging Guidelines](./logging-guidelines.md). diff --git a/.trellis/spec/backend/logging-guidelines.md b/.trellis/spec/backend/logging-guidelines.md new file mode 100644 index 0000000..33b2c73 --- /dev/null +++ b/.trellis/spec/backend/logging-guidelines.md @@ -0,0 +1,46 @@ +# Logging Guidelines + +`library/dever/log.dever` provides `debug`, `info`, `warn` and `error` with a +`Text` message and `Map<Text, Text>` fields. Native emission routes these calls +to `crates/dever-runtime/src/log.rs`; do not add a second logger in the CLI or +generated program. + +The runtime writes one JSON record per line to stderr, with `level`, `message` +and `fields`. The bounded writer may drop debug/info events under pressure and +reports the count in `dropped_events` on a later record; warn/error events wait +for room, and error records flush immediately. Native program exit flushes the +writer. Keep log messages and fields bounded instead of logging unbounded +payloads. + +LLVM applications with reachable log calls use the same invocation-owned +`application::Session` and load `config/setting.json` once, even for log-only CMD. +Configured level filtering and request identity fields come from that owner; +Scope drain precedes pool close and log flush. Pure scalar CMD/kernel entries +do not acquire an API owner. `llvm_api_source` tests warn-level filtering and +65 flush-before-return invocations; do not infer configured logging support +from the standalone kernel logger alone. + +The single writer uses one preallocated 128-command queue; debug/info overflow +and warn/error backpressure remain distinct. Flush callers are serialized onto +one persistent acknowledgement state. Release preceding records and flush the +output before acknowledgement; writer exit/panic clears queued owners and wakes +all waiters. Do not add per-flush channels or a second queue. A pure-flush native +ledger exposed lazy mpsc receiver waiter allocations after acknowledgement, so +ordinary first-flush success alone is not a stable exit/allocation boundary. +Keep the original strict ledger, not sleeps, extra warmup or relaxed counts. +Root test/runtime_logging.rs covers FIFO/drop counts, backpressure, concurrent +flushes and writer failure; native_runtime_link covers the actual ABI barrier. + +`crates/dever-runtime/src/http/mod.rs` records an HTTP request-boundary error +with `boundary`, `request_id`, `protocol` and `target`. Do not include secrets, +credentials or full request/response bodies in fields. The source-to-stderr +contract is covered by `test/dever-tests/tests/logging.rs`; extend that focused +test when changing the public logging surface or output format. + +Secret and complete Models containing private fields are non-observable types, +including nested containers and related Models. Compiler entry/error/API/assertion +render boundaries must reject them before native emission; log arguments remain +Text and Map<Text, Text>. Runtime Secret Debug emits one fixed redacted marker and +never reveals bytes. Do not add a generic conversion or derive Render for a +sensitive nominal type. Ordinary record private fields remain an access rule, +not a promise of cryptographic secrecy. diff --git a/.trellis/spec/backend/quality-guidelines.md b/.trellis/spec/backend/quality-guidelines.md new file mode 100644 index 0000000..a6769b7 --- /dev/null +++ b/.trellis/spec/backend/quality-guidelines.md @@ -0,0 +1,120 @@ +# Quality Guidelines + +Workspace-owned Rust forbids unsafe code. Decimal128 and insertion-ordered Map reuse the pinned `dec` and `indexmap` dependencies in `Cargo.lock`; do not reimplement these domains. Keep source loading, parsing, semantic checking, native emission, runtime and CLI separate. Parser success is not semantic success. + +Root dev/test profiles keep `debug = 0` and `incremental = false` so independent focused test binaries do not recreate multi-gigabyte debuginfo and incremental object trees. Release keeps fat LTO and one codegen unit. Native runtime feature combinations share `target/native-runtime/cargo`; exact per-build inputs live under `target/native-runtime/inputs`. Do not restore one complete Cargo target per feature combination. + +Native ABI release archives use the root `runtime-pack` profile and one shared `target/native-runtime-abi` Cargo target for all four driver combinations. Save each completed archive before the next feature build. Preserve link symbols, keep `embedded` disabled, and distinguish deterministic pack assembly from cross-host source reproduction; the author workflow and assertion points live in toolchain-and-library.md and sdk/native-release.md. + +Tests exercise source contracts and observable AST structure, not private helper implementations. Include the PRD's complete package, precedence and associativity, Unicode and line positions, lexical failures, excluded syntax, recovery, structural limits and directory loading. Filesystem tests create and remove only their own temporary directories; source loading never follows symbolic links. + +Explicit LLVM Worker acceptance consumes the prepared immutable `target/native-release-inputs/component-fixture`, with recorded source and binary digests. Cargo's `target/debug/component-fixture` is a mutable build output: feature unification can change its bytes while running the workspace matrix. Ordinary Worker tests use their Cargo-provided executable; never bypass the CI input-change gate to treat a mutable output as a fixed author input. + +Explicit single-libtest acceptance uses `run_command(..., single_harness=True)` and requires the final parent summary to contain exactly one passed test, no failures and no ignored tests. Nested probe summaries must not be added to that parent count; ordinary Cargo multi-harness commands retain aggregate counting. A zero-test parent, failed parent or missing summary is not a pass even if a child probe succeeded. `test/ci/test_gate.py` owns these rejection cases. + +The shared `test/native-runtime-abi/managed-driver.c` runs one warmup and 64 repeated entries on joined host threads, then checks strict allocation balance after each join/TLS destruction. Rust's bounded-channel producer may first allocate its 48-byte thread-local Context only after warmup when the queue fills; measuring before the host thread exits misreports that cache as a leak. Preserve process initialization before any thread, original result checks and strict zero allocation delta; do not add retries, random warmups or an allowed-leak threshold. This measures entry-plus-host-thread cleanup, not unbounded TLS growth on a long-lived embedding thread. `test/ci/test_driver.py` compiles the shared driver with the explicit existing author C tool and verifies late TLS cleanup succeeds while intentional per-entry leaks and changed results fail. Suite probes include the same driver instead of copying its ledger. + +Markdown source uses pinned pulldown-cmark in the compiler only. Preserve the `.dever` path and existing semantic/contract checks; shared Markdown extraction/formatting rules and the focused `markdown_source` target are specified in [Toolchain And Official Library Contracts](toolchain-and-library.md). + +Focused checks for the current milestone: + +```bash +cargo fmt --all --check +cargo clippy --offline -p dever-core -p dever-tests --test syntax_and_format -- -D warnings +cargo test --offline -p dever-tests --test syntax_and_format +git diff --check +``` + +For core language changes, select the affected targets and announce bounded native/resource checks before running: + +```bash +cargo test --offline -p dever-tests --test hello_native +cargo test --offline -p dever-tests --test core_semantics +cargo test --offline -p dever-tests --test core_native +cargo test --offline -p dever-tests --test runtime_foundations +cargo test --offline -p dever-tests --test syntax_and_format +cargo clippy --offline -p dever-core -p dever-runtime -p dever-tests --test hello_native -- -D warnings +cargo clippy --offline -p dever-cli --bin dever -- -D warnings +``` + +The hello target retains stdout, value mutation, package boundaries, missing toolchain and artifact cleanup regressions. Core targets cover the full PRD sample, static rejection, clause permutation, concrete native output, COW ordering, nullable flattening and source-located faults. Foundation-extension cases additionally cover contextual handler syntax, forwarding/specialization recursion, value-only clause domains, List/Bytes/Stream traversal, early stop/resume and one terminal stream failure. Resource tests create only their own temporary files and loopback sockets on port 0 with bounded reads/timeouts. CLI changes also require the actual check/run/build command. These are focused compiler checks, not an integration environment or workspace-wide test run. + +Parentheses must not affect context inference: use the shared Group-unwrapping helper for nulls, empty literals and named handlers. Handler signature types must reuse ordinary boundary validation, including generic arity and nested Map-key rules. Stream adapters must preserve `Pull::Last` rather than turning it into an ordinary item. Never retype an existing Decimal/Int local as Float, even after constant propagation. Float zero patterns match both signs, so substituting the pattern constant for the actual input changes division semantics. Preserve source field and Map pair evaluation order in generated code; declaration order controls layout/display only. + +Use an explicit installed toolchain or a command-scoped PATH if Cargo is not discoverable. Do not change global shell configuration or replace the existing `dever` command. + +Do not run workspace-wide tests, npm builds, service checks, native compilation benchmarks or integration environments by default. Focused syntax tests do not establish type correctness, formatting round trips or runtime equivalence. Record those later-phase checks as not run. + +Explicit Linux no-host release checks share `native_release::acceptance::isolated_os_command`: a real mount-namespace root with only declared OS files, proc and synthetic devices. Plain chroot blocks the nested user namespace required by Worker sandboxing. The explicitly built static `native-acceptance-init` replaces bwrap's masked proc with a complete private proc inside the already pivoted outer namespace; otherwise the kernel rejects the nested proc mount. Preserve the outer fixture's original capabilities and the product's inner capability removal; do not substitute a host proc bind. Verify the final standalone executable after removing source and machine stores; an earlier pre-sandbox artifact or managed-only pass is insufficient. + +Installed daemon and launcher checks must share that owned PID/user/mount namespace so kernel peer credentials remain visible. Use the owned bwrap PID report, verify its live process and root inode, then enter it explicitly for clients; never relax production peer authentication to accommodate sibling PID namespaces. Pin the namespace init with a pidfd and wait for namespace exit and daemon-lock release before starting the next daemon. Run real distinct-UID checks separately because a single-UID test user namespace cannot represent multiple host identities. + +Toolchain/library changes select `formatter`, `standard_library`, `native_ownership`, `concurrency`, `json_library` or `http_library`. The optional `reference` feature enables only `differential`; production CLI feature resolution must exclude it. HTTP tests independently assert wire bytes and malformed framing, with bounded loopback peers. Formatter changes also exercise the actual `fmt`/`--check` CLI and failure-before-rewrite behavior. + +Configured concurrency and reusable TLS clients select `async_composition`, `pooled_network` and `pooled_library`. Assert task cleanup even before first poll, complete-body connection reuse versus abandoned-body close, bounded physical connections including idle entries, stream deadlines, verified CA/name failures, WSS and graceful shutdown. Check real native source conversion alongside runtime peers; include compile rejection for mismatched context/output signatures and unhandled stream failures. Certificates and test private keys belong under root `test/`; peers use only owned loopback ports. + +Runtime root entries pin the caller Future once before forwarding it through cleanup adapters and task-local wrappers. Job execution scopes also pin before constructing their async wrapper; identity initialization and task-local entry remain lazy. Preserve borrowed and non-Send roots, concrete errors, component-close/child-drain/cleanup order, and nested-entry rejection. Exercise `runtime_entry_cleanup` and all-feature `durable_jobs`, including signal/transaction shutdown: default-feature success alone misses the larger combined-driver Future layout. Do not increase thread stacks to mask wrapper-induced stack overflow. + +Standalone C fixture drivers must acquire the existing process initialization before starting kernel/runtime threads, just like generated process main. The shared `managed-driver.c` also owns initialization for the indexed suite driver. Callable roots deliberately preserve an embedding host's signal policy; do not modify runtime network leaves or test subprocess inheritance to hide a missing driver initialization. Keep the direct SIGPIPE ABI assertion and real TLS upload/download allocation-ledger regression. + +The ignored `performance` target runs only for an explicitly announced bounded performance check. Match Decimal initial representation as well as numeric value; prepare parsing outside the timed region. Five native computation kernels cover Int/Float/List/Decimal/Map with exact results and a 1.15 baseline ratio budget. Alternate a fixed 25 independent processes per side, discard each process's warmup, retain every process's nine raw samples, then compare the medians of all 225 samples per side. The same ELF has exhibited clustered timing modes even with CPU affinity; five process groups can accidentally favor one side. More process groups improve coverage without proving host noise is eliminated. Never selectively omit groups or add samples until a failing ratio passes. Production and baseline share `native::OPTIMIZATION_ARGS`, including one codegen unit so generic collection inlining does not depend on generated program size. Record compilation separately and report CPU/environment without inferring HTTP throughput or whole-language performance. + +## Standalone Runtime Benchmarks + +### 1. Scope / Trigger + +Changes to root `test/performance/` and the `network_bench`, `async_bench`, `live_bench` Rust examples. These explicit opt-in benchmarks measure owned binaries/loopback peers, never existing services or default test execution. Operational commands and limitations live in `test/performance/README.md`. + +### 2. Signatures + +`python3 test/performance/run.py build --output <new-dir>` compiles fixtures and records binary fingerprints/configuration. `--selection protocols` selects runtime/HTTP/live and three explicitly prepared Rust peers; `profiles` selects four runtime profiles; `cms-profiles` also builds both CMS sources. Every Dever fixture uses the public LLVM project build and CMD dispatch, never the removed native Rust fixture builder. `run --artifacts <dir> --output <new-dir> --suite memory|async|http|live|http2|http2-recovery|all` executes only prepared binaries. The HTTP peer CLI is `network_bench load h1|h2 URL PATH RATE SECONDS PHYSICAL_CONNECTIONS STREAMS_PER_CONNECTION TIMEOUT_MS REUSE CA`; H1 requires one stream per connection, while H2 requires connection reuse. Service modes are runtime-http/runtime-https/hyper-http/hyper-https plus runtime-http2/runtime-https2/hyper-http2/hyper-https2. Async comparison uses `async_bench root|worker WORKERS TASK_CAPACITY ITERATIONS`. Long connections use `live_bench PROTOCOL URL CONNECTIONS CYCLES HOLD_MS TIMEOUT_MS`, selected by `--connection-tiers`, `--cycles` and `--duration` in the runner. + +For `run`, `--server-cpu <id>` optionally binds the server to one CPU; `--client-cpus <id,id,...>` optionally binds HTTP/1, HTTP/2 and live clients to a CPU set, and `--client-workers <positive-int>` defaults to 1. `Process(cpus=...)` changes only its owned child before exec. An omitted client set inherits the runner's allowed CPUs. The report records `arguments.client_cpus`, `arguments.client_workers` and the effective `environment.client_cpus`/`client_workers`. + +### 3. Contracts + +Servers publish `READY|port` after preparing shared payloads. JSON counters satisfy `scheduled=sent+dropped`, `sent=completed=succeeded+errors`, and `dropped=dropped_capacity+dispatch_expired`. `latency_ms` measures planned-to-successful-completion; `request_latency_ms` measures dispatched-to-successful-completion; dispatch lag includes completed failures. Empty histograms are null and success histograms have exactly succeeded samples. Record raw rounds, successful throughput, load-generator CPU/lag, config and limits; never average p99 values into a global p99. HTTP/1, HTTP/2 and live clients share explicit `--client-workers`/`--client-cpus` configuration; record both in the report. Python stages each peer beside an owned config/setting.json with a benchmark object; peers read it relative to their executable and never use application environment variables. The manifest records setting-json-v1 and build checks the peer's check-config output before accepting it. Server affinity remains one CPU, distinct from the specified client CPU set. + +HTTP/2 records `physical_connections`, `streams_per_connection`, their `request_slots` product and `successful_handshakes`. Preconnect the exact physical connection count before timing, share cloned senders across stream slots, and do not retry, replace or expand connections during measurement. H2 cleartext is prior knowledge; TLS advertises and requires only ALPN `h2`. Runtime and raw Hyper baselines use the same stream/window/header/frame/send-buffer limits. After each warmup, case or recovery cycle, close clients, wait the fixed settle interval and require service FDs to return to the READY baseline; preserve the RSS sequence without treating allocator retention alone as a leak. Any HTTP/2 request error or cgroup OOM event fails the run after completed case data is persisted; capacity drops remain saturation evidence rather than a pass/fail threshold. + +Manifest `application_configuration=llvm-cmd-v1` is mandatory alongside `setting-json-v1`; reject older application artifacts before launch. Each runtime/http/live project contains its group's CMD declarations, so per-command aliases share the same multi-command binary and recorded identity. `fixture_command` derives the qualified CMD from the same `ENTRIES` used for building. Live settings belong beside each owned executable; ORM keeps its separate `benchmark.command.run` entry. LLVM O2/LLD and the exact prebuilt runtime archives determine Dever output. Rust peers are separately prepared with Cargo release, copied and fingerprinted. Do not claim identical optimizers or scheduling: Dever uses production defaults, while explicit workers/capacity affect Rust peers. Both retain checked Int arithmetic and structured run/wait; root/worker modes differ by an untimed outer structured task. Sampled /proc context-switch totals count currently alive threads and can decrease as threads exit; use perf for complete process/thread totals. + +Live clients flush zero-based `PHASE|cycle|connecting/connected/closing/disconnected` in order. Connected holds perform validated interactions; disconnected holds last 300 ms before the next connecting marker. Compare progress before and after sampling; discard ambiguous transition observations from phase reports. Success requires attempted=connected=disconnected=connections*cycles, errors=0 and at least one validated message per connection (SSE also requires a heartbeat during hold). Counters alone do not prove server cleanup: preserve per-phase FD/RSS and their post-disconnect deltas. TCP/WS/SSE payload validation, finite limits and driver ownership apply on errors as well as success. + +CMS concurrent reads close the control client before the baseline. `Process.descriptors()` records FD-to-target maps; `validate_read_descriptor_recovery(load)` is shared by the runner and CI. Its input includes `descriptors_before`, `descriptors_after`, and fixed `pooled_descriptor_limits`: each existing SQLite database/main WAL target is bounded by configured `max_connections`, SHM by one. All remaining descriptor targets and multiplicities must match (`Counter(non_pooled_before) == Counter(non_pooled_after)`). Preserve total FD change as observation. Persist each case's `report.json`, including partial completed data, raw samples, final budget and errors; aggregate failures as well as successes. + +### 4. Validation & Error Matrix + +Wrong status/type/body -> counted error, no successful latency sample. Exhausted load capacity -> dropped_capacity. Missed measurement window -> dispatch_expired; one late timer wake is not itself a drop. Incomplete samples/checksum mismatch -> runner failure. Missing cgroup permission/controller -> explicit failure and owned-group cleanup, never unlimited fallback. Interrupted run -> terminate/reap owned processes and remove owned empty groups, preserving completed case records. + +An empty, duplicate, negative or non-integer `--client-cpus` set fails at argument parsing; an unavailable CPU or overlap with the specified server CPU fails before any output directory or process is created. A nonpositive `--client-workers` value fails at argument parsing. Do not claim independent client/server cores when affinity was omitted. + +CMS extra/replaced/duplicated sockets, unexpected files or SQLite handles over the configured bounds fail even if total FD count is unchanged. Record completed publication fields and baseline/limits before starting the read load, so an early read or sampling error cannot erase existing evidence. + +Process memory may disappear before waitpid reports exit: skip that unavailable observation and let the process exit check own failure reporting. Every owned server context rejects cgroup OOM after teardown without masking an earlier failure. HTTP warmup and measurement require at least one success and zero request errors; capacity drops remain explicit measurements. Live missing/out-of-order phases or incomplete connection cleanup fail the run. A valid partial SSE frame at the intentional measurement deadline is not a malformed peer frame; never require TCP chunk boundaries to coincide with events. + +### 5. Good / Base / Bad Cases + +Base: single-worker runtime, 32 server connections, 16 client slots. Good: each implementation prepares and shares the 64 KiB body once, validates exact response bytes and waits the entire measurement window. Bad: only one side reallocates the body per request, failed responses improve p99, or sub-window completion inflates QPS. + +CPU placement good: `--server-cpu 0 --client-cpus 1,2,3,4 --client-workers 4` on a host allowing CPUs 0–4. Base: omit client flags for one worker inheriting allowed CPUs, with no isolation claim. Bad: repeat CPU 1 or place CPU 0 in the client set, or infer multi-core use solely from a requested worker count. + +CMS FD base: one main/WAL/SHM handle per database. Good: a configured four-connection pool expands to four main and WAL handles, one SHM, and all non-database descriptors recover. Bad: a fifth database handle, an extra socket or a replacement socket masked by a closed control connection. + +### 6. Required Tests + +Run `python3 -m unittest discover -s test/performance -p test_runner.py -v` for counting/warmup/phase ordering/process teardown, HTTP/2 shape/summary validation, isolated JSON staging and missing-controller cleanup. Assert staged public projects declare the CMD used by every runner call and reject an obsolete application manifest. Configure test binaries in root config/setting.json at performance.network_peer and performance.live_peer. `python3 test/performance/test_network_bench.py -v` uses independent owned Python H1 peers and owned h2c/TLS services for exact-body success, one-connection multi-stream behavior and deliberate wrong-body failure. `python3 test/performance/test_live_bench.py -v` verifies TCP reconnect and rejects wrong TCP/SSE bodies or an SSE hold without heartbeats. Both also provide a no-network check-config test. Announce owned loopback checks before running them; select bounded protocol validation after driver changes, never the full suite by default. Network examples build offline with the workspace release profile; Dever archive/source provenance is recorded separately from Rust reference builds. + +After changing CPU placement, assert the child reports the requested affinity without changing the parent, reject empty/duplicate/out-of-range/overlapping IDs, and verify serialized report workers/CPU sets. Run a bounded h2c/TLS+h2 smoke and minimal HTTP/1/live smoke against fingerprinted artifacts. To establish that the client actually exceeded one core, inspect `load.client_resources.cpu_cores_sampled` in high-rate raw cases, alongside service CPU and slot/dispatch drops. + +CMS recovery changes require `test_runner.py` and `test/ci/test_gate.py`: bounded pool growth passes, socket replacement/duplication and excessive pool handles fail, and read/recovery failures preserve case and aggregate evidence after cleanup. A real read check longer than the default 30-second HTTP idle timeout exercises the closed-control baseline; it does not replace the 1800-second soak. + +### 7. Wrong vs Correct + +Wrong: call warmed cgroup current the application's cold-start RAM; subtract two p99 values to infer request latency; equate a completed report with zero errors/OOM. Correct: report RSS/PSS alongside charged cgroup current/peak and OOM/throttling counters, distinguish warmed file pages and pre-exec Python peaks, record request latency directly, and inspect all failure counters before drawing conclusions. No global cache drops. Raw Hyper is a normal-GET baseline, not an equivalent implementation of production slow-peer/fault/structured-cleanup semantics. + +Wrong: require CMS total FD delta zero across idle control-client expiry and lazy SQLite pool expansion. Correct: close that client first, preserve exact non-pooled descriptor identity/count and independently enforce the database pool's descriptor bounds. + +Wrong: equate a 5-second saturation platform or a complete report with a sustainable 60-second rate. Correct: pick a rate below the observed platform, require at least 99% planned-request success over 60 seconds in both transport modes and memory budgets, and preserve nonpassing calibration data separately. + +For the 2026-09-18 HTTP/2 ceiling check, `--client-workers 4 --client-cpus 1,2,3,4` removed the former single-core client constraint while the Dever server stayed on CPU 0. A 16x16 `/plain` scan at 30k/45k/60k/90k observed roughly 36k–40k successful req/s at the upper rates, with the service near one CPU and request-slot drops dominant; this is not an isolated server-only ceiling. Short scans cannot select a reliable steady rate by themselves: 28k failed the 99% planned-request success gate over 60 seconds, while 20k passed for both h2c and TLS+h2 at 128 and 64 MiB. `test/performance/README.md` owns exact commands, counts, RSS/latency, failure and calibration reports, and the limits of the 100-cycle recovery evidence. Do not extrapolate loopback or warm-cache results to production. diff --git a/.trellis/spec/backend/toolchain-and-library.md b/.trellis/spec/backend/toolchain-and-library.md new file mode 100644 index 0000000..2247def --- /dev/null +++ b/.trellis/spec/backend/toolchain-and-library.md @@ -0,0 +1,417 @@ +# Toolchain And Official Library Contracts + +## 1. Scope / Trigger + +Application deployment targets belong to `toolchain::BuildTarget`, independently of the compiler host. Public `build --target linux-aarch64` may run on Linux x86_64; omitted targets and `run/test` stay native. Propagate the selected target through Lib preparation, CompileRequest, runtime pack selection, signature closure verification and both caches. Verify ELF machine/type for archive members and ordinary objects; a matching manifest label/hash alone does not prove architecture. A warm shared cache must still validate the selected target pack. Do not globally replace `platform_identity` or require an ARM compiler core for cross builds. + +Target runtime/interpreter/extension/exec/sandbox assets must match the deployment target. Go build tools explicitly declare `build.host`, while Go output and stdlib headers match `build.goos/goarch` and version. Their sandbox uses host assets; emitted Workers use target assets. Product processes keep cleared environments; author-only cross linker settings or Go overlays must not become application configuration/environment knobs. Python sdist/npm hook execution requires a matching execution environment or a previously prepared complete target receipt; never relabel host native output. The canonical author maps and preparation commands are in `sdk/native-release.md`. + +Executable modes are also part of the cross-target resource contract. Sandbox packaging and application embedding share `asset_executable`; classify both supported target loaders independently of the author's host, while retaining target ELF validation. Regressions must check actual packaged modes from non-executable source inputs and execute the embedded Worker. Host-only loader assertions and successful linking cannot establish target bootstrap behavior. Full-system sandbox fixtures must switch from initramfs rootfs to a normal root mount before bubblewrap uses `pivot_root`; do not loosen the production sandbox to accommodate a fixture mount error. + +Applies to formatter/CLI, reference execution, native ownership and clause optimization, concurrent sequence actions and official source packages. JSON algorithms belong to `.dever`; HTTP reuses Hyper in the runtime, with source-owned public types and static business handlers. `intrinsic.rs` remains the exhaustive bridge for checker, native emission and reference evaluation. + +The compiler author workspace requires Rust 1.95 or newer. SQLite is pinned to `deadpool-sqlite` 0.14.0 and its locked bundled SQLite dependency chain, which removes the observed WAL-close/replacement-open VFS lock inversion. Do not lower this requirement by reverting to the defective 3.51.1 engine, changing global toolchains or mutating Cargo registry sources. Initial developer dependency preparation may fetch explicitly; product run/build still consume prepared artifacts offline. See database-guidelines §9 for the root/connection lifecycle and regression evidence. + +Compiler bridge exception (explicitly authorized 2026-09-30): workspace Rust remains `unsafe_code = forbid`. Only `dever-backend-bridge` may override this with `deny`, and only its private `ffi.rs` may allow necessary foreign calls, runtime C exports and their raw-memory operations. The rest of that crate, its build script, core, runtime, CLI and tests remain safe Rust; no public unsafe Rust API escapes the private boundary. Each foreign call and output transfer documents pointer lifetime, buffer length, allocation ownership and serialized LLD state. This is not a Dever-language FFI feature. LLVM/LLD object/link tests do not establish complete HIR semantic equivalence or a shipped runtime pack. + +Keep `embedded` (LLVM/LLD compiler bridge) and `runtime-abi` (application staticlib) independently buildable. Runtime-only builds never link the LLVM/C++ SDK. Versioned signatures belong to `include/runtime.h`: fixed-width scalars/status, explicit output pointers, borrowed slices and matching owner releases. Decimal uses explicit little-endian low/high u64 words, not aggregate-by-value or Rust layout. Numeric success leaves Buffer null/0; errors preserve the original diagnostic and use only `dever_rt_v1_buffer_free`; compiler Reply has its separate C++ allocator. Reuse existing runtime numeric/collection/resource/protocol/system implementations. Application slices include CMD/wire/Dever/external Adapter, Model/SQL/transactions, HTTP/REST/auth/tenant/permission/Cookie/Upload/log and Job/Test. `runtime-api` adds owned application resources independently of `embedded`; `runtime-sqlite` and `runtime-postgres` select real drivers; `runtime-external` adds verified Worker resources. Private and managed Linux compilation use LLVM; formal six-target packs remain unfinished. + +Each task root owns a component Session and its prepared bundle, propagated into child Tasks, blocking work and API requests. Copy external metadata into owned strings/vectors; never leak C name arrays or reset a process-global registry between entries. Register a Worker before awaiting startup and attach its supervisor to the root Scope. Explicit shutdown can be followed by a new start; terminal close permanently rejects admission. Retain task ownership while awaiting shutdown so cancellation can resume cleanup. Close Workers before cancelling/draining descendants, then close databases and other application owners in the same runtime. Generated typed roots use `run_entry_with_typed_resource_cleanup` to append resource failure to an existing typed fault; the original Rust backend preserves its explicit shutdown/error-cause path. + +Private task constructors without suspension remain ordinary functions. Scope adapters should directly return the existing Tokio scoped future when no additional await is needed. An extra async wrapper can store both its input future and the child future, multiplying debug-build stack usage through nested callers; do not compensate by increasing thread stacks or boxing individual test phases. + +Private `dever run/build/test` uses checked HIR → LLVM → object → in-process LLD, without Rust/Cargo/cc lookup or fallback. Keep LLVM calls binary-private; shared `toolchain/runtime_pack` validates compiler-relative `runtime/<platform>/manifest.json`, normalized paths, version/v1 ABI/target, four profiles and exact CRT/archive digests. Shared code must not reference bridge target methods: actual linkage pulls LLVM into launcher/daemon even when the call appears to be metadata only. Reject symlinks, scripts, thin archives and incomplete references; snapshot/recheck selected inputs before linking. Private compilation reuses `native::compile_artifact`; managed compilation requires authenticated daemon IPC and never falls back. Ephemeral signed Linux fixtures do not establish trimmed or formal six-target release assets. + +Native authoring now uses the independent four-profile `runtime-pack` build recipe and `sdk/native-release.md`; the old all-features debug archive remains a test fixture only. Signing prepared inputs and compiling source are separate operations. Do not move Cargo/tool discovery into the maker or application run/build. + +Linux first installation reuses the release signature and machine version owner. Author `bootstrap` inputs provide launcher/daemon and their own complete private ELF library closure. The independently authenticated installer reads only its absolute root-owned `config/setting.json`; its public key must be supplied outside the downloaded release, then pinned atomically without key replacement. A bounded journal may name only the seven fixed bootstrap destinations, including the sandbox helper tree and dedicated AppArmor policy. Persist final file modes and child directories before publication; recover uncommitted changes, but never roll back a committed publication because cleanup failed. Stop a newly started service before removing its unit; restore previous service state after restoring files. Offline image installation never invokes systemctl. Incoming packages must match the supported service template; existing signed versions retain their original template and remain usable by pinned projects. Actual global activation and public release require separate operational authorization and evidence. + +Process main is separate from callable roots. Initialize Unix SIGPIPE policy before application threads, preserving stdout EPIPE as the existing located runtime error. Do not change an embedding host's signals from callable roots. Reuse typed fault render, indexed Test fault size/render/release, output cleanup and logger flush; Test main accepts exactly one nonempty unsigned decimal index below case count. Verify actual env-cleared CLI execution, relocated embedded Worker, selected-pack tampering on cache hits, broken stdout, and launcher/daemon dynamic dependencies. Six-target objects do not establish cross-platform execution. + +## 2. Signatures + +- Project creation: `dever new <new-directory> [--markdown]`; `project::create(&Path, bool)` consumes the single template source under the pinned `skills/assets/template` submodule. The machine launcher selects the active core without reading nonexistent target configuration. +- AI guidance: `dever skill path [project-root]` resolves and verifies the active or project-pinned release's `skills/dever-language`; `dever skill install <new-directory>` atomically publishes a stable loader that invokes the absolute machine launcher. Skill resolution is a public read-only operation and must not open the root-only install lock. + +- Application target: `dever build <root> --output <new-file> [--target linux-x86_64|linux-aarch64]`; explicit dependency preparation accepts the same target on `dever lib add/update/doctor`. Omission selects `BuildTarget::host()`. `CompileRequest.target: BuildTarget` is required and serialized canonically; test requests must use the host. + +- Private author entry: `native-release <author-root> --output <new-dir>` calls `toolchain::packaging::create(&Path, &Path) -> Result<ReleaseManifest, String>`. It reads only `<author-root>/config/setting.json`; this is not an application configuration section or another public `dever` command. + +- Runtime ABI examples: `uint32_t dever_rt_v1_int_add(int64_t left, int64_t right, int64_t *out, DeverRtBuffer *buffer)` and `void dever_rt_v1_buffer_free(uint8_t *ptr, uint64_t len)`. Buffer is `{uint8_t *ptr; uint64_t len;}`; Decimal output is `{uint64_t low; uint64_t high;}` and input words are separate scalar parameters. Full declarations live in the canonical C header, not independently handwritten per consumer. +- Managed ABI: Text/Bytes/List/Map use opaque owner pointers and paired retain/release; Id shares Text representation. A process-static immutable `DeverRtType` holds size/align and clone/drop/equal/hash callbacks for the actual compiled type. `_take` consumes the non-null receiver on success, value error or invalid output; other inputs are borrowed. List/Map cursors consume their receiver and release unvisited rows on early release. Optional output uses a separate u8 presence flag, and the value slot holds the complete element/value type, including its own Nullable flag when applicable. +- Application wire ABI uses the same process-static `DeverRtType` and synchronous typed encode/decode callbacks. Borrowed runtime wire Nodes and CMD Inputs never escape a callback or survive suspension. Validate aligned writable slots before parsing, consuming a field, selecting an Adapter or calling user callbacks. A successful decode initializes a complete typed row; if subsequent CMD `Inputs.finish()` or error-buffer validation fails, drop that row once. Partial failed rows remain the compiled callback's guarded responsibility. Keep CMD top-level `Inputs` policy distinct from nested checked wire parsing; required/unknown field wording and field-error precedence must match the Rust backend. Runtime `wire` feature supplies these owners without enabling a second protocol engine or LLVM SDK. +- Adapter settings ABI reuses `Settings::load_adapter_settings`, name validation and selection. Runtime owns only the temporary opaque config handle; generated concrete Setting slots retain decoded fields after that handle is released. Decode errors never render Secret content. Scope drain precedes typed Setting release and the module-local entry gate reset; repeated success, partial init failure and child Task execution must preserve allocation balance. +- API ABI: `dever_rt_v1_api_serve(session, routes, count, tenant_components, component_count, error) -> DeverRtAsyncOp*`; process-static `DeverRtApiRoute` references concrete callable/input/failure/auth/permission metadata. Claims pack borrows subject/session/optional tenant/site Text handles and initializes one concrete input row. Identity unpack borrows the returned concrete row, retains one id Text, and writes separate i64/u8 user/tenant value/presence slots. Release both unpacked id and the original output once. Runtime dispatches checked callbacks; source App is never looked up by string or called through dynamic business Value. +- Resource ABI: File retain/release shares the runtime File; open/create/read/write/close/chunks borrow inputs, and create uses create_new. File read has an independent presence byte. `DeverRtReadEvent` supplies a concrete descriptor and Chunk/Failed callbacks, retaining borrowed Bytes/Text into the full original Choice. Stream retain/close/pull shares the runtime cursor; internal pull has typed row/presence outputs, emits a terminal Failed once and then End, and validates writable outputs before advancing the cursor or doing I/O. +- Async owned ABI: `DeverRtOwnedType {size, align, move_init, drop, transferable}` has process-static callbacks; move initializes matching aligned storage and deinitializes the source owner. `DeverRtAsyncFunction` carries output/fault descriptors and create/resume/destroy/done/send_safe; create clones borrowed input before initial suspend. `DeverRtSyncFunction` carries input/output/fault descriptors and invoke/send_safe; invoke borrows runtime-owned input, returns 0 initializing output or 1 initializing fault, and never drops the input itself. Internal Unit rows are transferable, and internal Outputs rows derive transferability from fields; the source type properties are unchanged. +- Async polling ABI: `dever_rt_v1_async_root(function, input, output, fault, error) -> u32` starts one task Scope; `dever_rt_v1_async_op_poll(state, operation, output, present, fault, error) -> u32` polls under only the current Context; `async_op_release` cancels/drops one operation. Statuses are Pending=0, ReadyOK=1, ReadyTypedFault=2, RuntimeError=3, InvalidInput=4, distinct from regular ABI 0/1/2 and kernel 0/1. Pending initializes no result, presence or fault. Ready moves only the appropriate typed owner; numeric faults and owned RuntimeAbi buffers remain typed faults, not business failures. Initial/pending/final coroutine destruction uses paired frame_alloc/free and does not retain poll Context. +- Async concurrency ABI: task/group run accepts checked async descriptors or checked sync descriptors routed through the bounded blocking pool. Task/Group retain clones a wrapper owner, not the affine task/group; wait/stop consumes one wrapper and takes its inner value once. Group run and Channel operations borrow handles. Successful non-null blocking/parallel/sync-run/send operation creation consumes its typed input even if later pending, failed or cancelled; null leaves the input caller-owned. Channel receive's separate presence flag distinguishes End from Some(null) before language-level flattening. SendForeignFuture is constructed only after checked transitive frame/output/fault transferability; local root/call polling never gains Send by blanket unsafe implementation. +- AsyncStream/TCP ABI: constructors borrow source handles and return a shared opaque producer. Rows use concrete managed clone or OwnedType move callbacks, never padding copies; separate presence distinguishes End from a nullable row. A consumption scope closes aliases on drop. Socket/Listener retain/release preserves runtime identity; async operations clone borrowed handles into their operation owner. TCP standard choices retain original value errors. Timeout returns a complete Task row plus presence, race validates matching canonical output descriptors before consuming its task wrappers, and both reuse structured runtime drain. ParallelFunction packs an element/context into one checked input row for the existing runtime dispatch owner, validating frame/input/output/fault transferability. +- `List::into_values()` consumes unique elements and lazily clones shared elements. Emit consuming loops only for ordinary movable values; resource-containing lists retain lexical ownership. +- `List::into_first()`, `Map::into_get()` and `Map::into_entries()` transfer uniquely owned values and clone only through a live alias. `Bytes::from_string()`/`into_text()` transfer compatible Text/Bytes storage. +- `Render::render_to(&mut String)` recursively appends composite values into one caller-owned buffer; `render()` remains the allocating convenience boundary. +- `concurrent::each_slice(&[T], workers, apply)` batches memory sequences; `each(iterator, workers, apply)` retains producer-owned Stream pulls. Both share cancellation/join logic. +- `dever.http.serve(handler, Listener, Limits) -> ()` owns bounded HTTP/1 or HTTP/2 connections; `send(address, port, Request, Limits) -> Response` owns a single request and driver without a client pool. `result(send(...))` captures ResponseResult. Handler suspension is inferred. `Limits.http2: Http2Limits?` selects HTTP/2 explicitly; null selects HTTP/1. +- `http.serve_live(handler(Request, HttpReply) -> (), Listener, Limits, LiveLimits)` owns streaming/SSE/WebSocket handlers. `respond/start/write/finish` share checked headers and a one-slot body channel. SSE Event has Text event/data and nullable Text id/Int retry_ms. WebSocket accept/connect/send/receive/messages/close reuse tokio-tungstenite; public source owns records/results/Message. +- `http.client(origin, ClientTls, Limits, PoolLimits) -> HttpClient` creates a fixed-origin shared resource in the caller's task scope. `request -> Response` buffers; `open` and `open_stream` return StreamResponse with `AsyncStream<dever.io.ReadEvent>`. StreamRequest uses the same stream body. `close_client -> ()` closes aliases and drains its driver. Result choices describe explicit captures, not ordinary output signatures. +- `tls.system/client/server` own public roots, explicit custom CA and server PEM configuration. `http.serve_tls/serve_live_tls` and `websocket.open(ws/wss URL, ClientTls, Limits)` reuse the same verified TCP/TLS transport. All four system http_serve intrinsics accept an optional final transferable Context, appended to the static handler signature. +- `native::compile_with_cache(..., cache_directory) -> NativeProgram`; `cache_hit()` reports reuse. This explicit development/test path uses the workspace `target/native-artifacts`; a released `dever` must request the protected machine `deverd` service and must not expose that directory as a fallback. +- `native::compile_test_suite(&Program, &SourceMap, rustc) -> NativeProgram`; every suite entry must come from successful checking and remains private. The suite selects one stable checked case index per process. +- `toolchain::artifact_put(&Layout, &[u8]) -> ArtifactReceipt` and `artifact_get(&Layout, sha256, bytes) -> Vec<u8>` use authenticated daemon IPC. Receipts carry lowercase 64-hex SHA-256 and exact length, not a client-selected path. `CacheStore::summary()` returns metadata only with `verified:false`; administrator `status()` verifies bytes and returns `verified:true`. +- `toolchain::compile(&Layout, &CompileRequest) -> Result<Vec<u8>, String>` submits canonical JSON `{version, kind, target, sources, bindings, resources}`. Sources are logical path/test origin/text; bindings contain only database names/drivers, provider verify names and site paths/auth keys. Resources are normalized paths, base64 bytes, digests and executable flags. No project root, secret Settings, IR, output path or client cache key is accepted. `SourceMap::from_compilation_sources` forces strict layout and compiler-owned standard sources; `check_with_bindings` and `validate_database_bindings` share the ordinary configuration rules. + +- `format::format(&SourceFile) -> Result<String, Vec<Diagnostic>>`; `dever fmt <project-root> [--check]` prepares both `module/` and optional `test/` before committing either source set. +- `SourceMap::load(root)` accepts `.dever` and `.dever.md`; plain `.md` is not program source. It derives package identity from the relative filename, and both source formats use the same `parse/check/api/run/build` contracts. +- `SourceMap::load_project(module_root, test_root)` preserves separate display/logical paths and assigns Test origin internally. A missing test root is empty; an existing non-directory or unreadable root is an explicit load error. +- Static API declarations are generated into the async application entry; application source does not call `dever.api.serve()`. Runtime `api::Inputs` binds GET/DELETE query parameters, POST/PUT JSON objects or a statically selected POST multipart shape, and runtime response helpers build the fixed JSON envelope. `config/setting.json` alone supplies HTTP, site/auth and upload limits; application config never uses environment variables. +- Optional feature `reference`: `reference::evaluate(&Program, entry) -> Result<Evaluation, String>`, with captured stdout and structured source-span fault. No interpreter production CLI path. +- `parallel_each(handler, List|Bytes|Stream, workers[, context]) -> ()`; zero-output synchronous handler, 1–256 workers, no recursively nested Stream in worker input/context. In async code the handler must be a named function with no transitive concurrency effect. +- Synchronous and asynchronous parallel dispatch use typed failure paths. Generated handlers keep `AppError` and business-error identity across worker boundaries; String-only runtime helpers delegate to the same generic implementation and are not a compiler fallback. +- List/Bytes accept both non-suspending and inferred-suspending handlers. Sequential collection operations await internally when required; suspending `parallel_each` reuses the AsyncStream Group path with capacity 1–65536 and the global task cap. Synchronous Stream remains synchronous. +- `AsyncStream<T>` shares one Send producer/cursor and close signal. Traversal accepts effect-checked handlers and infers suspension; stream `parallel_each` requires a zero-output handler and a Group limit of 1–65536. Both `AsyncStream<T>` and `Channel<T>` require transferable T in every declaration position. +- TCP connect/listen/accept/read/write infer suspension; metadata, close and stream construction do not suspend. `net.chunks` returns AsyncStream; file chunks return Stream. Explicit captures use their respective ReadStreamResult choices. No synchronous TCP backend or compatibility aliases. +- `run`/`wait`/`stop`, bounded Group and bounded Channel are structured under one task-local Scope. Direct calls complete sequentially; `run` is the only call-level explicit concurrency operation and accepts both suspending and non-suspending static calls. `receive(Channel<T>) -> T?`. +- Source code cannot start a runtime. A fully non-suspending entry executes directly; an inferred-suspending entry starts the single bounded runtime. `timeout(Task, millis, zero-input fallback)` and `race(Task, Task, ...)` consume affine tasks with identical output signatures. `stream(Channel|List|Bytes)` and `task.ticks(millis)` are pull-driven AsyncStreams. +- `text.at(Text, Int) -> Text?`, `slice(Text, start, end) -> Text?`, `codepoint(Text) -> Int?`, `from_codepoint(Int) -> Text?`; positions count Unicode scalars, slice is half-open. Empty-separator split yields scalars without synthetic empty edges. +- `int.parse`, `decimal.parse`, `float.parse` return nullable values. Decimal remains finite/exact at parse; Float supports its rendered IEEE special spellings. `to_text` uses runtime rendering. +- `dever.process.arguments -> List<Text>` excludes argv[0] and rejects non-UTF-8 arguments. There is no application environment-reading capability. Application and benchmark configuration uses config/setting.json only. +- `dever.time` owns UTC now, strict DateTime/Date/Time parse/format, millisecond Duration construction and checked DateTime add/subtract/difference. Deterministic operations are pure; now has a time effect. Result choices expose Read(value)/Failed(message). Existing Int clocks and zero-output sleep remain. Chrono 0.4.45 (std only) validates calendars without a local timezone dependency. `dever.net.connect_timeout(address, port, milliseconds)` requires a numeric IP and positive timeout. +- JSON: `dever.json.parse(Text) -> dever.json.value.Document`, `stringify(Document) -> Text`; ParseResult/WriteResult apply to explicit result captures. node/property/element access returns nullable values or node IDs. +- HTTP: source-defined Header/Request/Response/Limits. Headers are a List of Text names and Bytes values; preserve duplicate values. Bodies are bounded Bytes. Incoming names normalize to lowercase; no promise of ordering across distinct names. + +## 3. Contracts + +Developer onboarding has two official repositories: `shemic/dever-main` and `shemic/dever-main-skills`. The skill directory is an independent Git repository, pinned by the compiler's submodule; it distinguishes this language from Go Dever. Author `skill` is a required list of `{source,path,sha256}` entries published as signed `skills/dever-language/<path>` artifacts, including SKILL.md; at most 256 files, 1 MiB each and 8 MiB total. New-project source bytes come from the same skill templates; Markdown documentation wraps them without a second semantic copy. Keep the generated configuration CMD-only and free of database/auth prerequisites. + +Official `install` and `update` explicitly download from the compiled GitHub Releases origin; neither project setting.json nor environment variables choose that origin. Each platform publishes `dever-<platform>.manifest.json`, `.manifest.sig`, `.tar.gz`, with immutable version tags `v<major.minor.patch>`. Verify the pinned signature before unpacking, require a matching skill, reject unsigned/duplicate/nonregular/escaping archive entries, and bound both metadata and payload. USTAR contains only signed artifact files; the two metadata files are separate assets. Redirects remain on the enumerated official HTTPS asset hosts. Privately staged publication never overwrites an existing release; malformed downloads leave the active version unchanged. The existing active-version journal is the sole commit point for both core and skill. The stable skill entry resolves it on each task; do not maintain a second independently switched skill pointer or rewrite every user's home directory. + +The skill's first-install helper uses its independently distributed public key, verifies the full package before executing bootstrap, and generates only an owned temporary installer config. It never reads business configuration. Real-root installation explicitly starts the machine service unless --no-start is selected; image-root verification must not touch host services. Python/OpenSSL are installer prerequisites, not application runtime dependencies. `sdk/release-assets.py` publishes the three deterministic assets with Linux NOREPLACE directory rename and cannot clean another writer's public output. + +Native author settings use `dever-native-release-input-v1`: version/target/signing_key, core and `core_libraries` inputs, required base/sqlite/postgres/both archive inputs, and ordered start/libraries/end. Input `source` is a normalized author-relative path and `sha256` is the exact lowercase digest; link inputs also specify the destination `path`. Core libraries use plain SONAME filenames and publish below `lib/`; runtime link inputs remain runtime-relative. Reuse the runtime v1 manifest types and validator. Copy into exclusively owned private staging, verify both source and copied digest, sort manifest artifacts, and sign those exact bytes with the supplied owner-private Ed25519 PKCS#8 key. Reject key content as payload, symlinks, path escape and reserved/duplicate destinations. Keep source paths, key material and timestamps out of release metadata. Linux publication uses atomic NOREPLACE; failures clean only owned staging. No tool invocation, environment configuration, downloads, input hard links or implicit trust-key installation belongs to this owner. + +`toolchain::core_libraries` validates the same complete ELF closure during authoring and verification of signed native releases (identified by the signed native runtime manifest). Require matching GNU target/class/endian, exact SONAMEs and core `DT_RPATH=$ORIGIN/lib`, with no core RUNPATH. Libraries may resolve only their own private directory; every non-OS dependency must be declared and every supplied library used. Reject loader audit/filter hooks, host `/etc/ld.so.preload`, unsigned flat leaves, symlinks and subdirectories including glibc-hwcaps. The installed directory must exactly match the signed regular files. GNU libc-family/loader are the explicitly declared OS ABI exception; LLVM and its transitive non-OS dependencies are not host prerequisites. Generic signed payload fixtures without a native manifest do not become compilable releases. Verify a real signed compiler can check/build in an owned root containing only declared OS ABI libraries, then run the output after deleting its source. + +Build archives with the root `runtime-pack` profile, one shared Cargo target, and independent features: runtime-api/runtime-external for every profile, then runtime-sqlite/runtime-postgres for the selected drivers. Never enable embedded or strip linkable symbols. Deterministic packaging of fixed inputs does not establish cross-host reproducible source builds. Assert actual driver separation and run the produced programs; a file named base.a is not evidence of trimming. + +Runtime ABI status is u32: 0 success, 1 original runtime value error, 2 invalid ABI input. Output slots must be live, writable, aligned and disjoint; borrowed inputs stay live and immutable for the whole call. Null input is valid only at zero length; reject lengths outside usize/isize range before constructing a slice. Caller releases any previous Buffer allocation before reuse, and releases each returned non-null allocation once through the matching runtime ABI. Error paths do not initialize numeric output. These are compiler-owned internal interfaces, not a language-level unsafe capability; application configuration gains no environment variables. + +Managed element storage is checked size/alignment with 8-byte-aligned `MaybeUninit<u64>` backing; descriptors and callbacks stay valid for the entire collection lifetime. Clone/drop/equality/hash act only through the concrete typed callbacks, never by reading/copying raw bytes or padding. Reuse runtime `List<Element>` and `Map<MapKey, Element>` COW/cursors. Element implements PartialEq, preserving NaN semantics in List and Map values; only MapKey wraps an eligible key with Eq/Hash. A non-null receiver passed to `_take` is owned by the call even if output validation fails. Successful optional output with presence=0 leaves its row untouched: never read it. Presence=1 initializes the entire row; its own Nullable flag remains authoritative. Text split receives the same Text descriptor used by ordinary List<Text> construction, not an independent runtime type identity. + +The runtime C ABI's value-error status does not by itself define a recoverable language failure. LLVM BytesToText/BytesFromInts/BytesSlice explicitly map status 1 to their checked standard Failed(Text), then use ordinary library fail/result propagation; other value faults (such as duplicate Map keys) remain RuntimeAbi. Status 2 always propagates as an ABI fault. Convert the original error Buffer to an owned Text handle, free the Buffer exactly once, and read value output only on status 0. LLVM business payloads belong to its typed fault tail and reuse compiler capture/drop owners, not a new C runtime error registry; details live in compiler-contracts.md. + +Protocol async operations validate output shape/alignment and required presence before polling, including operations whose successful output is Unit. Invalid metadata returns async ABI status 4 without changing the operation or output/fault sentinels; a corrected call may retry. Do not perform network I/O and only then reject the result destination. Protocol getters validate destination slots before allocating owned Text/Bytes/stream fields. Their fixed records and handles live in runtime.h; this is not a generic serializer ABI. + +Nominal type properties are computed once after cycle validation. Contract analyses reuse a shared dependency graph; monotone summaries enqueue only affected callers. Bundled AST is cached in process; each thread retains only its preceding exact path/text checked Program or diagnostics. Separate CLI processes still check all source and the API baseline. + +Native optimization works on a cloned checked clause: coalesce record aliases only while old fields remain equivalent; snapshot only ordinary scalar future reads with no intervening host writes; move only the final overlapping read of a replaced field. Direct split consumers reuse runtime text::pieces Unicode/empty-separator semantics. Fuse only non-resource stages with structurally proven total, synchronous, effect-free handlers; pure alone never proves totality. Preserve argument evaluation, numerical faults and left-to-right Float sums. + +Native argument bindings are disposable owned temporaries: a final movable read transfers the source, while a still-live or resource-containing source first clones according to value/resource semantics. Collection lookup/extraction and compatible Text/Bytes conversions consume that temporary so uniquely owned payloads are not copied. Composite rendering recursively writes into one String, including generated records/choices and named entry outputs; numeric `to_text` keeps its direct `to_string()` hot path. + +Memory parallel batches are capped at 256 elements and check the shared cancellation state between elements. Stream dispatch remains item-by-item. HTTP uses Hyper's incremental protocol state. Bytes shares engine buffers and slices; multi-frame body collection merges incrementally instead of retaining metadata per tiny frame. Empty concat reuses storage without altering aliases. + +Native cache keys use SHA-256 and include emitted input, compiler/toolchain identity, exact runtime/dependency bytes, options/platform/environment. A compiler-owned path/size/mtime memo avoids rehashing unchanged runtime files on every cache hit; any metadata change recomputes the exact content identity, and a cache miss verifies copied runtime bytes against that identity before compilation. Publish complete private entries atomically; restore verifies identity and executable integrity, copies into per-call temp ownership, and never skips CLI API checking. Generated Rust is not persisted. Cache digests are content identities, not publisher signatures. + +The machine toolchain has one stable `dever` launcher and a signed version store. A project may select one exact `major.minor.patch` only through `config/setting.json` at `dever.version`; otherwise the machine active version is used. The launcher verifies the signed manifest and selected core before reading project source, then preserves arguments and exit status. Version publication uses staging and an atomic directory rename. Active-version state is a checksum-valid append journal so an interrupted write leaves the preceding complete record selectable on Unix and Windows. Cargo's default `dever` binary is the development compiler and becomes `dever-core` in a release. Cargo's `dever-launcher` binary is the machine entry installed as `dever`; it must stay independent of LLVM. This is one public command name, with no legacy command alias. Builds do not replace global commands. + +`dever clean <project-root>` is a version-core project command. It removes only current-user Dever temporary build directories and legacy `.dever-run-*` files that pass exact name/type/owner/age checks and whose recorded process is no longer active, then reports directory, file and byte counts. It never touches explicit build outputs, project data, Lib artifacts, Cargo caches or the machine store. `dever cache clean` remains an authenticated `deverd` machine-cache operation with no local fallback; compiler contributors use `cargo clean` for the repository `target/`. + +`deverd` is the only writer of released cross-user cache entries. Its storage engine uses complete length-delimited build inputs, SHA-256 identities, immutable atomic entries, integrity checks, active-operation/read leases and maintenance locking. Released clients require platform-authenticated IPC; service unavailability fails without a private or uncached fallback. OS service installation and remaining platform peer adapters belong to distribution, not a caller-writable cache directory. + +Linux cross-user IPC uses kernel `SO_PEERCRED`, verifies endpoint/server UID against the machine state owner and keeps the administrator token owner-only. Foreign users may transfer Lib artifacts and query metadata, never clean or trigger whole-cache hashing. Eight slots admit at most two foreign connections per UID and six total. Artifact transfer accepts at most 64 MiB and stores at most 2 GiB/4096 entries. Compiled entries have separate equal quotas and are inaccessible through ArtifactGet/Put: retrieving a program requires its complete canonical compilation request. + +Compilation admission is two requests, with 384 MiB wire input, 4096/16 MiB sources and 4096/256 MiB resources (128 MiB each). The service verifies the installed release and complete runtime reference closure, including its native manifest and LLVM library, on every request before cache lookup. Identity binds canonical input and signed manifest; a persistent private install lock is shared through compilation/transfer and exclusive for install/uninstall, never unlinked. The whole request holds a cache operation; transmission retains its read lease. Only the signed core's hidden compile worker may produce a publishable program. It reads a service-owned 0600 request inside 0700 staging; trusted state/cache ancestors remain traversable and non-writable by other users. The worker reconstructs/checks sources and links, without loading deployment Settings, preparing Libs, executing App/Test or writing shared cache. Caller retains dependency preparation, baseline checks, program execution and per-case Test settings. Worker CPU/address-space/file/FD/core limits do not constitute an OS sandbox. + +Compilation has a 180-second absolute service deadline, including output transfer, and a 190-second client download deadline. EOF or any byte following the exact request body cancels it. Every failure kills/waits for the owned child and removes staging; diagnostics are capped at 64 KiB and output at 320 MiB. This output budget applies only to trusted compilation receipts, never opaque ArtifactGet/Put. Apply remaining time before each write/read; a per-call socket timeout alone permits slow peers to retain version and cache locks indefinitely. + +Binary resources cross the compiler bridge separately from bounded textual LLVM IR (8 MiB). The shared resource index verifies digests and binds path, length and executable status into IR/cache identity; declarations are initialized from borrowed bytes before LLVM verification and optimization. Reject duplicate, unused, defined or incorrectly typed/sized attachment symbols; six targets preserve the same byte order. Do not hex-escape interpreter payloads into IR or duplicate large byte buffers merely to sort canonical requests. Attachment limits match the managed resource count/byte budgets, and bridge object output is bounded at 320 MiB. + +### Signed ecosystem pack contract + +#### Scope / trigger + +Native release authoring also packages Python/Node interpreters and Go build tools; Worker preparation consumes the same runtime descriptor validator. The author recipe lives in [sdk/native-release.md](../../../sdk/native-release.md). + +#### Signatures + +`packaging::create(author_root: &Path, output: &Path) -> Result<ReleaseManifest, String>` and the private `native-release <author-root> --output <new-directory>` command retain their signatures. `workers::runtime::validate(files, ecosystem, target)` owns descriptor/file-closure validation for both maker and consumer. + +#### Contracts + +Author `config/setting.json.runtimes` optionally contains `pip`, `npm`, `go`; each has `name`, `version`, `files[{source,path,sha256}]`, with Python `python_markers` and `python_wheel_tags`. Configuration is bounded at 1 MiB. Sources are explicit regular files below the author root, never env/PATH lookup. The deterministic archive and registry manifest are both signed artifacts; neither an upload nor a filename can substitute for their verified identity. Prepared trees have their own 256 MiB/65,536 file bound; final embedded applications must also fit the stricter compilation limits above. + +#### Validation and error matrix + +| Input | Result | +| --- | --- | +| Missing interpreter, Go tool or importcfg archive | Reject incomplete pack before publication | +| Wrong ecosystem/target, Python flags or registry metadata | Reject runtime identity/descriptor | +| Duplicate/escaping paths, file/directory conflict, symlink, wrong digest or signing-key alias | Reject author input; remove owned staging | +| Existing output or concurrent second publisher | Preserve published output; reject replacement | + +#### Good / base / bad cases + +Good: complete relocatable interpreter plus stdlib, or Go tools plus every referenced stdlib archive. Base: omit `runtimes` for an otherwise complete native-only release. Bad: copy only host Python and assume its host stdlib will exist on the destination; a descriptor alone cannot prove portability. + +#### Tests required + +`native_release` verifies deterministic archive/signature bytes, computed runtime identities, bad inputs and rollback. Explicit ecosystem acceptance must lock from a signed install, execute all three managed check/run/build paths, then run programs after removing source and machine stores inside a root with no system language installations. OS baseline libraries, test key, target and untested third-party registry behavior remain explicit. + +Linux no-host acceptance must use a genuine mount-namespace root, shared by core, Worker and bootstrap fixtures. A plain `chroot` leaves the process root different from the namespace root; Linux then rejects the Worker's nested `CLONE_NEWUSER` with EPERM, even for root. Use the explicitly prepared bwrap/loader assets to bind only the owned OS root, private proc and synthetic devices. Do not remove the product's `--unshare-all` or add host library/tool mounts to make the fixture pass. A successful managed run does not replace source-free standalone execution under this boundary. + +Managed Go compilation removes the invocation-owned stage prefix from recorded source paths with `compile -trimpath`. This applies to the SDK, Adapter and dependency packages through their single `Builder::compile` path. Preserve logical filenames/line numbers; do not strip diagnostics to hide path drift. The real `external_workers` Go fixture must prepare twice in fresh stages and compare every emitted resource identity, then execute after source removal. The signed three-ecosystem run/build test must publish exactly three compiled entries. A single successful Go execution does not prove reusable cache identity. + +#### Wrong versus correct + +Wrong: encode a 118 MiB Node binary as LLVM text, raise the IR cap, or relax opaque artifact upload limits. Correct: pass checked bytes through `emit_object_with_resources(ir, target, resources)`, keep the IR cap, and apply the separate trusted compile/output budgets. Test author compression deadlines do not change the service's compile deadline. + +Formatting is syntax-only, stable and idempotent. It preserves comments, literal values, explicit groups and source record-expression order. Prepare every source before rewriting any; snapshot changes, syntax errors, symlinks and staging failures do not modify originals. Each final replacement is atomic, not a multi-file OS transaction. + +Application-test execution reuses checked HIR, specialization, runtime feature trimming and native cache; it does not link the reference evaluator or add an interpreter. Each case owns its temporary executable/project directory. Production check/api/run/build source sets and cache inputs exclude `test/`; fmt intentionally includes it. + +Port native dispatch is a direct call for one implementation or a closed match for several. `adapter.<port identity>.use` in setting.json selects an immutable startup binding; unknown choices/fields, duplicate keys and missing required settings fail before user code. A selected Adapter's contextual `setting` is a private record decoded with shared SettingInput wire helpers into a OnceLock; only that implementation requires values. Secret errors never include input values. Port-only binaries enable the wire feature without API/database runtime; a no-setting single implementation may omit setting.json. Runtime configuration uses the bounded duplicate-aware wire parser before typed config parsing. + +External exec and managed dispatch remain behind the same selected Port binding. All executions require a verified embedded launch manifest, selected Worker tree and signed sandbox assets; no adjacent-file or bare-process fallback exists. CLI preparation reads sandbox leaves from the authenticated installed release, including for exec. Runtime clears the environment, maps only verified launch arguments, and uses length-framed stdio. Native and LLVM dispatch reuse one invocation settings snapshot for Adapter selection, business settings and deployment grants. Component tasks reuse the structured application Scope and bounded queues. `hello/ready/health` must finish before admission; request ids are sequential and exact; timeout/cancellation sends `cancel` and drains one matching `result` or `error`; application cleanup sends `shutdown`, joins, then uses a bounded kill only when the child misses the deadline. + +Python SDK input uses an owned asyncio pipe transport and bounded message queue. Cancel and join its reader on normal shutdown and protocol failure, then close the transport. A daemon thread blocked on `sys.stdin.buffer` can abort CPython finalization even after a successful reply. The SDK regression must wait for successful process exit after the shutdown reply while the parent's stdin writer remains open; closing it before waiting masks this lifecycle defect. SDK source changes require re-embedding the source through the host compiler before deployed application verification. + +`dever-sandbox` owns the Linux namespace/seccomp launch boundary for Workers and Go build tools. The signed release supplies bwrap, the safe Rust guard and a bounded, complete ELF library closure; the loader cannot select libraries through environment or cache, and a host preload configuration is explicitly unsupported. Mounts are opened with no-symlink `openat2` and passed as owned descriptors to bwrap bind-fd, preserving the authorized inode across rename/replacement. Never replace this with a path check followed by a path-based bind. Input trees are read-only; output grants cannot overlap executable inputs. The runtime additionally rejects writable grants overlapping `data/cache/lib`, including its ancestors. Sandbox descendants inherit the syscall filter; namespace termination stops even children that changed process groups. + +The signed bwrap must be a bounded static target ELF without an interpreter or dynamic dependencies and is executed directly. On Linux with restricted unprivileged user namespaces, a non-root launch requires the matching digest-addressed `/opt/dever/sandbox/<sha256>/bwrap`, root ownership of every ancestor, no shared writes/symlinks/setuid/setgid/file capabilities, and both dedicated AppArmor profiles loaded in enforce mode. The helper bytes must match the embedded signed asset. The runtime and real-host bootstrap require `kernel.apparmor_restrict_unprivileged_unconfined=1` before using/publishing this policy: at zero, unconfined callers can enter a capability-granting profile through `aa_change_profile`, independently of its executable attachment. Missing or invalid strict-policy state fails closed. The product only reads these flags; administrator-approved installation/loading owns host changes. Root or unrestricted-host execution uses the same embedded static helper, never an unsandboxed fallback. Test normal and immediate profile changes, with and without no-new-privileges, using the same-process negative probe; wrapper failure or unavailable setup is not a successful isolation check. + +With `process=true`, POSIX spawn may reset only the effective UID/GID to the guard's captured real identity: `setresuid(-1, uid, -1)` and `setresgid(-1, gid, -1)`. Compare all three arguments as 32-bit kernel IDs; changing real/saved IDs or choosing another effective ID remains denied. Without the process capability both calls remain denied. Keep clone3 returning ENOSYS so libc falls back to the flag-checked clone syscall. The actual C probe must cover both capability states, high-bit-equivalent IDs and rejected identity changes; an unrestricted setuid allowlist is not an acceptable build compatibility fix. + +Deployment file grants live only at `adapter.<port>.files.<name> = {"path":"data/upload", "write":true}` in setting.json. Checked `allow file` is required, names are bounded lowercase identifiers, paths stay below the application, and write defaults to false. Workers see `/data/<name>`; grant configuration is not business `setting`. Network capability permits IP sockets and supplies only bounded resolver files; host Unix sockets and raw sockets stay denied. Process capability controls fork/process clones separately from threads. GPU and unsupported OS backends fail explicitly. Linux additionally requires host permission for user/mount/PID/network namespaces and openat2; root fixture success does not establish non-root deployment support under AppArmor. + +Go tools consume a private read-only input tree, including parent-generated import/embed configuration, and a separate writable output tree. Parent reads of generated programs must use nofollow, regular-file and single-link checks with a byte limit; never follow a tool-created output symlink on the host. Runtime/SDK/build assets and source remain explicit, without host compiler discovery or implicit networking. + +When a build carries compiler-owned external resources, the native entry may include a deterministic resource table. `dever-runtime::external` releases it only below the executable's private `data/cache/lib/<digest>` directory, rejects symlinks/path escape and digest mismatches, and publishes a complete staging directory atomically. Existing entries are reverified; a concurrent publisher is reusable only after the same manifest verifies. This extraction path never downloads, consults `PATH`, or accepts runtime paths from environment variables. + +Resource SHA-256 must match its bytes before native cache lookup. The bootstrap emitter references private, fixed-name binary compiler inputs with `include_bytes!`, not decimal arrays; identical checked digests share one static payload/input while each resource retains its own path and executable bit. Emitter and build staging reuse one input-index owner so they cannot diverge. The build owner stages each unique input exclusively and removes it with the owned build directory. Runtime integrity checks compare exact length, every 64 KiB chunk and digest against embedded bytes without allocating another whole interpreter image. + +Markdown Port function metadata adds `- 允许失败：` after outputs, exactly matching the fails type. Qualified Adapter/fake names remain complete in their function sections and private in public-method lists. The setting declaration uses a Type section named Setting with its ordered field metadata. + +Markdown selects only top-level fenced blocks with literal info `dever` or `typescript dever`, using pulldown-cmark offset events rather than scanning fence-like lines. Quotes/lists/raw HTML/outer showcase blocks are presentation. Fences use backticks or tildes with CommonMark 0–3-space indentation and require explicit closure. Both formats own one package each; duplicates are C002, never merged or selected by priority. + +Every `.dever.md` has exactly one natural-language H1 source section. Its ordered metadata is path-derived source identity, role-derived public types/functions and logical App call examples; H1 has no executable program block. Every type or logical `(function name, arity)` group has one natural-language H2 section and one program block. All clauses in a logical function share that block and input names. Type fields/variants and checked function parameter/output names, types, counts and order must equal their documented inline-code values. Empty collections use explicit `无`; per-field/variant/input/output descriptions are nonempty. H3 and deeper supplemental content may follow the section's program block. + +Closing-fence trailing whitespace is spaces only; a tab is rejected as an unclosed program fence (M002). The Markdown parser receives a same-byte-length view with bare CR changed to LF because it does not recognize bare CR itself; source text, code lexing and diagnostic positions still use original bytes. CRLF is kept intact. Do not normalize entire files or introduce an offset-changing source transformation. + +SourceFile text, cache keys and CLI snapshots retain original Markdown. Lex each block in its original byte range so Text/comment scanning cannot enter prose. Parsing and formatting share extraction, structural documentation parsing and block validation. After function signature resolution, semantic checking compares documentation with syntax/HIR before checking bodies. Formatting reuses the AST emitter per block, keeps comments in their block, preserves external/fence bytes and the opening fence's newline style. Runtime faults keep original positions through HIR. The Markdown dependency belongs only to the compiler. + +Native last-use analysis walks actual HIR read order backwards, kills full assignments, retains field hosts and outputs, and identifies concrete expression nodes. It may move only ordinary values: resource aliases have observable lexical scope. Share recursive type-property traversal for comparability, transferability and movability. Full-domain Int/finite Decimal guards need no runtime comparison; actual clause bounds must remain. + +`native::OPTIMIZATION_ARGS` owns production and baseline optimization flags: level 3, one codegen unit, fat LTO and stripped symbols. The workspace release profile applies matching fat LTO/one-unit settings so the runtime rlib retains bitcode for the final native link. Multiple units caused larger generated programs to miss the same generic hash-table inlining performed in a small equivalent baseline. These settings preserve that optimization without weakening overflow, hashing or panic semantics, at the cost of compilation parallelism and higher compile time/memory. Stripping removes release-only debug/symbol payload without removing source-location strings used by Dever failures. + +Application test suites use level 1, 16 codegen units and stripped symbols without LTO. Tests validate observable semantics and source locations rather than production optimization quality; `run`, `build`, performance fixtures and baselines continue using `OPTIMIZATION_ARGS`. + +Concurrent actions use a caller-owned producer, bounded queue and joined workers. Exact-size finite iterators create at most one worker per item/batch; unknown-length Stream iterators retain the explicitly requested limit. Faults stop further dispatch at cooperative pull boundaries and already-running work drains. Pending blocking operations require explicit OS timeouts; there is no forced cancellation or detached task. The reference evaluator chooses a documented sequential valid schedule; concurrent stdout ordering is not a differential invariant. + +Inferred-suspending entry points own one bounded Tokio runtime. Child Scopes share task and blocking semaphores but keep independent work registries; body cancellation cannot cancel the supervisor responsible for recursively draining descendants and started blocking jobs. Transitive effect checks, including concrete handler substitution at wrapper calls, keep blocking system operations, stdout and hidden collection-handler concurrency off Tokio scheduling workers unless they use an explicit bounded boundary. + +Async stream consumers reserve capacity before pulling and observe child failures while the producer is idle. Close releases the producer's resource alias, not independently held Socket/Listener aliases. TCP has independent direction locks and a shared close signal. Read buffers allocate after readiness. Operation deadlines include direction-lock wait; partial writes close the socket on abandonment before releasing the write lock, with no whole-message retry. DNS uses the shared bounded blocking path. + +Single `read(resource, limit)` uses the caller-selected maximum buffer and must not silently clamp a valid limit: regular files may satisfy the whole request in one operation, and existing programs rely on that result. Reads may still be short according to the OS. Memory-constrained programs choose a smaller limit or use the pull-based `chunks` interface; the runtime cannot both return a `limit`-sized value and avoid storing that value. + +JSON uses an acyclic flat document: consecutive postorder IDs, child IDs smaller than parents, Map insertion order independent of ID order. Numbers retain raw JSON spelling, object names are unique, Unicode surrogate escapes must pair. Maximum depth is defined once in `dever.json.value`; stringifier checks references and output expansion before concatenating child graphs. + +Typed wire boundaries are separate from the ordinary source Document API. Runtime `wire` (enabled by api) owns a borrowed RawValue parser with duplicate checks at every object, total byte/value/depth budgets (16 MiB / 65536 / 64 containers), exact raw Json and bounded output writing. A missing object value must retain its incomplete frame; a byte-limit failure after a partial write permanently prevents `finish` from publishing Encoded JSON. Generated concrete decoders reject unknown/missing fields and never pass Node/Value to business signatures. API envelopes embed only validated Encoded JSON, preserving raw numeric spelling. Decimal/Id/Uuid use strings; DateTime normalizes offsets to UTC Z with optional three-digit milliseconds, Date is YYYY-MM-DD and Time is HH:MM:SS[.sss]. Reject leap seconds, submillisecond precision and calendar years outside 0000–9999. Duration remains a JSON i64. Select root `time_codec` with api/reference for the concrete native decoder, policy, parser budgets, native/reference time, Markdown parity and in-process API dispatch checks; it opens no listener. + +HTTP/1 server connections reuse AsyncStream/Group and the same task cap, with capacity reserved before accept. Hyper owns framing/keep-alive/pipelining; source handlers specialize directly. Preserve exact record-field contracts at the intrinsic boundary and include callbacks in effects, dependencies and specialization. Header/body/handler/stalled-write deadlines release peers. Configuration, accept and handler faults propagate through serve; ordinary peer failures end only that connection. send errors return Failed, with driver and request polled together. No old wire/transport API, detached driver, fallback parser, TLS/HTTP2, nonempty trailers, upgrades or streaming response in N2. Tests compare with independent wire peers. + +N3 live HTTP adds one structured handler per connection, using a Group of one under the same global cap. Response body Drop cancels even HEAD/buffered handlers; a Session guard ends the stream or upgraded socket and invalidates escaped aliases. Ordinary stream idleness is a peer failure; handler faults retain their original error and cannot become a successful chunked EOF. An upgraded Hyper driver finishes before its WebSocket handler: the connection owner must wait for that handler and keep its capacity reservation. Take the concrete TCP/TLS Transport and pre-read bytes out of Hyper on upgrade so the HTTP write timeout does not leak into WebSocket policy. + +LiveLimits owns maximum chunk bytes, stream idle deadline and shorter SSE heartbeat interval. A one-slot channel bounds pending chunks; business values, current writes, engine and OS buffers are additional memory. SSE normalizes CR/LF/CRLF without discarding trailing data lines, rejects event/id field injection and checks encoded length before allocation. Heartbeats are three-byte comments generated by polling the body; replay/history is application-owned through Last-Event-ID. + +WebSocket data message and write-buffer sizes are bounded, including fragmented messages; read buffer is 4 KiB. Independent direction locks permit simultaneous receive/send. Acquire the writer lock before arming the partial-frame cancellation guard: cancelling a lock waiter cannot close another task's active frame. Endpoint.take wakes ordinary operations while transferring the final transport owner to close-handshake cleanup. Control frames, masking and UTF-8 belong to tungstenite. receive/messages must be polled to process Ping/Close; no detached reader or automatic Ping task. Limits are per operation. N4 adds WSS; subprotocol negotiation, compression and close metadata remain deferred. + +N4 pool ownership uses separate request and physical-connection permits, each bounded by Limits.connections; idle and queued drivers count as physical connections. One structured task polls bounded queued/running Hyper drivers and idle expiry. Construct its owner guard before task creation so dropping an unpolled child closes escaped pool aliases. Never detach a connector/driver or retry partially sent requests. Return a connection only after complete EOF; body drop/fault/cancellation aborts that entry. Streamed transfer bounds chunks/read waits, buffered request additionally bounds total body/time. TLS shares trust/session caches with full certificate/name verification and small explicit cache capacities; public roots are bundled, custom roots replace them. + +N5 HTTP/2 reuses Hyper/h2 with fixed receive windows, 16 KiB frame/send-buffer settings and decompressed header budgets. TLS must negotiate h2 through HTTP-specific derived configs; do not mutate shared WSS trust/config or downgrade. Cleartext uses prior knowledge. Source targets remain path/query; normalize authority to Host when Host is absent and reject disagreement. Connection-specific fields are rejected rather than silently rewritten. HTTP/1 upgrades remain the WebSocket path; no extended CONNECT, h2c Upgrade, push or nonempty trailers. + +Hyper's synchronous Executor must spawn registered Dever child Scopes. A scoped connection owner drains these tasks and their started blocking descendants before releasing its physical guard. Serialize closing admission and child registration; task completion must include fault reporting and registry removal. Cleanup belongs to an uncancellable registered supervisor, even when the caller awaiting stop/finish is cancelled. Never take a Scope work registry in a cancellable caller and lose the captured completions on Drop. Each live HTTP/2 body owns its handler Group under that stream's Scope, so reset before/after response head cancels only that handler; report handler errors explicitly because Hyper consumes HTTP/2 stream errors internally. + +HTTP/2 clients lease streams independently from physical connections, with total requests bounded by connections × streams (at most 65536). Upload and download share the stream lease; neither idle expiry nor another stream's cancellation may abort an active neighbor. Create connection scopes when handshake jobs are polled by the persistent pool driver, never under a temporary request Scope. Queued, idle and draining connections share the physical budget. GOAWAY blocks new dispatch while accepted streams drain. Only Hyper TrySendError returning the unsent request permits requeueing within the original deadline; release the old upload lease first. Never replay a sent request or infer dispatch safety from an error string. Single-use send has one deadline spanning TCP, handshake and exchange, followed by mandatory cleanup. Server first-request and protocol liveness deadlines prevent silent peers from holding connections indefinitely. + +Listener close ends HTTP accept normally and triggers Hyper graceful shutdown while active response/upgrade owners drain. Public net.connections retains a terminal Failed on explicit listener close. Applications use timeout(server, grace_ms, zero-output fallback) for bounded drain. timeout/race borrow supervisor handles while waiting, cancel and drain losers/expired work before returning, and preserve runtime faults. Source stream adapters retain failure obligations and nullable Channel items; closing the adapter does not close independent Channel aliases. + +Create a response-body cancellation owner only after the pending-to-committed state transition is accepted. Dropping an uncommitted replacement body must not close a previously committed response. Rejected duplicate start/respond/SSE/upgrade calls leave the existing stream usable; head delivery failure still cancels the committed body. + +## 4. Validation & Error Matrix + +- New target already exists (including empty directory or dangling symlink), parent is missing, or two creators race: explicit error for the loser; preserve the existing target and remove only owned staging. +- Signature/payload/skill corruption, incomplete archive or absent official asset: explicit failure, no active-version switch or fallback. Publishing source code is not evidence that a downloadable binary release exists. +- Project has a pinned version: `skill path <project>` resolves that exact installed version. Foreign users must read matching guidance without obtaining the administrator install lock. + +Cross build: unknown/duplicate/missing target arguments fail before execution; absent target packs, wrong ELF/archive machine, unsigned target leaves, incompatible lock/runtime/build receipt and changed target inputs fail before cache reuse or linking. Sandbox loader mode must survive both release packing and embedding; a hash-valid non-executable loader is not a usable Worker resource. + +Native maker: wrong format/target/version, unknown or duplicate fields, missing profile, non-regular or escaping source, SHA mismatch, exposed/invalid signing key or payload alias to key → explicit failure, no output, owned staging removed. Existing or concurrently created destination → no overwrite. Correct bytes → complete signed v1 directory, without private key/configuration. Unsupported host publication → explicit failure, never a weaker rename fallback. + +Managed input with duplicate fields/noncanonical encoding, unsafe paths or forged resource digest -> reject before publication; invalid source -> original checker diagnostic from worker; unavailable service -> explicit failure; signature/runtime omission or tampering even on a cache hit -> reject; full admission/quota -> bounded error; cancellation/expiry -> reap child and release staging/locks without publishing a partial executable. + +External Lib declarations live only in the root `lib` array of `config/setting.json`; reject old `config/lib.json` rather than silently combining it. Canonical v2 locks bind checked Port/Adapter/schema/operations/capabilities and independent Worker roots. Doctor rejects a stale binding, missing/cyclic closure or within-environment version conflict; independent Workers may select different versions. Embedding compares complete normalized provider metadata, including dependencies, runtime, schema and artifacts, not just an artifact subset. + +Production registry access derives from a verified installed version core, not host tools or development fixtures. Signed assets `runtime/<ecosystem>/<target>/manifest.json` (`dever-registry-runtime-v1`) bind a RuntimePack identity and the adjacent `runtime.pack`; Python metadata additionally supplies target markers/wheel tags. Resolver owners implement PyPI PEP 508/440, npm semver/SRI and Go proxy/MVS with fixed HTTPS origins, disabled proxies and bounded work. Redirects are rejected except for Go `/@v/*.zip`: manually follow at most two hops into the exact HTTPS `storage.googleapis.com/proxy-golang-org-prod/` bucket. Validate normalized host/port/path and reject userinfo, fragments, missing/duplicate Location and loops before issuing a subsequent request; all hops share one 30-second deadline and diagnostics must not echo signed URLs. Loopback mirrors are explicit test transport, never project configuration. Unsupported extras/sdist, install hooks and Go sumdb are not silently approximated. Only explicit Lib commands resolve/download; run/build consume locked cached bytes. + +`workers::prepare(root, Program, resources)` expands verified runtime packs (`dever-worker-runtime-v1`: ecosystem, target, executable, arguments; Go additionally has `build`) and owned Adapter sources into a per-contract `workers/<digest>` tree. Python uses exactly `-I -S -B`, installs the packaged SDK path and imports that trusted SDK before exposing business source/dependency paths, and reserves stdout for protocol; JavaScript binds ESM/CommonJS exports. Both bind each checked Port operation by name using `sdk_manifest()`. Go consumes fixed-target compiler/linker/source selector and stdlib importcfg from the same exact locked pack, compiles the SDK and checked-operation runner alongside target-selected Adapter files and locked module ZIPs, then embeds only the executable and checked contract. Go source exports one exact `func(context.Context, any, any) (any, error)` per operation; `send` maps to `Send`, dotted segments to PascalCase. Cgo, assembly and unlisted imports fail explicitly. Build tools run by absolute path with an empty environment and bounded lifetime; product compilation does not invoke host `go build`, `PATH`, `GOOS`, `GOARCH`, `GOCACHE` or `GOPATH`. `<entry>.dever-worker.json` (`dever-worker-launch-v1`: ecosystem, entry, executable, arguments, working_directory) accepts only manifest-bound resource arguments in that same tree and bounded literal flags. Runtime revalidates the entire private bundle before every launch/restart and clears the inherited environment. Discard consumed raw runtime/archive resources after expansion unless config roots or exec Workers still need them. A private fixed-target pack fixture proves the local compile/link path; signed release packs, Go sumdb and cross-platform distribution remain separate acceptance work. + +Packaged exec Workers resolve only against the entry's embedded manifest. Bundle identity includes path, byte digest, length and executable bit. Validate all path segments, Unix owner/private permissions and file bytes on both cache reuse and every Worker restart; do not scan other bundles or fall back to source workers. Publish owner-only files/directories atomically before exposure. These protocol and integrity checks do not establish OS capability isolation; verify the sandbox launch boundary above separately. + +`dever.crypto` uses the existing source Result choices and exhaustive Intrinsic bridge. Secret storage lives in the base runtime with zeroize; Argon2id, SHA/HMAC and OS randomness are in the optional crypto feature. Native runtime feature selection inspects already-pruned emitted crypto calls so direct system intrinsics, API roots and Test roots are covered. Password hashing uses RustCrypto Argon2 0.5.3, Argon2id v19 and default Params; PHC verification bounds algorithm/version/memory/time/parallelism/output before computing. Failure messages contain no input. Hash/verify use the existing blocking-worker effect boundary. `dever.process` exposes arguments only; application settings never come from environment variables. + +Cache miss -> compile and publish; changed inputs -> separate key; incomplete/modified published artifact -> explicit error; unchanged sources with modified API baseline -> reject before native reuse. Concurrent writers may reuse only a complete same-identity entry. + +| Condition | Result | +| --- | --- | +| Async op Pending; ready typed fault; runtime error | No output initialization; move checked fault with original kind/source; move original error bytes into RuntimeAbi fault respectively | +| Consuming async operation creation returns null | Keep caller-owned input live for cleanup; a non-null operation owns the input through completion/cancel | +| Invalid File/Stream writable slot | ABI invalid-input status before I/O or shared cursor progress | +| Null/misaligned runtime ABI output, null nonempty input or unrepresentable length | ABI status 2; never dereference that pointer or read an uninitialized value output | +| Int overflow, numeric division by zero, nonfinite Decimal bits or invalid UTF-8 | ABI status 1 with owned original runtime diagnostic; caller frees Buffer exactly once | +| Invalid element descriptor/layout, or missing Map-key hash callback | ABI status 2 before constructing element storage; callback pointers and concrete row layouts remain compiler-owned lifetime/type obligations | +| Optional extraction succeeds with presence=0 | Do not read the uninitialized row slot; return zero Nullable and release consumed receiver/unvisited cursor rows | +| Recoverable Bytes operation returns status 1, or any returns status 2 | Bytes status 1 becomes its original Failed(Text); status 2 remains RuntimeAbi and is not caught by result | +| LLVM-generated Windows aggregate object linked without target runtime support | Explicit missing `__chkstk`/`_fltused` diagnostic; no successful output or fixture replacement symbols | +| Invalid format input anywhere in source root | Diagnostics, zero source rewrites | +| Markdown source has no program blocks | M001 at original source | +| Program fence is not explicitly closed | M002 at opening fence | +| Package header/type/function crosses program blocks | M003; no format writes | +| Package or declaration section is absent, duplicated or contains the wrong declarations | M004/M005; no format writes | +| Documented package/API/declaration name differs | M006 with documentation and code locations | +| Documented function input differs | M007 with documentation and code locations | +| Documented output/type field/variant differs | M008 with documentation and code locations | +| Same package in `.dever` and `.dever.md` | Existing C002 duplicate-package check | +| `fmt --check` finds changes | Exit 1, no writes | +| Ordinary alias remains live | Preserve independent value semantics | +| Resource-containing local reaches last read | Retain lexical lifetime; no earlier socket EOF | +| Invalid parallel count | Source-located failure before pulling or invoking handlers | +| Stream-containing worker input | C005 before native compilation | +| Invalid scalar index/number input | Null, never a default zero or lossy truncation | +| Exact finite input shorter than requested workers | Same visits/results with no more workers than items/batches | +| Valid one-shot read limit | Caller-selected maximum is passed to the OS read; no runtime clamp | +| Composite/named-output rendering | Exact existing bytes, appended through one result buffer | +| JSON grammar/reference/depth/output violation | Explicit Failed with offset/node and message | +| Component hello/ready/schema/operation/capability mismatch | Startup fault before request admission; child is killed and reaped | +| Component duplicate/unknown/out-of-order/truncated/oversized message | Fail the current and queued calls, close the generation and reap the child | +| Component call timeout or caller cancellation | Send one matching cancel, drain one matching result/error within a deadline, otherwise terminate the Worker | +| Component shutdown with active or queued calls | Close admission permanently, cancel the active call, drop queued calls, then bounded shutdown/kill/wait | +| Managed launch identity/path/argv/runtime drift | Reject before executing any interpreter; no host fallback | +| Shared artifact bad hash/length, overflow, truncation or quota exhaustion | Fail without publication; clean the owned staging and release operation/admission guards | +| Foreign UID requests cache clean | Reject by kernel caller identity, regardless of supplied token | +| HTTP peer framing/body violation | Hyper rejection or explicit error response/connection close, no handler invocation | +| HTTP invalid outgoing message | send Failed or serve task fault; never silently rewrite framing | +| HTTP/2 TLS peer cannot negotiate h2 | Explicit TLS/ALPN failure; no HTTP/1 downgrade | +| HTTP/2 connection-specific header, or response TE header | Explicit error; request TE allows only trailers | +| HTTP/2 reset before/after response head, including a blocked write | Cancel that handler and drain its descendants; neighboring streams survive | +| HTTP/2 handler program fault | Original error propagates to serve; never treat it as normal EOF | +| HTTP/2 GOAWAY while old streams remain | Stop admission and keep the physical connection permit until cleanup | +| HTTP/2 stream/connection budget or window outside its bounds | Explicit failure before accepting or connecting | + +## 5. Good / Base / Bad Cases + +Onboarding base: `dever new app` followed by `run app -- hello.greeting.greet '{"name":"Dever"}'` returns the standard envelope with `data:"Hello, Dever"`. Good: --markdown generates equivalent checked declarations; two projects/users share core and skill. Bad: manual main.dever scaffolding, Go service/Page JSON templates, update URLs in project setting.json, or a global skill copied once and allowed to drift from the installed compiler. + +Cross build good: an x86 core uses ARM profiles, ARM Worker assets and x86 Go tools emitting ARM objects. Base: omitted target retains native behavior. Bad: replacing global host identity, executing an ARM build tool on x86, labeling x86 native extensions ARM, or deriving target loader permissions from the host. + +Native author base: fixed inputs and temporary key produce a verifiable manifest. Good: two different output names have identical manifests/signatures and payload digests, source mutation does not affect either published copy, and optimized profile programs run through the managed compiler. Bad: hard-link output to build caches, accept a debug all-features archive as four trimmed profiles, or call an unused PostgreSQL configuration a real database test. + +Managed base: a signed Package project completes check/test/run/build and its built program runs without daemon. Good: different UIDs and project directories submit the same logical inputs and reuse one entry, while wrong source, unsigned CRT or modified pack fails. Bad: accept client IR, fetch compiled output through artifact digest, upload full Settings, run the application under the daemon UID, or make state/socket directories private to hide a worker permission bug. + +Resource/async base: create/read a private fixture file and lower sleep to a real switched-resume frame. Good: aliases share Stream progress; stop releases a pending child and Scope drains descendants; large typed records and errors survive Pending; sync Task inputs/outputs transfer through the same bounded blocking owner. Bad: declare async metadata safe from parameters alone, clone borrowed arguments after initial suspend, reinterpret a typed error as String, or let a same-content overwrite pass the test. + +Runtime ABI base: Int 40+2 yields status 0/value 42/null Buffer. Good: explicitly link a runtime-abi-only archive from a C or LLVM caller, read output only on status 0 and free an error/text buffer via the matching symbol. Bad: cast Rust DecimalValue into a C layout, pass aggregates by value, reuse a live owned Buffer slot, free compiler Reply via runtime buffer_free or treat author fixtures as signed release packs. + +Managed ABI good: typed C callbacks retain/drop a nested Text record; aliased List/Map updates preserve the original; early cursor release drops remaining rows; duplicate Map keys retain the exact original diagnostic. Bad: byte-copy a managed payload or padding, make ordinary Float elements implement Eq, or confuse outer row existence with a nullable element's null state. + +Base: format the hello sample twice; bytes are unchanged on the second pass. Good: move an unaliased Map payload into `get`, render a nested value into one buffer, or parse and stringify nested JSON with an exact large number. Bad: clamp a valid one-shot read behind the caller's back, move the last Socket alias into a short-lived function, accept conflicting Content-Length values, or compare concurrent stdout as an ordered trace. + +Markdown base: an application file documents its path-derived component/domain/role identity in H1 metadata; API sections document declarations and contain no handler bodies. Good: keep every type/function/declaration description synchronized with its checked signature and format without changing prose. Bad: add application `main.dever.md`, put code under H1, group two functions under one H2, split clauses across H2 sections, execute a quoted example or move a comment across prose during formatting. + +HTTP/2 base: null Limits.http2 retains HTTP/1; default_http2_limits() selects 16 streams and fixed 65535/262144-byte receive windows. Good: a TLS proxy makes a nested request through the same one-connection pool while its outer response remains open. Bad: expire that active connection using the idle timer, or reuse a cancelled stream's lease on a replacement connection. + +Component base: one selected external Adapter completes hello/ready/health, serves typed Port calls and acknowledges shutdown. Good: a protocol fault closes the old receiver before a later call creates a rate-limited new generation; shutdown rejects every racing or queued call and explicitly reaps the child. Bad: drain a full request queue during shutdown, restart after admission closed, rely only on `kill_on_drop`, or report handshake capability equality as an OS sandbox. + +Managed base: a zero-Lib Python/JavaScript Adapter still locks its signed runtime and generates a typed runner. Good: preserve raw archives shared with exec while removing managed-only copies. Bad: import the Python SDK before restoring the packaged path under `-I`, use a peer-supplied cache path, or treat `verified:false` counts as byte-integrity evidence. + +## 6. Required Tests + +For onboarding select CLI `new_project`, lib `release_source`, shared-toolchain version/skill tests and native-release skill authoring tests, plus `python3 -B -m unittest discover -s test/onboarding -v`. Assert both formats check/format, no-overwrite concurrency, real CMD/Test/build output, exact signed extraction, corrupt-download retention, project-pinned guidance and foreign-user reads. The foreign-user test lowers only an owned client UID. Bootstrap script tests mock final installation; report that separately from real native-template execution. Do not activate host services or run full release/CI suites as an onboarding check. + +Cross-target changes cover `build_arguments`, native-release target/ELF/mode cases, CompileRequest validation, cache isolation and Lib/Worker target selection. The opt-in `signed_host_compiler_builds_arm_profiles_and_runs_standalone_under_qemu` and `signed_arm_ecosystems_build_offline_standalone_apps_for_system_guest` require explicit author inputs. Run exported Workers/HTTP with `test/ecosystem-release/arm64/simulate.py` under a complete ARM kernel; assert each successful output, source-located expected failure and child cleanup. Do not substitute QEMU user emulation for namespace/seccomp execution or infer physical ARM performance. + +Native release changes select `native_release` plus affected private pack/installation checks. Default cases cover deterministic bytes/signature, private-copy ownership, malformed configuration/path/digest/key rejection, cleanup, and concurrent no-overwrite publication. Explicit `native_release::acceptance` consumes prepared optimized archives and author executable; inspect SQLite/PostgreSQL symbols, then repeat packaging, verify signed installation, compile/run all four profiles and run independent outputs after source removal/daemon stop. Report PG-server and cross-host source rebuild evidence separately. Keep native checks and Cargo serial under constrained disk space. + +Managed compilation selects `packages::managed_compile`, the installed Package execution fixture, affected `shared_toolchain` cache/install/IPC cases, and private `llvm_cli`. Cover secret-free snapshots, reserved/strict source checks, cross-UID reuse, changed sources, failure recovery, fake-only Test execution, signature reference omissions, cache-hit tampering, limits, real cancellation/disconnect/deadline reaping, and shared install lock lifetime. Tests use owned daemons and temporary signing keys. Check final `dever`/`deverd` DT_NEEDED independently: empty-environment success on an LLVM-equipped host cannot establish absence of a dependency. + +Runtime ABI changes select `native_runtime_abi` and opt-in `native_runtime_link`: exact Decimal word round-trip, checked errors/IEEE floats, borrowed UTF-8/Bytes including null-empty and invalid input, real C-header/staticlib linkage, LLVM scalar/out-pointer calls and matching buffer release. Managed changes include canonical C typed lifecycle callbacks, nested rows, COW aliases, cursors/early release, duplicate Map keys, split descriptor identity and Bytes conversion/slice. Build the offline runtime-only archive explicitly in `target/native-runtime-abi`; do not recursively build it from tests or silently use an embedded-compiler archive. Opt-in `llvm_managed_native` and `llvm_result_native` reuse the shared runtime-linked runner to verify real checked-source execution and repeated allocation balance; this is not a general memory profiler or capacity test. Typed lowering/bridge tests and target-pack limitations are specified in `compiler-contracts.md`. + +File/Stream/coroutine ABI changes additionally select `llvm_resource_native`, `llvm_async_native` and `llvm_async_source`. Keep hand-written LLVM/C ABI smoke separate from checked-source lowering evidence; execute both against the actual archive. Assert all initial/pending/final destroy paths, actual Pending/backpressure, typed error transfer, sync input borrowing, non-null creation consumption, nullable row/End, Scope drain and repeat allocation balance. Use an atomic allocation ledger for concurrent workers. Do not run broad database/service/performance checks or claim complete application/target-pack acceptance from these tests. + +HTTP/system lowering selects `llvm_http_system_source` plus the above affected ABI/async targets: repeated headers, independent chunked wire, pooled H1/H2 uploads/downloads, verified TLS, live/SSE, WebSocket typed variants/end, context ownership, failure-500-next-request, original diagnostics, calendar/UUID/Secret/crypto/argv and exact stdout/log JSON. The canonical C protocol fixture must reject null/misaligned destinations before poll, preserve sentinels, allow corrected retry, and reject completed repoll. The pure-log C fixture uses the same strict allocation ledger; logger concurrency/backpressure/panic tests stay under root test/runtime_logging.rs. Six-target object generation is separate from the actual Linux archive execution. + +AsyncStream/TCP/timeout/race/ParallelEach changes also select `llvm_stream_network_source` and the existing async composition/network reference fixtures as appropriate. Cover successful null/Unit versus genuine timeout, exact typed faults without fallback, shared stream close, terminal rows, all dispatch categories and descendant peer EOF after cleanup. C polling invalid input returns `DEVER_RT_ASYNC_INVALID_INPUT=4`; it must never look like a typed fault. Canonical signatures remain only in runtime.h. + +Performance changes select the affected native_ownership, runtime_foundations, concurrency or native_cache cases. HTTP changes select http_engine (independent wire, repeated headers, framing, body limits, deadlines, connection backpressure and task cleanup) and http_library (static handlers/effects, native bridging, examples). Bounded ignored performance tests require explicit scope; correctness checks do not establish throughput or RSS improvements. + +Choose relevant root targets: `formatter`, `standard_library`, `native_ownership`, `concurrency`, `json_library`, `http_library`, and opt-in `differential`. Assert independent expected results, diagnostics, source field/Map order, scope lifetime, actual worker overlap, limits, malformed protocols and wire bytes. Native/resource tests own all temporary files and loopback sockets. + +Markdown changes select `markdown_source` and affected `formatter`/`syntax_and_format` regressions. Cover mixed formats/duplicates and unchanged language rejection; H1 metadata without a block and H2 one-block structure; public names/use examples; fields/variants; resolved clause inputs and outputs; explicit empty lists and natural descriptions; original UTF-8/LF/CRLF/CR positions; hidden/nested/nonprogram content; closure and cross-block rejection; comments, prose preservation and idempotence. Verify actual CLI check/fmt/api/run/build with isolated small programs and failure-before-rewrite cases, without a full or service integration suite. + +`performance` is explicitly ignored by default. It measures bounded hot paths after setup against the same representation and runtime, with exact output checks and repeated median measurements. Keep the 1.15 ratio gate meaningful; never replace a baseline by a slower different representation to conceal a compiler regression. Report environment and actual scope, not generic speed claims. + +N3 selects `live_network` and `live_library`: independent HTTP chunks and WebSocket frame bytes, preread upgrades, masking/fragmentation/control frames/UTF-8, exact SSE field/line/size rules, backpressure, phase vs idle deadlines, task/connection limits, close/cancellation and native static bridging. Existing `http_engine`, `http_library` and affected `async_network` cases cover shared owners. Service examples are checked/formatted without starting their persistent entry point. + +N4 selects `async_composition`, `pooled_network` and `pooled_library`: source configuration, affine timeout/race, async memory fusion boundaries, Channel/ticks, exact typed context/effects/failure obligations; independent connection reuse, upload framing, stream backpressure, queue/body/idle deadlines, incomplete-body cancellation, unpolled pool cleanup, CA/name rejection, WSS and graceful draining. Native programs must exercise the complete checked source bridge. The self-contained backend example may run on its owned random loopback port; persistent examples remain check-only. + +N5 selects `http2_client`, `http2_server`, affected `http_library`/`pooled_library`, HTTP/1 pool/live regressions and structured task lifecycle cases. Independent h2 peers must prove actual multiplexing, stream reset isolation, flow-control recovery, stream/physical budgets, active-body idle behavior, ALPN success/rejection, GOAWAY draining and handler error propagation. Native source must cover HTTP/2 single-use send and a TLS proxy that issues a nested request through the same one-connection client, plus streamed upload, SSE and run/wait inside handlers. Functional checks do not establish HTTP/2 throughput or memory improvements. + +External Component Protocol changes select `external_component`, `port_adapter` and `application_testing`. The fixture must be an owned test executable, not Python/Node/Go or a service dependency. Assert two calls, declared business error, health, caller cancel, timeout cancel, crash/restart, malformed frames, queued shutdown, schema drift and native Worker execution. These tests establish protocol/lifecycle correctness only; they do not establish ecosystem resolver, packaging, OS sandbox, throughput or RSS. + +Resource and Lib changes also select `external_resources`, CLI `external_libs`, `application_config` and `shared_toolchain`. Include concurrent extraction, execute-bit identity, symlink/permission rejection, no-source/env-cleared packaged exec with repeat runs and tamper rejection, full provider metadata drift and independent Worker lock bindings. The opt-in `test/external-sdk/check.py` uses explicitly supplied host interpreters and checks all three SDKs, i64 extremes, JSON budgets, health/errors and cooperative/noncooperative cancellation; it does not prove managed runtimes or no-host distribution. + +Registry tests exercise real provider metadata through an injected bounded transport, candidate/dependency/version conflicts and runtime identity; owned loopback HTTP tests also cover the Go ZIP redirect and rejection-before-request boundary. They are not public-network or signed-release acceptance. `external_workers` tests cover Python isolation, ESM/CommonJS binding, wheel/npm rules, path/file conflicts and no-host lookup. Explicit host-copy subprocess cases remain ignored by default. Shared service tests cover real UID separation (opt-in root-owned fixture), bad/truncated concurrent uploads, entry/byte quotas, summary-vs-integrity semantics and owner maintenance. Native tests assert resource-independent generated source size, forged-digest rejection and actual packaged execution; tests derived solely from emit strings do not prove launch. + +## Durable Worker Lifecycle + +The compiler owns Job service startup. Both application backends validate settings and selected `runtime.mode` before preparing service databases or listening/claiming. API and worker roots stay in one binary; api/worker/all gate only the services, not source reachability. Starting a Job service requires runtime settings; CMD and management commands do not validate or start unrelated services. An enqueue-only CMD may use one database connection without HTTP/worker configuration. API-only applications use api/30000 ms defaults when omitted. Worker pools reserve at least one connection beyond worker slots for queue control and lease renewals. SQLite additionally requires `lease_ms > Job timeout_ms` before database preparation, because a spare connection does not bypass its single-writer lock. + +Tokio's existing signal feature owns SIGTERM/Ctrl-C registration; do not install handlers with unsafe code or detach signal tasks. Lifecycle supervises the application root with the existing task owner. Shutdown stops API accept and Job claim, drains within one deadline, then cancellation joins descendants before database shutdown and log flush. Generated normal/test entries close databases on success and application/startup failure. Worker work and heartbeat use registered tasks, bounded active slots and fencing tokens; cancellation leaves the lease for recovery. + +Test-only `dever.test.advance_clock` and bounded `drain_jobs` share production clock/store/dispatch logic. Each case has an independent clock/database/fake closure; empty drain needs no database. No test may start API/worker services, select a production Adapter or use deployment configuration. Job test roots must include enqueued handlers even though enqueue is not a synchronous dependency edge. + +LLVM Job/Test validation selects `llvm_job_test_source` and the Job cases in `llvm_api_source`, plus affected original `durable_jobs` contracts. Check six-target object emission separately from actual archive execution. Exercise single-suite case isolation, structured failure locations, fake-only external calls, virtual delay/retry/cron, atomic enqueue/handler rollback, timeout child drain and repeated-root allocation balance. Real owned API/worker cases verify tenant queue placement and current permissions after enqueue. Revocation fixtures keep stored references structurally valid; malformed storage ownership is an earlier fail-closed store error, not a successful dispatch-time authorization test. + +LLVM external validation selects `llvm_external_source`, the original `external_component`/`external_resources` tests and the existing fake-only Job/Test case. Verify six-target metadata/object layouts separately from actual runtime execution. Run real fixture processes for typed values/errors, malformed payloads, bad Setting/handshake, cancellation, and strict warmup + 64 repeated roots. Relocate only the linked executable and settings before checking embedded exec and managed launch manifests with an empty environment; synthetic managed launch fixtures are not real Python/npm/Go SDK or formal release acceptance. Retain resource SHA verification on every root/restart. Linux allocation ledgers must wrap `realpath(NULL)` as an owned allocation: glibc's internal malloc bypasses executable linker wrapping, while Rust's later free is wrapped. Account for that owner without relaxing balance or increasing warmups. + +Runtime config owns the shared Job bounds: timeout <= 3,600,000 ms and lease in 100..7,200,000 ms. Tests use the maximum lease so every legal timeout fits SQLite without bypassing production policy validation. + +Ordinary Linux commands use `workers/command.rs` with the shared `workers/elf.rs` dependency-closure checker. Prepare the domain-owned ELF and flat adjacent `lib/` from either the project or locked Package; never scan host libraries or require a Lib lock for command-only projects. Validate target, GNU interpreter, recursive DT_NEEDED, SONAME, symbol versions and loader hooks. All command RPATH/RUNPATH expansions must remain inside the logical `program` tree, which maps to `/worker`; accepting an escape such as `$ORIGIN/../data/grant` would allow a file grant to replace a validated dependency at execution time. Private libraries cannot shadow signed sandbox libraries. Dynamic launch uses the signed target loader with `--inhibit-cache --library-path lib:/lib`; user argv is appended literally after mapping only compiler-owned resource arguments. Commands use a distinct `dever-command-launch-v1` manifest and `commands/` tree under the existing verified bundle/extraction owner. + +Command limits are `adapter.<port>.command.timeout_ms` (default 30000, 1..3600000) and `output_limit` (default 1048576, 1..8388608 combined stdout/stderr); stdin is bounded at 8388608 bytes and argv at 65536 including NUL terminators, with embedded NUL rejected. A call owns one process, closes absent stdin, drains both output pipes concurrently and never replays. Nonzero program exit is output; failure before target exec must remain a runtime fault and cannot be inferred from stderr or a reserved exit code. Invocation shutdown and caller cancellation must wait for actual sandbox completion. Queued cancellation has no child to drain and must not wait behind an unrelated active call. Ordinary application tests remain fake-only. Focused evidence belongs to `external_command`, command cases in `external_workers`/`llvm_external_source`, and the opt-in signed native-release command acceptance; physical ARM execution must not be inferred from cross-target metadata tests. + +Command execution observes guard exec through a private CLOEXEC pipe. Bind the PID namespace init through bwrap `--info-fd`, keep it blocked with `--block-fd` until pidfd acquisition and PPid verification, and wait for that pinned pidfd before releasing caller-scope completion. Pipe EOF (`--sync-fd`) is not sufficient: Linux closes init descriptors before tearing down descendants. Cleanup always reaps the owned launcher even if namespace observation fails, retaining both errors. Old Worker launches keep their existing handshake path. + +## 7. Wrong vs Correct + +Wrong: update a skill symlink separately after changing the active core, treat a cached older version as a successful failed update, or delete a public output tree after a publication race. Correct: resolve core and signed skill from one active journal, return the original download failure, and publish exclusively owned staging with NOREPLACE. + +Cross-target wrong: check ELF headers but use the host loader name to assign executable mode. Correct: validate target architecture and independently preserve each declared executable's mode across signed packaging, embedding and extraction; exercise the resulting Worker bootstrap under the target kernel. + +Native authoring wrong: discover a compiler in PATH while signing, overwrite a completed release, or strip a hard-linked archive in place. Correct: separate the recorded source-build recipe from deterministic copy/verify/sign, preserve link symbols, and publish a wholly new directory with the existing signed installation contract. + +Wrong: reuse shared-file validation for an owner-only lock, require 0700 on public state ancestors, unlink a held advisory lock, or trust a successful test on a host with LLVM installed. Correct: validate each owner/permission contract separately, retain the lock inode, keep source staging private, and inspect final dynamic dependencies. + +Wrong: export a Rust aggregate directly, use C++ free on Rust ABI buffers or ship the LLVM author SDK inside the application runtime. Correct: fixed scalar arguments and explicit output pointers, one canonical header, private allocator-matched exports and separate runtime-abi/embedded features; validate formal target packs separately. + +Managed ABI wrong: trust a copied byte buffer as an owned typed value, create per-caller descriptor identities or read an optional out row regardless of presence. Correct: one process-static descriptor per concrete type, typed lifecycle callbacks over aligned uninitialized storage, explicit receiver consumption and full-row nullable extraction. + +Async ABI wrong: use regular 0/1/2 status checks for Pending/Ready, raw-copy fault payload padding, free normal guards twice during final destroy, or drop the runtime's input inside SyncFunction.invoke. Correct: explicit async status mapping, typed nominal move/drop with source zeroing, initial/pending reverse cleanup versus final frame-only free, and runtime-owned input release after invoke or cancellation. + +HTTP/2 wrong: spawn Hyper futures outside CURRENT_SCOPE, let a body-reset write error stop the whole server, or retry every GOAWAY error. Correct: register protocol tasks under a connection owner, distinguish transport cancellation before propagating handler faults, and only requeue a structurally returned unsent request within its original deadline. + +Wrong: formatter reorders effectful constructors; code generation moves resource aliases early; composite rendering first collects every child String; the runtime caps one-shot reads to an arbitrary internal chunk; a benchmark times unequal Decimal initial representations. + +Correct: reuse AST/HIR/runtime owners, preserve source ordering and lifetime, consume only disposable native bindings, stream rendering into one output String, keep read size under caller control, and match both semantics and representation before measuring. + +Markdown wrong: regex-extract every fence or format all extracted code as one document. Correct: identify program ranges through Markdown parsing, preserve original positions, and reuse the existing Dever pipeline and formatter within those ranges. + +Component wrong: use shell command strings, `PATH`, application environment variables, detached tasks, automatic call replay, a second JSON parser, or a per-ecosystem supervisor. Component correct: resolve one compiler-owned relative entry, clear inherited environment, reuse bounded wire and Scope owners, preserve the first runtime fault, and explicitly kill/wait on every abnormal exit. diff --git a/.trellis/spec/guides/code-reuse-thinking-guide.md b/.trellis/spec/guides/code-reuse-thinking-guide.md new file mode 100644 index 0000000..bb789e9 --- /dev/null +++ b/.trellis/spec/guides/code-reuse-thinking-guide.md @@ -0,0 +1,223 @@ +# Code Reuse Thinking Guide + +> **Purpose**: Stop and think before creating new code - does it already exist? + +--- + +## The Problem + +**Duplicated code is the #1 source of inconsistency bugs.** + +When you copy-paste or rewrite existing logic: +- Bug fixes don't propagate +- Behavior diverges over time +- Codebase becomes harder to understand + +--- + +## Before Writing New Code + +### Step 1: Search First + +```bash +# Search for similar function names +grep -r "functionName" . + +# Search for similar logic +grep -r "keyword" . +``` + +### Step 2: Ask These Questions + +| Question | If Yes... | +|----------|-----------| +| Does a similar function exist? | Use or extend it | +| Is this pattern used elsewhere? | Follow the existing pattern | +| Could this be a shared utility? | Create it in the right place | +| Am I copying code from another file? | **STOP** - extract to shared | + +--- + +## Common Duplication Patterns + +### Pattern 1: Copy-Paste Functions + +**Bad**: Copying a validation function to another file + +**Good**: Extract to shared utilities, import where needed + +### Pattern 2: Similar Components + +**Bad**: Creating a new component that's 80% similar to existing + +**Good**: Extend existing component with props/variants + +### Pattern 3: Repeated Constants + +**Bad**: Defining the same constant in multiple files + +**Good**: Single source of truth, import everywhere + +### Pattern 4: Repeated Payload Field Extraction + +**Bad**: Multiple consumers cast the same JSON/event fields locally: + +```typescript +const description = (ev as { description?: string }).description; +const context = (ev as { context?: ContextEntry[] }).context; +``` + +This is duplicated contract logic even when the code is only two lines. Each +consumer now has its own definition of what a valid payload means. + +**Good**: Put the decoder, type guard, or projection next to the data owner: + +```typescript +if (isThreadEvent(ev)) { + renderThreadEvent(ev); +} +``` + +**Rule**: If the same untyped payload field is read in 2+ places, create a +shared type guard / normalizer / projection before adding a third reader. + +--- + +## When to Abstract + +**Abstract when**: +- Same code appears 3+ times +- Logic is complex enough to have bugs +- Multiple people might need this + +**Don't abstract when**: +- Only used once +- Trivial one-liner +- Abstraction would be more complex than duplication + +--- + +## After Batch Modifications + +When you've made similar changes to multiple files: + +1. **Review**: Did you catch all instances? +2. **Search**: Run grep to find any missed +3. **Consider**: Should this be abstracted? + +### Reducers Should Use Exhaustive Structure + +When state is derived from action-like values (`action`, `kind`, `status`, +`phase`), prefer a reducer with one `switch` over scattered `if/else` updates. + +```typescript +// BAD - action-specific state transitions are hard to audit +if (action === "opened") { ... } +else if (action === "comment") { ... } +else if (action === "status") { ... } + +// GOOD - one reducer owns the transition table +switch (event.action) { + case "opened": + ... + return; + case "comment": + ... + return; +} +``` + +This matters when the event log is the source of truth. A reducer is the +documented replay model; display code and commands should not duplicate pieces +of that replay model. + +--- + +## Checklist Before Commit + +- [ ] Searched for existing similar code +- [ ] No copy-pasted logic that should be shared +- [ ] No repeated untyped payload field extraction outside a shared decoder +- [ ] Constants defined in one place +- [ ] Similar patterns follow same structure +- [ ] Reducer/action transitions live in one reducer or command dispatcher + +--- + +## Gotcha: Python if/elif/else Exhaustive Check + +**Problem**: Python's if/elif/else chains have no compile-time exhaustive check. When you add a new value to a `Literal` type (e.g., `Platform`), existing if/elif/else chains silently fall through to `else` with wrong defaults. + +**Symptom**: New platform works partially — some methods return Claude defaults instead of platform-specific values. No error is raised. + +**Example** (`cli_adapter.py`): +```python +# BAD: "gemini" falls through to else, returns "claude" +@property +def cli_name(self) -> str: + if self.platform == "opencode": + return "opencode" + else: + return "claude" # gemini silently gets "claude"! + +# GOOD: explicit branch for every platform +@property +def cli_name(self) -> str: + if self.platform == "opencode": + return "opencode" + elif self.platform == "gemini": + return "gemini" + else: + return "claude" +``` + +**Prevention**: When adding a new value to a Python `Literal` type, search for ALL if/elif/else chains that switch on that type and add explicit branches. Don't rely on `else` being correct for new values. + +--- + +## Gotcha: Asymmetric Mechanisms Producing Same Output + +**Problem**: When two different mechanisms must produce the same file set (e.g., recursive directory copy for init vs. manual `files.set()` for update), structural changes (renaming, moving, adding subdirectories) only propagate through the automatic mechanism. The manual one silently drifts. + +**Symptom**: Init works perfectly, but update creates files at wrong paths or misses files entirely. + +**Prevention**: +- **Best**: Eliminate the asymmetry — have the manual path call the automatic one (e.g., `collectTemplateFiles()` calls `getAllScripts()` instead of maintaining its own list) +- **If asymmetry is unavoidable**: Add a regression test that compares outputs from both mechanisms +- When migrating directory structures, search for ALL code paths that reference the old structure + +**Real example**: `trellis update` had a manual `files.set()` list for 11 scripts that `getAllScripts()` already tracked. Fix: replaced the manual list with a `for..of getAllScripts()` loop. See `update.ts` refactor in v0.4.0-beta.3. + +--- + +## Template File Registration (Trellis-specific) + +When adding new files to `src/templates/trellis/scripts/`: + +**Single registration point**: `src/templates/trellis/index.ts` + +1. Add `export const xxxScript = readTemplate("scripts/path/file.py");` +2. Add to `getAllScripts()` Map + +That's it. `commands/update.ts` uses `getAllScripts()` directly — no manual sync needed. + +**Why this matters**: Without registration in `getAllScripts()`, `trellis update` won't sync the file to user projects. Bug fixes and features won't propagate. + +**History**: Before v0.4.0-beta.3, `update.ts` had its own hand-maintained file list that frequently fell out of sync with `getAllScripts()`. This caused 11 Python files to be silently skipped during `trellis update`. The fix was to eliminate the duplicate list and use `getAllScripts()` as the single source of truth. + +### Quick Checklist for New Scripts + +```bash +# After adding a new .py file, verify it's in getAllScripts(): +grep -l "newFileName" src/templates/trellis/index.ts # Should match +``` + +### Template Sync Convention + +`.trellis/scripts/` (dogfooded) and `packages/cli/src/templates/trellis/scripts/` (template) must stay identical. After editing `.trellis/scripts/`, always sync: + +```bash +rsync -av --delete --exclude='__pycache__' .trellis/scripts/ packages/cli/src/templates/trellis/scripts/ +``` + +**Gotcha**: Running rsync with wrong source/destination paths can create nested garbage directories (e.g., `.trellis/scripts/packages/cli/...`). Always double-check paths before running. diff --git a/.trellis/spec/guides/cross-layer-thinking-guide.md b/.trellis/spec/guides/cross-layer-thinking-guide.md new file mode 100644 index 0000000..9686546 --- /dev/null +++ b/.trellis/spec/guides/cross-layer-thinking-guide.md @@ -0,0 +1,327 @@ +# Cross-Layer Thinking Guide + +> **Purpose**: Think through data flow across layers before implementing. + +--- + +## The Problem + +**Most bugs happen at layer boundaries**, not within layers. + +Common cross-layer bugs: + +- API returns format A, frontend expects format B +- Database stores X, service transforms to Y, but loses data +- Multiple layers implement the same logic differently + +--- + +## Before Implementing Cross-Layer Features + +### Step 1: Map the Data Flow + +Draw out how data moves: + +``` +Source → Transform → Store → Retrieve → Transform → Display +``` + +For each arrow, ask: + +- What format is the data in? +- What could go wrong? +- Who is responsible for validation? + +### Step 2: Identify Boundaries + +| Boundary | Common Issues | +| --------------------- | --------------------------------- | +| API ↔ Service | Type mismatches, missing fields | +| Service ↔ Database | Format conversions, null handling | +| Backend ↔ Frontend | Serialization, date formats | +| Component ↔ Component | Props shape changes | + +### Step 3: Define Contracts + +For each boundary: + +- What is the exact input format? +- What is the exact output format? +- What errors can occur? + +--- + +## Common Cross-Layer Mistakes + +### Mistake 1: Implicit Format Assumptions + +**Bad**: Assuming date format without checking + +**Good**: Explicit format conversion at boundaries + +### Mistake 2: Scattered Validation + +**Bad**: Validating the same thing in multiple layers + +**Good**: Validate once at the entry point + +### Mistake 3: Leaky Abstractions + +**Bad**: Component knows about database schema + +**Good**: Each layer only knows its neighbors + +### Mistake 4: Every Consumer Parses The Same Payload + +**Bad**: A command reads JSONL events and casts fields inline: + +```typescript +const thread = (ev as { thread?: string }).thread; +const labels = (ev as { labels?: string[] }).labels; +``` + +This looks local, but it means every consumer owns a private version of the +event contract. The next field change will update one command and miss another. + +**Good**: Decode once at the event boundary, then export typed projections: + +```typescript +if (!isThreadEvent(ev)) return false; +return ev.thread === filter.thread; +``` + +**Rule**: For append-only logs, JSON streams, RPC payloads, or config files, +create one owner for: + +- event / payload type definitions +- type guards and normalization from `unknown` +- metadata projections used by UI commands +- reducers that replay state from the source of truth + +Rendering code may format fields, but it must not redefine the payload contract. + +--- + +## Checklist for Cross-Layer Features + +Before implementation: + +- [ ] Mapped the complete data flow +- [ ] Identified all layer boundaries +- [ ] Defined format at each boundary +- [ ] Decided where validation happens + +After implementation: + +- [ ] Tested with edge cases (null, empty, invalid) +- [ ] Verified error handling at each boundary +- [ ] Checked data survives round-trip +- [ ] Checked that consumers import shared decoders / projections instead of + casting payload fields locally +- [ ] Checked that derived state points back to the source event identifier + (`seq`, `id`, `version`) instead of inventing a second cursor + +--- + +## Cross-Platform Template Consistency + +In Trellis, command templates (e.g., `record-session.md`) exist in **multiple platforms** with identical or near-identical content. This is a cross-layer boundary. + +### Checklist: After Modifying Any Command Template + +- [ ] Find all platforms with the same command: `find src/templates/*/commands/trellis/ -name "<command>.*"` +- [ ] Update all platform copies (Markdown `.md` and TOML `.toml`) +- [ ] For Gemini TOML: adapt line continuations (`\\` vs `\`) and triple-quoted strings +- [ ] Run `/trellis:check-cross-layer` to verify nothing was missed + +**Real-world example**: Updated `record-session.md` in Claude to use `--mode record`, but forgot iFlow, Kilo, OpenCode, and Gemini — caught by cross-layer check. + +--- + +## Generated Runtime Template Upgrade Consistency + +Some generated files are both documentation and runtime input. In Trellis, +`.trellis/workflow.md` is parsed by `get_context.py`, `workflow_phase.py`, +SessionStart filters, and per-turn hooks. Template changes must be validated +against both fresh init and upgrade paths. + +### Checklist: After Modifying A Runtime-Parsed Template + +- [ ] Identify every runtime parser that reads the template, not just the file + writer that installs it +- [ ] Check whether relevant syntax lives outside obvious managed regions + such as tag blocks +- [ ] Verify fresh `init` output and a versioned `update` scenario that writes + the older `.trellis/.version` +- [ ] Add an upgrade regression using an older pristine template fixture, then + assert the installed file reaches the current packaged shape +- [ ] Update the backend spec that owns the runtime contract + +--- + +## Versioned Documentation Boundary + +Versioned documentation is a cross-layer boundary: source paths, `docs.json` +version routing, and the rendered version selector must all describe the same +release line. + +### Checklist: Before Editing Versioned Docs + +- [ ] Identify the target release line: stable, beta, or RC +- [ ] Verify the edited MDX path matches that line: + - stable: `docs-site/{start,advanced,...}` and `docs-site/zh/{start,advanced,...}` + - beta: `docs-site/beta/**` and `docs-site/zh/beta/**` + - RC: `docs-site/rc/**` and `docs-site/zh/rc/**` +- [ ] Verify `docs.json` navigation points the version label to the same paths +- [ ] Grep the opposite tree for release-line-specific terms before committing +- [ ] Treat beta content appearing under root release paths as a source-path bug, + not a rendering bug + +**Real-world example**: A beta-only task workflow change documented +`prd.md` + `design.md` + `implement.md`, task-creation consent, and Codex +mode banners under root `start/` and `advanced/` paths. The docs site then +served 0.6 beta behavior under the Release selector. The fix was to restore root +release docs, move the 0.6 content to `beta/` and `zh/beta/`, and add a grep +audit for beta markers against the root release tree. + +**Real-world example**: Codex inline mode changed workflow platform markers from +`[Codex]` / `[Kilo, Antigravity, Windsurf]` to `[codex-sub-agent]` / +`[codex-inline, Kilo, Antigravity, Windsurf]`. Fresh init was correct, but +`trellis update` only merged `[workflow-state:*]` blocks and preserved stale +markers outside those blocks. Result: upgraded projects got new hook scripts +but old workflow routing, so `get_context.py --mode phase --platform codex` +could return empty Phase 2.1 detail. + +--- + +## Mode-Detection Probe Checklist + +When a CLI auto-detects a mode by probing a remote resource (e.g., checking if `index.json` exists to decide marketplace vs direct download): + +### Before implementing: + +- [ ] Probe runs in **ALL** code paths that use the result (interactive, `-y`, `--flag` combos) +- [ ] 404 vs transient error are distinguished — don't treat both as "not found" +- [ ] Transient errors **abort or retry**, never silently switch modes +- [ ] Shared state (caches, prefetched data) is **reset** when context changes (e.g., user switches source) +- [ ] **Shortcut paths** (e.g., `--template` skipping picker) must have the same error-handling quality as the probed path — check that downstream functions don't call catch-all wrappers + +### After implementing: + +- [ ] Trace every path from probe result to the mode-decision branch — no fallthrough +- [ ] External format contracts (giget URI, raw URLs) are tested or at least documented as comments +- [ ] Metadata reads consume a complete response or use a streaming parser — never parse a fixed-size prefix as full JSON +- [ ] When reconstructing a composite identifier from parsed parts, verify **all** fields are included and in the **correct position** (e.g., `provider:repo/path#ref` not `provider:repo#ref/path`) +- [ ] Verify that **action functions** called after a shortcut don't internally use the old catch-all fetch — they must use the probe-quality variant when error distinction matters + +**Real-world example**: Custom registry flow had 8 bugs across 3 review rounds: (1) probe only ran in interactive mode, (2) transient errors fell through to wrong mode, (3) giget URI had `#ref` in wrong position, (4) prefetched templates leaked across source switches, (5) `--template` shortcut bypassed probe but `downloadTemplateById` internally used catch-all `fetchTemplateIndex`, turning timeouts into "Template not found". + +**Real-world example**: Agent-session update hints fetched npm `latest` metadata with `response.read(4096)` and then parsed it as complete JSON. The `@mindfoldhq/trellis` package metadata exceeded 4 KB, so the JSON was truncated, parse failed silently, and the first session injection showed no update hint. Fix: read the complete response before parsing, and add a regression where `version` is followed by an 8 KB metadata tail. + +--- + +## Cross-Platform Template Consistency + +In Trellis, command templates (e.g., `record-session.md`) exist in **multiple platforms** with identical or near-identical content. This is a cross-layer boundary. + +### Checklist: After Modifying Any Command Template + +- [ ] Find all platforms with the same command: `find src/templates/*/commands/trellis/ -name "<command>.*"` +- [ ] Update all platform copies (Markdown `.md` and TOML `.toml`) +- [ ] For Gemini TOML: adapt line continuations (`\\` vs `\`) and triple-quoted strings +- [ ] Run `/trellis:check-cross-layer` to verify nothing was missed + +**Real-world example**: Updated `record-session.md` in Claude to use `--mode record`, but forgot iFlow, Kilo, OpenCode, and Gemini — caught by cross-layer check. + +--- + +## Generated Runtime Template Upgrade Consistency + +Some generated files are both documentation and runtime input. In Trellis, +`.trellis/workflow.md` is parsed by `get_context.py`, `workflow_phase.py`, +SessionStart filters, and per-turn hooks. Template changes must be validated +against both fresh init and upgrade paths. + +### Checklist: After Modifying A Runtime-Parsed Template + +- [ ] Identify every runtime parser that reads the template, not just the file + writer that installs it +- [ ] Check whether relevant syntax lives outside obvious managed regions + such as tag blocks +- [ ] Verify fresh `init` output and a versioned `update` scenario that writes + the older `.trellis/.version` +- [ ] Add an upgrade regression using an older pristine template fixture, then + assert the installed file reaches the current packaged shape +- [ ] Update the backend spec that owns the runtime contract + +**Real-world example**: Codex inline mode changed workflow platform markers from +`[Codex]` / `[Kilo, Antigravity, Windsurf]` to `[codex-sub-agent]` / +`[codex-inline, Kilo, Antigravity, Windsurf]`. Fresh init was correct, but +`trellis update` only merged `[workflow-state:*]` blocks and preserved stale +markers outside those blocks. Result: upgraded projects got new hook scripts +but old workflow routing, so `get_context.py --mode phase --platform codex` +could return empty Phase 2.1 detail. + +--- + +## Mode-Detection Probe Checklist + +When a CLI auto-detects a mode by probing a remote resource (e.g., checking if `index.json` exists to decide marketplace vs direct download): + +### Before implementing: +- [ ] Probe runs in **ALL** code paths that use the result (interactive, `-y`, `--flag` combos) +- [ ] 404 vs transient error are distinguished — don't treat both as "not found" +- [ ] Transient errors **abort or retry**, never silently switch modes +- [ ] Shared state (caches, prefetched data) is **reset** when context changes (e.g., user switches source) +- [ ] **Shortcut paths** (e.g., `--template` skipping picker) must have the same error-handling quality as the probed path — check that downstream functions don't call catch-all wrappers + +### After implementing: +- [ ] Trace every path from probe result to the mode-decision branch — no fallthrough +- [ ] External format contracts (giget URI, raw URLs) are tested or at least documented as comments +- [ ] Metadata reads consume a complete response or use a streaming parser — never parse a fixed-size prefix as full JSON +- [ ] When reconstructing a composite identifier from parsed parts, verify **all** fields are included and in the **correct position** (e.g., `provider:repo/path#ref` not `provider:repo#ref/path`) +- [ ] Verify that **action functions** called after a shortcut don't internally use the old catch-all fetch — they must use the probe-quality variant when error distinction matters + +**Real-world example**: Custom registry flow had 8 bugs across 3 review rounds: (1) probe only ran in interactive mode, (2) transient errors fell through to wrong mode, (3) giget URI had `#ref` in wrong position, (4) prefetched templates leaked across source switches, (5) `--template` shortcut bypassed probe but `downloadTemplateById` internally used catch-all `fetchTemplateIndex`, turning timeouts into "Template not found". + +**Real-world example**: Agent-session update hints fetched npm `latest` metadata with `response.read(4096)` and then parsed it as complete JSON. The `@mindfoldhq/trellis` package metadata exceeded 4 KB, so the JSON was truncated, parse failed silently, and the first session injection showed no update hint. Fix: read the complete response before parsing, and add a regression where `version` is followed by an 8 KB metadata tail. + +--- + +## When to Create Flow Documentation + +Create detailed flow docs when: + +- Feature spans 3+ layers +- Multiple teams are involved +- Data format is complex +- Feature has caused bugs before + +--- + +## Event Log / Projection Boundary + +Append-only logs are cross-layer contracts. A single event travels through: + +``` +CLI input → event writer → events.jsonl → reader → filter → reducer → display +``` + +### Checklist: After Adding A New Event Kind Or Field + +- [ ] Add the event kind to the central event taxonomy +- [ ] Add a typed event variant or type guard at the event layer +- [ ] Add normalization helpers for array/object fields that come from + user input or JSON +- [ ] Keep `seq` / `id` assignment in the event writer only +- [ ] Make filters and reducers consume the typed event guard, not local casts +- [ ] Make display code consume reducer output or typed events, not raw JSON +- [ ] Add at least one regression that proves history replay and live filtering + use the same filter model + +**Real-world example**: Thread channels added `kind: "thread"`, `description`, +`context`, labels, and `lastSeq`. The first implementation replayed thread +state correctly, but several commands still re-parsed event payload fields with +local casts. The fix was to make the core event layer own `ThreadChannelEvent` +and `isThreadEvent`, make `reduceChannelMetadata` the only channel metadata +projection, and make `reduceThreads` the only thread replay reducer. diff --git a/.trellis/spec/guides/index.md b/.trellis/spec/guides/index.md new file mode 100644 index 0000000..56c6d77 --- /dev/null +++ b/.trellis/spec/guides/index.md @@ -0,0 +1,97 @@ +# Thinking Guides + +> **Purpose**: Expand your thinking to catch things you might not have considered. + +--- + +## Why Thinking Guides? + +**Most bugs and tech debt come from "didn't think of that"**, not from lack of skill: + +- Didn't think about what happens at layer boundaries → cross-layer bugs +- Didn't think about code patterns repeating → duplicated code everywhere +- Didn't think about edge cases → runtime errors +- Didn't think about future maintainers → unreadable code + +These guides help you **ask the right questions before coding**. + +--- + +## Available Guides + +| Guide | Purpose | When to Use | +|-------|---------|-------------| +| [Code Reuse Thinking Guide](./code-reuse-thinking-guide.md) | Identify patterns and reduce duplication | When you notice repeated patterns | +| [Cross-Layer Thinking Guide](./cross-layer-thinking-guide.md) | Think through data flow across layers | Features spanning multiple layers | + +--- + +## Quick Reference: Thinking Triggers + +### When to Think About Cross-Layer Issues + +- [ ] Feature touches 3+ layers (API, Service, Component, Database) +- [ ] Data format changes between layers +- [ ] Multiple consumers need the same data +- [ ] You're not sure where to put some logic +- [ ] You are adding an event kind, JSONL record, RPC payload, or config field +- [ ] UI / command code starts casting raw payload fields directly + +→ Read [Cross-Layer Thinking Guide](./cross-layer-thinking-guide.md) + +### When to Think About Code Reuse + +- [ ] You're writing similar code to something that exists +- [ ] You see the same pattern repeated 3+ times +- [ ] You're adding a new field to multiple places +- [ ] **You're modifying any constant or config** +- [ ] **You're creating a new utility/helper function** ← Search first! +- [ ] Two files read the same untyped payload field with local casts +- [ ] Multiple branches update the same derived state from `kind` / `action` + +→ Read [Code Reuse Thinking Guide](./code-reuse-thinking-guide.md) + +### When Verifying AI Cross-Review Results + +- [ ] Reviewer claims "user input can be malicious" → Check the actual data source (internal manifest? user config? external API?) +- [ ] Reviewer flags "missing validation" → Is the data from a trusted internal source? +- [ ] Reviewer says "behavior change" → Read the code comments — is it intentional design? +- [ ] Reviewer identifies a "bug" in test → Mentally delete the feature being tested — does the test still pass? If yes → tautological test + +**Common AI reviewer false-positive patterns**: +1. **Trust boundary confusion**: Treating internal data (bundled JSON manifests) as untrusted external input +2. **Ignoring design comments**: Flagging intentional behavior documented in code comments as bugs +3. **Variable misreading**: Not tracing a variable to its actual definition (e.g., Map keyed by path vs name) + +**Verification rule**: Every CRITICAL/WARNING finding must be verified against the actual code before prioritizing. Budget ~35% false-positive rate for AI reviews. + +--- + +## Pre-Modification Rule (CRITICAL) + +> **Before changing ANY value, ALWAYS search first!** + +```bash +# Search for the value you're about to change +grep -r "value_to_change" . +``` + +This single habit prevents most "forgot to update X" bugs. + +--- + +## How to Use This Directory + +1. **Before coding**: Skim the relevant thinking guide +2. **During coding**: If something feels repetitive or complex, check the guides +3. **After bugs**: Add new insights to the relevant guide (learn from mistakes) + +--- + +## Contributing + +Found a new "didn't think of that" moment? Add it to the relevant guide. + +--- + +**Core Principle**: 30 minutes of thinking saves 3 hours of debugging. diff --git a/.trellis/tasks/09-04-dever-language-mvp/design.md b/.trellis/tasks/09-04-dever-language-mvp/design.md new file mode 100644 index 0000000..40c75ac --- /dev/null +++ b/.trellis/tasks/09-04-dever-language-mvp/design.md @@ -0,0 +1,793 @@ +# Dever Minimal Core 0.1 Design + +## N4 Concurrent Composition And Reusable Clients (2026-09-12, Complete) + +Source `dever.task.start` establishes the one runtime from a synchronous entry with an explicit configuration and static async handler. HTTP system primitives infer an optional final transferable context and pass it to a statically checked handler; applications can wrap this in their own concrete context signature. Ordinary records retain value semantics; Channel/client/resource fields share their existing resource identity. + +`timeout(task, millis, fallback)` consumes a Task, waits for cancellation cleanup on expiry, then invokes a zero-input async fallback with the identical output signature. `race(tasks...)` consumes equally typed tasks and drains losers before returning the first result. Neither operation suppresses program faults. `stream(Channel|List|Bytes)` and periodic `dever.task.ticks` are pull-driven; async memory traversal reuses collection handlers and bounded Group dispatch. + +Network clients own bounded reusable HTTP/1 connections and stream bodies. TLS uses tokio-rustls/ring, with certificate verification, bundled public roots and explicit custom roots/server PEM material. TCP/TLS share the concrete I/O owner used by HTTP and WebSocket upgrades. Protocol parsing stays in Hyper/tungstenite. Idle connections, pending requests, body chunks and cancellation have explicit owners. No HTTP/2, implicit retries, redirects or detached connection drivers. + +The implemented pool fixes one origin per shared HttpClient and uses Hyper's low-level HTTP/1 API. Separate request and physical-connection permits include idle/queued drivers in the same limit. One structured task drives the bounded queue and FuturesUnordered, expiring idle entries without a future request. Its owner guard is constructed before task creation, including the unpolled-child cancellation case. Response EOF returns its lease; early close/fault/cancellation aborts the connection. Listener.close triggers graceful HTTP shutdown; timeout(server, grace_ms, fallback) bounds draining active responses and upgrades. + +Locked additions are tokio-rustls 0.26.5, rustls 0.23.44 with ring/std/tls12 and webpki-roots 1.0.8. Shared TLS configuration uses small explicit session-cache capacities. Transport boxes only the large TLS variant; ordinary TCP stays direct. The backend example combines configured source entry, concrete Context with Channel, a pooled request and graceful shutdown on an owned random loopback port. + +## N3 Streaming HTTP, SSE And WebSocket (2026-09-12, Approved) + +Keep ordinary `serve` direct; `serve_live` binds an async zero-output `(Request, HttpReply)` handler. A Group of one owns each connection's live handler. Shared request decoding, checked response encoding and Hyper configuration avoid a parallel HTTP implementation. A one-slot body channel provides backpressure. Response-body Drop signals cancellation for streamed, buffered and HEAD responses; Session cleanup closes the producer or upgraded socket. Handler faults propagate and cannot be emitted as a successful truncated body. + +On upgrade, wait for Hyper's driver, extract its known TcpStream and preread bytes, then keep the connection reservation while awaiting the handler Group. This removes the HTTP write deadline and the need for an extra transport abstraction. WebSocket uses independent reader/writer locks around tokio-tungstenite split halves, the shared Endpoint close signal and bounded message/write buffers with a 4 KiB read buffer. Acquire the writer lock before arming cancellation cleanup. close takes final transport ownership, wakes other operations and completes a bounded handshake. + +SSE is a small field encoder plus response-body timer; no routing framework or heartbeat task is needed. Preserve CR/LF/CRLF and trailing data lines, compute final size before allocation and reserve Content-Type for the protocol. Request headers carry Last-Event-ID; replay belongs to application code. WebSocket control frames are processed while receive/messages is polled; no automatic reader or Ping task. Public exact limits and lifecycle are defined once in LANGUAGE.md. + +References checked: [tokio-tungstenite 0.30.0](https://docs.rs/tokio-tungstenite/0.30.0/tokio_tungstenite/), [WebSocketConfig](https://docs.rs/tungstenite/0.30.0/tungstenite/protocol/struct.WebSocketConfig.html), [EventSource parsing](https://html.spec.whatwg.org/multipage/server-sent-events.html#parsing-an-event-stream). No TLS, HTTP/2, pooling, protocol extensions or compatibility layer in N3. + +## N2 HTTP Engine (2026-09-12, Approved) + +Hyper HTTP/1 owns parsing/framing/persistence, using the existing Tokio runtime and only hyper-util's Tokio I/O/timer adapter. No Axum dependency is needed for the current single static route contract. This supersedes earlier source-only HTTP and per-request fault-isolation proposals below: handler program faults propagate through the structured server task, while peer protocol/I/O failures end only that connection. + +Official source owns Header/Request/Response/Limits, header helpers, send and serve. Header is Text name plus Bytes value; List preserves repeated values. Native conversions resolve fields by name after exact shape checks. Intrinsic carries an optional static callback; callback extraction is shared with Collection for dependency/cycle/specialization traversal, and handler effects/recovery propagate through the existing summaries. + +HTTP accepts unique Tokio streams through the existing Listener owner before constructing any aliased public Socket. Existing AsyncStream/Group reserves capacity before accept, observes connection faults while idle, and drains descendants on stop. Hyper never spawns a detached connection driver. Client send polls driver and response in the same bounded future and releases both on completion/cancellation; pooling is deferred. + +Bytes uses bytes::Bytes for shared engine storage, slicing and compatible Text ownership transfer. Bodies are buffered within explicit limits, merging fragments incrementally without retaining one metadata entry per tiny frame. Header, body, handler and stalled-write deadlines have separate owners. The public contract and exact protocol exclusions are in LANGUAGE.md; TLS/HTTP2, streaming responses, client pooling and WS/SSE are not N2 deliverables. + +## N1 Async Stream And TCP (2026-09-12, Approved) + +This new-language phase supersedes the historical compatibility baseline and network pause below. Migrate callers directly; no synchronous TCP backend, aliases or fallback paths. + +Use one existing Tokio runtime with its I/O driver enabled. AsyncStream owns one Send stream producer, serializes pulls, and shares a close signal across aliases. Closing or finishing a consumer releases its producer; there is no unbounded buffer or per-item boxed future. Stream remains the distinct synchronous file-sequence type. + +Socket and Listener share explicit resource closure. Socket read and write have independent async locks; closure wakes both. A write tracks partial progress and closes the socket if abandoned after writing any bytes, so subsequent operations cannot append to a truncated message. Operation timeout includes waiting for its direction lock. No retry of an entire interrupted write. + +AsyncStream traversal may call synchronous effect-checked handlers or async handlers. parallel_each on AsyncStream requires an async zero-output handler and uses the existing structured Group. It checks completed tasks and acquires group capacity before pulling, and observes handler failures even while the producer is idle. No new detached task model or private thread pool. + +Compiler types, effects, liveness, native specialization, official network API and existing HTTP transport move together. The synchronous reference evaluator explicitly excludes async execution; no second network implementation is retained for differential tests. + +## Structured Concurrency Design And Implementation Contract (2026-09-12) + +This preceding phase is implemented through the compiler frontend, checked HIR, native backend and bounded runtime, with evidence in `implement.md`. It superseded the first-stage no-async model. The current N1 decision permits direct source-contract changes without a compatibility baseline. + +### One Task Model + +Dever has one concurrency model rather than separate async, coroutine and thread abstractions: + +- an async function compiles to a stackless coroutine; +- `await` suspends that coroutine without occupying a worker thread; +- `run` schedules the coroutine as a child Task; +- the runtime may move transferable tasks across worker threads; +- `parallel` and `blocking` cross from async scheduling into bounded synchronous execution. + +The program creates one Tokio runtime at an async entry point. A program whose reachable entry graph is entirely synchronous retains the current direct native entry and does not initialize Tokio. The runtime configuration owns async worker count, blocking capacity and task limits once; packages do not create private runtimes or pools. + +### Source Surface + +`async` prefixes a function clause. Every clause in one logical function must agree on this modifier: + +```dever +async load_user(id: Id) (user: User) { + user = await(fetch_user(id)) +} +``` + +An async call is context-only, not an ordinary storable future. It may appear directly under `await`, `run` or the matching compiler-owned task operation. This avoids adding a general `Awaitable<T>` value and preserves the current static-call model. + +```dever +first = run(load_user(first_id)) +second = run(load_user(second_id)) +first_user = await(first) +second_user = await(second) +``` + +`await(async_call(...))` invokes and waits without creating a separately owned Task. `run(async_call(...))` starts first and returns an affine Task whose associated output signature remains in checked HIR. `await(task)` consumes it and yields the same zero, single or named multi-output shape as the function. Task operations consistently use the language's existing call appearance; no second prefix spelling is accepted. + +Static handler syntax places `async` before `handler`: + +```dever +async serve( + route: async handler(request: Request) (response: Response) +) () { + response = await(route(request)) +} +``` + +`async` has one canonical declaration position immediately before the function name. Existing `pure` and `recover` contracts retain their post-signature positions. Parser, formatter, Markdown signatures and examples must change atomically; no postfix async spelling is accepted. + +### Task Ownership + +Task is a compiler-owned affine local type, not a new user-defined generic facility. It may be held only in a local slot of the async function that created it. It cannot be copied, compared, stored in an aggregate, passed as an ordinary value, returned, captured by a handler or left live at any clause exit. + +Each async entry and spawned task has an internal runtime Scope; the compiler treats every async function as a lexical ownership scope. Directly awaited calls inherit the current Scope because runtime faults cannot be caught, while `run` and Group create a supervised child Scope. Every Task must be consumed exactly once by `await` or `stop`. On abnormal runtime exit, the root or child supervisor requests cancellation and waits for descendant and blocking-work cleanup before returning the failure. There is no detached task in the first release. + +`stop` means cancel-and-wait, not fire-and-forget cancellation. To keep failure and output obligations explicit, the first release accepts only zero-output Task values. Result-bearing tasks must be awaited. Cancellation is cooperative at await/yield boundaries; it neither kills a running synchronous operation nor rolls back effects that already occurred. + +### Dynamic Task Groups + +`group(limit)` creates an affine Group for a dynamic number of zero-output async calls. It is needed for later connection/session ownership without allowing detached children: + +```dever +workers = group(128) +run(workers, serve_connection(connection)) +await(workers) +``` + +`run(group, async_call(...))` transfers the child into the Group and returns no Task. It may suspend while the positive capacity limit is full, providing backpressure before starting more work. Ungrouped `run(async_call(...))` is likewise subject to the owning function scope's task limit. `await(group)` closes admission, waits for all children and consumes the Group. `stop(group)` closes admission, requests cancellation, waits for every child and consumes the Group. Group cannot be copied, stored or escape its creating async function. + +Only zero-output calls enter a Group, so the runtime cannot accumulate unbounded result values and expected failures cannot be silently discarded. A later result-collecting combinator must define ordering and memory limits separately. + +### Thread Transfer And Execution Classes + +The type model derives a `transferable` property recursively, alongside the existing comparison/move/transfer properties. Bool, numeric values, Text, Id and Bytes are transferable. Records, choices, nullable values, Lists and Maps are transferable only when every nested value is transferable. Each opaque resource is opted in only when its runtime representation and lifecycle are thread-safe; the current pull-based Stream is initially non-transferable. + +Values live across an `await`, captured by `run`, or passed through channel/parallel boundaries must be transferable. Last-use analysis may move uniquely owned values into a task; otherwise ordinary value semantics require a copy. This rule is checked before native emission and is not delegated to a Rust compiler error. + +Execution classes remain distinct: + +- ordinary synchronous calls execute inline; +- `run` schedules an async call on the shared runtime; +- `parallel(sync_call(...))` accepts only a pure synchronous target with transferable inputs/outputs and runs it through bounded CPU execution; +- `blocking(sync_call(...))` accepts a synchronous target whose transitive effect summary contains a compiler-known blocking system boundary and uses bounded blocking capacity. + +`parallel` and `blocking` are async-context operations in the first release: they suspend their caller until the synchronous call completes. Synchronous code continues to use `parallel_each`; it does not initialize Tokio solely to access these operators. + +Existing synchronous `parallel_each` keeps its public behavior and joined scoped workers. Async List/Bytes lowering requires a named synchronous function with no transitive concurrency effect, then submits bounded batches directly to the shared Tokio blocking executor, so it neither blocks a Tokio scheduling worker nor creates a nested thread pool. Other collection operations inspect the complete handler effect and cannot hide blocking, concurrency or an unconstrained handler effect on a Tokio scheduling worker. The first release has no handler effect bounds, so an async function cannot invoke a synchronous handler parameter inline. Both parallel paths share worker-limit and batch-size validation. Synchronous Stream remains unavailable in async functions until async Stream is designed. + +### Channels + +`channel(Type, capacity)` is compiler-contextual syntax: its first operand is a TypeRef, not a runtime type value. It creates a bounded `Channel<Type>` resource. Capacity is required, positive and capped. This deliberately avoids adding generic function invocation syntax solely for one constructor. + +Channel aliases share one state. `send(channel, value)` asynchronously waits for capacity; `receive(channel)` asynchronously waits for an item and returns `Item?`, where a value is an item and `null` means the Channel is closed. Because nullable types are idempotent, receiving from `Channel<Item?>` also returns `Item?`; native emission flattens the runtime's nested option. A runtime scheduler or Channel fault remains a located fatal execution failure rather than a domain value. `close(channel)` is idempotent and wakes blocked senders and receivers. No unbounded constructor exists. Values must be transferable. Channel itself cannot be a Map key or comparable value, and its close/alias behavior follows explicit resource semantics rather than ordinary value semantics. + +### Effects, Failures And Static Handlers + +Async is part of a function and handler signature. Sync callers cannot invoke async functions or async handlers. Async callers may invoke sync code directly and async code only through an explicit async operation. Collection handlers remain synchronous because collection operations do not await handler futures; async handlers are called only through `await` or `run`. Handler specialization keys include the async signature bit, and cycle detection continues across await/run/forwarding edges; async does not legalize recursion. + +`await`, `run`, `stop`, group operations, channel waits, timers, `parallel` and `blocking` are effects. The existing transitive effect graph owns these facts, so the first release rejects `async` together with `pure`. Task boundaries do not discharge error-bearing outputs: awaiting restores the original output obligations, while Group and stop accept only zero-output work. Runtime scheduler failure is a located fatal execution failure; expected domain failures remain explicit source choice values. + +### HIR And Native Boundaries + +The syntax tree records the async modifier and contextual concurrency expressions. Checked HIR owns explicit `AwaitCall`, `RunCall`, `AwaitTask`, `StopTask`, Group and Channel operations. It never represents a generic user-visible Future. Function metadata and HandlerSignature carry the async bit; Task stores its concrete checked output fields. + +Native emission generates concrete Rust futures and direct calls. Static handlers remain specialized direct targets; no boxed dynamic callback, generic runtime Value, reflection registry or garbage collector is introduced. The runtime owns scheduler construction, task/group/channel state, limits and cancellation primitives. Compiler checks own source legality, transferability, affine consumption and diagnostic locations. + +Runtime cancellation separates a business future from its supervisor. Task/Group stop aborts the business future; the supervisor itself remains alive, drains its child Scope, and only then publishes completion to the parent Scope. Started `spawn_blocking` jobs register a sticky completion signal because Tokio cannot abort them safely. Drop paths request cancellation and detach only supervisor handles whose completion is still owned and awaited by the parent Scope; no Drop implementation calls `block_on`. + +### Compatibility And Deferred Work + +Existing synchronous source and `parallel_each` behavior stay valid. `LANGUAGE.md`, Markdown contracts and API snapshots are updated with the implementation. The `run` source operator is distinct from the CLI command by grammar context. + +The task-foundation phase deferred networking; N1 above resumes AsyncStream and TCP on these primitives. The HTTP engine and WebSocket/SSE remain later work. Detached tasks, arbitrary shared mutation, locks, user thread creation, unbounded channels and forceful cancellation remain out of scope. + +## Markdown Source Follow-through (2026-09-09) + +`.dever.md` is an additional source container alongside `.dever`. Top-level literal `dever` and `typescript dever` fences contain ordinary Dever declarations; the latter supplies a TypeScript highlighting hint while retaining an explicit Dever marker. A dedicated Markdown contract parser maps one H1 to the package header and one H2 to each type or logical function group before formatting. After shared signature resolution, semantic validation compares package/API/use metadata, type fields/variants and function inputs/outputs with syntax and HIR, retaining both documentation and declaration spans in diagnostics. Shared fence extraction keeps original spans for parsing, diagnostics and per-block formatting. `.dever` and language/runtime semantics are unchanged. The authoritative source-format contract is `LANGUAGE.md` and `.trellis/spec/backend/toolchain-and-library.md`; earlier `.dever`-only and unconstrained-heading limits below are superseded. + +## Performance Implementation Boundary (2026-09-08) + +- Compiler analysis worker owns types.rs, check.rs integration and contracts dependency scheduling: memoized type properties and monotone reverse-dependency work queues, retaining handler substitution and proof limits. Existing graph traversal is reused. +- Cache worker owns source/parse caching, native/build.rs and its cache helper plus CLI cache integration if necessary. Cache identities include effective source, compiler/toolchain/runtime and options; checked API baselines stay outside cached native execution. No new external dependency or source format. +- Main owns native liveness/emission, runtime collections, JSON/HTTP source algorithms, performance fixtures and integration. Record updates preserve value aliases and evaluation order; consuming iteration retains resource scope; direct split consumption avoids eager character lists; HTTP parsing tracks incremental boundaries and framing in Dever. +- Root tests own semantic regressions and bounded before/after measurements; no services/full suite/commit. Workers are not alone and must preserve unrelated edits. Shared check.rs edits require coordination with the analysis owner. Independent final review checks values, failure order, cache invalidation and protocol bounds. + +## Follow-through Implementation Design (2026-09-07) + +This section owns the current follow-through; older extension boundaries below describe completed milestones. + +- Formatter: add an AST writer using existing spans, preserved comments and precedence. Preserve record initializer expression order. CLI owns filesystem mutation and prepares every result before any rewrite. Expose syntax-only formatting separately from semantic checking. +- Reference evaluator: add an opt-in `reference` compiler feature, not a runtime dependency. Consume checked HIR, normalized domains and static handler bindings; reuse runtime number/resource primitives. Frames hold value locals and preserve copy semantics. Capture stdout and named results for differential tests. Do not add an evaluator production CLI path. +- Standard library: extend the single intrinsic/checker/native catalog for atomic representation/OS operations only. Public result choices live in `library/`; share runtime implementations with the evaluator. Text indexes are Unicode scalar positions; Bytes indexes remain byte offsets. Number parse failures are nullable results, never a silent zero; finite Decimal rules remain unchanged. +- Optimization: analyze actual read order and writes per clause, retaining outputs and assignment hosts. Move dead values, preserve live aliases and short-circuit paths; transfer reduction state between iterations. Verify value semantics before performance comparison. +- Parallel ownership: formatter worker owns formatter modules, formatting CLI and root format tests; reference worker owns evaluator modules and root differential tests. Main agent owns libraries, primitive integration, native optimization, task documents and integration. Coordinate shared module registration edits. +- Verification uses offline focused targets and test-owned files/processes. Performance checks are bounded local computation, exclude compiler startup and use matching numeric/collection wrappers. Existing user sources are never formatter test fixtures. +- Production native compilation and matching performance baselines share one optimization-argument owner (`opt-level=3`, `codegen-units=1`). The single generated source compiles as one unit to keep generic collection inlining independent of source size; compilation time is measured separately from hot paths. + +- Status: Core foundations and foundation extension implemented and verified on 2026-09-07 +- Date: 2026-09-04 +- Product source of truth: `prd.md` + +## Completed Foundation Boundary (Historical) + +Extend the implemented core with the approved sequence and resource foundation: `reduce`, `reduce_until`, Bytes traversal, `Stream<Item>`, restricted static handler inputs, and file/TCP stream wrappers. Existing clause-only branching, linear bodies and the ban on source loops and user recursion remain permanent. Compiler distribution, cross-platform work and full HTTP/JSON/CRUD packages remain deferred. + +Extend the existing checker and HIR in place. Represent user types nominally, resolve every function signature before bodies, prove clause coverage over symbolic input domains, and emit concrete native types/functions. Keep source expression order, including record construction; formatting must not reorder effectful field expressions. Use one runtime implementation for numeric operations, ordered collections and resource primitives. Bundled `.dever` sources define official wrappers and result types through the same frontend as user packages; only fixed primitive signatures have native implementations. + +## Foundation Extension Design + +This section records the completed foundation-extension design. The Follow-through Implementation Design above owns subsequent additions; later numbered sections retain the original Core roadmap. + +### Sequence Contract And Reuse + +The checker extends the existing collection-operation owner rather than adding parallel validators. One internal sequence classifier returns the element type and source kind for `List<Item>`, Bytes and `Stream<Item>`; `each`, `reduce` and `reduce_until` reuse it for handler checking and lowering. + +- `reduce(handler, sequence, initial_state)` requires a two-input handler `(Item, State) -> State` and returns `State`. +- `reduce_until(handler, sequence, initial_state)` requires `(Item, State) -> (State, stop: Bool)`, processes the current item before stopping, and returns only `State`. +- List and Bytes use finite direct loops. Bytes yields Int values from 0 through 255 without allocating `List<Int>`. +- Stream consumption calls its pull boundary only when the next item is needed. `reduce_until` leaves the next item unread; `reduce` consumes through natural exhaustion. +- `each(Stream)` accepts only a zero-output handler. List/Bytes retain their existing zero-output action and one-output mapping behavior. `filter`, `find` and `sum` remain List-only. +- Core `close(stream)` is a zero-output, idempotent Stream operation. It shares the same sequence operation registry but is checked as resource termination, not traversal. + +Handler signature diagnostics, sequence classification and output-shape validation each have one semantic owner. Native lowering receives already classified sequence and handler facts; it does not repeat source-level type decisions. + +### Static Handler Representation + +`handler(inputs) (named_outputs)` is contextual syntax allowed only after a function input colon. The syntax tree represents value patterns and handler parameter declarations as separate input kinds; it does not disguise a handler as an ordinary `TypeRef`. The semantic model likewise keeps `HandlerSignature` separate from runtime `Type`, because a handler is a compile-time binding rather than a value. + +`HandlerSignature` stores positional input types and ordered named output fields. Input names do not participate in compatibility; output name, position and type all do. The checker accepts only a matching named function or a handler parameter already in scope. Handler references cannot enter local slots, aggregates, outputs, comparisons or clause domains. + +HIR uses explicit call and argument forms: + +```text +HandlerTarget = NamedFunction(FunctionId) | Parameter(HandlerParameterId) +CallTarget = NamedFunction(FunctionId) | Handler(HandlerTarget) +CallArgument = Value(Expression) | Handler(HandlerTarget) +``` + +Collection/sequence HIR stores the same `HandlerTarget`; it no longer carries a collection-specific optional function ID. This is the reusable representation for direct handler calls, sequence operations and multi-level forwarding. + +Native compilation treats a function with handler inputs as a template. A specialization key is the function ID plus its ordered concrete named-function bindings. Forwarding substitutes bindings until every handler call has a concrete target, then emits an ordinary direct call. The specialization graph rejects a repeated active key as direct or indirect recursion before any private Rust source is compiled. Generated handler plumbing contains no function pointers, closures, virtual dispatch or reflective registry; the opaque Stream producer described below is the only dynamic call boundary. + +### Stream Representation And Lifecycle + +The semantic type model adds `Stream(Box<Type>)`. It is the only new compiler-provided generic type; it is a valid parameter, local, field and choice payload but is not comparable and cannot be a Map key. Copying it preserves opaque resource semantics: every alias shares one cursor, exhausted flag and explicit-close flag. + +`dever-runtime` adds one generic pull-stream owner backed by shared synchronized state. Its producer returns either one item or natural end. The only dynamic producer abstraction is inside this opaque I/O boundary, where the PRD explicitly permits it; Dever handlers remain statically specialized. Pulling and closing are serialized, with no prefetch, worker thread or async task. + +The terminal rules are: + +1. Normal EOF marks the Stream exhausted and releases its producer without yielding an element. +2. A terminal file/socket read error is converted once to `ReadEvent.Failed`, then the Stream becomes exhausted. +3. A terminal accept error is converted once to `ConnectResult.Failed`, then the Stream becomes exhausted. +4. `close(stream)` marks all aliases closed, drops the producer and is a no-op when repeated; later traversal sees ordinary exhaustion. +5. Closing a separately held File, Socket or Listener invalidates the shared underlying resource. The dependent Stream then yields its one typed Failed item and ends. +6. Closing only the Stream drops its source alias but does not close a separately held source handle. Native destruction remains the final cleanup fallback. + +`reduce_until` never pre-pulls, so its stop item is consumed and the following item remains available to every Stream alias. Concurrent consumption is outside the language contract; the synchronized state protects resource integrity but does not introduce concurrency semantics. + +### Official Package And Runtime Boundary + +The public stream API remains ordinary bundled `.dever` source: + +```dever +type ReadEvent { + Chunk(bytes: Bytes) + Failed(message: Text) +} + +type ReadStreamResult { + Streaming(stream: Stream<ReadEvent>) + Failed(message: Text) +} +``` + +`dever.io.chunks(file, limit)` and `dever.net.chunks(socket, limit)` return `dever.io.ReadStreamResult`; `dever.net.connections(listener)` returns `Stream<dever.net.ConnectResult>`. Existing one-shot `read` remains unchanged. Invalid limits and already-closed sources fail during stream creation; later source failures become one terminal stream item. + +The ownership flow remains: + +```text +bundled .dever function and result choices + -> fixed dever.system intrinsic signature + -> generated typed adapter + -> dever-runtime Stream/resource primitive +``` + +The generated adapter maps runtime pull outcomes (item, natural end or error) into the source-defined concrete choices. Runtime code therefore owns pulling, locking and OS resources, while `.dever` owns public names and recoverable result shapes. No HTTP parser, route table, JSON codec or user model enters the runtime. + +### Implementation And Rollback Boundaries + +Implementation proceeds through coherent compiler boundaries: syntax/semantic handler declarations; shared handler and sequence HIR; List/Bytes reduce lowering; runtime Stream; file/TCP adapters; then native specialization and end-to-end fixtures. Each boundary must compile and pass its focused tests before the next. Unsupported handler or Stream positions remain compile diagnostics; there is no compatibility fallback to runtime values or eager Stream collection. + +The existing one-shot file/TCP APIs, List operations and generated-binary boundary remain compatible. No stored data or public artifact format needs migration. Reverting the extension removes its syntax, standard source declarations, HIR forms and runtime module together while leaving the completed core foundations intact. + +## Historical Hello Milestone (2026-09-06) + +The completed deliverable is a native `user.dever` whose `main() ()` prints `hello`. A future user API may start with in-memory fixtures; database selection is no longer a prerequisite. Existing parser and surface syntax are preserved. + +The implemented Text/action subset covers package and function resolution, Text parameters and local assignment, zero-output calls, checked HIR, native Rust emission and local check/build/run. It checks all source declarations and rejects unsupported constructs, duplicate symbols, invalid exposures, private cross-package calls, source/package mismatches and cycles. This is not full Core checking. + +The standard `dever.io.println(Text)` signature resolves to a typed intrinsic. Its implementation belongs to a small dependency-free `dever-runtime`; the native backend embeds that same runtime source in disposable generated Rust. Text uses owned Rust Strings to preserve Dever value semantics. User function and local identifiers lower to numeric IDs, not raw Rust identifiers. Output failures propagate internally to a nonzero process exit with the Dever call location; no new source exception syntax is introduced. + +The CLI owns argument parsing and process execution; the core owns checked programs and native compilation. Invoke rustc directly without a shell, using RUSTC or PATH. Compile into an exclusively created temporary directory, retain it through execution and remove owned artifacts afterwards. Explicit build output must not overwrite an existing file. No external dependency, package distribution, interpreter or service is needed. + +The following sections describe the wider roadmap. Implemented foundations are recorded in the Active Core Checklist and Core Foundation Evidence in `implement.md`; formatter, reference evaluator, performance gates and application packages are not part of this delivery. + +## Deferred Application Architecture + +This is a later HTTP/JSON/CRUD direction, not part of the approved foundation extension. It must be reconsidered against the implemented Stream and static-handler contracts before application work begins. + +The native path remains `.dever -> parser -> resolver/checker -> typed HIR -> private Rust -> native executable`. HTTP requests must execute the compiled business functions, not an interpreter or a handwritten Rust user service. + +| Owner | Required responsibility | +| --- | --- | +| `.dever` application packages | User/input types, validation, CRUD orchestration, parameterized queries, routes and business error mapping | +| `dever-core` | Symbol/type/clause/initialization checks, standard signatures, I/O effects, typed handler checks and native code generation | +| `dever-runtime` | Mature HTTP and JSON implementations, the selected database driver, connection/resource lifecycle and per-request fault boundary | +| CLI | Configuration inputs, `check`, native `build/run` and diagnostics | + +Necessary semantic adaptations, with no new surface keywords: + +- Allow named function references in fixed HTTP registration positions, extending the existing collection-handler mechanism. They remain statically resolved, not general first-class functions. +- Add opaque standard resource types for database pools and server resources. Ordinary records/collections keep value semantics; resource handles explicitly identify external state and cannot be serialized or mutated as records. +- Give standard I/O operations fixed effect metadata and propagate effects through the call graph. Preserve statement/expression evaluation order; internal async lowering does not introduce `async` or `await` into `.dever`. +- Reuse choice results for recoverable HTTP/JSON/database failures. Program faults are isolated to the failing request and retain Dever source locations. +- Bundle only the required standard HTTP/JSON/database contracts with the toolchain. External package distribution remains deferred. + +The current `TypeRef` AST can represent parameterized standard types, but `parser.rs::type_ref_inner` presently recognizes only List/Map/MapEntry generic spellings. Any required qualified standard generic type must be supported through the standard-type contract, without a User-specific parser branch. User-defined generics remain deferred. + +Persistence selection is pending. The initial design candidate uses parameterized SQL and static type-driven row decoding, with schema/query definitions owned by one application persistence package. Do not build an ORM or multi-database framework speculatively. JSON and row decoding should share resolved type facts, rather than duplicate user field definitions in Rust. + +Verification must show that changing `.dever` handlers, types or routes changes the compiled API. A second small schema/route fixture should prove that compiler/runtime code contains no user-API special case. Report actual CRUD throughput, latency percentiles, errors and memory against equivalent native code under identical database/durability settings; finalize the performance budget after storage and deployment scope are selected. + +## 1. Architecture Decision + +Core 0.1 is a textual language toolchain, not a semantic-graph workbench. It has one checked frontend and two deliberately different consumers of the same typed HIR: + +```text +.dever source root + | + v +source loader -> lexer -> parser -> syntax tree + | | + | +-> formatter + v + package/name resolver + v + type + clause + flow checks + v + typed HIR + / \ + v v + reference evaluator native specialization + v + private Rust emission + v + rustc / LLVM -> binary + +dever CLI -> load / check / format / build / run +``` + +Dever text in `.dever` files or program blocks of `.dever.md` files is the program source. The syntax tree and typed HIR are in-memory compiler representations; they are not editable JSON formats or public compatibility contracts. The evaluator defines a simple executable reference for conformance tests. Production `build` and `run` use ahead-of-time native compilation and never route through the evaluator. + +The bootstrap compiler is implemented in Rust because rustc provides a mature LLVM path. The repository starts without an existing workspace or implementation. This is an implementation choice, not a surface-language dependency: diagnostics, type identities and runtime semantics are owned by Dever, and generated Rust is a disposable private build artifact. + +## 2. Scope Boundaries + +The implementation contains three production crates: + +```text +crates/ + dever-core/ + src/ + source.rs + diagnostic.rs + token.rs + lexer.rs + syntax.rs + parser.rs + format.rs + package.rs + resolve.rs + types.rs + clauses.rs + check.rs + hir.rs + value.rs + stdlib.rs + eval.rs + native.rs + lib.rs + dever-runtime/ + src/ + number.rs + collections.rs + fault.rs + lib.rs + dever-cli/ + src/ + main.rs +``` + +Compiler phases remain in one `dever-core` library rather than separate syntax, semantic and IR crates. They share source spans and language contracts and will evolve together during 0.1. `dever-runtime` is the only justified split because generated native programs consume it without depending on parser, checker, evaluator or CLI code. It contains only language-owned numeric, ordered-collection and runtime-fault primitives; it does not contain a dynamic interpreter or plugin layer. + +`dever-cli` owns argument parsing, filesystem invocation, exit codes and rendering. It calls `dever-core` APIs and contains no parser, type or evaluation rules. + +Permanent integration tests stay under repository-root `test/`, matching project policy. Plain-source examples live under `examples/dever/`; Markdown examples live under `examples/markdown/`. Both may contain small fixture data when needed. + +## 3. Reuse And Replacement + +### Reuse + +- Pinned dependency discipline in the new Rust workspace. +- Small modules and explicit data structures. +- The idea that every diagnostic maps back to a source location. +- Root-level integration tests and deterministic fixtures. + +### Replace + +- `SemanticGraph`, canonical JSON loading and semantic hashing. +- Refiner candidates, approval, scrap and quarantine workflows. +- Dever IR designed around the old graph. +- Old target-facing Rust and React source generators and their public backend contracts. +- Workbench presentation and temperature-alarm example. +- Old child-task plan and old repository rules that call the semantic graph trusted source. + +No adapter translates old JSON into new `.dever`. Such a bridge would preserve the rejected architecture, expand the change surface and create two sources of truth. + +The replacement list describes rejected concepts, not files present at kickoff. No old implementation exists in the working tree or HEAD. Initialize `dever-core` and the root test crate first; add the runtime and CLI when their real consumers exist, without empty crates or partially working public commands. + +## 4. Source Model + +### 4.1 Source Files And Spans + +`SourceFile` owns: + +- canonical source-root-relative path; +- UTF-8 text; +- precomputed line-start offsets. + +`Span` contains a file identifier and half-open byte range. Line and column are calculated only when rendering a diagnostic, keeping compiler nodes small. Every token, syntax node, resolved reference and HIR instruction retains a span. + +The loader: + +1. walks the supplied source root for `.dever` files; +2. sorts paths lexically before parsing; +3. rejects invalid UTF-8 and duplicate canonical paths; +4. derives the expected package name from the path; +5. returns all diagnostics in deterministic file/span order. + +### 4.2 Tokens And Trivia + +The lexer emits semantic tokens plus newline and `#` comment trivia. Newlines matter for separating linear statements; whitespace otherwise does not. Comments attach to the following syntax item, or to the preceding item when trailing on the same line, so the formatter can preserve them. `//` is always one integer-division token and is never considered comment syntax. + +Numeric text remains unconverted in tokens. Parsing a very large literal directly into i64 or f64 would lose the ability to issue precise range and rounding diagnostics; conversion happens during checking with the expected type. An inexact Decimal literal is rejected rather than silently rounded, while checked Decimal arithmetic may produce the specified rounded result. + +### 4.3 Syntax Tree + +The parser produces a lossless-enough typed AST: + +- `PackageDecl` and exposed names; +- record/choice `TypeDecl`; +- `FunctionClause`; +- input patterns and output declarations; +- assignment/call statements; +- value construction, call, field, literal, unary and binary expressions. + +The parser does not resolve names or infer types. It supports bounded recovery at a newline, closing delimiter or next top-level declaration and returns a partial tree plus diagnostics. Later phases run only for packages without syntax errors. + +## 5. Formatter + +The formatter consumes the syntax tree and attached comments, never performs string replacements over raw source, and owns one layout: + +- two-space indentation; +- multiline package exposure and type members, one item per line; +- commas only where grammar requires positional separation; +- no semicolons; +- stable spaces around operators; +- record fields in declaration order after successful resolution; +- parentheses inserted only when required by precedence; +- one blank line between top-level definitions and no trailing whitespace. + +`format(format(source)) == format(source)` is an acceptance invariant. Invalid files are not rewritten in Core 0.1; the CLI reports diagnostics and leaves them untouched. + +## 6. Resolution And Symbols + +Resolution runs in three passes: + +1. Register every package, type, function name and function arity. +2. Register record fields, choice variants and full function signatures. +3. Resolve bodies, patterns, exposed names and cross-package references. + +Symbols use compiler-assigned integer IDs internally. User-visible identity remains the package-qualified source name, not an opaque semantic graph ID. + +The resolver enforces: + +- one file per package and exact path/package match; +- local source plus the fixed compiler-owned standard catalog as the only Core 0.1 package origins; +- reserved core single-segment names and the `dever.*` namespace, which local source cannot declare; +- unique names inside a package; +- one function signature per name/arity; +- contiguous clauses for the same signature; +- public access only through `exposes`; +- full qualification across packages; +- acyclic package, type and function-call graphs. + +Resolution never performs network access or searches outside the supplied source root in Core 0.1. An unavailable official or third-party package is an ordinary unknown-package diagnostic. Future lockfile-pinned components will be registered as an additional origin before these same resolution passes; this does not require imports, aliases or version text in `.dever` calls. + +### 6.1 Distribution Compatibility Contract + +External distribution is deferred to 0.2, but Core 0.1 must not close off its confirmed identity model: + +- a stable component ID such as `shemic.contact` owns the root of package and type identity; +- registry, Git and local paths are replaceable source transports and never appear in `.dever` references; +- Git additions require an explicit tag or revision, then lock the full commit and normalized SHA-256 content hash; +- CLI-maintained `dever.mod` records direct dependency intent while `dever.lock` records the complete exact graph; +- verified sources live in a global read-only content-addressed store shared across projects; +- only explicit `add`, `update` and `fetch` operations may access the network; +- every package supplied by a component must remain under that component ID, and duplicate providers are errors. + +No 0.2 parser, Git client, resolver, store or command is implemented speculatively in Core 0.1. The current resolver needs only reserved namespaces and origin-neutral internal symbol IDs so adding a locked package origin later does not change source names. + +The ordinary call graph is computed after all bodies resolve. Sequence handlers and handler-input forwarding are explicit HIR edges; native specialization checks the fully substituted instance graph, so indirect recursion cannot hide behind `each`, `reduce`, `reduce_until` or a handler parameter. + +## 7. Type System + +`TypeId` points to interned compiler types: + +- the six scalar types; +- user record and choice types; +- `Nullable(TypeId)`; +- `List(TypeId)`; +- `Map(KeyTypeId, ValueTypeId)`; +- compiler-owned `MapEntry(KeyTypeId, ValueTypeId)`; +- compiler-owned `Stream(ItemTypeId)` opaque resources; +- ephemeral named multi-output result types. + +There are no user-defined generic parameters in 0.1. List, Map, MapEntry, Stream and sequence functions are compiler-provided generic contracts. Static `HandlerSignature` is a separate function-parameter contract and is not a runtime `Type`. + +The type interner owns idempotent nullable construction for standard generic returns; explicit doubled `?` syntax remains invalid. Structural equality follows the PRD, including ordered Map pairs and IEEE Float comparisons inside aggregates. Checking, reference evaluation and native lowering must share these contracts. + +Checking is expression-directed: + +- function inputs and outputs provide declared boundary types; +- assignment provides an expected type once the local is initialized; +- an uninitialized local takes the inferred type of its first complete assignment; +- empty List/Map and context-sensitive numeric literals require an expected type; +- the only implicit numeric conversion is Int to Decimal; +- function output, static handler and sequence-operation contracts are checked before lowering. + +Linear bodies make definite assignment a forward pass. The checker tracks each local as uninitialized or initialized with one stable type. Record field writes require an initialized record and preserve its type. + +## 8. Clause Analysis + +Clause-based conditional dispatch is a permanent source-language constraint, not a Core 0.1 deferral. Native decision trees are its internal implementation and do not introduce another source-level branching form. + +Each same-name/same-arity group becomes one `FunctionDef` with multiple clauses. Analysis separates base input types from clause patterns. + +Patterns normalize into symbolic sets: + +- Bool and no-payload choice types become finite atoms. +- Nullable adds a distinct null atom. +- Payload choices keep the variant atom and bind payload fields after selection. +- Text/numeric constants become singleton sets. +- Int and Decimal comparisons become normalized intervals with open or closed endpoints. +- `other` is computed as the complement of all explicit patterns in that argument position and group. + +A multi-input clause is a product of its per-input sets. The analyzer incrementally subtracts each product from the function input universe: + +- intersection with prior coverage means overlap; +- an empty clause domain means unreachable; +- remaining universe after all clauses means non-exhaustive. + +The algorithm works on finite atoms and interval boundaries, not enumerated i64 or Decimal values. Float range patterns are rejected before this phase. The source order is never part of selection. + +At runtime, HIR stores a compiler-chosen decision tree built from the normalized domains. It can optimize tests while preserving order-independent semantics; no runtime “first matching clause” rule exists. + +## 9. HIR And Reference Evaluation + +The typed HIR contains only resolved IDs and typed operations: + +- constants and value construction; +- local/field load and store; +- checked numeric/text/boolean operations; +- direct function and statically bound handler calls; +- standard collection/sequence calls with explicit handler targets; +- clause decision nodes; +- named outputs. + +The evaluator uses an explicit call frame with owned locals and a source span for every operation. There is no global mutable program state. + +Runtime values are: + +```text +Bool +Int(i64) +Decimal(DecimalValue) +Float(f64) +Text +Id +Record +Choice +Nullable +List +Map +Stream +MultiOutput +``` + +Aggregate values use shared immutable storage with copy-on-write mutation. This implements source-level value semantics without eagerly cloning every nested List, Map or record. The optimization is internal and must pass alias-observation tests. + +Map storage uses an insertion-ordered map. In Rust, `indexmap::IndexMap` is the planned implementation; replacement keeps the same wrapper contract. Removal must use order-preserving removal, and overwrite must retain the existing position. + +Evaluation returns either named outputs or a `RuntimeFault`. A fault carries category, message and originating `Span` and unwinds only to the current CLI execution boundary. There is no language-level catch path. + +The evaluator is intentionally not optimized as the production runtime. Its job is to make language semantics easy to inspect, to evaluate focused tests and to serve as a differential oracle for native output. + +## 10. Native AOT Compilation + +The native path consumes only fully checked typed HIR. It performs whole-program specialization from the selected zero-input entry function and emits one transient Rust crate: + +```text +reachable typed HIR + | + v +specialize concrete List/Map/Stream and handler-binding combinations + | + v +emit static Rust records, enums, functions and loops + | + v +cargo build --release -> native executable +``` + +There is no generic `Value` in generated function signatures. Dever records lower to concrete Rust structs, choices to concrete enums, nullable values to an internal option representation, and calls to direct functions. Clause domains lower to compiler-selected decision trees. Because sequence handlers are concrete after specialization, `each`, `reduce`, `reduce_until`, `filter`, `find` and `sum` lower to direct calls and loops. List/Bytes loops have no virtual dispatch; only opaque Stream pulling may use the internal producer abstraction at the I/O boundary. + +Source-level value semantics do not require eager deep copies. A last-use/liveness pass may move a value when the source is dead; when both aliases remain live, generated aggregates use a language-owned clone or copy-on-write representation. This is invisible to `.dever` and is verified against the evaluator. Locals and non-escaping records remain eligible for normal Rust/LLVM stack and register optimization. The AOT runtime has no garbage collector and no reflection registry. + +Checked Int and Decimal behavior is emitted explicitly, so the release profile cannot remove language-required overflow or invalid-operation faults. Runtime helpers carry compact source-location IDs; the executable maps a fault back to the original `.dever` path and span. Internal Rust compiler errors are toolchain failures, not Dever diagnostics, and generated Rust names must never appear as user symbols. + +The transient crate depends only on `dever-runtime`. Its generated source and Cargo metadata live in a compiler-managed build directory, are not emitted by a public command and may be deleted at any time. The optimized profile starts with `opt-level = 3`, thin LTO and one codegen unit; benchmark evidence may change those internal settings without changing the language. + +The first backend deliberately uses stable Rust as a bootstrap bridge to LLVM. A later direct LLVM or Cranelift backend can consume the same checked HIR and runtime contracts without changing `.dever` source. No `dever-backend-rust` public crate or target-specific surface syntax is introduced. + +Self-hosting is a staged replacement, not a Core 0.1 syntax promise: + +1. The Rust bootstrap compiler establishes parser, checker, HIR, native semantics and conformance fixtures. +2. Later language phases add the explicit file/process/memory capabilities required to write compilers. +3. The frontend and standard library are reimplemented in `.dever` against the same fixtures and HIR contracts. +4. A stage-1 compiler builds stage 2; stage 2 builds stage 3; stage-2/stage-3 behavior and reproducible artifacts are compared before the Rust frontend becomes only a seed. + +## 11. Numeric Implementation + +`dever-runtime::number` is the only module allowed to depend on the decimal implementation. It exposes a language-owned `DecimalValue` and operations returning: + +```text +Result<DecimalValue, NumericFault> +``` + +Core 0.1 uses the Rust `dec` crate's `Decimal128` plus a reusable Context configured for 34 digits and round-half-even. Its status flags are cleared and checked around every operation: inexact/rounded is an accepted rounded result, while overflow, underflow, division by zero and invalid operation become Dever runtime faults. Decimal NaN/Infinity never enter a normal Dever `Value`. Compile-time constant evaluation and the reference evaluator call this same runtime wrapper, so native and reference behavior cannot acquire separate Decimal implementations. + +This boundary is necessary because `dec` uses libdecnumber internally and exposes more IEEE decimal states than the language permits. Conformance tests, not the dependency's default operators, define Dever behavior. + +Int operations use Rust checked arithmetic. Division and remainder explicitly handle zero and `i64::MIN / -1`. Int-to-Decimal conversion is exact. + +Float uses Rust f64 operations without fast-math transformations. List summation is left-to-right. NaN/Infinity classification is exposed through `math` functions. + +## 12. Standard Library Boundary + +Core names such as `each`, `reduce`, `reduce_until`, `close` and `get` are registered from a static table containing: + +- accepted arities; +- generic input/output constraints; +- whether an argument is a sequence, value or statically bound handler; +- evaluator function. + +Qualified standard packages (`text`, `decimal`, `float`, `math`) use the same table with qualified names. This is a fixed compiler-owned catalog, not a plugin registry. + +Every implemented standard operation has one checking contract and one native lowering rule. Sequence source classification and handler validation are shared so List, Bytes and Stream forms cannot drift. The deferred evaluator must later consume these same checked contracts rather than define a second interpretation. + +## 13. CLI Contract + +`dever-cli` builds one `dever` binary: + +```text +dever check <source-root> +dever fmt <source-root> +dever build <source-root> <package.function> --output <path> +dever run <source-root> <package.function> +``` + +- `check` loads every package and runs all static phases. +- `fmt` prepares every file before writing; if any source file is invalid, no file is modified. Record-field reordering requires successful resolution. +- `build` checks the complete source root, specializes one exposed zero-input function as the entry point and writes an optimized native executable to the requested path. +- `run` uses the same native compilation path in a compiler-managed directory and then invokes the executable; it does not run the reference evaluator. +- Outputs print in declaration order using deterministic Dever value formatting. +- Diagnostics go to stderr; program output goes to stdout. +- Exit 0 means success, exit 1 means source/runtime diagnostics, exit 2 means invalid CLI usage. + +The CLI is deliberately not a project/package manager. Source-root selection avoids introducing a manifest before dependency versions or executable targets are designed. + +Core 0.1 therefore has no `add`, `update`, registry, signature or component-cache commands. Those are a 0.2 distribution concern; `check`, `build` and `run` remain offline and deterministic. + +### 13.1 Command Identity And Coexistence + +The Cargo binary and canonical product command remain `dever`. Command examples, help semantics and the public CLI contract use that name. + +During Core 0.1 coexistence with the existing Dever framework CLI, a machine may expose this compiler on `PATH` as `deverc`. That entry is a symlink or launcher for the same compiler binary, not a second CLI implementation. It must not replace, shadow or modify the existing `dever` executable. + +Repository builds and automated tests invoke the built binary directly and never mutate global executables or shell configuration. Installing, updating or removing the temporary `deverc` entry is an explicitly authorized machine-level operation. Moving the compiler onto the global `dever` name requires a separate migration plan. + +## 14. Diagnostics + +All compile diagnostics use one structure: + +```text +severity +code +message +primary span +zero or more labeled related spans +optional help +``` + +Examples include both conflicting clause spans, declaration and bad-reference spans, or first initialization and later type-changing assignment. Rendering uses `path:line:column` and a source excerpt. Messages describe the violated Dever rule and never expose Rust type names, crate names or implementation stack traces. + +Runtime faults use the same renderer with a runtime category. The CLI may include a Dever call trace, but the first diagnostic always points to the failing `.dever` operation. + +## 15. Migration And Repository Cleanup + +Implementation starts from the documentation-only repository verified on 2026-09-05: + +1. Initialize `dever-core` and `dever-tests`; add `dever-runtime` and `dever-cli` at their implementation stages. +2. Implement `dever-core` around textual language phases. +3. Do not recreate old Refiner, IR, Workbench, generated backend or semantic JSON architecture. +4. Preserve the existing deletion of superseded planning documents; no old child links exist. +5. Update repository `AGENTS.md` and Trellis package specs so future work treats `.dever` as the trusted source. +6. Add the Core 0.1 examples and tests. + +No database, persisted user data or old implementation exists, so there is no migration. Git history retains the old planning documents. + +## 16. Verification Strategy + +Tests are organized by behavior rather than compiler module: + +```text +test/dever-tests/tests/ + syntax_and_format.rs + package_resolution.rs + type_checking.rs + clause_analysis.rs + value_semantics.rs + collections.rs + numbers.rs + failures.rs + native.rs + differential.rs + performance.rs + cli.rs +``` + +Fixtures include valid programs plus one focused invalid case per diagnostic. Tests assert diagnostic code and relevant spans, not full prose, except dedicated rendering tests. + +The highest-risk property tests are: + +- parse/format/parse semantic equivalence and formatter idempotence; +- clause order permutation does not change coverage or result; +- Int division/remainder identity across boundary samples; +- Decimal conformance vectors for precision, rounding and fault flags; +- copy-on-write values do not expose alias mutation; +- Map overwrite/remove/reinsert order; +- Float sum remains left-to-right. +- reference evaluator and native executable produce identical outputs/fault categories for the conformance corpus; +- generated executables do not link the evaluator or perform per-element dynamic dispatch; +- representative native hot paths stay within the PRD performance budget against equivalent safe Rust baselines. + +The foundation extension adds focused cases to the existing `syntax_and_format`, `core_semantics`, `core_native`, `runtime_foundations` and `hello_native` targets. They cover handler syntax and forbidden positions, signature matching and forwarding, specialization-cycle rejection, List/Bytes reduce behavior, Stream alias/EOF/stop/close/error behavior, official package compilation, and actual CLI check/run/build of a chunked state parser. File fixtures use test-owned temporary directories; TCP fixtures use only loopback port 0, bounded data and explicit timeouts. + +No existing service, browser, npm build, full workspace test or external integration environment is involved in this extension. + +## 17. Key Trade-offs + +- One compiler crate is less independently reusable than many compiler crates, but avoids unstable public boundaries during the first grammar; the native runtime is separate only because generated programs are a real independent consumer. +- No recursion/effects limits programs to finite pure computation, but makes the first language deterministic and prevents hidden control flow. +- Full clause checking is more compiler work than ordered pattern matching, but it directly enforces the user's requirement that AI-written code cannot hide behavior in ordering or nested branches. +- A decimal dependency is safer than hand-written decimal128 arithmetic; the wrapper and conformance suite prevent it from defining public language behavior. +- A small fixed standard catalog is less extensible than plugins, but keeps the first implementation inspectable and removes speculative architecture. +- Rust emission gets optimized native code much earlier than implementing an LLVM backend, while typed HIR, differential tests and a private artifact boundary prevent Rust from becoming the language definition. +- Keeping a reference evaluator adds implementation work, but it localizes semantic debugging and gives native code generation a deterministic oracle; it is excluded from production binaries. +- Separating component identity from Git/registry location adds two CLI-managed metadata files in 0.2, but prevents repository moves from becoming source-breaking type renames and enables one verified global source store. +# Compiler Contract Extension (2026-09-08) + +Extend existing AST fields and clause modifiers, normalize field bounds with the same interval owner as patterns, then analyze checked HIR before publishing Program. Keep private checks in field access/construction owners. Function/package metadata feeds effect/dependency checks and deterministic API snapshots. Failure obligations follow values across aggregates and calls, with explicit source recovery declarations as the only normal-value discharge boundary. Static handlers share existing specialization bindings. Facts must be conservative on mutation and unknown operations; no runtime interpreter, speculative fallback, new dependencies or source keyword for every domain. + +Independent syntax/formatter work owns syntax.rs/parser.rs/format.rs; effect/API work owns contracts/effects.rs/api.rs and CLI; main owns semantic types, HIR integration, fact/failure analysis, standard-source migration, tests and docs. All workers preserve shared changes. Core check is the single enforcement entry used by CLI and native execution. + +# AI-Oriented Source Architecture Design (2026-09-19) + +Source classification is one compiler-owned value derived from the relative source path. It records main or component/domain/role/topic identity and is reused by parsing, Model registration, symbol resolution, API route construction and architecture diagnostics. Those layers must not rediscover roles with independent string searches. + +Every physical source still has a unique internal owner name so diagnostics, HIR ownership and native symbol generation remain deterministic. App declarations additionally receive one stable logical alias, `<component>.<domain>.<name>`, regardless of whether they live in `app.dever` or `app/<topic>.dever`. Alias collisions are duplicate declarations. Topic paths remain an implementation detail and never leak into API snapshots or call sites. + +Name lookup tries the current physical owner first, then role-relative names in the current domain, same-component App aliases, and finally full App aliases. A resolved reference is accepted only when the role dependency matrix permits it. Main and API may call App; App may call private roles in its own domain and App aliases elsewhere; private roles may collaborate only inside their own domain. API functions are registered for HTTP routing but rejected as source call targets. Model type identity stays path-derived, while generated operations are restricted to their owning domain. + +Application `public` modifiers are rejected. App functions and App-declared types form the external contract automatically; Model record/choice/ID types remain type-visible where relations or App contracts require them. After failure inference, every App function is also checked so a private Domain/Adapter error type cannot leak through its effective error set. Bundled standard sources keep compiler-internal visibility metadata because they are not application architecture. The existing API baseline and unused-export checks consume logical App names. + +Architecture validation runs before semantic bodies: legal layouts, reserved names, file/directory exclusivity, flat non-API topic directories and the no-single-file-role-directory rule. Declaration-role validation then keeps API thin, Model storage-only and App as the sole cross-domain capability boundary. Port syntax itself is not expanded in this change; `port` is a reserved architectural role until a separately approved replaceable-boundary declaration contract exists. diff --git a/.trellis/tasks/09-04-dever-language-mvp/implement.md b/.trellis/tasks/09-04-dever-language-mvp/implement.md new file mode 100644 index 0000000..23ba125 --- /dev/null +++ b/.trellis/tasks/09-04-dever-language-mvp/implement.md @@ -0,0 +1,819 @@ +# Dever Minimal Core 0.1 Implementation Plan + +## HTTP/2 Ceiling And Soak (2026-09-18, Complete) + +子任务 `09-13-http2-ceiling-soak` 复用既有负载器，将客户端 worker/CPU 集合参数贯通 HTTP/1、HTTP/2 和长连接入口。34 项运行器定向测试及最小 loopback 回归通过；生产运行时未改。1 CPU、128 MiB 下 h2c/TLS+h2 的 16x16、30k/45k/60k/90k 双次扫描表明本机吞吐约 36k-40k req/s 时服务端接近 1 核，客户端仍占 1.4-2 核；请求槽丢弃使它不是纯服务端上限。28k 的首轮稳态未达 99% 成功率，故改用 20k：128/64 MiB 各 60 秒的四组成功率为 99.77%/99.92%/99.94%/99.80%，无请求错误、OOM 或 FD 残留。h2c/TLS+h2 各 100 轮重连无错误或 FD 残留；短时 RSS 数据不证明小时级无泄漏。原始报告、命令和限制见 `test/performance/README.md`。 + +HTTP/2 有界验收和 typed ORM 子任务均已完成。根任务保持进行中：历史计划的编译器发行/跨平台与自举仍未交付，小时级稳定性等扩展也没有在本轮获批或验证；不得凭子任务完成数宣称整个语言路线图结束。 + +## N5 HTTP/2 (2026-09-12, Complete) + +子任务 `09-12-http2` 完成：Hyper/h2 buffered/live 服务端、single-use/pooled 客户端、TLS ALPN、固定流控窗口与每连接流配额、逐流 handler/上传/下载/SSE、RST 隔离及 GOAWAY 排空。官方 `Limits.http2: Http2Limits?` 与检查器一起直接演进，静态原生桥接复用既有 Nullable/Record；没有新 intrinsic 或兼容层。 + +Hyper Executor 的协议任务接入 Dever 子 Scope，关闭 admission 与登记串行，故障上报/注销在 completion 之前；不可取消 supervisor 清理协议任务和已启动 blocking 后代后才释放物理许可。池连接归持久 driver；上传/下载共享流租约。只有 Hyper 结构化交回的未发送请求可在原期限重排，已发请求不重放。 + +88 个不同定向用例通过：HTTP/2 client/server 各6，HTTP engine12、HTTP library7、live16、pooled network14、pooled library6、async runtime10、structured concurrency11。独立 h2 对端覆盖流控、取消、容量、ALPN、语义与故障；原生程序覆盖单连接 TLS 嵌套代理、流式上传、SSE、run/await 和原有 HTTP/1/WSS。修复传输取消误升为整服故障、响应发布锁重入、GOAWAY 未发请求卡住旧租约和 Response TE 静默删除；测试端句柄和 WSS 独立连接预算也已修正。 + +最终 CLI release 构建、backend check/fmt-check、network_bench/reference 编译及源码检查通过；独立审查闭环。Rustfmt/Clippy 不可用，官方库 fmt-check 的原有 net.dever 差异未改。临时构建目录和暂存约378 MiB可再生缓存已清理，旧编译器与性能报告保留。不提交、不自动归档；N5 实现没有运行 HTTP/2 性能矩阵。后续仍有 HTTP/2 压测、小时级稳定性、入口调度优化、RFC 8441/协议扩展、数据库、发行和自举。 + +## P2 Async Comparison And Long Connections (2026-09-12, Complete) + +子任务 `09-12-runtime-performance-p2` 继续复用 P1 工具，完成同语义 Rust run/wait 对照、perf 调度定位、HTTP/HTTPS 高负载和 TCP/WS/SSE 分阶段回收测量。12 个运行器检查、4 个独立对端检查、11 个原生入口及新 Rust examples 的离线构建通过；未修改生产编译器/运行时。 + +1 CPU/128 MiB 下，Dever run/await root约15.50 µs、实验worker约2.003 µs，与Rust15.59/1.984 µs接近；任务入口跨线程切换是明确优化方向，根任务配额、Send和取消语义需保留。54组HTTP高负载发出1500830请求全部成功、299170计划请求未发出；压测槽位已限制结果，不是服务上限。 + +最终27组长连接覆盖32/128/512条，每组3轮；18144连接、377061消息均成功，81次FD回基线。另各协议512连接十轮，共15360连接、471207消息，无错误/OOM且30次FD回基线；SSE RSS在约25.3 MiB趋稳。修正采样退出与阶段交接竞态、SSE无心跳误通过和客户端清理；完整证据与限制集中在 `test/performance/README.md`。 + +P2 结束时尚未启用http2/h2依赖；当时规划的TLS ALPN、独立流/连接预算、流控、取消隔离和GOAWAY已由后续N5交付。小时级soak与入口调度优化另行落地，总语言路线图保持未完成。 + +## P1 Standalone Runtime Performance (2026-09-12, Complete) + +用户批准建立常驻内存、异步调度和 HTTP/HTTPS 三组基准，取代旧网络实施阶段的局部“不压测”限制。子任务 `09-12-runtime-performance` 完成：真实原生二进制、Rust runtime/Hyper 对照、有界固定速率负载、Linux RSS/PSS/CPU 和专属 cgroup 预算、原始 JSON 报告。生产语言/运行时未改动。 + +7 个运行器与 2 个独立 HTTP 对端检查通过；离线原生/release 构建和格式检查通过，Rustfmt/Clippy 不可用。最终 78 组主测、18 组新连接、15 组 64 MiB 空闲基线均完成且无 OOM；主测网络发出 107962 请求全部成功，38 个计划请求未发出并按原因记录。当前只是热缓存、短时、指定负载结果，不是吞吐极限、冷启动整机内存或长期稳定性结论。 + +运行命令、确切统计口径和首轮数据以 `test/performance/README.md` 为准，原始产物/报告保存在 `target/performance/`。后续候选：同语义 Rust 异步对照与调度剖析、高负载扫描、TCP/WS/SSE 长连接；其余语言/后端路线图仍未完成。 + +## N4 Concurrent Composition And Reusable Clients (2026-09-12, Complete) + +- [x] Implement source runtime configuration, typed handler context, timeout/race and pull-driven stream adapters; extend async memory traversal. +- [x] Add verified TLS transport and bounded reusable HTTP clients with streamed uploads/downloads; integrate HTTPS/WSS and TLS servers. +- [x] Wire compiler contracts, official source, native conversion and examples together. +- [x] Run focused compiler/native and independent loopback protocol/lifecycle tests; review final ownership and limits. +- [x] Update canonical language/spec docs and record evidence without a commit or archive. + +### N4 Evidence + +83 distinct focused cases passed: async_composition 7, pooled_network 14, pooled_library 5, structured_concurrency 11, async_network 9, http_engine 12, http_library 6 and live_network 16. Reruns are not counted twice. The old structured-concurrency case that rejected async List handlers was migrated to the newly accepted contract and checked for a concurrency effect. + +New coverage includes source runtime bounds/start-once, timeout cleanup of started blocking children, race loser cleanup/fault propagation, Channel null/drain/close and periodic stream timing, static handler/output checks, C012 stream failure obligations, async memory traversal/fusion and concrete HTTP Context transfer. Native programs exercise HTTPS, pooled proxy calls through shared Context, streamed uploads/downloads, SSE, WSS and graceful shutdown. + +Independent peers verify physical connection reuse, repeated headers, early chunks beyond the response-head deadline, chunked upload, slow/abandoned/truncated bodies, upload failure/cancellation, queue/body/idle deadlines, pool close wakeups, unpolled-task cleanup, certificate CA/name rejection and WSS frames. A reproduced lifetime defect allowed a pool's unpolled driver to escape its creator; constructing the driver owner guard before task creation fixes it. Native checks also exposed missing Channel Debug/Render for shared records and async handlers entering synchronous fusion; both are corrected. Final bounded cleanup removed an infallible Result from the private lease-completion path; all 14 pool cases passed afterward. + +Actual CLI backend example check and fmt-check passed without warnings. Offline build produced a standalone binary; running with an empty environment and PATH restricted to /usr/bin:/bin printed exactly `Hello /Dever\nrequests = 1\n` and exited successfully. Official source formatting, optional-reference core/runtime/CLI cargo check, whitespace/conflict-marker and final ownership/contract inspection passed. Rustfmt/Clippy are not installed, so their checks were unavailable; Rust layout was reviewed manually. + +Only initial dependency preparation fetched locked packages; validation/builds were offline. All network peers used owned loopback random ports and static test certificates under test/. No full suite, prior performance baseline, existing service, persistent example launch, global install or commit. No throughput/RSS improvement is inferred from functional checks. Temporary build, certificate-generation, formatting and demo directories are removed at handoff; workspace native caches remain. N4 is complete; HTTP2, protocol extensions/close metadata, database, distribution/cross-platform and self-hosting keep the umbrella task active. + +## N3 Streaming HTTP, SSE And WebSocket (2026-09-12, Complete) + +- [x] Add `serve_live`, HttpReply and LiveLimits while keeping ordinary serve direct. Reuse request/response codecs and connection configuration; one-slot body channel and one-handler Group per live connection. +- [x] Add SSE field encoding, bounded chunks, Last-Event-ID access and body-driven heartbeat timers; no event history or extra heartbeat task. +- [x] Reuse tokio-tungstenite 0.30.0 handshake/protocol for server/client Text/Binary/Ping/Pong, AsyncStream messages and close handshakes. Shared Endpoint lifecycle, 4 KiB read buffer, bounded messages/write buffers and independent directions. +- [x] Integrate resource types, exact nominal records/choices, async effects, static handlers, native conversion, API/Markdown names and Render. No new syntax, compatibility layer, dynamic interpreter or generic transport abstraction. +- [x] Verify owned wire peers and real native programs; update canonical docs/example and manually record the session without a commit or archive. + +### N3 Evidence + +46 distinct focused cases passed: live_network 16, live_library 3, http_engine 12, http_library 6, async_network 9. Repeated checks after corrections are not counted again. All final checks passed; the earlier slow-peer test assumed normal return and was corrected to observe cleanup on either return or cancellation. + +Wire evidence covers early chunk delivery, bounded slow-peer output, SSE exact encoding/heartbeats past the ordinary deadline, idle failure isolation, HEAD cleanup, handler faults, upgrade capacity/preread bytes, independent client masks, fragmentation/UTF-8/control frames, close, partial writes, waiting-writer cancellation and started blocking descendant drain. A reproduced regression showed duplicate response publication closed the existing body; cancellation ownership now starts only after publication is accepted, and the regression passes. + +The native source program exercised all four routes (buffered, stream, SSE, WebSocket), including client connect/send/receive/close and static async message-stream handlers. Existing native HTTP clients also matched independent wire peers. Final optional-reference cargo check, actual CLI example check (no warnings), CLI fmt-check, source format roundtrips and explicit whitespace/owner inspection passed. Rustfmt/Clippy are not installed and were not run. No throughput/RSS claims or old baseline. + +Only a one-time dependency fetch accessed the registry; checks/builds ran offline. No full suite, existing service, persistent example launch, global install or commit. N3 is complete; TLS/WSS, HTTP2, client pools, protocol extensions, close metadata, database/distribution and other roadmap work keep the umbrella task active. + +## N2 HTTP Engine (2026-09-12, Complete) + +The user approved continuing after N1. Execute inline; no further planning approval. Reuse Hyper HTTP/1 on the shared Tokio runtime. No legacy compatibility, old baseline, full suite, service checks or global installation. + +- [x] Replace the custom HTTP parser/transport with Hyper server/client primitives and migrate all source callers. +- [x] Preserve repeated headers as `List<Header>` with byte values; buffer bodies within explicit limits. Reuse byte storage across the engine boundary. +- [x] Add `serve(async handler, Listener, Limits) -> ()` with bounded structured connections, server keep-alive, chunked input and cancellation cleanup. Configuration/accept/handler faults propagate; peer protocol failures terminate only that connection. +- [x] Add one-shot `send(address, port, Request, Limits) -> ResponseResult`. Poll its driver in the request future; no detached tasks, implicit retries or client pool. +- [x] Check static handler contracts/effects and native record conversion. Replace obsolete parser tests with independent loopback wire cases and focused compiler/native tests. +- [x] Update canonical docs and record evidence. TLS, HTTP/2, client pooling, streaming response bodies, WebSocket and SSE remain later work. + +Limits: header buffer at least 8192 bytes (Hyper requirement), nonnegative body bytes, positive timeout, bounded connections. Defaults: 8192 / 1048576 / 5000 ms / 32. Runtime owns HTTP framing; outgoing Content-Length/Transfer-Encoding are rejected. Request header/body/handler and stalled write deadlines prevent one idle peer retaining a connection indefinitely. Static Dever handlers specialize directly; no dynamic interpreter dispatch. + +### N2 Evidence + +- 44 distinct focused cases passed: http_engine 12, http_library 6, async_network 9, structured_concurrency 11, native_ownership byte cases 3, runtime_foundations bytes case 1, contract_execution selected example checks 2. Repeats after local corrections are not counted again. +- Independent wire cases cover persistence/pipelining, HEAD and 204/205/304, fragmented/chunked/binary bodies, repeated and non-UTF8 headers, conflicting lengths, header/body limits, unsupported trailers/upgrades, truncated responses, all deadlines, connection capacity and descendant cleanup. Header timeout closes directly; body/handler timeouts produce 408/504. Native tests execute real compiled send/serve and compare client bytes to an independent peer. +- Bytes tests retain unique Text storage transfer, shared-alias behavior, shared slices and consuming iteration. Connection setup now configures TCP_NODELAY once through one helper. No old-performance baseline, throughput or RSS claims were derived from correctness tests. +- Core compiled offline with the optional reference feature. Actual CLI HTTP example check/fmt-check and formatting roundtrips passed; the persistent example was not started. Final owning-layer, dependency, obsolete-reference and whitespace review passed. Rustfmt and Clippy were unavailable (components not installed), not reported as passing. +- Hyper 1.11.1, hyper-util 0.1.20, http-body-util 0.1.5 and bytes 1.12.1 are pinned. Missing transitive development dependencies were fetched once; builds/tests remained offline. No Axum, TLS or HTTP2 features were added. +- No full suite, existing service check, load test, global install, commit or task archive. N2 is complete; the umbrella task remains active for N3 and other unshipped roadmap work. + +## N1 Async Stream And TCP (2026-09-12, Complete) + +- [x] Persist the approved new-language/no-compatibility policy; use existing inline execution routing. +- [x] Add bounded, cancellable async stream consumption and Tokio TCP resources to the shared runtime. +- [x] Integrate AsyncStream types, async handlers, effects/liveness and concrete native code generation. +- [x] Replace synchronous TCP source contracts and migrate official HTTP transport, examples and tests. +- [x] Verify stream backpressure, close/stop, duplex I/O, timeout and compiler/native behavior with focused tests. +- [x] Update public contracts, inspect the final changed flows and record evidence. N2 HTTP engine and N3 WS/SSE remain deferred. + +### N1 Evidence + +- 62 distinct focused cases passed: async_network 9, async_stream 6, structured_concurrency 11, async_native 6, async_runtime 10, http_library 11, core_native `official_` 5, runtime_foundations `stream` 3, and the reference/native file-stream differential case 1. The HTTP benchmark stayed ignored. Repeated individual checks after local fixes are not counted again. +- Network tests owned only loopback port-0 sockets. A one-worker runtime proved duplex progress, pending close wakeups, read timeout reuse, partial-write cancellation closure, pre-pull backpressure, failure while the producer is idle, child cleanup and listener release. The persistent server example was checked without starting it. +- Native execution exposed missing Render implementations for new resource types; those were fixed at the runtime owner and the affected native checks passed. Migrated TCP differential cases now assert independent native expectations; the synchronous evaluator no longer carries a duplicate TCP implementation. +- Compiler/default and optional reference builds passed offline. Source formatting round trips, explicit whitespace checks and final owning-layer inspection passed. Rustfmt/Clippy were unavailable because those components are not installed; no claim of passing them. +- No workspace-wide tests, service checks, old-performance baseline, network load test, global install or commit. The task remains active for N2 HTTP engine and N3 WebSocket/SSE, not for unfinished N1 work. + +## Structured Concurrency Implementation (2026-09-12, Complete) + +The user approved this preceding implementation. Networking was paused during it; N1 above resumes AsyncStream/TCP. Each phase uses only focused offline tests under root `test/`; do not run the workspace-wide suite, service checks, old-performance baselines or network load tests. + +### Phase A: Freeze Syntax And Semantic Model + +- [x] Add the canonical `async name(...)` modifier while retaining post-signature `pure`/`recover`; update lexer/parser/formatter and Markdown signature model together. +- [x] Add async to function identity metadata and HandlerSignature compatibility without changing package/name/arity lookup. +- [x] Reject `async` together with `pure` in the first release and classify every concurrency operator through the existing transitive effect owner. +- [x] Add contextual `await`, `run`, `stop`, `group`, `parallel`, `blocking` and `channel` syntax; reject async calls as ordinary stored values. +- [x] Add HIR task/group/channel operations and compiler-owned affine local metadata; do not add a user-visible Future or generic evaluator value. +- [x] Extend recursive type properties with thread transferability and explicit resource opt-in. +- [x] Add focused syntax/format/check cases for clause agreement, sync-to-async rejection, handler mismatch, forbidden Task escape and transfer failures. + +Rollback point: syntax/HIR-only changes must leave all existing synchronous HIR and formatted output unchanged. + +### Phase B: Await And One Shared Runtime + +- [x] Add Tokio with the smallest required features and one runtime owner in `dever-runtime`. +- [x] Emit async functions as concrete Rust futures while retaining direct sync emission for wholly synchronous entry graphs. +- [x] Implement direct `await(async_call(...))` and async entry execution; preserve zero/single/named multi-output and source-located failures. +- [x] Implement async timer sleep as the first real suspension boundary; do not use TCP/HTTP as the proof fixture. +- [x] Verify multiple sleeping tasks can make progress without one OS thread per task and sync binaries do not initialize the runtime. + +Rollback point: runtime dependency and native entry changes are isolated from Task ownership and can be removed while keeping Phase A diagnostics. + +### Phase C: Structured Task And Group Ownership + +- [x] Implement `run(async_call(...)) -> Task`, `await(task)`, and exactly-once affine consumption across local writes and every clause exit. +- [x] Implement zero-output `stop(task)` as cooperative cancel-and-wait; reject result-bearing stop and repeated consume. +- [x] Implement positive bounded `group(limit)`, backpressured `run(group, call)`, `await(group)` and `stop(group)`. +- [x] Ensure normal and abnormal parent completion cleans all children; no detached path or swallowed expected failure. +- [x] Verify start-before-await concurrency, output equivalence, cancellation cleanup, capacity backpressure and bounded task ownership. + +Rollback point: Task and Group runtime modules remain separate from scheduler construction and do not change direct await. + +### Phase D: Multi-thread And Blocking Work + +- [x] Implement `parallel(sync_call(...))` for pure transferable calls using bounded execution; preserve existing numerical/error semantics. +- [x] Implement the compiler-owned `blocking` boundary only for calls with a transitive compiler-known blocking effect; bound active workers plus queued submissions. +- [x] Route async use of `parallel_each` away from Tokio workers while retaining the current synchronous contract and shared worker validation. +- [x] Configure conservative defaults from available parallelism with explicit hard maxima; do not create per-call runtimes or unconstrained pools. +- [x] Verify multi-core execution, no scheduler starvation under bounded blocking work, exact output/error behavior and small-worker configurations. + +Rollback point: retain the existing synchronous `parallel_each` implementation until the common executor passes its focused compatibility cases. + +### Phase E: Bounded Channels And Contract Integration + +- [x] Add compiler-contextual `channel(Type, capacity)` and the runtime Channel resource. +- [x] Implement async `send`/`receive`, idempotent `close`, wakeup behavior and explicit source-defined result choices. +- [x] Extend effect, failure, dependency, redundancy, API snapshot and Markdown checks for every new signature/operation. +- [x] Add documentation only for implemented behavior and a runnable non-network producer/consumer example. +- [x] Run the available focused syntax/semantic/native/runtime/concurrency, reference-feature and whitespace checks; record unavailable rustfmt/Clippy components explicitly. + +Final gate: inspect generated native code for concrete futures/direct static handlers and absence of generic Value/handler dispatch. Record passed, failed and unrun checks without claiming network readiness. + +### Final Evidence + +- `structured_concurrency`: 11 passed; syntax, formatter shape, affine ownership, transferability, effects/failures, handler boundaries and Channel contracts. +- `async_runtime`: 10 passed; recursive cleanup, cancel-and-wait, Group backpressure, bounded channels, shared blocking limits and async `parallel_each` peak concurrency. +- `async_native`: 6 passed; real native execution of await/Task/Group/Channel/parallel/blocking/timer, sync-entry isolation and nullable Channel flattening. +- Existing synchronous `concurrency`: 5 passed and its explicit benchmark remained ignored; the async example and nullable-result native regressions each passed their exact case. +- `dever-core` with the optional `reference` feature compiled offline. `git diff --check` and explicit trailing-whitespace scans passed. Dependency inspection confirmed Tokio uses only runtime, sync and time features in this phase. +- The installed Rust toolchain has no rustfmt or Clippy component, so those two checks were unavailable rather than passed. No full suite, build, service, network, old-performance baseline, global installation or commit was run. +- Independent final review found no remaining blocking defect after nullable Channel emission, synchronous collection-handler enforcement and transitive async effect boundaries were corrected. + +### Deferred Follow-through + +- [x] Design and implement AsyncStream/Tokio TCP on the shared runtime in N1. HTTP engine and WebSocket/SSE integration remain later phases. +- [ ] Measure the new packaged binary and live network behavior only after network integration; the user explicitly waived an old-runtime baseline. + +## Active Performance Checklist (2026-09-08) + +- [x] Capture bounded compiler, record/text/list/map/JSON/HTTP and concurrency baselines. +- [x] Memoize type properties and schedule effect/failure summaries by dependencies. +- [x] Eliminate redundant record field copies and consume unique list elements safely. +- [x] Optimize direct character-list consumption and eligible collection fusion without changing fault order. +- [x] Implement incremental HTTP framing and test in-memory fragments/limits/malformed inputs. +- [x] Cache bundled frontend work and native artifacts with complete invalidation and API checks. +- [x] Measure concurrent task granularity; batch only where justified and preserve bounded Stream consumption. +- [x] Run affected offline tests, native/reference regressions, strict lint, formatting and independent final review; record measurements and update owning docs. + +User has authorized the complete implementation. This extends the active task and does not require another planning approval. Two bounded implement workers own analysis and cache respectively; main owns emission/runtime/protocols and benchmark integration. All permanent tests stay in root test/. Never run workspace-wide tests, existing services or global installation; use test-owned temporary inputs/processes only. + +## Packaged Runtime Footprint Follow-through (2026-09-09) + +- Composite runtime values and generated records/choices now append recursively through `Render::render_to`; named entry outputs reuse that same String instead of building a rendered value and then copying it into a second formatted String. Numeric `to_text` retains its direct `to_string()` path after the bounded hot-path gate exposed a slower generic formatting attempt. +- Native binding temporaries now transfer uniquely owned Text/Bytes storage and List/Map values through consuming operations. Live aliases retain copy-on-write behavior, and resource-containing values remain non-movable under the existing type-property contract. +- Exact-size finite parallel iterators create no more workers than items/batches. Unknown-length Stream dispatch keeps the caller-requested worker limit, bounded queue, cancellation and joined lifecycle. +- Production optimization arguments now use fat LTO and strip symbols; the workspace release runtime uses matching fat LTO/one-unit settings. The same allocation-performance fixture decreased from 4,680,864 to 452,352 bytes (90.3%) and measured 3,468 KiB peak RSS, versus 3,608 KiB without LTO. +- Thin LTO was rejected because ordered Map reached a 1.255 native/baseline ratio. Fat LTO passed all five gates: Int 0.713, Float 1.001, List 0.668, Decimal 0.985 and ordered Map 1.030. It trades substantially longer cold compilation for the user-approved packaged-runtime priority. +- An attempted fixed 64 KiB clamp for one-shot reads was rejected and removed: it changed a 1 MiB regular-file result to 64 KiB and failed the exact performance result. `read(limit)` remains caller-sized; low-memory programs use smaller limits or `chunks`. +- Focused runtime/concurrency/native ownership/core-native checks and both explicit performance scenarios cover the final contracts. No workspace-wide suite, service, external network, global installation or commit was used. + +## Performance Evidence (2026-09-08) + +- Nominal type properties are memoized after cycle validation. Facts, effects and failures share one dependency graph; summary changes reschedule callers instead of rescanning every function. The same 24-level shared-type debug regression went from 11.56 s to 0.47 s across the combined analysis/frontend changes. +- Native emission reuses checked HIR and last-use analysis. Safe record aliases are coalesced, scalar reads can be snapshotted before consuming large records, and exact field overwrites can move their old value. Unique Lists move elements; shared Lists clone only consumed elements. Resources retain their original lifetime and aliases retain value semantics. Empty Bytes concatenation reuses storage without weakening copy-on-write. +- Text splitting has one iterator implementation. Direct consumers avoid an intermediate List; eligible each/filter stages fuse only when handlers are proven free of observable effects and faults. A pure Int handler alone is insufficient because overflow can change fault order. Generated-code, alias, mutation and native/reference cases cover these boundaries. +- HTTP wire decoding now retains its header boundary and parsed frame between feeds. Transport uses that state; legacy one-shot scan delegates to the same implementation. Three new in-memory cases cover fragmentation, binary bodies, EOF, limits, duplicate headers and private/aliased state; five existing pure protocol tests passed. No live HTTP service was used. +- Bundled AST and the last complete frontend result are cached within a process. Native artifacts persist across processes, keyed by source-generated code, compiler/toolchain/runtime/dependency identities and native options. Atomic publication, fingerprint validation, isolated executable copies and create-new saves preserve correctness. All six cache tests and the explicit actual-CLI API-baseline regression passed. Independent CLI processes still check current sources and API baselines; no persistent HIR cache was introduced. +- Finite parallel inputs use bounded batches through the existing cancellation/join owner; Stream keeps bounded producer-driven pulls. With 32,768 inputs and four workers, median light-handler queue/batch/serial times were 140.945/3.894/0.247 ms; 512-round handlers measured 178.267/20.033/60.185 ms. Batching reduces dispatch cost but does not make tiny tasks preferable to serial execution. +- Main-agent final ownership/JSON/reference verification passed 9/7/10 tests respectively; only three network-dependent differential cases were skipped. Concurrency passed five functional cases, contracts passed 38 affected cases, and strict Clippy/type checks passed for core/runtime/CLI and affected test targets. Actual CLI library formatting and contracts/library checks passed. Independent final review found no blocking issue. +- After the final Bytes optimization, the main agent reran all three incremental HTTP cases and six default native-cache cases successfully. The actual-CLI cache test is ignored by default and had already passed when explicitly selected. Final targeted Rust formatting and new-file whitespace checks passed. + +The same source was compiled before and after this change. Linux x86-64, Rust 1.98.0, CPU affinity 4; five alternating processes per side, native internal timing excluding compilation/startup. Text/List/JSON each retain five samples per process after discarding warmup; Map retains one per process. All expected results were asserted. Medians: + +| Source workload | Before | After | +| --- | --- | --- | +| 16,384 record Text appends | 14.093 ms | 0.279 ms | +| 16,384 Text List elements | 4.786 ms | 1.342 ms | +| Long JSON string parsing | 28.072 ms | 6.977 ms | +| Record accumulation of 16,384 distinct Map keys | 548.061 ms | 1.999 ms | +| Process peak resident memory | 4,648 KiB | 3,604 KiB | + +- Separate HTTP strategy measurement used identical 16,947-byte input in 64-byte fragments, both paths on the new compiler, seven alternating samples after warmup: repeated one-shot scan 4.541 ms, incremental decode 0.366 ms. This is parser work, not server throughput or an old/new compiler comparison. +- Same release CLI hello: cold native miss 0.84 s; subsequent independent-process hits 0.17/0.17/0.16 s, including source/API checks and runtime freshness. In-process frontend cold/hit measured 171.273/3.742 ms. +- The existing five-kernel matching-runtime gate passed again with 45 effective samples per side: native/baseline Int 0.989, Float 0.998, List 0.578, Decimal 0.979, ordered Map 1.006 (limit 1.15). This checks code-generation overhead against the same runtime, not against the old compiler. Build times were measured separately: native 2,137–2,693 ms, baseline 252–680 ms. +- These are bounded local workloads, not whole-language speed guarantees. README, IMPLEMENTATION and owning specs describe the optimization boundaries. No full suite, service integration, global installation or commit; the performance scope is complete while the umbrella language roadmap remains active. + +## Active Follow-through Checklist (2026-09-07) + +- [x] AST formatter, comments/precedence/idempotence, all-file preparation and CLI fmt/check. +- [x] Text/number library expansion and explicit argument/environment/clock primitives. +- [x] Opt-in reference evaluator, core semantics and native differential cases. +- [x] Last-use and reduction-state ownership optimization with alias/order regressions. +- [x] Bounded reproducible native performance baseline and evidence. +- [x] JSON/HTTP source packages and focused protocol fixtures. +- [x] Controlled concurrent execution with explicit limits/lifecycle and focused verification. +- [ ] Resolve database choice from pending user answer; implement or record explicit deferral. +- [x] Integrate delivered flows, review, update canonical specs and actual validation evidence. + +Workers implement formatter and reference evaluator independently; the main agent owns libraries and integration. Checks are targeted, offline and bounded; no full suite, live service, global install, commit or cross-platform distribution. Later checklists are historical. + +## Follow-through Evidence (2026-09-07) + +- Implemented syntax-only AST formatting and CLI `fmt`/`--check`. Source snapshots are reloaded before staging and before replacement, comparing both the complete path set and file contents. Independent review found the initial missing path-set check; the shared source loader now closes that gap without a second traversal implementation. Per-file replacement remains atomic, not a multi-file OS transaction. +- Added Unicode Text operations, numeric parsing/rendering, process arguments/environment and explicit clocks/sleep through the existing intrinsic catalog and source-defined result choices. JSON and HTTP algorithms are ordinary `.dever` packages; no runtime JSON/HTTP parser or new dependency was added. +- JSON validates grammar, precise numeric spelling, duplicate keys, Unicode, graph references, nesting and expansion limits. HTTP supports a declared HTTP/1.1 single-message subset with binary bodies, Content-Length, strict ASCII headers and typed failures; independent peers check canonical request/response bytes, fragmentation, invalid requests/400, EOF and no-body statuses. Blocking read/write syscalls have explicit timeouts; no strict overall message deadline is claimed. +- `parallel_each` reuses static handlers/sequence classification and adds a bounded producer/worker owner with explicit worker limits, fault cancellation at pull boundaries and joined completion. Stream remains producer-owned; worker inputs/context cannot recursively contain Stream. +- Ordinary last-use moves and transferred reduction state preserve live aliases and output/field order. Resource-containing locals retain lexical lifetime; a differential socket regression verifies no premature peer EOF. Full-domain Int/Decimal guards omit redundant comparisons without altering bounded clauses. +- Passed 25 targeted core/library tests: `standard_library` 5, `native_ownership` 3, `concurrency` 4 and opt-in `differential` 13. `formatter` passed 11 tests including added/renamed source snapshots and all current official packages/examples. JSON passed 7 and HTTP passed 8 in worker validation; the main agent independently reran Unicode/exact-number JSON and the real HTTP server wire/400 case. Production CLI feature resolution excludes `reference`. +- Actual CLI `check`, `fmt`, `run` and `build` succeeded for `examples/library/src`; normal run and a standalone executable under `env -i PATH=/nonexistent` printed `json = {"message":"你好 Dever","total":6}` and `http_roundtrip = true`. +- Independent final review found no remaining blocker in formatter, reference, ownership, numeric domain optimization, concurrency and JSON after the snapshot fix. Main-agent HTTP review checked framing/status/timeout contracts. Database selection remains pending rather than implicitly chosen. +- Passed the explicitly ignored performance target on Linux x86-64, CPU affinity 4, Rust 1.98.0, cached runtime, optimization level 3 and one codegen unit. Each side used five alternating processes and 45 effective samples. Native/baseline median ratios: checked Int 0.952, Float 1.000, List each/filter/sum 0.207, Decimal 1.001, ordered Map 1.145; all met 1.15. Native build times were 1,206–1,793 ms, baseline compile times 255–668 ms, recorded separately from execution. These figures cover the selected kernels only. +- Decimal comparison was corrected to use identical lexical `0.0` representation with baseline parsing outside timing. Map diagnostics identified missed generic hash-table inlining under multiple codegen units; production and baseline now share one optimization-argument constant. This changes compiler optimization scope without changing hashing, panic, arithmetic or value semantics; it trades compilation parallelism for optimization consistency. +- After the final native compiler setting, all 13 differential cases passed again, including exact fault traces, signed zero, Map semantics and resource lifetime. Strict Clippy/type checks passed for core/runtime/CLI and every new test target; targeted rustfmt and explicit whitespace checks passed. The complete follow-through has 51 functional tests plus one five-kernel performance gate, not a full workspace test run. +- No full suite, existing service, external database, global toolchain/command change, Git commit or archive. Distribution/cross-platform remains deferred; self-hosting is still unshipped. + +- Status: Approved core foundations and foundation extension implemented and verified on 2026-09-07 +- Requirement source: `prd.md` +- Architecture source: `design.md` +- Execution mode: one tightly coupled task; no obsolete child-task dispatch + +## Completed Foundation Delivery Rule (Historical) + +Implement the foundation extension as one bounded delivery: `reduce`, `reduce_until`, Bytes traversal, `Stream<Item>`, restricted static handler inputs, file/TCP stream wrappers and focused verification. Reuse the existing collection checker, typed HIR, intrinsic catalog, native emitter and resource owner; do not build a parallel execution path. Compiler distribution, cross-platform work, formatter/evaluator work and full HTTP/JSON/CRUD/database packages remain deferred. The completed core and hello milestones are regression baselines; the old API and original phase checklists are historical. + +## Foundation Extension Checklist (Authoritative, Complete) + +### 1. Syntax And Semantic Contracts + +- [x] Parse contextual `handler(inputs) (named_outputs)` only in function input declarations, using a dedicated AST input kind rather than an ordinary value type. +- [x] Parse `Stream<Item>` through the standard generic-type path; reject wrong arity, nullable handler signatures and handler syntax in fields, outputs or local/value positions. +- [x] Add semantic `HandlerSignature` and handler-parameter symbols separately from runtime `Type`; add opaque `Stream(Item)` to the type model with non-comparable, non-Map-key resource semantics. +- [x] Centralize handler compatibility, allowed reference positions and sequence element classification so direct calls, forwarding, `each`, `reduce` and `reduce_until` cannot drift. + +### 2. Typed HIR And Static Specialization + +- [x] Replace collection-only handler IDs with shared `HandlerTarget`, `CallTarget` and handler/value call arguments. +- [x] Lower direct handler calls and multi-level forwarding without storing handlers in value locals or aggregates. +- [x] Key native specializations by function plus ordered concrete handler bindings, substitute every forwarded target, and emit only direct calls. +- [x] Detect direct and indirect cycles in the substituted specialization graph before private Rust compilation. + +### 3. Sequence Operations + +- [x] Add `reduce` and `reduce_until` to the single standard-operation registry and HIR, with exact state and `stop: Bool` output contracts. +- [x] Lower List and Bytes traversal as left-to-right direct loops; emit Bytes elements as Int values without an intermediate List. +- [x] Extend `each` to Bytes and zero-output Stream consumption while preserving existing List mapping/action behavior. +- [x] Leave `filter`, `find` and `sum` List-only and reject implicit Stream collection. +- [x] Add idempotent zero-output `close(stream)` and reject it for non-Stream values. + +### 4. Stream Runtime And Official Packages + +- [x] Add one generic runtime Stream owner with shared cursor/end/close state and serialized pull/close operations; do not add prefetch, threads or async machinery. +- [x] Implement natural EOF, one terminal typed failure item, idempotent close, alias continuation after `reduce_until`, and release of only the Stream-held source alias. +- [x] Add `ReadEvent`, `ReadStreamResult` and `chunks` to bundled `dever.io`; add socket `chunks` and listener `connections` to bundled `dever.net`. +- [x] Extend fixed `dever.system` intrinsic signatures and generated typed adapters only for Stream creation/pulling; keep public result choices in `.dever` source and OS/resource mechanics in runtime. +- [x] Preserve existing one-shot read/write/accept/close contracts without compatibility aliases or behavior changes. + +### 5. Focused Verification And Cleanup + +- [x] Add parser/semantic negatives for handler placement, type/arity/output mismatch, forbidden storage/return/comparison/clause use and indirect recursion. +- [x] Add native List/Bytes cases for empty input, order, state typing, stop behavior, zero-output each and absence of intermediate Bytes allocation. +- [x] Add runtime/native Stream cases for shared aliases, natural EOF, early stop/resume, repeated close, source close, invalid limit and one terminal failure. +- [x] Compile official wrappers through the normal source loader and test file chunks plus loopback socket chunks/connections with owned resources, bounded data and timeouts. +- [x] Exercise actual CLI `check`, `run`, `build` and the resulting standalone executable with a chunked state-parser fixture whose behavior is defined in `.dever`. +- [x] Run focused formatting/type/lint/test checks only, inspect generated code/artifacts for direct handler calls, validate Trellis context and run whitespace checks. +- [x] Review the final diff for duplicate contracts, runtime handler dispatch, eager Stream collection, HTTP/JSON leakage and unrelated changes; update the canonical specs and implementation evidence. + +Planned focused validation targets: + +```bash +cargo test --offline -p dever-tests --test syntax_and_format <handler-or-stream-filter> +cargo test --offline -p dever-tests --test core_semantics <reduce-or-handler-filter> +cargo test --offline -p dever-tests --test core_native <sequence-or-specialization-filter> +cargo test --offline -p dever-tests --test runtime_foundations <stream-or-loopback-filter> +cargo test --offline -p dever-tests --test hello_native <regression-filter> +cargo check --offline -p dever-core -p dever-runtime -p dever-cli +python3 ./.trellis/scripts/task.py validate 09-04-dever-language-mvp +git diff --check +``` + +Filters will be replaced by the concrete test names introduced during implementation. No full suite, workspace-wide test, performance gate, service start, external network call or global command mutation is authorized. File tests use test-owned temporary directories; TCP tests bind only `127.0.0.1:0` and terminate within explicit bounds. + +Rollback boundaries are syntax/semantic representation, HIR/specialization, finite sequences, runtime Stream and official wrappers. A boundary is not accepted while it relies on a runtime function value, duplicated checker rule, eager collection fallback or partially exposed public API. + +## Foundation Extension Evidence (2026-09-07) + +- Handler inputs use dedicated syntax, semantic symbols and HIR arguments. Clause coverage and runtime locals traverse value parameters only. One specialization graph resolves direct and multi-level handler forwarding, records real cross-package handler references and rejects substituted recursion; generated functions contain direct calls and no user-handler function values. +- `reduce`, `reduce_until` and `each` reuse one sequence classifier. List and Bytes lower to direct left-to-right loops, Stream pulls on demand, and `reduce_until` does not pull past the stopping item. Bytes traversal yields Int directly without constructing a List. +- Runtime `Stream<T>` shares cursor/end/close state across aliases and serializes pull/close. File/socket chunks validate creation, release their producer at EOF or after one terminal failure, and support early-stop continuation. Bundled `dever.io` and `dever.net` expose source-defined stream choices through the normal loader. +- Passed `cargo check --offline -p dever-core -p dever-runtime -p dever-cli` and strict Clippy for core/runtime/CLI plus the root test crate. `cargo fmt --all --check` passed. +- Passed 74 focused tests: 22 `syntax_and_format`, 16 `core_semantics`, 16 `core_native`, 8 `runtime_foundations` and 12 `hello_native`. File tests owned their temporary directories; TCP tests used `127.0.0.1:0`, explicit timeouts and bounded transfers. +- Final review made handler signatures reuse ordinary nested Map validation, preserved `Pull::Last` through mapped official streams, and added generated-code plus official creation-failure regressions. An independent read-only review then found no blocking correctness, safety or contract issue. +- Actual CLI `check`, `run` and `build` succeeded for `examples/stream/src`; run and the standalone executable both printed `total = 208`. The executable also ran with an empty environment and no Cargo/Rust path. +- No formatter/evaluator, async/concurrency system, HTTP/JSON/database package, external service, full workspace test, benchmark, distribution change, global command change or commit was included. + +## Active Core Checklist + +- [x] Record clause-only conditional dispatch as a permanent language rule and remove conditional blocks from the deferral list. +- [x] Retain settled repetition rules and remove the redundant planning blocker. +- [x] Record the user's approval and map the implementation boundary to core language and system interfaces. +- [x] Implement scalar/aggregate types, complete symbols, expressions, outputs and value-semantic assignments. +- [x] Implement clause type inference, normalization, overlap/exhaustiveness analysis and typed native dispatch. +- [x] Implement numeric and collection runtime operations plus shared intrinsic contracts. +- [x] Compile bundled official source and implement Bytes/file/TCP resource foundations. +- [x] Verify focused semantic/native/runtime cases and review the final cross-layer changes. + +## Core Foundation Evidence (2026-09-07) + +- Implemented six scalars, records/choices/nullable/List/Map/MapEntry, typed expressions, named outputs, stable assignments and value semantics through check/HIR/native execution. Added naming and unused-write diagnostics. +- Function groups infer one signature, use symbolic per-axis domains and rectangle subtraction to prove disjoint coverage, and lower into concrete native dispatch. No conditional blocks, source loops or user recursion were introduced. +- Official source in `library/` shares the same frontend; one intrinsic enum bridges the small system boundary. Decimal128 and ordered maps reuse pinned `dec`/`indexmap` runtime wrappers. Added Bytes, create-new/read-only files, TCP and shared-close opaque handles with explicit source-defined result choices. +- Passed 46 focused tests: 13 `core_semantics`, 9 `core_native`, 5 `runtime_foundations`, 12 existing `hello_native`, and 7 relevant `syntax_and_format` source/diagnostic regressions. Runtime tests used owned temporary files and loopback port 0 only. No full suite or integration environment was run. +- Passed targeted rustfmt and strict Clippy/type checks for compiler/runtime/CLI and affected test targets. Existing tool archives were invoked with command-scoped paths; a local ignored verification wrapper supplies the toolchain library path after Cargo sanitizes its environment. No global installation or environment changes. +- Actual CLI `run examples/core/src app.main` produced all 22 expected named outputs. `check` succeeded with nonexistent RUSTC. `build` produced `target/dever-core-example`, confirmed as a native Linux x86-64 ELF, and it produced the same outputs with `env -i PATH=/nonexistent`. +- Independent read-only review found and closed duplicate choice Map keys, Group-sensitive inference, null-only `first`, and Float signed-zero constant propagation. Added semantic/native regressions; Float literal contextual typing does not retype existing Decimal/Int locals. +- The initial wrapper/fixture errors and one stale numeric rejection fixture were corrected; final affected checks passed. Trellis context validation passed with implementation/check JSONL absent and skipped for inline work. Ordinary `git diff --check` passes but does not inspect the untracked compiler files; targeted formatting and explicit new-file whitespace checks cover those files. +- Formatter, reference evaluator/differential checks, performance gates, full HTTP/JSON/database packages, self-hosting and distribution/cross-compilation were not implemented or claimed by this delivery. No commit/archive; the wider language task remains active. + +## Completed Hello Checklist + +- [x] Correct the superseded API-first requirements and record the approved Text/action subset. +- [x] Add package/function resolution, semantic diagnostics and checked HIR. +- [x] Add standard output runtime and native compilation. +- [x] Add local check/build/run CLI and `examples/hello/src/app/user.dever`. +- [x] Run focused semantic/native/runtime tests and the actual hello command; announce local process execution first. +- [x] Review the final changes, update implementation evidence and record the session. + +## Native Hello Evidence (2026-09-06) + +- Implemented the approved Text/action subset, checked HIR, shared stdout runtime, native Rust generation and local `deverc` CLI. Example: `examples/hello/src/app/user.dever`. +- Passed 12 `hello_native` tests and the existing 20 `syntax_and_format` tests. After the final emitter readability change, reran all 12 native tests successfully. +- Passed rustfmt and strict Clippy for core/runtime, both targeted test binaries and the actual CLI binary. +- Actual CLI `check` succeeds even with a nonexistent RUSTC, proving it does not invoke compilation. `run` prints exactly `hello\n`; an unknown entry returns exit 1. `build` produced `target/hello-user`, identified as a native x86-64 ELF, which prints `hello\n` with PATH containing no Rust tools. +- Independent read-only reviewer found no blocking issue; the noted CLI verification gap was closed by the actual commands above. Tests exercise Unicode/escaping, source locations, value semantics and write failures, not a User/hello-specific execution branch. +- Added README run/build instructions and updated owning compiler specs. No global command replacement, external dependency, commit, service, database or load test. +- This bounded milestone is complete. The overall language roadmap remains open; HTTP/JSON/CRUD and persistence are deferred pending the user's next step. + +Preserve unrelated changes and do not create a commit without explicit authorization. Verification for the new API must use its own local process and test storage, with functional checks and bounded performance checks described before execution. Do not test against, restart or benchmark existing services. No runtime or load check has been run during this planning turn. + +## Historical API Replanning Checklist (Deferred) + +- [x] Record the real user API goal and supersede the pure-computation-only boundary. +- [x] Keep `.dever` source and the existing surface syntax; defer the speculative contract-declaration system. +- [ ] Confirm persistence selection. +- [ ] Specify typed HTTP/JSON/storage boundaries and necessary effect/concurrency semantics. +- [ ] Finalize user API behavior and meaningful performance acceptance. +- [ ] Replace the next execution steps with a bounded native application delivery plan. +- [ ] Review the concrete plan with the user before implementation. + +Proposed delivery order, to finalize after the open product decisions: + +1. Implement name/type/clause/flow checking and HIR for the language features the API actually uses. +2. Establish concrete native business-function execution and source-located failures. +3. Add typed HTTP/JSON boundaries, explicit resource/effect semantics and the selected persistent database adapter. +4. Implement `examples/user-api` entirely in `.dever`, with real CRUD and configuration supplied by the runtime environment. +5. Run announced, isolated functional/persistence/concurrency checks and a bounded native performance comparison. + +Complete Decimal/Float/Map libraries, the full reference evaluator, formatter, component distribution and AI indexing remain on the roadmap; they are not all prerequisites for the first user API. Preserve unsupported-source diagnostics rather than silently approximating unimplemented language behavior. + +## Initial Frontend Evidence (2026-09-05) + +- Implemented: dependency-free Rust workspace, source directory loading, UTF-8 and line/column handling, deterministic diagnostics, lexer, spanned AST and parser. +- Passed: 20 focused `syntax_and_format` tests, including the complete PRD package; Rust formatting check; focused Clippy with `-D warnings`; diff whitespace check. +- Independent review: fixed iterative expression-tree depth bypass and removed inaccurate terminal-width caret rendering. Follow-up review confirmed both fixes. +- Validation tools: rustc/Cargo 1.98.0 are installed but absent from the default PATH. Rustfmt and Clippy were missing, so their exact 1.98.0 official archives were downloaded to ignored `target/verification-tools/`, checked against the installed Rust manifest SHA-256 values and invoked directly with command-scoped library/tool paths. No global toolchain installation or command replacement occurred. +- At that checkpoint, native/differential/performance checks had not run. The Native Hello Evidence above records the subsequently implemented and verified native subset. +- The previous API-first next step was superseded by the approved hello-world milestone; formatter and other language work remain on the roadmap. + +## Historical Original Roadmap Notice + +The phase checklist below records the original repository bootstrap plan. Its unchecked boxes are not the current implementation status and do not expand the authoritative extension above. The Active Core Checklist and dated evidence describe completed work. + +## Phase 1: Initialize The Repository + +### 1.1 Preserve and classify current changes + +- [x] Record `git status --short`; current changes contain planning and agent metadata, plus two deleted old documents. +- [x] Confirm no old Rust workspace, implementation or test files exist in the working tree or HEAD. +- [x] Preserve existing user changes; no old implementation needs removal. + +### 1.2 Establish the new workspace + +- [x] Create `crates/dever-core` as the single language library. +- [ ] Add `crates/dever-runtime` in Phase 7 when numeric/fault consumers exist; keep parser, checker, HIR and evaluator code out of it. +- [ ] Add `crates/dever-cli` in Phase 10 for the `dever` binary; do not expose syntax-only success as the public semantic `check` command. +- [x] Create `test/dever-tests` as the only permanent test crate. +- [x] Create root `Cargo.toml`; pin external dependencies when they are first consumed. +- [ ] Add a decimal128 dependency behind `dever-runtime::number` and reuse that wrapper from the evaluator. +- [ ] Add an insertion-ordered map dependency behind `dever-runtime::collections` and reuse that wrapper from the evaluator. +- [x] Do not add parser generators, async runtimes, plugin systems or CLI frameworks. + +### 1.3 Establish current rules + +- [x] Confirm the rejected prototype and its fixtures/tests are absent; do not add migration scaffolding. +- [x] Preserve existing deletion of superseded plan/spec documents. +- [x] State in `AGENTS.md` that `.dever` source is the trusted source. +- [x] Replace relevant Trellis spec placeholders with the actual frontend ownership and verification rules. + +Rollback point: kickoff adds compiler files and updates canonical planning only; existing user deletions and unrelated metadata are preserved. + +## Phase 2: Source, Diagnostics And Lexer + +### 2.1 Source ownership + +- [x] Implement `SourceId`, `SourceFile`, `SourceMap` and half-open `Span`. +- [x] Precompute line starts and render stable one-based line/column positions. +- [x] Load `.dever` paths in lexical order and reject invalid UTF-8. + +### 2.2 Diagnostics + +- [x] Implement the shared diagnostic model; semantic phases will consume it when implemented. +- [x] Support primary and related spans plus stable diagnostic codes. +- [x] Sort diagnostics deterministically. +- [x] Add renderer tests for the syntax-phase messages, related source files and Unicode locations. + +### 2.3 Lexer + +- [x] Tokenize identifiers, keywords, delimiters, operators, newlines and comments. +- [x] Preserve numeric spelling and comment trivia. +- [x] Decode Text escapes with precise bad-escape diagnostics. +- [x] Reject non-ASCII identifiers while allowing UTF-8 strings/comments. + +Focused checks: + +```bash +cargo test --offline -p dever-tests --test syntax_and_format +``` + +## Phase 3: Parser And Formatter + +### 3.1 Syntax tree + +- [x] Define focused AST nodes for package, type, function clauses, patterns, statements and expressions. +- [x] Keep every node spanned; do not attach resolved IDs or runtime values. +- [x] Preserve ordered comments with source spans; formatter attachment remains part of Phase 3.3. + +### 3.2 Recursive-descent parser + +- [x] Parse the package header and exposure list. +- [x] Parse record and choice type declarations. +- [x] Parse function signatures, clause patterns and linear bodies. +- [x] Parse record/List/Map values, calls, field access and operators with fixed precedence. +- [x] Recover only at bounded declaration/statement delimiters. +- [x] Reject excluded constructs rather than accepting compatibility syntax. + +### 3.3 Canonical formatter + +- [ ] Format every valid syntax node from structure. +- [ ] Preserve comments and normalize whitespace, blank lines and indentation. +- [ ] Add parse-format-parse and twice-format idempotence tests. +- [ ] Refuse to rewrite invalid files. + +Focused check: + +```bash +cargo test -p dever-tests --test syntax_and_format +``` + +Milestone boundary: the parser is currently an internal library for focused tests. Do not expose a public checked-source or formatting command until its complete contracts are implemented. + +## Phase 4: Package And Name Resolution + +- [ ] Register package/type/function headers before resolving bodies. +- [ ] Validate path/package equality and one-file ownership. +- [ ] Load only the supplied source root and fixed standard catalog; do not search external paths or perform network access. +- [ ] Reserve core single-segment package names and `dever.*` against local declarations. +- [ ] Validate `exposes` against locally declared names. +- [ ] Resolve local short names and cross-package fully qualified names. +- [ ] Enforce visibility without import tables. +- [ ] Group same-name/same-arity clauses and enforce contiguity. +- [ ] Reject package, recursive type and direct/indirect function cycles. +- [ ] Include standard core and qualified package names through one fixed catalog. +- [ ] Report unavailable future official/third-party references as unknown packages without an implicit install fallback. + +Focused check: + +```bash +cargo test -p dever-tests --test package_resolution +``` + +## Phase 5: Type Checking And Linear Flow + +### 5.1 Type interning and declarations + +- [ ] Implement scalar, record, choice, nullable, List, Map, MapEntry, Stream and multi-output types, with static handler inputs represented separately from values. +- [ ] Reject record/choice member mixing and recursive user types. +- [ ] Validate choice payload arity and record completeness. +- [ ] Validate Map key eligibility. + +### 5.2 Expressions and calls + +- [ ] Check literals using expected type without premature numeric conversion. +- [ ] Implement operator tables exactly as specified in the PRD. +- [ ] Permit only lossless Int-to-Decimal implicit promotion. +- [ ] Resolve positional function calls and named multi-output access. +- [ ] Restrict named handler references to sequence operations, direct static invocation and approved handler-input forwarding. + +### 5.3 Definite assignment and mutation + +- [ ] Track uninitialized/initialized locals in one forward pass. +- [ ] Reject reads and field writes before complete initialization. +- [ ] Require every output on every clause path. +- [ ] Keep a local's inferred type stable across reassignment. +- [ ] Report unused writes without adding defensive runtime behavior. + +Focused check: + +```bash +cargo test -p dever-tests --test type_checking +``` + +## Phase 6: Clause Analysis And Typed HIR + +### 6.1 Normalize patterns + +- [ ] Normalize Bool, null, choice, scalar constant and `other` patterns. +- [ ] Normalize Int/Decimal comparisons into open/closed intervals. +- [ ] Support at most one lower and one upper bound joined by `and`. +- [ ] Reject Float ranges and dynamic/function-based guards. + +### 6.2 Prove clause validity + +- [ ] Compute overlap, reachability and exhaustiveness over symbolic domains. +- [ ] Support products of multiple input domains without enumerating numeric values. +- [ ] Emit diagnostics that label both conflicting clauses or the uncovered domain. +- [ ] Prove that permuting source clauses leaves the normalized decision model unchanged. + +### 6.3 Lower typed HIR + +- [ ] Lower only fully checked packages. +- [ ] Replace names with resolved IDs and attach static types/spans. +- [ ] Build decision nodes from normalized clause domains. +- [ ] Keep HIR internal to `dever-core`; do not serialize it as a second source format. + +Focused check: + +```bash +cargo test -p dever-tests --test clause_analysis +``` + +Rollback point: do not add an ordered first-match fallback if coverage analysis is incomplete. Unsupported patterns must fail compilation. + +## Phase 7: Values, Numbers And Reference Evaluator + +### 7.1 Runtime values + +- [ ] Implement the language-owned `Value` variants. +- [ ] Use shared immutable aggregate storage with copy-on-write updates. +- [ ] Wrap insertion-ordered Map behavior and use order-preserving removal. +- [ ] Implement deterministic value rendering and named output order. + +### 7.2 Numeric layer + +- [ ] Implement checked i64 arithmetic, including negative division/remainder edges. +- [ ] Wrap Decimal128 Context with round-half-even and per-operation status clearing/checks; accept inexact/rounded and fault on overflow, underflow, division by zero or invalid operation. +- [ ] Reject Decimal special values at the `DecimalValue` boundary. +- [ ] Implement exact Int-to-Decimal and explicit Int-to-Float conversion. +- [ ] Reject inexact Decimal literals while allowing specified inexact/rounded arithmetic results. +- [ ] Implement f64 arithmetic and IEEE classification functions without fast math. +- [ ] Add Decimal conformance and numeric fault fixtures. + +### 7.3 Evaluator + +- [ ] Execute HIR with explicit frames and no mutable global state. +- [ ] Select clauses through compiled decision nodes. +- [ ] Preserve source-level value semantics during local and field mutation. +- [ ] Return named outputs or an uncatchable `RuntimeFault` with source span. +- [ ] Add a Dever call trace only as diagnostic context. +- [ ] Keep evaluator entry points internal to compiler tests; production `build` and `run` must not depend on them. + +Focused checks: + +```bash +cargo test -p dever-tests --test value_semantics +cargo test -p dever-tests --test numbers +cargo test -p dever-tests --test failures runtime +``` + +## Phase 8: Standard Functions + +- [ ] Define one static signature/evaluator catalog for core and qualified standard names. +- [ ] Implement `append`, `first`, `get`, `put`, `remove` and `entries`. +- [ ] Implement `each`, `reduce`, `reduce_until`, `filter`, `find` and `sum` with shared sequence/handler helpers. +- [ ] Enforce one optional context and element-first handler inputs. +- [ ] Preserve order and short-circuit `find`. +- [ ] Keep Float sum left-to-right. +- [ ] Implement minimum `text`, `decimal`, `float` and `math` functions used by the PRD. +- [ ] Do not add unrequested collection aliases, lazy Stream transforms or reduction names besides `reduce` and `reduce_until`. + +Focused check: + +```bash +cargo test -p dever-tests --test collections +cargo test -p dever-tests --test numbers standard_functions +``` + +## Phase 9: Native AOT Compilation + +### 9.1 Whole-program specialization + +- [ ] Start from one checked, exposed, zero-input entry function and collect its reachable HIR. +- [ ] Specialize every concrete record, choice, nullable, List, Map, Stream and static handler-binding combination. +- [ ] Lower clauses to deterministic native decision trees and collection functions to direct loops. +- [ ] Perform last-use/liveness analysis so dead sources move instead of clone while observable value semantics remain unchanged. +- [ ] Reject any native path that would require a generic runtime `Value`, reflective lookup or virtual per-element dispatch. + +### 9.2 Private Rust emission + +- [ ] Emit concrete Rust types and direct functions from typed HIR through one structured writer with centralized identifier/literal escaping. +- [ ] Emit explicit checked numeric and source-located runtime-fault operations. +- [ ] Make generated programs depend only on `dever-runtime`, never `dever-core` or the evaluator. +- [ ] Keep generated Rust and Cargo metadata in a compiler-owned transient directory with no public emit command. +- [ ] Compile with an optimized release profile starting at opt-level 3, thin LTO and one codegen unit. +- [ ] Translate toolchain failures into stable compiler diagnostics without leaking generated identifiers as Dever names. + +### 9.3 Native conformance and performance + +- [ ] Differentially compare evaluator and native outputs/fault categories across the language conformance corpus. +- [ ] Inspect native artifacts to prove the evaluator is not linked. +- [ ] Benchmark scalar arithmetic and List `each`/`filter`/`sum` against equivalent safe Rust using the same algorithm, data layout and toolchain. +- [ ] Benchmark Decimal and ordered Map separately against Rust baselines using the same underlying semantic wrappers. +- [ ] Fail the performance gate when representative median native hot-path time exceeds 1.15 times the matching Rust baseline. + +Focused checks: + +```bash +cargo test -p dever-tests --test native +cargo test -p dever-tests --test differential +cargo test -p dever-tests --test performance -- --ignored +``` + +Rollback point: the evaluator remains the semantic oracle, but an incomplete native backend cannot silently become the production `run` path. Unsupported lowering is a compile diagnostic. + +## Phase 10: CLI And End-To-End Example + +### 10.1 CLI + +- [ ] Implement `check`, `fmt`, `build` and `run` as thin adapters. +- [ ] Do not add registry, `add`, `update`, signature or component-cache commands in Core 0.1. +- [ ] Validate exact argument counts and stable exit codes. +- [ ] Keep diagnostics on stderr and results on stdout. +- [ ] Make `fmt` all-or-nothing when any source file is invalid. +- [ ] Restrict `build` and `run` to exposed zero-input functions. +- [ ] Make `build` write an optimized native executable to `--output` and make `run` compile/execute the same native path. +- [ ] Keep the Cargo binary and canonical command name as `dever`; do not rename compiler internals or public command semantics to `deverc`. +- [ ] Treat `deverc`, when explicitly installed for coexistence, as a thin link or launcher to the same compiler binary with no divergent behavior. +- [ ] Do not replace the existing framework `dever`, modify system `PATH` or create global executables during repository implementation or automated tests. +- [ ] If global `deverc` setup is later authorized, resolve and record both command targets and verify that the existing global `dever` remains unchanged. + +### 10.2 Example and integration + +- [ ] Add the canonical account/user package from the PRD. +- [ ] Add a zero-input executable example covering types, clauses, collections and numbers. +- [ ] Verify multiple source files and full qualification. +- [ ] Verify the zero-input example produces identical evaluator and native results. +- [ ] Ensure no example source depends on Rust, JSON graph, React or a hidden host fallback. + +Focused check: + +```bash +cargo test -p dever-tests --test cli +``` + +## Phase 11: Cleanup And Final Verification + +### 11.1 Structural review + +- [ ] Search for duplicate operator, standard-function, diagnostic and type rules. +- [ ] Keep each language rule owned by one table/module. +- [ ] Remove obsolete compatibility code, commented code, placeholders and old architecture references. +- [ ] Confirm the CLI contains no language semantics. +- [ ] Confirm no public Rust API leaks into `.dever` diagnostics or docs. +- [ ] Confirm generated binaries depend on the minimal runtime but not compiler/evaluator modules. +- [ ] Confirm private generated Rust is not documented or tested as a public compatibility format. + +### 11.2 Acceptance mapping + +- [ ] Map all 20 PRD acceptance criteria to named test cases. +- [ ] Run each relevant focused test target. +- [ ] Run static formatting and diff checks. +- [ ] Record passed, failed, skipped and not-run checks separately. + +Permitted final checks, announced before execution: + +```bash +cargo fmt --check +cargo check -p dever-core -p dever-runtime -p dever-cli +cargo test -p dever-tests --test syntax_and_format +cargo test -p dever-tests --test package_resolution +cargo test -p dever-tests --test type_checking +cargo test -p dever-tests --test clause_analysis +cargo test -p dever-tests --test value_semantics +cargo test -p dever-tests --test collections +cargo test -p dever-tests --test numbers +cargo test -p dever-tests --test failures +cargo test -p dever-tests --test native +cargo test -p dever-tests --test differential +cargo test -p dever-tests --test cli +python3 ./.trellis/scripts/task.py validate 09-04-dever-language-mvp +git diff --check +``` + +The performance test is an explicit, separately announced gate and is not part of the default test list. Do not substitute `cargo test --workspace`, `npm run build` or service/integration tests. If a focused check reveals a shared compiler defect, fix the owning implementation once and rerun only affected targets. + +## Finalization + +- [ ] Run `trellis-check` after implementation. +- [ ] Update canonical Trellis specs for the new crate ownership and language rules. +- [ ] Review the final diff against PRD scope and remove accidental abstractions. +- [ ] Do not commit until the user explicitly authorizes a commit. +- [x] Confirm there are no obsolete planning children to archive. +# Active Compiler Contract Checklist (2026-09-08) + +- [x] Contract syntax, formatter and semantic metadata. +- [x] Field privacy, numeric facts, construction/write/output constraints. +- [x] Failure obligations and explicit recovery, including nested values and static handlers. +- [x] Transitive effects, package dependencies and API snapshot/CLI enforcement. +- [x] Evidence-backed redundancy diagnostics and application export analysis. +- [x] Official source/examples migration, targeted tests, native equivalence, docs/specs and final review. + +User approved the full contract implementation. Complete the list before claiming delivery. Keep earlier evidence below as historical; no full test suite, external services, distribution work or commit. + +## Compiler Contract Evidence (2026-09-08) + +- Implemented private record fields, normalized Int/Decimal field/payload/output/handler bounds, explicit error/recover and pure/depends. HIR facts reuse existing clause domains; handler boundaries support safe variance and concrete output inference without forcing wrapper annotations. +- Failure obligations follow copies, nested fields, calls and collections. Official Failed variants are explicitly marked; HTTP transport preserves earlier failure state and returns the inspected result with its stop decision. The Stream example now propagates a CountResult failure instead of silently returning zero. Presentation/test recovery reasons are explicit and consistent across clauses. +- Effects and recovery policies propagate through invoked static handlers. API generation is create-new; check/run/build compare an existing baseline. Adding a private recovering helper changes the effective public snapshot without exposing that helper's name. Depends includes normalized system resource types. W001 is reused; W002–W005 and entry-scoped W007 are advisory, not arbitrary abstraction requirements. +- Final six contract targets passed 47 tests: syntax 4, facts 16, semantics 10, API 9, redundancy 6, execution 2. Existing core semantics passed 16 (15 in the first run, W001 assertion narrowed to its diagnostic then rerun); existing syntax passed 22, formatter passed 11 and the new contracts example roundtrip passed. The independent expected-output core differential case passed. Forty-one migrated source fixtures were checked without executing their network/resource scenarios; the temporary probe was removed. +- Actual CLI api/check/run/build and standalone execution with `env -i PATH=/nonexistent` passed. A temporary source's new exposure was rejected by its unchanged baseline. `examples/contracts/src/dever.api` records the example's contract. Strict installed-driver Clippy/type checks passed for core/CLI and focused contract tests; formatting and explicit whitespace checks cover untracked files as well. +- Independent review found and closed empty-reduce seed consumption, primitive resource dependency omission, and exponential/deep proof graphs. Added positive/negative regressions; immutable shared fact/value graphs preserve aliases, proof work is capped at 16,384 and iterative nominal depth validation caps 128 before recursive analysis. Exhaustion reports C015; no incomplete proof becomes a successful Program. +- Updated LANGUAGE.md, IMPLEMENTATION.md, README and the owning compiler-contract spec. No database/distribution expansion, full suite, integration environment, existing service, global installation or Git commit. The umbrella task stays active for its previously deferred roadmap; this approved contract extension is complete. + +## Direct Example Roots And Coverage (2026-09-08) + +- Moved all five original source roots from `examples/<name>/src` to `examples/<name>`, including the unchanged contracts API baseline. Removed empty src directories; package-relative nesting, such as hello/app/user.dever, remains. Updated executable test references, current guides and owning specs; earlier dated evidence retains its historical paths. +- Added sequences, text, privacy, parallel and system examples, bringing runnable roots to ten. Added six separately checkable rejected roots for C006/C009/C011/C012/C013/C014. The examples index documents commands, outputs, nondeterministic ordering, runtime input paths and explicit recovery boundaries. +- Reused compiler/check/native APIs in one example runner. Automatic discovery checks every runnable root and any baseline; formatter coverage discovers all examples, including rejection fixtures. No compiler or runtime behavior changed. +- Passed seven example-target tests covering all ten successful roots, nine native executions and six rejection cases; the existing core native test separately verified its 22 expected outputs. The targeted formatter roundtrip passed over every official/example source. Actual CLI contract check and expected C012 rejection passed; example fmt --check passed. Strict Clippy/type checks compiled all affected test targets, including feature-gated differential references. +- No full suite, network/service integration, compiler feature expansion, global installation, commit or archive. + +## Isolated PostgreSQL And Dual-source CMS Measurements (2026-09-18) + +- In a nonprivileged, owned PostgreSQL 16 instance with a private Unix socket and no TCP listener, the ignored `postgres_orm_uses_the_configured_isolated_database` test passed 1/1. Its shared schema cleanup left no test schemas. Added a focused live named-migration probe for bound before/after SQL, atomic rollback on after failure, successful DDL and once-only re-execution. Test connection came only from a temporary root `config/setting.json`; no database URL environment variable or existing PostgreSQL service was used. After verification, the temporary setting was deleted, the owned instance stopped and its test data removed. +- Focused `cms-profiles` build compiles both current `examples/cms/{dever,md}` source formats using one shared benchmark entry and separately builds base/SQLite/PostgreSQL/both runtime profiles. Artifacts: `target/performance/cms-dual-build-09-18-v2/manifest.json` and `profile-report.json`. Profile bytes: 411,352 / 2,558,200 / 2,708,112 / 4,672,888. CMS Dever/Markdown bytes: 3,151,352 / 3,151,416. +- `target/performance/cms-dual-rss-09-18/report.json` completed all six runs (three per source, READY=3 throughout). Dever/Markdown median readiness 19.87/19.69 ms, idle RSS 5.37/5.37 MiB, PSS 3.28/3.33 MiB, VmHWM 5.37/5.37 MiB. This is warm-cache initialization and bounded idle sampling, not throughput. No hard 64/128 MiB CMS budget or cgroup peak: the local controller is not writable. No broad/full suite, existing service check, or global installation. +- Benchmark runner unit tests pass 40/40; focused test compiled with `--features postgres`; Trellis task context validation passed. Rustfmt and Clippy are unavailable in the installed toolchain. This root roadmap remains in progress; API stays deferred pending its redesign, and hard-budget CMS stress requires a delegated cgroup environment. + +# Source Visibility And Module API Implementation (2026-09-18) + +- Replaced `package/exposes` syntax with path-derived package identity and explicit `public` for cross-package type/function visibility; fixed private `main.main` as the only run/build entry. Updated bundled sources, CMS Dever/Markdown roots, old runnable examples and Markdown metadata. The hello example now delegates through a public library function to its own `main.dever`. +- Added statically checked module `api/` routes with GET/POST/DELETE naming, recursive paths, private handlers and one `response` output. `dever.api.serve()` opts the application into routing and the existing bounded HTTP engine; typed inputs, JSON serialization and `{code,message,data}` envelopes reuse a single runtime adapter. Listener settings are read from `config/setting.json`, not environment variables; no CMS business API or authentication policy was added. +- Focused checks: core/CLI/runtime/tests cargo check passed; `source_visibility` 10/10, `syntax_and_format` 22/22, `core_semantics` 16/16, `formatter` 11/11 and `markdown_source` 20/20 passed. `api_routes` passed 7/7 before the final test adjustment, then its application-mode native compilation passed 1/1 with config bootstrap and no listener. An isolated temporary-project CLI `check/run/build` and API-baseline rejection test passed. Maintained examples, CMS Dever/Markdown source parity and rejection fixtures passed selected contract tests. Two ORM fixture static-check tests passed without database connections. `deverc check` passed for hello and both CMS formats (CMS retains W001 unused-write warnings). +- Not run: full suite, live HTTP/API service check, real PostgreSQL, CMS runtime/build or benchmarks. Rustfmt and Clippy binaries are unavailable in the installed toolchain. The umbrella task remains in progress for separately deferred roadmap work; historical entries above retain their original date/context. + +# AI-Oriented Source Architecture Implementation (2026-09-19, Complete) + +- [x] Add one shared source-role classifier and project-layout validation. +- [x] Replace application `public` visibility with automatic App capabilities and logical App aliases. +- [x] Enforce role dependency direction, API non-callability and Model-operation ownership. +- [x] Migrate the Dever and Markdown CMS sources from `service/` to component/domain roles without function-per-file fragmentation. +- [x] Update the language, Markdown and backend compiler/database contracts. +- [x] Add focused layout, visibility, API, Model and CMS parity tests; run only the affected checks. + +The implementation does not add environment variables, implicit HTTP startup, authentication policy, a new Port declaration syntax, or backward compatibility for the superseded application layout. + +## AI-Oriented Source Architecture Evidence (2026-09-19) + +- `SourceLayout` is the single main/component/domain/role/topic classifier. Strict project loading rejects loose application paths, generic buckets, role file/directory conflicts, nested non-API topics and one-file role directories. The check cache includes strict-layout identity; in-memory compiler fragments remain isolated from project-layout enforcement and cannot weaken CLI projects. +- App declarations receive stable `<component>.<domain>.<name>` identities without exposing topic filenames. Role-aware lookup enforces Main/API/App/private-role direction, API non-callability and owning-domain Model operations. App signatures and inferred failures reject private type leakage. Port remains a reserved, rejected declaration role. +- Plain and Markdown CMS now contain only `main`, `user/account/{app,domain,model}` and `news/article/{app,model}`. Both forms have equal normalized declarations, Model/API snapshots and settings. Current examples contain no package/exposes/public syntax, old service/model roots or environment-variable configuration. +- The current performance packager now stages `benchmark/<domain>/app.dever` plus generated `main.dever`; it no longer generates package/exposes headers or passes arbitrary entries to `deverc build`. ORM, profile, Dever CMS and Markdown CMS staged projects all passed actual CLI check. The CLI CMS regression test compiles against the fixed main-only interface and uses a test-owned App probe instead of injecting a loose root source. +- Focused verification passed: core/runtime/CLI Cargo check; `source_architecture` 8/8; `source_visibility` 10/10; `orm_performance_fixture` 3/3; `concurrency` 5/5 with its explicit benchmark ignored; CMS source/config parity 1/1; archived parallel and source-root native regressions passed after the synchronous typed-error fix. Earlier in this same phase `api_routes` passed 7/7. Actual CLI `check` and `fmt --check` passed for both CMS formats, and Python syntax checking passed for the performance builder. +- Not run: full workspace suite, live CMS `run/build`, `cms_project` integration execution, HTTP listener, real PostgreSQL, performance/size/RSS builds or 64/128 MiB stress. Rustfmt and Clippy components remain unavailable. The umbrella language roadmap stays in progress for separately deferred work. diff --git a/.trellis/tasks/09-04-dever-language-mvp/prd.md b/.trellis/tasks/09-04-dever-language-mvp/prd.md new file mode 100644 index 0000000..b8022df --- /dev/null +++ b/.trellis/tasks/09-04-dever-language-mvp/prd.md @@ -0,0 +1,990 @@ +# Dever Language Definition + +## N5：HTTP/2（已完成） + +用户在 P2 性能验证结束后批准继续接入 HTTP/2。使用 Hyper 实现 buffered/live 服务端和 single-use/pooled 客户端；复用既有请求/响应、TLS、AsyncStream 与 SSE 接口。Limits.http2 显式配置协议和每连接流数、接收窗口，TLS 必须协商 h2，明文使用 prior knowledge。每流取消隔离，Hyper 内部任务保留 Dever Scope，GOAWAY 排空不自动重试已发请求。详细验收以 `../09-12-http2/prd.md` 为准。 + +N5 已由 88 项定向测试、原生执行和独立审查验收。后续 HTTP/2 性能矩阵与有界稳态/恢复压测已由子任务完成；小时级稳定性、编译器发行/跨平台、自举及其余路线图仍未完成。 + +仅定向协议、生命周期与原生桥接验证；不重跑 P1/P2 性能矩阵，不改根任务调度，不接 RFC 8441、h2c Upgrade、推送、trailer、数据库或发行工具。下文各轮排除范围仅约束相应历史阶段。 + +## 已完成阶段 N4：并发组合与可复用网络客户端（2026-09-12） + +用户批准完成上一轮清单的 1 和 2。交付源码运行参数、静态 HTTP handler 的类型化上下文、任务超时/多路竞争等待、周期与 Channel 流、异步内存序列遍历；有界 HTTP 连接池、域名、流式请求/响应、HTTPS/WSS 客户端和 TLS 服务端。保留 N3 SSE/WS 会话、取消和背压契约。HTTP/2、数据库、协议扩展和分发工具不属于本轮。 + +任务、连接及正文队列有界；取消和关闭回收所属工作，已开始的阻塞工作必须排空。复用 Tokio、Hyper、tungstenite 和 Rustls，不维护兼容接口或自写协议栈。只运行定向编译/测试及自有 loopback 随机端口，不测旧性能基线，不启动现有服务，不提交。 + +以上范围已接通 checked source、原生编译和运行时。83 项定向用例、实际 CLI check/fmt/build 与无 Rust 工具链环境中的独立后端示例通过；具体证据见 implement.md 的 N4 Evidence，公开用法以 LANGUAGE.md 为准。 + +## 已完成阶段 N3：流式响应、SSE 与 WebSocket（2026-09-12） + +用户已授权“做下一阶段”，沿用新语言直接演进和低内存运行目标。保留普通 HTTP 的直接 handler 路径，增加 `serve_live` 和 HttpReply：同端口处理完整响应、逐块正文、SSE 和 WebSocket。HTTP 请求仍有界缓冲；响应通道只有 1 个槽位，长连接配置与普通请求期限分开。 + +WebSocket 服务端/客户端复用 tokio-tungstenite 0.30.0（handshake feature），提供 Text/Binary/Ping/Pong、AsyncStream 消息消费和关闭握手。SSE 提供字段编码、Last-Event-ID 访问和空闲注释心跳，历史重放归应用所有。HttpReply/WebSocket 进入编译器资源、类型、effects、API、Markdown 和 native 契约；不引入新语法、旧 API 兼容或自写 WebSocket 协议栈。 + +连接、handler 及升级后会话共用 Scope/Group 上限与取消排空；等待写锁时取消不能破坏另一任务的帧。仅执行根 test/ 下的定向编译器/运行时验证和自有回环端口，不测试旧性能基线、常驻示例或已有服务。TLS/WSS、HTTP/2、客户端池、子协议/压缩扩展、关闭帧元数据和数据库不属于 N3。 + +## 前一阶段 N2：HTTP 引擎（2026-09-12，已完成） + +用户在 N1 完成后批准继续。采用 Hyper 1.x HTTP/1 引擎、hyper-util TokioIo/Timer 和共享 bytes 缓冲区；不额外引入路由框架。此阶段取代历史“HTTP 全部由 Dever 自写”的实现决定。官方类型与静态业务 handler 仍由 Dever 定义，不保留旧协议 API 或备用解析器。 + +- `serve(async handler, Listener, Limits) -> ()`：持久连接、流水线、chunked 输入，复用现有有界任务组、背压、超时和取消清理。accept/配置/程序故障沿任务传播，对端协议错误只结束对应连接。 +- `send(address, port, Request, Limits) -> ResponseResult`：单次请求，无客户端池和重试，driver 与请求同生命周期。 +- Header 列表保留重复头和字节值；消息体在显式上限内缓冲，单帧共享，多帧增量合并。 +- 直接迁移示例、检查器、native bridge 和测试。只用离线定向检查、自有 loopback 随机端口，不测旧基线、不跑全量或现有服务。 +- TLS、HTTP/2、流式响应、客户端池、非空 trailer、WebSocket/SSE 留待后续；N3 优先处理长连接协议。 + +## 前一阶段 N1：异步 Stream 与 TCP（2026-09-12，已完成） + +用户批准继续，并明确这是全新的语言：无需旧源码兼容，不增加兼容别名、回退实现或推测性的防御分支。仓库调用方直接迁移。此决定取代下文历史阶段中的“网络暂停”和“兼容基线”要求。 + +- 增加 `AsyncStream<Item>`，支持在 async function 中按需 `each`、`reduce`、`reduce_until`，以及有界 `parallel_each`；handler 可为 async，关闭/提前结束释放生产端。 +- `dever.net` 直接采用 Tokio 异步 TCP；connect、listen、accept、read、write 为 async。资源名称仍为 Socket、Listener；无 sync/aio 两套 API。 +- 连接流和字节流不预取；并发消费先等待容量再拉取。复用现有结构化 Task/Group，停止后等待子任务清理，连接支持同时读写。 +- 保留协议必须的输入约束、失败输出、超时、关闭和取消语义；写入中断不得自动重发已写字节。 +- 现有 HTTP 传输调用链随 TCP 改为 async；成熟 HTTP 引擎接入属于 N2，WebSocket/SSE 属于 N3，本轮不实施。 +- 只做根 test/ 下定向检查及自己创建的 loopback 随机端口验证；不测旧性能基线，不运行全量测试、业务服务、全量 build 或全局安装。 + +## 前一阶段：并发基础（2026-09-12，已完成） + +前一阶段先完成同步、异步和多线程（含协程/任务）基础，网络当时暂停；当前 N1 恢复异步 Stream/TCP。下述并发契约已经进入编译器前端、HIR、原生后端和 runtime；最终状态以本任务的定向验证和 `LANGUAGE.md` 公开契约为准。不测旧实现性能基线。 + +### 暂存的网络方案 + +- 后续需要 TCP、HTTP、WebSocket、SSE 长连接，复用成熟 Rust 库。 +- 候选栈：Tokio 统一网络调度，Hyper/hyper-util 处理 HTTP，Axum 复用服务路由、升级和 SSE，WebSocket 复用其 tungstenite 生态，bytes 管理共享缓冲区。实施前核对版本、工具链和 feature 兼容性。 +- 共用连接限额、有界队列、背压、超时、取消与关闭。长连接保活独立于普通请求超时；SSE 事件历史保存/重放属于应用责任。 +- 不在每次 I/O 上 block_on，也不把同步阻塞 handler 直接放入异步调度线程。HTTPS/WSS、HTTP/2 的首轮范围尚未确定。 +- 取消旧性能基线工作，不取消新能力必要的定向正确性验证；不启动业务服务或压测。 + +### 并发语言契约（已批准实施） + +#### Goal + +增加可明确推理的同步、异步协程和多线程能力，为 TCP、HTTP、WebSocket 与 SSE 提供统一运行基础。小机器只运行打包后的二进制；空闲任务不占用一个系统线程，任务、线程和队列都有明确上限。契约可以直接演进，维护值/资源语义的清晰性，不承担旧源码兼容义务。 + +#### In Scope + +- 普通 function 保持同步；`async` 声明异步 function，`await` 等待异步调用或已启动任务。 +- `run` 启动异步调用并返回受限 Task；`stop` 采用“请求协作取消并等待清理完成”的安全语义，而不是仅发送信号。 +- 每个 async function 是默认结构化任务域；动态数量的零输出任务可由有界 `group` 统一拥有、等待和停止。 +- `parallel` 将 pure 同步计算交给有界多线程执行；`blocking` 供明确的阻塞边界使用。`parallel_each` 按序列类型复用相应的有界执行器。 +- 有界 `channel` 以及 `send`、`receive`、`close` 用于任务通信；禁止无界队列。 +- async 静态 handler、跨线程可传递性、任务消费、effects/failures、API 快照、Markdown 契约、格式化和原生代码生成均由编译器检查。 +- 运行时复用 Tokio 的协程与多线程调度；整个程序最多建立一个异步运行时，CPU/阻塞任务使用受限执行通道。 + +#### Out of Scope + +- 本阶段不实现 TCP、HTTP、WebSocket、SSE、TLS 或数据库；网络方案按上节暂存。 +- 不加入裸线程、共享可变全局对象、用户锁、分离后无人管理的后台任务、无界 channel、强杀线程或副作用回滚。 +- 不同时加入 lambda、闭包、一等 function、用户泛型、用户递归、循环块或异常机制。 +- 同步 `Stream<Item>` 保持当前阻塞拉取契约；异步流在网络恢复前单独设计，不做隐式转换。 + +#### Acceptance Criteria + +1. 完全同步的入口不启动 Tokio runtime；接口发生变化时直接迁移仓库源码。 +2. async function 和 async 静态 handler 的调用都必须直接位于 `await(...)` 或 `run(...)` 上下文；同步 function 中的异步调用在原生构建前拒绝。 +3. `await(call(...))` 顺序执行并保持原函数的命名输出；`task = run(call(...))` 先启动，`await(task)` 恰好消费一次任务并取得相同输出。 +4. Task 不能复制、存入 record/List/Map/Stream、作为普通参数/输出逃逸、重复等待或在 function 返回时仍未消费；必须 `await` 或 `stop`。 +5. `stop(...)` 第一阶段只接受零输出 Task 或 Group，协作取消并等待清理；已发生的副作用不回滚，长时间同步计算不承诺立即停止。 +6. `group(limit)` 的 limit 必须为正且有实现上限；只拥有零输出异步任务。容量已满时 `run(group, call(...))` 施加背压，不无限排队；`await(group)` 或 `stop(group)` 消费该 Group。 +7. `parallel(call(...))` 只接受 pure 同步 function，参数和输出必须可跨线程传递；`blocking(call(...))` 只接受编译器传递 effect 分析确认包含阻塞系统边界的同步调用。二者第一阶段只能用于 async function，并使用明确的有界容量；同步代码继续使用现有 `parallel_each`。 +8. `channel(Type, capacity)` 容量必须为正且有实现上限；send 在满时等待，receive 在空时等待并返回 `Item?`（值表示 Item，`null` 表示 Closed），close 唤醒等待方；调度器或 Channel 内部故障作为带源码位置的 fatal error 传播。 +9. async handler 的同步/异步属性是签名的一部分；所有同签名分句必须一致，专门化和递归检查不能因异步边界失效。 +10. 任务启动、等待、停止、channel、时间等待和 blocking 均进入传递 effect 分析；pure function 不能隐藏这些效果，失败义务不能因任务边界被吞掉。 +11. 异步入口只创建一次运行时；多个挂起任务不按任务创建线程。工作线程、阻塞线程、group、channel 与待处理任务均有保守默认上限及显式配置边界。 +12. 仅运行根 `test/` 下针对新增语法、检查、原生执行、取消/关闭和有界调度的离线定向测试；不跑全量测试、服务测试、旧实现基线或网络压测。 + +#### Key Naming Decisions + +- 保留 `async`、`await`、`parallel`、`blocking`、`channel`、`send`、`receive`、`close`。 +- 使用 `run` 代替 spawn，使用 `stop` 代替 cancel，使用 `group` 表示动态任务所有者；不再增加 join，统一由 `await` 等待。任务操作沿用函数式调用外观，不提供第二套前缀语法。 +- 协程是 async Task 的运行时实现，不增加 coroutine 关键字。 + +## Markdown Source Follow-through (2026-09-09) + +用户批准保留 `.dever`，新增 `.dever.md` 源码容器，并进一步要求编译器强制说明与代码一致。顶层 `dever` 或 `typescript dever` 围栏内写正常 Dever 声明，共用原有语法、契约、API 和原生路径。一个 H1 说明 package；每个 H2 只说明一个 type 或一个 `(函数名, 参数数量)` 及其全部分句。编译器核对 package 名、公开类型/方法/使用方式、类型字段/分支、函数输入输出的名称、类型、数量和顺序，空项显式写 `无`。`typescript dever` 的首个标记供编辑器尝试 TypeScript 语法高亮。跨格式同包重复拒绝；格式化保留说明，诊断定位原文并关联代码。此决定替代下方历史阶段“只加载 `.dever` 文件”及“标题说明无约束”的范围限制，不开放其它语言或 HIR/JSON 作为源码。正式写法见 `LANGUAGE.md` 的 Markdown 小节。 + +## Approved Performance Follow-through (2026-09-08) + +用户批准实施本轮性能方案。保持现有语法、值/资源语义、错误位置、数值顺序和编译器契约。交付类型属性缓存、依赖驱动契约分析、记录更新复制消除、独占列表消费、直接消费字符列表的遍历优化、HTTP 增量扫描、标准库及原生产物缓存。循环融合必须证明副作用和失败顺序等价；并发分批以有限测量为依据，不要求线程池。补编译/运行/内存定向基准并记录优化前后证据。只运行离线定向测试、自有临时文件/进程和内存协议输入，不运行全量或服务压测，不提交或安装全局命令。 + +## Current Follow-through (2026-09-07) + +用户在剩余项盘点后要求“实施剩下的”。继续完成已确定语言的工具、标准库和验证工作，并推进后续运行能力；既有语法、分句、静态 handler、值/资源语义保持兼容。下方历史里程碑的暂缓项不阻止本次明确要求的工作。 + +- 提供 `.dever` 的规范格式化及 CLI `fmt`/只读检查，保留注释、表达式含义和求值顺序；任一输入语法错误时不改写文件。 +- 补常用 Text、数值解析/格式化、进程参数/环境和时钟接口。公开接口与组合逻辑仍由普通 `.dever` package 定义。 +- 实现仅用于验证的 HIR 基准解释器，覆盖已实现核心语义并与原生结果、故障位置对照；生产 build/run 始终使用原生编译。 +- 实现保持值语义的最后使用优化，提供可复现、有限工作量的同语义性能基线，不对现有服务压测。 +- 继续推进 JSON、HTTP 与受控并发等官方包所需能力；协议与应用逻辑使用 Dever 基础语法，系统边界保持最小。 +- 数据库引擎此前未定，已向用户异步询问；这一选择不阻塞其余工作。 +- 编译器发行、跨平台沿用此前暂缓安排；自举仍是尚未交付的长期项。本次不安装或替换全局命令，不提交，不访问既有服务。 + +验收按每项真实完成的行为和定向证据记录，不将已完成的基础语法测试当作新增工具和库的验收。 + +## Goal + +当前目标是实现已经确定的基础语言，使官方 package 与应用能够用同一套 `.dever` 语法编写。2026-09-07 用户批准实现基础类型、表达式、函数输出、分句、赋值和值语义、集合与处理函数、统一 package 编译及最小系统接口。原生 Hello World 里程碑已于 2026-09-06 完成，作为回归基线。 + +长期目标仍是改善 AI 开发中的硬编码、重复实现、无依据兜底和大项目上下文问题。表面语法保留 package、type、function、分句和值语义；用户所说的“用户类”用已有 `type User` 表达，不引入 class 或继承。Rust 等引导技术不进入用户源码或语言兼容契约。 + +基础阶段曾将 HTTP、JSON 和性能验证后移；当前 Follow-through 已推进这些能力。编译器发行与跨平台仍暂缓，数据库引擎未定；用户 CRUD 示例不是基础语言的额外交付。保留原生执行方向和既有表面语法，不增加契约 DSL。 + +## Approved Core Implementation + +- 沿用既定条件分句、线性 body 和集合函数；不再把循环语法作为待确认项，不增加条件块、循环块或用户递归。 +- 每项语言能力贯通源码检查、typed HIR 与原生执行；不能以 AST 已解析代替支持。 +- 官方 `.dever` package 与本地 package 共用语言规则。官方命名空间由编译器随附源码提供，本地源码不能冒充官方来源。 +- 系统接口提供 Bytes、opaque 文件/TCP 资源、读写和显式关闭；普通值保持值语义，资源别名标识同一外部资源，关闭后操作得到显式失败。 +- 可恢复系统错误以官方源码声明的选择型结果返回；程序故障保持源码定位。系统原语不包含 HTTP 路由或应用业务。 +- 验收覆盖完整语言样例、负例诊断、分句顺序变化、集合顺序与值语义，以及隔离临时文件和本地回环连接。使用必要的最小定向检查，不运行全量测试、既有服务或压测。 + +## Approved Foundation Extension + +Status: implemented and verified on 2026-09-07. + +### Goal And Scope + +补齐官方 `.dever` package 编写协议解析、持续 I/O 组合和宿主回调所需的最小表达能力，同时保留分句条件、线性 body、无循环 block 和无用户递归的长期规则。本阶段一次完成 `reduce`、`reduce_until`、Bytes 遍历、`Stream<Item>`、受限静态 handler 参数，以及文件/TCP 的流式包装；不在 runtime 中加入 HTTP 或业务特例。 + +### Sequence Operations + +- 带状态遍历对外名称固定为 `reduce`；此前讨论中的 `fold` 和 `accumulate` 名称作废。 +- `reduce(handler, sequence, initial_state)` 固定从左到右执行，始终显式提供初始状态；空序列直接返回初始状态。 +- handler 依次接收当前元素和上一步状态，必须输出一个与状态类型相同的下一步状态： + +```dever +add(value: Int, total: Int) (next_total: Int) { + next_total = total + value +} + +total(values: List<Int>) (result: Int) { + result = reduce(add, values, 0) +} +``` + +- `reduce_until(handler, sequence, initial_state)` 使用相同顺序，但 handler 必须有两个输出：第一项是下一步状态，第二项必须命名为 `stop` 且类型为 Bool。处理完当前元素后，`stop` 为 true 就立即返回，不再请求下一个元素。 +- `reduce` 和 `reduce_until` 不提供额外 context 参数；共享配置应组合进状态，避免同一调用同时维护两套跨元素数据。 +- sequence 只接受 `List<Item>`、Bytes 或 `Stream<Item>`。Bytes 的元素静态类型为 Int，运行时值保证在 0 至 255，不先复制为 `List<Int>`。Text 如需按 UTF-8 遍历，必须显式使用 `dever.bytes.from_text`。 +- `each(List/Bytes)` 保留零输出动作和单输出 List 映射；`each(Stream)` 只接受零输出 handler，不把未知长度或无限输出隐式收集到内存。Stream 的最终状态由 `reduce`/`reduce_until` 返回。 +- `filter`、`find`、`sum` 本阶段仍只处理 List；不增加 Stream 到 Stream 的惰性 map/filter 管道。 + +### Static Handler Inputs + +function 输入可以声明受限 handler 签名： + +```dever +serve( + route: handler(request: Request) (response: Response), + config: Config +) (result: ServeResult) { + result = serve_with(route, config) +} +``` + +- handler 类型只允许出现在 function 输入中；输入按位置匹配类型，命名输出同时匹配名称、顺序和类型。 +- 调用方只能传入签名匹配的具名 function，或转发当前 function 的 handler 参数。 +- handler 参数可以直接调用，也可以传给 `each`、`reduce`、`reduce_until` 或另一个 handler 输入；不能参与分句匹配，不能赋给局部值、存入 record/List/Map/Stream、比较或作为输出返回。 +- handler 不是运行时值。typed HIR 保留静态绑定，AOT 以 function 和实际 handler 绑定组成专门化实例，生成直接调用；不产生函数指针、闭包、虚调用或反射注册表。 +- 递归检查覆盖 handler 调用与转发后的专门化调用图，不能通过 handler 绕过无递归规则。 + +### Stream And Resource Contract + +- `Stream<Item>` 是编译器提供的唯一新增标准泛型类型，用户仍不能声明泛型 type 或泛型 function。 +- Stream 是拉取式、顺序、一次消费的 opaque 资源；复制 Stream 只产生同一资源的别名，所有别名共享读取位置、结束状态和显式关闭状态。Stream 不可比较，也不能成为 Map key。 +- `reduce_until` 提前停止时 Stream 保持可用，调用方可从下一个未消费元素继续处理。正常耗尽后继续消费只得到结束，不会重新执行生产者。 +- 正常 EOF 使用 Stream 自然结束表示，不作为普通元素。可恢复 I/O 错误必须作为官方 choice 的 `Failed(message: Text)` 元素交给 handler；终止性错误产生一个 Failed 后结束 Stream，不能转换成默认值或程序故障。 +- 流式读取使用 `dever.io.ReadEvent`，其分支固定为 `Chunk(bytes: Bytes)` 和 `Failed(message: Text)`；现有单次 `read` 继续返回 `ReadResult.Read/End/Failed`，两种接口不混用结束语义。 +- 流创建结果固定为 `dever.io.ReadStreamResult`，其分支为 `Streaming(stream: Stream<ReadEvent>)` 和 `Failed(message: Text)`。`dever.io.chunks(file, limit)` 与 `dever.net.chunks(socket, limit)` 都返回该类型；非正数、无法表示的 limit 或创建时已经关闭的源立即返回 Failed。 +- `dever.net.connections(listener)` 直接产生 `Stream<ConnectResult>`；创建后关闭 listener 或终止性 accept 错误产生一个 `ConnectResult.Failed` 后结束。 +- 核心操作 `close(stream)` 没有输出并且幂等。它停止该 Stream 及其别名并释放 Stream 持有的底层资源引用，但不主动关闭调用方仍单独持有的 File、Socket 或 Listener；关闭后的 Stream 与正常耗尽一样不再产生元素。调用方先关闭底层资源时，依赖它的未关闭 Stream 产生一个 Failed 后结束；最后一个资源别名离开作用域时仍由原生资源析构兜底。 +- 首阶段只有阻塞、顺序执行；Socket 延续显式 timeout。没有预取、后台线程、async、await、并发消费或共享可变业务全局状态。 + +### Acceptance Criteria + +1. List 和 Bytes 的 `reduce` 按源顺序传递状态，空序列返回初始状态；Bytes 不经过中间 List。 +2. `reduce_until` 在 stop 元素之后不调用 handler 或拉取 Stream；同一 Stream 随后从未消费位置继续。 +3. `each(Stream)` 接受零输出 handler，拒绝单输出或多输出 handler；List/Bytes 的既有映射和动作语义不回归。 +4. handler 输入支持具名 function、跨 package function 和多层静态转发；签名不匹配、存储、返回、比较和 handler 间接递归均在原生构建前拒绝。 +5. 生成程序中的 handler 是专门化直接调用；不引入通用 runtime Value、运行时函数指针或逐元素虚拟 handler 分派。 +6. Stream 别名共享游标、结束和关闭状态；自然 EOF、提前停止、幂等 `close`、底层关闭与终止性错误均有确定测试。 +7. 官方 `dever.io`/`dever.net` 流式接口作为普通 `.dever` 源码经过同一 parser、resolver、checker 和 HIR；runtime 只拥有系统读取和 Stream 拉取原语。 +8. 文件验证只使用测试自有临时目录，TCP 验证只使用 `127.0.0.1` 随机端口、有限数据和超时；不连接、启动或修改现有服务。 +9. 实际 `check`、`run`、`build` 与独立原生程序覆盖一个分块状态解析样例，证明结果来自 `.dever` handler 和状态，而非编译器特判。 + +### Out Of Scope + +- HTTP、JSON、CRUD、数据库、并发服务器和异步语法；本阶段交付的是这些 package 可复用的底层表达能力。 +- 一般化用户泛型、lambda、闭包、一等 function、Stream 惰性转换、循环 block、用户递归和异常机制。 +- formatter、语义基准解释器、性能门槛、自举、package 分发、跨平台构建和全局命令迁移。 + +### Known Risks And Controls + +- 静态 handler 转发可能产生重复原生实例；专门化键按 function 与完整 handler 绑定去重，并且只生成入口可达实例，不退化为运行时分派。 +- Stream 的内部生产者调用有固定边界成本；文件和 Socket 以 Bytes chunk 为单位拉取，不按单字节跨越动态边界，有限 List/Bytes 热路径仍是直接循环。 +- 首阶段的阻塞 Stream 不能承担并发服务器；这是明确范围限制，不能在 HTTP package 中用隐藏线程或全局队列规避。 +- 把 I/O 失败作为最后一个元素要求 handler 穷尽处理官方 choice；类型检查和终止状态测试共同防止错误被静默丢弃或无限重复。 + +## Completed Hello Milestone + +- 示例位于 `examples/dever/hello/app/user.dever`，与 `package app.user` 路径一致。 +- 支持 Text 字面量、Text 参数、局部赋值、普通零输出函数调用和公开零输入入口。 +- 内置标准签名为 `dever.io.println(text: Text) ()`，输出 UTF-8 文本和一个换行；写入失败报告 Dever 调用位置并以非零状态退出。 +- 修改文本、调用其他自定义函数、跨包调用公开函数都必须由实际源码驱动；不能特判示例名称或内容。 +- 名称、参数、可见性、包路径和重复定义检查在原生编译前完成；递归及包依赖循环按既有契约拒绝。 +- 未实现的类型、表达式、分句和输出签名明确报不支持，不静默生成近似代码。 +- 提供本地 `check <source-root>`、`run <source-root> <package.function>`、`build <source-root> <package.function> --output <path>`；不替换全局命令。 +- 验收真实 stdout、错误定位、原生可执行文件和输出失败；不启动任何服务或执行全量测试。 + +## Deferred User API Proposal + +以下保留之前的 API 讨论作为后续参考；其中持久化、并发与性能要求尚未重新确认，不属于当前验收或实施前置条件。 + +### Confirmed Requirements + +- 应用业务使用 Dever 源码（`.dever` 或 `.dever.md` 的程序块）；不能手写 Rust 用户 CRUD 再把 Dever 当作装饰。 +- 用户模型只有 ID、姓名、手机号，CRUD 和 HTTP JSON 交互必须实际可用。 +- 保留已经讨论的表面语法；只增加实际应用所需的标准 package 能力和必要语义。 +- 编译器与通用运行时不能特判 `User`、`name`、`phone` 或 `/users`；这些信息来自应用源码。修改业务类型和路由不能要求修改编译器。 +- 数据需要真实持久化；重启后仍可查询，不能用进程内 Map 冒充实际存储。 +- 性能验证覆盖真实读写、序列化和持久化，不用空接口成绩替代 CRUD 性能。 + +业务模型沿用现有语法： + +```dever +type User { + id: Id + name: Text + phone: Text +} +``` + +### Proposed API Acceptance + +以下为待最终收敛的最小 API 行为，不增加登录、管理页面或其它业务模块： + +| Request | Behavior | +| --- | --- | +| `POST /users` | 提交姓名和手机号；服务端生成 ID，返回创建的用户 | +| `GET /users/{id}` | 查询单个用户；不存在时返回明确的未找到结果 | +| `GET /users` | 按稳定顺序、有上限地分页查询 | +| `PUT /users/{id}` | 完整更新姓名和手机号，ID 不可变 | +| `DELETE /users/{id}` | 删除指定用户，删除后无法查询到 | + +- `name`、`phone` 使用 Text；不把手机号当数值，也不擅自添加特定国家的手机号规则或手机号唯一约束。 +- 缺少字段、非法 JSON、无效 ID、未知用户和存储失败需要明确响应；不能静默填值或伪装成功。 +- 并发访问不能产生重复 ID、半写入或未说明的丢失更新；具体并发写入契约随存储设计确定。 +- 吞吐、p50/p95/p99 延迟、错误率和资源使用需要在固定数据量、读写比例、并发和持久化配置下报告，并与同环境、同功能的原生基线比较。数值性能门槛仍待收敛，现有计算内核的 1.15 倍预算不能直接替代 API 指标。 + +### Open Decisions + +- 后续 API 先允许内存测试数据；需要持久化时再讨论数据库、并发更新和性能预算。 + +### Deferred From This Milestone + +- 契约 DSL、AI 生成器、全面重复语义检测和完整的大项目上下文工具。 +- 登录/RBAC、前端页面、多租户、分布式部署及其它没有被请求的用户业务。 +- 与用户 API 无关的完整数值标准库、全部集合组合、自举和组件分发系统；既有语言契约作为后续路线保留。 + +## Language Roadmap + +后续章节保留原 Minimal Core 的语言契约，作为本轮定义讨论的基础。已确认的长期规则与首版能力限制分别标明；讨论中的功能不代表已经实现，编译器仍须明确拒绝尚未支持的源码。 + +## Repository Evidence + +- 仓库当前没有已发布的 `.dever` parser、formatter 或兼容格式。 +- 2026-09-05 开工检查确认：当前仓库只有规划资料，没有 Cargo workspace、Rust 源码或测试。HEAD 也只包含两份旧规划文档。 +- 新语言没有兼容旧语义图、旧 CLI、旧生成结果或旧测试的义务。旧规划由 Git 文档历史保留，不在新实现中增加迁移层。 +- 从零建立 Rust workspace；复用源码位置诊断思路和仓库根目录 `test/` 约束。旧领域模型和旧后端契约不能决定新语法。 + +## Core Language Roadmap Scope + +1. `.dever` 源文件、UTF-8 文本、注释、literal 和唯一 formatter。 +2. package 路径、公开名称、完整限定名和源码根目录内的自动依赖解析。 +3. 被动 type、字段型值、选择型值、可空值、List、Map、Bytes 和 opaque Stream 资源。 +4. 显式 function 边界、线性 body、局部类型推断和值语义修改。 +5. 通过同名 function 分句表达的穷尽、互斥条件规则。 +6. 算术、比较、布尔和文本连接表达式。 +7. `each`、`reduce`、`reduce_until`、`filter`、`find`、`sum` 以及最小 List/Map 标准函数。 +8. Int、Decimal、Float 的确定数值语义和最小 `math`/`decimal` 标准 package。 +9. parser、formatter、resolver、type checker、clause analyzer、语义基准解释器、原生 AOT 编译和 CLI。 +10. Rust 编写的首代引导编译器，以及不依赖 Rust 表面语法、可被未来 Dever 自举编译器复用的 typed HIR 和运行语义。 + +## Language Contract + +### 1. Source And Package + +- 源文件后缀固定为 `.dever`，编码为 UTF-8。 +- 代码标识符只使用 ASCII 英文；注释、字符串和文档可以使用任意 UTF-8。 +- `#` 开始单行注释；Core 0.1 不提供块注释。`//` 只表示整数除法，不具有第二种词法含义。 +- 语句和声明按换行或结构边界分隔，不写分号。 +- 一个文件完整拥有一个 package；同一 package 不能拆到多个文件。 +- 源文件相对 `src/` 的路径必须与 package 点分名称一致。 + +```text +src/account/user.dever -> package account.user +``` + +文件第一项必须是 package 声明： + +```dever +package account.user exposes ( + User + create +) +``` + +`exposes` 只能列出当前 package 已声明的 type 或 function。未列出的名称只在当前 package 可见；公开 function 名称会公开该名称的全部签名和分句。 + +同 package 引用使用短名称，跨 package 引用必须使用完整限定名： + +```dever +normalized = normalize_email(email) +user = account.user.create(id, name, email) +``` + +Core 0.1 不提供 `import`、`use`、通配导入或别名。编译器从完整限定引用建立依赖图，并拒绝 package 循环依赖。`app`、`domain`、`feature`、`flow`、`policy`、`view` 和 `scenario` 都不是关键字，需要时只是普通 package 路径段。 + +#### Package Origins And Distributed Components + +语言层只有 package，不新增 `component` 关键字。package 是源码和 API 命名空间；“组件”只是 registry 中用于安装、签名和版本管理的发布单元，可以包含一个根 package 及若干子 package。组件不能绕过各 package 的 `exposes` 边界。 + +package 来源固定分为三类： + +| 来源 | 名称规则 | 示例 | 获取方式 | +| --- | --- | --- | --- | +| 核心标准 package | 编译器保留的单段名称 | `text`、`math`、`decimal` | 随编译器提供 | +| Dever 官方组件 | `dever.<component>` 根路径 | `dever.http`、`dever.sql` | 显式安装 | +| 公共第三方组件 | `<publisher>.<component>` 根路径 | `stripe.payment`、`shemic.robot` | 显式安装 | + +本地业务 package 继续使用项目内路径，例如 `account.user`、`order.checkout`。本地源码不能声明核心标准名称或 `dever.*` 官方命名空间。同一个构建中如果本地源码、标准库或未来已安装组件提供了相同 package 名称，编译器直接报告重复定义，不进行覆盖或静默择优。 + +调用形式对所有来源一致，版本不会进入业务源码： + +```dever +user = account.user.create(input) +name = text.trim(raw_name) +response = dever.http.send(request) +payment = stripe.payment.create(order) +``` + +从 0.2 起，组件的稳定身份与下载来源相互独立。`shemic.contact` 是进入 package 名称和类型身份的 component id；GitHub、其他 Git 服务或 Dever registry 只是可以替换的 source。仓库迁移不会迫使业务源码改名。 + +公共组件必须通过命令显式加入，不能因为遇到未知名称就在 `check` 或 `build` 时自动联网： + +```text +dever add dever.http@1 +dever add stripe.payment@3 +dever add shemic.contact --git https://github.com/shemic/contact.git --tag v1.2.0 +dever add shemic.contact --git https://github.com/shemic/contact.git --rev 8f34c2a +dever update stripe.payment +dever fetch +``` + +Git source 必须同时给出 tag 或 revision，不接受未固定的默认分支 HEAD。tag 在加入时解析为完整 commit；`dever.lock` 同时记录 source URL、完整 commit 和规范化源码内容的 SHA-256，后续 tag 被移动也不会改变构建。组件内全部 package 必须位于 component id 根路径下，例如 `shemic.contact` 可以提供 `shemic.contact.phone`，不能同时冒充 `account.user`。 + +0.2 的项目根目录增加两个由 CLI 维护、需要提交的元数据文件： + +```text +dever.mod # component id、直接依赖及其 registry/git/path 来源要求 +dever.lock # 完整依赖图、精确版本/commit、内容摘要和实际提供的 package +``` + +开发者仍只手写 `.dever`；`dever add`、`dever update` 和未来的本地 link 命令负责规范化更新元数据。分开保存意图和精确结果，使直接依赖保持可读，同时保证所有机器解析出同一份源码。 + +下载源码不复制到每个项目。Dever 使用平台级、可由 `DEVER_HOME` 改址的全局只读内容寻址仓库： + +```text +$DEVER_HOME/ + git/<source-url-hash>/ # Git bare mirror，只负责取对象 + store/sha256/<content-hash>/ # 已校验、不可修改的组件源码 + build/<target>/<artifact-hash>/ # 可删除的编译缓存 +``` + +项目目录只保留 `src/`、`dever.mod`、`dever.lock` 和可删除的 `target/dever/` 构建结果。多个项目依赖相同 commit 时共享同一份 store 内容。`dever add`、`dever update`、`dever fetch` 是仅有的默认联网入口；普通 `check`、`build` 和 `run` 只读取本地源码、内置标准 package、lock 与 store，缺失时明确要求执行 `dever fetch`，不隐式下载。 + +Core 0.1 只实现本地 package 与随编译器提供的核心标准 package，同时锁定上述命名和解析边界。registry、`dever add`、签名验证和外部组件缓存确定延后到 0.2，不影响现有 `.dever` 调用形式。 + +### 2. Canonical Naming + +| 对象 | 唯一格式 | 示例 | +| --- | --- | --- | +| package 路径段 | `lower_snake_case`，优先单数领域名 | `account.user`、`sales_order` | +| 文件名 | `lower_snake_case` | `user.dever` | +| type | `UpperCamelCase` | `User`、`CreateResult` | +| 选择项 | `UpperCamelCase` | `Pending`、`Created` | +| function | `lower_snake_case` | `create`、`can_sign_in` | +| 输入、输出、局部值、字段 | `lower_snake_case` | `user_id`、`created_at` | +| 标准泛型参数 | 有含义的 `UpperCamelCase` | `Item`、`Key`、`Value` | + +大小写不控制公开性。缩写按普通单词处理：`Id`、`Url`、`HttpClient`、`user_id`、`callback_url`。编译器强制可机械判断的大小写、ASCII 和路径格式；“是否是好名称”只产生规范诊断，不依赖英语词典阻止编译。 + +### 3. Types And Literals + +Core 0.1 的基础标量只有： + +| 类型 | 语义 | +| --- | --- | +| `Bool` | `true` 或 `false` | +| `Int` | 有符号 64 位整数，范围 `-9223372036854775808` 至 `9223372036854775807` | +| `Decimal` | decimal128，最多 34 位有效十进制数字，round half to even | +| `Float` | IEEE 754 binary64 近似数 | +| `Text` | UTF-8 文本；相等按实际字符序列判断，不做隐式 Unicode 规范化 | +| `Id` | 不透明稳定标识，只支持相等比较、传递和序列化 | + +Core 0.1 不提供 `UInt`、`BigInt`、`Percent`、字符类型或隐式动态 `Json` 类型。 + +- 整数 literal 默认是 Int。 +- 带小数点的 literal 默认是 Decimal；在明确要求 Float 的静态上下文中直接成为 Float。 +- 科学计数 literal，例如 `1.25e6`，是 Float。 +- Decimal literal 必须能被 34 位 decimal128 精确表示；源码常量不会被静默舍入，超出精度或范围时由编译器拒绝。运行时算术仍按已确定的银行家舍入产生结果。 +- Text 使用双引号，支持 `\\`、`\"`、`\n`、`\r`、`\t` 和 `\u{HEX}` Unicode 转义。 +- `true`、`false` 和 `null` 是保留 literal。 +- 源码不提供 NaN 或 Infinity literal；Float 运算可以产生这些值。 + +用户定义 type 只有两种形状，不能混写： + +```dever +type User { + id: Id + name: Text + phone: Text? +} + +type CreateResult { + Created(user: User) + InvalidPhone(reason: Text) + EmailAlreadyUsed(email: Text) +} +``` + +字段型 type 是固定字段数据；选择型 type 表示同一时间只能是其中一种选择。type 没有方法、构造器、hook、继承或行为，也不能递归包含自身。 + +选择项必须通过所属 type 限定： + +```dever +result = CreateResult.Created(user) +``` + +选择项携带的数据在声明中必须有名称和类型，构造时按声明位置传入。字段型值必须一次完整创建： + +```dever +user = User { + id = id + name = name + phone = null +} +``` + +未知、重复或遗漏字段都是编译错误，包括值为 `null` 的可空字段。formatter 按 type 声明顺序输出字段。点号只访问或修改已经完整存在的字段型值，不能逐字段创建半初始化对象。 + +`Value?` 表示 `Value` 或 `null`。普通类型永远不能为 `null`，`Value??` 非法，也没有 `?.`、`??`、强制解包或隐式默认值。可空值必须交给穷尽的 function 分句处理。 + +标准泛型函数产生可空返回值时，可空化是幂等的：`first(List<Int?>)`、`find` 的同类调用及 `get(Map<Text, Int?>, key)` 都返回 `Int?`，不产生嵌套可空类型。返回 `null` 不区分“没有元素/键”和“已有值为 null”；需要区分时，业务使用带显式选择项的 Value 类型。源码仍禁止显式书写 `Int??`。 + +### 4. List And Map Values + +List 使用方括号创建： + +```dever +users = [first_user, second_user] +empty_users() (users: List<User>) { + users = [] +} +``` + +List 元素必须是同一静态类型。空 List 必须能从 output、参数或其他静态上下文推断元素类型，否则编译错误。 + +Map 是 `Map<Key, Value>`，使用裸大括号创建： + +```dever +default_labels() (labels: Map<Text, Text>) { + labels = { + "env" = "production" + "region" = "china" + } +} +``` + +同一 Map literal 的 key 类型一致、value 类型一致，重复 key 是编译错误；空 Map 也必须能推断完整类型。`Type { ... }` 是固定字段型值，裸 `{ ... }` 是动态 Map，二者不会合并成万能对象。 + +Map key 只允许 Text、Int、Bool、Id 和不携带数据的选择型 type。Decimal、Float、可空值、字段型值、List 和 Map 不能成为 key。 + +Map 保证插入顺序。literal 保留源码顺序；`put` 新 key 时追加，覆盖已有 key 不移动位置；`remove` 后重新写入视为新的末尾项。`entries`、显示和未来的 JSON 边界都必须遵守这一顺序。 + +### 5. Function Definitions And Calls + +所有行为只有一种定义形式： + +```dever +line_total(line: Line) (total: Decimal) { + total = line.price * line.quantity +} +``` + +- 第一组括号是输入，第二组是命名输出。 +- 普通输入和输出必须写成 `name: Type`；body 内局部值可以推断。 +- 没有输入或输出时仍写空括号。 +- 不使用 `function` 或 `return` 关键字。 +- 输入名称是当前分句的局部绑定；输出名称和输入输出类型属于稳定签名。 +- 调用只按位置传参，不提供命名参数调用。 +- 用户 function 以“package、名称、输入数量”确定签名；同名同参数数量的全部分句必须具有相同的基础输入类型和输出契约。不同参数数量可以形成独立签名。 +- 同一签名的全部分句必须在同一文件中连续书写。 +- Core 0.1 禁止直接或间接递归；重复执行只能通过 `each`、`reduce` 和 `reduce_until` 这些受检查的具名 handler 组合器进入。 + +选择项会提供所属选择型 type，普通 type 会提供自身类型；`null` 和 `other` 则从同组其他分句推断基础输入类型。只写 `null`/`other` 而没有任何可推断类型的分句组是编译错误。`Text` 与 `null` 两个分句共同形成 `Text?` 输入，而不是两个运行时重载。 + +单输出调用直接得到该值，零输出调用作为一条动作语句，多输出调用得到只能按输出名称访问的结果： + +```dever +allowed = can_sign_in(user.status) +parts = split_name(name) +display = parts.first + parts.last +``` + +不提供按位置拆包。命名 function 只能在普通调用位置、作为标准序列/集合操作的 handler，或传给受限静态 handler 输入；Core 0.1 不提供 lambda、闭包，也不能把 function 存入局部值、字段、List、Map 或 Stream。 + +### 6. Function Clauses Replace Conditional Blocks + +function 分句是 Dever 唯一的条件分派机制。这是 2026-09-07 用户确认的长期语言规则；条件 block 和条件表达式不属于后续待增加能力。Bool 表达式及其短路规则仍按第 8 节定义，业务分支通过调用具名 function 的分句选择。 + +示例： + +```dever +can_pay(status: OrderStatus.Draft) (allowed: Bool) { + allowed = true +} + +can_pay(status: other) (allowed: Bool) { + allowed = false +} +``` + +冒号左侧始终是当前分句的局部输入名称；不同分句可以使用不同名称。右侧允许： + +- 普通类型，表示该类型的任意值； +- `true`、`false`、`null`、Text 或数值常量； +- 完整限定的选择项及其数据绑定，例如 `CreateResult.Created(user)`； +- 选择项中的 `_`，表示忽略该携带值； +- Int 或 Decimal 的静态常量比较； +- 上述数值比较的一个上下界组合； +- 上下文关键词 `other`。 + +数值范围直接放在输入中： + +```dever +absolute(value: Int < 0) (result: Int) { + result = -value +} + +absolute(value: Int >= 0) (result: Int) { + result = value +} + +grade(score: Int >= 60 and < 90) (name: Text) { + name = "pass" +} + +grade(score: Int < 60) (name: Text) { + name = "fail" +} + +grade(score: Int >= 90) (name: Text) { + name = "excellent" +} +``` + +Float 不进入静态范围分句，因为 NaN 不具有普通全序；先通过 `math.is_nan` 等 function 得到 Bool，再对 Bool 写分句。 + +分句没有源码顺序优先级。编译器把同一签名的全部分句作为一组规则，拒绝类型不一致、重叠、遗漏和不可达情况。多输入 function 中，每个 `other` 是该输入位置全部明确模式的补集，编译器再检查各位置组合是否覆盖完整输入空间。`other` 只在分句输入冒号右侧具有该含义，不能作为普通值存储、传递或组合。 + +分句输入中不能调用任意 function 或读取运行时状态。更深一层的业务选择必须调用另一个有名称的 function，源码不提供 `if`、`else`、`switch`、`match`、`when` 或 `by` block。 + +### 7. Linear Body And Value Semantics + +body 只包含顺序赋值和 function 调用： + +```dever +final_price(price: Decimal, rate: Decimal) (result: Decimal) { + discounted = price * (1 - rate) + result = decimal.round(discounted, 2) +} +``` + +表达式由 literal、名称、字段访问、值构造、function 调用、一元运算和二元运算组成。没有条件表达式、循环 block、匿名 function、复合赋值或隐藏控制流。 + +`=` 既用于初始化，也用于修改当前 function 的 input、局部值或 output。编译器强制： + +- 读取前已经完整初始化； +- 字段修改的宿主值已经完整初始化； +- function 结束时全部命名输出都已赋值； +- 赋值类型稳定，不通过再次赋值改变局部类型； +- 无意义覆盖和写入后未读取至少产生诊断。 + +普通值的赋值、传参和输出采用值语义。修改只属于当前局部名称，调用方或其他名称不会观察到共享别名变化；变化只能通过命名 output 或未来的显式 effect function 离开当前调用。实现可以用唯一所有权或写时复制消除真实复制，但源码不暴露借用、生命周期或 `clone`。File、Socket、Listener 和 Stream 是显式 opaque 资源例外：复制只建立同一资源状态的别名，不复制外部资源。 + +### 8. Expressions And Operators + +普通算术直接使用符号： + +```dever +subtotal = price * quantity +ratio = 5 / 2 +page = 5 // 2 +remainder = 5 % 2 +eligible = user.active and user.age >= 18 +``` + +| 运算 | 规则 | +| --- | --- | +| `+ - *` | Int、Decimal 或 Float 的同类运算；Int 与 Decimal 混合时 Int 无损提升为 Decimal | +| `/` | Int/Int 返回 Decimal；Decimal 与 Int 返回 Decimal；Float/Float 返回 Float | +| `//` | 只接受 Int，结果向零截断 | +| `%` | 只接受 Int，余数符号跟随被除数，并满足 `a = (a // b) * b + (a % b)` | +| `+` | 也可连接两个 Text | +| `== !=` | 相同可比较类型 | +| `< <= > >=` | Int、Decimal 或 Float 的同类比较；Int 可提升为 Decimal | +| `and or not` | 只接受 Bool，并具有短路语义 | + +可比较类型包括上述标量，以及成员递归可比较的字段型、选择型、可空、List、Map 和 MapEntry。字段型按声明字段逐项比较；选择型比较选择项和携带值；List 按位置比较；Map 按插入顺序比较 key/value 对，因此顺序不同的 Map 不相等；可空值中只有两个 null 相等，两个非 null 值继续比较其内容。Float 保持下文的 NaN 规则，包括嵌套在集合或字段中时。命名多输出结果只能访问输出字段，不能直接比较。相等运算不做 Int/Decimal 提升。 + +Decimal 与 Float 不隐式混合，实际 Int 值转为 Float 也必须显式调用 `float.from_int`。Core 0.1 不提供 Float 到 Decimal 的隐式或有损转换。 + +运算优先级从高到低固定为：调用和字段访问、一元 `-`/`not`、`* / // %`、`+ -`、比较、`and`、`or`。同级二元运算从左到右；括号可以显式改变顺序。formatter 不依赖空白表达优先级。 + +### 9. Collection And Sequence Functions + +Core 0.1 只保留以下集合行为，不增加循环语法： + +```dever +views = each(to_view, users) +views = each(to_view, users, context) + +active = filter(is_active, users) +active = filter(is_active, users, context) + +user = find(is_target, users) +user = find(is_target, users, context) + +total = sum(prices) +total = sum(line_total, lines) +total = sum(line_total, lines, context) + +state = reduce(update_state, values, initial_state) +state = reduce_until(update_until_done, values, initial_state) + +parallel_each(consume, values, 4) +parallel_each(consume, values, 4, context) +``` + +- `each`、`filter`、`find` 和投影式 `sum` 最多有一个末尾 context；需要多项共享数据时先组合成字段型值，其 handler 先接收元素、再接收 context。 +- `reduce` 和 `reduce_until` 的第三项是必需的初始状态，不是可选 context；handler 先接收元素、再接收当前状态，且不接受第四项参数。 +- `each` 的处理 function 有一个输出时按输入顺序产生 `List<Output>`；没有输出时只按顺序执行动作；多个输出不合法。 +- `filter` 的处理 function 必须输出 Bool，并保留原列表顺序。 +- `find` 找到第一个 Bool 为 true 的元素后停止，返回 `Item?`；没有结果返回 `null`。 +- `sum(List<Number>)` 直接求和；投影形式要求处理 function 输出 Int、Decimal 或 Float。 +- 空 List 的 `sum` 返回相应数值类型的零。 +- 除显式 `parallel_each` 外，List 操作按列表顺序执行，Bytes 按字节位置执行，Stream 按拉取顺序执行；Float 求和不得为了优化而重排。 +- `reduce` 和 `reduce_until` 的完整 List/Bytes/Stream、状态和停止契约由 Approved Foundation Extension 定义；它们不开放循环 block 或用户递归。 +- `parallel_each` 接受 List、Bytes 或 Stream、1–256 个工作线程和可选 context；handler 必须零输出，输入依次是元素和 context。复用既有静态 handler 与专门化规则，不产生用户可存储的 function 值。 +- 生产端顺序拉取且队列有界，工作端的副作用顺序不保证。handler 输入与 context 不能递归包含 Stream，源 Stream 只在生产端拉取；文件与 Socket 等别名仍遵守共享资源规则。 +- `parallel_each` 返回前等待全部已启动动作；程序故障在后续拉取边界停止分派，保留第一个已记录故障并传播源码位置。已经阻塞的拉取或系统调用由显式系统超时约束，不提供强制取消或后台遗留任务。非法线程数在拉取前产生程序故障。 + +其他最小标准函数： + +```dever +users = append(users, user) +user = first(users) + +value = get(labels, "env") +labels = put(labels, "env", "development") +labels = remove(labels, "env") +pairs = entries(labels) +``` + +`append` 返回新 List；`first(List<Item>)` 返回 `Item?`。`get` 返回 `Value?`；`put` 和 `remove` 返回新 Map；`entries` 返回按插入顺序排列的 `List<MapEntry<Key, Value>>`。没有 `list.add`、数字点号索引、`map.key` 或 `map[key]`。 + +`any`、`all`、`min`、`max` 不进入首版；真实重复需求出现后再增加普通标准 function。 + +### 10. Recoverable Failures And Runtime Faults + +可恢复业务失败使用选择型输出： + +```dever +create_message(result: CreateResult.Created(user)) (message: Text) { + message = user.name +} + +create_message(result: CreateResult.InvalidPhone(reason)) (message: Text) { + message = reason +} + +create_message(result: CreateResult.EmailAlreadyUsed(email)) (message: Text) { + message = email + " already exists" +} +``` + +这同时替代 `throw` 和 `catch`。调用方接收完整选择值，再通过命名分句显式转换。Core 0.1 不提供 exception、自动错误传播、隐藏提前退出或类似 Rust `?` 的语法。 + +编译器可以证明的不变量错误直接拒绝编译。只能在运行时发现的程序故障终止当前执行边界，并至少报告故障类别、文件、行和列；普通 function 不能捕获、忽略或转换程序故障。执行边界由宿主确定，例如一次 CLI 调用，未来可以是一次请求或任务。 + +数据库、网络等只要存在恢复可能，就必须在未来以显式选择型结果返回，不能伪装成程序故障。 + +### 11. Numeric Semantics + +- Int 是固定 i64；不环绕、不饱和。静态可证明的溢出是编译错误，运行时溢出是程序故障。 +- `Int / Int` 先无损提升为 Decimal 再除，因此 `5 / 2` 是 Decimal `2.5`。 +- `Int // Int` 向零截断；`Int % Int` 与商保持代数恒等式。除数为零以及 `-9223372036854775808 // -1` 属于溢出故障。 +- Decimal 使用 decimal128 的 34 位有效数字和银行家舍入；算术中的正常 inexact/rounded 状态产生舍入结果，指数上溢、下溢、除零或无效运算产生程序故障，不向源码暴露 Decimal NaN/Infinity。 +- 金额位数由 `decimal.round(value, places)` 显式处理；核心算术不根据变量名、字段名或业务场景猜测两位小数。 +- Float 遵守 IEEE 754 binary64 非捕获行为：溢出产生正负 Infinity，无效运算产生 NaN，除零产生 Infinity 或 NaN。 +- Float 的 NaN 与任何值的 `==` 都为 false，`!=` 为 true，有序比较均为 false；使用 `math.is_finite`、`math.is_nan` 和 `math.is_infinite` 分类。 +- 常用科学函数通过 `math.sqrt`、`math.sin`、`math.cos`、`math.log` 和 `math.pow` 等普通标准 package function 提供。 +- Decimal 与 Float 不隐式混合。任意精度整数/小数、向量、矩阵、复数、统计与单位系统以后由独立 package 提供。 + +## Standard Names + +无需 import 即可使用的核心名称只有： + +- Bool、Int、Decimal、Float、Text、Id、Bytes、List、Map、MapEntry、Stream； +- true、false、null、other； +- each、parallel_each、reduce、reduce_until、filter、find、sum、append、first、get、put、remove、entries、length、用于 Stream 的 close； +- 仅用于 function 输入签名的上下文关键词 handler； +- 算术、比较和布尔运算符。 + +其他标准行为使用完整 package 名，例如 `text.trim`、`text.lower`、`decimal.round`、`float.from_int` 和 `math.sqrt`。这避免大量隐式全局名称，也让标准库与业务 package 遵守同一调用模型。 + +## CLI Identity And Coexistence + +- 语言、编译器产品和正式命令名保持为 `dever`，文档与命令语义以 `dever` 为准。 +- Core 0.1 与现有 Dever 框架 CLI 共存期间，系统 `PATH` 中可以暂时用 `deverc` 暴露语言编译器；不得覆盖、替换或遮蔽现有的 `dever` 命令。 +- `deverc` 只是指向同一编译器二进制的安装入口，不形成第二套命令、配置或行为契约。 +- 仓库构建与自动测试直接调用构建产物，不修改系统 `PATH` 或全局可执行文件；创建、更新或移除全局 `deverc` 入口必须单独获得用户授权。 +- 未来是否让语言编译器接管全局 `dever` 命令属于独立迁移决策，不在 Core 0.1 默认实施范围内。 + +## Complete Package Example + +```dever +public type UserStatus { + Pending + Active + Disabled(reason: Text) +} + +public type CreateInput { + id: Id + name: Text + email: Text + phone: Text? + score: Int + labels: Map<Text, Text> +} + +public type CreateResult { + Created(user: User) + InvalidName(reason: Text) + InvalidEmail(reason: Text) +} + +public type User { + id: Id + name: Text + email: Text + phone: Text? + score: Int + labels: Map<Text, Text> + status: UserStatus +} + +public type UserView { + id: Id + name: Text + email: Text + phone: Text + score: Int + can_sign_in: Bool +} + +public create(input: CreateInput) (result: CreateResult) { + normalized = normalize(input) + result = create_name(normalized, normalized.name) +} + +normalize(input: CreateInput) (result: CreateInput) { + result = input + result.name = text.trim(input.name) + result.email = text.lower(text.trim(input.email)) +} + +create_name(input: CreateInput, name: "") (result: CreateResult) { + result = CreateResult.InvalidName("name is required") +} + +create_name(input: CreateInput, name: other) (result: CreateResult) { + result = create_email(input, name, input.email) +} + +create_email(input: CreateInput, name: Text, email: "") (result: CreateResult) { + result = CreateResult.InvalidEmail("email is required") +} + +create_email(input: CreateInput, name: Text, email: other) (result: CreateResult) { + user = User { + id = input.id + name = name + email = email + phone = input.phone + score = input.score + labels = input.labels + status = UserStatus.Pending + } + result = CreateResult.Created(user) +} + +public activate(user: User) (result: User) { + result = user + result.status = UserStatus.Active +} + +public disable(user: User, reason: Text) (result: User) { + result = user + result.status = UserStatus.Disabled(reason) +} + +public can_sign_in(status: UserStatus.Active) (allowed: Bool) { + allowed = true +} + +public can_sign_in(status: other) (allowed: Bool) { + allowed = false +} + +phone_text(phone: Text) (result: Text) { + result = phone +} + +phone_text(phone: null) (result: Text) { + result = "Not provided" +} + +public to_view(user: User) (view: UserView) { + view = UserView { + id = user.id + name = user.name + email = user.email + phone = phone_text(user.phone) + score = user.score + can_sign_in = can_sign_in(user.status) + } +} + +public to_views(users: List<User>) (views: List<UserView>) { + views = each(to_view, users) +} + +is_active(user: User) (result: Bool) { + result = can_sign_in(user.status) +} + +public active_views(users: List<User>) (views: List<UserView>) { + active = filter(is_active, users) + views = each(to_view, active) +} + +has_id(user: User, expected: Id) (matches: Bool) { + matches = user.id == expected +} + +public find_user(users: List<User>, id: Id) (user: User?) { + user = find(has_id, users, id) +} + +public put_label(user: User, key: Text, value: Text) (result: User) { + result = user + result.labels = put(user.labels, key, value) +} + +public label(user: User, key: Text) (value: Text?) { + value = get(user.labels, key) +} + +score_bonus(user: User, rate: Decimal) (amount: Decimal) { + amount = user.score * rate +} + +public total_score_bonus(users: List<User>, rate: Decimal) (total: Decimal) { + total = sum(score_bonus, users, rate) +} +``` + +该源码只要求开发者理解 package、type、function、值和普通表达式。它同时覆盖显式业务结果、局部修改、条件分句、可空处理、列表转换、筛选、查找、Map 更新和 Decimal 聚合，所有路径保持平铺且可由编译器完整检查。 + +## Original Core Deferrals + +以下保留原 Core 0.1 的暂缓能力，具体版本安排在后续需求中确定。HTTP、JSON、持久化以及异步/并发 I/O 的应用扩展仍属暂缓 API 提案。 + +- AI、自然语言转译、画布和业务 DSL； +- Web 页面、React，以及超出首个 API 需要的数据库、网络和部署能力； +- 超出首个 API 需要的 effect/capability、异步、并发和事务能力；不开放任意共享可变业务全局变量； +- package registry、版本解析和项目清单； +- `for/while/break/continue/iterate/recur/repeat`、用户递归和其它通用循环结构； +- class、继承、interface、annotation、macro、unsafe、FFI； +- import、别名、命名参数、lambda、闭包和一般化一等 function； +- 用户泛型 type、递归 type、Set 和动态 Json；首个 API 的 JSON 边界应由静态类型驱动； +- UInt、BigInt、任意精度 Decimal、Percent、日期时间和单位类型； +- 直接 LLVM/Cranelift backend、完成自举、所有权或生命周期表面语法。 + +这些内容保留为 Core 0.1 暂缓项，不改变 Dever 最终成为通用编程语言的方向。条件 block 已从暂缓清单移除，条件分派始终遵守第 6 节的长期规则。 + +## Core Roadmap Acceptance Criteria + +这些是完整语言路线的验收项。2026-09-07 的基础语法、最小系统接口和后续工具/库分别验收，实际状态与证据见 `implement.md`；发行和自举不因基础里程碑完成而视为已交付。 + +1. 一个源码根目录中的多个 `.dever` 文件可以被加载；Core 0.1 只读取该根目录和内置标准 package，不隐式联网或搜索外部目录；路径与 package 不一致、保留命名空间、重复 package、未知公开名、未知引用和循环依赖均得到带源码位置的确定诊断。 +2. parser 覆盖本 PRD 的 package、两种 type、function、分句、value、statement 和 expression；拒绝已排除的关键字或语法。 +3. formatter 对合法程序产生唯一布局，连续执行两次结果不变，并保持注释和程序含义。 +4. 编译器强制命名格式、显式 function 边界、位置调用、一文件一 package、连续分句和无递归调用图；同 package 使用短名称，跨 package 必须使用完整限定名，并且只能访问对方 `exposes` 中的名称。 +5. type checker 拒绝未知/重复/遗漏字段、半初始化字段写入、未初始化读取、遗漏 output、非法 null、类型漂移、非法 Map key 和未授权隐式转换。 +6. clause analyzer 对选择型、可空、Bool、Text 常量、Int/Decimal 范围和多输入组合检查互斥、穷尽与可达性；分句移动顺序不改变执行结果。 +7. 本 PRD 的完整 package 示例可以通过解析、格式化、名称解析和类型检查；独立的零输入示例在语义基准解释器和原生程序中产生相同的确定命名输出。 +8. 局部赋值和字段修改不会改变调用方或其他名称持有的原值；编译器实现可以优化复制但不能改变可观察语义。 +9. `each`、`reduce`、`reduce_until`、`filter`、`find` 和 `sum` 的顺序、handler、context、初始/空结果、停止和类型规则全部有定向测试。 +10. List、Map、Bytes 和 Stream，以及 `append`、`first`、`get`、`put`、`remove`、`entries` 遵守可空返回、值/资源语义和稳定顺序。 +11. `5 / 2` 得到 Decimal `2.5`，`-5 // 2` 得到 Int `-2`，`-5 % 2` 得到 Int `-1`，并满足商余恒等式。 +12. Decimal 能证明 `0.1 + 0.2 == 0.3`，`1 / 3` 使用 34 位有效数字和 round half to even；`decimal.round` 显式控制业务位数。 +13. 静态 Int/Decimal 溢出或零除数被拒绝编译；动态故障终止当前执行边界并定位 `.dever` 源码，不环绕、饱和或返回默认值。 +14. Float 使用 binary64 和科学计数法，能够产生并通过 `math.is_nan`、`math.is_finite`、`math.is_infinite` 判断 NaN/Infinity；Decimal 与 Float 混用被拒绝。 +15. 可恢复失败只能通过选择型输出和穷尽分句传播；源码中不存在 exception、`throw`、`catch` 或自动传播语法。 +16. 正式 CLI 名为 `dever`，至少提供 `dever check <source-root>`、`dever fmt <source-root>`、`dever build <source-root> <package.function> --output <path>` 和 `dever run <source-root> <package.function>`；与现有框架 CLI 共存时，临时全局入口 `deverc` 必须调用同一编译器且不得覆盖现有 `dever`；build/run 首版只接受公开的零输入 function，run 执行与 build 相同的原生路径并确定性打印命名输出。 +17. 新实现不依赖旧 canonical JSON、SemanticGraph、Refiner、Workbench、温度示例或 Rust/React 生成后端，仓库文档不再把旧路线描述为当前架构。 +18. 生产产物不链接通用解释器，不携带运行时 type tag 分派或垃圾回收器；record、choice、List、Bytes 和静态 handler 在 AOT 路径中具有具体原生表示，热循环不发生逐元素动态 handler 分派或无必要堆分配；opaque Stream 只在 I/O 资源拉取边界保留内部生产者抽象。 +19. 在同一算法、数据布局、工具链和机器上的 release 基准中，Int/Float 算术及 List `each`/`filter`/`sum` 代表性热路径的中位执行时间不得超过等价 safe Rust 基线的 1.15 倍；Decimal 与有序 Map 分别对使用相同底层语义的 Rust 基线比较，编译时间单独报告。 +20. Rust 生成文件只存在于编译器管理的临时构建目录，不是用户源码、公共输出格式或兼容接口；删除这些文件不影响 `.dever` 工程。 + +## Risks And Deferred Items + +- 条件分句的穷尽分析是首版最复杂的静态检查，应先支持有限选择域与一维 Int/Decimal 区间，再组合多个输入；不引入通用定理证明器。 +- decimal128 需要成熟实现与一致状态标志。Core 0.1 采用 Rust `dec` crate 的 Decimal128 和可配置 Context，并将其封装在内部数值适配层；语言级测试锁定行为，依赖 API 不得泄漏到 `.dever`。 +- 当前仍禁止用户递归，已按批准范围提供显式文件/TCP 系统操作和结构化异步并发；异步网络、一般化 effect 和事务模型尚未实现，因此不是最终完整运行时。 +- Float 运算遵守平台 IEEE 754 语义，但任何未来的快速数学优化都必须显式设计，首版不得重排计算。 + +# Current Source And Module API Contract (2026-09-18) + +This user-approved contract supersedes historical `package/exposes`, arbitrary entry arguments and "API deferred" wording above. Source filenames relative to `module/` define package names in both `.dever` and `.dever.md`; top-level type/function declarations default private and may opt into cross-package access with `public`. `deverc run/build <project>` always invokes private `main()` from `module/main.dever` (or `.dever.md`) and accepts no entry argument. Markdown H1 documents path-derived package and public declarations with no executable H1 block. + +Each module may own recursive `api/` source containing only private `get_`/`post_`/`delete_` ordinary functions with one `response` output. Routes are statically validated and compiled only when reachable `main()` calls `dever.api.serve()`. GET/DELETE query inputs and POST JSON-object inputs are typed Text/Int/Bool (nullable allowed), and HTTP responses have JSON `code/message/data` envelopes. Listener settings are read only from `config/setting.json`. No application-config environment variable, implicit listener, new auth policy or CMS HTTP business endpoint is part of this milestone. Current canonical contracts are in `LANGUAGE.md`, `MARKDOWN-SYNTAX.md` and `.trellis/spec/backend/`. + +# Current AI-Oriented Source Architecture Contract (2026-09-19) + +This section supersedes the 2026-09-18 application visibility and package rules above. An application starts only at `module/main.dever` or `module/main.dever.md`; `main()` is implicit and private, and HTTP starts only when it calls `dever.api.serve()`. Application source contains no `package`, `exposes`, `public`, `internal` or import declarations. + +Business source uses `module/<component>/<domain>/<role>.dever` or `module/<component>/<domain>/<role>/<topic>.dever`, where role is exactly `app`, `domain`, `model`, `port`, `adapter` or `api`. A role file and its same-name directory are mutually exclusive. Non-API role directories are flat; only `api/` may add nested URL scopes. A role directory containing only one source file is rejected because it creates structure without separation; keep that implementation in the role file instead. + +Files group cohesive behavior rather than individual functions. One role file may contain multiple related functions, and one topic file may contain multiple related functions. The topic filename never becomes part of an App call. All App functions are cross-domain capabilities and are addressed as `<component>.<domain>.<function>`; a same-component caller may use `<domain>.<function>`, and source in the same domain may use `app.<function>`. Every other role is domain-private. Cross-domain and cross-component calls may target only App. Model types and ID types may appear in relations and signatures across domains, but generated Model operations remain callable only inside their owning domain. API handlers are HTTP entry adapters and are never source-callable. + +App inputs, outputs and inferred failure sets are one public contract. They cannot expose a private Domain/Adapter type; a business error that may leave the domain is declared in App and referenced from same-domain private source as `app.<ErrorType>`. + +The dependency direction is `api -> app`, `app -> domain/model/port/adapter` plus another domain's App, while domain/model/port/adapter remain implementation details. Generic dumping grounds such as `common`, `shared`, `utils`, `helper`, `base` and physical `service/` are invalid application architecture. Keep one-use logic inline, same-topic reuse beside its callers, cross-topic domain rules in `domain`, persistence in `model`, and external integrations in `adapter`; introduce `port` only for a real replaceable boundary. + +API route prefixes and JSON `code/message/data` envelopes remain as approved on 2026-09-18. Runtime settings still come only from `config/setting.json`; environment-variable configuration remains prohibited. Authentication remains business composition through App/domain capabilities rather than a compiler-owned HTTP policy. diff --git a/.trellis/tasks/09-04-dever-language-mvp/research/package-distribution.md b/.trellis/tasks/09-04-dever-language-mvp/research/package-distribution.md new file mode 100644 index 0000000..e67a52b --- /dev/null +++ b/.trellis/tasks/09-04-dever-language-mvp/research/package-distribution.md @@ -0,0 +1,51 @@ +# Package Distribution Research + +## Question + +How should a Dever project consume a component downloaded from GitHub or another Git host without leaking transport details into `.dever` source or losing reproducible builds? + +## Primary Evidence + +- Go defines a module as a versioned collection of packages. The module path is also the prefix of package import paths and usually identifies where the module can be downloaded. Downloaded source is shared through a module cache, and hashes are verified against `go.sum`. + - https://go.dev/ref/mod + - https://go.dev/doc/modules/gomod-ref +- Cargo permits registry, Git and local path sources. Git dependencies may select a tag or revision, and Cargo locks the resolved commit in `Cargo.lock` until an explicit update. + - https://doc.rust-lang.org/cargo/reference/specifying-dependencies.html +- Cargo and Go both separate downloaded source caches from project build artifacts. Go also supports optional vendoring for a fully project-local source tree. + - https://go.dev/ref/mod#module-cache + - https://go.dev/ref/mod#vendoring + - https://doc.rust-lang.org/cargo/reference/config.html#cache + +## Alternatives + +### Git URL as package identity + +Example: `github.com.shemic.contact.phone.parse(raw)`. + +This is easy to locate but couples every source reference and type identity to the current hosting provider. Repository transfer, self-hosting or registry mirroring then becomes a source-breaking rename. It also creates long names that obscure the business API. + +### Project-local copied dependencies + +Example: clone every dependency under `vendor/`. + +This is transparent and offline-friendly but duplicates source across projects, increases repository size and makes accidental dependency edits likely. It is useful as an explicit export mode, not the default store. + +### Stable identity plus independent source + +Example identity: `shemic.contact`; example source: `https://github.com/shemic/contact.git` at an exact resolved commit. + +This preserves short, stable `.dever` names while allowing registry, Git, mirrors and local development to use the same package identity. It requires dependency metadata and identity verification, but those responsibilities belong to the package manager rather than business source. + +## Recommendation + +Use the third model: + +1. `component id` is the stable namespace and type-identity root. +2. `source` is a replaceable registry, Git or local transport selected outside `.dever`. +3. `dever.mod` records direct dependency intent and source requirements. +4. `dever.lock` records the complete graph, full Git commits and normalized content hashes. +5. A global immutable content-addressed store deduplicates verified source across projects. +6. `check`, `build` and `run` never perform implicit network access; explicit `add`, `update` and `fetch` do. +7. Package declarations must remain beneath the component root, preventing one dependency from supplying another component's namespace. + +This deliberately combines Go's small module/package model and shared verified cache with Cargo's source flexibility and exact Git locking, while avoiding Go's coupling between source-level identity and repository location. diff --git a/.trellis/tasks/09-04-dever-language-mvp/research/runtime-dependencies.md b/.trellis/tasks/09-04-dever-language-mvp/research/runtime-dependencies.md new file mode 100644 index 0000000..4dbeee0 --- /dev/null +++ b/.trellis/tasks/09-04-dever-language-mvp/research/runtime-dependencies.md @@ -0,0 +1,55 @@ +# Core 0.1 Runtime Dependency Research + +- Checked: 2026-09-03 +- Scope: dependencies required by already-approved Decimal and Map semantics + +## Decimal128 + +Selected for the MVP: `dec = "=0.4.11"`. + +Evidence: + +- The [dec crate documentation](https://docs.rs/dec/latest/dec/) describes `Decimal128` as a 128-bit decimal representation with 34 digits of precision and exposes a configurable `Context` with rounding and exceptional-condition status. +- The [Decimal128 API](https://docs.rs/dec/latest/dec/struct.Decimal128.html) supports the required arithmetic and classification operations. + +Reason: + +- It directly implements the approved decimal128 contract instead of approximating it with a lower-precision financial decimal. +- Context status lets Dever convert overflow, division by zero and invalid Decimal states into language-level runtime faults. +- A reusable Context avoids the per-operation default-context overhead documented by the crate. + +Constraint: + +- `dec` is a safe Rust wrapper over libdecnumber and therefore introduces a native C build dependency. It must remain isolated behind `dever_runtime::number::DecimalValue`. +- Dever conformance tests, not the crate's operator defaults or special-value surface, define public behavior. +- A future pure-Rust replacement is allowed only when it passes the same conformance suite; this does not change `.dever` semantics. + +Rejected for Core 0.1: + +- `rust_decimal`: pure Rust, but its 96-bit coefficient does not provide the approved 34-digit decimal128 contract. +- Hand-written decimal arithmetic: correct rounding, exponent handling and status behavior are too risky for the language MVP. +- Newly published decimal128 implementations: potentially useful later, but less established than the selected reference implementation. + +## Insertion-Ordered Map + +Selected for the MVP: `indexmap = "=2.14.0"`. + +Evidence: + +- The [IndexMap project](https://github.com/indexmap-rs/indexmap) documents hash-based lookup with iteration independent of hash order. +- The [IndexMap API](https://docs.rs/indexmap/latest/indexmap/map/struct.IndexMap.html) documents that insertion appends new keys, overwriting preserves the existing position, and order-preserving removal is available. + +Reason: + +- Those operations match the approved Dever Map contract without maintaining a duplicate vector-plus-index implementation. +- Lookup remains hash based while iteration is deterministic. + +Constraint: + +- Dever wraps IndexMap and exposes only `get`, `put`, `remove` and `entries`. +- Removal uses the order-preserving operation rather than swap removal. +- Dependency-specific index access never appears in `.dever` or public diagnostics. + +## Dependency Discipline + +These are planned pins; the initial parser milestone has no external dependencies. Verify availability and pin each version when the runtime first consumes it. No dependency becomes a public language contract. Upgrading either dependency requires running the focused Decimal/Map conformance tests and confirming that serialized/displayed Dever values remain unchanged. diff --git a/.trellis/tasks/09-04-dever-language-mvp/task.json b/.trellis/tasks/09-04-dever-language-mvp/task.json new file mode 100644 index 0000000..b55f2bf --- /dev/null +++ b/.trellis/tasks/09-04-dever-language-mvp/task.json @@ -0,0 +1,38 @@ +{ + "id": "dever-language-mvp", + "name": "dever-language-mvp", + "title": "Dever Language Compiler Contracts And Library Follow-through", + "description": "N5 HTTP/2、定向性能矩阵与有界稳态/恢复压测、typed ORM 及双源码 CMS 合同修正已完成；业务 API 等用户重新设计，编译器发行/跨平台与自举仍未交付，根语言路线图保持进行中。", + "status": "in_progress", + "dev_type": null, + "scope": ".", + "package": null, + "priority": "P1", + "creator": "Antigravity", + "assignee": "Antigravity", + "createdAt": "2026-09-04", + "completedAt": null, + "branch": "main", + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [ + "09-12-runtime-performance", + "09-12-runtime-performance-p2", + "09-12-http2", + "09-12-http2-performance", + "09-13-http2-ceiling-soak", + "09-13-typed-orm", + "09-18-cms-dual-source-contracts", + "09-19-application-testing" + ], + "parent": null, + "relatedFiles": [], + "notes": "2026-09-18: HTTP/2 有界饱和扫描、128/64 MiB 各 60 秒稳态与各 100 轮重连验收完成；typed ORM 子任务也已完成。原始结果见 test/performance/README.md，历史阶段证据见 implement.md。根任务仍需单独界定发行/跨平台、自举等后续路线；未运行全量测试、操作现有服务、安装全局命令或提交。", + "meta": { + "planning_workflow": "trellis", + "execution_mode": "inline" + } +} \ No newline at end of file diff --git a/.trellis/tasks/09-06-private-compiler-distribution/check.jsonl b/.trellis/tasks/09-06-private-compiler-distribution/check.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/.trellis/tasks/09-06-private-compiler-distribution/implement.jsonl b/.trellis/tasks/09-06-private-compiler-distribution/implement.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/.trellis/tasks/09-06-private-compiler-distribution/prd.md b/.trellis/tasks/09-06-private-compiler-distribution/prd.md new file mode 100644 index 0000000..45b389a --- /dev/null +++ b/.trellis/tasks/09-06-private-compiler-distribution/prd.md @@ -0,0 +1,72 @@ +# Dever 私有编译器与公开发行边界 + +## Goal + +让用户只接触 Dever 语言、工具链和发行物，不接触或依赖 Rust/Cargo/rustc；Rust 仅作为私有仓库中的首代引导实现，不进入公开源码、安装流程、命令输出或用户运行环境。 + +## Background + +- 当前 `/data/project/dever` 没有配置 Git 远端，Cargo workspace、Rust 源码、测试和 README 尚未提交，因此没有证据表明实现已经对外发布。 +- 当前仓库及 Git 历史包含 Rust 实现信息，不能直接转换为公开仓库。 +- 当前 `deverc run/build` 调用用户环境中的 `rustc`，并在临时目录生成 Rust 源码；这不符合对外隐藏实现和免 Rust 工具链的要求。 +- 二进制实现无法可靠抵抗专业逆向识别。本任务隐藏正常产品表面和源码实现，不承诺不可逆向。 +- 当前 Rust 编译器及计划使用的 LLVM 后端都有必须合规审查的第三方许可证；不能通过删除或伪造法定声明隐藏实现。 + +## Requirements + +### Repository Boundary + +- 当前编译器仓库保持私有，不向公开仓库推送现有 Git 历史、Cargo 文件、Rust 源码或内部设计资料。 +- 对外发行使用一个没有当前仓库历史的新仓库或等价的独立公开渠道。 +- 公开内容只包含 Dever 用户需要的文档、`.dever` 示例、安装入口和构建后的发行物，不包含编译器实现源码。 + +### Product Surface + +- 用户只使用正式的 `dever` 命令和 `.dever` 源码，不需要了解 Cargo、Rust、rustc 或内部生成语言。 +- 公开文档、安装器、帮助文本、普通诊断、日志和临时文件不得暴露 Rust/Cargo/rustc 实现细节。 +- `dever check/run/build` 不要求用户安装 Rust 工具链，也不得调用用户 PATH 中的 `rustc` 或 `cargo`。 +- `dever build` 生成的程序可独立运行，不依赖 Cargo、Rust 工具链或私有编译器源码。 +- 更换公开发行边界和原生后端不得改变已经确认的 `.dever` 表面语法、诊断位置或当前可执行语义。 + +### Native Backend + +- Rust 可以继续用于私有编译器本身，但用户侧原生编译路径必须由随 `dever` 交付的内置后端完成。 +- 不采用“把 rustc 改名或静默捆绑”来满足隐藏要求；底层工具链不能作为可观察的用户侧子进程或安装前置条件。 +- 编译必须在用户本机离线完成，不上传 `.dever` 源码或中间表示，也不依赖远程构建节点。 +- 不采用云端编译来回避本地后端，因为这会额外引入源码上传、网络依赖和服务可用性边界。 + +### Initial Platform Boundary + +- 首版支持 Linux、Windows 和 macOS；所有编译均在用户本机离线完成。 +- Linux、Windows 和 macOS 均提供 x86_64 与 ARM64 版 `dever`。 +- 任一受支持宿主上的 `dever build` 都必须能够离线生成 Linux、Windows 和 macOS 的 x86_64/ARM64 程序，形成 6 个宿主发行物到 6 个目标的完全对称矩阵。 +- 生成跨平台程序不能依赖远程构建节点或上传源码及中间表示。 +- 目标后端、链接能力和 Dever 平台运行时随发行物提供，不能要求用户另行安装目标平台编译器或 SDK。 +- `dever build` 为 macOS 目标生成平台运行所需的离线 ad-hoc 链接签名；它不代表开发者身份，也不能通过 Gatekeeper 分发验证。 +- Developer ID 身份签名和 Apple 公证是独立的可选发布阶段，不属于 `dever build`；Developer ID 安全时间戳和 Apple 公证均允许并必然需要联网。 + +## Acceptance Criteria + +- [ ] 从一台没有 `cargo` 和 `rustc` 的目标机器安装公开发行物后，可以执行当前里程碑对应的 `dever check/run/build`。 +- [ ] 安装、帮助、正常运行、普通错误和构建临时目录中均不出现 Rust、Cargo、rustc 或 `.rs` 源码。 +- [ ] `dever build` 的输出是可独立运行的目标平台程序，并保持当前 `.dever` 示例的可观察行为。 +- [ ] 公开仓库或公开发行包不包含当前私有仓库的 Git 历史、Cargo metadata、Rust 源码或私有 Trellis 资料。 +- [ ] 自动检查能够证明公开构建路径没有启动 `cargo` 或 `rustc`，并覆盖没有 Rust 工具链的干净环境。 +- [ ] 断网且未安装 Windows 编译工具链的 Linux 环境可以生成 Windows 程序，并在真实 Windows 环境验证其行为。 +- [ ] 断网且未安装 Xcode 或 Apple SDK 的受支持环境可以生成 macOS 程序，并在真实 Intel Mac 和 Apple Silicon Mac 上验证其行为。 +- [ ] 六个宿主发行物分别能够生成六个目标程序，36 种宿主到目标组合使用同一命令合同并通过确定性产物检查；每个目标程序均在对应真实系统与架构上执行验证。 +- [ ] macOS ARM64 产物包含可离线生成的有效 ad-hoc 链接签名；编译不访问开发者身份或公证服务。 +- [ ] 单独的发布流程明确区分 ad-hoc 链接签名、需要联网安全时间戳的 Developer ID 身份签名和需要联网的 Apple 公证。 +- [ ] 私有仓库仍可维护和验证 Rust 引导编译器，公开边界不迫使内部源码改名或制造重复实现。 + +## Out Of Scope + +- 阻止专业人员通过反汇编、二进制指纹或运行时特征推断实现语言。 +- 为隐藏实现而混淆、加壳或伪造二进制来源。 +- 在本任务中实现 HTTP、JSON、数据库、CRUD 或完整 Dever 语言路线。 +- 公开编译器源代码或建立外部贡献者的编译器开发环境。 +- 在 `dever build` 中管理用户的 Developer ID 凭据、执行身份签名或隐式提交 Apple 公证。 + +## Open Question + +- 是否允许依法必须提供的第三方许可证清单提及 Rust；如果不允许，则 Rust 引导编译器只能作为私有 seed，Dever 自举编译器必须成为首次公开发行的前置条件？ diff --git a/.trellis/tasks/09-06-private-compiler-distribution/research/cross-compilation.md b/.trellis/tasks/09-06-private-compiler-distribution/research/cross-compilation.md new file mode 100644 index 0000000..cf9dda2 --- /dev/null +++ b/.trellis/tasks/09-06-private-compiler-distribution/research/cross-compilation.md @@ -0,0 +1,35 @@ +# Linux Host Cross-Compilation + +## Confirmed Toolchain Facts + +- LLVM is designed as a cross-compiler, but target code generation alone is insufficient. A complete build also needs target-specific headers, libraries, linker behavior and usually a sysroot: <https://clang.llvm.org/docs/CrossCompilation.html>. +- LLD supports ELF, Windows PE/COFF and macOS Mach-O. Its PE/COFF port is described as complete; Mach-O support exists but is listed after ELF and PE/COFF in completeness: <https://lld.llvm.org/>. +- Microsoft publishes the PE/COFF executable and object format specification, so a Linux-hosted backend can produce Windows artifacts without invoking a Windows compiler: <https://learn.microsoft.com/en-us/windows/win32/debug/pe-format>. +- Apple's documented universal-binary workflow requires Xcode on a Mac and uses Apple target tooling. Distributed macOS software also has Developer ID signing and notarization requirements: <https://developer.apple.com/documentation/apple-silicon/building-a-universal-macos-binary> and <https://developer.apple.com/documentation/security/notarizing-macos-software-before-distribution>. +- Apple exposes a Notary REST API, so submission need not use `notarytool`, but this does not remove SDK, linking, signing or real-device validation concerns: <https://developer.apple.com/documentation/NotaryAPI>. + +## Product Consequences + +### Linux to Windows + +This is a practical offline target. A target-neutral Dever frontend can feed an embedded native backend, then a PE/COFF linker can combine generated code with a versioned Dever Windows runtime and required import libraries. The resulting `.exe` still needs validation on real x86_64 and ARM64 Windows systems. + +### Linux to macOS + +Generating x86_64 or ARM64 instructions and a Mach-O file on Linux is technically possible. Treating that file as a supported macOS product is harder: platform libraries, minimum OS versions, Mach-O signing details, Developer ID distribution, notarization and testing remain Apple-platform concerns. Redistributing an Apple SDK inside the Linux compiler also introduces a licensing boundary and must not be assumed acceptable. + +For the current Text/println subset, a self-contained Dever runtime and purpose-built Mach-O path could avoid most SDK use. That shortcut does not establish a maintainable path for the planned HTTP, TLS, files, database and other system integrations. + +Local compilation and public distribution are separate contracts. Once the target backend and versioned Dever macOS runtime are installed, `dever build` can remain offline. Apple Silicon requires every executable to be signed, so the linker must create an identity-free ad-hoc signature as part of offline output: <https://developer.apple.com/documentation/macos-release-notes/macos-big-sur-11_0_1-universal-apps-release-notes>. Developer ID distribution signing requires the developer's identity and an online secure timestamp: <https://developer.apple.com/documentation/technotes/tn3161-inside-code-signing-certificates>. Apple's notarization service then requires uploading the signed artifact. Apple exposes a REST API for non-macOS notarization clients, but the service is necessarily online: <https://developer.apple.com/documentation/NotaryAPI>. + +## Recommended Boundary + +- Keep parsing, checking, HIR and target selection shared. +- Support offline Linux-to-Windows cross-build as a normal target once the embedded backend exists. +- Keep all first-release compilation local and offline; do not transfer source or intermediate representations to a build service. +- Include macOS as an offline compilation target without introducing controlled Mac build infrastructure. Treat Developer ID signing and Apple notarization as separate optional publishing stages, and require real Intel and Apple Silicon validation for supported Mach-O output. +- Implement each target through small ABI/runtime/object-format adapters rather than duplicating compiler flows. + +## Open Product Decision + +Resolved: the first release uses a fully symmetric Linux, Windows and macOS x86_64/ARM64 host-to-target matrix. Six host distributions must each produce all six targets locally and offline. diff --git a/.trellis/tasks/09-06-private-compiler-distribution/research/distribution-licensing.md b/.trellis/tasks/09-06-private-compiler-distribution/research/distribution-licensing.md new file mode 100644 index 0000000..d4ad083 --- /dev/null +++ b/.trellis/tasks/09-06-private-compiler-distribution/research/distribution-licensing.md @@ -0,0 +1,23 @@ +# Distribution Licensing And Implementation Disclosure + +## Confirmed Facts + +- The Rust project and standard library are distributed under `MIT OR Apache-2.0`: <https://rust-lang.org/policies/licenses/> and <https://github.com/rust-lang/rust/blob/main/COPYRIGHT>. +- A public compiler binary built from the current Rust implementation can contain Rust standard-library code. Its exact notice obligations require a release-license audit; the project must not assume that deleting all Rust attribution is permitted. +- LLVM is `Apache-2.0 WITH LLVM-exception`. LLVM states that binaries including LLVM must reproduce its copyright notice, while permitting closed-source and commercial use: <https://llvm.org/docs/DeveloperPolicy.html#copyright-license-and-patents>. +- Apple licenses prohibit separately using Apple SDKs or running Apple Software on non-Apple-branded hardware, and generally prohibit redistributing Apple Software in whole or part without permission: <https://www.apple.com/legal/sla/docs/xcode.pdf>. + +## Consequences + +Hiding Cargo files, symbols and implementation-specific diagnostics is a product-surface concern. Required third-party notices are a separate compliance concern and must remain truthful. + +There are two viable disclosure boundaries: + +1. Ship the Rust bootstrap compiler initially. Normal product surfaces mention only Dever, but a compliant third-party notice may identify Rust and LLVM. Professional binary identification is already outside the task scope. +2. Never ship the Rust bootstrap compiler publicly. Use it only as the private stage-1 seed, complete enough Dever language and runtime capabilities to build a stage-2 compiler, and publicly distribute the verified stage-2/stage-3 Dever compiler. LLVM attribution can remain if LLVM is embedded, but Rust is absent from the public executable and notices. + +The second boundary matches a strict “no public Rust trace” requirement but makes self-hosting a release prerequisite rather than a later roadmap item. It materially expands the first public release. + +## Rule + +Do not use renamed tools, stripped license files, misleading notices, obfuscation or packaging tricks as a substitute for the chosen disclosure boundary. diff --git a/.trellis/tasks/09-06-private-compiler-distribution/research/native-backend-options.md b/.trellis/tasks/09-06-private-compiler-distribution/research/native-backend-options.md new file mode 100644 index 0000000..2de76f4 --- /dev/null +++ b/.trellis/tasks/09-06-private-compiler-distribution/research/native-backend-options.md @@ -0,0 +1,104 @@ +# Native Backend Options + +## Required Properties + +- Consume checked Dever HIR without changing `.dever` syntax or semantics. +- Generate x86_64 and ARM64 ELF, PE/COFF and Mach-O on every supported host. +- Run fully offline without Cargo, rustc, a target SDK or a remote build service. +- Preserve direct native calls, concrete value layouts and the existing native performance direction. +- Keep generated IR, objects and link details private, owned temporary artifacts. +- Avoid six target-specific compiler implementations. + +## Options + +| Option | Strengths | Blocking weaknesses | Decision | +| --- | --- | --- | --- | +| Pinned LLVM code generation plus LLD | Mature optimization; x86_64/ARM64; ELF, PE/COFF and Mach-O; supports in-process use | Large host binaries, native integration boundary, mandatory LLVM notices, target runtimes still required | Recommended | +| Cranelift plus LLD | Safe Rust-facing library, fast compilation, x86_64/ARM64 | Still needs a cross-format linker and target runtimes; optimization quality creates risk against the native performance contract | Keep as fallback only if measured LLVM integration cost is unacceptable | +| Custom machine-code and executable writers | Full control and small artifacts for the current subset | Reimplements two ISAs, three formats, relocation, unwind and linking; duplicates platform logic and does not scale to the language roadmap | Reject | +| WebAssembly or a bytecode VM inside native launchers | Simple portable execution model | Changes the AOT/runtime architecture, adds a production VM and jeopardizes native performance requirements | Reject for production build/run | +| Generated Rust plus bundled or renamed rustc | Reuses current implementation | Exposes the forbidden toolchain, produces Rust temporary source and makes offline target support depend on Rust sysroots/linkers | Replace completely | + +LLVM's cross-compilation documentation still requires target-specific sysroots/libraries beyond machine-code generation: <https://clang.llvm.org/docs/CrossCompilation.html>. LLD supports the three required executable formats and can be embedded for compiler-owned trusted objects: <https://lld.llvm.org/> and <https://lld.llvm.org/NewLLD.html>. + +## Proposed Ownership + +```text +dever-core + checked, target-neutral HIR and source diagnostics + +dever-native + fixed target catalog + HIR -> LLVM lowering + object emission + LLD integration + target-pack selection + owned temporary artifacts + +dever-runtime + Dever-owned ABI and runtime behavior + six release-built target packs + +dever-cli + --target parsing, host/default selection, output and process adaptation +``` + +`dever-native` is one justified private workspace boundary: it isolates the large native dependency and target mechanics from parsing/checking. It is not a public backend plugin API. Target variation uses one explicit catalog and small data/config adapters, not inheritance or a registry. + +## Build Data Flow + +```text +.dever -> check -> typed HIR -> specialize reachable program + -> shared LLVM lowering -> target object in memory + -> fixed target specification + versioned Dever runtime pack + -> embedded LLD driver -> new executable output +``` + +- LLVM IR and object files are never public output formats. +- `dever build` creates a new output and keeps the current no-overwrite contract. +- `dever run` always builds the host target and runs it; cross-target execution/emulation is not added. +- A cross-target `build` only generates an artifact. Real target runners validate it in release CI. + +## Target Catalog + +The public catalog has exactly six initial targets: + +```text +linux-x86_64 +linux-arm64 +windows-x86_64 +windows-arm64 +macos-x86_64 +macos-arm64 +``` + +Each target definition owns its LLVM triple, CPU baseline, object format, calling convention, executable suffix, runtime pack and deterministic linker settings. Shared OS and architecture helpers remove repetition; target entries remain explicit and reviewable. + +## Runtime Packs + +Generated code must use a Dever-owned ABI rather than Rust's unstable ABI or Rust `String` layout. Each target pack contains only Dever-owned startup/runtime objects and permitted platform import metadata. It must not contain Rust standard-library objects, Apple SDK files or target compiler binaries. + +The current `dever-runtime` source is a behavior reference, not a stable native ABI. The first backend milestone defines the minimal Text slice, result/fault and `println` ABI once, then supplies six compiled implementations or target-specific adapters behind that contract. Future collections, numbers, HTTP, TLS and storage extend this ABI deliberately rather than reaching into host-language layouts. + +## Determinism And Matrix Verification + +- Pin one LLVM/LLD source revision and runtime-pack version for all six host builds. +- Remove timestamps, absolute build paths, random UUID inputs and host-dependent ordering. +- Each of six host compilers builds all six targets, producing 36 artifacts. +- Artifacts for the same target and source should be byte-identical across hosts. Hash equality reduces execution validation to one artifact per target; all six target artifacts still run on real matching systems. +- Offline checks deny network access and remove Cargo, rustc, Xcode, Visual Studio, GCC and target SDKs from PATH. + +## Release And License Gates + +- LLVM permits proprietary distribution but requires its copyright notice in binaries that include it: <https://llvm.org/docs/DeveloperPolicy.html#copyright-license-and-patents>. +- No Apple SDK file may enter a target pack. Apple's agreement prohibits separately using SDKs on non-Apple hardware and redistributing Apple Software in part: <https://www.apple.com/legal/sla/docs/xcode.pdf>. +- Before implementation commits to the macOS target, prove that both Mach-O targets can link the minimal Dever runtime without redistributed Apple SDK material and pass real-machine execution. +- The public-compiler Rust attribution decision in `distribution-licensing.md` remains a product gate. + +## Rejected Shortcuts + +- Do not rename or conceal rustc, Cargo, LLD or license files. +- Do not special-case the hello example or patch prebuilt executable templates. +- Do not implement one backend per OS. +- Do not upload source or HIR when local target support is missing. +- Do not silently fall back to a VM, host execution or another target. diff --git a/.trellis/tasks/09-06-private-compiler-distribution/task.json b/.trellis/tasks/09-06-private-compiler-distribution/task.json new file mode 100644 index 0000000..9c7de7d --- /dev/null +++ b/.trellis/tasks/09-06-private-compiler-distribution/task.json @@ -0,0 +1,28 @@ +{ + "id": "private-compiler-distribution", + "name": "private-compiler-distribution", + "title": "Dever 私有编译器与公开发行边界", + "description": "隐藏 Rust 引导实现并建立不依赖 Rust 工具链的公开发行路径", + "status": "planning", + "dev_type": null, + "scope": null, + "package": null, + "priority": "P2", + "creator": "Antigravity", + "assignee": "Antigravity", + "createdAt": "2026-09-06", + "completedAt": null, + "branch": null, + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [ + "09-28-shared-toolchain" + ], + "parent": null, + "relatedFiles": [], + "notes": "", + "meta": {} +} \ No newline at end of file diff --git a/.trellis/tasks/09-12-http2-performance/check.jsonl b/.trellis/tasks/09-12-http2-performance/check.jsonl new file mode 100644 index 0000000..5b9a0a0 --- /dev/null +++ b/.trellis/tasks/09-12-http2-performance/check.jsonl @@ -0,0 +1,3 @@ +{"file":".trellis/spec/backend/toolchain-and-library.md","reason":"检查 HTTP/2 配置与运行时公开合同一致"} +{"file":".trellis/spec/backend/quality-guidelines.md","reason":"检查测量公平性、计数完整性、资源预算和验证范围"} +{"file":".trellis/tasks/09-12-http2-performance/research/current-benchmark-and-h2-api.md","reason":"核对实现是否真正拆分物理连接和并发流"} diff --git a/.trellis/tasks/09-12-http2-performance/design.md b/.trellis/tasks/09-12-http2-performance/design.md new file mode 100644 index 0000000..2a7be5a --- /dev/null +++ b/.trellis/tasks/09-12-http2-performance/design.md @@ -0,0 +1,84 @@ +# 设计 + +## 1. 边界与复用 + +继续使用 `test/performance` 的“构建与测量分离”结构：`build.py` 只生成并指纹化产物，`run.py` 只启动清单内的二进制并采样。进程、cgroup、直方图、逐 case 落盘和失败保留逻辑全部复用。新增的唯一核心抽象是 HTTP 连接拓扑： + +```text +ConnectionShape + physical_connections + streams_per_connection + request_slots = physical_connections * streams_per_connection +``` + +HTTP/1.1 要求 `streams_per_connection = 1`；HTTP/2 要求复用连接。这个模型避免继续把并发请求与 TCP 连接混为一个 `connections` 参数，也让报告能解释吞吐和内存差异来自哪里。 + +## 2. Rust 基准对端与负载端 + +`network_bench` 的统一负载入口调整为： + +```text +network_bench load h1|h2 URL PATH RATE SECONDS PHYSICAL_CONNECTIONS STREAMS_PER_CONNECTION TIMEOUT_MS REUSE CA +``` + +所有仓库调用者一次性迁移，不保留第二套旧参数解析。 + +- HTTP/1.1 保留现有“一名 worker 独占一条连接”的实现，并要求 streams 为 1。 +- HTTP/2 在计时前主动建立指定数量的物理连接；每条连接由 Hyper HTTP/2 client driver 持有，`SendRequest` 按该连接的请求槽克隆给 worker。总 worker 数为 shape 的乘积，worker 固定映射到连接，不额外建立连接。 +- HTTP/2 连接中断后不重试请求或静默补建连接。失败必须进入错误计数，避免恢复逻辑污染吞吐数据。 +- 两种协议共享现有速率调度、队列、响应校验、延迟直方图和结果守恒逻辑，不复制 metrics/validation。 +- 结果新增 `http_version`、`physical_connections`、`streams_per_connection`、`request_slots` 和 `successful_handshakes`。H2 的握手数必须等于物理连接数。 + +H2 请求使用完整 scheme/authority/path 和 `Version::HTTP_2`。明文使用 prior knowledge；TLS client/server 配置按协议派生 ALPN，H2 只允许 `h2`，不改变运行时用于 WSS 的共享 TLS 配置。 + +服务模式新增 `runtime-http2`、`runtime-https2`、`hyper-http2`、`hyper-https2`。`ServiceKind` 集中派生 runtime/TLS/H2 三个维度，避免模式分支散落。直接 runtime 使用 `Http2Limits`；原生 Hyper 使用 `http2::Builder<TokioExecutor>`，显式配置与 N5 相同的 streams、窗口、header/frame/send-buffer 上限。两者继续复用同一响应路由和连接生命周期。 + +## 3. Dever 夹具与构建清单 + +HTTP 夹具暴露 `http`、`https`、`http2`、`https2` 四个入口。`serve` 接收 `dever.http.Limits` 参数，四个 worker 只负责选择 TLS 和协议配置，避免复制路由与关闭流程。 + +`build.py` 增加 HTTP/2 构建参数并写入 manifest： + +- `http2_streams = 16` +- `http2_stream_window_bytes = 65535` +- `http2_connection_window_bytes = 262144` + +`config.dever` 分别生成 HTTP/1.1 和 HTTP/2 limits；`task_capacity` 继续按有界并发计算，并校验正式拓扑的总请求槽不超过语言上限。旧产物目录和报告不修改，新构建必须使用新的输出目录。 + +## 4. Python 运行器 + +新增两个显式 suite： + +- `http2`：吞吐、延迟与活跃资源矩阵。 +- `http2-recovery`：同一服务进程上的多轮连接建立、请求和断开。 + +`--http2-connections` 接受物理连接档位，默认 `1,4`；每连接流数从 manifest 读取。`--implementations` 可将 smoke 或 64 MiB 场景限制为 `dever`，正式矩阵默认为 `dever,runtime,hyper`。已有 `http` suite 的 `--concurrency` 和连接新建/复用语义不变。 + +吞吐 case 的流程为：启动服务并取得 READY，运行独立 warmup，等待连接释放并采样基线，运行正式负载，同时采样服务端与客户端；负载退出后再次采样以记录 FD/RSS 回落。case 名和 JSON 主键包含实现、传输、路径、速率、物理连接、每连接流和 repeat，避免结果覆盖或混淆。 + +恢复 case 使用独立参数 `--recovery-connections`、`--recovery-rate`、`--recovery-path`，复用 `--cycles` 与 `--duration`。每轮启动一次负载进程，在同一服务上建立固定 shape，完成请求后关闭连接；运行器等待稳定窗口并保存该轮末样本。FD 不回到 READY 基线、请求错误、握手数不符或服务提前退出均使 case 失败。RSS 只报告趋势，不以“没有立即回到启动值”单独判断泄漏。 + +报告继续保留 `cases.jsonl` 和 `report.json`，并在完整或部分结果上生成结构化 `summary`。摘要按实现、协议、TLS、路径、速率和 topology 分组，对成功 QPS、p99、RSS/PSS/FD 峰值取各 repeat 中位数；不把不同 offered rate 或饱和/未饱和 case 混合。 + +## 5. 测量矩阵 + +正式测量固定服务端 1 worker、1 CPU、128 MiB cgroup、swap=0；客户端绑定另一 CPU且位于服务预算之外。每组 0.5 秒预热、2 秒正式负载、3 个独立服务进程 repeat： + +| 正文 | offered rate | topology | 实现 | 传输 | +| --- | --- | --- | --- | --- | +| `/plain` 5 B | 5k / 15k / 30k req/s | 1x16、4x16 | Dever/runtime/Hyper | h2c、TLS+h2 | +| `/bytes` 64 KiB | 100 / 500 / 1k req/s | 1x16、4x16 | Dever/runtime/Hyper | h2c、TLS+h2 | + +另运行 10 轮、1x16、`/plain` 的恢复矩阵，以及 64 MiB 下 Dever 的 1x16 明文/TLS 活跃代表场景。若当前机器不能创建指定 cgroup，工具必须明确失败；不得回退为无预算结果后仍标记完成。 + +## 6. 解释与决策 + +- 主要比较同一 H2 topology 下 Dever、runtime 和 Hyper 的差异，定位语言桥接、运行时结构化任务和底层协议栈各自成本。 +- 1x16 与 4x16 比较多路复用和多物理连接扩展；它们不是“连接数相同”的横向比较。 +- `/bytes` 的速率按响应字节量降低，避免只测客户端或 loopback 带宽耗尽；65536 字节正文会越过 65535 字节默认单流窗口，可观察 WINDOW_UPDATE 路径。 +- `dropped_capacity` 表示负载端无法在计划时间取得请求槽，不算服务错误；分析时必须同时看 actual QPS、请求延迟和服务 CPU，不能只看 offered rate。 +- 不设预定胜负线。若 Dever 相对 direct runtime 出现稳定的大幅差距，或内存随 recovery 周期持续增长，则把证据和复现命令交给后续独立优化任务。 + +## 7. 回退形态 + +本阶段不迁移生产 API。若 H2 基准实现无法通过独立 smoke，删除新增 suite、服务模式和夹具入口即可回到原有 H1 基准；历史产物目录不受影响。任何正式测量都写入新的 `target/performance/*` 目录，失败结果保留且不覆盖旧报告。 diff --git a/.trellis/tasks/09-12-http2-performance/implement.jsonl b/.trellis/tasks/09-12-http2-performance/implement.jsonl new file mode 100644 index 0000000..b330ae6 --- /dev/null +++ b/.trellis/tasks/09-12-http2-performance/implement.jsonl @@ -0,0 +1,3 @@ +{"file":".trellis/spec/backend/toolchain-and-library.md","reason":"HTTP/2 streams、窗口、ALPN、结构化任务和资源预算合同"} +{"file":".trellis/spec/backend/quality-guidelines.md","reason":"独立二进制基准、cgroup、报告和定向验证规则"} +{"file":".trellis/tasks/09-12-http2-performance/research/current-benchmark-and-h2-api.md","reason":"现有 H1 基准限制、锁定 Hyper API 和 H2 负载模型结论"} diff --git a/.trellis/tasks/09-12-http2-performance/implement.md b/.trellis/tasks/09-12-http2-performance/implement.md new file mode 100644 index 0000000..1f9e423 --- /dev/null +++ b/.trellis/tasks/09-12-http2-performance/implement.md @@ -0,0 +1,50 @@ +# 执行计划 + +## 1. 任务启动 + +- [x] 用户审核本 PRD、设计和执行计划后，运行 `task.py start`；启动前不改产品或基准源码。 +- [x] 使用 `trellis-before-dev` 重新加载 backend 规范，核对工作区状态和已有性能产物，不清理用户文件。 + +## 2. 基准协议模型 + +- [x] 在 `network_bench` 中加入 `h1|h2` 与 `ConnectionShape` 参数校验；H1 限制每连接一槽，H2 限制复用且乘积有界。 +- [x] 保留一个速率调度和 metrics 路径；把 H1 独占连接与 H2 共享 sender 封装为两个小的协议执行分支，不引入通用 trait 层级。 +- [x] H2 计时前建立固定数量物理连接，按连接分配 sender clone，关闭时排空/终止 driver；不重试、不自动扩容。 +- [x] 在结果中输出并校验协议、shape 和握手计数，继续保证全部请求计数守恒和完整正文校验。 + +## 3. 三类服务实现 + +- [x] 扩展 `ServiceKind` 和 TLS ALPN 派生，新增 runtime/raw Hyper 的 h2c 与 TLS+h2 模式。 +- [x] direct runtime 与 raw Hyper 使用相同的 streams、窗口、header/frame/send-buffer 约束；复用响应路由、accept 限额和关闭流程。 +- [x] 重构 Dever HTTP fixture，让四个入口共享 serve/route/lifecycle，只在入口选择协议和 TLS。 +- [x] 扩展构建配置与 manifest，生成并指纹化 `http2`、`https2`，保留 H1 产物。 + +## 4. 运行器与摘要 + +- [x] 增加 `http2`、`http2-recovery` suite、物理连接档位、实现筛选及恢复参数，并按 manifest 校验 capacity、streams 和窗口。 +- [x] HTTP/2 吞吐 case 记录 warmup、正式负载、断开后样本、服务/客户端资源和 cgroup 状态；case 标识包含完整维度。 +- [x] 恢复 case 在一个服务进程内运行多轮固定 shape，逐轮校验请求和握手并记录 FD/RSS 回落。 +- [x] 从逐 case 结果生成分组中位数 summary；失败时仍保存已完成 case 和可诊断日志。 + +## 5. 定向验证 + +- [x] 运行 `python3 -m unittest discover -s test/performance -p 'test_runner.py' -v`，覆盖参数、计数、summary、进程回收和错误路径。 +- [x] 离线 release 构建 `network_bench`，运行现有独立 Python H1 对端测试，确认一次性 CLI 迁移没有改变 H1 测量语义。 +- [x] 运行 H2 loopback smoke，覆盖 1 条物理连接、至少 16 个请求槽、明文/TLS、`/plain`/`/bytes`、三类服务模式和错误正文拒绝。 +- [x] 用新的输出目录构建 Dever 性能夹具，核对 manifest 指纹、HTTP/2 limits 和四个 HTTP 入口。 +- [x] 运行最小 `http2`/`http2-recovery` runner smoke，检查失败报告、FD 回落和原始计数完整性。 + +## 6. 有界正式测量 + +- [x] 先确认可用 CPU affinity、cgroup v2 cpu/memory 控制器、磁盘和已有服务边界，再说明将执行的有界命令。 +- [x] 运行 128 MiB `/plain` 矩阵：5k/15k/30k，1x16/4x16，三实现、h2c/TLS、3 repeats。 +- [x] 运行 128 MiB `/bytes` 矩阵：100/500/1000，1x16/4x16，三实现、h2c/TLS、3 repeats。 +- [x] 运行 128 MiB 10 轮恢复矩阵，并运行 64 MiB Dever 1x16 明文/TLS 活跃场景。 +- [x] 核对全部报告 status、errors、dropped、OOM、FD、RSS/PSS、CPU 和摘要分组；不以单次异常样本下结论。 + +## 7. 收尾审查 + +- [x] 更新 `test/performance/README.md` 和 `.trellis/spec/backend/quality-guidelines.md` 的 CLI、矩阵、实测结果和边界说明。 +- [x] 检查最终改动是否仍有重复协议分支、混合职责或无必要依赖；确认未修改生产运行时、旧报告和用户文件。 +- [x] 运行任务 JSONL 校验、Python 语法检查及受影响的最小回归；不运行全量测试。 +- [x] 记录实际通过、失败、跳过和不可用项；完成 Trellis 检查与会话记录，不提交 Git。 diff --git a/.trellis/tasks/09-12-http2-performance/prd.md b/.trellis/tasks/09-12-http2-performance/prd.md new file mode 100644 index 0000000..839a233 --- /dev/null +++ b/.trellis/tasks/09-12-http2-performance/prd.md @@ -0,0 +1,40 @@ +# HTTP/2 性能基准 + +N5 已完成 HTTP/2 服务端、客户端和连接池的功能接入，但现有独立二进制基准仍只测 HTTP/1.1。当前阶段为 HTTP/2 建立可重复的小机器性能证据，不凭功能测试或理论值宣称性能提升。 + +## 需求 + +- 扩展现有显式运行的独立二进制基准，新增 HTTP/2 suite；保留已有 HTTP/1.1、异步和长连接基准的语义及历史报告。 +- 覆盖明文 prior knowledge 和 TLS ALPN 两种 HTTP/2 传输，统一比较原生 Dever 程序、直接 `dever-runtime` 和原生 Hyper 三种实现。 +- 负载模型必须把物理 TCP 连接数与每连接并发流数分开。每个报告明确记录协议、TLS、物理连接数、每连接请求槽、总请求槽、服务端流/窗口配置和正文大小。 +- 至少覆盖单连接多流和多连接多流；请求必须校验状态码、Content-Type 和完整正文，所有计划、发送、完成、失败及丢弃请求均可核对。 +- 记录实际成功吞吐、总延迟/排队延迟/请求延迟的 p50/p95/p99，以及服务端和客户端 RSS、PSS、峰值、线程、FD、CPU 和 cgroup 内存/OOM 数据。 +- 同一服务进程上重复建立、使用和关闭 HTTP/2 连接，记录每轮断开后的 FD/RSS 差值，验证物理连接能够回收且内存不按轮持续增长。 +- 生成按实现、传输、路径、速率和连接拓扑分组的中位数摘要，同时保留逐 case 原始样本和失败时已完成的数据。 +- 完成 1 worker、1 服务 CPU、128 MiB、无 swap 的正式矩阵，并完成 64 MiB 下的 Dever 活跃 HTTP/2 代表场景；负载端使用独立 CPU且不计入服务预算。 + +## 约束 + +- 本任务只改基准夹具、基准对端、运行器、定向测试和对应性能文档；不修改 `crates/dever-runtime`、编译器或语言公共 API。基准揭示的生产瓶颈另建优化任务，避免边测边改导致基线失效。 +- 复用仓库锁定的 Hyper 1.11.1、h2 0.4.19、rustls 和现有进程/cgroup 采样器，不引入新的协议栈或压测依赖。 +- HTTP/2 明文只使用 prior knowledge；TLS 只协商 `h2`，不降级 HTTP/1.1。HTTP/1.1 每连接请求槽固定为 1，HTTP/2 只测复用连接，不测每请求新建连接。 +- 服务端与客户端固定使用 N5 默认值：每连接 16 流、65535 字节单流接收窗口、262144 字节连接接收窗口，以及运行时已有的 16 KiB frame/send-buffer 上限。 +- `/plain` 用于小响应调度和协议开销，`/bytes` 使用 65536 字节正文并跨越默认单流窗口；`/json` 不进入正式矩阵，因为当前夹具只返回预制 JSON，信息与小响应重复。 +- 不设与机器无关的绝对 QPS 或内存门槛。错误、OOM、计数不守恒和 FD 未回落属于失败；容量丢弃与相对性能只作为结果，不伪装成通过或失败阈值。 +- 仅运行本任务的 Python 单元测试、独立 loopback smoke、必要的离线 release 构建和显式有界性能命令；不运行全量测试，不替换全局二进制，不接触已有服务，不提交 Git。 + +## 验收标准 + +- [x] HTTP/1.1 旧入口仍通过现有独立 Python 对端测试；HTTP/2 smoke 证明一条实际物理连接承载至少 16 个并发请求槽，明文和 TLS 均完整成功。 +- [x] 构建清单包含 `http2`/`https2` Dever 产物、HTTP/2 配置及所有二进制指纹；运行阶段只使用清单中的已构建产物。 +- [x] `http2` suite 覆盖三种实现、两种传输、`/plain` 与 `/bytes`、`1 x 16` 与 `4 x 16` 两种拓扑，并保存可核对的逐 case 数据与聚合摘要。 +- [x] `http2-recovery` suite 在同一服务进程上完成至少 10 轮连接周期；每轮请求正确，断开后 FD 回到 READY 基线，且报告保留 RSS 变化序列。 +- [x] 128 MiB 正式矩阵中无请求错误、无 OOM；丢弃请求如实报告。64 MiB 代表场景无 OOM，并报告峰值而非只报告进程 RSS。 +- [x] 文档写明运行命令、环境、矩阵、限制和实测结论；不得将 HTTP/2 结果外推为所有后端负载或真实网络的性能保证。 +- [x] 定向测试覆盖参数校验、计数守恒、连接拓扑字段、摘要分组、H2 明文/TLS 正文校验、进程失败和连接回收；相关检查全部通过。 + +## 排除项 + +- RFC 8441 WebSocket-over-HTTP/2、h2c Upgrade、HTTP/3、服务器推送、非空 trailers、HTTP/2 客户端自动重试。 +- 小时级 soak、跨主机网络、丢包/限速故障注入、真实公网站点和多平台硬件矩阵。 +- 本阶段根据一次测量直接改写运行时热路径或承诺固定的 Fiber/Go 对比数字。 diff --git a/.trellis/tasks/09-12-http2-performance/research/current-benchmark-and-h2-api.md b/.trellis/tasks/09-12-http2-performance/research/current-benchmark-and-h2-api.md new file mode 100644 index 0000000..7599ded --- /dev/null +++ b/.trellis/tasks/09-12-http2-performance/research/current-benchmark-and-h2-api.md @@ -0,0 +1,31 @@ +# 现有基准与 HTTP/2 API 调研 + +## 现有结构 + +- `test/performance/build.py` 负责生成 Dever fixtures、复制 Rust peers、记录配置和 SHA-256；`run.py` 只执行已指纹化产物。这一分离可直接复用。 +- 当前 HTTP suite 覆盖 Dever、direct runtime、raw Hyper，但三者都只使用 HTTP/1.1。`config.dever` 明确写入 `http2 = null`。 +- 当前 loader 的 `connections` 同时决定 worker 数和连接数，每个 worker 串行持有一条 HTTP/1.1 连接；它无法证明 HTTP/2 多路复用。 +- 现有 runner 已有固定速率调度、三类延迟直方图、进程 RSS/PSS/FD/CPU 采样、cgroup memory/peak/OOM 采样、逐 case 落盘和异常清理，不应重写。 +- `live` suite 已有多轮连接后 FD/RSS 差值的报告方式，可复用其 phase 后采样语义，但 HTTP/2 负载和连接创建应继续由 `network_bench` 负责。 + +## N5 已有合同 + +- `Http2Limits` 的公开维度为 streams、stream window 和 connection window；默认分别为 16、65535、262144。 +- cleartext HTTP/2 使用 prior knowledge；TLS 配置必须协商 `h2`，不能降级或影响 WSS 的 HTTP/1.1 TLS 配置。 +- 客户端按流租约复用物理连接，服务端连接和流均受限；固定 16 KiB frame/send-buffer，关闭自适应窗口。 +- `/bytes` fixture 为 65536 字节，恰好跨过默认 65535 字节单流窗口，适合覆盖 flow-control 热路径。 + +## 锁定依赖可用能力 + +- Cargo.lock 固定 Hyper 1.11.1、h2 0.4.19、rustls 0.23.44 和 tokio-rustls 0.26.5，不需要新依赖。 +- Hyper HTTP/2 `SendRequest` 实现 `Clone`，可让多个请求 worker 共享同一物理连接的 driver。 +- Hyper HTTP/2 client/server builder 可显式设置 initial stream/connection window、adaptive window、max concurrent streams、max frame、max send buffer 和 header list size。 +- Hyper HTTP/2 request future 的取消会对单流发送 RST_STREAM；基准不需要自己实现帧协议。 + +## 设计结论 + +1. 物理连接与请求槽必须拆开；只给现有 loader 换 HTTP/2 handshake 会得到错误的“一 worker 一连接”模型。 +2. H2 连接应在计时前主动建立，worker 固定映射到已建 sender；这样报告的物理连接数由 loader 自己控制和证明。 +3. H1 和 H2 共享调度、计数和响应校验，但连接执行策略分开。不存在足够稳定的公共行为值得引入 trait 或继承式抽象。 +4. raw Hyper 对照必须显式复制 N5 的协议参数；使用 Hyper 默认值会把配置差异误判为 Dever/runtime 开销。 +5. 性能任务不同时修改生产 runtime。测量发现的瓶颈需要在固定基线后另立任务验证，否则优化前后结果不可复现。 diff --git a/.trellis/tasks/09-12-http2-performance/task.json b/.trellis/tasks/09-12-http2-performance/task.json new file mode 100644 index 0000000..6fb6010 --- /dev/null +++ b/.trellis/tasks/09-12-http2-performance/task.json @@ -0,0 +1,26 @@ +{ + "id": "http2-performance", + "name": "http2-performance", + "title": "HTTP/2 performance benchmarks", + "description": "扩展现有独立二进制基准，测量小内存下 HTTP/2 多路复用的吞吐、尾延迟、内存和连接回收。", + "status": "completed", + "dev_type": null, + "scope": "test/performance + test/dever-tests/examples/network_bench* + performance docs/spec", + "package": null, + "priority": "P2", + "creator": "Antigravity", + "assignee": "Antigravity", + "createdAt": "2026-09-12", + "completedAt": "2026-09-13", + "branch": "main", + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [], + "parent": "09-04-dever-language-mvp", + "relatedFiles": [], + "notes": "HTTP/2 benchmark complete: 224 formal cases plus 60 recovery cycles, fixed 1x16/4x16 topology, h2c/TLS and Dever/runtime/Hyper comparisons. All request errors, OOM and FD deltas were zero; 64 MiB active Dever cases passed. Sixteen runner tests, three loopback tests, release cargo check, Dever fmt-check and independent review passed. No full suite, Rustfmt/Clippy, commit or archive.", + "meta": {} +} diff --git a/.trellis/tasks/09-12-http2/check.jsonl b/.trellis/tasks/09-12-http2/check.jsonl new file mode 100644 index 0000000..02213de --- /dev/null +++ b/.trellis/tasks/09-12-http2/check.jsonl @@ -0,0 +1,4 @@ +{"file":".trellis/spec/backend/toolchain-and-library.md","reason":"检查 HTTP2 的结构化归属、资源容量和桥接"} +{"file":".trellis/spec/backend/quality-guidelines.md","reason":"定向测试和无全量测试约束"} +{"file":".trellis/tasks/09-12-http2/prd.md","reason":"完整范围和验收"} +{"file":".trellis/tasks/09-12-http2/design.md","reason":"协议配置与生命周期"} diff --git a/.trellis/tasks/09-12-http2/design.md b/.trellis/tasks/09-12-http2/design.md new file mode 100644 index 0000000..211bd78 --- /dev/null +++ b/.trellis/tasks/09-12-http2/design.md @@ -0,0 +1,21 @@ +# 设计 + +## API 与复用 + +在官方 `dever.http.Limits` 增加 `http2: Http2Limits?`；`Http2Limits` 三个 Int 字段为 `streams`、`stream_window_bytes`、`connection_window_bytes`。默认关闭 HTTP/2，`default_http2_limits()` 返回 16 流、65535 字节单流窗口、262144 字节连接窗口。沿用现有 serve/serve_tls/serve_live/client/open/open_stream/send，不增加平行 API。 + +`Limits.check` 统一校验；HTTP/2 streams 和 connections 的乘积不得超过 65536，窗口遵循 HTTP/2 31 位限制（连接窗口至少 65535）。关闭自适应窗口，保持帧和发送缓存有界。源语言 exact-record 检查与原生结构转换同步修改。 + +## 运行时边界 + +- 协议配置、Hyper builder 和 HTTP 专用 TLS ALPN 配置集中在 HTTP 模块，原有 TLS trust/session cache 和 Transport 复用。不能修改共享 TLS 配置使 WSS 错误协商 h2。 +- 编解码复用 message；按协议校验头字段，HTTP/2 使用路径查询而不是绝对 URI 作为 source Request.target，authority 与 Host 一致。 +- HTTP/1 仍独占连接 lease；HTTP/2 持有共享连接与独立流 lease。EOF 或取消释放流，只有连接 owner 结束才释放物理连接许可。空闲回收不能中止活跃流。 +- Hyper Executor 必须接入已有结构化任务 Scope，不能直接用脱离作用域的 TokioExecutor。每流清理独立，程序故障明确传给服务任务，peer reset 只结束该流。 +- GOAWAY 排空与业务取消分开；应用已有 timeout(server, grace_ms, fallback) 控制最长排空时间。 + +## 验证 + +仅运行本次相关 Rust 定向测试、编译器官方库检查和隔离 loopback 原生程序。独立 HTTP/2 对端验证真正单连接多流、SETTINGS/流控、RST_STREAM、GOAWAY、ALPN 和请求语义。保留已有性能报告；缓存可重建部分暂存到专用 /dev/shm 目录以腾出构建空间。 + +协议依据：[RFC 9113](https://www.rfc-editor.org/rfc/rfc9113.html) 与当前锁定 Hyper 1.11.1 源码。 diff --git a/.trellis/tasks/09-12-http2/implement.jsonl b/.trellis/tasks/09-12-http2/implement.jsonl new file mode 100644 index 0000000..8c6037b --- /dev/null +++ b/.trellis/tasks/09-12-http2/implement.jsonl @@ -0,0 +1,5 @@ +{"file":".trellis/spec/backend/directory-structure.md","reason":"Rust 语言分层与 root test 布局"} +{"file":".trellis/spec/backend/toolchain-and-library.md","reason":"HTTP、TLS、流、结构化任务及原生桥接合同"} +{"file":".trellis/spec/backend/error-handling.md","reason":"程序故障与 peer 失败边界"} +{"file":".trellis/spec/backend/quality-guidelines.md","reason":"有界定向验证与原生测试"} +{"file":".trellis/tasks/09-12-http2/design.md","reason":"本阶段已批准接口与生命周期设计"} diff --git a/.trellis/tasks/09-12-http2/implement.md b/.trellis/tasks/09-12-http2/implement.md new file mode 100644 index 0000000..8f5db6b --- /dev/null +++ b/.trellis/tasks/09-12-http2/implement.md @@ -0,0 +1,49 @@ +# 执行记录 + +## 实现 + +- [x] 用户批准方案后实施，复用 Hyper 1.11.1；启用 http2 feature，锁定 h2 0.4.19 和四个传递依赖，不引入第二套协议栈。 +- [x] 官方 Limits.http2/Http2Limits、精确记录检查和仓库调用者一起迁移；原生桥接复用 Nullable/Record 转换，无新增 intrinsic。 +- [x] protocol/message 集中拥有流控预算、ALPN 派生、authority/Host 和方向相关的头校验。 +- [x] buffered/live 服务端和 single-use/pooled 客户端接通；流式上传、下载、SSE、GOAWAY 和取消沿用现有 API。 +- [x] ScopedOwner/ScopedSpawner 接入已有 Task Scope、全局容量和 blocking 清理；每流独立 Group，物理许可持有到协议后代清理完成。 +- [x] 按工作流完成服务端/客户端实现分工及独立并发、全范围检查；主代理复查关键所有权和运行证据。 +- [x] 更新现有 LANGUAGE、IMPLEMENTATION、README、toolchain-and-library 和性能说明；P1/P2 报告仍保留为 HTTP/1 数据。 +- [x] 最终定向验证与检查闭环。 +- [x] 完成任务状态、会话记录和自有临时目录清理，不提交。 + +## 复现及修正 + +- Hyper Executor 不携带 Dever CURRENT_SCOPE：协议 future 改由连接 owner 创建子 Scope；关闭 admission 与登记使用同一把锁。故障报告/注销先于完成信号；不可取消 supervisor 排空后代，等待者被取消也不能遗失 completion。 +- 池握手的连接 owner 必须在持久 driver 首次 poll 时创建，不能归临时请求 Scope。 +- RST/正文 idle 超时使背压 write 同时失败：先记录传输取消，避免把该流结束当成全局 handler 故障；真正程序错误仍保留原消息。发布失败时先释放 state 锁再析构响应体，避免取消标记重入锁。 +- GOAWAY 后 Hyper dispatch ready 不等于能建新流：仅 TrySendError 结构化返回的未发送请求可在原期限内重排；先释放旧上传/流租约，不重放已发送请求。 +- HTTP/2 Response TE 必须显式拒绝，不能交给 Hyper 静默删除；请求 TE 只接受 trailers。 +- 测试修正：Dever 子任务用 await；静默 h2 peer 可能先收到 SETTINGS，需读到 EOF；独立 h2 的 FlowControl/RecvStream/SendStream 在等 driver 结束前释放。共享原生 fixture 的 HTTP/1 服务端保留三条连接容量，两条供池内嵌套代理，一条供独立 WSS；HTTP/2 客户端仍只允许一条物理连接。 + +## 验证约束 + +使用 PATH 指向已有 stable 工具链、CARGO_TARGET_DIR=target/native-runtime、TMPDIR=/dev/shm/dever-http2-native.fW3PNS，Cargo 均 --offline --locked --release。一轮 fetch 后不再联网。仅自有随机 loopback 端口和 test/ 下测试证书，不运行全量测试、既有服务、性能矩阵或全局安装。 + +首轮 CLI release 构建和 backend 示例 check 通过。官方库 fmt-check 仅报告原有 net.dever 格式差异；修改的 http.dever 无差异。Rustfmt/Clippy 未安装，不作为通过项。 + +低磁盘处理仅将可再生 target/debug/{incremental,deps,build} 暂存到 /dev/shm/dever-http2-cache.wOoaI9；现有 debug/deverc、源码和 target/performance 报告保留。最后统一记录验证结果与缓存清理。 + +## 最终测试证据 + +88 个不同定向用例全部通过（修复后重跑不重复计数）：http2_client 6、http2_server 6、http_engine 12、http_library 7、live_network 16、pooled_network 14、pooled_library 6、async_runtime 10、structured_concurrency 11。 + +- 独立 h2 对端证明一条物理连接的并发请求、流式上传/下载、RST 前后隔离、背压写取消、WINDOW_UPDATE 恢复、流上限与活跃正文不会被 idle 回收、GOAWAY 的物理预算与连接替换、TLS ALPN 成功/拒绝、Host/target、413 与原始 handler 故障。 +- 原生 Dever 程序证明 single-use HTTP/2、TLS 同池嵌套代理（客户端只有一个物理名额）、上传、SSE 和 handler 内 run/await。共享 HTTP/1 变体另覆盖 WSS 与优雅停机；测试服务端三连接容量的修正经实际运行确认。 +- 最新响应发布取消/锁序改动已重新编译，并由 http2_server 6 项和 live_network 16 项覆盖；已有结构化任务、HTTP/1 连接池与源码语义检查通过。 +- 源码空白/冲突标记扫描、性能构建器 Python 语法和任务 context JSONL 验证通过。Rustfmt/Clippy 未安装；未运行性能矩阵、全量测试或既有服务验证。 + +复现上述范围： + +```bash +cargo test --offline --locked --release -p dever-tests --test http2_client --test http2_server --test http_engine --test http_library --test live_network --test pooled_network --test pooled_library --test async_runtime --test structured_concurrency +``` + +最终 CLI release 构建通过，产物为 `target/native-runtime/release/deverc`；backend 示例的实际 CLI check/fmt-check 均无输出并退出 0。迁移后的 network_bench example 及启用 reference 的 core/runtime 离线 release check 通过。未替换全局命令。 + +独立审查与主代理最终所有权、调用链、规范和改动范围核对闭环，无剩余生产阻断。清理专属 shm 构建目录，并删除暂存的约 378 MiB 可重建 debug 中间缓存；源码、现有编译器和全部性能报告保留。按用户约束手动更新状态/日志，不提交、不调用自动提交归档流程。N5 完成，HTTP/2 性能测量、RFC 8441、数据库与发行等仍在后续路线图。 diff --git a/.trellis/tasks/09-12-http2/prd.md b/.trellis/tasks/09-12-http2/prd.md new file mode 100644 index 0000000..ef7df6d --- /dev/null +++ b/.trellis/tasks/09-12-http2/prd.md @@ -0,0 +1,16 @@ +# N5 HTTP/2 + +用户已批准上一阶段提出的 HTTP/2 接入方案，并要求继续实现。复用 Hyper，面向小机器控制运行内存；不实现旧源码兼容层，不改运行时根任务调度策略。 + +## 范围与验收 + +- buffered/live 服务端和 single-use/pooled 客户端支持 HTTP/2；HTTPS 必须通过 ALPN 协商 h2，明文为 prior knowledge。 +- `Limits.http2: Http2Limits?` 显式选择协议；null 为 HTTP/1，非空为 HTTP/2。配置包含 streams、stream_window_bytes、connection_window_bytes，连接限额仍由 connections 持有。仓库调用者直接迁移。 +- 连接内复用请求，流式上传/下载与 SSE 复用现有正文接口；并发流、窗口、发送缓存有界，单流中止不取消相邻流。 +- Hyper 内部任务保留 Dever 的作用域归属；子任务、已启动 blocking、取消及故障完整排空，容量持有到清理完成。 +- 监听器关闭发送 GOAWAY 并排空在途流；客户端不向已 GOAWAY 连接派发新请求，不自动重试已发送请求。 +- 使用独立线协议对端和原生 Dever 程序验证多路复用、TLS ALPN、流控、取消隔离和生命周期，选取 HTTP/1 与 WS 回归。 + +## 排除 + +本阶段不接 RFC 8441 WebSocket-over-HTTP/2、h2c Upgrade、服务器推送、非空 trailers、自动协议降级，不跑全量测试或重新做性能矩阵，不替换全局二进制，不提交 Git。 diff --git a/.trellis/tasks/09-12-http2/task.json b/.trellis/tasks/09-12-http2/task.json new file mode 100644 index 0000000..9932924 --- /dev/null +++ b/.trellis/tasks/09-12-http2/task.json @@ -0,0 +1,26 @@ +{ + "id": "http2", + "name": "http2", + "title": "HTTP/2 server and multiplexed client", + "description": "复用 Hyper 接入 HTTP/2 服务端、客户端多路复用、TLS ALPN、流控和结构化取消；验证流式正文与 SSE，保留 HTTP/1 和普通 WebSocket。", + "status": "completed", + "dev_type": null, + "scope": null, + "package": null, + "priority": "P2", + "creator": "Antigravity", + "assignee": "Antigravity", + "createdAt": "2026-09-12", + "completedAt": "2026-09-12", + "branch": "main", + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [], + "parent": "09-04-dever-language-mvp", + "relatedFiles": [], + "notes": "N5 complete: 88 focused cases passed, native HTTP/2/TLS proxy/upload/SSE and HTTP/1/WSS verified; CLI build/check and independent review passed. No performance matrix, global install or commit.", + "meta": {} +} diff --git a/.trellis/tasks/09-12-runtime-performance-p2/check.jsonl b/.trellis/tasks/09-12-runtime-performance-p2/check.jsonl new file mode 100644 index 0000000..13a30a8 --- /dev/null +++ b/.trellis/tasks/09-12-runtime-performance-p2/check.jsonl @@ -0,0 +1,3 @@ +{"file":".trellis/spec/backend/quality-guidelines.md","reason":"Benchmark verification contracts"} +{"file":".trellis/tasks/09-12-runtime-performance-p2/prd.md","reason":"Scope and acceptance"} +{"file":".trellis/tasks/09-12-runtime-performance-p2/design.md","reason":"Semantic comparison and lifecycle"} diff --git a/.trellis/tasks/09-12-runtime-performance-p2/design.md b/.trellis/tasks/09-12-runtime-performance-p2/design.md new file mode 100644 index 0000000..3d84786 --- /dev/null +++ b/.trellis/tasks/09-12-runtime-performance-p2/design.md @@ -0,0 +1,11 @@ +# Design + +现有 test/performance 负责产物和采样，Rust examples 负责压力对端。保留该分工，不增加第二套预算与报告框架。 + +异步对照使用独立 async_bench example：相同 run/wait、checked Int 加法、输入与批次；runtime-root 与 runtime-worker 仅改变批次所在任务。worker 模式增加一个结构化外层任务，不修改生产调度。对照程序输出 P1 SAMPLE 协议，Python 复用校验和与样本汇总。 + +长连接使用新增 Dever fixture 与 Rust 对端。Rust 对端有固定连接上限、阶段标记、消息校验和超时；Python 根据阶段采样同一个服务进程，复用 Process/Budget。每轮连接、保持、断开后等待采样，最后重连探测服务可用。TCP/WS 做小正文交互；SSE 校验有限事件，保留流直到断开。 + +高负载扫描复用现有 HTTP load，不重写压测器。使用独立 CPU 亲和性和 128 MiB 服务预算，合理短窗口与重复次数；保留客户端瓶颈和短窗口限制。 + +HTTP/2 只做本地依赖/池/协议边界核查：Hyper h2、TLS ALPN、流与连接预算、复用客户端、GOAWAY、取消隔离；HTTP/1 WebSocket 升级保持独立。 diff --git a/.trellis/tasks/09-12-runtime-performance-p2/implement.jsonl b/.trellis/tasks/09-12-runtime-performance-p2/implement.jsonl new file mode 100644 index 0000000..8f6caf9 --- /dev/null +++ b/.trellis/tasks/09-12-runtime-performance-p2/implement.jsonl @@ -0,0 +1,4 @@ +{"file":".trellis/spec/backend/index.md","reason":"Rust language owning layers"} +{"file":".trellis/spec/backend/quality-guidelines.md","reason":"Bounded standalone benchmark contracts"} +{"file":".trellis/tasks/09-12-runtime-performance-p2/prd.md","reason":"Approved scope and acceptance"} +{"file":".trellis/tasks/09-12-runtime-performance-p2/design.md","reason":"Reuse and ownership"} diff --git a/.trellis/tasks/09-12-runtime-performance-p2/implement.md b/.trellis/tasks/09-12-runtime-performance-p2/implement.md new file mode 100644 index 0000000..159bfff --- /dev/null +++ b/.trellis/tasks/09-12-runtime-performance-p2/implement.md @@ -0,0 +1,21 @@ +# Execution + +- [x] 主代理：async_bench、Python 集成、HTTP 扫描、采样、结果分析与 HTTP/2 就绪评估。 +- [x] trellis-implement：long connection Rust 对端及 Dever fixture，独立文件所有权，不修改生产代码。 +- [x] 定向 offline release 构建、生成原生 fixture、校验工具错误路径、执行有界矩阵。 +- [x] trellis-check：复查统计、语义、连接清理和最终结果；主代理复核关键证据。 +- [x] 更新现有性能 README、任务与日志，不提交。 + +磁盘空间有限；复用 target/native-runtime 的 release 依赖。原生构建 TMPDIR 使用任务独有 /dev/shm 临时目录，结束移除自有目录。不并行执行 Cargo 构建。 + +## 已取得证据 + +- HTTP/HTTPS 高负载完整 54 组，report: target/performance/http-load-p2-final/report.json。1800000 计划、1500830 实际成功、0 错误、299170 未发出；三实现均有压测槽位丢弃，不能推断服务端吞吐上限。无 OOM。 +- 首次 http-load-p2 在子进程退出采样竞态中断：/proc memory teardown 早于 waitpid 状态。已修复采样拥有层，补定向回归，保留中断报告。 +- run/wait Rust 对照用生产 task runtime、Bytes 和 checked Int；build 工具使用与 native 相同优化参数和 runtime rlib。async-p2 已完成 18 组：Dever root/worker 15.50/2.003 µs，Rust 15.59/1.984 µs，三轮校验均通过且无 OOM。独立 perf 确认 root/worker 线程切换为 465149/56 次；生产运行时未修改。 +- 独立 review 修正了 live 总超时遗漏关闭阶段预算、SSE 保持期无心跳仍通过的问题；静态发现并修正 SSE driver 早期错误所有权和任意截止点半帧误判。所有修改均在 benchmark 工具/fixture 内。 +- 首次 live-p2 消息校验通过，但发现阶段读/采样间跨轮竞态：下一轮 FD 混入上轮 disconnected。修复为采样前后 progress 必须一致，丢弃过渡阶段观察，并新增回归。最终 live-p2-final 完整重跑 27 组、18144 连接、377061 消息；错误/OOM 为零，81 次断开 FD 差值均为零。 +- live-cycles-p2 对三个协议各做 512 连接十轮重连、每轮 3 秒；15360 连接、471207 消息，错误/OOM 为零，30 次 FD 均回基线。SSE RSS 在前三轮升到约25 MiB，后七轮在25.24–25.36 MiB；TCP约8.8、WS约17 MiB。短时证据不宣称小时级稳定性。 +- 12 个 Python 运行器定向检查、4 个独立 TCP/SSE 对端检查通过；11 个 Dever 入口生产优化构建和两个新 Rust example 离线 release/原生优化构建通过。Dever fmt-check、Python解析、源码空白/冲突和范围检查通过；Rustfmt/Clippy 不可用，未运行。 +- /dev/shm/dever-perf-p2.E84QV8 已空并移除。保留原始构建/剖析/中断与最终报告以供复核；不提交，不运行自动归档或自动日志提交脚本。 +- 最终独立检查无剩余阻断；主代理复核四份最终报告、全部 OOM/FD 计数及无残留预算组。P2 完成，可进入 HTTP/2 阶段，未将实验调度变化当作已实现的生产优化。 diff --git a/.trellis/tasks/09-12-runtime-performance-p2/prd.md b/.trellis/tasks/09-12-runtime-performance-p2/prd.md new file mode 100644 index 0000000..a858b0b --- /dev/null +++ b/.trellis/tasks/09-12-runtime-performance-p2/prd.md @@ -0,0 +1,12 @@ +# Runtime performance P2 + +用户已批准继续上一阶段；本阶段完成同语义异步对照、高负载 HTTP/HTTPS 扫描和 TCP/WebSocket/SSE 有界长连接测试，并回答 HTTP/2 的接入条件。HTTP/2 实现留到下一阶段。 + +## 验收 + +- Rust 对照调用现有 Dever task runtime，保留 run/wait 的任务额度、结构化取消和清理；单独标注工作线程调度实验的边界。 +- 复用 P1 构建、进程采样、cgroup 和报告逻辑，记录校验和、样本、CPU、上下文切换、RSS/PSS 与环境；不以不同语义的裸 Tokio 数字宣称语言损耗。 +- HTTP/HTTPS 固定正文在更高输入速率下记录成功、错误、容量丢弃、调度落后及服务端/客户端 CPU；限速结果不宣称极限吞吐。 +- TCP、WebSocket、SSE 支持有限数量的长连接保持、正文验证和多轮断开重连；记录服务端连接前、保持中、断开后的内存和 FD。 +- 运行仅限自有 loopback 服务和独立预算组；所有任务、连接和进程有明确停止边界，结束无残留。执行最小定向测试，不运行全量测试、不修改全局安装、不提交。 +- 不添加兼容分支、无证据的运行时重构或新依赖；发现实际阻断后再定位归属。 diff --git a/.trellis/tasks/09-12-runtime-performance-p2/task.json b/.trellis/tasks/09-12-runtime-performance-p2/task.json new file mode 100644 index 0000000..90ff435 --- /dev/null +++ b/.trellis/tasks/09-12-runtime-performance-p2/task.json @@ -0,0 +1,26 @@ +{ + "id": "runtime-performance-p2", + "name": "runtime-performance-p2", + "title": "Runtime performance and long connections", + "description": "Continue approved runtime benchmarks: equivalent async Rust comparison, bounded HTTP load scan and TCP/WebSocket/SSE connection recovery; assess HTTP/2 readiness without implementing it.", + "status": "completed", + "dev_type": null, + "scope": null, + "package": null, + "priority": "P2", + "creator": "Antigravity", + "assignee": "Antigravity", + "createdAt": "2026-09-12", + "completedAt": "2026-09-12", + "branch": "main", + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [], + "parent": "09-04-dever-language-mvp", + "relatedFiles": [], + "notes": "", + "meta": {} +} diff --git a/.trellis/tasks/09-12-runtime-performance/check.jsonl b/.trellis/tasks/09-12-runtime-performance/check.jsonl new file mode 100644 index 0000000..badd547 --- /dev/null +++ b/.trellis/tasks/09-12-runtime-performance/check.jsonl @@ -0,0 +1 @@ +{"file":".trellis/spec/backend/quality-guidelines.md","reason":"Benchmark validation and evidence limits"} diff --git a/.trellis/tasks/09-12-runtime-performance/design.md b/.trellis/tasks/09-12-runtime-performance/design.md new file mode 100644 index 0000000..ac4f5e6 --- /dev/null +++ b/.trellis/tasks/09-12-runtime-performance/design.md @@ -0,0 +1,9 @@ +# 测量边界 + +现有 performance.rs 拥有计算热路径基准，保持原有语义与 1.15 门槛。新增根 test/performance/ 的 Python 标准库驱动，分离构建、进程采样、场景执行和报告。Dever fixtures 经过现有 CLI build 成为独立二进制。Rust 辅助程序作为 dever-tests 的显式 example，复用 workspace 已锁定的 Tokio/Hyper/Rustls。 + +网络辅助程序同时提供等价 Rust runtime / 原生 Hyper 对照服务和独立负载端；不自行解析 HTTP。固定速率调度以计划发送时间计算延迟，并记录有界并发容量不足或调度逾期导致的丢弃。TLS 校验证书和主机名，复用已有测试 CA；新连接与连接复用分别测量。 + +进程通过 READY|port（空闲/计算用 READY|0）报告可采样状态。配置统一使用 BENCH_WORKERS、BENCH_CONNECTIONS、BENCH_TIMEOUT_MS、BENCH_LIFETIME_MS、BENCH_CERT、BENCH_KEY 等环境变量。负载 CLI 为 network_bench load URL PATH RATE SECONDS CONNECTIONS TIMEOUT_MS REUSE CA，JSON 标准输出。服务 CLI 为 network_bench runtime-http/runtime-https/hyper-http/hyper-https。各服务 /plain 返回 hello，/json 返回 {"ok":true}，/bytes 返回 65536 个 x；带对应 Content-Type，不启用日志。 + +Linux /proc 采样独立于子程序；cgroup 为可选且显式启用的进程预算，使用新建的专属子组，不调整宿主或已有服务的 cgroup。负载发生器不进入服务预算。默认不声称隔离或峰值吞吐；报告采样间隔、负载端 CPU 和限额状态。汇总保留每轮原始数据，不建立未经校准的绝对性能门槛。 diff --git a/.trellis/tasks/09-12-runtime-performance/implement.jsonl b/.trellis/tasks/09-12-runtime-performance/implement.jsonl new file mode 100644 index 0000000..eab87a2 --- /dev/null +++ b/.trellis/tasks/09-12-runtime-performance/implement.jsonl @@ -0,0 +1,2 @@ +{"file":".trellis/spec/backend/quality-guidelines.md","reason":"Focused owned benchmarks and unchanged runtime semantics"} +{"file":".trellis/spec/backend/toolchain-and-library.md","reason":"Native optimization and HTTP/TLS/task contracts"} diff --git a/.trellis/tasks/09-12-runtime-performance/implement.md b/.trellis/tasks/09-12-runtime-performance/implement.md new file mode 100644 index 0000000..d5f2df1 --- /dev/null +++ b/.trellis/tasks/09-12-runtime-performance/implement.md @@ -0,0 +1,18 @@ +# 执行顺序 + +- [x] 主代理实现 Dever 场景和 Python 构建/采样/报告入口。 +- [x] 独立 implement 代理实现 test/dever-tests/examples/network_bench.rs 及其私有模块；复用现有网络依赖。 +- [x] 编译定向 fixtures/example，最小验证测量统计和清理，短时 loopback HTTP/HTTPS 对照。 +- [x] 检查最终文件、响应校验、延迟分母、丢弃/失败计数、资源预算和退出清理；记录初始结果与限制。 +- [x] 更新 test/performance/README.md、现有质量规范及任务证据；不提交。 + +工具链使用 /root/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/bin 的命令级 PATH；所有 Cargo 构建 offline。只运行本任务的最小定向检查，不执行旧 performance target 或全量测试。 + +## 最终证据 + +- 七个 Python 运行器定向测试和两个独立 Python HTTP 对端测试通过；负例响应正文必须记 error、成功延迟为空。覆盖热身去除、正确结果/数量、完整窗口、计数守恒、已退出进程和缺控制器时清理。 +- Rust example 离线 release check/build 通过；七个 Dever 入口经 CLI 格式化、语义检查和生产优化原生构建。Rustfmt/Clippy 未安装，未报告为通过；显式源码空白/冲突标记检查与 Dever fmt-check 通过。 +- 最终基线 78 组：15 内存、9 异步、54 HTTP/HTTPS；另 18 组新连接和 15 组 64 MiB 空闲基线。各组实际调用环境见 target/performance/*/report.json，结果解释以 test/performance/README.md 首轮结果为准。 +- 54 组正式网络共计划 108000 请求，发出并成功 107962，错误 0；未发出 38（窗口过期 32、容量不足 6）。预热单独记录。新连接 900 请求全部成功，无丢弃；全部自有预算组无 OOM，结束后没有残留 dever-perf-* cgroup。 +- 对照复查修正了大正文逐请求重复分配、短窗口吞吐分母、定时器抖动错误计丢弃、请求 driver 取消所有权、负载线程环境污染、缺控制器清理以及实际请求延迟校验。独立只读 review 最终没有剩余阻断。 +- 仅构建/测试工具、fixtures、Cargo 测试依赖和文档发生变化；生产编译器/运行时未修改。保留产物和原始报告，不提交、不运行自动归档脚本。更高负载、长连接/持续稳定性和运行时优化是后续工作。 diff --git a/.trellis/tasks/09-12-runtime-performance/prd.md b/.trellis/tasks/09-12-runtime-performance/prd.md new file mode 100644 index 0000000..22880cb --- /dev/null +++ b/.trellis/tasks/09-12-runtime-performance/prd.md @@ -0,0 +1,14 @@ +# 独立二进制运行性能基准 + +用户批准实施上一轮提出的第一阶段：常驻内存、异步调度、HTTP/HTTPS。小机器只运行已编译程序；构建资源不是本阶段优化目标。此批准取代历史网络开发阶段不运行压测的局部限制，仅限本任务拥有的有界基准。 + +验收： +- 根 test/ 内提供显式运行的构建与测量入口，默认测试不启动压测。 +- 测量真实 Dever 编译产物，复用现有优化设置；编译不进入运行计时。 +- 覆盖同步/异步/HTTP/HTTPS 空闲内存，任务启动/Channel/取消，以及 HTTP/HTTPS 固定速率请求。 +- HTTP 对照使用相同 Dever runtime 的 Rust 服务和原生 Tokio/Hyper 服务；校验响应内容，记录成功、失败、未发出请求和尾延迟。 +- 输出机器可读结果、环境/配置/二进制身份、CPU/RSS/PSS/线程/FD；可选独立 cgroup 限额，缺少权限时报错，不虚构限额。 +- 自有随机 loopback 端口、测试证书、有界时间和负载，退出清理子进程；不访问已有服务，不跑全量测试，不提交。 +- 完成短时定向验证和至少一组可解释的初始测量；长时间、多机器/多档压力矩阵明确留待显式运行。 + +不包含运行时优化、数据库、服务端流式请求、信号 API、TCP/WS/SSE 长连接稳定性和发行工具链。 diff --git a/.trellis/tasks/09-12-runtime-performance/task.json b/.trellis/tasks/09-12-runtime-performance/task.json new file mode 100644 index 0000000..3f96765 --- /dev/null +++ b/.trellis/tasks/09-12-runtime-performance/task.json @@ -0,0 +1,26 @@ +{ + "id": "runtime-performance", + "name": "runtime-performance", + "title": "Runtime performance baselines", + "description": "Measure standalone Dever memory, async scheduling and HTTP/HTTPS with reproducible bounded workloads and equivalent Rust baselines.", + "status": "completed", + "dev_type": null, + "scope": null, + "package": null, + "priority": "P2", + "creator": "Antigravity", + "assignee": "Antigravity", + "createdAt": "2026-09-12", + "completedAt": "2026-09-12", + "branch": "main", + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [], + "parent": "09-04-dever-language-mvp", + "relatedFiles": [], + "notes": "", + "meta": {} +} diff --git a/.trellis/tasks/09-13-typed-orm/check.jsonl b/.trellis/tasks/09-13-typed-orm/check.jsonl new file mode 100644 index 0000000..2b10ee7 --- /dev/null +++ b/.trellis/tasks/09-13-typed-orm/check.jsonl @@ -0,0 +1,7 @@ +{"file":".trellis/spec/backend/error-handling.md","reason":"数据库错误、事务失败、取消和资源回收必须显式传播"} +{"file":".trellis/spec/backend/quality-guidelines.md","reason":"检查数据库测试边界、内存限制、无重复驱动逻辑和最小验证"} +{"file":".trellis/tasks/09-13-typed-orm/research/architecture-options.md","reason":"复核实现只暴露 Model 包短函数，没有混入实例 method、链式查询器或动态 Value 路径"} +{"file":".trellis/tasks/09-13-typed-orm/prd.md","reason":"逐项核对已确认的 Model、查询、配置、事务、错误、迁移和小内存验收合同"} +{"file":".trellis/tasks/09-13-typed-orm/design.md","reason":"复核编译器/runtime/dialect 边界、driver profile、单连接事务与错误根边界没有重复实现"} +{"file":".trellis/tasks/09-13-typed-orm/research/schema-change-matrix.md","reason":"确认所有自动 schema 变化均无静默数据损失，危险变化必须由命名 migrate 表达"} +{"file":".trellis/tasks/09-13-typed-orm/research/driver-selection.md","reason":"确认 SQLite/PostgreSQL 只复用驱动和连接池，不引入第二套 ORM 或无界池"} diff --git a/.trellis/tasks/09-13-typed-orm/design.md b/.trellis/tasks/09-13-typed-orm/design.md new file mode 100644 index 0000000..c9155fb --- /dev/null +++ b/.trellis/tasks/09-13-typed-orm/design.md @@ -0,0 +1,409 @@ +# 类型安全 ORM 技术设计 + +## 设计目标 + +实现一个静态、无反射、默认有界的 ORM，并让数据库能力自然接入 Dever 的顺序调用、结构化并发、错误传播和 AOT 产物裁剪。Model 只描述持久化事实和数据访问；数据库驱动不拥有语言层查询语义。 + +## 总体边界 + +```text +Dever source + -> parser / formatter + -> model + function semantic checking + -> effect / error / database binding inference + -> typed QueryPlan + normalized ModelSchema + -> native emitter + -> shared ORM runtime + -> SQLite dialect/executor + -> PostgreSQL dialect/executor + -> packaged binary + external config/setting.json +``` + +- 编译器拥有 Model 识别、字段类型、查询表达式、静态关系、错误集合和 transaction 合法性。 +- ORM runtime 拥有配置、连接生命周期、绑定值、行流、事务、schema 执行和统一错误分类。 +- dialect 只拥有 SQL/DDL 差异；driver executor 只拥有具体连接、参数编码和行解码入口。 +- 生成代码拥有每个 Model 的具体创建输入、更新输入、查询返回、字段绑定和行 decoder。 +- 不建立通用 Repository/BaseModel 层，也不让 SQLite/PostgreSQL 分别复制 CRUD 流程。 + +## 源码合同 + +### Model 文件 + +```dever +package app.model.user exposes ( + User +) + +type UserStatus { + Active = "启用" + Disabled = "禁用" +} + +type User { + uuid: Uuid generated unique + email: Text(254) unique + display_name: Text(1, 64) from nickname + status: UserStatus default UserStatus.Active index + organization_id: app.model.organization.id? +} + +index(status, created_at) + +seed { + { + email = "admin@example.com" + display_name = "Admin" + status = UserStatus.Active + } +} + +migrate remove_legacy_code { + drop legacy_code +} +``` + +`database <name>`、`index`、`unique`、`relation`、`seed` 和 `migrate` 只在 Model 文件中合法。需要显式使用报告库时，在 package header 后增加 `database report`；未写时按根包/default 规则绑定。普通文件继续只有普通 type/function 语义。 + +### 查询与事务 + +```dever +users = user.list({ + where = status == UserStatus.Active and organization_id == organization_id + order = [created_at.desc, id.desc] + page = 1 + size = 20 + with = [organization] +}) + +transaction disable_organization( + target_organization_id: app.model.organization.id +) () { + user.update({ + where = organization_id == target_organization_id + }, { + status = UserStatus.Disabled + }) + organization.update(target_organization_id, { + active = false + }) +} +``` + +匿名创建/更新/查询块只在对应参数位置出现，由目标 Model 提供字段命名空间。它们不会成为可传递的动态 Map 或持久查询对象。 + +### 类型化原生 SQL + +复杂查询通过 Model 文件内的专用 `sql` 声明提供，不引入通用泛型或 type-as-value： + +```dever +type UserSummary { + status: UserStatus + total: Int +} + +sql active(status: UserStatus) (users: List<User>) { + sqlite = "SELECT id, created_at, email, status FROM user WHERE status = ?1" + postgres = "SELECT id, created_at, email, status FROM user WHERE status = $1" +} + +sql summary(status: UserStatus) (summary: UserSummary?) { + sqlite = "SELECT status, COUNT(*) FROM user WHERE status = ?1 GROUP BY status" + postgres = "SELECT status, COUNT(*) FROM user WHERE status = $1 GROUP BY status" +} +``` + +- 两种方言都必须是静态 Text 字面量。SQLite 显式使用 `?N`，PostgreSQL 显式使用 `$N`；编译器不重写 SQL。 +- 参数按声明顺序和具体字段类型绑定；禁止插值、动态列名及动态行 `Map`。 +- 结果只能是一个具体 `Model`、`Model?`、`List<Model>`，或同文件私有辅助 record 的对应形状。 +- Model 文件仍只有文件名对应且 exposed 的 record 注册为 Model。辅助 record 必须被 SQL 结果使用，不生成表，也不能 exposed。 +- Model 结果复用生成的 Model decoder；辅助 record 生成具体字段顺序 decoder。缺列、多列、类型不匹配，以及单行结果的零行/多行都作为运行时数据库数据错误传播。 +- 调用形式与内置 CRUD 一致，例如 `user.active(status)`。执行复用既有 statement cache、值绑定和隐藏 transaction executor；`List<Record>` 只物化该静态 SQL 实际返回的行，不建立额外动态缓存。 + +## 编译器设计 + +### 1. 语法与格式化 + +- 根据源文件相对路径判定是否属于 `model/`；Model 文件仍解析一个普通 record type，不新增通用 type 种类。 +- 在 AST 中增加目的明确的 Model 元数据节点：数据库绑定、字段修饰、复合索引、关系、Seed 和命名迁移。不要把它们编码为字符串或普通函数调用后再反解析。 +- 函数声明使用 `FunctionKind::Ordinary | Transaction`，取代继续叠加多个布尔修饰位。 +- 删除源码 `async` 和 `await` 节点；`run`、`wait`、`stop` 保留为明确的并发上下文表达式。 +- `result(call)` 和 `fail(error_variant)` 作为两个目的单一的核心表达式进入 AST/HIR，不借此加入 try/catch、annotation 或 macro。 +- formatter、Markdown 合同解析和 API snapshot 与新语法同步更新；不保留旧语法分支。 + +### 2. Model 注册与静态元数据 + +- SourceMap 加载后先建立 `ModelId`，验证文件名、package 路径、唯一公开 record 和 Model 表名。 +- 为每个 Model 合成： + - 不透明 `<model-package>.id`； + - 完整 record 中的 `id`/`created_at`； + - 创建和更新输入 shape； + - CRUD/query/relationship 符号； + - choice options 常量； + - 具体 row decoder 与 bind encoder 描述。 +- 同一逻辑连接中发生表名、索引名或内部对象名冲突时，在生成 SQL 前报告源码诊断。 +- Model CRUD 是编译器合成的包能力，不要求开发者在 `exposes` 中逐个列出，也不扩展成通用 type-as-value。 + +### 3. 规范化 Schema + +新增与方言无关的 `ModelSchema`： + +- 稳定 Model/package/table identity； +- 有序字段、逻辑类型、可空性、默认值、generated、旧字段来源； +- 主键、外键、choice CHECK、单列/复合索引； +- relation、Seed 和命名迁移规范化表示。 + +revision 只对逻辑 schema 和迁移意图计算，不包含 SQLite/PostgreSQL SQL 文本、连接串、文件绝对路径或源码格式。这样两个方言完成后记录相同逻辑身份。 + +### 4. QueryPlan + +所有短函数共用一个静态 IR： + +- operation：select/insert/update/delete/upsert/count/exists； +- projection 与具体结果 shape； +- condition tree、稳定 order、page/cursor、limit； +- relation load plan； +- 参数槽位及 Dever 类型； +- transaction requirement 与 Model binding。 + +查询块在 checker 中解析为 Model 字段 ID 和类型化运算，不把字段名字符串留到 runtime。native emitter 将固定 SQL shape 和参数提取代码写入生成程序。 + +类型化原生 SQL 作为同一 Model operation HIR 的专用分支：HIR 只保存已校验的 operation 索引和类型化参数，方言文本保留在 `ModelSchema`。native emitter 选择 runtime profile 对应文本，并复用普通 ORM 的 bind encoder、database executor 与 row decoder；不会把 SQL 或结果 shape 推迟到 runtime 解释。`Record`/`Record?` 最多拉取两行以判定基数错误；`List<Record>` 最多保留 `database.max_page_size` 行并额外探测一行，超限失败而不静默截断，因此漏写 SQL `LIMIT` 也不会形成无界进程内结果集。 + +#### Relation 结果与加载计划 + +- 编译器公开内置 `Related<T>`，只有 `Unloaded` 和 `Loaded(value)` 两态。ORM 负责构造值；业务代码可以用 `Related.Unloaded<T>()` 和 `Related.Loaded<T>(value)` 分句穷尽匹配。 +- `Related` 使用间接 native 布局 `Loaded(Box<T>)`。Model 类型环只有穿过 `Related` 才合法，基础 row decoder 始终把合成关系字段初始化为 `Unloaded`。 +- 持久化字段 `<name>_id: target.id` 自动合成 `<name>: Related<Target>`；可空外键合成 `Related<Target?>`。显式 `relation children = target.parent_id` 合成 `children: Related<List<Target>>`。持久化 schema、bind encoder 和基础 decoder 不包含这些合成字段。 +- `with = [organization, children]` 在 checker 中解析为 `QueryPlan` 的静态 relation 索引，只允许一层裸 relation 名称；未知、重复和嵌套关系均拒绝。`with` 只允许 `first`、`list`、`cursor`，`stream` 明确拒绝。 +- to-one 使用受控 `LEFT JOIN` 或 `INNER JOIN`，支持同一父查询的多个关系；`list` 的 count 查询保持只访问父表，`cursor` 不生成 count。 +- to-many 在父结果确定后按父 ID 批量加载并归组，不逐父查询。子记录按 `id ASC` 稳定排序，每个父记录最多加载 `database.max_page_size` 条；父 ID 每 998 个一批，为 SQLite 的第 999 个参数保留 relation limit。空父列表不会生成 `IN ()`。 +- 关系查询数量由生成代码 shape 测试验证：每个 relation 只有一个按固定参数批次循环的 `query_owned` 调用点，不在生产热路径加入计数原子。 + +### 5. 挂起 Effect + +- intrinsic 和官方库函数标记基础挂起 effect；调用、静态 handler 和 collection handler 通过现有依赖图传递。 +- AST 不再存 `is_async`，HIR 在 effect 收敛后记录内部 `suspends`。native emitter 仅对可达挂起函数生成 Rust `async fn` 和内部 `.await`。 +- 普通调用根据目标 effect 降低为同步 call 或内部 await call；源码语义始终是顺序完成。 +- 同步入口的完整可达图没有挂起 effect 时沿用直接 `main`；否则生成唯一 Tokio runtime 入口。 + +### 6. Error Effect + +- `error` choice 变体仍是命名、带类型 payload 的领域失败；`fail(...)` 把对应变体加入当前函数错误集合。 +- 调用图为每个函数推导封闭错误集合。普通调用在 HIR 中自动传播失败，native 使用具体 Rust `Result<Success, AppError>`；源码不写 `?`。 +- 对存在多个正常状态的操作，把正常状态保留为非 error choice，例如 `ReadState.Read/End`，错误集合与成功类型分开。 +- `result(call)` 阻止当前调用自动传播，并生成编译器内部 `CapturedResult<SuccessShape, ErrorSet>`。它不引入用户泛型；checker 只允许通过现有穷尽函数分句消费、继续返回或传递，且必须覆盖成功形状和全部 error 变体。 +- native 为当前可达程序生成一个具体的错误 enum，并为函数记录精确子集；API/docs/editor 显示每个函数的静态错误集合。没有动态错误对象、字符串类型分派或按变体名猜 HTTP 状态。 +- 现有 C012 改为检查 error effect 和 captured result 的消费，继续拒绝丢弃、覆盖、仅打印或仅记录。 + +### 7. Database Effect 与 Transaction + +- 每个 Model operation 在 HIR 标记 `DatabaseEffect(ModelId)`；普通 helper 通过调用图传递 Model 集合。 +- 每个 Model 保存声明式 `ConnectionSelector::Explicit(name) | PackageRoot(root)`。source check 可拒绝显然不同的显式绑定；native build 读取 setting 后做最终连接解析和 transaction 单连接验证。 +- Transaction 函数必须具有至少一个数据库 effect，且所有 effect 在构建配置中解析为同一连接；并发子调用中的数据库 effect 直接诊断。 +- HIR 为 transaction 调用链增加隐藏 `DatabaseContext` 参数。普通路径使用 registry slot，事务路径使用同一 connection 的 transaction handle；不查询线程局部或任务局部状态。 + +## 配置与构建能力 + +### 项目根与源码根 + +- 应用命令接收项目根；项目根内的 `module/` 是唯一 SourceMap 根，package 按文件相对 `module/` 的路径校验和派生。 +- `module/user/model/user.dever` 因而对应 `user.model.user`，根包数据库绑定解析为 `database.user`；不存在时才使用 `database.default`。 +- `config/`、`data/` 与输出二进制都在项目根下，但不参与 Dever 源码扫描。编译器内部只传递规范化后的项目根和源码根，不在 parser/checker 中重复猜目录。 +- 旧的单目录语言 fixture 由测试工具显式传入源码根；应用 CLI 不增加自动回退或第二套模糊目录探测。 + +### setting.json + +配置路径固定为 `<executable-dir>/config/setting.json`。数据库对象直接按连接名组织： + +```json +{ + "app": {}, + "http": {}, + "database": { + "default": { + "type": "sqlite", + "path": "data/db/app.db", + "max_connections": 1, + "max_page_size": 100 + }, + "report": { + "type": "postgres", + "url": "postgres://user:password@db.example.com/report", + "tls": "system", + "min_connections": 0, + "max_connections": 4, + "max_page_size": 100, + "wait_timeout_ms": 5000, + "io_timeout_ms": 30000 + } + }, + "log": {} +} +``` + +- 使用严格的结构化 JSON 解析；未知数据库 type、缺失 default、无效数值或被引用连接不存在均失败。 +- PostgreSQL 的 `tls` 必须显式为 `system` 或 `disabled`；`system` 使用现有 rustls 系统根并校验证书与主机名，不提供从 TLS 静默回落明文的 `prefer` 模式。 +- PostgreSQL 的 `wait_timeout_ms` 限制池等待，`io_timeout_ms` 独立限制 SQL、事务和迁移 I/O；二者范围均为 1–300000 毫秒。 +- 相对数据库/日志/upload/cache/tmp 路径都相对可执行文件目录，不相对 cwd。 +- 启动时创建 `data/db`、`data/upload`、`data/log`、`data/cache`、`data/tmp`；不创建或修复 setting。 +- 连接 selector 按“显式名称 -> 根包同名键 -> default”解析一次，Model 保存连接 slot。 + +### RuntimeProfile + +发布构建读取 setting 中全部数据库 `type`，生成 `RuntimeProfile { sqlite, postgres, async, log }`： + +- 只读取能力类型，不把 URL、密码、路径或池参数嵌入二进制或缓存 key。 +- runtime 的 SQLite/PostgreSQL 依赖使用 optional Cargo features。 +- profile、runtime rlib/dependency 快照和生成源码共同进入 native cache identity。 +- 编译器 parse/check 不依赖 setting；生成可部署原生程序时需要 setting。程序启动会再次严格解析实际 setting。 +- 运行时配置可以在已打包驱动集合内修改。若改成未打包的数据库类型，启动明确失败并要求重新构建，不做动态下载或回退。 + +### CMS 纵向样例 + +```text +examples/dever/cms/ + config/setting.json + data/ + module/ + cms.dever + user/model/user.dever + user/service/register.dever + news/model/news.dever + news/service/publish.dever +``` + +- `user.model.user` 保存账户及初始化管理员，`news.model.news` 通过 `author_id: user.model.user.id` 建立静态外键。 +- `user.service.register` 与 `news.service.publish` 使用 transaction 函数表达真实业务边界，直接复用编译器生成的 Model 操作。 +- `cms.main` 只编排可观察的纵向流程，不复制 CRUD、迁移或连接选择逻辑。 +- 默认 setting 只启用 bundled SQLite，既是新项目模板也是 SQLite-only 原生产物和小内存基准的真实用例。 + +## Runtime 设计 + +### 连接注册表 + +`DatabaseRegistry` 在应用入口启动 HTTP/任务前完成： + +1. 定位并解析 setting； +2. 创建数据目录； +3. 构造所有被 Model 引用的连接池； +4. 解析 Model -> connection slot； +5. 验证 transaction 单连接不变量； +6. 获取每个连接的 schema 锁，执行 migration 和 Seed； +7. 成功后才进入用户入口。 + +热路径通过整数 slot 取连接，不执行字符串查找。连接实现使用小枚举静态分派，不使用每行/每值 trait object。 + +### SQLite + +- 使用 `deadpool-sqlite` + `rusqlite bundled`。 +- pool 上限不超过 Dever 全局 blocking 容量；每次同步交互都通过既有 blocking permit 和 pool object 生命周期。 +- 默认一个连接，开启 foreign keys，设置 busy timeout；WAL 是否启用由连接配置明确决定，不在 runtime 猜测。 +- transaction 独占同一个 pool object 直到 commit/rollback；RowStream 不能在释放连接后继续读取。 + +### PostgreSQL + +- 使用 `deadpool-postgres` + `tokio-postgres`；TLS 连接通过 `tokio-postgres-rustls` 复用当前 rustls/webpki roots。 +- pool 获取、连接建立、查询和 row stream 都使用 Tokio，且有独立的连接上限、获取等待和 I/O 超时。 +- driver future 归属连接/pool 的结构化 scope；关闭 registry 时停止接收、等待借出连接归还并排空 driver task。 +- 对固定 QueryPlan 使用 statement cache；迁移 DDL 与一次性动态片段不进入无界缓存。 + +### 统一错误 + +driver 错误在所属 executor 边界映射为稳定 `DatabaseError`： + +- Connection、Timeout、Cancelled； +- Constraint（含 unique/foreign-key/check/not-null 分类）； +- NotFound； +- Decode/Encode； +- Migration/SchemaConflict； +- Busy/PoolExhausted； +- Driver（保留可记录的内部 code/cause，但不直接暴露到 HTTP）。 + +非幂等写入不自动重试。原始 driver cause 进入诊断链，公开 error 只暴露稳定字段。 + +## Schema 与 Seed 执行 + +### 内部状态 + +每个数据库使用一个保留前缀的内部 history 表记录： + +- Model identity； +- kind：schema/migration/seed； +- logical name/revision； +- normalized checksum； +- applied_at。 + +history 不是目标 schema 的替代品。每次启动仍核对数据库 catalog；history 与实际 schema 冲突时拒绝继续，不猜测修复。 + +### 启动流程 + +- PostgreSQL 使用数据库级 advisory lock；SQLite 使用写事务锁，防止多个进程同时迁移同一 schema。 +- 表不存在：直接创建当前最终表、索引与约束，不回放 `from` 或历史 migrate。 +- 表存在：catalog -> normalized actual schema -> diff -> validation plan -> DDL plan。 +- 收紧变化先运行只读验证 SQL，任一失败都不执行 DDL。 +- 命名 migrate 校验未执行/内容未改，再与 DDL/history 在一个连接事务中提交。 +- Seed 在 schema 成功后执行，按唯一键 conflict-do-nothing；只忽略目标唯一冲突。 +- 多连接无法拥有全局原子迁移。各连接独立加锁和提交，按连接名稳定顺序执行；后续连接失败时应用不启动，已完成连接依靠 checksum 幂等重跑。 + +## 查询执行数据流 + +```text +generated Model function + -> evaluate typed arguments + -> acquire connection or reuse hidden transaction + -> dialect renders fixed QueryPlan + -> prepare/cache statement + -> bind concrete values + -> execute / pull rows + -> generated row decoder + -> Model/List/Page/Cursor result + -> release connection or retain for bounded RowStream +``` + +- to-one include 与主查询共享受控 join 计划。 +- to-many include 在获得父页后用一个有界 `IN (...)` 批次查询并按父 ID 归组；超过参数上限时按固定上限分批，不按父记录逐条查询。 +- list 的 count 与 data 查询按同一条件树生成；cursor 只执行 data 查询。 +- stream 持有连接和 statement 生命周期，消费者停止、Task 取消或 fault 时通过 Drop/scope 清理归还。 + +## Transaction 执行 + +- 最外层 transaction 从已验证 slot 获取一个连接并 begin。 +- 生成的 `DatabaseContext` 携带 `ConnectionSlot + TransactionHandle`，只沿隐藏参数传递。 +- 普通 Model call 检测到上下文时验证 slot 相同并直接使用 handle；无上下文时从 pool 获取。 +- 同连接嵌套 transaction 只增加逻辑作用域，不 begin/commit。 +- 函数成功时最外层 commit；error/fault/取消路径统一进入 rollback guard，rollback 完成后再传播原始失败。 +- rollback 自身失败追加到诊断 cause，不覆盖原始业务 error/fault。 +- transaction context 不能进入 `run`、Group、Channel、Stream producer 或返回值，因此不存在逃逸连接。 + +## 日志与执行边界 + +- runtime 维护一个结构化 `DiagnosticContext`，包含 fault/error code、source span、request/task id 和 cause chain。 +- lower layer 只附加 cause/context，不记录根日志；HTTP、ownerless Task、CLI 等根边界决定是否记录。 +- HTTP 未映射失败在 rollback 后返回通用 500；显式 handler 可把预期 error 映射为 4xx。 +- 日志 front-end 先检查 level，再构造字段。队列容量固定；debug/info 满时丢弃并累计，warn/error 等待容量。 +- 文件 sink 的编码、轮转和压缩策略不在本任务冻结；先完成有界队列、stderr emergency path 和一次记录合同。 + +## 性能设计 + +- 生成代码只保留可达 Model、字段 decoder、QueryPlan 和 driver profile。 +- 相同静态 SQL shape 使用每连接 statement cache；值变化不创建新 SQL 字符串或缓存项。PostgreSQL 每连接缓存暂定上限为 64；to-many 参数批次生成的动态 SQL 不进入缓存，最终上限由阶段 10 基准收敛。 +- page size、to-many 子项、IN 批次、批量写入、pool、blocking、RowStream、日志全部由一个明确上限约束。 +- SQLite 默认一连接，PostgreSQL 默认零预热小池；最终默认值由 64/128 MiB 定向基准确定，并固化为命名常量和文档。 +- 性能基准实测 RSS/PSS/VmHWM、cgroup、线程/FD、吞吐、p50/p95/p99 和二进制体积；配置的 pool capacity 单独记录。生产 runtime 尚无低成本观测点的 pool wait、SQL 数量和数据库连接数必须标记为 `unavailable`，不能用 FD 或配置值推算冒充实测。 +- 构建产物以 manifest 的 SHA-256 和字节数校验后才运行；每个 ORM case 使用独立 bundle。PostgreSQL URL 的唯一 `{case}` 必须位于数据库名，敏感 URL 只进入权限为 `0600` 的 case setting，不进入 manifest/report。 +- transaction cancel 使用单连接 pool：先证明连接可用，再证明长事务占用 pool、客户端在响应前断开、连接在 handler timeout 前归还且事务行数为零，避免把 handler timeout 回滚误报为请求取消。 +- 先验证稳定内存和正确回收，再优化 SQL 生成、statement 命中或 decoder；不以缓存未绑定 SQL 或扩大队列换吞吐。 + +## 风险与处理 + +- 隐式挂起和错误传播会触及编译器、官方库和全部异步 fixture，应先单独完成语义迁移并保持 Rust runtime 行为，再叠加 ORM。 +- SQLite DDL 经常需要重建表；schema planner 必须以真实 catalog fixture 验证，不能只比较生成 SQL 字符串。 +- 外部 setting 与驱动裁剪可能不一致；构建记录 capability，启动严格校验，错误要求重建，不加载动态插件。 +- 多数据库迁移无法跨连接原子提交；稳定顺序、单连接事务和 checksum 幂等是唯一可维护恢复路径。 +- PostgreSQL 测试依赖真实服务；实施阶段只运行隔离的定向 fixture，不连接开发或生产数据库。 + +## 回滚边界 + +- 每一实施阶段必须在独立语义门槛下完成；不在半迁移状态同时保留新旧源码语法。 +- 自动 schema 操作在提交前不改 history；失败回滚。跨连接已有成功迁移不反向回滚，由幂等重启继续。 +- 发布二进制始终保留上一个版本；危险命名 migrate 需要先有数据库备份，自动回滚程序版本不能逆转已提交的数据迁移。 diff --git a/.trellis/tasks/09-13-typed-orm/implement.jsonl b/.trellis/tasks/09-13-typed-orm/implement.jsonl new file mode 100644 index 0000000..bf60dd1 --- /dev/null +++ b/.trellis/tasks/09-13-typed-orm/implement.jsonl @@ -0,0 +1,10 @@ +{"file":".trellis/spec/backend/compiler-contracts.md","reason":"model、查询计划、资源和公开 API 必须在 checker/HIR/native 中保持静态合同"} +{"file":".trellis/spec/backend/toolchain-and-library.md","reason":"数据库公共包、驱动选择、runtime 构建与缓存必须复用现有工具链边界"} +{"file":".trellis/tasks/09-13-typed-orm/research/architecture-options.md","reason":"依据现有无反射 AOT 和资源语义选择 ORM 表面"} +{"file":".trellis/spec/backend/error-handling.md","reason":"实现默认错误传播、事务回滚、根边界处理和 C012 合同"} +{"file":".trellis/spec/backend/quality-guidelines.md","reason":"检查数据库测试边界、内存限制、无重复驱动逻辑和最小验证"} +{"file":".trellis/tasks/09-13-typed-orm/research/implicit-suspension.md","reason":"统一删除 Dever async/await，并保留编译器推导协程、run/wait/stop 和同步入口裁剪合同"} +{"file":".trellis/tasks/09-13-typed-orm/research/field-types.md","reason":"实现隐式字段、常用数据库类型、Uuid、choice、关联 ID、Seed 和索引语义"} +{"file":".trellis/tasks/09-13-typed-orm/research/schema-change-matrix.md","reason":"按变更类别实现自动 diff、数据验证、from、命名 migrate 和两种方言的等价结果"} +{"file":".trellis/tasks/09-13-typed-orm/research/query-api.md","reason":"实现已确认的 Model 短函数、块式查询、分页、关联、事务和连接解析表面"} +{"file":".trellis/tasks/09-13-typed-orm/research/driver-selection.md","reason":"接入 rusqlite/deadpool-sqlite 与 tokio-postgres/deadpool-postgres，并按 runtime profile 裁剪驱动"} diff --git a/.trellis/tasks/09-13-typed-orm/implement.md b/.trellis/tasks/09-13-typed-orm/implement.md new file mode 100644 index 0000000..a4950d9 --- /dev/null +++ b/.trellis/tasks/09-13-typed-orm/implement.md @@ -0,0 +1,297 @@ +# 类型安全 ORM 实施清单 + +## 开始前门槛 + +- [x] 用户在最终规划摘要之后明确批准进入实现，并要求增加 `examples/dever/cms` 项目。 +- [x] 运行 `python3 ./.trellis/scripts/task.py start 09-13-typed-orm`，加载 `trellis-before-dev` 上下文。 +- [x] 核对 `Cargo.lock` 中待加入驱动与当前 Rust 1.85/MSRV、license 和 feature；只锁定实际使用的最小 feature。 +- [x] 为每个阶段建立对应的根目录 `test/` 定向测试目标；不运行 workspace 全量测试或集成压测。 + +## 阶段 1：隐式挂起语言表面 + +目标：删除 Dever 源码 `async`/`await`，保留 Rust runtime 的真实异步执行。 + +- [x] 调整 lexer/parser/syntax/formatter，删除 function/handler async 与 `await(...)`，保留 `run`/`wait`/`stop`。 +- [x] 将 `FunctionClause.is_async` 拆为编译器推导的 HIR `suspends`，复用现有调用依赖图传递基础挂起 effect。 +- [x] 更新 checker：普通调用自动按目标 effect 顺序等待；`run` 接受可启动调用，Task/Group 仍保持结构化所有权。 +- [x] 更新 native emitter：同步可达图生成直接入口，挂起可达图生成唯一 Tokio 入口和内部 await。 +- [x] 更新 API snapshot、Markdown 合同、官方 `dever.task/net/http/websocket/sse` 源码、examples 与相关 fixture，不保留旧语法。 +- [x] 清理旧 `AwaitCall`、`TaskStart`、公开 async 专属诊断和死分支，保留 runtime Rust async 节点。 + +定向验证： + +```bash +cargo fmt --all -- --check +cargo test -p dever-tests --test structured_concurrency +cargo test -p dever-tests --test async_native +cargo test -p dever-tests --test async_network +cargo test -p dever-tests --test http_library +cargo test -p dever-tests --test live_library +``` + +完成条件：HTTP/TCP/WS/SSE 的 Dever fixture 不含源码 async/await，协议、背压、取消和同步程序无 Tokio 入口的定向测试通过。 + +## 阶段 2：默认错误传播与执行边界 + +目标：建立 `fail`、`result(call)`、推导错误集合和运行时统一兜底。 + +- [x] 在 AST/HIR 增加 `Fail`、`CaptureResult` 和静态 `FailureSet`，通过现有 dependency worklist 传递。 +- [x] 将普通调用降低为自动传播，`result(call)` 降低为编译器内部 captured result；不增加用户泛型或每函数 error 声明。 +- [x] 拆分官方库中“正常状态 choice”与 error effect，迁移现有 Result choice 的调用方。 +- [x] 扩展 C012，覆盖默认传播、captured result 穷尽消费、覆盖、丢弃、仅日志和 `recover`。 +- [x] 生成可达应用的具体 Rust error enum、cause chain 和 source span；API snapshot 显示每函数精确错误集合。 +- [x] 建立 HTTP、Task、CLI 根边界：事务回滚后分别返回 500/Task failure/非零退出，且只记录一次。 +- [x] 增加 `dever.log` level API、共享诊断上下文和有界队列；文件 sink 格式/轮转仍不在本阶段冻结。 + +定向验证： + +```bash +cargo fmt --all -- --check +cargo test -p dever-tests --test contract_syntax +cargo test -p dever-tests --test contract_semantics +cargo test -p dever-tests --test contract_execution +cargo test -p dever-tests --test hello_native +cargo test -p dever-tests --test http_engine +``` + +完成条件：默认传播、主动捕获、自定义 fail、C012、HTTP/Task/CLI 边界及一次日志都有正反 fixture。 + +## 阶段 3：Model 语法与静态 Schema + +目标：编译器能从一个 Model 文件得到完整静态持久化合同。 + +- [x] 增加 Model 路径识别和 Model 专属 declaration/field metadata AST。 +- [x] 实现文件名/package/唯一公开 record 校验，合成 `id`、`created_at`、创建输入和更新输入。 +- [x] 实现 `Bool/Int/Float/Decimal/Text/Bytes/Uuid/DateTime/Date/Time/Duration/Json/choice/model.id` 及 nullable/static bounds。 +- [x] 实现 `generated/default/index/unique/from`、复合 index/unique、relation、seed、migrate 的解析、格式化和源码诊断。 +- [x] 实现不透明 Model ID 的解析与类型检查、隐式 FK/index、choice labels 与 `Choice.options`。 +- [x] 建立与方言无关的 `ModelSchema` 规范化和稳定 checksum；检查同连接内表/对象名冲突。 +- [x] API/Markdown 输出 Model、隐式字段、choice options 和持久化元数据，但不暴露内部输入类型噪音。 + +建议新增测试目标：`model_syntax`、`model_semantics`、`model_schema`。 + +定向验证： + +```bash +cargo fmt --all -- --check +cargo test -p dever-tests --test model_syntax +cargo test -p dever-tests --test model_semantics +cargo test -p dever-tests --test model_schema +``` + +完成条件：字段/索引/关联/Seed/migrate 的合法和非法边界在 native build 前有稳定诊断，schema checksum 不受格式或方言影响。 + +## 阶段 4：配置、目录与 RuntimeProfile + +目标：严格加载外部 setting，并按实际能力构建最小 runtime。 + +- [x] 增加严格 `setting.json` 结构化解析和可执行文件目录定位。 +- [x] CLI 应用命令以项目根为输入，统一把 `<project>/module` 交给 SourceMap；fixture 的显式源码根入口保持内部测试用途，不做应用级自动回退。 +- [x] 创建固定 `data/db|upload|log|cache|tmp` 目录；setting 缺失/无效直接失败。 +- [x] 实现显式连接、根包连接、default 的一次性解析和生成 Model 句柄缓存。 +- [x] 将 SQLite/PostgreSQL/config 依赖拆为 runtime optional features。 +- [x] native build 从 setting 提取驱动 capability，构建对应 runtime rlib；capability 进入 cache identity，敏感配置不进入生成源码或 identity。 +- [x] 启动时复核实际 setting 的驱动集合和 transaction binding；未打包驱动明确失败。 +- [x] 增加 base/sqlite/postgres/both 四种产物的依赖和二进制体积报告。 + +建议新增测试目标：`application_config`、`native_profiles`。 + +定向验证： + +```bash +cargo fmt --all -- --check +cargo test -p dever-tests --test application_config +cargo test -p dever-tests --test native_cache +cargo test -p dever-tests --test native_profiles +``` + +完成条件：cwd 改变不影响定位，缺失配置无回退；SQLite-only/PostgreSQL-only 产物不含另一驱动。 + +## 阶段 5：共享 QueryPlan 与 SQLite 纵向闭环 + +目标：先用一个真实数据库打通静态 CRUD 和有界查询。 + +- [x] 实现 Model 短函数符号与匿名 create/update/query 块的上下文类型检查。 +- [x] 建立共享 QueryPlan、条件树、参数槽、稳定排序、page/cursor/stream 和结果 shape。 +- [x] 接入 `deadpool-sqlite` + `rusqlite bundled` 和有界 pool。 +- [x] 实现当前 CRUD 所需 SQLite dialect：标识符、占位符、类型/值编码、RETURNING、CHECK/FK/index。 +- [x] 生成 Model 专属 bind/row decoder；所有值参数化，不使用反射或动态 Map。 +- [x] 实现 create/get/first/list/cursor/count/exists/stream/update/delete/upsert 与有界 create_many。 +- [x] 验证 stream/取消/超时/pool exhaustion 后连接归还。 + +建议新增测试目标：`orm_query`、`sqlite_orm`。 + +定向验证： + +```bash +cargo fmt --all -- --check +cargo test -p dever-tests --test orm_query +cargo test -p dever-tests --test sqlite_orm +``` + +完成条件：临时 SQLite 数据库完成 CRUD、分页、游标、stream、约束错误和清理；目标机器路径不需要系统 SQLite。 + +## 阶段 6：Schema Diff、迁移与 Seed + +目标：SQLite 先完成空库、升级、危险变化拒绝和幂等初始化。 + +- [x] 建立 catalog 列 introspection 与 history 中的规范化 actual schema。 +- [x] 实现内部 history 表、schema/seed checksum 和并发启动锁。 +- [x] 实现空库最终 schema 创建，不回放 `from`/历史 migrate。 +- [x] 实现安全字段 diff、约束数据库验证和 SQLite 表重建。 +- [x] 实现字段 `from`、稳定命名 migrate、内容不可改和事务性 history。 +- [x] 实现 Seed 唯一身份检查、缺失插入、非唯一冲突失败和 checksum 快速跳过。 +- [x] 对每一种自动/拒绝/显式变化保留真实 catalog 和数据断言，不只断言 SQL 字符串。 + +建议新增测试目标：`orm_migration`、`orm_seed`。 + +定向验证： + +```bash +cargo fmt --all -- --check +cargo test -p dever-tests --test orm_migration +cargo test -p dever-tests --test orm_seed +``` + +完成条件：新库、旧库 rename、长度放宽/收紧、唯一/FK、重建失败回滚、重复启动和 Seed 追加全部通过。 + +## 阶段 7：PostgreSQL 复用接入 + +目标：只增加方言和 executor，不复制 Model/QueryPlan/schema 语义。 + +- [x] 接入 `deadpool-postgres` + `tokio-postgres`；使用 `tokio-postgres-rustls` 支持系统根 TLS 校验和显式禁用。 +- [x] 实现当前 CRUD 所需 PostgreSQL dialect、值编码/解码、RETURNING、catalog introspection 和 advisory migration lock。 +- [x] 对固定 QueryPlan 使用 statement cache；缓存 key 只来自编译器生成的静态 SQL shape。 +- [x] 复用 SQLite 同一 CRUD/schema/Seed fixture 数据集和断言，仅替换隔离数据库适配。 +- [x] 验证 pool wait、连接断开、取消、timeout、constraint code 映射、TLS 主机名/证书拒绝和 shutdown drain。 + +建议新增测试目标：`postgres_orm`，默认标记为 ignored；仅在仓库根 `config/setting.json` 的 `database.postgres_test` 明确配置隔离测试 PostgreSQL 时手动运行，不接触现有服务。 + +定向验证： + +```bash +cargo fmt --all -- --check +cargo test --offline -p dever-tests --features postgres --test postgres_orm -- --ignored --exact postgres_orm_uses_the_configured_isolated_database +``` + +完成条件：PostgreSQL 与 SQLite 对共享 fixture 返回相同逻辑结果和 schema revision，差异仅存在于 dialect/executor。 + +## 阶段 8：Transaction 与多数据库 + +目标：实现无显式句柄、单连接静态约束的 transaction 函数。 + +- [x] 实现 `FunctionKind::Transaction` 检查和 `DatabaseEffect(ModelId)` 传播。 +- [x] build/runtime 分别解析 transaction Model 集合到唯一 connection。 +- [x] native 生成隐藏数据库上下文参数和最外层 begin/commit/rollback guard。 +- [x] 同连接嵌套 transaction 复用 handle；不创建 savepoint。 +- [x] 拒绝无数据库 effect、跨连接、动态切库、transaction context 逃逸和并发数据库子任务。 +- [x] 验证 error/fault 时先 rollback，再传播原始失败；rollback failure 只追加 cause。 +- [x] 验证 default、根包、显式 report 三种连接绑定和多连接启动顺序/幂等恢复。 + +建议新增测试目标：`orm_transaction`、`orm_connections`。 + +定向验证： + +```bash +cargo fmt --all -- --check +cargo test -p dever-tests --test orm_transaction +cargo test -p dever-tests --test orm_connections +cargo test --offline -p dever-tests --features postgres --test postgres_orm -- --ignored --exact postgres_orm_uses_the_configured_isolated_database +``` + +完成条件：两种数据库都验证提交、回滚、嵌套复用、取消和连接归还，跨数据库在执行前被拒绝。 + +## 阶段 9：Relation 与原生 SQL 逃生口 + +目标：完成后端常用关联读取和复杂查询能力。 + +- [x] 实现 to-one relation 推导、显式反向 to-many relation 和 `with` 计划。 +- [x] to-one 使用受控 JOIN；to-many 按父页批量加载、稳定排序、数量和参数批次上限。 +- [x] 加入生成代码 query-shape 探针，断言父记录数增长不会产生 N+1。 +- [x] 实现参数化 typed native SQL，显式 SQLite/PostgreSQL 方言和具体 Model/record decoder。 +- [x] 拒绝 SQL 插值、动态列 Map、结果列缺失和类型不匹配。 +- [x] 建立 `examples/dever/cms/module/{user,news}/{model,service}`，用用户注册、新闻发布和 author 外键覆盖 Model/Service/transaction 纵向流程。 + +建议新增测试目标：`orm_relations`、`orm_native_sql`。 + +定向验证： + +```bash +cargo fmt --all -- --check +cargo test -p dever-tests --test orm_relations +cargo test -p dever-tests --test orm_native_sql +``` + +完成条件：两种数据库返回相同关系结果，to-many SQL 数量有固定上界，原生 SQL 保持参数化和静态解码。 + +## 阶段 10：小内存与压力验证 + +目标：在正确性完成后冻结小机器默认值并找出真实瓶颈。 + +- [x] 扩展 `test/performance` 构建器，产出 sync/base、SQLite-only、PostgreSQL-only 和 HTTP+JSON+DB fixture。 +- [x] 构建和测量分离；运行阶段只启动已打包二进制。 +- [x] 添加 SQLite/PG idle、CRUD、list/cursor/stream、pool saturation、transaction cancel 场景；PG 仅在根 `config/setting.json` 显式配置 `database.postgres_test` 时运行。 +- [x] 报告 binary size、RSS/PSS/VmHWM、cgroup current/peak/OOM、线程/FD、配置 pool capacity、吞吐和 p50/p95/p99；当前无法直接观测的连接数、pool wait、SQL 数明确标记 unavailable。 +- [x] 先跑 1 核/128 MiB，再跑 64 MiB；不设置假 QPS 门槛，以无 OOM、无错误、内存有界、连接回收和相对对照为硬条件。 +- [x] 根据报告收敛 SQLite pool、PG min/max、page size、row buffer、IN batch 和 statement cache 上限；两档最大 RSS 高水位均约 7 MiB、OOM 为 0，现有有界默认值有充分余量，因此不做无数据依据的调参。 +- [x] 最后做 cleanup：删除临时 fixture、重复驱动分支、旧 async/await 代码和未使用 feature。 +- [x] 以 `examples/dever/cms` 验证项目级 check/run/build 和 SQLite-only 打包产物。 +- [x] 以 `examples/dever/cms` 验证真实业务内存基线。 + +最小基准检查： + +```bash +python3 -m unittest discover -s test/performance -p 'test_runner.py' -v +python3 test/performance/run.py run \ + --artifacts target/performance/orm-current \ + --output target/performance/orm-128m \ + --suite orm --duration 10 --warmup 2 --repeats 3 \ + --cgroup-parent /sys/fs/cgroup/system.slice \ + --memory-mib 128 --cpu-quota 1 +``` + +该运行会启动隔离的本地测试进程并使用 cgroup；执行前必须说明，确认不会影响现有服务。PostgreSQL 场景只使用专门测试实例。 + +## 最终自检 + +- [x] `rg` 确认 Dever 官方库、examples、fixture 不再出现源码 `async`/`await`。 +- [x] `rg` 确认没有旧 ORM、链式 Query、显式 Database/Transaction 或双 CRUD 表面。 +- [x] 检查生成 SQL 的所有值都通过 bind 参数，只有编译器生成的标识符进入 SQL。 +- [x] 检查 SQLite/PostgreSQL 共享 QueryPlan/schema/error 流程，没有复制业务算法。 +- [x] 检查所有队列、池、page、batch、stream、日志和 blocking 路径有明确上限。 +- [x] 检查最终改动没有调试输出、TODO、临时测试、无用依赖或未使用兼容分支。 +- [x] 只报告实际运行的定向验证；未运行 PostgreSQL、压测或受权限限制的 cgroup 检查必须明确标注。 + +## 高风险文件与回滚点 + +- 编译器语法/HIR：`crates/dever-core/src/{lexer,parser,syntax,hir,format}.rs`。 +- 效果与错误：`crates/dever-core/src/contracts/*`、`check/*`、`native/*`、`api.rs`。 +- native profile/cache：`crates/dever-core/src/native/build.rs`、`native/build/cache.rs`、workspace/runtime Cargo 配置。 +- Runtime：新增 config/database/orm/dialect 模块与 `crates/dever-runtime/src/task.rs` 的 bounded blocking 接口。 +- 官方库和示例：`library/dever/*`、`examples/dever/*`。 +- 所有长期测试只能放在仓库根 `test/`。 + +阶段 1、2 属于语言合同迁移，完成后不回退旧语法；阶段 3-9 每阶段保持编译器与 runtime 同步可用。schema 变更只靠数据库事务/history 回滚，已成功提交的命名迁移不由程序版本回退。 + +## 2026-09-17 收口验证 + +- `cargo check --offline -p dever-runtime --features postgres` 通过。 +- `application_config` 3/3 通过；PostgreSQL 配置默认值和超时范围已覆盖。 +- 原 `postgres_orm` 无环境变量时提前返回却显示 1/1 通过，该证据无效。测试现已改为默认 ignored，并只从根 `config/setting.json` 读取 `database.postgres_test`；真实 PostgreSQL 验收仍未运行。 +- PostgreSQL 配置入口回归：performance runner 27/27 定向单测通过，`cargo check --offline -p dever-tests --features postgres --test postgres_orm` 通过；未连接数据库。 +- handler effect 替换回归通过；全部维护中示例按 `<project>/module` 统一源码根通过静态检查。 +- `cms_project` 已完成 1/1 check/run/build/打包程序验证；最终复验因根文件系统空间不足在链接测试二进制前中止，不是测试逻辑失败。 +- rustfmt、clippy 组件未安装；全量测试、真实 PostgreSQL、二进制体积、64/128 MiB 和 cgroup 压测未运行。 + +## 2026-09-17 最终验收 + +- 共享 fixture 已落在 `test/dever-tests/tests/support/orm_fixture.rs`；SQLite 与 PostgreSQL 使用同一 schema/Seed、CRUD、create_many、upsert、list/cursor/stream、transaction 和 catalog 断言。SQLite 1/1、隔离 PostgreSQL 1/1 最终复验通过。 +- PostgreSQL 真实验收修复并覆盖三处仅靠编译无法发现的问题：cursor 可空参数显式 SQL 类型 cast、UUID 原生二进制参数直接使用 `::uuid`、shutdown SQLSTATE 映射为 connection error。pool wait、断连、timeout、constraint、TLS 拒绝和 drain 均通过。 +- PostgreSQL 测试与性能工具只从仓库根 `config/setting.json` 读取 `database.postgres_test`；没有数据库 URL/TLS 环境变量或命令行覆盖入口。测试实例为独立临时 PostgreSQL，未接触现有 5432/5433 服务。 +- 四 profile 使用同一最小源码：base 413,880 B、sqlite 2,562,808 B、postgres 2,711,632 B、both 4,676,472 B。依赖报告确认单驱动产物不包含另一驱动，证据见 `target/performance/typed-orm-build-v2/profile-report.json`。 +- 1 CPU、无 swap、128 MiB 与 64 MiB 各完成 48/48 ORM case（SQLite/PostgreSQL × 8 场景 × 3 重复），两档 OOM/OOM kill 均为 0，最大 RSS 高水位分别 7,184,384 B 与 7,102,464 B；所有 cancel case 均确认池饱和、零提交和超时前连接回收。报告见 `target/performance/typed-orm-128m-v2/report.json` 与 `target/performance/typed-orm-64m/report.json`。 +- 现有 SQLite pool=1、PostgreSQL min/max=0/4、page size=100 以及有界 row/IN/statement 路径均有充分内存余量；本轮数据不支持进一步缩减，保持默认值。 +- CMS 128 MiB 与 64 MiB 各 3/3 完成，业务基线每次为 2 条记录，OOM 为 0，RSS 高水位中位数分别 5,513,216 B 与 5,562,368 B。报告见 `target/performance/cms-128m/report.json` 与 `target/performance/cms-64m/report.json`。 +- 定向回归：performance runner 32/32、`application_config` 3/3、`orm_query` 8/8、`cms_project` 1/1（实际 check/run/build/打包程序）通过；`cargo check --offline -p dever-runtime --features postgres` 通过。 +- 最终独立复核修正了两个收口问题：普通 SQLite `setting.json` 缺少 `database.postgres_test` 时现在按未启用 PostgreSQL 处理，只有显式但非法的测试连接才失败；真实 PostgreSQL fixture 使用 PID+纳秒+计数隔离 schema，并在断言失败/panic 后清理再恢复原始失败。修正后 runner 32/32、PostgreSQL cargo check 与真实验收 1/1 通过，结束后隔离 schema=0、活动连接=0。 +- rustfmt、clippy 组件未安装，未运行全量 workspace 测试；以上限制不影响本任务已列出的定向验收。 diff --git a/.trellis/tasks/09-13-typed-orm/prd.md b/.trellis/tasks/09-13-typed-orm/prd.md new file mode 100644 index 0000000..1db57db --- /dev/null +++ b/.trellis/tasks/09-13-typed-orm/prd.md @@ -0,0 +1,211 @@ +# SQLite 与 PostgreSQL 类型安全 ORM + +## 目标 + +为 Dever 增加编译期类型检查、AOT 生成具体代码且无需反射的 ORM。第一轮完整支持 SQLite 和 PostgreSQL，共用 Model、查询计划、事务、迁移和错误合同；小内存机器只运行打包后的二进制，不承担 Rust 编译成本。 + +## 背景与约束 + +- Dever 是全新语言，不保留旧 ORM、旧 `async`/`await` 或其它源码兼容层，也不增加防御性回退路径。 +- 当前编译器没有用户泛型、反射、动态 `Value`、method 或 type-as-value；公开 ORM 必须保持静态、直接和可 AOT 降低。 +- Rust runtime 继续使用 Tokio 和真实异步 I/O。删除 Dever 源码中的 `async`/`await` 只改变语言表面，不把 HTTP、TCP、WebSocket、SSE 或数据库驱动改成阻塞线程模型。 +- SQLite 面向单机、小内存程序；PostgreSQL 面向并发后端。二者共享语义，不强行共享执行机制。 +- Model、Transaction、RowStream 等数据库资源沿用现有所有权、结构化任务、背压和取消清理原则。 + +## Model 与字段 + +- `model/` 下一个源文件定义一个 Model，继续使用普通 record `type`，不增加 `model` 关键字。 +- 文件只公开一个与文件名对应的 Model record；表名和列名按文件名、字段名精确派生，不做复数化猜测。 +- 同一文件集中字段、索引、关联、choice、Seed、迁移和紧贴持久化的数据访问操作，不增加外部 schema 映射文件。 +- 每个 Model 隐式拥有只读 `id` 和 `created_at: DateTime`。二者出现在完整查询结果中，但不出现在创建输入中；`updated_at` 不隐式生成。 +- `<model-package>.id` 是数据库生成的有符号 64 位正整数，也是 Model 专属不透明类型。PostgreSQL 使用 64 位 identity，SQLite 使用 `INTEGER PRIMARY KEY`；不同 Model 的 ID 不能混用，也不能按普通 `Int` 运算。 +- 关联字段直接引用目标主键，例如 `owner_id: app.model.user.id`、`parent_id: app.model.category.id?`。作为 Model 字段时自动生成外键和普通索引，但不触发隐式加载。 +- 第一轮字段类型包括 `Bool`、`Int`、`Float`、`Decimal(P, S)`、`Text`、`Bytes`、Model `.id`、`Uuid`、`DateTime`、`Date`、`Time`、`Duration`、`Json`、无载荷 choice 及其可空形式。 +- `List`、`Map` 和普通 record 不隐式序列化为列；结构化持久化必须显式使用 `Json`。 +- 字段元数据使用可解析、可格式化、可静态检查的语法，不使用字符串 tag。 + +### 字符串、字节与数值 + +- `Text` 表示无界 UTF-8 文本，`Text(max)` 表示最多 max 个 Unicode codepoint，`Text(min, max)` 表示 min 至 max 个 Unicode codepoint；不增加语义重复的 `Char`。 +- `Bytes`、`Bytes(max)`、`Bytes(min, max)` 使用相同边界形式，但长度单位是字节。 +- PostgreSQL 有界 Text 使用 `VARCHAR` 与必要的 CHECK；SQLite 使用 `TEXT` 与等价 CHECK，不能依赖 SQLite 忽略 VARCHAR 长度。 +- `Decimal(P, S)` 必须满足 decimal128 的精度边界；两种数据库往返后保持精确值，不静默舍入。 + +### UUID 与时间 + +- `Uuid` 是固定 128 位值，运行时不使用堆分配 Text。PostgreSQL 使用原生 UUID，SQLite 使用 16-byte BLOB；API/JSON/文本统一为小写、带连字符的 36 字符格式。 +- 普通 `Uuid` 由调用方提供；`Uuid generated` 由 ORM 创建 UUIDv7 并从创建输入省略；`generated` 与可空不能同时使用。 +- `Uuid` 不隐式表示关系，关系仍使用 Model `.id`。 +- `DateTime` 表示 UTC 毫秒时间点；`Date` 表示公历日期；`Time` 表示日内毫秒时间；`Duration` 表示固定毫秒时长，不使用包含月份换算的数据库 interval。 + +### Choice + +- Model 文件可声明供字段使用的无载荷 choice；只有文件名对应的 record 生成表。 +- 可选项写成 `Pending = "待处理"`。数据库和 API 存储左侧稳定变体名，右侧显示名称不入库。 +- 编译器生成保持声明顺序的只读 `Choice.options: Map<Choice, Text>`；一个 choice 的变体必须全部带显示名或全部不带。 +- 第一轮不支持 choice payload，也不创建 PostgreSQL 专有 enum。 + +## 索引、Schema 与迁移 + +- 单字段索引写为字段后的 `index` 或 `unique`；复合索引写为 `index(a, b)` 或 `unique(a, b)`。 +- 索引描述当前目标 schema，不写版本号或手工名称；编译器稳定生成数据库对象名并静态检查字段引用和重复列。 +- 当前 Model schema 是唯一目标。ORM 比较数据库系统目录、内部 history 与二进制嵌入的规范化 schema，不生成源码级 `table/` 状态目录。 +- 新表直接按当前最终 schema 创建；新增可空字段、有合法常量默认值的非空字段、普通索引等意图唯一的变化自动应用。 +- 长度、精度、可空性、默认值和约束直接修改目标字段。放宽变化自动应用；收紧变化先验证现有数据，数据不满足时整次迁移失败并要求命名 migrate，绝不截断、舍入、猜测补值或删除数据。 +- 简单字段重命名写为 `display_name: Text(64) from nickname`。已有旧表执行 rename；全新数据库忽略历史来源，只创建最终字段。 +- `from` 只供 schema 规划，不进入运行时字段、CRUD 输入、查询 SQL 或最终约束。 +- 开发者不维护数字 schema 版本。编译器根据规范化 schema 和变更计划生成 revision 校验值，嵌入二进制并记录到数据库内部 history。 +- 复杂数据变化使用 Model 文件内稳定命名的 `migrate <name> { ... }`，包括拆分、合并、跨行转换、删除前归档和显式 drop。 +- 一个命名迁移只成功执行一次；已执行迁移的规范化内容改变时拒绝启动。迁移和 history 写入在同一数据库事务内完成。 +- SQLite 无法直接 ALTER 的变化由方言层在事务内重建表；PostgreSQL 使用对应 DDL。两者完成后必须得到相同逻辑 schema revision。 +- 只有结构化迁移无法表达时，命名 migrate 才可包含明确方言的参数化原生 SQL；不承诺该 SQL 跨数据库复用。 + +## Seed + +- 每个 Model 文件最多一个无版本 `seed { ... }` 块，块内直接写多个匿名创建数据块，不重复 Model 名或 `create`。 +- Seed 数据按 Model 创建输入静态检查，不能填写 `id`、`created_at` 或其它 generated 字段。 +- 每条 Seed 必须完整提供至少一个单字段或复合唯一键，用于稳定识别初始化记录。 +- 编译器为规范化 Seed 生成校验值。内容未变化时启动不执行 Seed SQL；变化时在 schema 迁移成功后，以一个事务重新执行“缺失则插入”。 +- 唯一键冲突保留数据库现值；其它非空、CHECK、外键或解码错误必须失败。Seed 不覆盖或删除线上数据，修改/删除内置数据使用命名 migrate。 + +## 查询与内置操作 + +- Model 包生成唯一一套短函数：`create`、`create_many`、`get`、`first`、`list`、`cursor`、`count`、`exists`、`stream`、`update`、`delete`、`upsert`。 +- 业务代码直接调用 `user.list(...)`、`user.update(...)`，不暴露 `User.Query`、`User.List`、Active Record 实例方法或第二套 CRUD。 +- 创建、更新和查询配置使用上下文推导的匿名块；底层生成具体静态输入和结果类型，不引入动态字段 Map。 +- 查询只采用一次性配置块： + +```dever +users = user.list({ + where = status == UserStatus.Active + and age >= 18 + and (city == "北京" or city == "上海") + order = [created_at.desc, id.desc] + page = 1 + size = 20 + with = [owner] +}) +``` + +- 不提供 `user.where(...).order(...).list()` 链式查询器；链式查询不会提升数据库性能，却会要求新增 method、泛型查询对象和所有权语义。 +- 多个条件使用现有 `and`、`or`、`not` 和括号；支持比较、`in`、`between`、`contains`、`starts_with`、`ends_with`、null 与关联字段路径。 +- 查询表达式在编译期降低为类型检查后的 `QueryPlan`。字段、排序、关系和 SQL 结构不能来自运行时字符串；所有值通过参数绑定。 +- `list()` 默认返回第一页，默认 size 为 20，并受 `setting.json` 的统一上限约束；排序不是唯一键时自动追加 `id` 保证稳定分页。 +- `list` 返回 `items/page/size/total/pages`；`cursor` 返回 `items/next/has_more` 且不执行 count；`stream` 全量遍历但保持有界内存，不提供无界 `all()`。 +- 正向 to-one 由关联 ID 推导；反向 to-many 在 Model 中显式声明 `relation tasks = app.model.task.owner_id`。 +- `with` 才加载关系。to-one 使用受控 JOIN 或等价单次查询；to-many 按当前父页 ID 批量加载并归组，必须有排序和数量上限；禁止 lazy loading 和 N+1。 +- 多对多使用显式中间 Model，不生成隐藏表。 +- 复杂查询保留参数化、类型化的原生 SQL 逃生口。调用必须声明具体 Model/record 结果和方言；不返回动态行 Map，不允许 SQL 字符串插值。 + +## 数据库配置与绑定 + +- 应用项目根采用固定布局：`module/` 是唯一 Dever 源码根，`config/` 保存 `setting.json`，`data/` 保存运行数据。编译器从项目根加载时按 `module/` 的相对路径派生逻辑 package，`module` 本身不进入 package 名；`config/` 和 `data/` 不参与源码扫描。 +- 可执行文件按自身所在目录解析固定布局，不使用进程当前目录，也不提供环境变量覆盖： + +```text +app/ + app + config/ + setting.json + data/ + db/ + upload/ + log/ + cache/ + tmp/ +``` + +- `config/setting.json` 是唯一配置文件；缺失、无法解析或数据库配置无效时启动失败。运行时按需创建 `data/` 子目录，不生成默认 setting。 +- `setting.json` 顶层使用 `app`、`http`、`database`、`log` 等对象；`database` 直接映射连接名，不增加 `connections` 包装层。 +- `database.default` 必须存在；其它名称由应用定义，例如 `database.report`。 +- Model 连接按以下顺序在启动时解析并缓存： + 1. Model 文件显式 `database report`； + 2. 否则按根包名查找，例如 `app.model.user` 查 `database.app`； + 3. 根包连接键不存在时使用 `database.default`。 +- 只有“根包连接键不存在”允许回落 default。显式连接不存在、default 不存在或被选连接无效都直接启动失败。 +- 普通 CRUD 不接收 Database 或 Transaction 参数，也不允许按运行时字符串切换连接。 +- SQLite 使用 bundled `rusqlite`，同步调用进入现有有界 blocking 通道，默认小连接数并明确单写者约束。 +- PostgreSQL 使用 `tokio-postgres` 与有界异步连接池；池容量、等待和超时全部受配置上限约束。TLS 必须显式选择系统根验证或禁用，不允许验证失败后静默回落明文。 +- 编译器按可达数据库能力选择 runtime profile。SQLite-only 二进制不链接 PostgreSQL 驱动，PostgreSQL-only 二进制不链接 SQLite；驱动集合进入原生产物缓存身份。 + +## CMS 示例项目 + +- 在 `examples/dever/cms/` 建立可检查、可运行、可构建的完整项目，而不是只展示孤立语法片段。 +- 固定目录为 `config/setting.json`、`data/`、`module/cms.dever`、`module/user/model/`、`module/user/service/`、`module/news/model/`、`module/news/service/`。 +- `module/user` 和 `module/news` 分别是组件根；逻辑 package 使用 `user.model.*`、`user.service.*`、`news.model.*`、`news.service.*`，入口为 `cms.main`。 +- Model 展示隐式 `id`/`created_at`、Uuid、choice、Seed、索引和跨组件关联；Service 只承载注册、发布等事务业务，不包装 Model 已生成的普通 CRUD。 +- 示例默认使用 SQLite 和相对路径 `data/db/cms.db`，因此打包二进制可在无外部数据库服务的机器上直接运行；PostgreSQL 通过独立定向 fixture 验证，不让 CMS 示例依赖现有服务。 + +## 顺序调用、并发与事务 + +- Dever 源码删除 function/handler 的 `async` 标记和直接调用外层的 `await`。普通调用表示顺序等待完成，编译器根据传递挂起 effect 自动降低为 Rust coroutine。 +- 只有 `run(call)` 显式开始并发，`wait(task/group)` 消耗并等待结果，`stop(task/group)` 请求取消并等待结构化清理。 +- 同步入口的可达图没有挂起 effect 时不启动异步 runtime；包含网络、数据库、定时器、Channel 或异步 Stream 时只启动一个有界 Tokio runtime。 +- 事务使用专用函数声明： + +```dever +transaction create_order(...) (...) { + order = order.create({...}) + inventory.update({...}) +} +``` + +- 源码不出现 `@transaction`、Database、Transaction、begin、commit、rollback、`async` 或 `await`。 +- 编译器分析完整受控调用链中的 Model 数据库 effect，静态推导唯一连接，并通过隐藏参数传递事务上下文；不使用线程局部、任务局部或全局“当前事务”。 +- 最外层 transaction 函数开启物理事务；正常完成提交，error、fault 或取消先回滚再传播原始失败。 +- 同连接的嵌套 transaction 函数复用外层事务，不开启 savepoint，也不提前提交。 +- 无数据库 effect、跨连接、动态无法确定连接，或在 `run`/并发子任务中执行数据库 effect 的 transaction 函数在编译期拒绝。第一轮不支持分布式事务。 + +## Error、Fault 与日志边界 + +- choice 变体继续用 `error` 标记可恢复失败；编译器保留并扩展 C012，禁止丢弃、覆盖或仅用日志观察失败。 +- 普通可失败调用直接返回成功值，失败自动向当前边界传播，不写 `?`： + +```dever +user = user.get(id) +``` + +- 需要主动处理完整结果时使用一个核心操作 `result(call)`，再由现有穷尽函数分句处理成功和 error 变体。 +- 自定义业务失败使用 `fail(OrderError.OutOfStock(id))`。错误集合由编译器沿调用图静态推导并显示在 API、文档和编辑器信息中，不要求每个函数重复声明错误类型。 +- `error` 表示可恢复业务/外部失败；`fault` 表示程序不变量、编译器已知资源或 runtime 故障。普通业务函数第一轮不能捕获 fault。 +- HTTP 未处理 error/fault 只终止当前请求，回滚事务后返回不泄露内部细节的 500，并在服务端根边界记录一次；预期 4xx 必须由 handler 显式映射。 +- Task 失败由 `wait`/`result` 返回；无 owner 的任务边界记录一次。CLI 输出 stderr 并返回非零退出码。 +- 日志是观察，不是错误处理；`log.error` 不满足 C012。底层驱动、事务和 Service 不重复记录同一失败。 +- 正式日志 API 采用 `dever.log.debug/info/warn/error(message, fields)` 与共享结构化诊断上下文。队列有界：debug/info 满时丢弃当前事件并累计数量，warn/error 施加背压；logger 自身失败或最终 fault 直接写 stderr。 + +## 性能与资源边界 + +- ORM 只生成具体 Model 输入、行解码和静态查询计划；共享查询规划、schema diff 和方言实现，不为每个 Model 复制算法。 +- 查询使用绑定参数和可复用预编译语句；结果按行拉取，有界缓冲，不默认收集全表。 +- 连接池、SQLite blocking、批量写入、分页、关系子列表、stream 和日志队列全部有明确上限。 +- 不静默重试非幂等写入；约束冲突、连接失败、超时、取消和解码失败保持可区分。 +- 小机器验收以打包二进制的稳定 RSS、空闲连接成本、并发请求内存斜率和取消后的资源回收为准，不以构建机编译耗时或峰值内存为主要目标。 + +## 验收标准 + +- 合法 Model 在原生构建前生成确定 schema、创建输入、查询结果和 CRUD；字段、索引、Seed、关系或查询中的非法引用产生源码定位诊断。 +- 未声明 `id`/`created_at` 的 Model 仍能读取这两个字段，创建时无需填写；不同 Model ID 无法误传。 +- SQLite 与 PostgreSQL 对全部首批字段类型、默认值、长度、精度、choice、外键和索引产生等价逻辑约束并正确往返；PostgreSQL 系统根 TLS 校验和显式明文模式分别通过定向验证。 +- 同一个查询配置块在两种数据库生成正确的参数化 SQL，支持分页、游标、stream、复合条件和显式关联加载；to-many 不产生 N+1。 +- `list` 默认有界并返回 total/pages，`cursor` 不执行 count，`stream` 在慢消费者下保持有界内存。 +- 从空数据库启动时直接创建最终 schema 并执行缺失 Seed；重复启动无重复数据。已存在数据库能安全自动升级，危险变化在没有有效 migrate 时保持原 schema/数据并启动失败。 +- 带 `from` 的稳定二进制既能初始化新数据库，也能保留旧列数据完成 rename；自动 revision 和命名迁移 history 可重复验证且不可静默改写。 +- 默认、根包和显式连接按既定优先级绑定；显式缺失连接和无效配置启动失败。事务调用链只能使用一个已解析连接。 +- create/get/update/delete/list、事务提交、事务回滚和取消清理分别通过 SQLite 与 PostgreSQL 的定向真实数据库 fixture。 +- 普通 Dever 网络和数据库调用无需 `async`/`await` 仍是非阻塞 I/O；只有 `run` 建立业务并发，同步程序不启动 Tokio。 +- 可恢复 error 不能被丢弃或用日志代替处理；默认传播、`result(call)`、`fail(...)`、HTTP/Task/CLI 边界和事务回滚具有定向编译与运行验证。 +- SQLite-only 与 PostgreSQL-only 发布二进制只包含所用驱动；目标机器不需要 Cargo、Rust、SQLite 动态库或 PostgreSQL 客户端 SDK。 +- 在 64 MiB 与 128 MiB 约束下分别测量 SQLite、PostgreSQL 的空闲基线、稳定请求负载、流式读取和连接池上限；持续运行时 RSS 不无界增长，取消后连接与事务槽位回到基线。 +- `deverc check examples/dever/cms`、`deverc run examples/dever/cms cms.main` 和以同一项目根执行的原生构建均使用 `module/` 作为源码根，并能由 CMS Service 完成初始化数据、注册和发布的纵向流程。 + +## 暂不纳入第一轮 + +- 旧源码兼容、第二套 CRUD、链式查询、动态 Model、反射字段访问或运行时动态 SQL 结构。 +- MySQL/MariaDB、SQL Server、Oracle 驱动；只保留真实共享边界，不提前增加空适配器。 +- 非默认主键、自定义主键类型、以任意 UUID 唯一键建立关系、主键策略迁移。 +- choice 稳定变体改名和表/Model 重命名的通用简写；第一轮使用命名 migrate。 +- lazy loading、自动 N+1、隐藏多对多表、跨数据库 join、分布式事务、savepoint、读写分离和自动故障转移。 +- 数据库专有数组/enum/interval、分区表、表空间、物化视图、触发器、行级安全和在线无锁迁移。 +- 通用 annotation、decorator、macro、AST 改写和用户编写的编译期函数处理器。 +- 前端、管理页面、package registry、跨平台发行和编译器自举。 +- 日志文件 sink 的最终格式、轮转默认值和压缩策略；该项在错误基础设施子阶段收敛，不阻塞 ORM 合同。 diff --git a/.trellis/tasks/09-13-typed-orm/research/architecture-options.md b/.trellis/tasks/09-13-typed-orm/research/architecture-options.md new file mode 100644 index 0000000..d900b76 --- /dev/null +++ b/.trellis/tasks/09-13-typed-orm/research/architecture-options.md @@ -0,0 +1,76 @@ +# ORM 架构选项 + +## 现有语言约束 + +- 一个源文件完整拥有一个 package，源码相对路径必须与 package 名匹配，因此可用 `model/` 路径承载 Model 身份，不需要再增加 `model` 声明。 +- `crates/dever-core/src/native/records.rs` 为用户 record/choice 生成具体类型；生产路径没有通用运行时 Value 或反射分派。 +- 系统资源由 `types.rs`、`check/symbols.rs`、`intrinsic.rs`、native emitter 和 runtime 共同定义，错误由官方 `.dever` choice 暴露。 +- static handler 是编译期绑定而非运行时函数值，可复用于行映射与查询回调，但不能单独解决表名、主键、索引和关联元数据。 +- 当前 native build 复用一个预编译 runtime rlib；数据库驱动选择必须进入可达性、runtime build 和缓存身份，不能只依赖链接器偶然消除代码。 + +## 表面模型比较 + +### 显式 Data Mapper（已否决） + +概念形态： + +```text +find(User, database, id) +insert(database, user) +update(database, user) +``` + +Model 是数据/schema，Database 和 Transaction 都由调用方显式传递。虽然实现直接，但业务表面重复且容易把同一事务漏传给某次操作；用户已明确否决，不作为公开或兼容 API 保留。 + +### Active Record（已否决） + +概念形态： + +```text +User.find(database, id) +user.save(database) +``` + +该方案会把生成类型和成员暴露给业务代码，用户已明确否决，不作为兼容表面保留。链式查询同样需要新增 method 和中间查询对象语义，且不改善生成 SQL；首版已经确定不采用。 + +### SQL-first typed mapper + +概念形态：参数化 SQL + 静态 row decoder。它不单独承担 ORM，只作为模型包短函数之外的高级逃生口保留：参数必须绑定，调用方必须声明具体 Model/record 结果，数据库专有语句明确选择对应 dialect。 + +## Model 文件约定 + +推荐形态： + +```dever +package app.model.user exposes ( + User +) + +type User { + email: Text(254) unique + name: Text(1, 64) + bio: Text? +} +``` + +- 路径 `app/model/user.dever` 对应 `package app.model.user`，文件名 `user` 精确对应公开 record `User` 和表名 `user`；不做复数化猜测。 +- 文件只能公开一个 Model record。私有辅助声明是否允许由实现阶段按真实需要决定，不能形成第二个持久化类型。 +- 主键 `id` 和 `created_at` 由 Model 文件约定隐式生成，不进入创建输入；查询结果仍可读取 `user.id` 与 `user.created_at`。其它 Model 通过 `app.model.user.id` 引用该强类型主键。 +- 编译器可合成 `User.New` 创建输入和固定 CRUD 成员；这属于 Model 专用静态语义，不意味着加入通用嵌套 type、反射或 type-as-value。 +- 简单字段属性紧邻字段；复合唯一约束、复合索引、初始化数据和有序迁移使用同文件的独立声明，避免把复杂结构压进字段行或外部配置。 +- 安全且意图唯一的 schema 变化自动生成；索引和 Seed 不使用版本号。简单重命名通过新字段的 `from <old_field>` 明确，编译器自动生成 revision；删除、类型收窄和数据转换等复杂操作才使用 Model 文件底部的命名 migrate。revision/迁移校验值记录在 history 中，SQLite 表重建由 dialect 隐藏。 +- Model 文件可以声明自定义持久化操作，但优先组合生成的查询/事务能力；不能覆盖通用 CRUD 的事务、错误和资源合同。 + +## 驱动边界 + +- SQLite：成熟 Rust 驱动支持 bundled SQLite，适合产生无需目标系统 SQLite 的单文件程序；同步 API 必须通过 Dever 已有 bounded blocking 资源执行。 +- PostgreSQL：成熟 Tokio 驱动原生异步并支持流水线；连接 driver 必须属于 Database/Pool 的结构化作用域。 +- 两者共享 QueryPlan、绑定值、行 schema、错误分类和迁移意图；SQL 编码与执行机制保持独立。 + +## 结论 + +采用 Model 包短函数作为唯一首版表面，例如 `user.list(...)`、`user.update(...)`。Model 用一文件一普通 `type` 的约定识别，不增加 `model` 关键字；创建、更新和查询块由调用上下文推导，不暴露 `User.Query` 等生成类型。复杂查询使用参数化、类型化的原生 SQL，不引入动态行对象。 + +跨操作事务采用 `transaction <function>` 专用函数声明。普通调用按顺序等待完成，编译器根据传递挂起 effect 自动降低为协程；Dever 源码不出现 `async`、`await`、Database、Transaction、begin、commit 或 rollback。编译器从 Model 数据库效果推导唯一连接，并把内部事务上下文沿受控调用链传递；正常完成自动提交，error、fault 或取消自动回滚。首版不允许事务中的数据库效果进入并发子任务，也不提供跨库事务、嵌套物理事务或 savepoint。 + +不引入 `@` 或通用函数装饰器。注解本身并不能减少事务实现，只会额外增加解析、注解解析、顺序、参数、扩展权限和源码映射合同；用户自定义处理还需要通用函数泛型、闭包捕获、async 包装与错误转发。完全自动推导事务和调用点 `transaction(handler)` 也分别存在边界隐蔽与包装遗漏问题，因此都不作为公开表面。 diff --git a/.trellis/tasks/09-13-typed-orm/research/driver-selection.md b/.trellis/tasks/09-13-typed-orm/research/driver-selection.md new file mode 100644 index 0000000..c99b3a0 --- /dev/null +++ b/.trellis/tasks/09-13-typed-orm/research/driver-selection.md @@ -0,0 +1,39 @@ +# 数据库驱动与连接池选择 + +## 结论 + +- SQLite 使用 [rusqlite](https://github.com/rusqlite/rusqlite) 并启用 `bundled`，目标机器不依赖系统 SQLite。 +- SQLite 池与同步任务适配使用 [deadpool-sqlite](https://docs.rs/crate/deadpool-sqlite/latest)，它直接基于 rusqlite，并能把 `bundled` feature 传递给 rusqlite。 +- PostgreSQL 使用 [tokio-postgres](https://github.com/sfackler/rust-postgres)，保留原生异步连接与流水线能力；TLS 使用 [tokio-postgres-rustls](https://docs.rs/tokio-postgres-rustls/latest/tokio_postgres_rustls/) 复用项目现有 rustls 根证书和主机名校验。 +- PostgreSQL 连接池使用 [deadpool-postgres](https://docs.rs/deadpool-postgres/latest/deadpool_postgres/)，复用其连接回收和静态语句缓存，不自行实现通用连接池。 + +## 选择理由 + +- 四个 crate 属于两组相互配套的驱动/池实现，不需要引入完整 ORM 或第二套查询语义。 +- rusqlite 的 bundled 模式静态编译 SQLite，符合打包后二进制直接运行的要求。 +- tokio-postgres 与当前 Dever Tokio runtime 一致；它的连接 driver 可以纳入现有结构化任务生命周期。 +- deadpool-postgres 已包装 `tokio_postgres::Client`/`Transaction` 并提供 statement cache，适合 Dever 已静态确定 SQL 形状的查询。 +- deadpool-sqlite 负责连接池和同步连接的安全交互；Dever 仍在外层施加全局 blocking 上限、取消和超时合同。 + +## 不采用的方案 + +- 不采用 SQLx、Diesel 或 SeaORM：Dever 自己拥有 Model、QueryPlan、迁移和类型检查，再引入完整 ORM 会形成重复语义与额外二进制负担。 +- 不直接手写 PostgreSQL/SQLite 连接池：池的回收、等待、关闭与失效连接处理不是 Dever 的业务差异点。 +- 不让 SQLite 和 PostgreSQL 共用同一个执行 trait object 热路径；共享的是静态 QueryPlan 和错误分类，具体执行通过编译时 profile 与连接枚举分派。 + +## 二进制裁剪 + +`dever-runtime` 将数据库依赖设为 optional features： + +- `sqlite`：deadpool-sqlite + rusqlite bundled +- `postgres`：deadpool-postgres + tokio-postgres + tokio-postgres-rustls +- `database`：共享 config、ORM 值和错误基础 + +构建阶段从 `config/setting.json` 读取所有连接的数据库类型，只把驱动能力集合加入 runtime profile 和原生产物缓存身份，不嵌入账号、连接串或其它配置值。运行时设置可以在同一能力集合内改变；改用未打包的数据库类型必须重新构建。 + +## 验证重点 + +- 分别比较 base、SQLite-only、PostgreSQL-only 和双驱动二进制大小。 +- 64/128 MiB 环境测量空池、最大池、稳定负载和连接归还后的 RSS。 +- 验证连接获取超时、取消、driver task 失败、SQLite busy 和 shutdown drain。 +- 对静态 SQL 验证 statement cache 命中；迁移 DDL 不进入无界 statement cache。 diff --git a/.trellis/tasks/09-13-typed-orm/research/field-types.md b/.trellis/tasks/09-13-typed-orm/research/field-types.md new file mode 100644 index 0000000..4faf7a0 --- /dev/null +++ b/.trellis/tasks/09-13-typed-orm/research/field-types.md @@ -0,0 +1,192 @@ +# ORM 字段类型与隐式字段 + +## 默认字段 + +`model/` 中的每个 Model 即使源码不写，也具有： + +```dever +id: <model-package>.id +created_at: DateTime +``` + +- `id` 是编译器合成、数据库生成的 64 位正整数主键字段；它的公开静态类型就是当前 Model 的 `<model-package>.id`，不能当作普通 Int 运算，不同 Model 之间也不能混用。 +- `created_at` 由 ORM 在插入时写入当前 UTC DateTime。 +- 两个字段都存在于完整 Model 和查询结果中，不存在于 `Model.New` 创建输入中。 +- `updated_at` 不默认生成。Go Dever 现有 Model 中 created_at 明显比 updated_at 普遍；只有确实需要跟踪修改时间的 Model 才显式声明自动更新字段。 +- `deleted_at`、version、tenant_id 和操作者字段都具有业务含义，不作为所有 Model 的隐式字段。 + +因此最小 Model 可以写成： + +```dever +type User { + name: Text(1, 64) + email: Text(254) unique +} +``` + +读取 `user.id`、`user.created_at` 合法，但创建时只提供 name/email。 + +## 通用类型矩阵 + +| Dever 类型 | 逻辑语义 | PostgreSQL | SQLite | +| --- | --- | --- | --- | +| Bool | 布尔值 | BOOLEAN | INTEGER + CHECK | +| Int | 有符号 64 位整数 | BIGINT | INTEGER | +| Float | binary64 | DOUBLE PRECISION | REAL | +| Decimal(P, S) | 最多 34 位十进制定点数 | NUMERIC(P,S) | 精确定点编码与约束 | +| Text | 无界 UTF-8 文本 | TEXT | TEXT | +| Text(N) | 0 至 N 个 Unicode codepoint | VARCHAR(N) | TEXT + CHECK | +| Text(M, N) | M 至 N 个 Unicode codepoint | VARCHAR(N) + CHECK | TEXT + CHECK | +| Bytes | 无界字节 | BYTEA | BLOB | +| Bytes(N) / Bytes(M, N) | 有界字节 | BYTEA + CHECK | BLOB + CHECK | +| `<model-package>.id` | 具体 Model 的不透明 64 位主键/外键投影 | BIGINT identity | INTEGER PRIMARY KEY | +| Uuid | 128 位 UUID | UUID | 16-byte BLOB | +| DateTime | UTC 时间点，毫秒精度 | TIMESTAMPTZ(3) | epoch-millis INTEGER | +| Date | 公历日期 | DATE | epoch-day INTEGER | +| Time | 一天内时间，毫秒精度 | TIME(3) | millis-of-day INTEGER | +| Duration | 固定毫秒时长 | BIGINT | INTEGER | +| Json | 已验证 JSON 值 | JSONB | UTF-8 TEXT | +| 无载荷 choice | 封闭枚举 | TEXT + CHECK | TEXT + CHECK | + +List、Map、普通 record 不自动变成 JSON 列。调用方必须显式选择 Json，避免字段重构在不知情时改变持久化格式。 + +## UUID 字段 + +UUID 不是每个 Model 的隐式字段。只有业务需要公开不可预测标识、跨系统交换标识或在写入数据库前生成标识时才显式声明： + +```dever +type User { + uuid: Uuid generated unique + name: Text(1, 64) +} +``` + +- `uuid: Uuid` 由调用方提供，存在于创建输入中。 +- `uuid: Uuid generated` 由 ORM 在创建时生成 UUIDv7，不存在于创建输入中。 +- `uuid: Uuid?` 接受空值；`generated` 与可空不能同时使用。 +- 运行时使用固定 16 字节值，不用堆分配的 Text 保存 UUID。 +- PostgreSQL 使用原生 UUID，SQLite 使用 16-byte BLOB，索引不会保存 36 字符文本。 +- API、JSON、日志及显式文本转换统一输出小写、带连字符的 36 字符形式；解析只接受语言标准库定义的 UUID 文本格式。 +- Uuid 字段不会自动成为关联。关联仍写成 `app.model.user.id`，以保持外键类型检查和紧凑索引。 + +## 初始化数据写法 + +Seed 是当前 Model 的可重复初始化数据。一个块可以包含多条数据，值使用 Model 创建输入进行静态检查： + +```dever +type Status { + code: Text(32) unique + name: Text(64) + enabled: Bool default true +} + +seed { + { + code = "pending" + name = "待处理" + } + + { + code = "done" + name = "已完成" + enabled = true + } +} +``` + +以后增加初始化记录时直接追加数据块，不维护 Seed 版本号。编译器按 Seed 内容校验值判断是否需要执行；执行时整体使用事务，唯一冲突保留已有记录。源码移除 Seed 也不删除数据，需要修改或删除时使用显式命名迁移。 + +## 可选项 + +固定可选项复用语言现有的无载荷 choice `type`，不另造 ORM enum： + +```dever +type TaskStatus { + Pending = "待处理" + Running = "进行中" + Done = "已完成" +} + +type Task { + title: Text(1, 160) + status: TaskStatus default TaskStatus.Pending +} + +seed { + { + title = "初始化任务" + status = TaskStatus.Pending + } +} +``` + +`Task` 是文件对应的 Model record；`TaskStatus` 是同文件的辅助 choice，不生成表。ORM 第一轮只接受不带 payload 的 choice，并将左侧变体名映射为带 CHECK 约束的 Text。右侧显示名称不存入业务列。 + +推荐由编译器把显示名称提供为保持声明顺序的只读 Map，不再重复定义映射： + +```dever +task_status_options() (options: Map<TaskStatus, Text>) { + options = TaskStatus.options +} +``` + +需要选项子集或自定义顺序时再返回 `List<TaskStatus>` 或组合 `TaskStatus.options`。choice 负责限制合法值，生成的 Map 负责名称和默认顺序。 + +## 索引与迁移写法 + +单字段索引直接跟在字段后，复合索引放在 Model 后；它们都描述当前目标 schema，不使用版本号或手工数据库名称： + +```dever +type Task { + tenant_id: app.model.tenant.id + code: Text(32) + status: TaskStatus index + title: Text(1, 160) +} + +unique(tenant_id, code) +index(status, created_at) +``` + +简单字段重命名直接写在新字段旁边： + +```dever +type Task { + summary: Text from description +} +``` + +编译器生成 `RENAME COLUMN`，不会把它误判成删旧列再加新列。revision 由编译器根据规范化 schema 和计划自动生成并写入数据库 history，源码不维护版本号。 + +拆分、合并、删除前归档等无法附着在单个新字段上的数据变化，才保留一次性命名迁移： + +```dever +migrate remove_legacy_code { + drop legacy_code +} +``` + +迁移名称是复杂操作的稳定身份，内容校验值用于阻止已经执行的迁移被悄悄修改。`from` 需要保留到所有可能升级的数据库都已越过该变更。 + +## 关联 ID + +```dever +type Article { + author_id: app.model.user.id + category_id: app.model.category.id? + title: Text(1, 160) +} +``` + +- `app.model.user.id` 与 `app.model.category.id` 是不同静态类型，不能误传。 +- 点分路径的最后一段 `id` 是目标 Model 的隐式字段投影，不是新增的普通小写 type；它也能用于 function 输入、输出和局部类型位置。 +- 非空和可空关联第一轮都使用 `ON DELETE RESTRICT`；SET NULL 或 CASCADE 的通用关系选项不在第一轮，必要时使用明确方言的命名 migrate。 +- 每个关联 ID 自动创建普通索引和外键约束。它只表达持久化关系，不自动查询目标对象。 +- 关系预加载必须显式请求并保持有界，不能形成隐藏 N+1。 + +## 时间 API 方向 + +- DateTime 值始终代表 UTC 时间点，不在值内携带可变时区数据库。 +- 时区只参与解析、格式化和日历换算；数据库比较、排序和相减使用统一时间点。 +- `created_at` 自动取当前 DateTime。`updated_at` 不隐式生成；第一轮需要该字段时按普通 `DateTime` 明确写入，自动更新修饰词后续单独设计。 +- DateTime/Date/Time 的 parse、format、比较和加减由标准 package 提供；数据库函数通过 ORM 查询表达式或明确的原生 SQL 使用。 diff --git a/.trellis/tasks/09-13-typed-orm/research/go-orm-reference.md b/.trellis/tasks/09-13-typed-orm/research/go-orm-reference.md new file mode 100644 index 0000000..b55a232 --- /dev/null +++ b/.trellis/tasks/09-13-typed-orm/research/go-orm-reference.md @@ -0,0 +1,45 @@ +# Go Dever ORM 参考结论 + +## 已核对实现 + +- `/data/project/shemic/backend/dever/orm/schema_registry.go` 通过 Go 反射读取 `dorm` 字符串 tag；Text 默认 `VARCHAR(255)`，`size:N` 生成 `VARCHAR(N)`，也允许直接写 `type:varchar(N)` 或 `type:text`。 +- `ModelConfig.Index`/`Indexes` 使用额外 Go struct 表达单列或复合索引；`ModelConfig.Seeds` 使用 `[]map[string]any` 表达初始化数据。 +- `/data/project/shemic/backend/dever/orm/schema_apply.go` 能建表、加字段、修改部分数据库的字段类型、删旧字段，并同步新增、变化和删除的索引。 +- 当前 Seeds 只在表首次创建时插入；已有表后来新增 Seed 不会自动补入。Seed 使用动态 Map，字段名和值类型不能在编译期完整检查。 +- 当前自动迁移直接对数据库执行，再可选记录 SQL；它会删除不在目标 schema 中的字段和索引。字段重命名只对大小写或下划线差异做有限推断，无法可靠判断任意重命名。 +- `database.persist=true` 会把目标 schema 写入 `data/table/*.json`，`dever migrate` 再读取这些文件应用到目标数据库。该目录是 schema 传递/发布产物，不包含某个数据库实例是否成功执行的可靠状态；同一文件可以对应多个状态不同的数据库。 +- Go Model wrapper 可以嵌入通用 `orm.Model[T]` 并增加 `AfterSave`、`AfterDelete` 或自定义查询方法。普通 Model 直接复用通用 CRUD，不需要 wrapper。 + +## 新语言应复用的设计 + +- 一个 Model 文件集中字段、索引、默认排序、初始化数据和少量紧贴持久化生命周期的扩展。 +- 通用 CRUD、过滤、排序、分页、事务和批量操作由 ORM 提供，不为每个 Model 重写。 +- 单列与复合索引都从 Model 目标 schema 自动同步。 +- Model 声明生成确定的数据库 schema，SQLite/PostgreSQL 只负责方言差异。 + +## 不应照搬的部分 + +- 不使用字符串 tag、反射、`any`、动态字段 Map 或运行时猜测字段名。 +- 不把无界 Text 默认截成 `VARCHAR(255)`；`Text` 表示无界文本，只有显式长度上限才产生 VARCHAR/CHECK。 +- 不只在首次建表时处理初始化数据。每条 Seed 必须有稳定身份并可重复应用。 +- 不把“字段从目标结构中消失”直接等同于可以删列，也不把任意新增/缺失列猜成重命名。 +- 不把本地 `table/` 文件是否存在当作数据库建表或迁移完成标记。新语言将目标 schema 嵌入二进制，实际状态记录在数据库并通过系统目录核对。 +- 不允许自定义方法覆盖并改变通用 CRUD 的基本事务、错误和资源合同；自定义操作组合通用查询或参数化 SQL。 + +## 推荐字段形态 + +```dever +type User { + name: Text(1, 64) + email: Text(254) unique + bio: Text? + credit: Decimal(18, 2) default 0 + status: Int >= 0 and <= 2 default 1 +} +``` + +- `Text(N)` 表示最多 N 个 Unicode codepoint，`Text(M, N)` 表示 M 至 N 个；无参数 Text 是无界文本。 +- `Bytes(N)` 表示最多 N 个字节，`Bytes(M, N)` 表示 M 至 N 个字节。 +- `Decimal(P, S)` 描述数据库精度与小数位，必须满足 `1 <= P <= 34`、`0 <= S <= P`，与 Dever decimal128 的 34 位有效数字上限一致。 +- `?` 已经表达可空，不再增加 `null/not null` 两套写法。 +- `id` 与 `created_at` 由 Model 隐式生成；关联主键用 `app.model.user.id` 这种字段投影引用。默认值和简单 unique 靠字段元数据；复合索引、无版本 Seed 和命名迁移使用同文件的独立声明，避免把复杂结构塞进一行字段。 diff --git a/.trellis/tasks/09-13-typed-orm/research/implicit-suspension.md b/.trellis/tasks/09-13-typed-orm/research/implicit-suspension.md new file mode 100644 index 0000000..5e378e7 --- /dev/null +++ b/.trellis/tasks/09-13-typed-orm/research/implicit-suspension.md @@ -0,0 +1,48 @@ +# 隐式挂起的顺序调用 + +## 决定 + +Dever 源码删除 function/handler 的 `async` 标记和直接异步调用外层的 `await(...)`。普通调用始终表示调用完成后才执行下一条语句；如果调用链包含数据库、网络、定时器、Channel 或异步 Stream 等挂起效果，编译器自动把对应函数降低为协程。 + +```dever +route(request: dever.http.Request) (response: dever.http.Response) { + user = user.get(request.user_id) + response = render(user) +} +``` + +等待 I/O 时只挂起当前协程，Tokio 工作线程可以执行其它连接和任务。语法简化不意味着阻塞线程，也不改变 HTTP/TCP/WS/SSE runtime、连接池、背压、超时与取消清理。 + +## 并发边界 + +普通调用严格顺序；只有 `run` 显式启动并发： + +```dever +first = run(load_user(first_id)) +second = run(load_user(second_id)) +first_user = wait(first) +second_user = wait(second) +``` + +- `run(call(...))` 创建受限 Task。 +- `wait(task_or_group)` 消耗任务或任务组并取得结果/等待排空。 +- `stop(task_or_group)` 请求取消并等待结构化清理。 +- CPU 同步计算继续使用 `parallel`，明确阻塞调用继续使用 `blocking`。 +- 普通调用不产生 Task，也不允许把可挂起调用结果当 Future 保存。 + +## 编译器边界 + +- 系统 intrinsic 和官方库入口提供基础挂起效果；调用图、静态 handler 专门化和 collection handler 传播效果。 +- checker 在效果推导后区分顺序调用、`run` 和 Task `wait`；HIR/native 仍可保留内部 coroutine/await 节点，但不把它们暴露到源码。 +- 同步入口的完整可达图没有挂起效果时沿用直接原生入口，不初始化 Tokio。 +- handler 是否允许挂起由其使用位置和实际效果静态检查，不再由源码 `async handler` 标记。 +- API snapshot、Markdown 合同和格式化输出的是新语法；旧 `async`/`await` 不保留兼容分支。 + +## 迁移范围 + +- lexer、parser、syntax AST、checker、effect/cycle/failure 分析、HIR 与 native emitter。 +- `dever.task`、`dever.net`、`dever.http`、`dever.websocket`、`dever.sse` 的官方 Dever 声明。 +- HTTP/1、HTTP/2、TCP、TLS、WebSocket、SSE、AsyncStream、Channel 和结构化并发测试中的嵌入源码。 +- backend/http/live/async 示例、性能 fixtures、LANGUAGE 与 Trellis backend 规范。 + +Rust runtime 内的 Rust `async/.await` 不在迁移范围；删除它们会把高并发 I/O 退化为阻塞或线程堆叠，与目标相反。 diff --git a/.trellis/tasks/09-13-typed-orm/research/migration-practices.md b/.trellis/tasks/09-13-typed-orm/research/migration-practices.md new file mode 100644 index 0000000..2a8117e --- /dev/null +++ b/.trellis/tasks/09-13-typed-orm/research/migration-practices.md @@ -0,0 +1,20 @@ +# 字段重命名与迁移版本调研 + +## 官方实现结论 + +- [Alembic Autogenerate](https://alembic.sqlalchemy.org/en/latest/autogenerate.html) 明确说明无法检测表名和列名变化；它会生成 add/drop 候选，要求开发者审查并改为 rename。 +- [Prisma 自定义迁移](https://docs.prisma.io/docs/orm/prisma-migrate/workflows/customizing-migrations) 的字段重命名示例同样默认得到 drop/add，需要先生成草稿迁移，再人工改成 `RENAME COLUMN`。Prisma 用迁移目录和数据库 `_prisma_migrations` 表维护历史。 +- [EF Core 管理迁移](https://learn.microsoft.com/en-us/ef/core/managing-schemas/migrations/managing) 说明框架无法判断“删除并新增”和“重命名”的真实意图；生成代码需要改为 `RenameColumn`，否则会丢失旧列数据。 +- [Django 迁移操作](https://docs.djangoproject.com/en/5.2/ref/migration-operations/) 提供显式 `RenameField(old_name, new_name)`；[Django 迁移流程](https://docs.djangoproject.com/en/6.0/topics/migrations/) 把迁移文件视为数据库 schema 的版本控制，并在数据库记录应用历史。 +- [Atlas 声明式重命名](https://atlasgo.io/changelog/declarative-schema-renames) 使用紧邻目标列的 `renamed_from` 显式表达意图。Atlas 早期依赖交互式询问，但非交互部署无法回答，因此把 rename 意图写回声明式 schema。 +- [Atlas 声明式与版本式对比](https://atlasgo.io/concepts/declarative-vs-versioned) 表明声明式流程可以从目标 schema 规划实际数据库差异；版本式流程则保留完整迁移文件历史。两者都不能依靠一个数字版本推导 rename 意图。 + +## 对 Dever 的结论 + +- 只增加 Model 数字版本不能解决字段重命名；版本只标识状态先后，不包含旧列与新列的对应关系。 +- 不采用字段名相似度自动猜测。相同类型的删一列、加一列既可能是 rename，也可能是两个独立业务变化。 +- 简单 rename 采用声明式字段来源：`new_field: Type from old_field`。它比单独 migrate 块短，并把意图放在所属字段旁。 +- 编译器为规范化目标 schema 与迁移计划自动产生 revision id/校验值，嵌入二进制；数据库内部 history 记录实际应用状态。开发者不写数字版本。 +- `from` 必须保留到所有需要升级的数据库越过该变化；这与 Atlas 把 `renamed_from` 放进 schema 的原则一致。 +- `from` 只描述已有表的升级路径。目标表不存在时直接创建当前最终 schema，只创建新字段，不要求旧字段存在，也不先创建旧结构再回放 rename。 +- 拆分、合并、跨行回填、删除前归档等没有单字段对应关系的变化，仍需要命名 migrate。该复杂度来自数据转换本身，不能由版本号或启发式推断消除。 diff --git a/.trellis/tasks/09-13-typed-orm/research/query-api.md b/.trellis/tasks/09-13-typed-orm/research/query-api.md new file mode 100644 index 0000000..9324180 --- /dev/null +++ b/.trellis/tasks/09-13-typed-orm/research/query-api.md @@ -0,0 +1,199 @@ +# ORM 查询与内置操作方案 + +## 结论 + +公开表面只使用 Model 包短函数和一次性查询配置块： + +```dever +users = user.list({ + where = status == UserStatus.Active and age >= 18 + order = [created_at.desc, id.desc] + page = 1 + size = 20 + with = [owner] +}) +``` + +不提供 `User.Query`、`User.List`、Active Record 实例方法或 `user.where(...).order(...).list()` 链式查询器。编译器把匿名块直接静态降低为 `QueryPlan`；源码没有 `async`/`await`，普通调用顺序等待完成。 + +## 为什么选择配置块 + +块式和链式可以生成相同 SQL，链式本身没有数据库性能优势。Dever 当前没有通用 method、lambda、用户泛型或 type-as-value，链式方案会额外引入查询对象类型、所有权、复用和执行时机合同。配置块则有以下特点： + +- `create`、`update` 和 `list` 保持同一种上下文推导形式。 +- 一次调用完整展示条件、排序、分页和关系加载。 +- 编译器直接生成一个静态计划，不产生运行时中间查询对象。 +- 公开 API 只有一套写法，不需要兼容或维护等价表面。 + +将来若动态 HTTP 参数确实需要逐步组合条件，应增加受限的类型化 where 值，而不是改造全部 CRUD 为链式查询。 + +## 基本 CRUD + +```dever +users = user.list() +current = user.get(user_id) +created = user.create({ + name = "Dever" + status = UserStatus.Active +}) +updated = user.update(user_id, { + name = "New name" +}) +deleted = user.delete(user_id) +``` + +`create` 和 `update` 的匿名数据块由调用上下文推导字段。不能写入的 generated 字段、拼错的字段和不兼容值在编译期拒绝；底层仍是具体输入 record，不是动态 Map。 + +普通可失败调用成功时直接产生成功值，失败自动传播。需要主动处理完整结果时写： + +```dever +result = result(user.get(user_id)) +``` + +随后使用现有穷尽函数分句处理成功和 error 变体，不为 ORM 增加 try/catch 或 `?`。 + +## List、Cursor 与 Stream + +`user.list()` 返回有界默认第一页。完整配置为： + +```dever +users = user.list({ + where = status == UserStatus.Active and created_at >= start + order = [created_at.desc, id.desc] + page = 1 + size = 20 + with = [owner] +}) +``` + +查询块处于当前 Model 上下文，所以字段直接写 `status`、`created_at`、`owner`，不重复 Model 名。它不是动态 Map，字段、运算符、值、排序和关系都由编译器检查。 + +- `page` 默认 1，`size` 默认 20，并受统一配置上限约束。 +- 默认排序为 `id.desc`；自定义排序不是唯一键时自动追加 `id`，保证页边界稳定。 +- 返回 `items`、`page`、`size`、`total`、`pages`。 +- 页码查询执行数据查询和 count；不需要 total 时使用 cursor。 + +深翻页使用同一个配置块： + +```dever +users = user.cursor({ + after = next + size = 100 + where = status == UserStatus.Active + order = [created_at.desc, id.desc] +}) +``` + +`cursor` 返回 `items`、`next`、`has_more`，不执行 count。全表处理使用 `user.stream(query?)`，按行拉取并保持有界缓冲；不提供无界 `all()`。 + +## Where + +多个条件直接组成一个布尔表达式，不增加条件数组： + +```dever +users = user.list({ + where = status == UserStatus.Active + and age >= 18 + and age < 60 + and (city == "北京" or city == "上海") +}) +``` + +- 比较支持 `==`、`!=`、`<`、`<=`、`>`、`>=`。 +- 逻辑组合使用既有 `and`、`or`、`not` 和括号。 +- 集合与区间使用 `in`、`between`。 +- Text 匹配使用 `contains`、`starts_with`、`ends_with`。 +- 可空字段可直接与 `null` 比较。 +- 关系条件使用字段路径，例如 `owner.status == UserStatus.Active`；筛选关系不会自动把关系放进结果。 + +ORM 将表达式规范化为静态条件树，按源码顺序收集绑定值。相同字段可重复出现，矛盾条件由数据库返回空结果；不引入运行时条件解释器。 + +## 关联 + +正向 to-one 从关联 ID 推导： + +```dever +type Task { + owner_id: app.model.user.id +} +``` + +查询时使用 `with = [owner]`。反向 to-many 无法可靠推断名称，因为目标 Model 可能有多个外键，因此在当前 Model 文件显式命名： + +```dever +relation tasks = app.model.task.owner_id +``` + +- to-one 使用受控 JOIN 或等价单次查询。 +- to-many 按当前父页 ID 批量加载并归组，不逐父记录查询。 +- to-many 子列表必须有稳定排序和数量上限。 +- 多对多使用显式中间 Model，不生成隐藏表。 +- 不提供 lazy loading，读取普通字段不会在背后访问数据库。 + +## 内置函数 + +| 函数 | 用途 | +| --- | --- | +| `user.create(data)` | 创建一条并返回完整 Model | +| `user.create_many(list)` | 有界批量创建 | +| `user.get(id)` | 按主键获取一条 | +| `user.first(query?)` | 按条件获取第一条 | +| `user.list(query?)` | 自动页码分页 | +| `user.cursor(query?)` | 稳定游标分页 | +| `user.count(where?)` | 统计数量 | +| `user.exists(where?)` | 判断是否存在 | +| `user.stream(query?)` | 有界流式遍历 | +| `user.update(id_or_query, changes)` | 单条或条件批量更新 | +| `user.delete(id_or_query)` | 单条或条件批量删除 | +| `user.upsert(key, create, update)` | 按唯一键新增或更新 | + +`update`/`delete` 根据首个参数静态区分主键和条件，不增加重复的 `update_many`/`delete_many` 名称。批量写入返回影响行数；业务状态流转、跨表原子编排和外部调用由 transaction 函数组合。 + +## 数据库绑定 + +普通调用不传 `Database`： + +```dever +user.list({...}) +user.update(user_id, {...}) +``` + +Model 的连接在程序启动时解析并缓存： + +1. Model 文件显式 `database report`； +2. 否则按根包名查找，例如 `app.model.user` 查 `database.app`； +3. 根包键不存在时使用必需的 `database.default`。 + +显式连接不存在、default 不存在或选中配置无效时启动失败。生成代码访问固定连接槽位，不按字符串进行每请求查找，也不允许 CRUD 临时切库。 + +## 自动事务 + +事务不属于 Model API，源码不取得或传递 Transaction： + +```dever +transaction assign_task( + user_id: app.model.user.id, + title: Text +) (result: AssignResult) { + user.update(user_id, { + active = true + }) + task.create({ + owner_id = user_id + title = title + }) + result = AssignResult.Assigned +} +``` + +编译器分析完整调用链，从 Model 数据库 effect 推导唯一连接。进入最外层函数时自动开启物理事务，直接或经普通顺序 helper 执行的操作都使用隐藏事务上下文；正常完成提交，error、fault 或取消先回滚再离开。 + +同连接嵌套 transaction 复用外层事务，不使用 savepoint。跨连接事务和事务中的并发数据库子任务在编译期拒绝。该设计不引入 `@`、decorator、显式 begin/commit/rollback 或动态当前事务。 + +## 实现与性能 + +- 所有 Model 复用一个编译期 `QueryPlan`、schema 引擎和方言执行层，只生成字段专属的薄代码。 +- 参数绑定、预编译语句和具体行解码不经过反射或动态 Map。 +- to-many 批量加载避免 N+1，也避免多个 to-many JOIN 产生笛卡尔膨胀。 +- 分页、关联子列表、批量写入、行流、连接池和 SQLite blocking 全部有界。 +- SQLite 和 PostgreSQL 可以用不同执行器，但必须消费同一逻辑查询计划并返回同一 Dever 类型。 diff --git a/.trellis/tasks/09-13-typed-orm/research/schema-change-matrix.md b/.trellis/tasks/09-13-typed-orm/research/schema-change-matrix.md new file mode 100644 index 0000000..3c333ac --- /dev/null +++ b/.trellis/tasks/09-13-typed-orm/research/schema-change-matrix.md @@ -0,0 +1,90 @@ +# Model schema 变更分类 + +## 设计原则 + +Model 始终声明当前目标 schema。ORM 只保留三种演进机制： + +1. 目标 schema diff：处理意图唯一且不会静默丢失数据的变化。 +2. `from`：补充 diff 无法推断的旧身份，例如字段重命名。 +3. 命名 `migrate`：处理需要业务决定的数据转换、修复和删除。 + +不为长度、默认值、可空性、索引等每类变化增加单独版本或迁移关键字。SQLite 和 PostgreSQL 的 DDL 差异由 dialect 处理。 + +## 变更矩阵 + +| 变化 | Model 写法 | 默认处理 | +| --- | --- | --- | +| 新建表 | 新增 Model 文件 | 直接创建当前最终 schema | +| 新增可空字段 | 新增 `field: Type?` | 自动添加 | +| 新增必填字段并有常量默认值 | 新增 `field: Type default value` | 自动添加并由数据库填充已有行 | +| 新增必填字段但无默认值 | 新增 `field: Type` | 空表可自动添加；有数据时要求 migrate 回填 | +| 字段重命名 | `new_field: Type from old_field` | 旧表执行 rename；新表只创建新字段 | +| 字段重命名并放宽类型约束 | `new_field: WiderType from old_field` | rename 后自动修改类型/约束 | +| 字段重命名并收紧类型约束 | `new_field: NarrowerType from old_field` | 先验证数据；不满足时要求 migrate | +| Text/Bytes 最大长度增加 | 直接修改类型参数 | 自动修改 | +| Text/Bytes 最大长度减小 | 直接修改类型参数 | 先验证现有值长度；不截断，失败时要求 migrate | +| Text/Bytes 最小长度增加 | 直接修改类型参数 | 先验证现有值；失败时要求 migrate | +| Text/Bytes 最小长度减小 | 直接修改类型参数 | 自动放宽约束 | +| Decimal 精度/小数位变化 | 直接修改类型参数 | 仅可精确往返时自动；可能溢出或舍入时要求 migrate | +| 同族无损类型扩大 | 直接修改字段类型 | 按编译器固定无损转换表自动执行 | +| 跨族或可能有损类型转换 | 直接修改字段类型并提供 migrate | 必须明确转换表达式，禁止依赖数据库隐式强转 | +| 必填改可空 | `Type` 改为 `Type?` | 自动移除 NOT NULL | +| 可空改必填 | `Type?` 改为 `Type` | 验证无 NULL；否则要求 migrate 回填 | +| 增加默认值 | 增加 `default value` | 自动修改，只影响后续写入 | +| 修改默认值 | 修改 `default value` | 自动修改，只影响后续写入 | +| 删除默认值 | 删除 `default value` | 自动修改，不改已有数据 | +| 普通索引增加/删除 | 增删 `index` 或 `index(...)` | 自动同步，不改业务数据 | +| 普通索引列或顺序变化 | 修改 `index(...)` | 删除旧索引并创建新索引 | +| 增加唯一约束 | 增加 `unique` 或 `unique(...)` | 先验证无重复；失败时要求 migrate 去重 | +| 删除唯一约束 | 删除 `unique` | 自动删除约束/索引 | +| 普通索引改唯一 | `index` 改为 `unique` | 先验证无重复 | +| 唯一索引改普通 | `unique` 改为 `index` | 自动放宽 | +| 增加外键/修改关联目标 | 改为 `<model-package>.id` | 验证无孤儿记录；失败时要求 migrate | +| 删除外键 | 改为普通字段类型或删除关联 | 自动删除约束；字段数据保留 | +| 修改外键删除行为 | 第一轮使用命名 migrate | 默认 RESTRICT；通用关系选项语法后续设计 | +| choice 增加变体 | 增加变体 | 自动放宽 CHECK | +| choice 修改显示名称 | 修改右侧文本 | 无 schema 迁移，存储值不变 | +| choice 删除稳定变体 | 删除变体 | 验证没有旧值；否则要求 migrate 映射 | +| choice 稳定变体改名 | 第一轮使用命名 migrate | 显式更新已有存储值并重建 CHECK | +| 删除字段 | 从 record 删除并写命名 migrate | 必须显式确认 drop；关联约束和索引按依赖顺序删除 | +| 删除表 | 删除 Model 并写命名 migrate | 必须显式确认，禁止仅因文件消失自动删表 | +| 表/Model 重命名 | 第一轮使用命名 migrate | 显式执行表 rename 并更新 schema identity | +| 字段拆分 | 当前字段 + 命名 migrate | migrate 负责回填多个目标字段 | +| 字段合并 | 当前字段 + 命名 migrate | migrate 负责合并与冲突规则 | +| 主键策略或主键类型变化 | 命名 migrate | 重建引用关系，首轮不自动处理 | +| generated 策略变化 | 直接修改声明并按需 migrate | 只影响新值时自动；需要改已有值时 migrate | +| Seed 新增记录 | 修改 `seed` | 按唯一键只插入缺失记录 | +| Seed 修改/删除已有记录 | 命名 migrate | Seed 不覆盖或删除线上数据 | +| 字段声明顺序变化 | 调整源码顺序 | 不改变已有数据库列顺序，不产生迁移 | +| 注释、标签、自定义方法变化 | 直接修改源码 | 不产生数据库迁移 | + +## 长度修改示例 + +```dever +type User { + display_name: Text(128) from nickname +} +``` + +如果旧字段是 `nickname: Text(64)`，规划顺序是先 rename，再把最大长度放宽到 128。全新数据库只创建 `display_name` 的最终定义。 + +收窄时仍只修改目标类型： + +```dever +type User { + display_name: Text(32) from nickname +} +``` + +ORM 先验证已有值都不超过 32 个 Unicode codepoint。全部满足就应用约束；存在超长值则整个迁移失败，不做自动截断。开发者必须通过命名 migrate 明确选择截断、拒绝、归档或其它业务规则。 + +## 方言执行边界 + +- PostgreSQL 原生支持 rename、类型变更、默认值、可空性和约束变更；类型变更可以使用 `USING` 表达式。 +- SQLite 直接 ALTER 能力较少。不能原地完成的同一逻辑变化由 dialect 在事务中创建目标临时表、复制转换后的数据、替换旧表并重建索引/约束。 +- 两个方言必须得到相同的规范化目标 schema 和 revision；底层是否重建表不能泄漏到 Dever Model 语法。 +- Dever 不使用静默截断、自动填充猜测、名称相似度推断或数据库特有隐式类型转换。 + +## 不属于首轮通用 Model 的数据库管理能力 + +分区表、表空间、存储参数、物化视图、触发器、行级安全策略、数据库专有索引方法和在线无锁迁移不进入首轮跨数据库 Model 协议。这些能力需要 PostgreSQL 专有扩展或显式原生迁移，不能伪装成 SQLite/PostgreSQL 通用行为。 diff --git a/.trellis/tasks/09-13-typed-orm/task.json b/.trellis/tasks/09-13-typed-orm/task.json new file mode 100644 index 0000000..ae2cb7a --- /dev/null +++ b/.trellis/tasks/09-13-typed-orm/task.json @@ -0,0 +1,26 @@ +{ + "id": "typed-orm", + "name": "typed-orm", + "title": "Typed ORM for SQLite and PostgreSQL", + "description": "为 Dever 增加隐式挂起与错误传播基础、一文件 Model、数据库无关查询计划，以及 SQLite/PostgreSQL 的类型安全 ORM、事务和迁移能力。", + "status": "completed", + "dev_type": null, + "scope": "implicit suspension/error contracts + compiler model/query/transaction contracts + runtime config/orm/sqlite/postgres/log foundations + focused database and memory fixtures", + "package": null, + "priority": "P1", + "creator": "Antigravity", + "assignee": "Antigravity", + "createdAt": "2026-09-13", + "completedAt": "2026-09-17", + "branch": "main", + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [], + "parent": "09-04-dever-language-mvp", + "relatedFiles": [], + "notes": "", + "meta": {} +} diff --git a/.trellis/tasks/09-19-application-testing/check.jsonl b/.trellis/tasks/09-19-application-testing/check.jsonl new file mode 100644 index 0000000..838d423 --- /dev/null +++ b/.trellis/tasks/09-19-application-testing/check.jsonl @@ -0,0 +1,8 @@ +{"file": ".trellis/tasks/09-19-application-testing/prd.md", "reason": "逐项核对应用测试验收标准"} +{"file": ".trellis/tasks/09-19-application-testing/design.md", "reason": "核对实现未偏离最终架构与隔离边界"} +{"file": ".trellis/tasks/09-19-application-testing/implement.md", "reason": "核对分阶段交付与允许执行的验证"} +{"file": ".trellis/spec/backend/compiler-contracts.md", "reason": "检查语言合同和生产编译路径无回归"} +{"file": ".trellis/spec/backend/error-handling.md", "reason": "检查诊断与测试失败报告"} +{"file": ".trellis/spec/backend/quality-guidelines.md", "reason": "检查重复、职责和定向覆盖"} +{"file": ".trellis/spec/backend/database-guidelines.md", "reason": "检查临时 SQLite 隔离且不触达正式数据库"} +{"file": ".trellis/spec/guides/cross-layer-thinking-guide.md", "reason": "检查源码到 CLI/runtime 的跨层闭环"} diff --git a/.trellis/tasks/09-19-application-testing/design.md b/.trellis/tasks/09-19-application-testing/design.md new file mode 100644 index 0000000..e50a041 --- /dev/null +++ b/.trellis/tasks/09-19-application-testing/design.md @@ -0,0 +1,143 @@ +# Dever 应用测试体系技术设计 + +## Architecture + +```text +module/ + test/ + -> one SourceMap with compiler-assigned Application/Test origins + -> existing parser and Markdown extraction + -> existing declaration registration and semantic checker + -> Test layout/access rules + -> filename-matched TestCase metadata + -> test-only assert/assert_eq lowering + -> checked Program + -> one native suite containing every checked TestCase specialization + -> case-id dispatch into one isolated native process + temporary project directory per case + -> captured result + deterministic CLI summary +``` + +There is no test interpreter, dynamic test registry, source macro, runtime reflection or production test symbol. Test behavior is a thin compiler-owned entry mode over the same checked Program and native backend. + +## Source Ownership And Identity + +`SourceFile` gains a compiler-assigned source origin rather than inferring test authority from a user-controlled path. Production loading remains the current `SourceMap::load(module_root)` behavior. A dedicated project-test loader loads `module/` as application sources and an existing `test/` as test sources; a missing test root is an empty set, while a non-directory or unreadable root is an explicit load error. + +Test sources retain two paths: + +- display path: `test/user/account/register.dever`, used by diagnostics and CLI output; +- logical path: `user/account/register.dever`, used for package identity and Markdown package documentation. + +The new `SourceLayout::Test { component, domain, topic }` is valid only for exactly three logical segments. It applies existing lower-snake-case and generic-bucket rules. `.dever` and `.dever.md` with the same logical stem are one duplicate test identity. + +The checker cache identity includes source origin/logical path so a strict application source and a test source with equal bytes cannot share the wrong checked result. + +## Test Entry Contract + +For `test/user/account/register.dever`, the logical package is `user.account.register`; the checker selects its only ordinary `register() ()` function and records a `TestCase` containing: + +- stable public identity `user/account/register`; +- checked private function id; +- source span and whether its reachable graph uses a Model. + +Other declarations stay private to the same test file. A same-name overload, transaction/recovery entry, parameters or outputs are rejected before native compilation. Test functions are never marked public and never enter API/model snapshots. + +`Program` exposes read-only test metadata to the CLI. Native compilation accepts an already checked test-case id through a dedicated method instead of weakening `Program::entry` or making test functions public. + +## Visibility + +Test resolution extends the existing role-aware symbol owner rather than adding a parallel resolver: + +- same file: private helper functions/types; +- same domain: App and Domain functions/types; +- other domains: App public surface only; +- Model values/types: readable under existing public/type rules; +- Model operations: still rejected because only the owning App role may issue CRUD; +- Adapter, Port, API and other Test owners: not callable. + +All production callers reject Test targets before ordinary same-domain checks, preventing reverse dependencies. Test assertions are available only while checking a Test owner. + +## Assertions + +`assert` and `assert_eq` are compiler-owned test intrinsics, not bundled library functions. They reuse the existing `Intrinsic` expression shape so dependency, effect, liveness and native traversal do not gain a second assertion IR hierarchy. + +- `assert(value)` checks one `Bool` expression. +- `assert_eq(actual, expected)` checks the left expression first, uses its exact type as the right-hand expectation, and requires the existing `Type::comparable` contract. +- both return Unit and are legal only as ordinary zero-output actions in Test-owned functions; +- declarations named `assert` or `assert_eq` remain ordinary outside tests and are reserved inside a test file. + +Native emission evaluates each operand once and preserves source order. On failure it creates a source-located assertion error. `assert_eq` renders concrete typed values through the existing `Render` implementations; generated records/choices retain their source-shaped renderer. No generic runtime `Value`, serialization round-trip or reflection is introduced. + +## Native Execution And Reporting + +The CLI checks the combined source set once, sorts `TestCase` metadata, then compiles one native test suite and runs each case serially. The suite unions the existing specialization graphs and dispatches one checked private entry by its stable sorted case index. It does not introduce a dynamic function registry or runtime reflection. + +Every case still starts a fresh process from the same immutable suite executable. The runner stages that executable beside a new temporary project for each process, so runtime singletons, generated SQLite settings, Model initialization and data remain isolated without an in-process reset protocol. A suite that contains any database test links the SQLite-capable runtime once; only a selected database branch loads its generated setting and initializes Models. A non-database branch does neither. + +Native cache identity continues to cover generated code, compiler identity, runtime bytes, toolchain, platform, environment and compiler options. Tests use a dedicated fast compilation profile because they validate behavior rather than production code generation performance; `run` and `build` retain the production optimization contract. + +For each case the runner: + +1. creates an owned temporary project directory; +2. reuses the single compiled suite and selects the checked private test entry by internal case index; +3. stages the executable in that directory; +4. writes a generated `config/setting.json` only when the reachable graph uses Model operations; +5. runs the executable with the temporary project as its executable/config root; +6. captures stdout/stderr and records pass/failure; +7. drops all owned native and runtime artifacts before continuing. + +Successful process output is captured and omitted from runner status. Failed output is printed in clearly labelled blocks below the case result. A source/check failure prevents all execution; a per-case compile/start/run/assert/business/fault failure is recorded and later cases continue. The final exit status is nonzero when any case failed or was not run. + +The first release has no child timeout or signal manager. Explicit cancellation may terminate the command; recoverable success/failure paths own deterministic cleanup. + +## SQLite Isolation + +The runner never loads the project's deployment database configuration for tests. If no test graph has a database effect, the suite uses the base runtime profile. A mixed suite links the SQLite-capable runtime once, but a selected non-database case writes no setting file and does not call database bootstrap or Model initialization. + +If the graph has a database effect, the runner generates a temporary setting with: + +- `database.default` as SQLite; +- one SQLite entry for each explicit Model connection name; +- paths below that case's `data/db/`; +- one connection and the existing bounded page defaults. + +Package-root selectors retain runtime resolution semantics: an undeclared root falls back to `default`; explicit selectors retain separate connections. The generated settings are parsed by `dever_runtime::config::Settings`, validated by `Program::validate_database_settings`, and consumed by the normal bootstrap, schema migration, Seed, transaction and shutdown code. No test-only database handle reaches Dever source. + +The emitter currently initializes the application's complete Model schema once a reachable test uses the database. This preserves foreign keys, migrations and Seed as an application-wide contract. A non-database test uses no database profile even when unrelated Models exist in the module. + +## CLI And Formatting + +`Action::Test` is added to the existing explicit CLI argument parser; there is no generic subcommand framework. The command performs semantic checking, normal warnings and API baseline validation before execution. + +Formatting remains syntax-only. `fmt` discovers both source roots, prepares every replacement before committing any, validates both snapshots again, then performs atomic per-file renames. A missing `test/` contributes no files. This prevents module files from changing when a test Markdown contract or file permission fails. + +Human output follows one stable shape: + +```text +running 2 tests +test news/article/publish ... ok +test user/account/register ... FAILED + +test result: FAILED. 1 passed; 1 failed; 0 not run +``` + +No JSON/JUnit output or test filter is introduced. + +## Markdown And CMS + +Test Markdown uses the existing strict document contract. Its package metadata names the logical package (for example `user.account.register`), and its function section documents the filename-matched entry. No test-specific Markdown headings are added. + +The maintained CMS Dever and Markdown projects receive equivalent application tests. They call existing App boundaries and assert observable business values and Model counts. They do not add test-only production App functions or duplicate CRUD. + +## Compatibility And Trade-offs + +- Production `check/api/run/build` continue loading only `module/`; their checked Program and native cache inputs remain unchanged. +- `fmt` intentionally expands to the optional test root because tests are first-class Dever source. +- One suite link removes per-case native compilation. Process-per-case dispatch retains runtime/database isolation, while the suite may be larger than one reachability-pruned test binary. +- Real PostgreSQL application tests are deferred. Adding them later requires an explicit safe test connection and cleanup contract; it must not alter default SQLite behavior. +- There are no fixture hooks or shared test modules. A later addition must prove that it does not introduce hidden order or shared mutable state. + +## Cleanup And Rollback + +All test execution artifacts are owned by RAII temporary-directory and native-program guards. Cleanup errors do not replace the primary test result. No project source, deployment setting or data path is rewritten by `dever test`. + +The change is additive. Rollback removes the Test source origin/layout, assertion lowering, CLI action and example tests; no production source migration or database migration is required. diff --git a/.trellis/tasks/09-19-application-testing/implement.jsonl b/.trellis/tasks/09-19-application-testing/implement.jsonl new file mode 100644 index 0000000..6861dfb --- /dev/null +++ b/.trellis/tasks/09-19-application-testing/implement.jsonl @@ -0,0 +1,11 @@ +{"file": ".trellis/tasks/09-19-application-testing/prd.md", "reason": "当前应用测试需求、边界与验收标准"} +{"file": ".trellis/tasks/09-19-application-testing/design.md", "reason": "已收口的编译器与运行器技术设计"} +{"file": ".trellis/tasks/09-19-application-testing/implement.md", "reason": "分阶段实施与最小验证清单"} +{"file": ".trellis/spec/backend/compiler-contracts.md", "reason": "保持语言、检查器、HIR 与原生后端单一路径合同"} +{"file": ".trellis/spec/backend/directory-structure.md", "reason": "约束源码目录、crate 与测试文件归属"} +{"file": ".trellis/spec/backend/error-handling.md", "reason": "约束源码诊断、运行失败与错误传播"} +{"file": ".trellis/spec/backend/quality-guidelines.md", "reason": "约束复用、复杂度与回归测试质量"} +{"file": ".trellis/spec/backend/toolchain-and-library.md", "reason": "约束 Rust 工具链、依赖与定向命令"} +{"file": ".trellis/spec/backend/database-guidelines.md", "reason": "约束 Model、SQLite 隔离与 PostgreSQL 边界"} +{"file": ".trellis/spec/guides/code-reuse-thinking-guide.md", "reason": "避免测试路径形成平行 parser/checker/runtime"} +{"file": ".trellis/spec/guides/cross-layer-thinking-guide.md", "reason": "核对 CLI 到 checker/native/runtime 的完整数据流"} diff --git a/.trellis/tasks/09-19-application-testing/implement.md b/.trellis/tasks/09-19-application-testing/implement.md new file mode 100644 index 0000000..a498d61 --- /dev/null +++ b/.trellis/tasks/09-19-application-testing/implement.md @@ -0,0 +1,123 @@ +# Dever 应用测试体系实施清单 + +## Start Gate + +- [x] 用户在最终规划摘要之后明确批准实施。 +- [x] 运行 `python3 ./.trellis/scripts/task.py start 09-19-application-testing`，确认任务进入 `in_progress`。 +- [x] 重新加载 `trellis-before-dev`，核对当前 PRD、设计、执行清单和 backend specs。 +- [x] 开工前记录实际 git/worktree 状态，只处理本任务拥有的文件，不清理用户已有改动。 + +## Phase 1: Test Source Layout And Static Contract + +- [x] 在 `source.rs` 增加编译器赋予的 Test 来源、显示/逻辑路径和组合项目加载；保持现有生产 `SourceMap::load` 合同。 +- [x] 在 layout/parser/checker 中加入严格的 `<component>/<domain>/<topic>` Test 布局，不增加源码关键字或第二套 parser。 +- [x] 在检查结果中建立按身份排序的 `TestCase`，强制同名 ordinary 零输入零输出入口和文件内私有辅助声明。 +- [x] 扩展 role-aware function/type resolution：同领域 App+Domain、跨领域 App、禁止 Model CRUD/Adapter/Port/API/其他测试。 +- [x] 把来源/逻辑路径加入 check cache identity，防止 Test 与普通 strict source 误复用。 +- [x] 增加路径、入口、重复身份和可见性的定向正反测试。 + +## Phase 2: Assertions And Native Entry + +- [x] 复用 `Intrinsic` HIR 形状加入仅 Test owner 可构造的 `assert`/`assert_eq`，保持普通同名业务函数语义不变。 +- [x] `assert` 强制 Bool；`assert_eq` 强制相同 comparable 类型并保持左右各求值一次、源顺序不变。 +- [x] 在 native emitter 生成源码定位的 assertion failure，复用现有 `Render` 输出 actual/expected。 +- [x] 增加从 checked `TestCase` 编译私有入口的 native API，不放宽生产 `Program::entry` 的公开性要求。 +- [x] 验证标量、record、choice、nullable、List、Map、MapEntry、Model ID，以及不可比较/类型不匹配/生产源码调用断言的拒绝。 + +## Phase 3: CLI Runner And Isolation + +- [x] 在 CLI 增加 `test <project-root>`，复用现有 module 检查、warning 和 API baseline 路径。 +- [x] 建立单一 `test_runner` owner，负责排序、单套件 compile、逐用例独立进程、输出捕获、状态汇总和临时目录生命周期；不创建通用 runner/framework 层。 +- [x] 无测试时输出 `0 tests` 成功；源码检查失败时零执行；单用例失败后继续后续用例。 +- [x] 对无数据库 effect 的用例使用 base profile，不读写 setting。 +- [x] 对有数据库 effect 的用例生成临时 SQLite setting，复用正式 Settings 校验、Model schema/Seed/transaction/shutdown。 +- [x] 每用例使用独立临时项目和数据库；确认项目 `config/setting.json`、`data/` 与环境变量不参与数据库选择。 +- [x] 失败时分区展示捕获的 stdout/stderr，runner 状态行不与测试输出混合。 +- [x] 覆盖编译失败、启动失败错误映射、断言失败、未捕获业务失败、runtime fault 和清理。 + +## Phase 4: Formatter And Maintained CMS + +- [x] 调整 CLI formatter，使 module/test 两个可选源码集合先完整诊断和 prepare，再统一 commit；任一失败时零改写。 +- [x] 为 CMS Dever 项目增加 `test/<component>/<domain>/<topic>.dever` 业务用例。 +- [x] 为 CMS Markdown 项目增加相同路径、入口和合同的 `.dever.md` 用例，遵守现有 Markdown 文档协议。 +- [x] 删除 CMS CLI 回归中被正式应用测试替代的临时 probe 源码注入，仅保留仍验证 build/package 的独立职责。 +- [x] 更新 `LANGUAGE.md`、`MARKDOWN-SYNTAX.md`、根 README 和 examples README 的正式测试合同与命令。 + +## Phase 5: Refactor And Verification + +- [x] 检查 SourceMap 双根加载、test metadata、assert lowering、runner 报告和 temp ownership 是否存在重复 owner 或平行逻辑。 +- [x] 搜索并删除调试输出、过渡 API、无用 helper、死分支、临时测试文件和过期 CMS probe。 +- [x] 确认生产 source set、API snapshot、main entry、native cache、runtime profile 与无测试项目行为不变。 +- [x] 检查磁盘空间后再运行会链接原生程序的定向测试；不运行 workspace 全量测试、真实 PostgreSQL 或性能压测。 + +## Verification Record + +- `cargo check --offline -p dever-core --features reference -p dever-cli -p dever-tests --test application_testing`：通过。 +- `application_testing`：8/8 通过；包含 plain/Markdown 发现、可见性、类型、`result(call)`、复合断言、源码位置及含未引用 Model 的 base profile。 +- `formatter`：12/12 通过；module/test 统一 prepare 和只读失败零改写通过；两套 CMS `fmt --check` 通过。 +- `source_architecture`：8/8 通过；`markdown_source`：20/20 通过。 +- `source_visibility`：10/10 通过；`contract_execution`：9/9 通过；`api_routes`（`api` feature）：7/7 通过。 +- `cms_project_checks_runs_and_builds_from_an_isolated_copy`：通过；Dever/Markdown 各连续测试两次，部署 setting 被替换为非法 JSON 时测试仍为 2/2，通过后 check/run/build/打包程序仍通过，且无效 test 源码不进入生产命令。 +- CLI 零测试、失败继续/输出分区、成功/失败临时目录清理：分别通过；`RUSTC=/bin/false` 定向故障注入确认两个编译失败均汇总且零临时残留。 +- 未捕获业务失败和动态数值 fault 的既有原生定向用例：通过。 +- `git diff --check` 和变更文件尾随空白扫描：通过；CLI 默认依赖仅启用 `dever-core` default feature，不含 reference。 +- `rustfmt`、`clippy` 组件未安装，未运行；未运行 workspace 全量测试、真实 PostgreSQL、性能/内存压测或持久服务。 + +## Phase 6: Test Suite Compilation Performance + +- [x] 在 native owner 中生成一个包含全部 checked TestCase specialization 的测试套件，不增加解释器或动态注册表。 +- [x] 复用 native 构建缓存并使用测试专用快速编译参数；生产 `run/build` 优化参数保持不变。 +- [x] CLI 只编译一次套件，按稳定用例编号逐个启动独立进程并保留失败继续、输出捕获和临时目录清理。 +- [x] 混合数据库/非数据库套件仅在数据库分支加载临时 SQLite 配置并初始化 Model。 +- [x] 增加多入口分派、一次编译、缓存命中、配置隔离和失败继续的定向回归。 +- [x] 记录 CMS 冷/暖执行耗时和峰值 RSS；不把本机数据外推为生产性能。 + +### Phase 6 Verification Record + +- `cargo check --offline -p dever-core -p dever-cli -p dever-tests --test application_testing --test native_cache`：通过。 +- `application_testing`：8/8 通过；一个 native suite artifact 按两个稳定编号执行成功/失败入口，并确认第二次编译命中 native cache。 +- `native_cache`：6/6 通过，另有 1 个既有显式 ignored；runtime 内容变化、工具链/参数变化、并发发布、损坏产物拒绝、原始快照与元数据摘要备忘均通过。 +- CLI 定向回归：混合数据库/非数据库套件 3/3 通过；失败后继续与 stdout/stderr 捕获通过；零测试不调用编译并成功；套件编译失败时 2/2 用例统一标记失败并清理临时目录。 +- 普通单入口 native hello 定向用例通过，确认 `run/build` 共用的生产发射路径未被套件抽取破坏。 +- CMS Dever 实际 `deverc test`：2/2 通过。稳定 runtime 下首次单套件链接为 21.37 秒、峰值 346068 KiB；随后内容缓存命中为 1.79 秒、峰值 54132 KiB。优化前同项目为 119.40 秒、594772 KiB。 +- 在 Cargo 测试环境的绝对 `RUSTC` 与普通 PATH `rustc` 之间切换会触发 Cargo 重建 native runtime；一次 75.56 秒、594976 KiB 的观测包含该重建，不作为测试套件链接基线。 +- runtime 摘要目录只保留当前元数据身份；`/tmp` 无 `dever-application-test-*`、`dever-build-*` 或 `dever-cms-project-*` 残留，CMS 示例无数据库或测试可执行文件残留。 +- `git diff --check` 与变更文件尾随空白扫描通过。`rustfmt`、`clippy` 组件仍未安装；未运行 workspace 全量测试、真实 PostgreSQL、64/128 MiB/cgroup 压测或持久服务。 + +## Focused Validation + +允许且计划执行的最小检查： + +```bash +cargo check --offline -p dever-core -p dever-cli +cargo test --offline -p dever-tests --test application_testing +cargo test --offline -p dever-tests --test source_architecture +cargo test --offline -p dever-tests --test markdown_source +cargo test --offline -p dever-cli --test cms_project +git diff --check +``` + +如 `rustfmt`/`clippy` 组件可用，再执行： + +```bash +cargo fmt --all -- --check +cargo clippy --offline -p dever-core -p dever-runtime -p dever-cli -p dever-tests --test application_testing -- -D warnings +``` + +不运行： + +- workspace 全量测试； +- `cargo build` 或应用全量发布构建； +- 真实 PostgreSQL； +- 64/128 MiB、cgroup 或吞吐压测； +- 持久 HTTP/网络服务。 + +## High-risk Files And Review Points + +- `crates/dever-core/src/source.rs`、`check.rs`、`check/layout.rs`、`check/symbols.rs`：来源身份和可见性不能泄漏到生产。 +- `crates/dever-core/src/check/expressions.rs`、`intrinsic.rs`、`native.rs`、`native/intrinsics.rs`、`native/records.rs`：断言必须静态类型化、单次求值并保留源码位置。 +- `crates/dever-core/src/native/build.rs`、`model.rs`：私有测试入口不能放宽生产 entry，SQLite profile 必须来自测试生成配置。 +- `crates/dever-cli/src/main.rs`、`format.rs`、新增 runner：临时目录、输出捕获、失败继续和格式化原子性。 +- `examples/cms/{dever,md}`：两套测试合同必须同步，不给生产 App 增加测试专用能力。 + +任何实现如果需要 Test 关键字、共享动态值、生产公开测试函数、环境变量数据库配置或真实 PostgreSQL，均视为超出当前设计，回到 planning 复核。 diff --git a/.trellis/tasks/09-19-application-testing/prd.md b/.trellis/tasks/09-19-application-testing/prd.md new file mode 100644 index 0000000..44c11d5 --- /dev/null +++ b/.trellis/tasks/09-19-application-testing/prd.md @@ -0,0 +1,101 @@ +# Dever 应用测试体系 + +## Goal + +让 Dever 应用可以用 Dever 源码编写和运行确定性的领域回归测试，使 AI 修改 App、Domain 或 Model 后能够验证业务合同，而不依赖 Rust 宿主测试、HTTP 接口或正式数据库。 + +测试是可选能力。没有测试的项目保持现有开发和运行方式；测试能力不得引入第二套解释器、生产可见测试入口或环境变量配置。 + +## Background + +- 当前 CLI 只有 `check`、`api`、`fmt`、`run` 和 `build`，应用自身没有测试命令。 +- 编译器仓库已有 Rust 回归测试，但它们验证编译器实现，不能表达具体 Dever 应用的业务合同。 +- 生产源码从 `module/` 加载，`main.main` 是 `run/build` 的唯一启动入口；App 是跨领域能力边界，Domain、Model、Port、Adapter 和 API 是领域实现边界。 +- parser、checker、HIR、原生后端、源码位置、错误传播、Model 迁移和 Seed 已有单一正式路径，应由测试复用。 +- ORM 的真实 PostgreSQL 验收已经由独立编译器测试覆盖，不属于首版应用测试命令。 + +## Requirements + +### Source And Discovery + +- 应用测试固定放在项目根 `test/`，与 `module/` 生产源码分离。 +- 测试路径必须是 `test/<component>/<domain>/<topic>.dever` 或 `test/<component>/<domain>/<topic>.dever.md`，不支持更深层级。 +- 每个测试文件由文件名选择同名普通函数作为唯一入口。例如 `test/user/account/register.dever` 执行其中的 `register() ()`。 +- 测试入口必须是零输入、零输出的 ordinary 函数。缺少同名函数、同名入口签名错误或重复测试身份必须在执行前产生源码诊断。 +- 测试文件中的其他函数和类型只属于该文件，不自动执行，也不能被其他测试文件或生产源码调用。 +- 不增加 `test` 关键字、`test_` 前缀、每文件 `main()`、package/expose 声明或顶层可执行语句。 +- `dever test <project-root>` 同时加载生产源码和测试源码，并在运行任何用例前完成全部语法、名称、类型、分句、effect、失败义务和架构检查。 +- 发现顺序按测试身份稳定排序，不依赖文件系统遍历顺序。 +- 项目没有 `test/` 或目录为空时，合法项目执行 `dever test` 输出 `0 tests` 并成功退出。 +- `check/api/run/build` 不加载测试源码。测试声明不进入生产二进制、API 快照或 HTTP 路由。 +- `fmt` 作为源码工具应同时格式化 `module/` 和存在的 `test/`，保持两者统一的 `.dever`/`.dever.md` 格式合同；任一源码失败时不得部分改写。 + +### Architecture And Visibility + +- 测试文件是所属 `<component>/<domain>` 的测试友元，可以调用同领域 App 和 Domain。 +- 跨领域调用只能经过目标 App。 +- 测试可以读取 App 返回的 Model 值和同领域类型，但不能直接调用 Model CRUD、Adapter、Port 或 API。 +- 生产 App、Domain 和其他测试文件都不能反向调用测试函数或访问测试私有类型。 +- 测试不能声明 Model、HTTP 路由或生产 App 能力。 +- 测试文件必须表达一个内聚业务场景；不建立 `common`、`shared`、`utils`、`helper`、全局 fixture 等测试垃圾桶。 + +### Assertions And Failures + +- 首版只提供测试源码专用的 `assert(condition)` 和 `assert_eq(actual, expected)`。 +- `assert` 只接受 `Bool`。`assert_eq` 要求两侧是相同且可比较的类型，不做数值提升、字符串求值或运行时反射。 +- `assert_eq` 复用现有比较与渲染合同，支持标量、record、choice、可空值、List、Map、MapEntry 和 Model ID；失败报告测试身份、源码位置、实际值和期望值。 +- 不增加 `assert_true`、`assert_false`、`assert_not_equal`、`assert_null`、异常断言或自定义 matcher。 +- 业务失败继续使用现有 `result(call)` 和 choice 分句验证；不增加 `try/catch/throws`。 +- 测试成功表示入口正常完成；断言失败、未捕获业务失败或程序 fault 表示该用例失败。 +- 一个用例失败后继续运行其余独立用例，最终汇总通过、失败和未运行数量，并以非零状态结束。 + +### Isolation And Lifecycle + +- 首版没有 `before/after`、全局 setup、共享 fixture、用例排序依赖或测试间共享状态。 +- 每个测试文件在独立原生进程和独立临时项目目录中运行。 +- 不触达 Model 的测试不初始化数据库。 +- 触达 Model 的测试由运行器生成每用例独立的临时 SQLite 配置，走正式配置解析、迁移、Seed、事务、连接池和 shutdown 路径。 +- 测试运行器不读取、复制或连接项目正式数据库 URL；不使用环境变量或命令行数据库覆盖。 +- 临时数据库、数据目录、配置、可执行文件和运行产物在成功或失败后由运行器清理，不写入项目 `data/`。 +- 首版不使用外层事务回滚代替隔离，因为这会改变被测 App 的真实提交和回滚语义。 +- 测试按稳定顺序串行运行；并行、随机顺序、重试和自动超时不属于首版。 + +### CLI And Reporting + +- 新增正式命令 `dever test <project-root>`；当前开发二进制对应 `deverc test <project-root>`。 +- 默认输出简洁的人类可读状态和最终汇总。测试进程 stdout/stderr 被捕获，不得冒充 runner 状态；失败时按独立区块展示。 +- 源码诊断、原生编译失败、进程启动失败、断言失败、业务失败和 runtime fault 保留各自可辨认的信息。 +- 测试复用与 `run/build` 相同的原生后端、runtime feature 裁剪、源码位置和构建缓存，不链接 reference evaluator。 +- 首版不增加测试名过滤、机器报告格式或新的数据库配置结构。 + +### Maintained Example + +- CMS 的 Dever 与 Markdown 项目各自增加等价测试，验证账户规范化、Model 写入/Seed 和文章发布等现有 App 合同。 +- 两套测试使用相同路径、入口名称和断言，只因源码容器格式不同而不同。 +- 旧 `examples/old/` 不迁移为应用测试模板。 + +## Acceptance Criteria + +- [x] `.dever` 与 `.dever.md` 测试均能按 `<component>/<domain>/<topic>` 稳定发现，文件名选择同名零输入、零输出函数。 +- [x] 非法路径、缺失/错误入口、重复身份、测试跨域越权、直接 Model/API 调用和生产源码反向访问测试均在执行前被拒绝并定位源码。 +- [x] `assert` 和 `assert_eq` 有正反类型检查；相等失败包含测试身份、原源码位置、实际值和期望值。 +- [x] 测试可以通过 `result(call)` 验证业务失败；未捕获业务失败、断言失败和 runtime fault 均使当前用例失败而不阻止后续用例。 +- [x] 无 `test/` 的合法项目执行 `dever test` 报告 `0 tests` 并成功；非法生产源码仍被拒绝。 +- [x] 纯非数据库套件使用 base runtime；混合套件的非数据库用例不读取配置或初始化数据库；使用 Model 的两个测试各自执行迁移和 Seed，写入互不影响，顺序互换结果不变。 +- [x] 测试不读取正式 `setting.json` 的数据库 URL，不连接 PostgreSQL，不在项目 `data/` 或源码目录留下临时文件。 +- [x] 测试进程成功和失败后均清理临时配置、SQLite 文件、可执行文件和 native 临时目录。 +- [x] `check/api/run/build` 的源码集合、API 快照、CMS 运行结果和生产产物不因存在 `test/` 而变化。 +- [x] `fmt` 对 module/test 的变更采用同一次准备后提交；任一格式诊断或准备失败时两处源码均保持原样。 +- [x] CMS Dever/Markdown 测试覆盖相同业务合同并由 `dever test` 通过。 +- [x] 同一项目连续运行两次时测试顺序和汇总一致，生成测试产物不链接 reference evaluator。 +- [x] 一次 `dever test` 只生成和编译一个测试套件程序；每个用例仍在独立子进程和临时项目中运行。 +- [x] 测试套件复用 native 内容寻址缓存；未改变有效生成代码时再次运行不调用 rustc 编译测试程序。 +- [x] 混合数据库/非数据库用例时，只有数据库用例进程读取运行器生成的 SQLite 配置并初始化 Model；非数据库用例不读取配置或初始化数据库。 + +## Out Of Scope + +- 真实 PostgreSQL 应用测试配置、schema 管理和外部数据库清理；继续由现有 `postgres_orm` 专项验收负责。 +- 自动 `before/after`、共享 fixture 文件、mock/stub、Port/Adapter 替身和依赖注入协议。 +- 测试筛选、watch、并行、随机顺序、重试、自动超时、覆盖率、快照、属性测试和模糊测试。 +- 浏览器/UI、HTTP 录制回放、外部服务沙箱和机器可读报告。 +- API 新设计、本体、智能体、package registry、跨平台发行和编译器自举。 diff --git a/.trellis/tasks/09-19-application-testing/task.json b/.trellis/tasks/09-19-application-testing/task.json new file mode 100644 index 0000000..7a5e057 --- /dev/null +++ b/.trellis/tasks/09-19-application-testing/task.json @@ -0,0 +1,28 @@ +{ + "id": "application-testing", + "name": "application-testing", + "title": "Dever 应用测试体系", + "description": "为 Dever 应用建立根 test 目录、dever test 命令、类型安全断言、隔离 fixture 与可重复执行合同", + "status": "in_progress", + "dev_type": null, + "scope": null, + "package": null, + "priority": "P1", + "creator": "Antigravity", + "assignee": "Antigravity", + "createdAt": "2026-09-19", + "completedAt": null, + "branch": "main", + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [], + "parent": "09-04-dever-language-mvp", + "relatedFiles": [], + "notes": "", + "meta": { + "execution_mode": "inline" + } +} \ No newline at end of file diff --git a/.trellis/tasks/09-20-durable-jobs/check.jsonl b/.trellis/tasks/09-20-durable-jobs/check.jsonl new file mode 100644 index 0000000..eff2287 --- /dev/null +++ b/.trellis/tasks/09-20-durable-jobs/check.jsonl @@ -0,0 +1,6 @@ +{"file":".trellis/spec/backend/quality-guidelines.md","reason":"Focused runtime/database/process checks"} +{"file":".trellis/spec/backend/error-handling.md","reason":"Retry, dead-letter and cancellation error behavior"} +{"file":".trellis/spec/backend/directory-structure.md","reason":"Job owner separation and no parallel queue implementation"} +{"file": ".trellis/tasks/09-20-durable-jobs/research/implementation-map.md", "reason": "Verify corrected transaction, lease, cron and shutdown contracts"} +{"file": ".trellis/spec/backend/compiler-contracts.md", "reason": "Verify language and native emitter contract parity"} +{"file": ".trellis/spec/backend/database-guidelines.md", "reason": "Verify transaction and lease correctness"} diff --git a/.trellis/tasks/09-20-durable-jobs/design.md b/.trellis/tasks/09-20-durable-jobs/design.md new file mode 100644 index 0000000..bdeacdb --- /dev/null +++ b/.trellis/tasks/09-20-durable-jobs/design.md @@ -0,0 +1,100 @@ +# Dever 持久化 Job 设计 + +## Source Contract + +Job role 使用 `job.dever` 或平铺的 `job/*.dever`，继续遵守角色文件与目录互斥、单 topic 必须合并的规则。Job 只是持久任务入口，不是可调用的普通函数： + +```dever +database default + +job publish_due(input: app.PublishInput) () retry(5) timeout(30000) { + app.publish_due(input.article_id) +} + +job cleanup_sessions() () retry(3) timeout(30000) { + app.cleanup_expired() +} + +schedule cleanup_sessions = "0 * * * *" +``` + +Payload record 由同领域 App 声明为公开契约，Job 不新增第二套 DTO 可见性。Job 只能有零个输入，或一个 wire-safe record 输入；输出必须为空；`retry(n)` 表示包含首次执行在内最多执行 `n` 次。`timeout(ms)` 和 `retry` 都是必填、有硬上限的正整数。 + +共享 wire schema 增加 nominal ModelId 节点。编码仍使用既有 ModelId 数值表示，但 fingerprint 包含所属 Model identity，不能把两个 Model 的 id 降级为同一种 Int/Id。Job payload 仍拒绝 Secret、private Model data、Choice、handler 和不稳定类型。 + +App 通过 compiler-owned 静态 target 入队： + +```dever +id = dever.job.enqueue(job.publish_due, input, key) +id = dever.job.enqueue_at(job.publish_due, input, key, run_at) +id = dever.job.enqueue(job.cleanup_sessions, key) +id = dever.job.enqueue_at(job.cleanup_sessions, key, run_at) +``` + +Target 不是普通值，不能存储、转发或用于普通 call/run/group。enqueue 使用专用 HIR，不把 Job handler 加入同步调用/effect/递归图；Job handler 作为 worker 独立 root 生成并检查。 + +## Effect And Transaction Integration + +Model 和 Job 都映射到一个统一 database binding。effect 系统记录有类型的 database owner，并通过唯一 accessor 解析为现有 `ConnectionSelector`；Model CRUD 仍保留 Model schema owner，不制造虚假 Model，也不增加第二套 transaction context。 + +enqueue 只传播队列连接的 database/suspension/failure effect，不同步传播 Job handler 的 Model、Port 或业务 failure。transaction App 的 enqueue 使用现有隐藏 `_database: Option<&Transaction>` 和 `Executor::new`，与同连接 Model 写入共享物理 transaction；无 ambient transaction 时开启并提交短 transaction。显式不同连接在语义检查拒绝，依赖 default/package fallback 的冲突在 settings-aware run/build bootstrap 拒绝。 + +## Private Schema And States + +每个使用 Job 的逻辑 database 初始化 runtime-owned versioned schema，独立于 Model history、snapshot、seed 和 migration。至少包含： + +- Job：id、target identity、schema fingerprint、payload、dedupe key、state、run_at、attempt、max attempts、timeout、claim token、lease deadline、安全错误分类/摘要和时间戳。 +- Schedule cursor：稳定 schedule identity、cron fingerprint、last processed UTC minute 和更新时间。 + +状态为 pending、running、succeeded、dead、blocked。业务 key 只在 pending/running 上保证 `(target, key)` 唯一，终结后可重用；schedule occurrence 由持久 cursor 保证永久单窗口物化一次，不能依赖 active-only 唯一索引。 + +未知 target、schema mismatch 和 corrupt payload 不执行 App，保留原记录并进入 blocked。历史记录的实际 timeout 也必须满足当前 SQLite lease 约束，否则以固定 `invalid_policy` 进入 blocked。attempt 耗尽进入 dead，并由独立有界清理处理，不占用领取可执行任务的槽。运行时不保存 Secret、完整 payload、backtrace 或未经控制的 `AppError::to_string()`；只保存固定类别和有界安全摘要。 + +## Claim, Lease And Retry + +PostgreSQL 使用短 transaction、bounded batch 和 `FOR UPDATE SKIP LOCKED`；SQLite 使用短 `BEGIN IMMEDIATE` 与条件 update。只按当前空闲执行槽 claim，不能预占后在无界内存队列等待。 + +每次成功 claim 时原子增加 attempt，并生成独立随机 claim token。这样进程在 handler 返回前崩溃也会消耗一次 attempt。过期 running 在 attempt 未达上限时可重领，已达上限时转 dead。 + +renew、success、retry、blocked/dead 更新均使用 `id + claim token + running state + 未过期 lease` CAS。claim 在获得写锁后读取时钟并建立租约；renew/complete 先锁定 claim 行，再读取同一 Clock，不能用锁等待前的时间续租或确认已过期任务。CAS 失败表示已失去所有权，旧执行者不得覆盖新 owner 状态。worker 在 lease 周期内续租；handler timeout 或安全业务失败按 checked exponential backoff 重新 pending，且不超过配置上限。语义为 at-least-once，外部副作用仍由业务 idempotency 保证。 + +## UTC Schedule + +Cron 固定为五字段 UTC：minute、hour、day-of-month、month、day-of-week。支持 `*`、单值、逗号列表、闭区间和步长；范围分别是 `0..59`、`0..23`、`1..31`、`1..12`、`0..6`，其中 Sunday 为 0。day-of-month 与 day-of-week 同时受限时采用 cron 的 OR 语义，否则匹配受限字段。 + +首次启用从当前 UTC minute 建 cursor，不回放历史。以后在同一 transaction 锁定 cursor、插入 occurrence、推进 cursor；每轮最多扫描并推进 256 个分钟槽，只对匹配 cron 的槽生成任务，剩余分钟后续继续补。slot key 包含稳定 schedule identity 和 UTC minute，与业务 key 分离。相同 schedule identity 的 cron fingerprint 不一致时启动失败关闭，禁止多版本互相改写 cursor。 + +## Runtime Topology And Shutdown + +严格配置只来自 `config/setting.json`： + +```json +{ + "runtime": { "mode": "all", "shutdown_ms": 30000 }, + "job": { + "workers": 4, + "poll_ms": 250, + "lease_ms": 60000, + "retry_base_ms": 1000, + "retry_max_ms": 300000 + } +} +``` + +`runtime.mode` 只能为 api、worker、all。generated entry 在 listen、schema materialize 或 claim 前统一校验 source capabilities、HTTP/Job 配置、driver 和所有上限。api 模式中的 `dever.job.serve()`、worker 模式中的 `dever.api.serve()`安全 no-op；同一 binary 保留两套静态 dispatch roots，不按运行模式裁剪。 + +没有 worker capability 的纯 API 应用可省略 runtime，默认 api 和 30000 ms；有 worker capability 时必须显式配置。worker 连接池必须比 worker 槽至少多一个连接，供 claim/heartbeat 使用；不引入独立控制连接池。SQLite 只有一个写者，额外连接无法绕过业务写事务，因此启动前还必须验证 `lease_ms > Job timeout_ms`，让租约覆盖 handler 的最长执行时间；PostgreSQL 可继续使用较短租约并续租。 + +generated entry 拥有唯一 lifecycle：结构化等待应用 root 与 SIGTERM/Ctrl-C。收到信号后广播 shutdown；API 关闭 listener 并复用 HTTP connection drain，Job 停止 schedule/claim 后等待 active handlers。到 `runtime.shutdown_ms` 才取消剩余作用域；未确认 success 的 Job 保留/恢复 lease。应用 root 汇合后执行 database shutdown，再 flush log。Tokio signal 是既有依赖的 feature 扩展，不引入 signal/queue/cron 框架或 unsafe handler。 + +Job timeout 上限为 3,600,000 ms，lease 的有界范围为 100..7,200,000 ms。测试使用 lease 上限，保证所有合法 Job timeout 都能在隔离 SQLite 中执行，仍复用生产 claim/dispatch/finish 和策略检查。 + +## Application Testing + +每个 test case 保持独立 native module、进程、临时目录、Port fake graph 和 SQLite connections。闭包同时包含：test 直接 enqueue 的 Job、Job handler 调用的 App/Port，以及 handler 的 Model connections；生产 Adapter 和部署 settings 不得回退进入 case。 + +测试专用 compiler-owned 操作提供 case-local clock advance 和 bounded due-job drain。它们复用生产 claim/dispatch/finish 状态机，不启动永久 `serve()`，不真实 sleep，也不允许测试直接调用 Job entry。无 database/Job effect 的纯测试仍不加载 settings。 + +## Explicit Non-Goals + +不增加动态 target string、通用 DI/queue framework、外部 broker、非 UTC schedule、exactly-once 外部副作用、payload 自动迁移或 Job 管理 API/front。真实 PostgreSQL 并发、锁与取消仍需隔离 fixture 验收；feature check 不能替代运行证据。 diff --git a/.trellis/tasks/09-20-durable-jobs/implement.jsonl b/.trellis/tasks/09-20-durable-jobs/implement.jsonl new file mode 100644 index 0000000..0015871 --- /dev/null +++ b/.trellis/tasks/09-20-durable-jobs/implement.jsonl @@ -0,0 +1,5 @@ +{"file":".trellis/spec/backend/compiler-contracts.md","reason":"Source role, static target, effect and visibility contracts"} +{"file":".trellis/spec/backend/database-guidelines.md","reason":"Transaction, connection, migration and cancellation invariants"} +{"file":".trellis/spec/backend/toolchain-and-library.md","reason":"Structured concurrency, native runtime and shutdown"} +{"file":".trellis/tasks/09-13-typed-orm/design.md","reason":"Existing SQLite/PostgreSQL transaction and pool architecture"} +{"file": ".trellis/tasks/09-20-durable-jobs/research/implementation-map.md", "reason": "Verified source ownership and implementation risks"} diff --git a/.trellis/tasks/09-20-durable-jobs/implement.md b/.trellis/tasks/09-20-durable-jobs/implement.md new file mode 100644 index 0000000..5f151b2 --- /dev/null +++ b/.trellis/tasks/09-20-durable-jobs/implement.md @@ -0,0 +1,127 @@ +# Dever 持久化 Job 实施计划 + +## Start Gate + +- [x] time-typed-codec 与 port-adapter 完成。 +- [x] 用户批准父任务规划并允许继续实现。 +- [x] 读取 structured concurrency、ORM transaction/settings、native profile、application test runner 和 HTTP drain owner。 +- [x] 完成源码 implementation map 与独立设计复核。 +- [x] 收紧 ModelId、connection effect、claim attempt、cron cursor、lifecycle 和测试 harness 契约。 + +## 1. Source And Static Contract + +- [x] 增加 Job SourceRole、layout、contextual declaration、policy、schedule、formatter 和 Markdown parity。 +- [x] 增加 Program Job metadata、静态 target identity、App-owned payload schema 和 nominal ModelId wire codec。 +- [x] 增加 enqueue/enqueue_at 专用检查与 HIR；拒绝普通 call、handler/run/group 转发和非法 role/type/signature。 +- [x] 将 Job handler 作为独立生成 root，不把 enqueue 视为同步依赖或递归 edge。 + +Gate：新增 `durable_jobs` 源码/Markdown测试覆盖合法声明、路径、签名、可见性、payload、policy、cron和target正反例。 + +## 2. Connection Effect And Atomic Enqueue + +- [x] 将 Model-only database effect 推广为统一的有类型 connection owner，并集中解析 ConnectionSelector。 +- [x] enqueue 只传播队列连接 effect，不传播 handler 业务 effect/failure。 +- [x] native emitter 复用 `_database`、Executor 和 wire codec，实现 standalone short transaction 与 ambient transaction enqueue。 +- [x] settings-aware bootstrap/test runner 使用统一 connection effect；拒绝解析后的跨连接 transaction。 + +Gate：enqueue-only transaction 可生成；Model write + enqueue 同时 commit/rollback；跨连接拒绝；同 transaction 不二次 checkout。 + +## 3. Private Store State Machine + +- [x] 实现 SQLite/PostgreSQL runtime private schema、版本校验和初始化锁，不进入 Model history。 +- [x] 实现 active-key 原子幂等 enqueue、run-at、bounded row/value limits。 +- [x] 实现 bounded claim、claim-time attempt、随机 token、lease renew/expiry 与 owner CAS。 +- [x] 实现 success、retry/backoff、dead、blocked 及受控错误分类/摘要。 + +Gate：SQLite temp DB 覆盖并发去重、终结后 key 重用、到期、双 claimant、崩溃耗尽、过期重领、旧 token 拒绝和 schema/decode/target blocked；PostgreSQL 验证 feature/SQL 生成。 + +## 4. Schedule, Worker And Test Harness + +- [x] 实现共享五字段 UTC cron parser、匹配与 bounded next-slot 搜索（每轮扫描最多 256 个分钟槽，不做无界寻找）。 +- [x] 实现持久 cursor、同 transaction occurrence materialize/advance、首次启用和每轮最多 256 个补偿窗口。 +- [x] 实现 bounded worker pool、timeout、heartbeat 和静态 payload dispatch。 +- [x] 扩展 application test closed graph、case-local clock 和 bounded due-job drain，复用生产状态机。 + +Gate：多 scheduler 同 slot、completed occurrence 不重建、cursor rollback/restart、cron fingerprint 冲突和补偿上限；不同 case fake/clock/database 不泄漏且不读取部署配置。 + +## 5. Mode And Lifecycle + +- [x] 增加严格 runtime/job settings、source capability 预检和 runtime profile feature/cache identity。 +- [x] 实现 api/worker/all gating，保留同 binary 的 API/Job dispatch roots。 +- [x] 增加结构化 SIGTERM/Ctrl-C owner；API stop-accept/drain 与 Job stop-claim/drain 共享 deadline。 +- [x] generated entry 在应用汇合后统一 database shutdown、log flush，覆盖启动失败和超时取消。 + +Gate：缺失/非法配置在 listen/claim 前失败；三 mode 只启动应有服务；test-owned 进程验证信号后停止新工作、drain/超时取消和关闭顺序。 + +## 6. Documentation And Focused Verification + +- [x] 更新 LANGUAGE、MARKDOWN-SYNTAX、compiler/database/toolchain 中的 application-testing 契约。 +- [x] 检查无环境变量入口、动态 target、第二套 codec/transaction/pool、脱管 task 或生产 Adapter 测试回退。 +- [x] 完成独立 Trellis check，修复 in-scope findings。 +- [x] 主代理最终复核并更新父任务 checklist。 + +```bash +cargo check --locked --offline -p dever-core -p dever-runtime -p dever-cli +cargo check --locked --offline -p dever-runtime --features sqlite,postgres +cargo test --locked --offline -p dever-tests --test durable_jobs +cargo test --locked --offline -p dever-tests --test source_architecture +cargo test --locked --offline -p dever-tests --test error_effects +cargo test --locked --offline -p dever-tests --test orm_query +cargo test --locked --offline -p dever-tests --test sqlite_orm +cargo test --locked --offline -p dever-tests --test application_config +cargo test --locked --offline -p dever-tests --test application_testing +cargo test --locked --offline -p dever-tests --test structured_concurrency +cargo test --locked --offline -p dever-tests --test pooled_network graceful_shutdown +git diff --check +``` + +所有 Cargo 命令使用无增量、debug info 关闭和串行测试。默认不运行 workspace 全量测试、真实性能套件或持久服务。真实 PostgreSQL 仅在仓库隔离 `config/setting.json` fixture 明确存在且用户知情时运行；否则明确记录为未运行。 + +## 实施记录 + +- 源码、共享 wire/connection owner、生产 store/worker、native/test roots、mode/lifecycle 已接通；以上勾选表示实现完成，最终 Gate 证据记录在下方，不代替独立 Trellis check。 +- 启用既有 Tokio `signal` feature；经主代理明确授权解析依赖，Cargo.lock 仅新增 `errno 0.3.14` 与 `signal-hook-registry 1.4.8` 两个传递项。后续命令均 `--locked --offline`。 +- API-only 保留既有省略 runtime 配置的兼容行为（api、30000 ms）；有 worker capability 的应用必须显式配置 runtime。worker pool 比执行槽至少多一个连接，以免 handler 事务耗尽 heartbeat/claim 容量。 +- enqueue 使用单条 active-key upsert + RETURNING，避免 PostgreSQL 上冲突 insert 后、查询 Id 前任务终结的竞态。 +- 新增测试发现 case 裁剪不可达 transaction 后被误报空事务；修复共同 effects validator，真实空事务仍有 clause 并继续被拒绝。 +- 数据库服务、真实 PostgreSQL、全量测试、性能/内存压测未运行；rustfmt/clippy 未安装，不安装。 + +### 最终实施验证 + +- `cargo check --locked --offline -p dever-core -p dever-runtime -p dever-cli`：通过。 +- 同一 check 加 `--features dever-runtime/sqlite,dever-runtime/postgres,dever-runtime/api,dever-core/reference`：通过。PostgreSQL 仅 feature 编译证据，未连接真实数据库，不把它算作锁、并发或 SQL 执行验收。 +- `durable_jobs`（sqlite）：14/14。覆盖源码/Markdown、跨连接拒绝、nominal ModelId native 往返、同物理事务 commit/rollback、active-key 并发幂等、定时到期、claim-time attempt、旧 token CAS、崩溃耗尽、schema/corrupt/unknown 的 generated dispatch blocked、多 scheduler、完成窗口不重建、cursor 中途失败回滚/重启、256 窗口上限、case-local SQLite/clock/fake 与无数据库 empty drain。 +- 同一 `durable_jobs` 目标包含真实 native 二进制的 api/worker/all 三 mode：临时回环监听和临时 SQLite；api 不 claim、worker 不监听、all 同时工作，均 SIGTERM 正常退出。独立 runtime 子进程另覆盖 heartbeat、停止 claim、在途排空与 shutdown deadline 取消后关闭数据库/flush 日志。 +- `application_config`：3/3；`application_testing`：8/8；`source_architecture`：8/8。 +- `pooled_network graceful_shutdown`：1/1，验证已有 HTTP 停止 accept 并完成在途响应。 +- `time_codec wire_schemas_do_not_change_model_time_storage_identity`：1/1。该条旧断言随 nominal ModelId wire 合同更新；无 private/bounded 字段的 Model record 也可通过共享 codec，存储 snapshot 不变。该 fixture 已调整为当前 App 路径，未顺带改造其余旧测试。 +- 上述测试均串行；`CARGO_INCREMENTAL=0`，dev/test debug info 为 0，固定 stable toolchain PATH。`git diff --check` 通过；新 Job/lifecycle 源码检查未发现环境变量读取、动态 target、脱管 spawn 或 unsafe。 +- 独立 Trellis check 结果见下方；父任务 checklist 由主代理收口，任务仍保持 `in_progress`。 + +## 独立检查与修复 + +- 复现 SQLite handler 持写事务时短租约无法续租：业务已提交，但任务仍为 running。当前源码的启动预检要求 SQLite `lease_ms > timeout_ms`；PostgreSQL 继续支持短租约续租。领取在获得锁后建立 lease，续租/完成先锁 claim 行再读注入 Clock，避免使用锁等待前的旧时间。 +- worker 在调用业务前重新验证持久化记录的实际 timeout/max attempts；旧策略不满足当前 SQLite lease 或已越界时进入 `blocked/invalid_policy`，不执行 App。公共 timeout 上限仍为 3,600,000 ms，lease 上限为 7,200,000 ms；测试使用最大 lease，合法长任务不会因测试固定短租约被误拒绝。 +- 耗尽 attempts 的到期记录改为同一 claim 事务内独立有界清理；eligible 查询跳过耗尽记录，避免它占据执行槽并使 drain 误判队列为空。 +- 信号测试原来对同一 run_at 的任务假定按业务 key 执行，但实际稳定排序使用 run_at/UUID；同毫秒 UUID 带随机部分。改用不同到期时间表达本测试的执行次序，同时保留实际 heartbeat、持事务 drain 和超时取消检查。 +- 保持既有已审阅的 API-only 默认配置契约；有 worker capability 时 runtime 必填。统一 cron 文档为每轮扫描/推进最多 256 个分钟槽，只有匹配 cron 的槽生成任务。 +- 同步 LANGUAGE、design、compiler 目录 owner 图以及 database/toolchain 规范；未增加环境变量、第二套 pool/transaction/codec、动态 target 或脱管任务。 + +### 独立验证证据 + +- 最终 `cargo check --locked --offline -p dever-core -p dever-runtime -p dever-cli --features dever-runtime/sqlite,dever-runtime/postgres,dever-runtime/api,dever-core/reference`：通过。 +- 最终 `cargo test --locked --offline -p dever-tests --features sqlite --test durable_jobs -- --test-threads=1`：14/14，通过，122.86 秒。包含最大合法 timeout 的 case-local SQLite/fake、claim/renew/complete 锁等待时钟回归、exhausted 后正常任务 drain、旧持久策略 blocked、三 mode 和 SIGTERM 的 grace/timeout/持事务三场景。 +- 主代理最终复核再次运行 `sqlite_durable_state_machine`：1/1 通过；core/runtime/CLI 的 SQLite+PostgreSQL+API+reference feature check 再次通过。 +- Cargo 命令统一固定 stable PATH、`CARGO_INCREMENTAL=0`、dev/test debug info 为 0。`git diff --check` 和改动 Job 文件尾空白检查通过。 +- 一次中间 suite 为 13/14：信号 fixture 的同到期时间随机顺序断言失败；已按上述原因修复，最终完整定向 target 14/14。 +- rustfmt/clippy 二进制未安装，未安装或运行；全量测试、真实 PostgreSQL、性能/内存压测未运行。PostgreSQL 证据仅 feature 编译与代码审查，不宣称已完成真实锁/并发/SQL 执行验收。 +- 本次独立检查没有剩余已知实现阻塞；任务状态按调度要求保持 `in_progress`，交由主代理最终复核。 + +## 待复核风险 + +- 业务 commit 与 Job insert 必须共享同一 physical transaction，不能 after-commit 补写。 +- attempt 必须在 claim 时增加；finish/renew/retry 必须比较独立 claim token 和有效 lease。 +- cron materialization 必须由 cursor transaction 保证，不能复用 active-only key 或单进程内存 timer。 +- enqueue 不得引入 handler 同步 effect/递归；worker/test root 必须包含 handler 的真实 Model/Port/fake 闭包。 +- runtime mode 必须在副作用前统一预检；signal drain 不能调用立即 abort 的 Group::stop 伪装优雅退出。 +- payload/schema identity 是持久协议；未知版本必须 blocked/fail closed，不能 best-effort decode。 diff --git a/.trellis/tasks/09-20-durable-jobs/prd.md b/.trellis/tasks/09-20-durable-jobs/prd.md new file mode 100644 index 0000000..55527f4 --- /dev/null +++ b/.trellis/tasks/09-20-durable-jobs/prd.md @@ -0,0 +1,50 @@ +# Dever 持久化 Job + +## Goal + +为 Dever 提供数据库持久化、事务一致、可恢复且有界的后台任务和 UTC 定时任务，使 API 与 worker 可使用同一编译产物独立或组合部署，禁止用脱离结构化作用域的异步任务代替可靠 Job。 + +## Dependencies + +- `time-typed-codec`：DateTime、payload codec 和 schema fingerprint。 +- `port-adapter`：Job 调用的 App 可以通过 Port 执行外部副作用。 +- 复用既有 SQLite/PostgreSQL ORM transaction、pool、settings 和 shutdown。 + +## Requirements + +- 新增 Job source role；允许 `job.dever` 或平铺 `job/*.dever`，角色目录遵守现有 file/directory exclusivity 和 no-single-topic 规则。 +- Job entry 使用明确 contextual declaration，零输入或一个 App-owned wire-safe record 输入、零输出，只能调用同领域 App，不能直接访问 Domain/Model/Port/Adapter/API；Job payload 支持保留 Model identity 的 ModelId。 +- Job entry 不可被普通 source call；App 只能通过 compiler-owned enqueue/enqueue_at target reference 创建持久化任务。 +- 每个 enqueue 必须提供业务 idempotency key；同 Job identity + key 的未终结任务不重复插入。 +- Job 声明绑定一个逻辑 database connection，并声明有界 timeout/max attempts；`retry(n)` 表示包含首次执行在内最多执行 n 次，不允许零/负/越界值。 +- 立即和 run-at Job 持久化 target identity、payload、schema fingerprint、key、run_at、attempt/state、lease 和错误摘要。 +- 周期 Job 使用编译期验证的五字段 UTC cron declaration且只能绑定零输入Job；持久 cursor、occurrence insert 和推进在同一 transaction，每个计划窗口生成确定性key，多进程及已完成任务不能重复创建同一occurrence。带payload的业务计划使用enqueue_at。 +- transaction App 内 enqueue复用隐藏 transaction context；Model effect与Job connection不同则编译失败。非 transaction enqueue使用自身短事务。 +- SQLite/PostgreSQL使用runtime private versioned schema；初始化、claim、lease renew/expiry、success、retry、dead-letter均有明确事务边界。 +- worker采用至少一次执行；attempt 在 claim 时原子增加，每次 claim 使用独立 fencing token；失败按有界指数退避重试，达到上限进入 dead-letter，不无限循环或静默丢弃。 +- payload schema不兼容、未知 Job target和解码失败进入dead-letter/blocked状态并记录脱敏原因，不best-effort执行。 +- `dever.job.serve()`显式启动worker。combined main使用现有Group并发启动API/Job；不增加隐式后台线程。 +- `config/setting.json`提供严格 `runtime.mode=api|worker|all` 和 job worker/lease/poll/shutdown上限；不接受环境变量或CLI覆盖。 +- SIGTERM/Ctrl-C停止API accept和Job claim，排空在途工作到deadline，再停止、释放lease/连接并执行数据库/log shutdown。 +- 应用测试的Job数据库由runner生成隔离SQLite；不读取部署setting或自动连接PostgreSQL。 + +## Acceptance Criteria + +- [x] Job合法路径/签名可检查和格式化；ModelId payload 保持 nominal identity；普通调用、非法输出/类型、跨域或直接Model/Port访问被拒绝。 +- [x] 业务Model写入与Job入队在同一transaction一起commit/rollback；跨连接组合在编译期拒绝。 +- [x] 相同identity+key并发入队只保留一个可执行任务，不丢失已提交业务状态。 +- [x] 到期前不执行，UTC run-at到期执行；周期任务在多个worker下每个window最多创建一次。 +- [x] worker崩溃/取消会消耗 attempt，lease 到期可由另一 worker 恢复；活跃 lease 不被重复 claim，旧 fencing token 不能提交迟到结果。 +- [x] handler失败按策略重试，成功终结，达到attempt上限进入dead-letter；错误信息不含Secret/payload全文。 +- [x] schema mismatch、unknown target、corrupt payload不会执行App并保留可诊断记录。 +- [x] api/worker/all三种mode分别只启动应有服务；缺少相应配置在监听/claim前失败。 +- [x] SIGTERM/Ctrl-C定向runtime test验证停止新工作、排空或超时取消以及最终数据库/log关闭顺序。 +- [x] SQLite端到端通过；PostgreSQL feature编译通过，真实行为只在隔离 `postgres_test` 明确存在时运行。 + +## Out Of Scope + +- Exactly-once外部副作用、分布式事务和跨数据库原子提交。 +- Redis/Kafka/RabbitMQ/SQS和第三方队列。 +- 非UTC cron、DST和动态用户时区调度。 +- Job管理front；dead-letter运维界面后续由业务API/front实现。 +- 无界任务fan-out、优先级抢占和动态代码任务。 diff --git a/.trellis/tasks/09-20-durable-jobs/research/implementation-map.md b/.trellis/tasks/09-20-durable-jobs/research/implementation-map.md new file mode 100644 index 0000000..ff9df80 --- /dev/null +++ b/.trellis/tasks/09-20-durable-jobs/research/implementation-map.md @@ -0,0 +1,109 @@ +# Research: Durable Job implementation map + +- Query: 核实持久化 Job 的源码接入点、最小复用架构、SQL/生命周期风险和定向验收。 +- Scope: mixed；以当前本地源码为准，外部资料仅核对 SQL/信号基础能力。 +- Date: 2026-09-20 + +## Findings + +### 当前事实和实施前须解决的契约矛盾 + +1. 当前没有 Job source role、Job runtime、cron parser 或 SIGTERM/Ctrl-C owner。`rg signal|SIGTERM|ctrl_c|cron|Job crates library` 仅发现 `core/wire.rs` 的 `Policy::Job`。不可将设计中的这些能力当作已有实现。 +2. `wire::Policy::Job` 已存在，拒绝 Secret/private Model；现有 wire 同样拒绝 ModelId。设计示例 `PublishInput.article_id: content.article.model.id` 因而不成立。需要明确选择：扩展共享 wire owner 支持保持 nominal 身份的 ModelId，或修改示例使用当前 wire-safe DTO；不得只给 Job 私加类型白名单。考虑 Job 最常见用途正是实体异步操作，建议在共享 schema/codegen 增加 ModelId 节点，协议用字符串，fingerprint 保留所属 Model 身份，同时明确适用 policy。当前 `native/wire.rs:32` 的普通 `Id` 已映射 runtime `Id`，不可把 ModelId 在类型系统里降级成普通 Text/Id。 +3. `hir.rs:21`、`contracts/effects.rs:18` 的 database effects 是 Model ID 集合；`native.rs:436` 的 transaction 取第一个 Model 找连接。纯 Job enqueue 无 Model 时无法运行。必须推广为连接资源 effect，不能伪造 Model 或复制第二套 transaction context。 +4. 现有跨连接检查在 `model.rs:256` 的 `validate_database_settings`，依赖设置解析后连接名，而非仅 AST check。明确显式不同连接可在 check 拒绝；PackageRoot fallback 仍需要 settings-aware CLI 检查/启动校验。不要错误拒绝不同 selector 最终落在同一个 default 的合法情形。 +5. `retry(5)` 的含义必须固定为最多 5 次 attempt，还是首次之外 5 次重试。PRD 用 max attempts，建议前者并准确写明；不能只在失败后加 attempt，否则每次执行中崩溃会无限重领。 +6. 设计只规定 UTC cron，未规定字段语法、DOM/DOW 组合、停机补跑和首次部署起点。最小实现应固定五字段数字语法、范围/列表/步长和 DOM/DOW 规则；持久 checkpoint、每轮有界补齐。拒绝不能满足的日期组合或规定有界搜索失败，不能无穷找下个时间。 + +### Owner 文件与复用入口 + +| 文件/符号 | 当前职责与 Job 接入 | +| --- | --- | +| `crates/dever-core/src/source.rs:25` SourceRole/from_name/name | 当前 App/Domain/Model/Port/Adapter/Api；增加 Job，沿用 path-derived identity。 | +| `check/layout.rs:14,56` RoleFiles | 已有 role file/directory 排他、非 API 平铺、目录至少两文件；增加枚举后复用，不另写 Job scanner。 | +| `syntax.rs:169,184` FunctionClause/FunctionKind | 已有 ordinary/transaction、fails/bodyless；Job kind 和 bounded policy 作为声明元信息。schedule 用独立声明。 | +| `parser.rs:210,707,761` declaration/function/function_contracts | contextual transaction 的 dispatch 可复用；增加 job/retry/timeout/schedule；勿让 retry/timeout 成普通函数调用。 | +| `format.rs:120,258` declaration/function | 共享 AST formatter，输出 job 前缀和稳定 policy 顺序。 | +| `markdown/contract/parse.rs:198,237,261` / `validate.rs:247` | function 元信息、非 function directive 标识已有 owner；schedule 需加入 directive 分支；Markdown policy 元信息应与 source 一致。 | +| `check.rs:321,423,438` Program/test registration | 注册 Job metadata 和 target map；test case closed graph 与 fakes 已完成，必须沿用。 | +| `check/symbols.rs:222` can_call / `:277` can_access_type | API 不可调用的现有模式适用于 Job；普通 function resolution 和 handler resolution 都必须拒绝 Job。Job 仅调用同域 App，DTO 类型访问另定规则。 | +| `check/expressions.rs:521,620,655` static call/handler reference | 可复用 named-path 解析形态，但 enqueue target 不是 Handler 参数，不能转发。新增专用 checked enqueue path，参数先解 target 再按 payload schema 检查。 | +| `hir.rs:164,507,524` ExpressionKind/static_call/CallArgument | 用 JobEnqueue {target,payload,key,run_at} 明确表达入队；遍历/变更遍历要覆盖三个值表达式。enqueue 不等价同步 call。 | +| `contracts/dependencies.rs`, `contracts/effects.rs:61,505`, `contracts/error_effects.rs` | enqueue 引入本 Job 连接、database failures 和 suspension；不得传播 Job handler 的业务失败/网络 effect 到 enqueue 的 App。 | +| `specialize.rs:77,120,178` database_effects/concrete/successors | 独立执行根收集 Job handlers；不要把 enqueue->handler 当普通递归 edge，App 入队自己未来工作的循环是正常队列行为。 | +| `native.rs:292,418,434,569` DB setup/function/call | 统一隐藏 `_database: Option<&Transaction>`；普通 enqueue 继承，Job handler 执行从 None 开始。 | +| `native/orm.rs` | 现有 Executor/参数/错误转换供 Job emitter 复用；不复制数据库错误到字符串再猜类别。 | +| `native/api.rs`, `native/ports.rs` | 静态 dispatcher 和 startup initialization 可参考；Job 生成 concrete decoder + direct function invocation，不造 runtime Value 解释器。 | +| `native/build.rs:120,309,349` | suite profile、runtime features、缓存身份；Job 无 Model 仍需 database+wire+已选驱动。feature/cache 与源码变化同步。 | +| `crates/dever-runtime/src/database.rs:167,317,458` | Database/begin、Executor::new、database_for；所有 Job SQL 通过既有 Sql 双方言和 Value/Row 编解码。 | +| `sqlite.rs:174,223,279` | BEGIN IMMEDIATE、transaction 方法、未确认完成 Drop 丢弃 pool object。可直接用于 claim 短事务。 | +| `postgres.rs:236,302,383` | BEGIN、带 I/O deadline 的事务、Drop 丢弃连接；claim 使用 SKIP LOCKED。 | +| `sqlite/migration.rs`, `postgres/migration.rs:73,109` | Model 自己的 schema/history owner；Job 私有 schema 不进入它们，但复用事务/advisory lock 纪律。 | +| `config.rs:11,70,144,325,356` | 单一 Settings，严格 top-level/duplicate wire 验证，bootstrap/http lazy read。扩展 typed runtime/job config。 | +| `task.rs:78,488`, `task/scoped.rs` | 单 runtime、Scope supervisor、Group；worker/heartbeat/signal 等所有活动必须有结构化 owner。 | +| `api.rs:13`, `http/mod.rs:147,176`, `net.rs:47,232` | API 内创建 listener；HTTP 已响应 listener.close 并 graceful drain。新增 signal owner 触发此路径，不用取消 serve 代替优雅关闭。 | +| `database.rs:496,502`, `postgres.rs:462,473` | prepare/shutdown 当前只负责 PostgreSQL；SQLite 没有显式 pool shutdown 路径。若 PRD 保证最终关闭 SQLite，需补 owner。 | +| `core/wire.rs:10,55,64`, `native/wire.rs:8` | Job policy、稳定版本 schema identity、concrete codec；当前 fingerprint 是确定性描述串，不是固定长度加密 hash。持久字段可存完整 identity，不误称 SHA。 | +| `cli/test_runner.rs:73,126` / `check.rs:466` | 每 case 新进程/临时目录，runner 生成 SQLite setting；无 DB case 无 setting。加入 Job connection、harness，仍不读部署配置。 | + +### 建议最小架构和数据流 + +保留 Job body 为 Function/HIR，额外 Program.jobs 持有 identity/function/connection/payload schema/policy/schedule。compiler 专用 enqueue 表达式只保存静态 target ID 和普通参数；不将 target 暴露成值类型。Job entry 的 ordinary/handler 调用拒绝统一放符号访问 owner。 + +数据库 effect 推荐抽为统一 `DatabaseBinding`（selector + fallback root 的稳定值或 intern ID），Model 与 Job 都引用。Model 的 CRUD schema/index 仍保持 Model ID，仅 effects/transaction/startup/test 配置改用连接 binding。为最小改动也可用明确的资源枚举 Model/Job 并统一 `binding()` accessor，但不应让消费者每处重复 match。 + +源码 Job → checked metadata/payload Schema → enqueue HIR → 编码 Encoded JSON → 同一 Executor 写入 → commit → worker claim → 校验 target/schema → concrete decode → direct App 调用 → owner-CAS finish/retry。enqueue 的调用图不包含 handler 业务 effect；worker 启动根和 test harness 根要独立包含 handler/App/Port fake graph，避免漏链接及错误递归判定。 + +runtime 可由 `job.rs` 拥有 public compiler bridge，`job/store.rs` 拥有 DB 状态机和 SQL，`job/schedule.rs` 拥有 cron/slot，`job/worker.rs` 拥有有界执行；只有职责确实需要时拆这些文件，不添加通用 queue framework。compile-time cron 验证可调用纯 runtime cron parser，避免编译期/运行时双实现。chrono std-only 已存在，复用 UTC calendar；无须引入时区库。 + +### SQL/租约策略 + +- runtime private version 表 + job 表 + schedule checkpoint 表。SQLite BEGIN IMMEDIATE，PG 独立 runtime namespace advisory transaction lock；version 不认识时明确失败，不能 CREATE IF NOT EXISTS 后假装兼容。 +- enqueue 的 active-only 唯一索引覆盖 `(target,dedupe_key)` 且 predicate 只含待执行/执行中状态；参数化 INSERT ON CONFLICT DO NOTHING，再返回已有 active ID。不能用 SELECT-before-INSERT 作为唯一保障。业务失败 rollback 必须撤销同物理事务的入队。 +- PostgreSQL claim：短事务选取到期 pending 或过期 running，`FOR UPDATE SKIP LOCKED LIMIT ...`，写 owner、lease_until、attempt 后提交。SQLite 同过程已有 BEGIN IMMEDIATE 序列化。只 claim 有空闲执行容量的条目，不能先批量占 lease 再在无界内存队列等候。 +- attempts 建议在成功 claim 时递增；超时/失败/崩溃恢复都耗费次数。过期且 attempts 达上限的条目需进入 dead-letter，不能永远排除于查询而残留 running。 +- owner token 每次 claim 新生成，不能仅以进程 worker ID 作为 fence；旧执行在同 worker 中过期重领后仍可能迟到。finish/renew/retry 带 id + token + state，必要时判断 lease_until；影响行数为 0 即丢失租约，不可继续确认成功。复用现有 OS randomness/Uuid owner，不硬编码 ID。 +- renewal 不与 handler 持有同一 transaction；需预算 worker 数与 pool 容量，避免单连接长事务使 heartbeat 饿死。取消后复用当前丢弃未知连接的规则。若 handler 无法强制立即停止，最多只能保证至少一次，外部副作用依赖业务 idempotency。 +- PG runtime Int bind 是 i64，SQL 数值列和参数使用 BIGINT 语境；时间全部 epoch ms，勿调用 SQL now 导致测试时钟不一致。错误摘要写固定错误类别/有界安全信息，不保存 AppError Display（业务 message 可含 payload）。 +- cron occurrence 幂等不能只用 active-only job 索引：已经成功的 occurrence 可能被另一 scheduler 再建。锁定 checkpoint 行，插入 jobs 与推进 last slot 同事务；或者永久唯一 occurrence ledger。checkpoint 必须按规范化 cron identity，计划变更不能回放全部历史。每轮扫描/补跑有界。 +- key/payload/target/schema/error 等都有确定上限；wire 16 MiB 默认上限已经有界，但 worker 并发乘以该值仍需评估，不宣称低内存性能。 + +### Mode、启动与关闭 + +当前 RuntimeProfile 只有 sqlite/postgres；Tokio RuntimeConfig 只有线程/任务容量，均没有 api/worker/all。严格 runtime/job 文档必须进入 Settings 的真实解析，不能仅在 serve 里临时读 JSON。主程序调用 serve 的能力图用于校验 mode：worker-only 不要求 HTTP；API-only 不 claim；all 按 main 的 Group 并行拥有两个 server。启动前校验对应模式的缺失配置及所有启用 Job timeout/lease 关系，不能 API 已监听后才发现 Job 不可用。 + +API runtime::serve 需在创建 listener 后把关闭与 drain 接到同一 shutdown token。signal watcher 是 generated entry/运行时生命周期拥有的 future，不可 detached spawn；SIGINT/SIGTERM 一次广播，停止 accept/claim/schedule 后等待在途工作到统一 deadline。deadline 到达使用现有 Task/Group cancel-and-drain，再执行 database shutdown 和 log flush。`Group::stop` 当前直接 abort body，正常收到 signal 时先合作排空，超时才 stop。 + +Tokio 1.53.1 当前 features 不含 signal（根 Cargo.toml:20），Cargo.lock 未找到 signal-hook。需启用 Tokio signal feature 并确认 offline 所需依赖已缓存；不要用 unsafe libc handler（workspace 禁 unsafe）或 shell signal 轮询做替代。PG driver shutdown 当前存在；SQLite pool explicit close 和 startup failure 清理须单独覆盖。generated entry 当前仅 profile.postgres 时调用 shutdown，需按完成的生命周期设计调整。 + +### Application test 隔离 + +保持每 case 单独 native module、closed fake graph、进程和临时目录。test metadata 收集 enqueue 与 harness 执行的 Job 连接以及 handler 访问的 Model；不能只看 test 主函数的直接 Model effects。每 case SQLite default/explicit connections 由 runner 生成。Job-only case 也生成 setting；非数据库 case 仍无 setting。 + +设计还没有给 compiler-owned test harness 的源码调用协议，不能只实现 Rust store 单元测试便声称应用测试可用。建议最小 test-only drain-due/advance-clock 操作，返回有界观察结果或保证已处理 due work，测试通过 App 查业务状态；不开放 DB handle 或普通直接 Job entry 调用。假时钟作为 case-owned Clock 注入 store/scheduler，不改全局系统时间、不真实 sleep。 + +### 定向验收建议 + +1. 新 `durable_jobs` 源码/formatter/Markdown 定向 target：合法声明、错误 role/直接 call/handler reference、0/1 record 参数、Secret/private/非法 payload、policy 上下界、cron 非法、纯 Job transaction 与跨连接。 +2. runtime SQLite temp DB：业务写入+入队 commit/rollback；并发 active key 去重；终结后 key 重用；run_at；多 claimant；租约过期、旧 token finish/renew 拒绝、崩溃耗尽 attempts；有界退避/溢出；损坏/schema/unknown target 不执行。 +3. cron 两个独立 scheduler 同 slot、已完成 job 后再次调度、checkpoint rollback/restart、停机补跑上限、闰年/月底/DOM-DOW 规则。 +4. `application_testing` 精确 case：同二进制不同 case fake 完全隔离；Job-only SQLite setting；部署 PostgreSQL 配置未读取；受控时钟/harness 触达 App。 +5. config 三 mode 缺失/未知/重复字段、timeout/lease/poll/shutdown 约束。测试专有子进程 signal，明确 deadline；仅所属随机回环 listener（若批准）验证停止新请求和 HTTP drain；不向已有服务发送信号。 +6. `structured_concurrency`/`pooled_network graceful_shutdown` 受影响 case；SQLite one-connection cancellation 后可恢复；PG 仅 feature cargo check 与 SQL 生成断言，真实 DB 另行授权条件。 + +## External references + +- PostgreSQL SELECT locking 官方说明：`SKIP LOCKED` 适合多消费者 queue 场景，但仅跳过 row lock，仍有表级锁：https://www.postgresql.org/docs/10/sql-select.html 。这是 SQL 策略依据，不是对本仓库 PostgreSQL 实测。 +- SQLite 官方 partial index 文档：https://www2.sqlite.org/partialindex.html ，UNIQUE 可只约束满足 predicate 的行。 +- Tokio signal 官方 Rustdoc：https://docs.rs/tokio/latest/tokio/signal/unix/fn.signal.html ，Unix signal API 要求 signal feature。仓库实际 pin 1.53.1，应以本地锁定版编译验证依赖。 +- 本地版本：Tokio 1.53.1、chrono 0.4.45 std-only；SQLite/Postgres pool 沿现有锁定依赖，不建议增加队列或 cron 框架。 + +## Related specs + +已查 `.trellis/workflow.md`、任务 prd/design/implement/task.json，以及 backend `index.md`、`directory-structure.md`、`error-handling.md`、`quality-guidelines.md`、`compiler-contracts.md`、`database-guidelines.md`、`toolchain-and-library.md`、`logging-guidelines.md`。Job 新约束尤其会改变 compiler-contracts 的 Model-only effect 描述及 test runner 数据库触发条件。 + +## Caveats / Not Found + +- 子 agent `task.py current --source` 返回 none；父 dispatch 明确提供本任务绝对路径，本研究仅写此 research/，不改 active state。 +- 未改源码、spec、任务状态；未运行 build、测试、服务、外部数据库或性能测量。 +- 当前研究证明复用边界和缺失点，不证明 Job 可执行。policy、cron、payload ModelId 和 test harness 上述未落实契约必须在实施/验收中明确，不能悄悄删需求。 diff --git a/.trellis/tasks/09-20-durable-jobs/task.json b/.trellis/tasks/09-20-durable-jobs/task.json new file mode 100644 index 0000000..7627c19 --- /dev/null +++ b/.trellis/tasks/09-20-durable-jobs/task.json @@ -0,0 +1,26 @@ +{ + "id": "durable-jobs", + "name": "durable-jobs", + "title": "Dever 持久化 Job", + "description": "实现事务内原子入队、延迟和周期任务、重试租约与 API/worker/all 运行模式", + "status": "completed", + "dev_type": null, + "scope": null, + "package": null, + "priority": "P1", + "creator": "Antigravity", + "assignee": "Antigravity", + "createdAt": "2026-09-20", + "completedAt": "2026-09-20", + "branch": "main", + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [], + "parent": "09-20-production-application-language", + "relatedFiles": [], + "notes": "", + "meta": {} +} diff --git a/.trellis/tasks/09-20-port-adapter/check.jsonl b/.trellis/tasks/09-20-port-adapter/check.jsonl new file mode 100644 index 0000000..4d4476c --- /dev/null +++ b/.trellis/tasks/09-20-port-adapter/check.jsonl @@ -0,0 +1,3 @@ +{"file":".trellis/spec/backend/quality-guidelines.md","reason":"Focused architecture/effect/config tests"} +{"file":".trellis/spec/backend/directory-structure.md","reason":"Single symbol/dispatch owner and legal role layout"} +{"file":".trellis/spec/backend/error-handling.md","reason":"Port failure subset and diagnostics"} diff --git a/.trellis/tasks/09-20-port-adapter/design.md b/.trellis/tasks/09-20-port-adapter/design.md new file mode 100644 index 0000000..b6a446a --- /dev/null +++ b/.trellis/tasks/09-20-port-adapter/design.md @@ -0,0 +1,47 @@ +# Dever Port 与 Adapter 设计 + +## Source Contract + +Port role按路径给出边界身份。根 `port.dever` 的 identity 为 `<component>.<domain>`；topic Port 增加 topic。bodyless contract 示例： + +```dever +send(message: Text) () fails app.DeliveryError +``` + +`fails` 必须引用同领域 App 的公开 error choice（例：App 定义 `DeliveryError`），或公开标准 error choice；至少有一个显式 error variant。Port 自有类型仍私有。这样保留 App 全部函数公开的规则，App 能传播自己的错误，无需增加隐式私有 helper。Port/Adapter 只能引用 Port 签名使用的 App 类型，不能调用 App。普通有 body function 仍沿调用图推断 failure，不新增通用 throws 声明。 + +Adapter/Test 通过 qualified definition 实现，例如 `port.send(...) () { ... }` 或 `port.mail.send(...) () { ... }`。普通单名 function 是该文件 helper。一个生产 Adapter 文件完整实现一个 Port identity；禁止跨 Port 聚合。一个 TestCase 可以完整 fake 多个 Port。 + +## Registration And Dispatch + +Checker 在所有签名完成后建立 Port contract/implementation table。实现身份来自 adapter role topic；根 `adapter.dever` 使用 `default`。逻辑 Port identity 不包含 Adapter 文件路径。每个可达 Port 必须至少有一个生产实现；不可达声明按既有 unused contract 报 warning。 + +单实现 native call 直接静态调用。多实现生成固定选择和 concrete match，分支调用具体 specialization。effect analysis 在 HIR 中看到生产实现保守并集；failure 摘要固定为声明全集，实现实际失败必须是子集。运行时选择不改变类型正确性。 + +## Settings + +Adapter 文件可声明一个 contextual `setting` record，并以只读 `setting.<field>` 访问。编译器把它纳入共享 setting wire schema。建议 JSON： + +```json +{ + "adapter": { + "notification.mail": { + "use": "smtp", + "setting": { + "endpoint": "https://mail.example.test", + "token": "secret-value" + } + } + } +} +``` + +`use` 在多实现时必需，单实现可省略但若提供必须匹配。端点/证书/凭据属于 Adapter setting；重试业务规则和权限不放配置。错误只报告 Port/Adapter/字段位置，不回显 Secret 值。 + +## Test Fakes + +Test qualified implementation使用相同 signature/failure checker，但 owner 是 TestCase。Suite emission 为每个 case 生成独立 closed binding；case selection 同时选择 test entry 和 fake table。没有 fake 的可达 Port 在测试检查阶段失败，不退回生产 Adapter，确保测试不会意外访问外部系统。 + +## Role Matrix Migration + +`check/symbols.rs` 继续作为唯一 call/type access owner。新增明确矩阵而不是负向条件组合。Model operation owner 检查保持独立但与 role matrix 一致。现有 CMS 尚未使用 Adapter，迁移只需修复拒绝 fixtures 和文档，不保留 App-to-Adapter compatibility。 diff --git a/.trellis/tasks/09-20-port-adapter/implement.jsonl b/.trellis/tasks/09-20-port-adapter/implement.jsonl new file mode 100644 index 0000000..08364c0 --- /dev/null +++ b/.trellis/tasks/09-20-port-adapter/implement.jsonl @@ -0,0 +1,3 @@ +{"file":".trellis/spec/backend/compiler-contracts.md","reason":"Source role, visibility, failure and effect contracts"} +{"file":".trellis/spec/backend/toolchain-and-library.md","reason":"Native dispatch, setting and official package boundary"} +{"file":".trellis/spec/backend/database-guidelines.md","reason":"Transaction effect propagation and settings source"} diff --git a/.trellis/tasks/09-20-port-adapter/implement.md b/.trellis/tasks/09-20-port-adapter/implement.md new file mode 100644 index 0000000..9593e9b --- /dev/null +++ b/.trellis/tasks/09-20-port-adapter/implement.md @@ -0,0 +1,68 @@ +# Dever Port 与 Adapter实施计划 + +## Start Gate + +- [x] secure-data-contracts 和 time-typed-codec 已完成。 +- [x] 用户批准父任务规划。 +- [x] 读取 source layout、parser declarations、symbol access、effects、application testing 和 config owners。 + +## Implementation + +- [x] 增加 Port bodyless contract、fails modifier、AST/formatter/Markdown contract。 +- [x] 注册 Port types/contracts 和 Adapter/Test qualified implementations。 +- [x] 集中实现 role call/type matrix，删除宽泛 private-role互调分支。 +- [x] 检查 implementation signature、outputs、failures、effects 和覆盖完整性。 +- [x] 生成单实现直接调用和多实现 closed dispatch。 +- [x] 增加 Adapter setting declaration、共享 codec 接入和严格 setting.json binding。 +- [x] 把 Port effect 纳入 specialization/transaction/concurrency 检查。 +- [x] 扩展 TestCase fake binding 和 suite case隔离。 +- [x] 更新 LANGUAGE、Markdown、compiler/toolchain/application-testing specs。 + +## Implementation evidence (2026-09-20) + +- 保留 Port 的 Function ID，附加生产实现 successors；shared dependencies/effects/specialization 纳入这些边，failure 直接 seed 声明全集。新 `check/ports.rs` 拥有声明、完整实现、setting schema 和 fake 绑定验证；`native/ports.rs` 生成直接调用/closed match 与不可变 setting/binding。 +- 合同修正：App 只有公开函数且语言用函数分句匹配，故“Port 私有失败再由 App 私有 helper 映射”不可表达。采用主代理确定的 `fails app.ErrorChoice`；App/Port/Adapter 共享已公开错误合同，禁止反向业务调用，不放宽私有类型导出。 +- 每个测试进程选定 case fake；check 与套件生成共用逐 case 的闭合图，Port successors 只替换为该 case 的 fake，生产 Adapter/配置不可达。每个 case 独立原生模块与同步/挂起入口，仍只编译一次套件。一个 case 可完整 fake 多个 Port；缺 fake 编译拒绝。 +- 新反例发现 Port 无 HIR body 导致原有挂起阻塞检查漏过合成 dispatch；已在同一 shared effects owner 校验 Port successors，拒绝挂起 Port 内联选择阻塞实现。 +- 当前已通过：`port_adapter` 10/10（单实现无配置、多实现同二进制切换、Secret setting、错误配置/重复字段、Markdown/fails、角色与 effects、逐 case fake 隔离）；`source_architecture` 8/8；`source_visibility` 10/10；`application_config` 3/3；`markdown_source` 20/20。 +- 最终 `core/runtime/CLI` feature check（reference + api + postgres）通过；`application_testing` 8/8；`error_effects::result_capture_requires_matching_success_and_error_variants` 1/1；`git diff --check` 通过。 +- `structured_concurrency` 3/11 通过、8 个旧 fixture 失败：仍把 time/task.sleep 的 Unit 当 SleepResult、chunks 当 ReadStreamResult、直接构造 error variant，以及 Markdown H1 放代码。失败先发生在这些旧协议校验，未扩到无关 fixture 迁移；新 Port 挂起/阻塞定向反例已通过。 +- 未运行全量、外部服务、真实数据库或性能测试；rustfmt/clippy 组件未安装。Port 当前明确拒绝 handler 参数和输入/输出范围 bounds；配置选择/setting 仅原生后端执行，reference 对单个无 setting 实现可执行，其余给显式不支持错误。 +- 已完成清理自审：声明注册与实现注册分开；role I/O 验证收敛到 shared effects；配置可执行目录定位复用同一函数；Port/setting 合成函数不参与重复实现建议。任务状态留给主会话复核后推进，未提交或归档。 + +## Independent review (2026-09-20) + +已修复四项实际问题： + +- 测试 fake 曾在 native 阶段跨 case 合并；一个阻塞、一个挂起 fake 会触发错误的 Port 阻塞诊断。生产合同推断也曾使用 Test body。现在生产推断排除 Test body，check/native 共用 `ports::test_program` 的 case 闭合图；共享 dependency owner 保留未实例化 helper 的声明检查，逐 case 校验所有 fake operation 的 failure 子集。原生套件按 case 生成独立模块与同步/异步入口，仍一次 rustc、一个二进制。混合 fake 回归先失败、修复后通过；同 case 阻塞违规、未调用 fake 的越界 failure、未实例化 pure helper 的副作用均被拒绝。 +- `setting()` 普通函数曾被 parser 误判为 setting record；现在只在 `setting {` 时识别 contextual 声明。 +- 显式 `adapter: null` 曾被 serde Option 吞成缺省，单实现可意外启动；保留原始 null 交给 shared wire validator 拒绝，单实现无文件仍可运行。 +- Port-local DTO 内嵌 App DTO 曾在合同图展开之前触发 C006。现在在 shape access check 前按既有名字解析规则展开合同类型图；合法嵌套 DTO 通过，无关 App 类型仍被拒绝。正例已先复现失败再通过。 + +最终验证：`port_adapter` **10/10**、`application_testing` **8/8**、`source_visibility` **10/10**；core/runtime/CLI `cargo check`（reference + api + postgres）通过。命令使用 stable PATH、CARGO_INCREMENTAL=0、locked/offline、dev/test debug=0、serial tests。`git diff --check` 与本轮修改文件逐文件 `git diff --no-index --check /dev/null` 均无空白错误（后者有文件差异时退出 1，不代表空白错误）。LANGUAGE 与 compiler-contracts 已同步。 + +rustfmt/clippy 未安装，未运行且未安装组件。未运行全量测试、服务、真实数据库或性能测试。此前 structured_concurrency 的旧 fixture 失败记录仍适用，不在本轮扩改。无新增未修复的 in-scope 阻塞；未提交、未归档、未更改任务状态。 + +## Focused Verification + +```bash +cargo check --offline -p dever-core -p dever-runtime -p dever-cli +cargo test --offline -p dever-tests --test source_architecture +cargo test --offline -p dever-tests --test source_visibility +cargo test --offline -p dever-tests --test error_effects +cargo test --offline -p dever-tests --test structured_concurrency +cargo test --offline -p dever-tests --test application_config +cargo test --offline -p dever-tests --test application_testing +cargo test --offline -p dever-tests --test markdown_source +git diff --check +``` + +不运行外部网络、全量测试、真实数据库或性能基准。Adapter runtime 正例使用 test-owned本地确定性实现，不连接服务。 + +## Review Risks + +- Port implementation不能被普通 name lookup 当成可调用函数。 +- 多实现 effect/failure union不能因当前 setting 选择而缩窄静态合同。 +- setting schema identity必须包含实现选择，且错误不得回显 Secret。 +- 单 suite 的 fake registry不能跨 TestCase共享可变状态或生产绑定。 +- role matrix应由一个表/函数拥有，避免 call与type access规则漂移。 diff --git a/.trellis/tasks/09-20-port-adapter/prd.md b/.trellis/tasks/09-20-port-adapter/prd.md new file mode 100644 index 0000000..f905881 --- /dev/null +++ b/.trellis/tasks/09-20-port-adapter/prd.md @@ -0,0 +1,42 @@ +# Dever Port 与 Adapter + +## Goal + +把外部 HTTP、邮件、对象存储等依赖变成编译期验证、运行时可配置替换的 Port/Adapter 边界，同时收紧应用角色依赖，禁止 App 绕过 Port 和 Domain/Adapter 反向依赖业务层。 + +## Dependencies + +- `secure-data-contracts`：Secret setting 和不可观察边界。 +- `time-typed-codec`：类型化 setting 解码和共享 wire schema。 + +## Requirements + +- `port.dever`/`port/*.dever` 只允许类型和 bodyless function contract；每个 contract 显式 `fails` 同领域 App 公开 error choice 或公开标准 error choice，至少一个显式 error variant。Port 自有 DTO 仍私有。 +- Port contract 输入、命名输出、failure contract 和逻辑身份稳定，不从任一 Adapter 实现反推。 +- Adapter 通过 qualified Port target 显式实现；签名必须相同，实际 failure set 必须是声明集合的子集。 +- Adapter 可包含同文件私有类型/helper，但不能调用 App、Domain、Model、其他 Adapter 或 Port；只能调用标准库和自身 helper。 +- App 只能调用同领域 Port；Domain 只能调用同领域 Domain；Model 无业务函数；API/Job 只调用同领域 App。 +- Port 调用的 suspend/effect 是所有可选实现的保守并集，仍由现有静态 effect 系统决定 worker/await 边界。 +- 单实现自动绑定；多个实现必须由 `config/setting.json` 的稳定 Port identity 选择，未配置、未知实现或未知 setting 字段启动失败。 +- Adapter 可声明一个 typed setting record；只有被选择实现的 setting 必须存在并解码，Secret 字段不进入日志/错误。 +- 编译产物包含闭合实现集合和直接 match，不使用动态库、反射、trait-object business dispatch 或运行时源码扫描。 +- Test 可在同一个测试文件完整 fake 多个 Port；fake 只绑定当前 TestCase，测试不读取项目 Adapter setting 或回退到生产实现。 +- Port/Adapter 的 `.dever.md` 文档合同与普通源码一致。 + +## Acceptance Criteria + +- [x] 合法 bodyless Port 和单 Adapter 可被 App 调用并生成直接静态实现调用。 +- [x] Port body、Adapter 缺失/重复实现、签名/命名输出/failure 不匹配在 check 阶段拒绝。 +- [x] App 直调 Adapter、Domain 访问 Model/Port/Adapter、Adapter 反调 App/Domain/Model 和跨领域 Port 调用全部拒绝。 +- [x] 多 Adapter 通过 `setting.json` 在不重新编译的情况下选择；未知选择、缺配置、未知字段和错误类型启动失败。 +- [x] 未选择 Adapter 的 setting 不要求存在，其代码仍通过静态检查并包含在闭合产物中。 +- [x] Secret setting 可以用于批准 sink，但不能出现在日志、错误、API/Job output 或普通字符串中。 +- [x] 两个应用测试可以对同一 Port 使用不同 fake，单套件编译后各自运行且互不泄漏。 +- [x] Port effect/failure 通过 App、transaction、handler specialization 正确传播，不出现动态 fallback。 + +## Out Of Scope + +- 运行时安装插件、动态库、服务发现和远端代码加载。 +- 跨领域直接共享 Port；领域间继续通过 App 合同协作。 +- 通用依赖注入容器、Service locator 或应用可变全局 registry。 +- Adapter 热切换；setting 在进程启动时验证并固定。 diff --git a/.trellis/tasks/09-20-port-adapter/research/implementation-map.md b/.trellis/tasks/09-20-port-adapter/research/implementation-map.md new file mode 100644 index 0000000..51e981b --- /dev/null +++ b/.trellis/tasks/09-20-port-adapter/research/implementation-map.md @@ -0,0 +1,138 @@ +# Research: Port / Adapter 最小落地路径 + +- Query: 按现有编译器、静态合同、共享 wire、应用测试机制实现 bodyless Port、闭合 Adapter 选择和 case-local fake。 +- Scope: internal +- Date: 2026-09-20 +- Task: `/data/project/dever/.trellis/tasks/09-20-port-adapter`，来自父代理显式分派；本代理 `task.py current --source` 返回 none，未修改 task 状态。 + +## Findings + +### 1. 语法与声明注册 + +现状：`syntax.rs:168` FunctionClause 的 name 是单个 Name，body 为 Vec；`FunctionKind` 只有 Ordinary/Transaction。`parser.rs:685` 强制单名和 `{}`，不存在可复用 bodyless 声明；`function_contracts:718` 已将 pure/recover 作为 contextual Name 处理，可复用 `at_name`，无需新增全局 token 关键字。Parser 已拥有 source layout，但结构只保留 model/application，可增加 role 或必要布尔上下文。 + +建议最小 AST：保留普通 name，不将所有函数名改 Path；加 `implementation: Option<Path>`、`failure: Option<TypeRef>` 与明确 bodyless 标记（FunctionKind::Port 或独立 declaration kind 均可，不能用空 Vec 判断，因为普通空函数合法）。qualified definition 的末段仍为 name，前缀保存 Port target。仅 Port role 接受 bodyless/fails；仅 Adapter/Test 接受以 port 开始的 qualified definition。普通函数继续当前推断合同，遇到 fails 直接拒绝。Port 仅一个无模式、无 handler 参数的签名起步，拒绝 transaction/recover、重载歧义和多 clause；若要支持 handler 合同，必须额外定义实现方 failure/effect 参数依赖，不能默默沿用未闭合语义。 + +推荐语法（其中 `setting` 是新 contextual record 声明，不是 `type setting`）： + +```dever +// module/notification/mail/port.dever +type DeliveryError { + error Unavailable(message: Text) + error Rejected(message: Text) +} +send(message: Text) () fails DeliveryError + +// module/notification/mail/adapter/smtp.dever +setting { + endpoint: Text + token: Secret +} +port.send(message: Text) () { + deliver(message, setting.endpoint, setting.token) +} +// deliver 为同文件 helper；真实外部失败需捕获并映射到 port.DeliveryError + +// test/notification/mail/delivery.dever +port.send(message: Text) () { + assert_eq(message, "hello") +} +delivery() () { + app.deliver("hello") +} +``` + +注意：目前 role 目录至少两文件，不能以只含 smtp.dever 的 adapter/ 作为合法单实现例子。单实现使用 adapter.dever（identity default），多个实现再用 adapter/smtp.dever 与 adapter/local.dever。 + +`check.rs:97-177` 按 qualified name + arity 合并 clauses；qualified 实现必须有自己的物理内部函数 ID，另存 target contract ID，不能注册为原 Port 名或让两个 Adapter 合并成 clauses。`check.rs:236` 先完成全部签名，适合在此后建立 contract/implementation table；bodyless 跳过 clauses coverage/body 输出赋值检查。`check/names.rs:60` 对 qualified 各段独立 snake 检查。`check.rs:485` 现有 Port 一律拒绝应替换成正向声明校验。 + +### 2. 路径、调用与类型权限 + +路径已有足够信息：`source.rs:25-105` SourceRole 含 Port/Adapter，Role 保存 component/domain/topics，Test 保存 component/domain/topic。`check/layout.rs` 负责 role 文件/目录互斥、flat topics、至少两文件；不需新目录体系。 + +`check/symbols.rs:206` can_call 当前：App 能调全部同域 private role；Domain/Model/Port/Adapter 可相互调同域非 App；Test 只能同域 Domain / 任意 App / 自身 Test；API 完全不可作为目标。需要集中 role policy，至少区分 Call 与 Type，不能用同一布尔强行覆盖所有访问。 + +建议调用矩阵：Main→App；API→本域 App；App→任意 App、本域 Domain/Port；Domain→本域 Domain；Adapter→同文件普通 helper；Test→同文件 helper、本域 Domain、任意 App。Port contract 无 body。生产 qualified implementation 永远不能被 source lookup/handler capture 直接引用。Adapter 不能调其他 Adapter 的 helper，即使同域也不行。Model 操作仍由 `check/orm.rs:247-258` 独立限定本域 App，不能只靠 function call 矩阵，因为 ORM bypass 普通函数 lookup。 + +`can_access_type:256` 目前先放行 public Model type、自己文件，然后同域 Domain/Port/Adapter 类型互通；因此“Domain 不访问 Model”“Adapter 不反向依赖业务层”必须同时收紧类型 owner，而不是只收紧调用。建议 Adapter 可引用自身类型、本域所实现 Port 的合同类型、public 标准类型；Port 可引用本 Port 类型/public 标准类型，避免通过 App/Domain 造成反向依赖；Domain 可引用本域 Domain 与允许的 App DTO（若保留现有 business error 约定），但不可访问 Model/Port/Adapter。App 可引用本域 Domain/Port、公开 App/Model，不能引用 Adapter setting/helper type。Test 为 fake 新增本域 Port 类型访问，但不新增直接 Port 调用权限。类型矩阵是否允许 Domain 引用 App DTO 与“Domain 只调用 Domain”并不冲突，需明确不要顺手删除现有 App 错误类型引用约定。 + +重要合同矛盾：`check.rs:559` 拒绝 App 传播 private error 类型。Port 内定义的 DeliveryError 是 private，不能直接从 App 泄漏。最小解法是 App 用既有 result capture + error-only choice wrapper 捕获，再映射 App 自己的公开 error；不要将 Port 类型整体 public 或放开跨域 Port。Port 返回私有 DTO 同理在 App 映射 View。 + +另一个绕过：can_call 在 `target.bundled` 提前放行所有标准公开函数。单纯角色矩阵不能阻止 App/Domain 直接调用 HTTP/socket/process/file，也不能阻止 Adapter 调用 dever.api.serve。若目标包括“不绕过 Port 外部 I/O”，必须添加 capability 校验并覆盖 direct system intrinsics、别名、handler 与间接调用。现有 effects 是 time/network/blocking 等粗粒度，不能只禁止全部 network（API serve 需要）；应按具体允许的边界/effect owner 定义规则，不从模块名 substring 猜。需要父代理固定本期准确范围。 + +### 3. HIR / failure / effect / specialization + +`hir.rs:11` Program 存摘要向量，`Function:71` 存普通函数签名与 clauses，`CallTarget:513` 仅 Function/Handler。最低侵入路径：Port 仍拥有一个 Function ID，新增 Program.port_contracts / adapter_implementations；调用保留 CallTarget::Function(port_function)，native 对该函数发出 compiler-owned dispatcher。这样签名解析、result、run/blocking、handler reference 继续用一个 ID，不必对所有 CallTarget match 添加 Port 分支。但必须让 contract Function 明确无 body，并在每个分析 owner 注入 Port 摘要和实现边，不能仅生成 native wrapper。 + +推荐数据：PortContract { function, identity, allowed_failures: BTreeSet<Failure>, implementations: Vec<ImplementationId> }；AdapterImplementation { owner, name, port, operations: BTreeMap<contract_function, implementation_function>, setting_type: Option<TypeId> }；TestCase.fake_bindings: BTreeMap<port_id, implementation_id>。identity 根为 component.domain、topic 为 component.domain.topic；实现 name 为 adapter topic 或 default，不使用 package 路径、源码 ID、排序下标作为 JSON 稳定身份。不同 operation 同名/同 arity 用完整 Port identity 区分。 + +`contracts/dependencies.rs:11` 是共享依赖 owner：contract→所有生产实现；effects/failure worklist reverse callers 必须看到这些边。`specialize.rs:174` Frame::new 当前只遍历 expression static calls，应扩展 synthetic Port successors；`concrete()` 会枚举所有普通函数，也要避免把 fake 当全局生产实现。native `expressions.rs:310` 统一 call_instance/invoke 可复用；dispatcher 每个分支直接调用具体 specialization，并按分支实际 suspension await。所有候选实现必须参与 reachable/codegen，不能按 build-time setting 剪枝，否则无法不重编译切换且 runtime features 错漏。 + +failure 不是简单 union：`contracts/error_effects.rs:42` 每轮从空摘要推断；Port 函数需每轮 seed **声明允许集合**，而非实现实际 union。否则一个实现暂未抛出的声明错误会消失，`result(call)` 精确错误集合验证随实现变化，违反合同稳定性。允许集从指定 Shape::Choice 中 `variant.error == true` 变体映射 `Failure {ty,variant}`；不得按名字或消息匹配。实现实际 program.failures 是允许集子集，且必须无未约束 handler error 集。实现不写 fails；实现 errors 超集在 contracts 完成后报错。凡 choice 非 error variant 不属于允许集；建议要求至少一个 error variant。是否允许 error-free Port 需明确无失败语法，当前 PRD “每个显式 choice”没有表达空集合的办法，不能假造 error。 + +`contracts/effects.rs:228` Effects 摘要要 union 所有候选的 known/recoveries/databases/blocking/suspends/handler dependencies；Port 无 body时目前会推成空。`contracts/failures.rs:571` 为值内 failure obligation，不等于 error_effects 的传播错误，不能混淆。`contracts/facts` / clause bounds 如果 Port 带输出 bounds，需要使用已声明输出保证且严格校验实现，不能因空 body绕过保证。最低范围可拒绝 Port bounds，文档明确，而非静默忽略。 + +### 4. config 与 Secret setting + +`runtime/config.rs:72` Document deny_unknown_fields，目前仅 app/http/database/log；app/log 被丢弃，http 延迟解析。加入 adapter 顶层必须修改此唯一 owner。`Settings::load:138` 使用 serde_json::from_str，map 会丢失重复 key；不要先 from_value 再 wire decode，因为重复 key 已不可恢复。建议 adapter 部分保留 raw JSON 子树或用既有 runtime wire Node 从原文本定位，再交共享 concrete decoder；对 Port key、use、setting 重复/未知字段都严格检查。以 bounded wire parse 先验证整个文档也可，但要确认既有配置大小/深度合同变化。 + +`wire.rs:7` Policy::SettingInput 已允许 Secret；`Schema::build/from_field` 是 eligibility owner，支持 visible record，不允许 bounded/private fields、Choice/Map/ModelId/Related。`native/wire.rs:7` emit 已产生 `<prefix>_decode(text)` 返回具体 T{id}，SettingInput 不产生 encoder；Secret 使用 from_input(text bytes)，无 need 自制 JSON 转换。register setting 为 owner-private 合成 record（如物理 Adapter 下 Settings），字段本身不可 private，schema fingerprint 包含 policy/nominal identity。setting 字段读取应新 HIR SettingField 或 Setting record expression；禁止赋值、局部 shadow 绕过、从 Adapter 输出 Secret。无需暴露 generic runtime Value 给业务。 + +JSON 建议仍使用设计稿 `{ "adapter": { "notification.mail": { "use":"smtp", "setting":{...} } } }`：它表达一个 Port identity 选一个实现，该实现完整实现此 Port 的所有 operations。一份 Adapter 实现多个 Port 时，setting 生命周期/多份配置 ownership 尚不一致，MVP 应禁止并以明确诊断说明，或先规定每 Port 独立配置，不能暗中跨 Port 复用某一个 setting。 + +单实现无 setting 时可完全省略 adapter 条目；单实现有 setting 时条目/setting 必需，use 可省略；多实现必需 use；若配置 use 必须匹配。未选实现 setting 不解码，但所有候选源码/Schema 要检查。未知 Port identity 需按编译闭合表拒绝。选中实现无 setting 时若提供非空 setting 应拒绝（建议完全不允许 setting key，规则更清晰）。错误上下文只报 Port/Adapter/字段路径，不回显原值。 + +`config::bootstrap:272` 当前将 settings/数据库初始化与 OnceLock 绑定；`native.rs:96` 仅有 profile 才调用 database setup。Port-only 应用也要初始化选择，不能搭便车要求 database 或 HTTP。建议从数据库 setup 抽取共享 settings 初始化/读访问，再生成 Adapter 初始化，保持只读固定绑定；按全部 required bindings 校验完再执行用户 main 或任何外部 I/O。generated immutable OnceLock<closed enum with concrete settings> 符合“一次启动固定”且不是可变 service registry。Secret 不可 Debug/render、不要 derivation泄漏。 + +`native/build.rs:346` runtime feature selection 与 generated code关联；所有候选 emission保证 HTTP/crypto特征不丢。`native/build/cache.rs:163` cache 包含生成源码和 runtime 输入；选择值不应进入生成源码，metadata/schema/候选集合必须进入。CLI run/build 会预读部署 Settings，须允许新 adapter section且 build不把 use 静态固化；测试运行时不读取部署 Settings。 + +### 5. case-local fake + +`hir.rs:28` TestCase 只有 checked function/span/database metadata；`check.rs` finalize_tests 在 contracts 后；适合加入 fake table 以及每 case Port reachable 验证。相同测试文件就是一个 entry/TestCase，qualified fake 私有实现归属该 owner。fake 与生产实现都走同一个签名/allowed failure checker，fake 不加入生产 adapter table；无 fake的可达 Port check 失败，即使生产恰好单实现也不能退回。 + +`native.rs:142` suite 单次 emit_definitions 全部 test roots，selected index选择 case；`test_runner.rs:70` 每 case 独立进程/临时项目，仅数据库 case写隔离SQLite setting，现成强隔离可复用。 + +建议使 specialization 带 immutable binding context（Production / Test(case id)），并在传播 App/helper/handler调用时保持此上下文，Port 在 Test context仅追加对应 fake successor。同一个 App 被两个 case调用将生成两个必要 specialization，不会共享错误 fake。alternative 是每进程选定 immutable fake dispatch enum，但 suite静态 effect/reachability必须仍排除生产实现，并处理同 App不同 fake suspension；该路线较易误把生产代码/features带入 suite。切忌用可变全局 fake registry且每 case reset。 + +静态生产合同仍是生产实现 effect union，fake不得让业务合同缩窄；测试 codegen可使用各 fake实际 suspension决定 await，但函数签名/specialization应一致，必要时保守 async化。finalize_tests 的 database metadata 要基于 Test context可达集合，防止从生产Adapter推导数据库（理论上Adapter禁止Model操作，但其他effects同理）。测试缺fake检查应检查经App、普通helper、静态handler、run/blocking所有到达路径，不能只扫Test直接calls。 + +### 6. Markdown / formatter + +`format.rs:264-309` 从token重新组装函数名、pure/recover、body；需要输出qualified path、fails类型、bodyless无花括号，保留注释消费顺序。setting 新 Declaration 分支须覆盖 formatter、markdown::declaration_span、source section validation、names及所有 Declaration exhaustive matches。 + +`markdown/contract/model.rs:36/60` 文档函数仅name/inputs/outputs；`metadata.rs:52` strict ordered元数据，建议Port段新增 `- 允许失败：` 只在Port存在，与 AST failure choice精确比较；Adapter段 name 使用完整 `port.send`，无额外猜测。setting段作为独立record-like Section，字段类型/顺序/数量与源码一致。`validate.rs:232` 用 owner + 最后一段函数名 + arity寻找HIR，会错误定位qualified实现，改为AST declaration span或显式 declaration↔HIR ID映射，不能 `.rsplit('.').next()` 猜。public_functions metadata继续角色导出规则，不能把实现列为公开App能力。 + +### 7. 建议分阶段实现与最小定向测试 + +1. AST/parser/formatter/Markdown + contract/implementation registration + role矩阵；只检查，不先运行native。正反例包括Port body、普通fails、qualified非法owner、缺/重复operation、参数类型/输入名/输出名/顺序/arity不匹配、跨域target。 +2. 固定failure declaration seed、effect union、synthetic dependencies、handler/run/transaction分析；测试一个blocking实现+一个suspending实现的保守拒绝、错误子集/超集、捕获声明全集，生产配置不能改变合同。 +3. shared wire setting + strictJSON选择 +直接native dispatch；确定性本地实现两分支，不联网；同一二进制仅换配置得到不同结果、未知use、缺use、未知字段、重复key、错误类型、未选配置不需存在、选中Secret错误不回显。 +4. TestCase绑定与specialization context；两个测试同一个App各自fake结果、同套件单编译、无fake失败、生产配置坏/含真实地址仍不读取、生产实现不作为fallback、case执行失败不影响后续。 +5. 同步LANGUAGE/Markdown/compiler/toolchain与root test fixtures，独立复核。 + +现有入口：`test/dever-tests/tests/{source_architecture,source_visibility,error_effects,structured_concurrency,application_config,application_testing,markdown_source,formatter}.rs`。建议新 `port_adapter.rs` 聚合端到端闭合合同；只运行必要exact测试和cargo check，无真实HTTP/数据库/全量。旧source_architecture中“port reserved”和App→Adapter正例要按新合同更新，不能保留兼容开关。 + +## Related specs + +### 补充：建议本期固定的执行决定 + +Port failure 合同建议本期采用声明全集、实现子集、App显式映射三条规则。普通App不透传Port私有错误；`fails DeliveryError` 只提取error variants，result仍要求全集匹配。先要求每个Port都有至少一个error variant，error-free语法另行设计，避免本期引入第二种fails语义。 + +“App不能绕过Port”必须本期实施，否则PRD目标未满足。最小可执行限制是：App/Domain禁止未经Port边界的 `file` 和 `network` intrinsic可达路径；Domain的Model操作另由既有owner检查拒绝。复用 `contracts/effects.rs:547` 的穷尽intrinsic_effect分类，不另写模块名白名单。新增一个内部摘要 `unmediated_io`，沿普通函数和具体handler传递，但穿过Port contract时清空这一项；正常effect/blocking/suspension摘要仍完整union，不能一并清空。这能允许App→Port→Adapter→HTTP，并拒绝App→helper/标准包装→HTTP。检查direct intrinsic同样经过此摘要，handler绑定用既有specialization保守展开。Adapter只允许自己的helpers/标准库，仍需单独禁止 `ApiServe`，并将该入口限制为Main显式启动；不要把整个network类禁在Adapter上。日志、时间、随机、密码散列、process.arguments本期保持已有授权，不把“Port约束”扩大为全新纯函数制度。若希望Domain完全无任何副作用，属于更强约束，不能偷偷从本期文件/网络隔离推出。 + +- `.trellis/workflow.md`：研究仅写task research；main负责后续 implement/check。 +- `.trellis/spec/backend/compiler-contracts.md`：source角色、App公共边界、failure/effects/shared dependencies、Model owner、Test权限、wire sole owner。 +- `.trellis/spec/backend/toolchain-and-library.md`：Markdown严格合同、native cache/feature选择、配置只用setting.json、Secret不可观察。 +- `.trellis/spec/backend/directory-structure.md`：目录/role归属。 + +## External references + +本研究仅依据当前仓库，不引入新依赖，不需要外部版本或API资料。 + +## Caveats / Not Found + +- 当前没有 bodyless contract、qualified实现、setting record、Port dispatch 或 fake机制；均不能靠修改一条调用矩阵完成。 +- PRD未给error-free Port表示法、App绕过Port的标准库I/O精确限制、跨Port Adapter setting owner；这些应在实现前由主代理用既有范围做一致决定并同步设计。 +- 明确区分“Port已声明failure全集”和“所有实现实际failure并集”；后者不能替代前者。 +- 查不到单独 `.trellis/spec/backend/application-testing.md`，应用测试规则当前在compiler-contracts与toolchain文档；不要引用不存在的规范文件。 +- 未运行测试，未修改源码/规范/状态。 diff --git a/.trellis/tasks/09-20-port-adapter/task.json b/.trellis/tasks/09-20-port-adapter/task.json new file mode 100644 index 0000000..a0980cd --- /dev/null +++ b/.trellis/tasks/09-20-port-adapter/task.json @@ -0,0 +1,26 @@ +{ + "id": "port-adapter", + "name": "port-adapter", + "title": "Dever Port 与 Adapter", + "description": "实现可替换 Port 合同、Adapter 实现和 setting.json 绑定，并收紧角色调用方向", + "status": "completed", + "dev_type": null, + "scope": null, + "package": null, + "priority": "P1", + "creator": "Antigravity", + "assignee": "Antigravity", + "createdAt": "2026-09-20", + "completedAt": "2026-09-20", + "branch": "main", + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [], + "parent": "09-20-production-application-language", + "relatedFiles": [], + "notes": "", + "meta": {} +} diff --git a/.trellis/tasks/09-20-production-api/check.jsonl b/.trellis/tasks/09-20-production-api/check.jsonl new file mode 100644 index 0000000..10b7cb3 --- /dev/null +++ b/.trellis/tasks/09-20-production-api/check.jsonl @@ -0,0 +1,3 @@ +{"file":".trellis/spec/backend/quality-guidelines.md","reason":"Bounded HTTP/1, HTTP/2 and resource tests"} +{"file":".trellis/spec/backend/error-handling.md","reason":"Input/API/internal error mapping"} +{"file":".trellis/spec/backend/directory-structure.md","reason":"Thin API owner and shared HTTP path"} diff --git a/.trellis/tasks/09-20-production-api/design.md b/.trellis/tasks/09-20-production-api/design.md new file mode 100644 index 0000000..2cb9828 --- /dev/null +++ b/.trellis/tasks/09-20-production-api/design.md @@ -0,0 +1,130 @@ +# Dever 生产 API 边界设计 + +## 2026-09-28 自动权限设计 + +### 静态权限身份 + +权限是 checked HIR 的入口元数据，不是业务 Text 常量。每个非 public API 入口拥有一个 `Permission`：`key/component/domain/site/action/method`。site 由现有目录映射得到，custom action 来自 API 声明，REST 展开为 read/create/replace/delete。key 不包含 HTTP method；同一 component/domain/site 下重复自定义 action 必须静态拒绝。public 入口没有 Permission。 + +权限清单由 native application 内嵌。应用数据库初始化后、服务开始前，Runtime 在控制数据库事务中创建并同步 `_dever_permission`：当前清单 upsert 为 active，清单外旧行标记 inactive。build/check 不连接数据库，同步失败时服务不得启动。 + +### 标准角色存储与请求授权 + +官方 auth runtime 在授权数据库维护 `_dever_auth_role`、`_dever_auth_role_permission`、`_dever_auth_user_role`，并以 `_dever_auth_version` 管理私有 schema。关系使用稳定 permission key，不使用环境相关自增权限 ID。Role 以 `(site, id)` 为身份，user-role 和 role-permission 同样携带 site 复合边界；不同站点可以复用同一角色 ID，且不能互相覆盖或授权。旧单列角色 ID schema 在一个事务中按 role.site 回填关系并替换三表，未知或不完整 schema 拒绝启动。授权只做 allow 并集，`all_permissions` 受 site 和 enabled 限制。 + +verify 输出收缩为 `id: Text, user_id: ModelId?, tenant_id: ModelId?`。Runtime 完成身份 scope 后，使用当前 user、site 和 permission key 对角色关系执行单次 EXISTS 查询；没有用户或授权记录返回 Forbidden。权限不进入 token、Session claims、Identity 或源码能力。租户应用在 tenant scope 内查询本租户角色库；平台入口查询控制数据库。 + +认证、组件可用性、接口授权和业务执行顺序固定为：路由/站点与凭据校验 -> verify -> 建立 tenant scope -> 检查可达租户组件 -> 检查 permission key -> 解码请求与调用 App。public 入口跳过 verify 与角色检查，并且其完整调用图必须静态证明不触达租户组件。 + +角色写入由 Runtime 的窄接口统一完成：列出 active 权限、创建/更新角色、事务替换角色权限、分配/撤销用户角色。写入时验证 site、active permission、角色状态和当前租户；这些能力只能从受保护的写 API 到达，业务组件不直接写私有表。租户 provision 时创建 site-scoped owner role并分配首个用户，避免授权死锁；保留 Owner 不能通过普通 save/grant/revoke/disable 修改。 + +### Job 与租户组件 + +User Job 只保存一个入口 permission key 和既有 provider/site/subject/session/tenant 身份引用。执行顺序固定为重新 verify、确认当前编译权限、检查租户组件、查询当前角色；失败转 blocked 且不进入业务 dispatch。v4 capability 列表迁移到新 schema 时，旧活跃 User Job 统一阻断为 `authorization_contract_changed`，物理删除旧 capability 列，历史终态记录保留。 + +编译器以 tenant Model 和 tenant Job 的静态归属识别租户组件，并为 API/CMD/Job 入口计算可达租户组件集合。混合组件整体视为租户组件，它拥有的全部 Job 都绑定租户队列，即使单个 Job 只访问 global 数据。控制库 `_dever_tenant_component(tenant_id, component, disabled_at)` 只存禁用例外。禁用优先于角色授权，不删除租户数据或改变 schema。 + +### 配置收缩 + +租户配置只有 `database/max_pools/idle_timeout_ms`。`database` 选择控制连接；物理租户数据库仍由现有 database 配置的 tenant target 派生。旧 `mode`、`control_database` 及 table/field 值明确失败，不保留双读或迁移别名。 + +## 2026-09-26 身份、站点与租户修订 + +此节是用户批准后的当前合同，替代旧 Auth Boundary 中的无身份运行时限制。实现仍复用既有 AST/HIR、wire、静态路由、隐藏请求上下文、ORM、Job 和 Port，不增加 Go 应用框架层。 + +### 配置与路由 + +`sites.<key> = { path, auth, hosts? }`。path 相对每个领域 api 目录，只匹配物理父目录（SourceLayout topics 排除文件名），空 path 用于单站根 API。保留完整 topics 生成 URL，站点不改写路径。每条 HTTP API 必须得到无歧义站点归属，Host 是进一步限制，不是认证结果。CMD 不从 HTTP 目录映射取得可信身份。 + +API 仅有 rest 和动词到 App 的绑定，增加单个 HTTP 声明的 public 修饰；禁止源码 site 声明、隐式全站共享接口及强制按站点复制 App/Model。注册后的路由元信息是认证 Provider 选择的唯一依据。 + +### 认证与权限 + +`auth.providers.<key>` 保存凭据验证配置和业务 App verify 引用。配置引用必须经过静态签名/Effect 校验并进入生产调用图、native emission、数据库初始化和 API 契约。密码/角色/成员业务复用账号领域，认证代码并不要求名为 auth 的目录。 + +运行时校验凭据后由受控桥接调用 verify；普通源 record 不能写入当前身份。verify 在租户业务自动事务之前完成。只读上下文包含身份来源、站点和可选用户/租户，不携带权限集合。App 可读 auth，Domain 保持纯规则。租户一经建立在当前调用链内不变，切换接口签发新凭据。 + +操作权限由 API/REST 静态入口自动生成并由 Runtime 查询标准角色关系；源码不登记或检查权限字符串。自动 REST 与 App 共享权限失败及标准 JSON Error 映射。身份对象不包含权限集合或可渲染原始 Token/Cookie。 + +### 数据边界 + +Model 的静态结构与物理存储分离：global 使用平台连接，其他持久化 Model 在 tenant 开启时要求可信租户。持久化模式只支持 database。数据库名只由内部稳定租户标识导出，不提供 table/field 分支。 + +普通 ORM 与自动 REST 使用共同存储/范围合同；不可证明符合隔离的原生 SQL 拒绝。连接由显式可回收拥有者管理，不永久泄漏每租户的 static handle。租户初始化有受控状态与 schema 版本，业务请求不能自行建库。跨平台库引用为逻辑引用，不能产生跨库 FK/Join；写入口只拥有单物理库事务。 + +### 后台执行与差异实现 + +Job 的租户身份由入口系统生成并与入队原子持久化，不信任 payload 内自报身份。User Job 只保存 Provider、站点、subject、session、租户声明和一个入口 permission key，不保存 token 或权限集合；执行时重新调用 verify，并查询最新角色关系复核该权限。受控 System 与 User 身份明确区分，缺少可信身份或旧 schema 不能降级。Worker 使用有界调度，租户任务不得回退平台队列。CMD 使用 System 身份，不能读取 HTTP Header/Cookie 或自报用户。 + +租户功能、额度及配置是业务数据。Port 候选实现静态编译、共享同一签名/失败/Effect 合同，选取依据可信租户配置。log 配置使用现有 logger 并关联调用元数据；runtime 缺省按 API/Job 能力选择模式，显式 CMD 不改变部署配置。 + +### 验证边界 + +先静态反例和 runtime in-process 验证，再生成代码/AOT 及双源码 CMS 定向用例。网络/数据库用例只使用 test 自有临时目录、随机 loopback 端口或明确配置的隔离数据库，运行前说明，不启动现有服务，不运行全量测试，不使用环境变量配置。 + +## Source And Generated CRUD + +本节记录目标方向。声明式 API/CMD、编译器生成入口、基础 REST 和部分隐藏请求上下文能力已实现；Model 字段绑定、可信 `owner`、完整 Cookie 策略、标准 API Error 与 Upload 仍是待实施合同，不能把下述示例全部视为现有语法。 + +```dever +# module/user/profile/model.dever +type Profile { + owner user_id: user.account.model.id = user.auth.id() + name: Text(1, 64) index + bio: Text? +} + +# module/user/profile/api.dever +rest +``` + +路径是 `/user/profile`，不重复目录名。仅当同领域主 Model 唯一时允许无参数 `rest`；多 Model 必须显式指定，歧义时报编译错误。生成 REST 覆盖 GET 列表与 `/{id}` 详情、POST、PUT 全量替换、DELETE，默认只传输本表公开字段和外键 ID，不自动展开或嵌套写关联。分页有界，外部筛选仅允许显式 `search` 或已索引的静态字段，不接受任意 QueryPlan。 + +Model 字段不重复列读/写名单：普通字段由编译器按字段名推导；`create`、`replace`、`search` 表达式仅描述不同的输入来源或纯转换。`owner field = context_read()` 对生成 REST 的创建赋值及所有读写施加同一行范围；GET/PUT/DELETE 的身份条件必须进入同一数据库查询或写语句，客户端不得设置或修改 owner。字段表达式仅在生成的 REST 入口执行，不改变普通 App `model.create/get/...` 的行为。只读上下文能力和纯函数可作为静态调用目标；数据库、网络、任务和写 Effect 不允许出现在字段绑定中。多个接口需要不同权限策略时使用明确的 App 操作。 + +```dever +# 自定义动作才需要 App；App 签名就是 API 输入输出合同。 +post publish = app.publish +cmd rebuild = app.rebuild +``` + +`get`、`post`、`put`、`delete` 是 API 声明而非函数名前缀；`cmd` 独立于 HTTP。GET 调用链禁止数据库写 Effect；POST/PUT/DELETE 和 CMD/Job 的数据库写入在入口调用链自动建立事务，失败或取消回滚。`rest` 与同路径同方法的显式声明冲突时报错，不暗中覆盖。无需 App 透传 CRUD、`crud ... on model`、`expose`、`package` 或手写启动 `main.dever`。 + +应用业务源码由 `<component>/<domain>/<role>` 唯一确定身份；固定 role 为 app/domain/model/port/adapter/api/job。App 函数均为领域公开能力，Domain 保留私有纯业务规则；topic 文件名不进入 App 调用名。非 API 主题目录只允许一层。App/Domain/Port/Adapter/Job 的根文件与主题目录互斥；主 `model.dever` 可以和附属 `model/<topic>.dever` 共存，`api.dever` 可以和 `api/**` 共存。`user/` 仅为 component，`user/profile/` 为 domain。主 Model 的逻辑身份与表名保持 domain 名，附属 Model 的身份与表名取 topic；移动文件不是纯重构。 + +生成的应用入口收集 API、Job 和 CMD；没有命令参数时按 setting 的 runtime mode 启动服务，CMD 由稳定的 `<component>.<domain>.<name>` 名称和一个 JSON 输入显式选择。CMD 选择不能改写部署配置；没有可运行入口或 CMD 名称歧义时失败。基础 REST 和自定义 HTTP/CMD 共享类型化 wire 与固定 JSON envelope，不能维护第二套动态解码器。 + +函数约束使用已检查 HIR 和生产入口调用图，不按长度或函数数量猜测质量。未被生产入口或其他真实函数引用的私有实现、完全不增加类型/范围/失败/效果合同的私有转发、只有原样 Model CRUD 的公开 App 包装是硬错误。应用 build/run 对不可达 App 能力报错；库式 check 不推测外部调用方，保留提示。W003 相似实现仍是提示，不自动合并不同业务含义。所有分句保持同文件连续声明，合法单用途复杂规则不受影响。 + +## Custom Handler Shapes + +GET/DELETE 从类型化 query/path 输入读取，POST/PUT 从静态 JSON 请求体读取；multipart 入口接收一个包含受限 scalar/Upload 字段的 record。API 自动继承绑定的 App 签名，不再机械重写参数和唯一 `response` 输出。请求值和业务失败继续使用共享静态 codec 与固定 JSON envelope。 + +## Context And Response Metadata + +Runtime 为每个 request 创建仅在调用链内有效的 opaque Context handle，不作为源函数参数传递。受控 API 能力返回复制的 Text 或 Secret 值，并修改 Context-owned response metadata。Native route 在 handler 成功/标准失败后统一合并 headers/cookies 并构造 envelope；handler 不能直接获得底层 Response。 + +Cookie使用具体record/choice表示属性。Secret cookie sink接受Secret并直接编码header，不产生Text中间值。普通response header仅接受Text且经过token/CRLF和forbidden-name检查。 + +## Error Mapping + +官方 `dever.api.Error` 是唯一可离开API handler并被协议映射的error choice。API通常通过 `result(app.call(...))` 捕获业务错误，再用小的API-local clause映射。Native route区分： + +- success -> 200/0/ok/data； +- api.Error ->对应status/code/message/null； +- InputError -> 400； +- other inferred failure/fault -> log request metadata, return generic500。 + +错误message有长度上限且不得包含Secret/private value。业务层不依赖HTTP status。 + +## Upload Lifecycle + +HTTP request reader为multipart路径提供bounded streaming/spool模式。Parser逐part验证headers和limits，将文件写入 `data/tmp` 下create-new随机名，业务只看到opaque Upload、原始filename展示值、declared MIME和size。Scalar fields经过shared decoder。 + +Upload资源owner跟踪临时文件。Storage Adapter通过批准runtime file/stream操作消费并显式commit ownership；未commit drop/close删除。API response完成前检查所有Upload已消费/关闭，防止隐式长期文件。 + +普通JSON路径保留现有bounded full body，因为全局body limit明确；multipart不复用会全量聚合的`Request.body`。 + +## Auth Boundary + +登录、Session 和资源级授权仍由业务实现。业务 verify 向隐藏调用上下文提供只读身份，供 `owner` 绑定或 App 中显式 `dever.auth` 能力使用；接口角色关系由官方 auth runtime 读取。CMD 使用受控 System 身份；认证请求入队的 User Job 持久化最小身份来源和入口 permission key，并在执行时重跑 verify 与当前授权查询。`owner` 只约束自动 REST，手写 App 的 Model 操作必须明确检查其数据范围。 diff --git a/.trellis/tasks/09-20-production-api/implement.jsonl b/.trellis/tasks/09-20-production-api/implement.jsonl new file mode 100644 index 0000000..479bdc1 --- /dev/null +++ b/.trellis/tasks/09-20-production-api/implement.jsonl @@ -0,0 +1,3 @@ +{"file":".trellis/spec/backend/compiler-contracts.md","reason":"API role, typed inputs, public outputs and failure contracts"} +{"file":".trellis/spec/backend/toolchain-and-library.md","reason":"HTTP/native/runtime resource and codec boundaries"} +{"file":".trellis/spec/backend/logging-guidelines.md","reason":"Request context and secret-free error logging"} diff --git a/.trellis/tasks/09-20-production-api/implement.md b/.trellis/tasks/09-20-production-api/implement.md new file mode 100644 index 0000000..c39caf4 --- /dev/null +++ b/.trellis/tasks/09-20-production-api/implement.md @@ -0,0 +1,173 @@ +# Dever 生产 API 边界实施计划 + +## 2026-09-28 自动权限与租户收口 + +- [x] 用户确认最终方案：API 自动权限、分层权限目录、标准角色关系、database-only 租户、组件禁用例外和 Job 当前权限复核。 +- [x] 修改前完整备份：`/data/project/dever-backups/pre-auto-rbac-20260928-105925/dever.tar.gz`；gzip 校验通过，SHA-256 `700c2756dfcb4c5bceaff9dc1a16b86cfe94599fdeea798740ebc5fe3a461eb0`，约 1.5 MiB。 +- [x] HIR/API checker 生成 custom/REST 权限元数据，public 不生成权限，重复 action 静态拒绝。 +- [x] Runtime 控制库同步 `_dever_permission`，并提供按 component/domain/site/action 分组的只读目录。 +- [x] 官方 auth runtime 建立角色、角色权限和用户角色私有表，提供事务化授权写入与单 key EXISTS 校验；verify Identity 删除 permissions。 +- [x] 删除 `dever.auth.require`、`AuthRequire` intrinsic、源码权限字符串和 Job capability 集合；所有非 public API/REST 在 handler 前自动授权。 +- [x] Job schema 升级并保存单一 permission key；执行时重新 verify 和查询当前角色，旧活跃 User Job 安全阻断。 +- [x] 租户配置改为 `tenant.database`，删除 mode/control_database，不保留兼容分支。 +- [x] 自动识别租户组件，生成入口可达组件集合；实现 `_dever_tenant_component` 和 API/CMD/Job 统一禁用检查及 CLI enable/disable。 +- [x] CMS Dever/Markdown 改为标准角色分配和自动权限；删除 membership 硬编码权限列表及所有手工 require。 +- [x] 增加编译器、配置、Runtime SQLite、Job、组件闭包和双源码 CMS 定向正反测试。 +- [x] 运行允许的最小 cargo check/定向测试、diff 检查和独立终审；真实 PostgreSQL 作为需要外部隔离实例的单独验收项记录。 +- [x] 抽取只读根 `config/setting.json` 的共享 PostgreSQL fixture，并让 ORM 与授权验收复用同一隔离数据库/随机 schema/失败清理合同；SQLite 与 PostgreSQL 授权目标复用同一数据库无关断言。 +- [x] 在真实 PostgreSQL 上运行授权目录、v1→v2 角色迁移、跨站同名角色、多角色、Owner 与失效权限的 ignored 定向验收；2026-09-30 再次通过，使用显式自有实例和临时根配置，结束后恢复配置文件不存在，不探测默认服务。 + +## 2026-09-26 已授权执行清单 + +- [x] 用户批准完整修订方案，并要求修改前备份；认证优先复用现有账号领域。 +- [x] 备份完整工作目录（包含 untracked 源码与 .git，排除 target）到 `/data/project/dever-backups/pre-auth-tenancy-20260926-rm6kL6/dever.tar.gz`；gzip 检查和 tar 对源逐项比较均通过。SHA-256 `e4f0666d054ca241484e8333c16270f039a26c3c616b7ddd0f0bea50476ca9b7`，约 1.4 MiB。 +- [x] 读取现有调用链和规范；确认站点 path 必须匹配 API 父目录，认证 hook 需进入生产根，Model/Job/Adapter 当前启动固定绑定。 +- [x] 配置、目录站点归属、public HTTP 声明、静态认证 hook、只读身份和标准错误。 +- [x] global/tenant Model、database 模式按租户选择物理库、有界池、显式初始化迁移和单库事务。 +- [x] 多用户成员角色示例、能力检查、REST owner 行范围与字段来源；普通 ORM 保持显式业务范围。 +- [x] Job/CMD 可信租户、日志配置、runtime 缺省、app 配置消费者迁移；CMD 不接受 JSON 自报租户/用户。 +- [x] 租户组件启停由核心私有表和受控 CLI 完成；table/field 隔离及租户级动态 Adapter 已被后续 database-only 决策取消，不保留兼容模式。 +- [x] 完成普通/Secret Cookie、标准错误和受限 Upload 合同。 +- [x] Dever/Markdown CMS 同步、定向正反验收和现有规范同步。 +- [x] 独立最终审查收口；限定复核外部 Origin、密码事务、public POST 错误、契约夹具和 Job 身份撤销，未发现 P0/P1 阻断。 + +实施与检查使用 Trellis 子代理；最多两个独立代理并行，共享文件必须先分配唯一拥有者。主会话负责合同、整合及最终证据。现有用户改动不回退、不提交 Git。测试仅限当前改动相关定向目标。 + +## Start Gate + +- [x] secure-data-contracts、time-typed-codec、port-adapter完成。 +- [x] 用户批准父任务规划，并批准 2026-09-25 的文件/函数/入口/API 改造方案。 +- [x] 核对路径冲突、主 Model 推导、CMD 输入合同及现有 API/native/runtime、SourceLayout、ORM、HTTP 边界。 + +## Implementation + +- [x] 收紧 role 布局：Model/API 根文件可与主题目录共存，其余 role 保持互斥；主 Model 身份沿用现有规则。 +- [x] 实现 `get/post/put/delete/cmd/rest` 声明、App 绑定、路径生成与 `{id}` 路由；应用严格模式拒绝旧前缀 API 函数体。 +- [x] 实现生成的 run/build 入口与 CMD JSON 选择；移除维护中 CMS 的手写 main。 +- [x] 实现主 Model 基础 REST：公开字段、GET 列表/详情、POST/PUT/DELETE、静态 SQL 和有界分页。 +- [x] 实现 Model 字段 `create/replace/search` 来源、可信 `owner` 身份绑定与所有自动 REST 读写同语句行范围；普通 App ORM 不受隐式修改。 +- [x] 实现 GET 写 Effect 拒绝和写入口自动事务；覆盖 API/CMD/Job 生成路径，持续补足提交、回滚与取消的定向运行验证。 +- [x] 基于生产调用图拒绝可证明的无用私有函数、无契约纯转发、简单 Model CRUD App 包装及应用构建不可达 App；保留警告和库检查。 +- [x] 扩展 API input shape 检查和 shared decoder generation，CMD 使用同一严格 JSON 对象解析。 +- [x] 增加 runtime 请求级隐藏作用域，不暴露可保存或伪造的 Context 源参数；HTTP/CMD/Job 调用链静态区分。 +- [x] 增加受控 Header 读取/响应写入、Secret Cookie 原始字节的版本化 base64url 读取/写入和 typed CookieOptions；禁止敏感 Header 经 Text 读取、Header 注入及 Secret Cookie 缺少 Secure/HttpOnly。 +- [x] `client_address()` 由 HTTP/1、HTTP/2 的 TCP peer 提供，不信任转发 Header；HTTP/1 loopback 定向验证通过。 +- [x] 拒绝 CMD/Job 或脱离请求的 Task/parallel/blocking 调用 HTTP Context；HTTP/2 并发流的 request id、path、peer 和响应 Header 隔离定向测试通过。 +- [x] 完成普通 Text Cookie 的安全读策略；Secret Cookie 保持独立非 Text 通道。 +- [x] 增加官方API Error及native status/envelope映射。 +- [x] 重构 request read boundary，认证/路由先于正文，multipart 使用 bounded streaming/spool。 +- [x] 增加 affine Upload runtime owner、App→Port→Adapter 单次传递、storage consume/close/cleanup 和 checker 规则。 +- [x] JSON request、request id 与现有 HTTP/1、HTTP/2 dispatch 路径共享；后续仍需定向运行验证 HTTP/2/drain。 +- [x] 更新 README、LANGUAGE、Markdown 语法与 CMS 两套源码，迁移至编译器生成入口；其余协议文档随 Context/Error/Upload 实施更新。 +- [x] Job 私有存储最终升级到 v5：只持久化可复核身份引用和单一入口 permission key，拒绝无身份入队；CMD/cron/Test 显式 System，User 每次执行重新 verify，v4 活跃 User 行阻断并删除旧 capability 列。 +- [x] 响应 Header/Cookie 改为 pending/committed 两阶段，序列化和数据库提交成功后才发布；登录类请求使用严格同源校验，Cookie 签发/清理统一读取配置。 +- [x] PostgreSQL database 租户由显式 `tenant migrate` 通过控制连接幂等创建后迁移；普通请求/worker 保持无创建、无回退。 + +## 2026-09-25 定向结果 + +- `cargo check --offline -p dever-core -p dever-runtime -p dever-cli` 通过。 +- API 声明测试 10/10、语法/格式 22/22、基础 REST 4/4、目录规则 12/12、旧 HTTP 路由 7/7 通过。 +- CMS Dever/Markdown `check`、`fmt --check` 通过；CMS 同源与生成入口测试 1/1 通过。 +- 无 main 的 API/CMD AOT 编译和实际 CMD JSON 调用通过，输出固定 envelope；Job AOT 编译通过。生成入口定向测试 11/11 通过。独立审查指出的必填 Json、事务并发、日志泄漏和 REST Text 边界已修复，并增加定向反例。请求上下文和 TCP peer 测试 5/5、HTTP/2 多路复用定向测试 1/1 通过；CMS 同源测试 1/1 通过。 +- 仍需确定 `owner` 的可信业务 actor 来源，以及声明式 API 如何把 App 业务失败映射到官方 API Error；两项不能用客户端 Header/Cookie 自报身份或让 App 隐式依赖 HTTP 状态来代替。Model `create/replace/search` 绑定、普通 Text Cookie 策略及 multipart Upload 还未实现。 +- 未运行全量测试、服务级 CMS/HTTP 集成测试、真实数据库或性能/内存压测；rustfmt/clippy 组件未安装。 + +## 2026-09-26 身份与配置定向结果 + +- typed `auth/sites/log` 配置、API 物理目录站点归属、单声明 `public`、静态 verify hook、只读 `site/auth` 上下文、HS256 凭据、Cookie Origin 约束和标准 API Error 映射已实现。 +- `app` 顶层配置已移除，性能工具迁移到顶层 `performance`；runtime 缺省按 API/Job 能力推导。UUID 官方 parse/to_text 能力已补齐，供业务会话与租户标识使用。 +- `/root/.cargo/bin/cargo check --offline -p dever-core -p dever-cli` 通过；`/root/.cargo/bin/cargo check --offline -p dever-runtime --features api` 通过；`api_routes` 7/7 通过（含 native/AOT）；此前 `auth_runtime` 2/2、`application_config` 5/5、API 声明 14/14、性能工具 Python 测试 40/40 通过。 +- database 模式多租户物理存储、Job/CMD 可信租户、普通 Text Cookie、Upload、REST owner 与字段绑定已经完成；CMS 双源码正在最终复验。 +- 此处当时记录的 table/field 隔离与租户级动态 Adapter 后续已由 database-only 决策取消；当前不保留这些模式或兼容分支。 + +## 2026-09-26 多租户、REST 字段与 Upload 定向结果 + +- database-only 租户已贯通 global/tenant Model、无回退物理库选择、ready/fingerprint、显式 `tenant migrate`、有界池回收、单库事务和 tenant Job 恢复；当前配置入口是 `tenant.database`。SQLite 无回退/幂等迁移 CMD 1/1、tenant Job 物理库隔离 1/1 通过。 +- REST `owner/create/replace/search` 已进入 checked HIR、生产可达图与 native；owner+search 复合索引、双静态 SQL、pure Domain/可信上下文正例和数据库/时间/响应/并发反例通过。SQLite AOT 1/1 通过（74.46 秒）。 +- multipart/Upload 边界 5/5 通过，包含流式分块、limits/重复字段/路径穿越、取消清理、affine 静态反例和 loopback POST→App→Port→Adapter→storage 实际 AOT；临时目录在成功后为空。 +- 写入口事务允许等待完成且目标无数据库/并发的 `blocking`，仍拒绝 `run` 等逃逸并发；自动事务、拒绝反例和显式 transaction 定向 3/3 通过。 +- API AOT 路由复验 1/1 通过；认证运行时此前 2/2、CMD tenant migration 1/1 通过。CMS Plain/Markdown 使用部署 settings 的同源/application reachability 复验 1/1 通过。 +- 未运行全量测试、真实 PostgreSQL、HTTP/2 上传 reset、外部对象存储原子发布、服务压测或性能/内存压测；rustfmt/clippy 组件未安装。 + +## 2026-09-26 可信身份与事务收口结果 + +- CMS Plain/Markdown 的发布操作统一调用 `dever.auth.owns_user(stored.author_id)`；租户 verify 同时要求 `platform.tenant.active`，避免仅凭有效会话绕过资源所有者或停用租户。 +- public API 不再从浏览器 Cookie 建立身份，显式无效 Authorization 仍拒绝；Origin 与 Host 按站点配置的外部 HTTP/HTTPS scheme、主机和有效端口精确匹配，不根据内部监听协议或转发 Header猜测。Cookie 名称、TTL、签发和清理均来自 `config/setting.json`，没有环境变量入口。 +- `_dever_jobs` 当前 v5 区分队列物理租户与执行租户，保存 provider/site/subject/session/tenant 引用及单一入口 permission key；不保存 token、Cookie、角色或权限集合。User Job 每次重试均重新 verify 并查询当前角色，legacy v4 活跃 User 行转为 `blocked/authorization_contract_changed`，匿名/无身份路径不能退化为 System。 +- SQLite 写事务继续 `BEGIN IMMEDIATE`。含 password verify/hash 的 HTTP 调用链不再生成外层事务，编译器要求全部写入进入显式短 `transaction` 并禁止事务内密码计算；CMS 登录在短事务内重新检查账户、租户和成员关系后创建 session。响应元数据只在响应序列化和数据库提交后转为 committed。 +- PostgreSQL 租户迁移会通过配置的控制连接创建派生数据库，SQLSTATE `42P04` 作为并发/重复创建成功处理；尚未连接真实 PostgreSQL 验证权限和服务端行为。 +- 定向结果：核心/runtime 的 API、SQLite、PostgreSQL feature `cargo check` 通过；durable Job 静态、生成入口、SQLite 状态机、v5 迁移/持久化、public 边界和 tenant 隔离各 1/1 通过；认证 runtime 3/3、配置 6/6、安全契约 12/12、生成 User Job AOT 1/1 通过。 +- 新增撤销回归通过 1/1：测试拥有的临时 SQLite 应用先以有效会话入队 User Job，再撤销 session 并启动 worker；任务转为 `blocked/identity_rejected`，业务副作用表保持为空。该验证使用临时 loopback 与子进程，无外部服务。 +- 密码窗口并发回归包含在安全契约中：账号读取与 session 短事务之间模拟密码计算，同时另一连接可完成无关写入；短事务重新读取状态后正常创建 session，不持有跨密码计算的 SQLite 写锁或旧读快照。 +- CMS Plain/Markdown `check` 与 `fmt --check` 均通过；两套源码/Model/API/config 同源检查 1/1 通过。两套 `check` 各保留 3 个预期 W001，不影响编译。 +- 独立终审只读复核上述五项实现与执行路径，未发现 P0/P1 阻断；真实 PostgreSQL、性能、table/field 和动态 Adapter 不属于本轮收口范围。 +- 未运行全量测试、持久服务、真实 PostgreSQL、HTTP 集成、性能/内存压测；rustfmt/clippy 组件未安装。table/field 租户隔离和租户级动态 Adapter 仍按已确认范围延期。 + +## 2026-09-28 自动权限与租户收口结果 + +- 权限由 checked HIR 自动生成，custom 使用 action，REST 使用 `read/create/replace/delete`；method 仅作元数据。跨 method/path 重复 key、public 触达租户组件、public/GET 调用角色管理能力均在 native 前拒绝。 +- `_dever_permission` 在服务启动前同步；授权库使用版本化 `_dever_auth_role`、`_dever_auth_role_permission`、`_dever_auth_user_role`，三表以 `(site, role_id)` 隔离同名角色，旧单列 ID schema 事务迁移并保留授权数据。精确权限使用索引化 EXISTS，多角色取并集，`all_permissions` 只覆盖同站点。保留 `_dever_owner:<site>` 只能由 `tenant owner` provision，普通 save/grant/revoke/disable 全部拒绝。 +- Identity 已固定为 `id/user_id/tenant_id`；源码 `auth.require`、AuthRequire intrinsic、权限列表和 Job capability 集合已删除。受保护 API 顺序固定为 verify、tenant scope、组件检查、精确权限、输入解码和 App。 +- Job 已升级到 v5，只保存一个入口 permission key；每次执行按 verify、编译目录、租户组件、当前角色的顺序复核。v4 活跃 User Job 迁移为 `blocked/authorization_contract_changed` 并删除旧 capability 列，System 任务保持可执行。 +- 租户只支持 `tenant.database/max_pools/idle_timeout_ms`。租户组件从 tenant Model/Job 自动推导，混合组件拥有的全部 Job 统一使用租户队列，并通过 `deverc tenant component enable|disable` 控制；public 入口不能静态触达租户组件。 +- CMS Plain/Markdown 已同步为核心角色管理薄适配，admin/front 各有同站点管理入口，启动说明要求迁移后分别 provision Owner；membership 只表达业务成员关系。两套 `test` 此前各 1/1 通过；本次最终源码改动后，两套 `check`、`fmt --check` 及当前编译器双源码同源/完整应用可达检查通过，check 仅保留 3 个既有 W001。受磁盘空间限制未重复链接两套完整 CMS 测试程序。 +- 定向验证通过：core/runtime/CLI cargo check；`application_config` 6/6、`api_declarations` 16/16、`contract_api` 9/9、`auth_runtime` 3/3、`authorization_runtime` 1/1（含旧角色 schema 迁移与跨站同名角色）；REST 权限、Identity、Job v4->v5 删列、撤销后 Job 阻断、混合组件 Job 租户 AOT、租户组件 disable/enable、CMS 双源码同源及受保护 auth/Job 顺序 AOT 各 1/1。 +- 两轮独立终审指出的站点角色主键、混合组件 Job、CMS 首次 Owner、v5 旧列、Job 复验顺序和旧授权 schema 升级问题均已修复并复验；最终未发现新的 P0/P1。 +- 未运行全量 workspace 测试、真实 PostgreSQL、持久服务或性能/内存压测；rustfmt/clippy 组件未安装。本轮实现没有环境变量配置入口。 + +## 2026-09-28 PostgreSQL 授权验收准备结果 + +- 新增共享 PostgreSQL 测试 fixture，只接受仓库根 `config/setting.json` 的 `database.postgres_test`；URL 的数据库名必须包含且只包含一个 `{case}`。fixture 为每个目标派生专用数据库名并创建随机 schema，测试成功或 panic 后都从独立 runtime 尽力清理。 +- `postgres_orm` 已迁移到共享 fixture；新增 ignored `authorization_postgres` 目标。SQLite 与 PostgreSQL 共用唯一授权合同，覆盖权限目录、旧 v1 schema 数据保留迁移、跨站同 role id、多角色并集、角色禁用/撤销、保留 Owner 限制和失效权限拒绝。 +- `/root/.cargo/bin/cargo check --offline -p dever-tests --features api,postgres --test postgres_orm --test authorization_postgres` 通过；SQLite 授权 exact 1/1 通过。无 feature 的 `syntax_and_format` 与仅 SQLite 无 API 的 `sqlite_orm` 编译检查由实施子任务验证通过。 +- 独立检查发现 PostgreSQL URL 的 `dbname` 查询参数可覆盖路径数据库，已拒绝明文和百分号编码的覆盖形式；纯 URL 隔离回归 1/1 通过。复验期间磁盘一度归零，两个重复定向链接已中止并清理本轮生成的可重建测试二进制/增量对象，未删除源码、配置或用户数据。 +- 真实 PostgreSQL 未运行：仓库根当前没有 `config/setting.json`。测试不会读取环境变量或命令行连接覆盖，也不会自动连接、创建或清理未明确配置的数据库。 +- `cargo fmt --all --check` 未运行成功，因为 rustfmt 组件未安装；新增 Rust 已手工按现有样式整理。 + +## 2026-09-29 真实 PostgreSQL 定向验收 + +- 使用仓库根 `config/setting.json` 的 `database.postgres_test` 配置（测试凭据仅存在于被 `.gitignore` 忽略的临时文件，验收后已删除），在本机隔离 PostgreSQL 数据库上运行 `authorization_postgres` ignored 定向目标，1/1 通过。 +- 本次实际覆盖权限目录同步、v1 角色 schema 迁移、跨站点同名角色、多角色权限并集、角色禁用、用户角色撤销、Owner 保留限制和失效权限拒绝；测试使用随机 schema，临时数据库在验收后清理。 +- `postgres_orm` 未通过，失败发生在数据库连接前的共享 fixture 编译阶段：旧 fixture 仍从 `main.dever` 调用 `user.create`，当前 Model 规则要求 Model 操作只在 owning-domain App 中调用。SQLite 共享 fixture 同样复现该编译失败，因此这是独立的 ORM fixture/语言契约迁移项，不是 PostgreSQL 实例或配置问题。 +- 没有读取环境变量或命令行连接覆盖；仓库工作区未保留真实 PostgreSQL 凭据或 `config/setting.json`。 + +## 2026-09-30 当前 PostgreSQL 组合补验 + +- 共享 ORM fixture 已按 owning-domain App/Model 合同迁移，真实 PostgreSQL ORM/授权 Store ignored 各 1/1 通过，旧 09-29 ORM 失败已关闭。 +- `postgres_api` exact ignored 1/1 通过 stock Dever CMS 的真实 HTTP、两个物理租户数据库、同 slug 独立发布、跨站 Cookie/跨租户登录拒绝、普通非 Owner 的多角色/跨站同名角色及同一会话撤销立即失效。测试注册仅在副本追加 `test/dever-tests/fixtures/postgres_api/` 片段，不修改生产 CMS 或 Owner 保留角色规则。 +- 新报告 `target/performance/postgres-http-09-30-v1/report-1968019-1790739069361983512.json`；`test-rbac-final.log` 和 `server-rbac.log` 保留真实结果与服务日志。失败轮的站点目录读取和 DELETE body 工具错误已按现行合同修正，未放宽 checker/runtime。 +- 临时控制库/租户库全部清理，自有 PG 已停，根 `config/setting.json` 恢复原先不存在；PG 下载/数据目录移入回收站可恢复。Python 工具 57/57、Rust postgres_api 非 ignored 3/3；不宣称 Markdown PostgreSQL/HTTP2 所有组合或全 workspace 复验已运行。 + +## Focused Verification + +```bash +cargo check --offline -p dever-core -p dever-cli +cargo check --offline -p dever-core -p dever-runtime --features dever-runtime/api,dever-runtime/sqlite,dever-runtime/postgres +cargo test --offline -p dever-tests --features api,crypto,sqlite --test secure_contracts --test auth_runtime --test application_config +cargo test --offline -p dever-tests --features api,crypto,sqlite --test durable_jobs revoked_user_job_is_blocked_before_business_dispatch -- --exact +cargo test --offline -p dever-tests --features api,crypto,sqlite --test contract_execution cms_formats_share_source_model_api_and_configuration_contracts -- --exact +cargo test --offline -p dever-tests --features api,postgres --test authorization_postgres postgres_authorization_preserves_site_scoped_roles_and_catalog -- --ignored --exact +cargo run --offline --quiet -p dever-cli -- check examples/cms/dever +cargo run --offline --quiet -p dever-cli -- check examples/cms/md +cargo run --offline --quiet -p dever-cli -- fmt examples/cms/dever --check +cargo run --offline --quiet -p dever-cli -- fmt examples/cms/md --check +cargo test --offline -p dever-tests --features api --test api_routes +cargo test --offline -p dever-tests --test http_engine +cargo test --offline -p dever-tests --test http2_server +cargo test --offline -p dever-tests --test logging +cargo test --offline -p dever-tests --test pooled_network +cargo test --offline -p dever-tests --test source_architecture +cargo test --offline -p dever-tests --test markdown_source +git diff --check +``` + +上传测试只使用test-owned临时目录和bounded loopback peer，提前说明后运行；不启动持久服务、外部网络、全量测试或性能基准。 + +## Review Risks + +- Context/Upload不能通过record、Task、Channel、App参数或输出逃逸request scope。 +- multipart错误/取消所有路径必须删除已创建part，且不能信任filename作为路径。 +- Secret cookie不能生成可被Render/log捕获的中间Text。 +- API Error必须与未知App failure区分，不能把内部错误message返回客户端。 +- HTTP/2并发stream的Context/temporary ownership不能串请求。 +- 目录或文件重排不得隐式改变 Model identity、表名、App 调用名和 API 路由；旧 API/Main 方案不能与新方案共存。 diff --git a/.trellis/tasks/09-20-production-api/prd.md b/.trellis/tasks/09-20-production-api/prd.md new file mode 100644 index 0000000..cd79dea --- /dev/null +++ b/.trellis/tasks/09-20-production-api/prd.md @@ -0,0 +1,86 @@ +# Dever 生产 API 边界 + +## 2026-09-28 自动权限与 database-only 租户修订 + +本节替代下文所有关于手写 `auth.require`、verify 返回权限列表、框架不提供 RBAC、`tenant.mode`、`tenant.control_database`、table/field 租户模式和 Job capability 列表的旧合同。 + +- `public` 是唯一匿名入口。所有非 public HTTP API 默认同时要求可信身份和当前接口权限；源码不得手写权限字符串或调用 `dever.auth.require`。 +- 编译器根据 `<component>/<domain>/api/<site>/**` 和 API 声明自动生成权限目录。权限 key 为 `component.domain.site.action`；HTTP method 只保存为元数据，不进入 key。`rest` 自动生成 `read/create/replace/delete`，自定义 API 使用声明 action。public API 不生成权限项。 +- 权限目录由构建产物携带，并在应用初始化时幂等同步到控制数据库 `_dever_permission`；移除的 API 标记 inactive，不直接删除授权历史。源码、组件和业务代码不注册权限。 +- 官方 auth runtime 提供租户内 `_dever_auth_role`、`_dever_auth_role_permission`、`_dever_auth_user_role`。用户只通过角色取得权限，多角色取并集；不支持直接用户授权、deny、角色继承或权限优先级。`all_permissions` 仅覆盖同一站点的 active 权限。 +- verify 只建立 `id/user_id/tenant_id` 可信身份，不返回权限列表。Runtime 针对当前 permission key 做索引化 EXISTS 查询，不把大量权限写入 JWT、Session 或请求上下文。`dever.auth` 只公开身份读取及签发/清理凭据能力。 +- API 权限只决定能否调用动作。资源归属、状态流转和其它数据权限仍由 App 读取可信身份后调用纯 Domain 规则；Domain 不读取隐式身份。未认证返回 401，接口权限或业务授权失败返回 403。 +- 多租户只保留 database 隔离。配置改为 `tenant.database`，删除 `tenant.mode` 和 `tenant.control_database`，不保留兼容分支。 +- 编译器自动识别组件：拥有 tenant Model 或 tenant Job 的组件属于租户组件，其余为平台组件；混合组件按租户组件处理。租户组件默认启用，控制库 `_dever_tenant_component` 只保存禁用例外。入口携带静态可达组件集合，Runtime 在 API/CMD/Job 入口统一检查。 +- User Job 保存可重新验证的身份引用和一个发起操作 permission key，不保存权限集合；每次执行前重新 verify，并按当前角色关系复核该 permission。旧 capability schema 的活跃 User Job 在迁移时阻断，不能降级为 System。 +- `_dever_permission` 和 `_dever_tenant_component` 是 Runtime 私有控制表；角色表由官方 auth runtime 统一拥有。项目仍拥有 User、Session、Tenant 业务 Model。 + +## 2026-09-26 已批准修订 + +用户已批准在备份后实施完整站点、可信身份、多用户权限和多租户方案。本节替代下文涉及身份、租户、入口配置的旧限制；尚未完成的字段绑定、Cookie、标准错误和上传仍属任务，不因新增范围而消失。 + +- `sites.<key>.path` 是每个组件/领域 `api/` 下的相对源码目录，递归包含子目录；不是 URL 前缀。`admin` 只匹配 `api/admin/**`，不能匹配同名 `api/admin.dever`。URL 保持现有组件/领域/API topic 生成规则，不增加源码 `site` 声明。站点匹配遗漏、歧义和路由冲突在服务开始前拒绝。 +- `public` 只修饰 HTTP API 声明，普通接口默认认证。站点选择配置中的认证 Provider，Host 可进一步约束入口；客户端提供的站点/用户/租户值不能自行成为可信身份。 +- Provider 验证签名和站点凭据后，调用静态检查的业务 App verify，检查账号、会话、成员关系并建立只读 `site.key`、`auth.id/user_id/tenant_id`。优先复用已有账号领域，不要求新建 `auth` 领域或 `admin` 组件。 +- 身份、接口权限和数据范围分别检查。接口权限由 Runtime 自动校验，数据范围由业务 App/Domain 处理，所有入口不能绕过对应边界。 +- 开启多租户后，普通持久化 Model 属于当前租户，`global` Model 使用平台库；只支持 database 分库。不影响普通 DTO，不因缺租户而回退平台库。CRUD、REST、统计、关联和批量路径共用隔离合同。 +- 租户连接池有界且可回收；初始化/迁移属于受控租户生命周期。跨库引用保留类型但不能隐式建立物理外键/Join；自动事务只覆盖一个物理数据库。隔离模式切换必须经过数据迁移。 +- 租户功能开通和规则配置属于业务数据；实现差异复用已编译 Port/Adapter。禁止租户 ID 分支、复制整套项目或执行数据库内代码。 +- Job 原子入队并保存受控租户/执行身份，执行时恢复并复核资格；CMD 不能通过自报用户 ID 伪造身份。日志关联站点/身份/租户而不泄漏凭据。 +- 所有部署配置仅来自 `config/setting.json`，无环境变量。保留并应用 log，runtime 可选并按能力选择服务，移除无用途 app 配置及迁移其工具消费者。 +- 同步 Dever/Markdown CMS，验证站点目录、跨站凭据、两个租户相同记录 ID、不同角色、匿名边界、Job 上下文、事务回滚和平台共享数据。 + +本轮已完成并逐项比对的修改前备份：`/data/project/dever-backups/pre-auth-tenancy-20260926-rm6kL6/dever.tar.gz`。SHA-256：`e4f0666d054ca241484e8333c16270f039a26c3c616b7ddd0f0bea50476ca9b7`。 + +## Goal + +在保持固定 JSON envelope 的前提下，以 Model + `rest` 覆盖受约束的普通 CRUD，以显式 API 动词绑定 App 业务动作；同时让 API 安全承载结构化请求、会话 Cookie、显式业务错误、隐藏请求上下文和受限文件上传。本任务正在实施，以下是完整目标合同；已实现的子集以 implement.md 为准。 + +## Dependencies + +- `secure-data-contracts`：Secret Cookie/token 和私密输出拒绝。 +- `time-typed-codec`：结构化请求/响应和完整标量 codec。 +- `port-adapter`：上传后的对象存储由业务 App 调用 storage Port。 + +## Requirements + +- API 使用 `get`、`post`、`put`、`delete`、`cmd` 和 `rest` 声明；移除 `get_`/`post_`/`delete_` 前缀语法，不加入 `patch`。同领域唯一主 Model 可由无参数 `rest` 推导，自动路径为 `/<component>/<domain>`；自定义动作以 `post publish = app.publish` 等形式绑定 App 签名。 +- 纯 CRUD 无需 App；编译器生成的 `rest` 才可直接降低到同领域 Model 操作。手写 API 不直接调用 Model；需要业务处理时调用同领域 App。App 中所有函数都是领域公开能力，私有辅助逻辑归 Domain，不增加 `expose` 或 `crud ... on model` 声明。 +- Model 普通字段按名称推导 CRUD 输入输出；只有例外才声明 `create`、`replace`、`search` 或 `owner` 字段绑定。`owner <field> = <context-read expression>` 为自动 REST 创建赋值及 GET/PUT/DELETE 行范围，客户端不能覆盖。字段绑定表达式只能使用静态类型可检查的纯计算或只读上下文能力，禁止数据库、网络和写 Effect；它们不隐式作用于手写 App 的 Model 调用。 +- `get` 调用链禁止数据库写 Effect；`post`、`put`、`delete` 写调用链默认同一事务。`cmd`、Job 的数据库写入也按 Effect 建立事务，不依赖 HTTP 动词。无需在 App 源码重复声明 `transaction`。 +- 源码身份固定为 `<component>/<domain>/<role>.dever[.md]`，role 只接受 `app/domain/model/port/adapter/api/job`。一个领域优先每个角色一个文件；按主题拆分时非 API 主题只允许一层。App、Domain、Port、Adapter、Job 的根文件与主题目录互斥；主 `model.dever` 可与附属 `model/<topic>.dever` 共存，基础 `api.dever` 可与路径分组 `api/**` 共存。主题必须表达业务含义，不按函数机械拆文件；搬动 Model 文件可能改变持久化身份，不能作为无迁移的整理。 +- `deverc run/build` 的目标入口由编译器生成，不要求应用编写 `main.dever`。无命令参数时按 `config/setting.json` 启动 API/worker；指定 CMD 时命令行 JSON 由该 CMD 的静态签名解码。没有可运行入口时报错；部署选择继续只读 `config/setting.json`，不用环境变量覆盖。 +- App 函数只表达有生产用途的公开业务能力；私有纯业务规则属于同领域 Domain。静态拒绝可证明无生产调用的私有函数、没有增加契约的纯转发以及只透传 Model CRUD 的 App 包装；应用构建还应拒绝不可达的 App 能力，测试调用不算生产用途。结构相似但业务含义可能不同的实现先保留警告，不用函数数量或行数配额。 +- 保留 JSON envelope：成功 HTTP 200、`code=0`、`message="ok"`；错误使用 HTTP status 对应数值 code，`data=null`。 +- GET/DELETE 接受 wire-safe scalar 和 nullable query 参数；POST/PUT JSON 使用静态类型请求体，不接受动态字段名或任意查询 Map。 +- POST multipart 接受一个包含受限 scalar/Upload 字段的 static record；JSON 和 multipart 模式由签名静态决定且互斥。 +- Runtime 隐式传递请求上下文，不在业务函数签名中声明 `dever.api.Context` 参数，也不允许存储或输出。API 的受控能力提供 request id、method/path、受信任边界内的 client address、受限 Header 读取、Text/Secret Cookie 读取，以及受控 response Header/Secure Cookie 写入；身份解析仍由业务实现。 +- Header name/value严格校验；禁止应用设置Content-Length、Transfer-Encoding、Connection和原始Set-Cookie。Cookie属性使用typed API并限制Domain/Path/SameSite/Secure/HttpOnly/Max-Age。 +- 自定义 API handler 必须把业务失败显式映射为标准 `dever.api.Error`：Invalid/Unauthorized/Forbidden/NotFound/Conflict/TooManyRequests。生成的 REST 使用固定输入、归属和数据库错误映射；未捕获失败返回通用 500 并只写脱敏结构化日志。 +- API response继续直接返回静态业务值，不暴露 `dever.http.Response`，不允许自定义绕过envelope的JSON。 +- Upload采用流式读取到runtime-owned临时文件或等价有界spool，不将大文件整体复制到内存；限制总字节、单文件、part数、字段数、header和filename长度。 +- Upload filename不作为路径，MIME只作声明且需业务/Adapter复核；拒绝路径穿越、重复字段和不完整multipart。 +- Upload handle为affine资源：必须由批准storage操作消费或显式close；正常、失败、timeout、取消和进程退出均清理未接管临时文件。 +- 登录、Session 和资源级授权由项目 App/Domain 实现；接口 RBAC 由官方 auth runtime 自动执行，不增加业务 middleware。 + +## Acceptance Criteria + +- [x] `rest` 自动生成 GET 列表/详情、POST、PUT、DELETE；显式动词/CMD 和固定 envelope 在 Dever 与 Markdown 源码中一致，DateTime 等标量按 shared codec 格式。 +- [x] 路径、角色、Model 自动识别、字段绑定 Effect、`owner` 的所有 REST 读写范围和无歧义规则均有正反例；手写 App Model 调用不受 REST 字段绑定的隐藏影响。 +- [x] GET 写入被静态拒绝，POST/PUT/DELETE 及 CMD/Job 写调用链的提交、错误回滚和取消行为有定向验证。 +- [x] POST/PUT record 和 GET/DELETE query/path 严格验证 duplicate/unknown/missing/type/size；非法输入返回 400 固定 envelope。 +- [x] 请求上下文由 runtime 隐式传递，客户端无法伪造；源码不得声明、存储或返回原始 Context。 +- [x] Cookie parse/write覆盖Secure/HttpOnly/SameSite/Path/Max-Age，header injection和受禁header拒绝。 +- [x] Secret session token不能日志/JSON/string泄漏，但可通过typed secret cookie sink发送。 +- [x] 标准API Error映射到400/401/403/404/409/429；未映射App failure返回脱敏500。 +- [x] multipart正常上传不全量驻留内存；越界、畸形、取消和handler失败后无临时文件残留。 +- [x] 手写 API 只绑定同领域 App；自动 REST 仅通过编译器生成的 Model 路径。认证示例由业务解析会话并提供只读身份能力。 +- [x] HTTP/1和HTTP/2已有deadline、request id、connection drain回归不退化。 +- [x] Model/API 根文件和主题目录共存、其他 role 冲突、无 main 的 run/build/CMD JSON、缺少入口以及无意义私有/App 包装都有正反测试；合法单用途业务规则和库检查不被误拒绝。 + +## Out Of Scope + +- GraphQL、gRPC、WebDAV和任意动态路由。 +- 外部 IAM、OAuth Provider、直接用户权限、deny、角色继承和复杂策略语言。 +- front CORS策略和浏览器页面。 +- 断点续传、分片合并、病毒扫描和对象存储协议；由后续业务Port/Adapter扩展。 diff --git a/.trellis/tasks/09-20-production-api/task.json b/.trellis/tasks/09-20-production-api/task.json new file mode 100644 index 0000000..bb579ca --- /dev/null +++ b/.trellis/tasks/09-20-production-api/task.json @@ -0,0 +1,26 @@ +{ + "id": "production-api", + "name": "production-api", + "title": "Dever 生产 API 边界", + "description": "补齐结构化请求、请求上下文、Cookie、响应 Header 与受限上传能力", + "status": "in_progress", + "dev_type": null, + "scope": null, + "package": null, + "priority": "P1", + "creator": "Antigravity", + "assignee": "Antigravity", + "createdAt": "2026-09-20", + "completedAt": null, + "branch": "main", + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [], + "parent": "09-20-production-application-language", + "relatedFiles": [], + "notes": "", + "meta": {} +} \ No newline at end of file diff --git a/.trellis/tasks/09-20-production-application-language/check.jsonl b/.trellis/tasks/09-20-production-application-language/check.jsonl new file mode 100644 index 0000000..05738cb --- /dev/null +++ b/.trellis/tasks/09-20-production-application-language/check.jsonl @@ -0,0 +1,3 @@ +{"file":".trellis/spec/backend/quality-guidelines.md","reason":"Focused verification and no-full-suite policy"} +{"file":".trellis/spec/backend/error-handling.md","reason":"Diagnostics, failure propagation and cleanup review"} +{"file":".trellis/spec/backend/directory-structure.md","reason":"Owner boundaries and duplicate implementation review"} diff --git a/.trellis/tasks/09-20-production-application-language/design.md b/.trellis/tasks/09-20-production-application-language/design.md new file mode 100644 index 0000000..6a0299f --- /dev/null +++ b/.trellis/tasks/09-20-production-application-language/design.md @@ -0,0 +1,113 @@ +# Dever 生产级大型应用语言支持设计 + +## 1. Delivery Shape + +父任务只拥有跨阶段合同、子任务顺序和最终集成验收，不直接实施源码。六个子任务按下列依赖执行： + +```text +secure-data-contracts ---+ + +--> port-adapter --------+ +time-typed-codec --------+ +--> large-cms-acceptance + +--> durable-jobs --------+ + +--> production-api ------+ +``` + +Port 的类型化配置依赖 Secret 和 typed codec；Job 依赖 DateTime、typed codec 和数据库事务；生产 API 依赖 Secret、typed codec 和既有 HTTP 引擎。CMS 只在所有语言能力分别通过后扩展。 + +## 2. Source Architecture + +最终角色矩阵： + +| Caller | Allowed application targets | +| --- | --- | +| Generated entry | configured API/Job startup and shutdown | +| handwritten API | own-domain App | +| generated REST | own-domain Model operations under checked field/access rules | +| Job | own-domain App | +| App | own Domain, own Model operations, own Port, any App | +| Domain | own Domain | +| Model | no business functions; field bindings are checked at generated REST entry | +| Port | declarations only | +| Adapter | implemented Port target, same-file helpers, bundled packages | +| Test | same-domain App/Domain, any App, test-local Port fakes | + +Type visibility is narrower than call visibility. App contracts may use public business records and Model IDs, but cannot expose private Model fields, Adapter/Domain implementation types or non-observable runtime resources. Port types remain domain-private. Job/API types must pass the shared wire-schema validator. + +## 3. Shared Static Value Schema + +The checker derives one internal wire schema from existing concrete `Type` and `Definition` data. It is compiler metadata, not a source-visible reflection API. Three emitters consume it: + +- setting decoder: strict JSON object to concrete Adapter/runtime settings; +- Job codec: concrete payload to/from persisted JSON bytes plus schema fingerprint; +- API codec: query/JSON request decode and response encode. + +Private fields, Secret and resources are represented in the schema policy, not rediscovered by each emitter. The runtime may reuse bounded JSON token parsing and scalar conversion, but generated programs retain concrete Rust values; no universal application `Value` interpreter crosses business calls. + +## 4. Secure Values And Model Privacy + +`Secret` is an opaque source scalar with no Render, comparison, ordinary JSON, Map-key or Model storage implementation. Approved constructors are secure random/token generation and typed setting/API decoding. Approved consumers are password/HMAC operations and narrowly typed transport sinks such as secure Cookie or bearer-auth construction. Secret cannot be converted to Text by a generic helper. + +Model `private` fields remain stored normally but receive domain-owned access metadata. Generated CRUD blocks in the owning App may assign them; field projection is available only to that App. A type containing a private Model field is rejected at App public signatures, API/Job wire boundaries and test assertions. This keeps the existing Model record internally useful without adding DTO directories; App declares explicit view records beside capabilities. + +Password hashing uses a maintained Argon2id implementation with runtime-selected secure salt and encoded hash text. Verification accepts Secret plaintext plus stored private hash and returns Bool/failure without exposing intermediate bytes. Security parameters are named runtime constants with tests, not application configuration knobs. + +## 5. Port And Adapter Dispatch + +Port role declarations use bodyless function signatures and explicit allowed failure choices. Adapter/Test sources may define a qualified Port implementation; ordinary unqualified functions remain local helpers. The checker builds one Port contract and statically verifies implementation inputs, named outputs and effective failure subsets. + +Single implementation binds directly. Multiple implementations receive a compiler-generated closed enum and direct match; `config/setting.json` selects one stable implementation identity. This is runtime selection among compiled functions, not dynamic linking or trait-object business dispatch. Only the selected implementation's setting object is required and decoded. + +Test-local fakes are associated with the owning TestCase. The single compiled test suite contains all fakes, while the selected case installs only its own compile-checked binding before entry. Production bindings and project deployment settings are not read by application tests. + +## 6. Durable Job Data Flow + +Each Job contract has a stable logical identity, concrete input shape, schema fingerprint, logical database binding, timeout and retry policy. Job inputs are zero or one wire-safe record; outputs are empty. Job handlers are registered entries and cannot be called as ordinary functions. + +```text +App write call under an automatically owned entry transaction + -> typed enqueue/schedule(job target, payload, dedupe key) + -> hidden transaction context writes _dever_job row + -> business rows and job row commit together + +Job worker + -> atomically claims due row with lease + -> decodes exact schema + -> calls own-domain App + -> marks success, schedules retry, or moves to dead-letter +``` + +SQLite uses a short write transaction and compare-and-update claim. PostgreSQL uses row locking/skip-locked semantics and the existing typed driver boundary. Leases have owner and expiry so a terminated worker does not permanently lose a task. A completed external side effect followed by process failure may execute again; this is documented at-least-once behavior. + +Immediate and one-shot scheduled tasks are inserted by App calls. Recurring UTC schedules are compile-time validated declarations and materialize deterministic slot keys so multiple workers do not create duplicate occurrences. Payload signature drift never falls back to best-effort decoding; incompatible work is retained with an explicit failure reason. + +## 7. Runtime Modes And Shutdown + +`config/setting.json` gains strict runtime/job sections. A single compiled program supports: + +- `api`: API server active, Job worker does not claim work; +- `worker`: Job worker active, API listener is not opened; +- `all`: both active for local/small deployments. + +The target source contract no longer requires `main.dever` or source-level `serve()` calls. The compiler generates API/Job startup for the configured mode and owns their bounded group, shutdown and resource cleanup. Disabled services return without opening resources. This is a planned migration; the current compiler still uses Main. + +The process signal owner is runtime-level and broadcasts to active services. API closes its listener and reuses existing connection drain. Job stops polling/claiming and drains active leases to its deadline. The generated entry then closes databases and flushes logs. Service failure cancels and drains its Group siblings. + +## 8. Production API + +API route discovery remains compiler-owned. The target syntax uses `get/post/put/delete/cmd/rest`; `rest` lowers plain CRUD to the same-domain Model, while custom actions bind the same-domain App signature. POST/PUT accept typed bodies; query routes accept named wire-safe scalars. A request Context is hidden in the runtime rather than declared as a source parameter. Validated transport observations and bounded Cookie/Header mutation are available through controlled API capabilities; no raw request/response handle reaches App. The detailed contract is owned by the `09-20-production-api` child task. + +Handler success is encoded into the existing `{code,message,data}` envelope. Recoverable App failures must be captured and mapped to a standard `dever.api.Error`; uncaptured failures produce a generic internal envelope and structured, secret-free log. This keeps business errors independent of HTTP while allowing explicit 400/401/403/404/409/429 mapping. + +Uploads are streamed to an owned temporary file under runtime data/tmp with checked total size, part count, filename and MIME metadata. API receives an opaque upload handle/metadata, and a storage Port consumes it. Drop, cancellation and handler failure remove unclaimed temporary files. The CMS does not expose arbitrary filesystem paths. + +## 9. CMS Integration + +The maintained CMS is the cross-feature acceptance application, not a second implementation of framework logic. Plain and Markdown forms retain equal normalized declarations. Plain CRUD uses generated REST; domains own explicit App views and commands when business behavior requires them. No dto/service/repository folders are introduced. + +Authentication uses Secret password input, private password hash storage, hashed session token storage and secure Cookie output. Media uses a storage Port with a local production Adapter and test fake. Publishing writes the state transition, audit row and scheduled Job atomically. Job execution rechecks state/idempotency before publishing and can retry safely. + +## 10. Compatibility And Rollback + +This language is pre-release and repository callers migrate atomically; no legacy aliases, environment fallback, old JSON date encoding or dual role matrix remains. Each child task is a rollback boundary. A child is not accepted while canonical docs, Markdown contracts or maintained CMS callers use two protocols. + +No database destructive migration is run against user data during compiler tests. Runtime private Job schema uses additive/versioned initialization and test-owned databases. Real PostgreSQL verification requires an explicitly configured isolated database. diff --git a/.trellis/tasks/09-20-production-application-language/implement.jsonl b/.trellis/tasks/09-20-production-application-language/implement.jsonl new file mode 100644 index 0000000..2a7916c --- /dev/null +++ b/.trellis/tasks/09-20-production-application-language/implement.jsonl @@ -0,0 +1,4 @@ +{"file":".trellis/spec/backend/compiler-contracts.md","reason":"Source roles, visibility, effects, API and application contract baseline"} +{"file":".trellis/spec/backend/database-guidelines.md","reason":"Model privacy, transactions, settings and SQLite/PostgreSQL invariants"} +{"file":".trellis/spec/backend/toolchain-and-library.md","reason":"Official package, native runtime and lifecycle contracts"} +{"file":".trellis/spec/backend/logging-guidelines.md","reason":"Secret-free logging and HTTP request boundary"} diff --git a/.trellis/tasks/09-20-production-application-language/implement.md b/.trellis/tasks/09-20-production-application-language/implement.md new file mode 100644 index 0000000..43e3f79 --- /dev/null +++ b/.trellis/tasks/09-20-production-application-language/implement.md @@ -0,0 +1,90 @@ +# Dever 生产级大型应用语言支持执行计划 + +## Start Gate + +- [x] 用户审阅并明确批准本 PRD、设计和执行顺序。 +- [x] 父任务保持 planning；不把父任务作为实现目标。 +- [x] 依次完善并启动拥有当前交付物的子任务。 +- [x] 开始每个子任务前读取 `trellis-before-dev`、子任务文档和 backend specs。 +- [x] 保留现有脏工作区和历史任务，不提交、不归档、不清理用户改动。 + +## Child Order + +### 1. secure-data-contracts + +- [x] Model private 字段的领域所有权和公共边界。 +- [x] Secret 类型、批准 sink、泄漏拒绝。 +- [x] secure random、Argon2id、SHA-256/HMAC、常量时间验证。 +- [x] 删除应用 `dever.process.environment`，迁移仓库调用方和文档。 +- [x] 定向安全、Model、API/log 边界测试。 + +### 2. time-typed-codec + +- [x] UTC DateTime/Date/Time/Duration API 和无溢出运算。 +- [x] 共享静态 wire schema 与具体编码/解码生成。 +- [x] 严格 JSON、日期格式、大小/深度和私密/Secret 拒绝。 +- [x] 迁移当前 API response codec，保持 envelope。 +- [x] 定向 codec、API、Markdown、ORM 表示回归。 + +### 3. port-adapter + +- [x] Port bodyless contract、明确 failure contract 和 formatter/Markdown 文档。 +- [x] Adapter/test fake 实现检查与静态 dispatch。 +- [x] 收紧角色调用矩阵和类型可见性。 +- [x] setting.json 多实现绑定、Adapter-local typed setting 和 Secret 字段。 +- [x] 单实现、多实现、错误绑定、fake 隔离和 effect/failure 回归。 + +### 4. durable-jobs + +- [x] Job role、声明、目标引用、payload schema 和 effect 分析。 +- [x] SQLite/PostgreSQL 私有 schema、原子入队、claim/lease/retry/dead-letter。 +- [x] immediate、run-at、UTC recurring schedule 和 schema drift 行为。 +- [x] `dever.job.serve()`、runtime mode、signal/drain 和数据库关闭顺序。 +- [x] 应用测试隔离、SQLite 端到端和 PostgreSQL 定向编译/可选隔离验收。 + +### 5. production-api + +- [ ] POST record/query scalar decode 与共享 codec 接入。 +- [ ] 注入 Context、Cookie/Header/request metadata。 +- [ ] 标准 API Error 映射和未映射失败脱敏。 +- [ ] 受限上传、临时文件生命周期和 storage Port 路径。 +- [ ] 路由、输入、错误、上传取消和 HTTP drain 回归。 + +### 6. large-cms-acceptance + +- [ ] 扩展 plain/Markdown CMS 领域和等价文档。 +- [ ] 实现账户、会话、权限、内容、修订、分类、媒体、发布和审计业务。 +- [ ] 接入 secure values、Port/Adapter、Job、API 和 runtime all mode。 +- [ ] 增加等价应用测试和无外部服务 fake。 +- [ ] 执行最终定向 check/test；任何 run/build 或持久服务验证先向用户说明。 + +## Cross-child Gates + +- [ ] 每个阶段的公共合同只有一个 owner；JSON、配置、Job payload 不复制类型转换。 +- [ ] 新增 Intrinsic 时 checker、effects、native emitter、reference policy 和官方 package 同步穷尽。 +- [ ] `.dever` 与 `.dever.md` 使用同一 AST/check/HIR，不增加 Markdown 专用语义。 +- [ ] 所有长期测试位于仓库根 `test/`，一次性 fixture 在交付前删除。 +- [ ] 任何配置只来自 `config/setting.json`；搜索确认无应用环境变量兼容分支。 +- [ ] 不添加 front、分发、外部队列、动态插件或内置 RBAC。 + +## Focused Validation Policy + +每个子任务在自己的 `implement.md` 中列出具体命令。共同允许： + +```bash +cargo check --offline -p dever-core -p dever-runtime -p dever-cli +cargo test --offline -p dever-tests --test <affected-target> +git diff --check +python3 ./.trellis/scripts/task.py validate <child-task> +``` + +如 rustfmt/clippy 组件存在，再运行受影响 package/target 的检查；组件缺失时如实记录。默认不运行 workspace 全量测试、性能套件、真实 PostgreSQL、front build、全量应用 build 或持久服务。 + +## Parent Completion Gate + +- [ ] 六个子任务分别完成质量检查、规范同步和归档。 +- [ ] 父任务重新核对所有 Requirement 与跨子任务数据流。 +- [ ] CMS plain/Markdown 的 schema/API/Job/Port/settings/test 快照一致。 +- [ ] 搜索确认不存在旧环境变量入口、App 直调 Adapter、公开私密字段或无界后台任务。 +- [ ] 汇总实际通过、跳过和受阻的验证，不把定向测试描述成全量证明。 +- [ ] 未经用户明确授权不创建 Git commit。 diff --git a/.trellis/tasks/09-20-production-application-language/prd.md b/.trellis/tasks/09-20-production-application-language/prd.md new file mode 100644 index 0000000..1a638dd --- /dev/null +++ b/.trellis/tasks/09-20-production-application-language/prd.md @@ -0,0 +1,102 @@ +# Dever 生产级大型应用语言支持 + +## Goal + +在不引入 front、编译器分发或任意动态插件系统的前提下，补齐 Dever 编写生产级大型后端所需的安全数据、时间与静态编解码、Port/Adapter、持久化 Job、生产 API 和大型 CMS 验收能力。最终应用保持路径驱动、静态类型、显式副作用和 `config/setting.json` 单一部署配置来源，AI 生成代码不能通过新增随意目录、直接调用实现层或脱离作用域的后台任务绕过架构边界。 + +## Background + +- 当前应用架构已经支持 `main/app/domain/model/port/adapter/api` 路径角色，但 `port` 仍拒绝所有声明，尚无 `job` 角色。 +- 当前 App 可以直接调用同领域 Adapter，Domain/Model/Port/Adapter 之间的调用范围也过宽，不能形成真实可替换边界。 +- 当前 Model 拒绝私密字段；CMS 的密码哈希仍是普通公开 Model 字段。 +- 当前 API 只接收 Text/Int/Bool 参数，无法安全承载登录、Cookie、结构化输入和上传。 +- `config/setting.json` 的 `app`、`log` 对象当前没有运行时合同，应用仍能调用环境变量读取函数。 +- DateTime 等类型已有数据库表示，但缺少 UTC 构造、解析、格式化和运算；API 输出仍使用内部整数表示。 +- HTTP 底层已有 listener close 和连接排空；本任务只补进程信号、API/Job 组合生命周期，不重写 HTTP 引擎。 +- 应用测试体系已经具备根 `test/`、类型安全断言、独立进程、隔离 SQLite fixture 和单套件编译缓存，可承接新能力的长期回归。 + +## Requirements + +### R1. Stable Application Architecture + +- 应用源码角色最终限定为 `app`、`domain`、`model`、`port`、`adapter`、`api`、`job`；不增加 service、repository、controller、dto、middleware、common、shared、internal 或 manager 目录。 +- 手写 API/Job 只调用同领域 App；自动 REST 由编译器生成同领域 Model 操作；App 调用同领域 Domain/Model/Port 和其他领域 App；Domain 只调用同领域 Domain；Adapter 只实现 Port 并调用标准库或自身私有辅助函数。`deverc run/build` 的应用入口由编译器生成，不要求源码 `main.dever`。 +- Model 不声明业务函数；Port 不含实现；API 和 Job 不承载业务规则；跨领域只能调用 App。 +- 单文件与同名角色目录继续互斥；除 API 可按 URL 嵌套外，角色主题目录保持平铺且单主题文件不得机械拆分。 + +### R2. Secure Data And Configuration + +- Model 支持领域私密字段；只有拥有领域的 App 和编译器生成的 Model 代码可以读写，其他领域、API、Job、测试和公开 App 合同不能观察该字段。 +- App/API/Job 输出和日志不能包含私密字段或 Secret；静态检查必须在生成本地代码前拒绝。 +- 提供不渲染、不可比较、不可序列化到普通 JSON 的 Secret 值，以及安全随机 token、Argon2id 密码 hash/verify、SHA-256/HMAC 和常量时间验证能力。 +- Secret 只能进入编译器认可的安全 sink；不得通过字符串拼接、日志、普通 HTTP Header 构造或错误信息隐式解密。 +- 部署配置只来自可执行文件相邻的 `config/setting.json`。移除应用层环境变量读取；命令行参数保留为显式程序输入，但不能覆盖应用配置。 +- Adapter 和运行时配置具有静态 schema、未知字段拒绝、Secret 字段保护和启动前验证；App/Domain 不直接读取部署配置，业务可变数据继续放 Model。 + +### R3. Time And Shared Typed Codec + +- 后端时间统一使用 UTC；提供 DateTime 当前时间、RFC3339 解析/格式化、Date/Time 解析/格式化和 DateTime/Duration 安全运算。 +- 数据库存储继续使用整数毫秒，不改变既有 SQLite/PostgreSQL 列表示。 +- 编译器建立一套静态类型值 schema 和具体代码生成路径，供 Adapter setting、Job payload、API 输入/输出共同使用；不得增加运行时反射 Value 解释器。 +- JSON 解码拒绝重复键、未知字段、越界数值、错误日期和过深/过大的输入；编码拒绝私密字段、Secret、资源、Task、Stream 和非有限 Float。 +- DateTime 的外部 JSON 表示为 UTC RFC3339 字符串，Decimal/Id/Uuid 保持无损文本表示，Duration 明确使用整数毫秒。 + +### R4. Port And Adapter + +- Port 源只声明类型、签名和允许失败，不包含函数体；App 通过 Port 调用外部边界。 +- Adapter 显式实现 Port，签名必须完全匹配，实际失败必须落在 Port 声明的集合内；不依赖运行时 trait object 或动态代码加载。 +- 单实现自动选择；多实现由 `config/setting.json` 按稳定 Port 身份选择。缺失、未知或不完整绑定在启动前失败。 +- 每个 Adapter 可声明自身类型化 setting；未选择 Adapter 的配置不得强制存在。 +- 应用测试可为可达 Port 提供用例私有 fake；fake 不进入生产源码、部署配置或其他测试用例。 + +### R5. Durable Job + +- 新增 `job.dever` 或平铺 `job/*.dever`；Job 声明为非源码可调用的后台入口，只能调用同领域 App，输入必须是可持久化静态类型，输出必须为空。 +- 支持立即任务、指定 UTC 时间任务和编译期校验的 UTC 周期计划。 +- Job 采用数据库持久化、租约领取、并发上限、执行超时、有界退避重试、dead-letter 和崩溃后租约恢复；语义明确为至少一次执行。 +- Job payload 保存稳定任务身份和 schema 指纹。部署后遇到不兼容 payload 时进入明确失败状态，不静默误解码。 +- App 事务内入队必须复用同一逻辑数据库事务；Job 与业务事务连接不一致时编译失败，避免业务提交成功但任务丢失。 +- 任务表、租约和调度状态属于运行时私有 schema，不要求业务创建 Job Model。 +- 同一产物支持 `api`、`worker`、`all` 三种运行模式，仅由 `config/setting.json` 选择；编译器生成对应的 API/Job 启动与排空入口，不要求应用手写 `main()` 或 `serve()` 调用。 +- SIGTERM/Ctrl-C 停止接收新请求和领取新任务，在配置的期限内排空 HTTP 与在途 Job，再执行数据库和日志关闭。 + +### R6. Production API Boundary + +- 使用 `get/post/put/delete/cmd/rest` 声明，保留固定 `{code,message,data}` JSON envelope，不返回低层 `dever.http.Response`。唯一主 Model 的 `rest` 自动生成受约束的 CRUD，手写 API 绑定同领域 App。 +- POST/PUT 支持静态类型请求体；GET/DELETE 支持完整可表示标量的 query/path 输入；复用共享 typed codec 和严格未知字段检查，不暴露动态查询 Map。 +- Model 可为自动 REST 声明受 Effect 约束的字段来源和 `owner` 行范围，普通字段由编译器推导；这些绑定不隐式改变手写 App 的 Model 操作。 +- 请求 Context 在 runtime 隐式传递；受控 API 能力读取 Header/Cookie/request ID/client address，并设置 Cookie/响应 Header。原始 Context 不出现在源函数签名，也不得存储或输出。 +- GET 调用链禁止数据库写入；POST/PUT/DELETE 和 CMD/Job 的写调用链自动建立事务，业务源码无需重复标记 `transaction`。 +- API 失败必须显式映射到标准 API Error，未映射业务失败继续视为 500 并脱敏记录。 +- 上传使用有总量、单文件、字段数、MIME 和文件名限制的流式/临时文件边界；拒绝路径穿越，临时资源在成功、失败和取消后均清理。 +- 鉴权、会话和授权属于业务 App/Domain；不增加 middleware 业务层或框架内置 RBAC。 + +### R7. Large CMS Acceptance + +- 同步维护 `examples/cms/dever` 与 `examples/cms/md`，两者具有等价声明、Model schema、API、Job、Port binding 和测试合同。 +- CMS 至少覆盖 user/account、user/session、user/access、content/article、content/revision、content/category、media/asset、publishing/schedule 和 audit/operation。 +- 验证密码 hash、Secret token、私密字段、会话 Cookie、权限判断、媒体存储 Port、fake Adapter、事务内发布任务、定时发布、失败重试和审计记录。 +- CMS 不引入 front；普通 CRUD 只使用编译器生成的 `rest`，不写透传 App/API 函数。登录、命令、上传等业务入口仍通过手写 API 绑定 App。 + +## Acceptance Criteria + +- [ ] 非法角色路径、App 直调 Adapter、Domain 访问 Model/Port/Adapter、API/Job 跨域调用等在 `deverc check` 阶段稳定拒绝。 +- [ ] Model 私密字段可由所属 App 持久化和验证，但无法进入 App/API/Job 输出、日志、其他领域或测试观察面。 +- [ ] Secret、密码和安全随机能力有成功、错误和泄漏拒绝回归；应用源码无法读取环境变量配置。 +- [ ] UTC 时间和静态 typed codec 对 `.dever`/`.dever.md` 结果一致，API、setting 和 Job 不维护平行编解码逻辑。 +- [ ] Port 单实现、多实现配置选择、缺失绑定、签名/失败不匹配和测试 fake 均有正反测试。 +- [ ] SQLite Job 可验证原子入队、租约恢复、延迟/周期调度、重试、dead-letter、payload 不兼容和优雅停止；PostgreSQL 代码路径通过定向编译，真实 PostgreSQL 只在隔离配置存在时验收。 +- [ ] `api`、`worker`、`all` 三种模式只读取 `config/setting.json`，无环境变量或命令行配置覆盖。 +- [ ] 生产 API 验证结构化输入、标准错误、Cookie/Header、请求上下文和受限上传清理，仍保持固定 JSON envelope。 +- [ ] Dever 与 Markdown CMS 的 check、test 和行为合同一致；最终定向运行验证不依赖 front 或外部服务。 +- [ ] 每个子任务只运行对应最小离线检查；不运行 workspace 全量测试、默认性能压测、真实外部服务或未说明的持久服务。 + +## Out Of Scope + +- front、Page JSON、React 插件和浏览器界面。 +- 私有编译器分发、交叉编译、自托管、包注册表和动态插件加载。 +- 内置用户/RBAC/租户业务模型；这些由 CMS App/Domain 实现。 +- Exactly-once 外部副作用保证；Job 明确采用至少一次语义，业务动作负责幂等。 +- 任意 cron 时区和 DST 规则；周期计划首版固定 UTC。 +- Redis、Kafka、RabbitMQ 或外部队列依赖；首版复用 SQLite/PostgreSQL。 +- 全量测试、默认真实 PostgreSQL、64/128 MiB 压测和 front 构建。 diff --git a/.trellis/tasks/09-20-production-application-language/task.json b/.trellis/tasks/09-20-production-application-language/task.json new file mode 100644 index 0000000..2aea388 --- /dev/null +++ b/.trellis/tasks/09-20-production-application-language/task.json @@ -0,0 +1,34 @@ +{ + "id": "production-application-language", + "name": "production-application-language", + "title": "Dever 生产级大型应用语言支持", + "description": "补齐 Port/Adapter、安全基础、持久化 Job、生产 API 与大型双源码 CMS 验收，front 和编译器分发延期", + "status": "planning", + "dev_type": null, + "scope": null, + "package": null, + "priority": "P1", + "creator": "Antigravity", + "assignee": "Antigravity", + "createdAt": "2026-09-20", + "completedAt": null, + "branch": null, + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [ + "09-20-secure-data-contracts", + "09-20-time-typed-codec", + "09-20-port-adapter", + "09-20-durable-jobs", + "09-20-production-api", + "09-20-large-cms-acceptance", + "09-29-external-libs" + ], + "parent": null, + "relatedFiles": [], + "notes": "", + "meta": {} +} \ No newline at end of file diff --git a/.trellis/tasks/09-20-secure-data-contracts/check.jsonl b/.trellis/tasks/09-20-secure-data-contracts/check.jsonl new file mode 100644 index 0000000..49ed001 --- /dev/null +++ b/.trellis/tasks/09-20-secure-data-contracts/check.jsonl @@ -0,0 +1,3 @@ +{"file":".trellis/spec/backend/quality-guidelines.md","reason":"Focused compiler/runtime test selection"} +{"file":".trellis/spec/backend/error-handling.md","reason":"Typed crypto failure and diagnostic rules"} +{"file":".trellis/spec/backend/directory-structure.md","reason":"Single owner across checker, emitter and runtime"} diff --git a/.trellis/tasks/09-20-secure-data-contracts/design.md b/.trellis/tasks/09-20-secure-data-contracts/design.md new file mode 100644 index 0000000..f3f4072 --- /dev/null +++ b/.trellis/tasks/09-20-secure-data-contracts/design.md @@ -0,0 +1,29 @@ +# Dever 安全数据合同设计 + +## Model Privacy + +复用现有字段 `private` 语法，不增加 Model 专用修饰符。类型检查为字段记录声明 owner 的 domain identity：普通 record private 继续 package-owned，Model private 改为 domain-App-owned。Model 生成 CRUD 的匿名 create/update block允许所属 App 写 private 字段，但投影、记录构造和公开边界仍由统一字段访问检查控制。 + +App 是跨领域能力边界，因此任何直接或嵌套包含 private Model 字段的 nominal record 都不能出现在 App 参数、输出或可传播错误中。Model ID 和不含 private 字段的显式 View 仍可公开。API/Job/wire 检查复用同一 `observable` 类型属性，不再各自遍历字段猜测。 + +## Secret Type + +Secret 是独立非泛型标量，内部持有受保护字节，不实现普通 Render/Clone-to-Text/JSON。值语义允许受控传参和 record 字段，但以下静态能力为 false：comparable、Map key、persistent、ordinary wire output、fault rendering。 + +构造来源： + +- `dever.crypto.token(bytes)`； +- 后续 typed setting/API Secret 字段解码； +- 必要的测试固定 Secret 构造仅存在于 compiler-owned Test source，不进入生产标准库。 + +消费 sink 由 Intrinsic 精确枚举，不能通过一个通用 reveal 函数扩散。后续 API/Adapter 新 sink 必须在 checker effect/role policy 中显式加入。 + +## Crypto Runtime + +使用固定版本的成熟 Rust crates 实现 Argon2id、HMAC/SHA-256 与 constant-time 比较；随机源复用 OS getrandom。Argon2id 参数是经过测试的运行时常量，避免应用通过 setting 降低成本。hash 输出为标准 PHC Text，verify 对格式错误返回显式失败而非 panic。 + +标准 `.dever` package 定义公开 Result choice 和简单函数；runtime 只承载密码学实现。checker 校验官方 choice shape，native emitter 复用现有 resource-result 模式，不创建第二套失败协议。 + +## Environment Removal + +从官方 `dever.process`、Intrinsic、checker effect、native emitter 和 runtime system bridge 同步删除 environment。仓库调用方整体迁移到 setting.json 或测试显式输入，不添加 feature flag 或旧名兼容。 diff --git a/.trellis/tasks/09-20-secure-data-contracts/implement.jsonl b/.trellis/tasks/09-20-secure-data-contracts/implement.jsonl new file mode 100644 index 0000000..7876e05 --- /dev/null +++ b/.trellis/tasks/09-20-secure-data-contracts/implement.jsonl @@ -0,0 +1,4 @@ +{"file":".trellis/spec/backend/compiler-contracts.md","reason":"Private fields, public boundaries and type properties"} +{"file":".trellis/spec/backend/database-guidelines.md","reason":"Model field schema and generated CRUD ownership"} +{"file":".trellis/spec/backend/toolchain-and-library.md","reason":"Official package and intrinsic implementation rules"} +{"file":".trellis/spec/backend/logging-guidelines.md","reason":"Secret-free render and log requirements"} diff --git a/.trellis/tasks/09-20-secure-data-contracts/implement.md b/.trellis/tasks/09-20-secure-data-contracts/implement.md new file mode 100644 index 0000000..f728a41 --- /dev/null +++ b/.trellis/tasks/09-20-secure-data-contracts/implement.md @@ -0,0 +1,71 @@ +# Dever 安全数据合同实施计划 + +## Start Gate + +- [x] 用户批准父任务最终规划后启动本子任务。 +- [x] 读取 compiler-contracts、database、logging、toolchain/library specs。 +- [x] 记录当前 private 字段、Model field access、Render/JSON/log 和 process intrinsic 所有调用点，见 research/secure-contracts-source-map.md。 + +## Implementation + +- [x] 扩展类型属性和字段访问 owner，允许 Model private 并限制为所属 App。 +- [x] 在现有 App/API/入口/错误/测试边界复用 observable 检查；Job 的消费检查由 durable-jobs 子任务接入相同属性。 +- [x] 增加 Secret 类型、语法类型解析、HIR/native/reference 表示和禁止能力。 +- [x] 增加 crypto 官方 package、Intrinsic、runtime 实现和依赖；保证失败类型化与错误脱敏。 +- [x] 删除 environment 官方函数、Intrinsic/runtime 分支、effect 分类、文档与调用方。 +- [x] CMS 账户使用 private password_hash: Text(255)，AccountView/ArticleView 隐藏私有值及作者关联；两套源码同步，没有新增登录 API。 +- [x] 更新 LANGUAGE/Markdown/compiler/database/logging canonical docs。 + +## Focused Verification + +```bash +cargo check --offline -p dever-core -p dever-runtime -p dever-cli +cargo test --offline -p dever-tests --test model_syntax +cargo test --offline -p dever-tests --test source_visibility +cargo test --offline -p dever-tests --test contract_semantics +cargo test --offline -p dever-tests --test standard_library +cargo test --offline -p dever-tests --test logging +cargo test --offline -p dever-tests --test markdown_source +git diff --check +``` + +如组件存在，增加受影响 package/target 的 rustfmt 和 `clippy -D warnings`。不运行全量测试、真实数据库、服务或性能测试。 + +## Review Risks + +- private Model 字段不能破坏生成 SQL 列顺序、迁移 fingerprint 或 row decode。 +- Secret 不得因派生 Render/Debug、错误路径或 generic collection 自动获得观察能力。 +- crypto 失败不得包含输入；随机源和 password verify 不得 panic。 +- 删除 environment 后必须穷尽 checker/emitter/reference/effect match，不能留不可达旧分支。 + +## Implementation Notes — 2026-09-20 + +- Secret 位于基础 runtime/secret.rs，Clone 保持值语义、Debug 固定脱敏、zeroize 清除已持有字节；argon2/sha2/hmac/getrandom/subtle 由独立 crypto feature 控制。原生构建根据已裁剪代码中的 crypto 调用选特性，也覆盖直接 system intrinsic。 +- Model 私有字段访问统一经过 Context::owns_private_fields。类型安全属性缓存沿 Related 关联图做单调反向传播，不把 ORM 关联环当成布局递归。 +- Argon2 0.5.3 官方文档核对后采用 Argon2id v19 / Params::default；verify 在计算前限制 PHC 算法、版本、成本及结果长度。执行一次 cargo fetch，锁定 12 个新增依赖包；zeroize 复用已存在的 1.9.0。 +- `secure_contracts` 新定向目标 8/8 通过（含 reference、原生与 Test source 执行）。覆盖隐私所有权/Model schema 不变、嵌套非观察边界、Related 自环、环境入口拒绝、密码错误/损坏/资源上限、SHA/HMAC 已知向量、token 独立性及脱敏 Debug。 +- 编译检查 core/runtime/cli + crypto/reference 通过。首次验证因磁盘只剩 65 MiB 导致 source_visibility 链接 Bus error，随后 rlib 报 No space left on device；主会话清理 target/debug/incremental 后恢复验证。后续使用 `--config build.incremental=false --config profile.dev.debug=0 --config profile.test.debug=0`，始终 offline + locked，原生测试串行。 +- rustfmt/clippy 组件未安装；没有安装全局工具。没有运行全量测试、真实 PostgreSQL、持久服务、CMS 打包运行或性能测试；OS 随机源不可用分支未做系统故障注入。 +- contract_semantics 完整目标初次 6/12：6 个旧样例直接构造 error variant，在既有 `construct_variant_with_kind` 禁令处收到 C005；本次没有改该 error 构造语义，也未重写这些旧样例。最终报告保留未通过，不据此宣称全目标成功。 + +## Verification Results + +| 定向验证 | 结果 | +| --- | --- | +| cargo check：core/runtime/cli + crypto/reference | 通过 | +| secure_contracts（crypto,reference，串行） | 8/8 通过 | +| model_syntax | 3/3 通过 | +| source_architecture | 8/8 通过 | +| source_visibility | 10/10 通过 | +| markdown_source | 20/20 通过 | +| standard_library | 5/5 通过 | +| logging | 1/1 通过 | +| differential::time_failures_are_source_defined_results | 1/1 通过；移除 environment 样例后保留显式 result 的时间失败 native/reference 一致性 | +| contract_execution | 首轮 7/9；两个 CMS 失败因关联传播要求 ArticleView，修复后 CMS parity 与 maintained root checker 各 1/1 通过；未重复整个目标 | +| contract_semantics | 完整目标 6/12；6 个既有 error 直接构造样例未重跑；当前 private_fields_protect_access_mutation_and_construction 精确用例 1/1 通过 | +| git diff --check / 本次文件尾随空白扫描 | 通过；项目源码目前未跟踪，因此额外执行显式文件扫描 | +| rustfmt/clippy | 未安装组件；rustfmt 启动失败，未安装或绕过 | + +contract_semantics 未通过用例：failure_cannot_be_discarded_overwritten_logged_or_laundered、errors_can_be_returned_translated_or_explicitly_recovered、failure_summaries_follow_long_calls_and_unused_handler_dependency_cycles、nested_failure_survives_value_copies_and_projection、empty_traversal_does_not_handle_context_and_singleton_projection_transfers_failure、possibly_empty_or_null_outputs_cannot_hide_another_failure。首个诊断均为 C005 `error choice variants can only be used with fail`。 + +当前没有新配置入口、没有登录业务、没有真实数据库/全量/服务/性能验收。Model create/update/读取 private 字段通过静态检查与 schema 不变验证；未据此声称已完成真实数据库运行验收。 diff --git a/.trellis/tasks/09-20-secure-data-contracts/prd.md b/.trellis/tasks/09-20-secure-data-contracts/prd.md new file mode 100644 index 0000000..a2f59a4 --- /dev/null +++ b/.trellis/tasks/09-20-secure-data-contracts/prd.md @@ -0,0 +1,37 @@ +# Dever 安全数据合同 + +## Goal + +让密码、令牌和部署密钥在 Dever 中具有编译器可识别的非观察边界，使大型应用能够存储私密 Model 字段、处理 Secret 和调用成熟密码算法，同时彻底移除环境变量配置入口。 + +## Dependencies + +- 无前置子任务；这是 Port setting、认证 API 和 CMS 会话的基础。 + +## Requirements + +- Model `private` 字段由所属 component/domain 的 App 拥有；生成的 Model CRUD 可以持久化，所属 App 可以访问，其他角色和领域不能访问。 +- 含 private Model 字段的完整 Model 不能出现在 App 输入/输出、API/Job wire 输出、日志或跨领域类型观察面；App 使用显式 View record。 +- 新增 opaque `Secret` 标量：不可 Render、比较、拼接、普通 JSON 编码、Map key、Model 存储或错误 payload。 +- Secret 只允许进入批准 sink：Argon2id hash/verify、HMAC、常量时间验证、后续受控 Cookie/bearer setting sink；不提供通用 Secret-to-Text。 +- 提供操作系统 CSPRNG token、Argon2id 密码 hash/verify、SHA-256、HMAC-SHA256 和常量时间字节/摘要验证；使用成熟依赖，不自实现密码算法。 +- 密码哈希存储为 private Text，明文密码使用 Secret；错误不得包含明文、密钥或摘要原文。 +- 删除 `dever.process.environment` 及对应 intrinsic/runtime bridge、文档和测试；不保留别名、fallback 或 deprecated 路径。 +- `dever.process.arguments` 保留，但不得成为 setting override；配置来源规则由后续 Port/Job/API setting 合同继续约束。 +- `.dever` 与 `.dever.md` 使用相同语法/语义；所有正式测试放根 `test/`。 + +## Acceptance Criteria + +- [ ] 所属 App 可创建、更新、读取 private Model 字段，并能用其验证密码；其他角色/领域的字段访问在 check 阶段拒绝。 +- [ ] App 签名、API response、日志和普通 JSON 对 private/Secret 的直接或嵌套泄漏均被静态拒绝。 +- [ ] Secret 不能被渲染、比较、拼接、存入 Model/Map key 或放进普通错误 payload。 +- [ ] secure token 两次调用不复用值且使用 OS CSPRNG；无随机源时返回显式安全失败。 +- [ ] Argon2id hash/verify 验证成功、错误密码、损坏 hash 和资源上限；SHA/HMAC 与已知向量一致。 +- [ ] 搜索和负向编译测试确认应用无法调用环境变量读取，仓库没有兼容入口。 +- [ ] 既有普通 record private 字段、Model schema、API envelope、日志和 Markdown 回归保持通过。 + +## Out Of Scope + +- 通用密钥管理服务、密钥轮换协议和硬件安全模块。 +- JWT/OAuth、会话业务模型、RBAC 和登录 API；由后续 API/CMS 子任务实现。 +- 数据库列加密；private 表示可见性，不声称静态或磁盘加密。 diff --git a/.trellis/tasks/09-20-secure-data-contracts/research/secure-contracts-source-map.md b/.trellis/tasks/09-20-secure-data-contracts/research/secure-contracts-source-map.md new file mode 100644 index 0000000..e586ac2 --- /dev/null +++ b/.trellis/tasks/09-20-secure-data-contracts/research/secure-contracts-source-map.md @@ -0,0 +1,70 @@ +# Research: 安全数据合同源码边界 + +- Query: Model private、Secret、密码学及 environment 移除的最小实现路径。 +- Scope: internal +- Date: 2026-09-20 + +## Findings + +### 字段与领域身份 + +- `crates/dever-core/src/syntax.rs:124` 的 AST `Field.private` 已存在；`parser.rs:517`、`:556` 解析普通 record 的 private，选择分支不允许 private。无需新增语法关键字。 +- `check/symbols.rs:373` 将 AST private 原样落到 `types.rs:188` 的 checked `Field`。HIR 表达式直接持有 checked Type，因此不需要另造字段隐私 HIR。 +- `types.rs:209` 的 `Definition` 已有 owner、kind；`DefinitionKind::Model` 可区分普通 record 与 Model。`source.rs:25` 的 SourceRole、`:84` 的 `SourceLayout::domain()` 是领域身份唯一来源，不应拆字符串判断归属。 +- `check/model.rs:539` 当前硬拒绝 Model private；移除此拒绝后已有字段落地流程可保留 metadata。`synthesize_fields`（`:219`）仅插入公开 id/created_at，生成 Page/Cursor 及 Related 字段。持久 schema `ModelField` 不必承载语言隐私，因为数据库列格式不因访问策略改变。 +- `check/expressions.rs:167` 的 `Body::field_type` 按 type.owner == context.owner 检查私有字段；写路径复用字段解析。`check/values.rs:41` 独立进行 record 构造授权。两处应改用同一 owner 检查：普通 record 保留 package 私有；Model 只允许同领域 App。可直接复用 `check/orm.rs:247` 对 SourceRole::App 和 layout.domain 的判定。 +- `check.rs:506` 的 validate_public_boundaries 只拒绝 private nominal type，并跳过 public record 的 private 字段；它不会拒绝一个 public Model 内含 private 字段。`:557` 的 validate_exported_failures 只检查错误类型自身 public 标志，嵌套 Model 同样需要补覆盖。 +- `check/api.rs:149` 的 json_type 已拒绝 record private，但自写递归，与 App 合同尚未共享。建议统一类型可观察性属性，再由 API 保留具体可编码标量白名单。 + +### Secret 与类型能力唯一所有者 + +- `types.rs:7` 的 Type 增加独立 Secret；`check/symbols.rs:300` 附近是源码内置标量解析入口；`native.rs:766` 附近是 Rust 表示映射；`reference/value.rs:17` 是 reference Value 枚举。不要用 Named record 或 Bytes 别名模拟 Secret。 +- `types.rs:232` 的 Properties + `cache_properties` 已按 nominal DAG 缓存 comparable/transferable/movable。建议在此集中增加可渲染/敏感类型属性；避免在 API、App、fault 三处复制 nominal 图遍历。 +- `check/symbols.rs:452` 的 map_key 是独立白名单，Secret 默认不匹配；`check/model.rs:1320` 的 normalize_field_type 是持久化白名单，Secret 默认不可持久化。无需把所有存储参数检查迁入通用 Properties；仅共享安全属性。 +- 风险：`Type::Related` 目前直接返回 Properties::ALL（types.rs:92），且 referenced_types（check/symbols.rs:471）特意不展开 Related，避免合法关联 Model 环。敏感信息属性若沿用该路径会漏检，若简单递归则可能死循环。使用有界反向传播/固定点处理关联图，或者在对外可观察类型中拒绝 Related；不能影响现有布局/可比较缓存算法。 +- `native/records.rs:40`、`:82` 无条件为 record/choice 生成 Render，逐字段调用 render_to，当前 private 字段同样被渲染。`reference/value.rs:134`、`:177` 同样逐字段展示。Secret 的静态禁令必须覆盖容器和选择 payload；不应靠运行期脱敏代替编译拒绝。 +- 原生类型统一 derive Clone/Debug（native/records.rs:25）；Secret Rust 表示可有安全 Debug 占位，但不能输出原始字节。不提供普通 Render 实现时，生成器需要避免为包含 Secret 的 nominal 类型生成不可编译 Render，同时错误/入口输出等 render sinks 必须先检查。 +- 日志内置签名在 `check/intrinsics.rs:108`，当前接收 Text；应保持此边界，不增加任意值日志转换。JSON 标准库采用显式 JSON 类型，API 泛型序列化是需额外保护的边界。 + +### environment 完整删除链 + +1. `library/dever/process.dever:6` EnvironmentResult、`:23` environment 及两个 environment_value 分支。 +2. `crates/dever-core/src/intrinsic.rs:26` ProcessEnvironment 枚举、`:140` process_environment 名称解析。 +3. `check/intrinsics.rs:85` 签名及官方 EnvironmentResult shape 验证。 +4. `contracts/effects.rs:549` process effect 分类，保留 ProcessArguments。 +5. `native/intrinsics.rs:72` resource_result → `dever_runtime::process::environment`。 +6. `reference/intrinsics.rs:69` recover → 同一个 runtime 函数。 +7. `crates/dever-runtime/src/process.rs:16` environment 桥接函数。 +8. `test/dever-tests/tests/standard_library.rs:77` process 输入测试及 `:115` 直接 runtime 错误测试；`differential.rs:331` environment 错误一致性样例。 +9. `README.md:65` 文档明确宣传该入口，也需要同步删掉。 + +已有 argv arguments 是独立能力，保留。测试环境配置不要换成另一个应用环境变量入口；删除 environment 专项并增加旧 API/旧 intrinsic 被拒绝的定向断言。 + +### 依赖与密码学复用 + +- `Cargo.toml:36` 已精确锁定 getrandom = 0.2.17；runtime `orm.rs:86` 用 getrandom::getrandom。runtime Cargo.toml 当前只在 database feature 下启用，非数据库 crypto 不能依赖 database 才有随机源。 +- Cargo.lock 中另有 getrandom 0.4.3（433）、hmac 0.13.0（490）、sha2 0.11.0（1059）、subtle 2.6.1（1143）；这些只是已解析的间接依赖，不代表 runtime 可直接 import。 +- Cargo.lock 已有 zeroize（1650）；没有 argon2。采用 Argon2 crate 时需核对其 digest/password-hash/rand_core 版本，不应假定已锁 sha2/hmac 与任意版本 Argon2 相容。 +- 官方包装遵循 process/time 的 Result choice + checker shape 校验 + native resource_result + reference recover；crypto 重用该协议。运行时集中到 crypto 模块；Secret 字节访问保持 crate 内部，禁止公共 reveal。 + +### 最小定向测试 + +- `contract_semantics::private_fields_protect_access_mutation_and_construction`（218）保留普通 record package-owner 基线。 +- `source_architecture::model_operations_belong_to_the_owning_domain_app`（209）扩展 Model private 字段读写、同领域不同 App topic、Domain/API/Test/跨域拒绝、App 嵌套参数/输出/错误泄漏。 +- `model_syntax` 覆盖 private 持久声明及 Secret 存储拒绝；`markdown_source` 验证 private 往返和两种源码一致。 +- `contract_api` 覆盖快照不会暴露私有表示，以及公开边界敏感类型拒绝。 +- crypto 新的根 test/dever-tests/tests 定向文件可以集中运行时已知答案、native/reference 一致性、Secret 禁止 Render/eq/Map-key/JSON/persistence；`test/dever-tests/Cargo.toml` autotests=false，必须注册新目标。 +- `standard_library` 和 reference feature 下 `differential` 修改 environment 案例。优先 cargo test -p dever-tests --test <target> <filter>，无需真实数据库、服务、全量测试。 + +### Related specs / External references + +- `.trellis/spec/backend/compiler-contracts.md`：字段隐私、App 边界、角色及有界类型图。 +- `.trellis/spec/backend/database-guidelines.md`：Model 所有权、静态 schema、settings-only 配置。 +- 当前任务 `design.md`：Model domain-App owner、opaque Secret、无环境变量兼容路径。 +- 外部资料未访问；版本事实来自当前 Cargo.lock，密码算法参数/具体 crate API 需实施者按实际选定版本的官方资料核实。 + +## Caveats / Not Found + +- 子会话 `task.py current --source` 返回 none；输出路径由父代理明确提供，未修改任何任务指针。 +- 未执行构建、测试或服务；结论为源码研究，不是运行验收。 +- 未发现可复用 Secret 或 argon2 实现。上述 Render 风险是当前实现事实；静态渲染入口完整枚举应在实现阶段继续检查 native entry/failure/assert 渲染，不能只修改 native record emitter。 diff --git a/.trellis/tasks/09-20-secure-data-contracts/task.json b/.trellis/tasks/09-20-secure-data-contracts/task.json new file mode 100644 index 0000000..6c84504 --- /dev/null +++ b/.trellis/tasks/09-20-secure-data-contracts/task.json @@ -0,0 +1,26 @@ +{ + "id": "secure-data-contracts", + "name": "secure-data-contracts", + "title": "Dever 安全数据合同", + "description": "实现 Model 私密字段、Secret、密码与安全随机能力，并移除应用环境变量配置入口", + "status": "completed", + "dev_type": null, + "scope": null, + "package": null, + "priority": "P1", + "creator": "Antigravity", + "assignee": "Antigravity", + "createdAt": "2026-09-20", + "completedAt": "2026-09-20", + "branch": "main", + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [], + "parent": "09-20-production-application-language", + "relatedFiles": [], + "notes": "Model private/Secret/crypto/environment removal/CMS View contracts complete. Independent quality gate and final review found no blocker; focused checks and secure_contracts 8/8 passed. Known unrelated contract_semantics legacy failures remain documented.", + "meta": {} +} diff --git a/.trellis/tasks/09-20-time-typed-codec/check.jsonl b/.trellis/tasks/09-20-time-typed-codec/check.jsonl new file mode 100644 index 0000000..f1d169b --- /dev/null +++ b/.trellis/tasks/09-20-time-typed-codec/check.jsonl @@ -0,0 +1,3 @@ +{"file":".trellis/spec/backend/quality-guidelines.md","reason":"JSON/time/API focused test matrix"} +{"file":".trellis/spec/backend/error-handling.md","reason":"Parse failure and source diagnostic behavior"} +{"file":".trellis/spec/backend/directory-structure.md","reason":"Single codec owner across consumers"} diff --git a/.trellis/tasks/09-20-time-typed-codec/design.md b/.trellis/tasks/09-20-time-typed-codec/design.md new file mode 100644 index 0000000..9a34dd3 --- /dev/null +++ b/.trellis/tasks/09-20-time-typed-codec/design.md @@ -0,0 +1,28 @@ +# Dever 时间与类型化编解码设计 + +## Time Representation + +保留现有 i64 毫秒运行时表示和数据库 codec。新增成熟、固定版本的日期时间库只用于严格解析、日历校验、UTC 格式化和 checked arithmetic，不手写日历算法。DateTime format 统一输出 `Z`，parse 可接受 RFC3339 offset 后转换 UTC。Duration 是有符号毫秒。 + +官方 `dever.time` 继续用普通 `.dever` Result choice 暴露失败，system intrinsic 返回具体 i64/Text。`unix_millis` 可保留为底层数值能力，但业务首选 `now() -> DateTime`。 + +## Shared Wire Schema + +在 checker/HIR 边界新增一个只读的具体 wire schema 描述，节点仅包含已解析类型、record 字段顺序和策略标记。schema 构建复用 nominal cycle/depth 和 private/Secret 属性，不独立遍历 AST。 + +Schema policy 区分： + +- ordinary/API input：可按协议解码，显式 Secret 字段生成 opaque Secret； +- ordinary output：必须可观察且无 Secret/private； +- setting input：允许 Secret，但不允许资源和业务 failure choice； +- persisted Job payload：要求跨进程稳定、无资源和 Secret。 + +Native emitter 为每个被使用 schema 生成具体转换函数并缓存 specialization。runtime 只提供重复键可见的 bounded JSON parser、标量 parse/format 和错误位置；业务值保持具体 Rust struct/list/scalar。 + +## API Migration + +先将 `native/api.rs` 的 response `json_value` 迁移到共享 encoder，保持 envelope 构造仍归 API owner。输入扩展留给 production-api 子任务。迁移后删除原有平行 Type match，添加生成代码断言防止回退。 + +## Compatibility + +数据库和 Model snapshot 不变。DateTime API JSON 从内部整数改为 RFC3339 是预发布协议的直接替换；仓库内 API fixture 同步迁移，不保留双格式输入输出。 diff --git a/.trellis/tasks/09-20-time-typed-codec/implement.jsonl b/.trellis/tasks/09-20-time-typed-codec/implement.jsonl new file mode 100644 index 0000000..40157ea --- /dev/null +++ b/.trellis/tasks/09-20-time-typed-codec/implement.jsonl @@ -0,0 +1,3 @@ +{"file":".trellis/spec/backend/compiler-contracts.md","reason":"Type properties, private boundaries and generated contracts"} +{"file":".trellis/spec/backend/toolchain-and-library.md","reason":"Official time/json packages and native runtime boundary"} +{"file":".trellis/spec/backend/database-guidelines.md","reason":"Date/time storage representation and ORM compatibility"} diff --git a/.trellis/tasks/09-20-time-typed-codec/implement.md b/.trellis/tasks/09-20-time-typed-codec/implement.md new file mode 100644 index 0000000..8265643 --- /dev/null +++ b/.trellis/tasks/09-20-time-typed-codec/implement.md @@ -0,0 +1,64 @@ +# Dever 时间与类型化编解码实施计划 + +## Start Gate + +- [x] `secure-data-contracts` 完成并提供稳定 Secret/private 类型属性。 +- [x] 用户已批准父任务规划。 +- [x] 读取 compiler/toolchain/database/error specs 和当前 API codec、JSON runtime、DateTime ORM codec。 + +## Implementation + +- [x] 增加 checked DateTime/Date/Time/Duration runtime 操作与官方 time API。 +- [x] 增加共享 wire schema 类型分类、策略和 schema fingerprint 基础。 +- [x] 增加 bounded strict JSON input parser/decoder 和 concrete value encoder。 +- [x] 生成具体 scalar/list/record/nullable codec，缓存重复 schema。 +- [x] 将 API response 迁移到共享 encoder，删除 `native/api.rs` 平行转换。 +- [x] 保持 ORM storage codec/fingerprint 不变并增加回归。 +- [x] 更新 LANGUAGE、Markdown、compiler/toolchain/database specs。 + +### 实施合同（2026-09-20） + +- 固定 chrono 0.4.45，std-only；主代理统一 fetch 后全部定向检查使用 locked/offline。Date 是 UTC 午夜 epoch 毫秒，Time 是午夜起毫秒；RFC3339 offset 转 UTC，拒绝闰秒、超过三位小数和四位日历范围以外值。checked 算术仅检查 i64 溢出，超日历范围在格式化边界失败。 +- `core/wire.rs` 是唯一类型允许表，policy 进入 identity；`native/wire.rs` 生成具体类型的双向 helper，API 按 identity 缓存。允许 Secret 的输入 policy 只生成 decoder。private/bounds/Choice/Map/ModelId/Related/resource 稳定拒绝；没有放行尚未实施的字段约束验证。 +- runtime wire 使用借用 RawValue 保留原始 JSON 数字，逐层 visitor 检查重复键；16 MiB、64 层容器、65536 个值。concrete decoder 拒绝未知和缺失必填字段，缺失 nullable 得到 None。API envelope 只嵌入 Encoded，移除旧 response Value round-trip。 +- 官方标准库原来没有 `.dever.md` 副本。经主代理确认，不新增同包重复源码；以 LANGUAGE 合同和 plain/Markdown 调用方同签名、同输出 parity 覆盖 Markdown 验收。 +- 没有改 ORM/model fingerprint owner、配置来源、API 输入协议或 CMS 业务；无应用环境变量配置。 + +### 验证结果（2026-09-20） + +- core/runtime/cli，api + reference，以及 postgres + api + reference 两次定向 cargo check：通过。 +- 新 `time_codec` 最终 9/9 通过（40.39 秒）。覆盖实际编译运行的 concrete decoder、Decimal/Id/Uuid/负零/原始 Json 精度、缺失/未知/重复字段、无监听器 API dispatch、native/reference 时间成功和失败、plain/Markdown、预算与 policy。随后仅增强 Model 专项为当前 App → Model 调用，独立复验 1/1 通过：四种时间值仍编码为 ORM Int，存储 logical type 和 i64 表示不变。 +- 既有 `api_routes` 7/7、`json_library` 7/7、`markdown_source` 20/20、`model_syntax` 3/3、`standard_library` 5/5 均通过。 +- `sqlite_orm::shared_orm_fixture_remains_valid_without_opening_a_database`：失败，未打开数据库。`support/orm_fixture.rs` 的旧 fixture 在 `main.dever` 中直接调用 `user.create`，当前 `check/orm.rs::resolve_model` 只允许 Loose 使用 table 短名，当前 Main/App 角色合同不再接受该用法，报 C004。该 fixture 和角色/ORM owner 本次均未修改；按主代理确认记录现存不匹配，不扩大本任务迁移旧 fixture。其余 SQLite 执行用例未运行。 +- `git diff --check` 通过；仓库主体为既有未跟踪文件，另对本次新增源码做显式空白和旧转换路径扫描。没有旧 `json_type`、`json_value` 或 API `parse_json` 输出路径残留。 +- 未运行全量、真实 PostgreSQL、服务、性能；rustfmt/clippy 组件缺失，未安装。 + +## Focused Verification + +### Trellis 独立检查（2026-09-20） + +- 核对共享 schema policy/identity、nominal ID 位移稳定性、根 response bounds、private/Secret/资源拒绝、具体 decoder 与 API envelope；没有发现并行输出类型允许表或 ORM storage/fingerprint 改动。 +- 修复 runtime Encoder 的异常状态：对象缺值时 `end()` 保留未完成 frame；写入部分 JSON 后触及字节上限时锁定失败，`finish()` 不再可能发布残缺 Encoded。新增缺值、闭合括号超限和逗号写入后值超限三个回归，同步 toolchain spec。生成代码原本逐步 `?` 传播，修复针对公共 Encoder 自身的 Encoded 不变量。 +- 独立 `cargo check --locked --offline -p dever-core -p dever-runtime -p dever-cli --features dever-runtime/postgres,dever-runtime/api,dever-core/reference` 通过（16.79 秒）。命令使用上述 incremental/debug/toolchain 设置。 +- 修复后 `time_codec` 9/9 通过（44.22 秒），包括新增 Encoder 回归、native/reference 时间、生成 concrete codec 与无监听器 API dispatch。 +- `git diff --check` 及新增/修改源码显式尾随空白扫描通过。rustfmt/clippy 二进制未安装，不能宣称 lint 全通过；未安装组件。 +- 未运行全量、真实 PostgreSQL、SQLite 执行、服务或性能测试。已有 SQLite fixture 角色不匹配仍按上述记录保留；无本任务未修复阻塞。 + +```bash +cargo check --locked --offline -p dever-core -p dever-runtime -p dever-cli --features dever-runtime/postgres,dever-runtime/api,dever-core/reference +cargo test --locked --offline -p dever-tests --features api,reference --test time_codec -- --test-threads=1 +cargo test --locked --offline -p dever-tests --features api,reference --test standard_library --test json_library --test api_routes --test model_syntax --test markdown_source -- --test-threads=1 +cargo test --locked --offline -p dever-tests --features sqlite --test sqlite_orm shared_orm_fixture_remains_valid_without_opening_a_database -- --exact --test-threads=1 +git diff --check +``` + +不运行全量测试、真实 PostgreSQL、服务或性能基准。PostgreSQL 仅做受影响 feature 的定向 `cargo check`，除非隔离连接被明确提供。 + +以上命令实际均额外使用 `CARGO_INCREMENTAL=0`、`--config profile.dev.debug=0 --config profile.test.debug=0` 和已安装 stable toolchain 的命令级 PATH；这些是编译工具设置，不是应用配置。 + +## Review Risks + +- JSON parser 必须保留 duplicate key 可见性，不能先解析成会覆盖键的 Map。 +- DateTime 外部格式变化不能修改数据库列类型、migration history 或内部排序。 +- schema 缓存 identity 必须包含 policy，避免 setting 允许的 Secret 被普通 output 复用。 +- 生成 codec 必须保留 record 声明顺序和输入求值顺序，且不泄漏 runtime Value 到业务层。 diff --git a/.trellis/tasks/09-20-time-typed-codec/prd.md b/.trellis/tasks/09-20-time-typed-codec/prd.md new file mode 100644 index 0000000..90cfb49 --- /dev/null +++ b/.trellis/tasks/09-20-time-typed-codec/prd.md @@ -0,0 +1,37 @@ +# Dever 时间与类型化编解码 + +## Goal + +补齐生产业务所需的 UTC 日期时间操作，并建立唯一的静态类型 wire 编解码合同，为 Adapter setting、Job payload 和 API 输入输出提供同一套安全、无损、可审计的转换。 + +## Dependencies + +- `secure-data-contracts`：提供 Secret 和统一不可观察类型属性。 + +## Requirements + +- DateTime 内部和数据库继续使用 Unix epoch 整数毫秒；Date/Time/Duration 保持既有 i64 存储兼容。 +- `dever.time` 提供 UTC now、RFC3339 DateTime parse/format、Date/Time parse/format、Duration 构造，以及 DateTime 加减 Duration、DateTime difference；溢出和非法文本返回显式失败。 +- 外部 wire 表示固定：DateTime 为规范 UTC RFC3339 字符串，Date 为 `YYYY-MM-DD`，Time 为带可选毫秒的 `HH:MM:SS[.sss]`，Duration 为整数毫秒。 +- 编译器从具体 Type/Definition 派生共享静态 wire schema；setting、Job 和 API 只能通过该 owner 请求 encode/decode 代码。 +- 支持 Bool/Int/Float/Decimal/Text/Id/Uuid/DateTime/Date/Time/Duration、nullable、List 和无 private 字段的 record；Json 作为已验证原始 JSON 时保留既有语义。Secret 只允许在 API/setting input policy 中解码，禁止普通 output 和 Job payload。 +- Float 编码拒绝 NaN/Infinity；Decimal/Id/Uuid 使用无损字符串；整数严格检查 i64。 +- 对象解码拒绝重复键、未知字段和缺失必填字段；嵌套深度、集合长度和总字节受运行时已有/新增明确上限约束。 +- 编解码器生成具体值，不把动态 `serde_json::Value` 传入业务函数，不增加反射或 runtime 类型注册表。 +- 迁移当前 API response 编码使用共享 schema，保持 `{code,message,data}` envelope 和既有路由行为。 + +## Acceptance Criteria + +- [ ] 时间 parse/format round-trip、UTC 规范化、闰年边界、负 epoch、非法文本和 i64 溢出均有定向测试。 +- [ ] SQLite/PostgreSQL Model 的整数毫秒表示和现有 schema fingerprint 不因外部格式变化而改变。 +- [ ] 支持类型的 encode/decode round-trip 无损；Decimal/Id/Uuid/DateTime 不经 Float。 +- [ ] duplicate/unknown/missing/over-depth/over-size/non-finite/private/resource 稳定拒绝；Secret 仅在批准输入策略成功，普通输出和 Job payload 稳定拒绝。 +- [ ] API response 改用共享 codec 后，已有 envelope、状态码和 API route tests 保持一致；DateTime 输出变为 RFC3339。 +- [ ] `.dever` 与 `.dever.md` 的时间签名、文档合同和生成行为一致。 +- [ ] 后续 setting/Job/API 能消费一个公共 compiler schema API，而无需复制类型匹配分支。 + +## Out Of Scope + +- 本地时区数据库、DST 推断和用户界面格式化。 +- 任意 schema registry、Avro/Protobuf 或动态 JSON schema 发布。 +- Model 结构自动存储为 JSON；持久化仍需显式 Json 字段。 diff --git a/.trellis/tasks/09-20-time-typed-codec/research/time-wire-source-map.md b/.trellis/tasks/09-20-time-typed-codec/research/time-wire-source-map.md new file mode 100644 index 0000000..80e766b --- /dev/null +++ b/.trellis/tasks/09-20-time-typed-codec/research/time-wire-source-map.md @@ -0,0 +1,81 @@ +# Research: 时间与共享类型化 wire codec 源码地图 + +- Query: 查清现有时间表示、API JSON 路径、可复用 parser/schema/type graph、依赖和最小验证入口。 +- Scope: mixed +- Date: 2026-09-20 + +## Findings + +### 已有时间表示和必须保留的合同 + +- `crates/dever-core/src/types.rs:18` 已有 DateTime/Date/Time/Duration 四个独立 Type；`check/symbols.rs:313` 负责源码类型解析。它们不是 Int 别名，源码不能随意把 Int 赋给 DateTime。 +- `native.rs:770` 四种类型均生成 Rust i64；`native/orm.rs:1312` 编码为 ORM 整数，`:1375` 通过 `orm::int` 解码；`:1746` PostgreSQL DDL 为 BIGINT。不要为外部字符串格式修改这些分支。 +- `model.rs:146` Model revision、`:158` model_snapshot、`:364` field_type 共同拥有 Model 语义身份；四种时间 logical type 字符串应保持原样。SQLite/PostgreSQL fingerprint 属持久化合同，wire schema 不应写入它。 +- `.trellis/spec/backend/database-guidelines.md:83` 已明确 Date/time 使用整数毫秒。建议 Date 为 epoch 起 UTC 午夜毫秒，Time 为午夜起毫秒且限制 `[0, 86400000)`；不能悄悄改为 Date epoch 天数。当前没有更具体的 Date 单位/范围实现，需在实现文档明确。 +- `reference/value.rs:14` 只有 Int 值载体，没有四种单独时间变体；HIR 保留具体 Type，reference 新 intrinsic 可复用 Value::Int，不必新建动态时间对象。 +- `crates/dever-runtime/src/time.rs:4` 仅有 unix_millis、monotonic_nanos、sleep；unix_millis 当前拒绝系统时钟早于 epoch，但 parse/format 应独立支持负 epoch。 +- `library/dever/time.dever:1` ClockResult.Read(Int)、SleepResult；普通函数调用系统 intrinsic 后分派并 fail，继续复用这种真实源码 Result choice。 +- 桥接修改位置：`intrinsic.rs:33/152` enum/name mapping；`check/intrinsics.rs:91` 签名和 Result shape；`native/intrinsics.rs:79` resource_result；`reference/intrinsics.rs:76` recover；`contracts/effects.rs:529/553` blocking/time effect。只有 now/时钟读取有 time effect；确定性 parse/format/算术应 pure，不能给所有时间 intrinsic 统一外部 time effect。 + +### API 与 parser 路径 + +- `check/api.rs:121` 调用私有 `json_type` (`:149`) 递归判断输出可序列化，`native/api.rs:72` 有平行 `json_value` Type match。共享 schema 应同时替换这两份允许类型逻辑，否则 checker 和 emitter 会继续漂移。 +- `native/api.rs:51` 是 response 转换接入点，转换失败保持日志脱敏和 500；`:32` handler 输入只绑定 Text/Int/Bool(nullable)，本任务不要顺手扩大 API 输入协议。 +- `runtime/api.rs:188` success 和 `:216` envelope 拥有 `{code,message,data}`；404/405 Allow/500 保持 API owner，不塞入通用 codec。 +- `runtime/api.rs:140` InputObject 仅拒绝顶层重复键；nested `serde_json::Value` 会静默覆盖重复键。`:191` parse_json 是普通 serde_json parse，没有项目字节/集合上限。不能宣称已有全面严格 parser。 +- `library/dever/json/parse.dever:31` 普通 Dever parser；`:212` depth guard、`:288` duplicate rejection；`json/value.dever:27` maximum_depth=256；`json/write.dever:24` 输出预算 16 MiB。它使用 postorder Document/Number(raw Text)，保持原始数字准确性；不能把它当现成可调用 Rust runtime parser。没有发现 parser 输入字节或集合总量上限。 +- `runtime/config.rs:142` serde_json::from_str(Document)，`:182` HTTP from_value；未来 setting 需接新 parser，当前无需实现 setting 功能。 + +### 最小清晰 owner/API + +1. 新 `crates/dever-core/src/wire.rs`：只依赖已检查 Type/Definition，导出 crate 可见的 Policy、Schema/Node、从具体 Type 构建 schema 的 Result API。当前 Type 本身 crate-private，因此所谓公共 compiler API 是 compiler 各消费者共用，不必把所有 HIR 类型变成外部 public。 +2. 新 `native/wire.rs`：从 schema 生成具体 Rust encode/decode helper；名义类型按 type-id + policy + direction 缓存 specialization，用 schema 引用而非展开共享 DAG。API、setting、Job 只能调用此 owner。最终业务参数必须具体 T{id}/List/scalar，Value 只停留在转换边界。 +3. 新 `runtime/wire.rs`：serde DeserializeSeed/Visitor 式递归 bounded parser，每层 map 在插入前检测重复键，进入容器/接收元素时扣预算。输入总字节在 parse 前检查、显式 end() 拒绝尾随内容；未知字段/必填字段由具体 decoder 检查。不要先 Value parse 再检查 duplicates。 +4. runtime scalar conversion 复用 `number.rs:44` DecimalValue::parse（严格拒绝 inexact/nonfinite）、`orm.rs:53` Uuid::parse、`lib.rs:40` Id(String) 和新增 time parse/format。Float decode/encode 拒绝非有限。Id/Decimal/Uuid 只收字符串，Int 只收精确 i64 JSON number。 +5. 新 wire runtime feature 最自然由 api/database 等消费者启用；当前 serde/serde_json 在 runtime 是 optional，`native/build.rs:309` runtime artifact cache identity 与 feature 选择要同时包含任何新 feature。不要让生成 decode helper 依赖未启用模块，也不要为未使用 codec 的普通程序开启整套 JSON。 + +### 类型图与敏感值 + +- `types.rs:84` observable 复用缓存的 Secret/private Model 敏感性；缓存 `cache_properties` 在名义类型校验后生成。不要在其初始化之前调用 observable。 +- `types.rs:290` cache_properties 用依赖顺序算属性；`nominal_sensitivity` 独立 worklist 传播 Related 环。`check/cycles.rs` 拒绝正常名义布局环。新 schema 不该再从 AST 建图，也不能把 Related 当普通记录递归展开。 +- wire 支持 record 且每个 field 必须非 private；observable 单独不够，因为普通 record private 不属于 private Model 敏感性，仍须逐字段拒绝。 +- API/setting input 允许 Secret，ordinary output/Job payload 禁止；不要用 transferable 代替 wire eligibility（Secret 可 transferable，资源也可能 transferable）。Choice、Map、Related、资源均不在当前 PRD 支持集。 +- `DefinitionKind::ModelId` 是特殊 nominal（native为 T{id}(i64)），不是 builtin Id(String)；当前 API json_type 对非 record 已拒绝它。不可意外将 ModelId 当空 record 编码 `{}`，若本任务不扩展其合同应稳定拒绝。 +- 当前 bounds 是 Field.hir::Domain；任务只要求类型转换，但 decoder 若接收受约束 record，必须防止生成值绕过既有字段范围保证。应明确生成 bounds 校验或 schema 构造拒绝未支持 bounds，不能默认通过。 + +### 日期依赖与精度风险 + +- 检查 Cargo.lock 和 `/root/.cargo/registry/{src,cache}`，未发现 chrono/time/jiff 日期库。不能按离线已缓存处理；引入固定版本后需由主代理统一 fetch/lock，再使用 offline+locked 验证。 +- 官方 chrono 0.4.45 文档给出 MSRV 1.62，满足工作区 Rust 1.85；可固定 `=0.4.45`, default-features=false, features=["std"]，复用已有 SystemTime，不开启 clock/local timezone。相关官方文档见下面链接。 +- Chrono 的 from_timestamp_millis 返回 Option，日期范围小于全 i64；格式化超出日历范围应返回明确失败。RFC3339 秒小数与 leap-second 不能直接 timestamp_millis 截断：对非毫秒精度和 `:60` 要显式拒绝或先固定精确可表示策略，避免静默丢失。 +- 四位 `YYYY-MM-DD` wire 合同意味着需要限制年份表示；DateTime canonical 精度（固定 .sss 或零毫秒省略）须统一定义并测试。Time 可选毫秒只能接受可无损表示文本。 +- DateTime 加减 Duration、difference、Duration 倍率构造复用 checked_add/sub/mul；i64 checked 成功不代表仍能格式化成约定 RFC3339。 + +### 定向验证与现有缺口 + +- `test/dever-tests/tests/standard_library.rs:105` clock_failures_are_results 可扩展时间 wrapper；更完整的新 `time_codec` target 可集中 runtime 时间边界和 compiler native/reference 最小回环，避免散入一堆测试。 +- `api_routes.rs:37` 生成路径检查、`:64` native compile（不启动监听）、`:118` typed input、`:148` errors/envelope；运行需 `--features api --test api_routes`。 +- 注意 `api_routes.rs:148` malformed_or_extra_inputs 中 if-let 仅检查解析成功分支，未断言坏输入一定失败；新 parser 回归必须直接 assert error，覆盖 nested duplicate、unknown/missing、i64 extremes、depth/bytes/list limits、NaN/Infinity。 +- `json_library.rs:47/85/146/237` 覆盖 ordinary Dever JSON 正常/坏输入/depth/构造图限制。新 Rust parser 不应改变此官方源码 JSON API；需要原始 Json 数字精度保真测试，serde_json 默认 Value 可能把巨大原始数字转浮点并改变值。 +- `model_syntax`/`markdown_source`/`contract_execution` 可以验证 Model snapshot/plain-Markdown parity，schema/int codegen 字符串断言可确认 PostgreSQL BIGINT 而不连真实数据库。 +- 已有 API DateTime 序列化是整数 (`native/api.rs:75`)，本任务将明确改成 UTC 字符串，需要新 fixture 证明，而不只是测试 time runtime。 + +## External References + +- https://docs.rs/crate/chrono/0.4.45 — 固定版特性、MSRV、日期范围。 +- https://docs.rs/chrono/0.4.45/chrono/struct.DateTime.html — parse_from_rfc3339、from_timestamp_millis、timestamp_millis、to_rfc3339_opts。 + +## Related Specs + +- `.trellis/spec/backend/{index,directory-structure,error-handling,quality-guidelines}.md` +- `.trellis/spec/backend/compiler-contracts.md`：type depth、敏感属性、API 输入/输出。 +- `.trellis/spec/backend/toolchain-and-library.md`：源码 JSON 与 intrinsic/reference 合同。 +- `.trellis/spec/backend/database-guidelines.md`：整数毫秒和 fingerprint。 +- 当前任务 `prd.md`、`design.md`。 + +## Caveats / Not Found + +- 子代理 `task.py current --source` 返回 none；输出目录由主代理 dispatch 显式指定，没有猜路径或更改 task 状态。 +- 此研究未改源码、规范或任务状态；未运行服务、数据库、全量或性能测试。 +- Shared schema 与 typed decoder 目前不存在；上面的文件名/API 是最小新增 owner 建议，不是已有事实。 +- 普通 JSON 的 exact raw numeric 保真与 serde_json Value 的数字语义有真实差异，需要实现前固定 Json passthrough 策略。 diff --git a/.trellis/tasks/09-20-time-typed-codec/task.json b/.trellis/tasks/09-20-time-typed-codec/task.json new file mode 100644 index 0000000..a7e9bdd --- /dev/null +++ b/.trellis/tasks/09-20-time-typed-codec/task.json @@ -0,0 +1,26 @@ +{ + "id": "time-typed-codec", + "name": "time-typed-codec", + "title": "Dever 时间与类型化编解码", + "description": "补齐 UTC 日期时间能力和编译器共享的静态类型 JSON 编解码合同", + "status": "completed", + "dev_type": null, + "scope": null, + "package": null, + "priority": "P1", + "creator": "Antigravity", + "assignee": "Antigravity", + "createdAt": "2026-09-20", + "completedAt": "2026-09-20", + "branch": "main", + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [], + "parent": "09-20-production-application-language", + "relatedFiles": [], + "notes": "UTC time API, shared typed wire schema/codegen, strict bounded JSON and API response migration completed. Focused checks passed; one pre-existing SQLite fixture role mismatch remains recorded outside this task.", + "meta": {} +} diff --git a/.trellis/tasks/09-28-shared-toolchain/check.jsonl b/.trellis/tasks/09-28-shared-toolchain/check.jsonl new file mode 100644 index 0000000..d60a564 --- /dev/null +++ b/.trellis/tasks/09-28-shared-toolchain/check.jsonl @@ -0,0 +1,2 @@ +{"file":".trellis/spec/backend/toolchain-and-library.md","reason":"复核共享缓存的身份、完整性、并发和验证要求"} +{"file":".trellis/tasks/09-28-shared-toolchain/prd.md","reason":"按当前子任务的验收标准进行最终检查"} diff --git a/.trellis/tasks/09-28-shared-toolchain/design.md b/.trellis/tasks/09-28-shared-toolchain/design.md new file mode 100644 index 0000000..5bc117e --- /dev/null +++ b/.trellis/tasks/09-28-shared-toolchain/design.md @@ -0,0 +1,86 @@ +# Design + +## Boundary + +本任务增加机器级 Dever 管理层。它拥有稳定入口、版本目录、项目版本分派、更新事务和共享缓存服务；版本化编译器继续拥有源码检查、生成和运行语义。项目仍通过现有 CLI 合同进入编译器。 + +## Directory Contract + +发行安装器按平台选择机器级根目录，但根内布局一致： + +```text +<toolchain>/ + bin/ + dever stable launcher + deverd cache/build service + state/ + active-version + install.lock + versions/ + <version>/ + dever-core + manifest.json + runtime/ + cache/ + native/ + downloads/ + staging/ +``` + +`bin/dever` 是机器上唯一公开入口。`versions/*`、`state/*` 与缓存目录不加入 PATH，也不要求用户直接操作。目录根由安装器写入 launcher 自身的安装位置，运行时不读取环境变量。 + +## Command Flow + +1. launcher 解析命令和项目根，不读取 Dever 源码。 +2. 对项目命令只解析 `config/setting.json` 的 `dever.version`；存在时精确选择，否则读取机器 `active-version`。 +3. launcher 校验版本 manifest 和核心可执行文件身份，然后把原参数分派给版本核心。 +4. 核心完成现有 check/run/build 流程；需要原生产物时通过本机受保护 IPC 请求 `deverd`。 +5. `deverd` 根据已安装版本和完整构建输入自行计算缓存键、编译或恢复产物，再将结果返回请求用户，客户端不能指定缓存文件路径。 + +`config/setting.json` 增加可选配置： + +```json +{ + "dever": { + "version": "0.1.0" + } +} +``` + +runtime 配置解析器接受并忽略已经由 launcher 消费的 `dever` 段，确保同一配置可以同时用于 `run` 和独立构建产物；构建产物不使用该字段选择自身 runtime。 + +## Ownership + +- `dever-cli`：稳定 launcher 命令合同、项目版本读取和版本核心分派。 +- 独立 machine manager 模块：安装、更新、切换、卸载、签名验证和原子状态变更；不把这些分散进每个命令分支。 +- `dever-runtime::config`：只增加严格的 `dever.version` 配置合同，不承担安装或版本选择。 +- `dever-core::native::build`：把默认缓存请求交给共享服务；保留显式隔离缓存入口供测试使用。 +- cache service：唯一可写共享缓存的进程，负责密码学键、编译、恢复、并发发布、租约和清理。 +- 平台 adapter：只封装全局路径、权限提升、服务管理和 IPC 差异；上层状态机和版本规则只有一套。 + +## Install And Update Transaction + +下载先进入 `cache/downloads`，验证发行 manifest 签名、包摘要、平台和版本后解包到 `cache/staging`。核心自检成功后原子发布到 `versions/<version>`，最后原子更新 `active-version`。任一步失败都删除 staging 并保留旧版本。卸载同样持有 `install.lock`，不能删除当前版本或正在服务请求的版本。 + +初次安装由签名的系统安装器建立全局入口、机器服务、权限和初始版本；之后统一由 `dever install/update/use/uninstall` 管理。需要管理员权限时使用平台标准提权流程，不保存凭据。 + +## Cache Security And Cleanup + +- `deverd` 使用专用系统身份，缓存目录只允许该身份写入。 +- 本地 IPC 使用操作系统凭据识别调用者；请求只包含规范化构建输入，不接受任意读写路径。 +- 缓存键使用 SHA-256 等密码学摘要；命中前复核 manifest 与产物摘要。 +- 客户端只有提交完整相同输入才能取得同一产物，服务不提供缓存目录列表或其他项目索引。 +- 运行中的条目持有租约。清理只删除无租约的 staging、损坏条目和按策略选出的旧条目，并使用原子删除/替换。 +- `cache status` 输出容量、条目数和版本占用，不输出项目源码、路径或用户信息。 + +## Failure Contract + +全局入口、活动版本状态、项目锁定、签名、版本 manifest、IPC 或缓存完整性任一验证失败时返回明确错误。不得静默切换版本、使用用户 PATH 中的编译器、改写项目配置或执行未验证产物。 + +## Compatibility + +开发态 `deverc` 保持为私有仓库入口直到正式发行链完成。正式 `dever` 不读取开发仓库 `target/native-artifacts`。已有项目无需新增配置；只有需要固定版本的项目才增加 `dever.version`。 + +## Deferred Complexity + +本任务不重复实现父发行任务的原生后端、目标包生产和发布基础设施。安装管理层必须以父任务提供的签名发行包为输入；在发行包合同完成前可以实现和测试本地 fixture，但不能宣称公开安装链已完成。 diff --git a/.trellis/tasks/09-28-shared-toolchain/implement.jsonl b/.trellis/tasks/09-28-shared-toolchain/implement.jsonl new file mode 100644 index 0000000..281b0e9 --- /dev/null +++ b/.trellis/tasks/09-28-shared-toolchain/implement.jsonl @@ -0,0 +1,2 @@ +{"file":".trellis/spec/backend/toolchain-and-library.md","reason":"定义原生产物缓存、工具链边界和定向验证合同"} +{"file":".trellis/tasks/09-06-private-compiler-distribution/prd.md","reason":"约束本子任务不扩大到公开发行、内置后端和跨平台矩阵"} diff --git a/.trellis/tasks/09-28-shared-toolchain/implement.md b/.trellis/tasks/09-28-shared-toolchain/implement.md new file mode 100644 index 0000000..6437681 --- /dev/null +++ b/.trellis/tasks/09-28-shared-toolchain/implement.md @@ -0,0 +1,46 @@ +# Implementation Plan + +## Phase 1: Contracts + +- [x] 定义唯一的机器目录布局、发行 manifest、版本语法、签名输入、active-version 状态和平台权限合同。 +- [x] 在 `config/setting.json` 增加严格的可选 `dever.version`，并覆盖未知字段、非法版本和独立产物读取行为。 +- [x] 固定用户命令：`install/update/use/uninstall/version/cache status/cache clean`，现有项目命令不改变。 + +## Phase 2: Global Launcher And Version Store + +- [x] 把稳定 launcher 与版本核心分开；所有项目命令通过同一选择函数分派。 +- [x] 实现版本安装、签名/摘要验证、staging、自检、原子发布和 active-version 可恢复原子切换。 +- [x] 实现更新回滚、当前版本卸载保护、缺失锁定版本诊断和显式降级拒绝。 +- [ ] 实现 Linux、macOS、Windows 平台标准提权适配。 +- [ ] 为 Linux、macOS、Windows 分别实现机器级入口、目录权限和服务注册，平台差异只留在 adapter。 + +## Phase 3: Cross-user Build Cache + +- [ ] 将版本核心原生编译接入受保护的 daemon build service；已完成的 Status/Clean/Lib artifact IPC 不代替编译请求。 +- [ ] 使用密码学内容摘要统一编译器、runtime、目标、生成内容与选项身份；SHA-256 与 store 已实现，但版本核心尚未通过受保护服务使用它。 +- [x] 实现认证 IPC、并发请求合并、不可变原子发布、租约、完整性复核和安全结果交付。（IPC 客户端只接受 token-authenticated `deverd` 响应，未恢复同进程 fallback。） +- [x] 实现 `cache status/clean` 与更新/卸载后的损坏、staging 和无引用条目清理。 +- [x] Linux 采用内核 peer credential 接受不同 UID 的 content-addressed Lib artifact put/get 和 metadata-only status；拒绝非 owner clean/管理 token 读取，限制每 UID/总连接、单资产/总容量/条目数。 + +## Phase 4: Integration And Documentation + +- [x] 将现有 `check/api/fmt/test/run/build/tenant` 命令接入统一 launcher，验证参数和退出码透传。 +- [x] 更新 `README.md`、`LANGUAGE.md`、`IMPLEMENTATION.md` 和 backend toolchain spec，明确用户只有一个 `dever`。 +- [x] 使用签名本地 fixture 验证双版本、双项目、并发缓存发布、升级故障与缓存清理。 +- [x] 使用 root 与自有 uid=65534 fixture 客户端验证同一 launcher、不同 UID 的 status/artifact IPC、token/clean/admission 隔离；不创建系统账号或安装真实全局服务。 +- [x] 运行受影响 crate 检查和定向测试；不默认运行全量测试或替换当前机器的真实全局命令。 +- [ ] 最终安全审查安装提权、路径穿越、符号链接、TOCTOU、缓存投毒、降级与跨用户信息泄露。 + +## Rollback + +安装和更新始终保留上一活动版本，active-version 只在新版本自检通过后切换。launcher 或服务升级失败时恢复上一套文件和服务定义；缓存是可再生数据，回滚不迁移或修改任何项目目录。 + +## Implementation Record (2026-09-29) + +- 已完成稳定 `dever` 二进制、严格项目版本读取、签名/摘要/平台验证、双版本安装与切换、活动版本追加校验日志、失败回滚、缺失锁定版本诊断、参数与退出码透传。 +- 已完成开发态原生缓存 SHA-256 化，以及未来 `deverd` 内部 store 的不可变发布、完整性复核、读/写租约、状态与清理算法；该 store 明确不构成 IPC 或跨用户权限边界。 +- 复审后，launcher 的 `cache status/clean` 在认证 IPC 未完成前统一明确失败，不再直接构造 store；安装复制固定为机器共享只读权限，launcher 在执行前复核状态、公钥、版本目录和发行文件的属主与可写位，并拒绝 `update` 通过 `latest` 隐式降级。 +- 本地签名 fixture 已覆盖双版本、双项目、并发缓存发布、篡改拒绝、健康检查失败、部分 active 记录恢复和清理租约。没有创建真实系统用户、安装全局命令或变更系统服务。 +- 真实 Linux/macOS/Windows 提权与服务注册 adapter、认证 IPC、版本核心到 `deverd` 的构建请求、签名发行目录与下载（当前 `latest` 只是未签名本地 fixture 指针）、内置后端发行包和六宿主发行验证仍由父任务 `09-06-private-compiler-distribution` 阻塞；这些项保持未勾选，不能用同进程 store 代替。 +- 最终定向复验：受影响的 `dever-cli`、`dever-runtime`、`dever-core` 离线 `cargo check` 通过；`shared_toolchain` 6/6、`application_config` 6/6 通过。`rustfmt`、`clippy` 未安装；未运行全量、性能、真实系统安装或跨平台测试。 +- 本轮补充 `deverd --root <machine-root>` 本地服务和长度分帧 IPC：token 文件与 socket 均限制为 owner-only，`cache status/clean` 通过服务访问 CacheStore；服务不可用时客户端失败，不构造本地 store。跨平台 peer credential、系统服务注册、提权和真实双用户验证仍未完成。 diff --git a/.trellis/tasks/09-28-shared-toolchain/prd.md b/.trellis/tasks/09-28-shared-toolchain/prd.md new file mode 100644 index 0000000..3cca00b --- /dev/null +++ b/.trellis/tasks/09-28-shared-toolchain/prd.md @@ -0,0 +1,81 @@ +# Dever 本机共享工具链 + +## Goal + +让一台机器上的所有用户和所有 Dever 项目始终使用同一个全局 `dever` 命令。Dever 内部负责安装、升级、版本选择、项目锁定、安全共享缓存和清理；用户不需要理解或指定编译器真实路径。 + +## Background + +- `deverc check/run/build <project-root>` 已经可以处理任意项目根目录，但当前只是开发仓库中的单个引导编译器。 +- 当前没有正式的全局 `dever` 入口、机器级安装目录、版本选择、项目锁定、升级回滚或卸载合同。 +- 当前 `compile`、`compile_application`、`compile_project` 和 `compile_test_suite` 把默认缓存固定到编译器源码工作区的 `target/native-artifacts`，不能作为多用户机器级缓存。 +- 现有缓存键已经包含编译器、后端工具、runtime、生成内容、平台和编译选项，并且发布过程具有完整性校验和并发原子性。 +- 项目配置只读取各项目自己的 `config/setting.json`；工具链选择也必须遵守这一配置边界，不增加环境变量或第二个项目配置文件。 + +## Requirements + +### Global Command + +- 每台机器只公开一个 `dever` 命令；普通用户和所有项目都调用这个入口。 +- 全局入口不得要求用户设置 `PATH` 之外的 Dever 环境变量，也不得要求用户知道内部版本目录。 +- `check/api/fmt/test/run/build/tenant` 等现有命令参数保持项目根合同，入口只负责选择工具链后原样分派。 + +### Installation And Versions + +- 提供 `dever install <version|latest>`、`dever update`、`dever use <version>` 和 `dever uninstall <version>`。 +- 初次安装、版本安装、切换和卸载按操作系统要求获取管理员权限；日常项目命令不需要管理员权限。 +- 多个版本安装在机器级版本目录，安装与切换使用 staging + atomic rename，失败时保留当前可用版本。 +- 下载的发行物必须校验受信签名和内容摘要；不得执行未验证的编译器。 +- `dever update` 安装并切换到新版本；失败或健康检查不通过时自动保留旧版本。 +- 卸载当前默认版本必须先切换到其它版本；项目锁定了已卸载版本时给出明确的安装命令，不静默改用其它版本。 + +### Project Selection + +- 项目可以在 `config/setting.json` 中可选声明 `dever.version`。 +- 已锁定项目精确选择该版本；未锁定项目使用机器默认版本。 +- 版本缺失、格式非法或不受支持时在读取项目源码和运行项目程序前失败。 +- 版本选择不读取环境变量，不新增 `.dever-version`、lock 文件或用户级配置。 + +### Machine Sharing And Isolation + +- 所有操作系统用户共享同一个全局入口、已安装版本、标准库、runtime/target packs 和机器级缓存服务。 +- 每个项目的源码、`config/setting.json`、`data/`、构建输出、端口和运行进程保持独立，不复制到工具链目录。 +- 普通用户不能直接修改全局入口、已安装版本、默认版本状态或缓存条目。 +- 跨用户缓存不得让一个用户写入随后由另一个用户直接执行的未验证文件，也不得提供按目录浏览其他项目产物的能力。 + +### Shared Cache + +- 共享缓存由机器级 Dever 缓存/编译服务独占写入；客户端只提交完整构建输入并接收验证后的结果。 +- 缓存键改为密码学内容摘要，并继续覆盖编译器版本、runtime、目标平台、生成内容和编译选项。 +- 相同版本和相同完整输入可以跨项目、跨用户复用；不同输入、版本或目标不得复用。 +- 缓存发布保持不可变、完整性校验和并发原子性；普通用户不能伪造缓存命中。 +- 提供 `dever cache status` 和 `dever cache clean`；更新或卸载后自动清理没有引用的临时文件和损坏条目。 +- 缓存服务不可用时明确失败，不退回不受保护的全局可写目录；是否允许本次不使用共享缓存必须是显式 CLI 选项，而不是隐藏回退。 + +## Acceptance Criteria + +- [ ] 安装后，机器上的两个不同系统用户都通过同一路径的 `dever` 命令运行各自项目，无需用户级 Dever 安装。 +- [ ] 未锁定项目自动使用机器默认版本；`config/setting.json` 锁定版本的项目自动使用对应已安装版本。 +- [ ] 安装、升级、切换和卸载具有原子性；故障注入后旧的可用版本和全局入口不受损。 +- [ ] 签名错误、摘要错误、版本缺失和卸载当前版本均返回明确错误，不执行错误版本。 +- [ ] 两个项目或用户产生相同完整原生输入时命中同一缓存；输入、版本或目标改变时不会错误复用。 +- [ ] 非特权用户无法修改版本、默认状态或伪造缓存条目供其他用户执行。 +- [ ] 项目源码、配置、运行数据和显式构建输出不会进入共享安装目录或被其他用户通过 Dever 浏览。 +- [ ] `dever cache status/clean` 可检查和清理共享缓存；并发构建与清理不会暴露半成品或删除正在使用的条目。 +- [ ] 所有选择和运行行为不读取 `HOME`、`XDG_*`、`DEVER_*`、`RUSTC` 或其它 Dever 配置环境变量。 +- [ ] Linux、macOS 和 Windows 分别验证全局入口、系统权限、版本分派和升级回滚；实际支持平台必须与父发行任务一致。 + +## Out Of Scope + +- 修改 `.dever` 语言语法或应用业务目录规范。 +- 将项目运行配置移出 `config/setting.json`。 +- 云端编译、上传源码或依赖远程构建服务。 +- 在本任务重复实现父任务负责的 LLVM/LLD 后端和六目标交叉编译器；本任务消费父任务交付的可安装发行物。 +- 承诺不同源码产生的程序可以共享缓存；共享仅发生在完整输入一致时。 + +## Key Decisions + +- “简单”指用户只面对一个 `dever` 命令和项目现有的 `config/setting.json`，不是减少产品功能。 +- 全局入口与真实编译器版本分离：入口稳定，版本可以并存、切换和回滚。 +- `dever.version` 是唯一项目级工具链选择；机器默认版本是唯一未锁定项目选择，不存在更多优先级。 +- 跨用户写共享可执行缓存是安全边界，因此由机器服务持有，不能使用所有用户可写目录代替。 diff --git a/.trellis/tasks/09-28-shared-toolchain/task.json b/.trellis/tasks/09-28-shared-toolchain/task.json new file mode 100644 index 0000000..6f608e1 --- /dev/null +++ b/.trellis/tasks/09-28-shared-toolchain/task.json @@ -0,0 +1,26 @@ +{ + "id": "shared-toolchain", + "name": "shared-toolchain", + "title": "Dever 本机共享工具链", + "description": "让一台机器上的所有用户和项目通过一个全局 dever 命令共享版本化工具链，并自动完成版本选择、更新与安全缓存", + "status": "in_progress", + "dev_type": null, + "scope": null, + "package": null, + "priority": "P2", + "creator": "Antigravity", + "assignee": "Antigravity", + "createdAt": "2026-09-28", + "completedAt": null, + "branch": "main", + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [], + "parent": "09-06-private-compiler-distribution", + "relatedFiles": [], + "notes": "", + "meta": {} +} \ No newline at end of file diff --git a/.trellis/tasks/09-29-build-cache-storage/check.jsonl b/.trellis/tasks/09-29-build-cache-storage/check.jsonl new file mode 100644 index 0000000..ddf593b --- /dev/null +++ b/.trellis/tasks/09-29-build-cache-storage/check.jsonl @@ -0,0 +1,4 @@ +{"file":".trellis/spec/backend/quality-guidelines.md","reason":"核对允许的离线定向检查和测试入口兼容性"} +{"file":".trellis/tasks/09-29-build-cache-storage/research/cache-boundaries.md","reason":"复核 clean 与机器级 cache clean 的安全边界"} +{"file":".trellis/spec/backend/error-handling.md","reason":"复核失败清理、符号链接拒绝和原始错误保留"} +{"file":".trellis/spec/backend/directory-structure.md","reason":"复核 owner 唯一性并避免平行缓存实现"} diff --git a/.trellis/tasks/09-29-build-cache-storage/design.md b/.trellis/tasks/09-29-build-cache-storage/design.md new file mode 100644 index 0000000..c81608b --- /dev/null +++ b/.trellis/tasks/09-29-build-cache-storage/design.md @@ -0,0 +1,76 @@ +# Design + +## Boundaries + +本任务只修改编译器仓库构建策略、native runtime 准备路径和 CLI 临时产物清理。Cargo 开发产物由 Cargo profile 管理；native runtime 依赖由 `dever-core::native::build` 管理；项目命令解析由 `dever-cli` 管理。机器级缓存继续归 `deverd` 合同所有。 + +## Cargo Profiles + +根 `Cargo.toml` 增加 dev/test profile： + +```toml +[profile.dev] +debug = 0 +incremental = false + +[profile.test] +debug = 0 +incremental = false +``` + +这把多个任务已经稳定使用的命令级覆盖固化成仓库默认值。release profile 不变。测试目标、文件和选择命令全部保留。 + +## Native Runtime Layout + +当前每个 profile 都有完整 target： + +```text +target/native-runtime/<profile>/release/... +``` + +改为一个共享 Cargo target 和不可变的精确输入视图： + +```text +target/native-runtime/ + cargo/ shared Cargo target + inputs/<identity>/ exact runtime/dependency files for one build identity + operations/ build locks/staging +``` + +`runtime_library` 根据 database、api、crypto、wire、external 得到规范化 feature 集。Cargo 使用 `--message-format=json-render-diagnostics` 报告本次图中的产物；代码通过结构化 JSON 解析收集当前 `dever-runtime` 及其依赖的 rlib/dylib，不扫描并吞入整个历史 `release/deps`。 + +收集后的文件先写入同一文件系统的 staging，再按内容身份原子发布到 `inputs/<identity>`。相同文件优先使用硬链接以复用共享 target 的磁盘块；不支持或失败时复制。已发布输入视图只读且不可变，`RuntimeInputs` 继续对精确文件集合计算身份并在编译前复核。并发操作使用 create-new 锁和 staging，失败只清理本次 staging。 + +## Temporary Directory Contract + +`BuildDirectory` 保留 RAII 清理。创建目录时增加固定格式 owner marker，至少包含格式版本、创建时间、进程 ID 和当前编译器身份。清理逻辑集中在 `native/build`，不在 CLI 复制一套删除规则。 + +自动清理和显式 clean 共同调用这一逻辑： + +1. 只枚举系统临时目录直属的 `dever-build-<pid>-<id>`。 +2. 使用 `symlink_metadata`，拒绝符号链接和非真实目录。 +3. marker 必须完整匹配且创建时间达到 24 小时过期阈值。 +4. 当前进程目录永不删除；不满足任一条件即跳过。 +5. 删除前统计普通文件字节数，返回结构化 summary。 + +24 小时阈值避免误删并发或长时间构建；正常流程仍由 `Drop` 立即清理，因此阈值只影响异常退出遗留物。 + +## Run Flow + +`NativeProgram` 已拥有可执行文件和临时目录，`run_native` 直接执行 `native.executable()`。借用在子进程退出前持续有效，随后 `NativeProgram` 的 `BuildDirectory` 统一回收。删除 `StagedExecutable` 和项目根二次复制路径。 + +## Clean Command + +CLI 增加 `Action::Clean` 和 `clean <project-root>` 参数合同。launcher 把它视为普通项目命令，用项目的 `config/setting.json` 选择版本后原样分派。 + +核心不解析或修改项目源码，只调用 native build 的限定清理 API，并兼容清理项目根中符合旧 `.dever-run-<pid>` 命名、普通文件、当前用户和 24 小时阈值的历史遗留物。输出固定摘要，不把未删除的未知文件当作成功删除。 + +该命令不接触 `target/debug`；编译器贡献者仍使用 `cargo clean` 管理 Cargo 开发产物。它也不接触机器共享缓存；后者仍是 `dever cache clean`。 + +## Compatibility And Rollback + +- Dever 源码、配置、语言行为和 release 构建参数不变。 +- 现有测试目标名与定向命令不变。 +- 旧 `target/native-runtime/<profile>` 不再读取；可由 `cargo clean` 删除，不提供双路径兼容。 +- 共享 runtime target 若出现问题，可回滚 `native/build` 布局而不迁移项目数据；所有相关内容均可重建。 +- clean 只删除严格识别的过期临时产物，回滚不需要恢复数据。 diff --git a/.trellis/tasks/09-29-build-cache-storage/implement.jsonl b/.trellis/tasks/09-29-build-cache-storage/implement.jsonl new file mode 100644 index 0000000..8817898 --- /dev/null +++ b/.trellis/tasks/09-29-build-cache-storage/implement.jsonl @@ -0,0 +1,4 @@ +{"file":".trellis/spec/backend/directory-structure.md","reason":"确认 native build、CLI、测试目录和单一 owner 边界"} +{"file":".trellis/tasks/09-29-build-cache-storage/research/cache-boundaries.md","reason":"保持 locked/offline 工具链、launcher 与机器共享缓存合同"} +{"file":".trellis/spec/backend/error-handling.md","reason":"保持临时目录生命周期、原子发布和清理错误合同"} +{"file":".trellis/spec/backend/quality-guidelines.md","reason":"遵守定向测试与低磁盘验证约束"} diff --git a/.trellis/tasks/09-29-build-cache-storage/implement.md b/.trellis/tasks/09-29-build-cache-storage/implement.md new file mode 100644 index 0000000..14a10f2 --- /dev/null +++ b/.trellis/tasks/09-29-build-cache-storage/implement.md @@ -0,0 +1,67 @@ +# Implementation Plan + +## 1. Cargo Development Footprint + +- [x] 在根 `Cargo.toml` 固化 dev/test 的 `debug = 0` 与 `incremental = false`，保留 release profile。 +- [x] 增加静态回归，确认 profile 合同及所有既有 `[[test]]` 目标仍存在，不移动或合并测试源码。 + +## 2. Shared Native Runtime Target + +- [x] 把 runtime feature/profile 规范化为单一 build key，复用现有 runtime 身份与 native cache owner。 +- [x] 使用一个 `target/native-runtime/cargo` 执行 locked/offline Cargo build，并解析 Cargo JSON 产物消息。 +- [x] 原子发布只包含当前构建图精确文件的不可变 `inputs/<identity>`；复用公共文件且拒绝符号链接、缺失文件和不完整 staging。 +- [x] 让 `RuntimeInputs` 消费显式文件清单，不再扫描整个 `release/deps`。 +- [x] 增加 base、API、SQLite、PostgreSQL、component/external profile 隔离、复用、失败和并发定向测试。 + +## 3. Temporary Build Cleanup + +- [x] 为 `BuildDirectory` 增加 owner marker 与统一 cleanup summary；正常 `Drop` 行为保持。 +- [x] 在创建 native build 前执行有界的 24 小时过期清理，跳过当前、未知、未过期和符号链接目录。 +- [x] `run_native` 直接执行 `NativeProgram::executable()`，移除项目根 staging 和 `StagedExecutable`。 +- [x] 增加成功、编译错误、子进程失败、过期目录和恶意同名路径的定向回归。 + +## 4. Clean Command + +- [x] 给版本核心增加 `clean <project-root>` 解析和执行；输出删除目录、文件及字节汇总。 +- [x] 给稳定 launcher 增加项目根识别和帮助文本，保持 `dever cache clean` 的认证服务失败合同。 +- [x] 兼容回收严格匹配且过期的旧 `.dever-run-*` 普通文件，不删除显式 output 或其它隐藏文件。 +- [x] 增加 CLI 参数、版本分派、限定删除和重复 clean 幂等测试。 + +## 5. Documentation And Verification + +- [x] 更新 README、backend directory/toolchain/error/quality 规范，说明三种清理边界与默认低磁盘 profile。 +- [x] 运行 `git diff --check`。 +- [x] 运行 core/CLI/runtime 的 offline focused check，不运行 workspace 全量测试。 +- [x] 分别选择 default、API、SQLite、PostgreSQL、component 的最小定向编译或测试；PostgreSQL 只编译，不连接数据库。 +- [x] 测量 fresh target 和重复执行后的占用，验证低于 16 GiB 基线的 40% 且无变化重复增长不超过 5%。 +- [x] 复查磁盘空间、临时目录和项目根，确认没有遗留本轮测试产物。 + +## Completion Evidence (2026-09-29) + +- `cargo check --offline --locked -p dever-core -p dever-cli` passed. +- `deverc` argument tests passed 5/5; local clean, selected-version launcher dispatch and authenticated `cache clean` refusal regressions passed. +- Exact runtime input publication/reuse/concurrency regression passed. Actual native base, API, SQLite, PostgreSQL and external/component profile checks passed; PostgreSQL was compile-only and opened no database. +- The runtime layout contains one `target/native-runtime/cargo` plus content-addressed `inputs`; no per-feature Cargo target remains. Repeating the external native check changed target usage from 1,625,111,299 bytes to 1,625,111,299 bytes (0%). +- The final focused matrix used 1,642,122,909 bytes (about 1.64 GB), 10.2% of the former 16 GiB baseline and below the 6.4 GiB budget. `target/debug/incremental` is an empty Cargo-created directory (0 files, 4 KiB), not an incremental object cache. +- No `/tmp/dever-build-*` or project `.dever-run-*` artifact remained. Final filesystem availability was about 17 GiB. +- `cargo fmt --all --check` and `cargo clippy ...` were unavailable because the installed toolchain lacks rustfmt and clippy. Workspace-wide tests, real PostgreSQL, frontend builds and performance/RSS benchmarks were not run by scope. +- Two pre-existing source fixtures were not counted as task verification: `rest_model::rest_field_bindings_are_checked_and_emit_static_search_sql` references missing generated `api_require_components`/`job_migrate_tenant`, and `sqlite_orm::database_failures_are_captured_after_transaction_rollback_and_task_wait` exposes a private `ReadResult` from `public main`. The selected SQLite and PostgreSQL profile checks passed through independent valid fixtures. + +## Final Review Evidence (2026-09-29) + +- Final review found and fixed three correctness gaps: runtime publication now selects the current `dever-runtime` manifest's hashed `release/deps` rlib instead of the mutable top-level Cargo alias; runtime input views reject symbolic links and unknown entries; stale cleanup preserves artifacts owned by any still-active recorded PID, not only the current process. +- The clean regression now keeps both a marked build directory and legacy `.dever-run-*` file for a live child process. Input-view regressions cover concurrent publication, exact dependency sets, unknown files and symbolic links. +- `cargo check --offline --locked -p dever-core -p dever-cli` passed after the fixes. +- `cargo test --offline --locked -p dever-tests --test native_cache runtime_input_views -- --nocapture` passed 2/2; the real concurrent native publication test passed twice and left no temporary artifact. +- `cargo test --offline --locked -p dever-cli --test cms_project clean_removes_only_stale_owned_dever_artifacts -- --exact --nocapture` passed 1/1. +- Selected-version launcher dispatch and authenticated `cache clean` refusal passed 1/1 each; `deverc` argument tests passed 5/5; the Cargo profile/test-target contract passed 1/1. +- Repeating the same concurrent native check left `target/` at exactly 1,689,816,322 bytes (0% growth). Final follow-up checks left it at 1,689,817,710 bytes (about 1.69 GB, 10.6% of the former 16 GiB baseline). Only `cargo` and `inputs` exist directly under `target/native-runtime`; `target/debug/incremental` contains zero files. +- No `/tmp/dever-build-*` or repository-root `.dever-run-*` remained. `git diff --check` passed and about 17 GiB remained available. +- `rustfmt` and `clippy` remain unavailable because those toolchain components are not installed. Workspace-wide tests, real PostgreSQL, frontend builds and performance/RSS benchmarks remain not run by scope. + +## Risks And Rollback Points + +- Runtime input discovery depends on Cargo's JSON artifact contract. Missing or ambiguous current-manifest runtime artifacts fail explicitly instead of scanning historical files. +- Process-state detection is conservative: an inspection failure preserves the candidate, so cleanup may leave an old artifact rather than risk deleting an active one. +- The shared target and input views are rebuildable. Rollback removes this layout through `cargo clean`; no project data migration is involved. +- Native test linking can still consume significant time and space even with deduplicated runtime inputs, so broad matrices remain opt-in. diff --git a/.trellis/tasks/09-29-build-cache-storage/prd.md b/.trellis/tasks/09-29-build-cache-storage/prd.md new file mode 100644 index 0000000..5f2a18c --- /dev/null +++ b/.trellis/tasks/09-29-build-cache-storage/prd.md @@ -0,0 +1,74 @@ +# Dever 构建缓存与磁盘占用治理 + +## Goal + +在不改变 Dever 程序语义、发布构建性能和现有定向测试入口的前提下，显著降低编译器开发与原生运行时构建的磁盘占用，并让 Dever 自有临时产物可以可靠回收。 + +## Background + +- 2026-09-29 清理前，仓库 `target/` 占用约 16 GiB：`target/debug/deps` 约 11 GiB、`target/debug/incremental` 约 2.3 GiB、`target/native-runtime` 约 2.3 GiB。 +- `test/dever-tests` 有 67 个独立集成测试目标。每个目标都会单独链接 Dever runtime；清理前单个 debug 测试程序约 120-173 MiB。 +- 多个既有任务已经通过命令行反复使用 `profile.dev.debug=0`、`profile.test.debug=0` 和 `build.incremental=false` 避免磁盘耗尽，说明这些设置是已验证的仓库开发约束，但尚未固化。 +- `crates/dever-core/src/native/build.rs` 当前按 `base/database/sqlite/postgres` 与 `api/crypto/wire` 组合创建独立 Cargo target；公共依赖因此被完整复制到每个 profile 目录。 +- `BuildDirectory` 已在正常返回和普通错误时通过 `Drop` 清理，但进程被强制终止或链接器崩溃时仍会留下 `/tmp/dever-build-<pid>-<id>`。 +- `dever cache clean` 已属于机器级共享缓存合同，必须等待认证 `deverd` IPC；本任务不得以本地直写方式绕过该安全边界。 + +## Requirements + +### R1. Cargo Development Profiles + +- 在仓库 Cargo 配置中固化 dev/test 的低磁盘设置，不再依赖每条验证命令重复传入临时覆盖项。 +- dev/test 不生成完整调试信息且禁用增量对象；release 的 `fat LTO`、优化级别和最终程序行为保持不变。 +- 保留现有独立测试目标及 `cargo test --test <name>` 合同；本阶段不通过合并测试文件换取空间。 + +### R2. Shared Native Runtime Build Target + +- 所有 runtime feature profile 共享一个 Cargo 构建 target，公共依赖只保存一份。 +- 每次 native 编译只消费当前 Cargo 调用实际报告的 runtime 与依赖产物，不把其它 profile 的历史依赖混入缓存身份或链接输入。 +- 不同数据库/API/加密/wire/external profile 仍有独立、完整且可验证的 runtime 身份；并发构建不得读到正在改写的半成品。 +- 继续使用 locked/offline Cargo，不引入网络、环境配置或第二套构建协议。 + +### R3. Temporary Build Ownership + +- 正常成功、编译失败和运行结束继续即时清理本次临时目录。 +- `run` 直接执行 `NativeProgram` 已拥有的临时可执行文件，不再向项目根复制 `.dever-run-<pid>`。 +- 临时目录写入 Dever 自有标记；启动构建和显式清理只处理名称、标记、类型、属主和年龄均符合合同的目录。 +- 清理拒绝符号链接、未知目录、活跃或未达到过期阈值的目录；清理失败不得掩盖原始编译或运行错误。 + +### R4. Local Clean Command + +- 版本核心提供 `clean <project-root>`，开发入口 `deverc` 与正式 launcher 分派后的 `dever` 使用同一实现。 +- 命令只回收当前用户的 Dever 过期临时构建目录和旧版遗留的过期 `.dever-run-*`，并输出删除数量与字节数。 +- 命令不得删除显式 build 输出、项目源码/配置/data、Cargo registry、Lib 缓存、机器共享缓存或其它项目文件。 +- `dever cache clean` 的名称、失败合同和认证 IPC 边界保持不变。 + +### R5. Documentation And Regression Coverage + +- README 和 toolchain 规范明确区分 `clean`、`cache clean` 与编译器仓库维护用的 `cargo clean`。 +- 测试覆盖参数解析、限定删除、符号链接拒绝、未过期目录保留、runtime profile 隔离和共享依赖复用。 + +## Acceptance Criteria + +- [ ] dev/test profile 默认不生成完整 debuginfo，也不创建 incremental 目录；release profile 保持原值。 +- [ ] 现有 `cargo test --test <name>` 入口不变，至少一个默认、API、SQLite、PostgreSQL 和 component 定向目标可以继续被单独选择。 +- [ ] 连续准备至少两个不同 runtime profile 时只存在一个 Cargo target，公共依赖不再按 profile 复制；每个 profile 链接到自身准确的 runtime 产物。 +- [ ] 并发或失败的 runtime 准备不会发布半成品，也不会让一个 profile 复用另一个 profile 的 runtime 身份。 +- [ ] `run` 不再创建项目根 `.dever-run-*`；成功和普通失败后本次临时目录不存在。 +- [ ] `clean <project-root>` 只删除符合 Dever 标记和过期规则的自有临时产物，并报告回收结果；未知文件、符号链接、未过期目录及显式输出保持不变。 +- [ ] `dever cache clean` 仍明确走机器共享缓存服务，未实现认证 IPC 时不允许本地回退。 +- [ ] 在全新 target 上完成允许的代表性 feature 定向检查后，`target/` 总占用不超过清理前 16 GiB 基线的 40%；重复同一组检查且源码不变时占用增长不超过 5%。 +- [ ] `git diff --check` 和受影响 crate 的离线定向检查通过；不运行 workspace 全量测试、真实 PostgreSQL、前端构建或性能压测。 + +## Out Of Scope + +- 实现或绕过尚未完成的认证 `deverd` IPC 和跨用户共享缓存清理。 +- 删除用户业务数据、显式构建输出、第三方包缓存或正在使用的缓存。 +- 改变 release 优化、最终程序语义、Dever 语言语法或应用配置格式。 +- 本阶段合并 67 个测试目标；只有 profile 与 runtime 去重后仍不能达到磁盘预算时再单独评估。 +- 把 Cargo 开发目标暴露成 Dever 应用运行时配置。 + +## Key Decisions + +- 根因是完整 debuginfo、增量对象、独立测试链接和 profile target 重复，不是源码本身或最终 Dever 二进制达到 16 GiB。 +- 保留小而可定向的测试目标比合并成少量巨大 runner 更有维护价值；先使用低磁盘 Cargo profile 解决体积。 +- 本地 `clean` 和机器级 `cache clean` 是不同安全边界，不能为了“一个命令”让普通项目进程直接写共享缓存。 diff --git a/.trellis/tasks/09-29-build-cache-storage/research/cache-boundaries.md b/.trellis/tasks/09-29-build-cache-storage/research/cache-boundaries.md new file mode 100644 index 0000000..0caf59e --- /dev/null +++ b/.trellis/tasks/09-29-build-cache-storage/research/cache-boundaries.md @@ -0,0 +1,14 @@ +# Cache Boundary Evidence + +## Existing Owners + +- `.trellis/spec/backend/toolchain-and-library.md`：正式 `dever` launcher 统一选择版本并分派项目命令；机器共享缓存只能通过认证 `deverd` 服务访问，服务不可用时必须明确失败，不能退回普通用户可写目录。 +- `.trellis/tasks/09-28-shared-toolchain/prd.md`：`dever cache status/clean` 属于跨项目、跨用户的机器级缓存合同，运行条目使用租约，清理与构建必须互斥。 +- `crates/dever-cli/src/toolchain/launcher.rs:58`：当前 `cache status/clean` 调用统一返回认证缓存服务尚不可用，未直接构造 `CacheStore`。 +- `crates/dever-cli/src/toolchain/cache.rs:157`：`CacheStore::clean` 是未来特权服务内部的文件系统引擎，不是项目进程可调用的公开回退路径。 + +## Task Boundary + +- 新的 `clean <project-root>` 只能清理当前用户、当前 Dever 版本严格识别的过期临时构建目录和旧项目 staging 文件。 +- `clean` 不读取、修改或删除机器共享缓存条目，不改变 `dever cache clean` 的命令和失败合同。 +- 编译器仓库的 `target/debug` 继续由 Cargo 管理；普通 Dever 项目不会通过应用配置控制 Cargo 开发缓存。 diff --git a/.trellis/tasks/09-29-build-cache-storage/task.json b/.trellis/tasks/09-29-build-cache-storage/task.json new file mode 100644 index 0000000..f851354 --- /dev/null +++ b/.trellis/tasks/09-29-build-cache-storage/task.json @@ -0,0 +1,26 @@ +{ + "id": "build-cache-storage", + "name": "build-cache-storage", + "title": "Dever 构建缓存与磁盘占用治理", + "description": "降低 Rust 测试和原生运行时构建占用，可靠清理临时产物并提供统一清理命令", + "status": "in_progress", + "dev_type": null, + "scope": null, + "package": null, + "priority": "P2", + "creator": "Antigravity", + "assignee": "Antigravity", + "createdAt": "2026-09-29", + "completedAt": null, + "branch": "main", + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [], + "parent": null, + "relatedFiles": [], + "notes": "", + "meta": {} +} \ No newline at end of file diff --git a/.trellis/tasks/09-29-external-lib-ecosystems/check.jsonl b/.trellis/tasks/09-29-external-lib-ecosystems/check.jsonl new file mode 100644 index 0000000..4d3188d --- /dev/null +++ b/.trellis/tasks/09-29-external-lib-ecosystems/check.jsonl @@ -0,0 +1,3 @@ +{"file":".trellis/spec/backend/quality-guidelines.md","reason":"CLI、filesystem、offline fixture 和确定性测试"} +{"file":".trellis/spec/backend/error-handling.md","reason":"resolver/cache/protocol failure 分类"} +{"file":".trellis/tasks/09-29-external-lib-ecosystems/prd.md","reason":"生态子任务验收标准"} diff --git a/.trellis/tasks/09-29-external-lib-ecosystems/design.md b/.trellis/tasks/09-29-external-lib-ecosystems/design.md new file mode 100644 index 0000000..d56eab9 --- /dev/null +++ b/.trellis/tasks/09-29-external-lib-ecosystems/design.md @@ -0,0 +1,23 @@ +# Design + +## Resolver Boundary + +CLI 使用一个 closed ecosystem dispatch：parse requested spec、resolve fixture/registry metadata、fetch verified artifact、produce normalized locked graph。生态实现只处理自己的 metadata；缓存发布、摘要、路径安全、lock 排序和诊断由共享 owner 负责。 + +## Lock + +`dever.lock` 是 compiler-owned canonical document，包含 format version、Dever toolchain identity、Adapter identity、Port schema hash、ecosystem、requested spec、runtime pack、target artifacts、dependency graph 和 SHA-256。未知字段、重复键或手工修改在 doctor/project command 中拒绝并提示重新解析。 + +每个 Adapter 拥有隔离 dependency graph，因此两个 Adapter 可以锁定同一生态包的不同版本。共享 cache 只按已验证内容去重，不合并运行环境。 + +## Runtime Packs And SDK + +- pip：固定 CPython pack，优先验证 wheel；source distribution 只能进入隔离构建 sandbox。 +- npm：固定 Node pack，lifecycle scripts 默认禁用，需要的 build hook 进入隔离 sandbox。 +- go：固定 Go pack，按 target 编译 worker；需要 cgo 时必须存在受管 target toolchain，否则明确拒绝。 + +三个 SDK 共用生成的 operation/type/error schema fixture；语言层只负责数据映射和协议循环。日志写 stderr，不能读取应用 setting 文件或环境变量配置。 + +## Network Boundary + +网络只由显式 add/update resolver 使用。registry URL 和信任根属于签名工具链发行合同，不由项目或环境变量覆盖。测试使用本地 fixture registry 和 test-owned cache，不连接真实 registry。 diff --git a/.trellis/tasks/09-29-external-lib-ecosystems/implement.jsonl b/.trellis/tasks/09-29-external-lib-ecosystems/implement.jsonl new file mode 100644 index 0000000..09ca814 --- /dev/null +++ b/.trellis/tasks/09-29-external-lib-ecosystems/implement.jsonl @@ -0,0 +1,4 @@ +{"file":".trellis/spec/backend/toolchain-and-library.md","reason":"CLI、cache、offline、runtime pack 和 config boundary"} +{"file":".trellis/spec/backend/compiler-contracts.md","reason":"Adapter schema identity 与业务边界"} +{"file":".trellis/tasks/09-28-shared-toolchain/design.md","reason":"机器级共享 cache、版本和权限 owner"} +{"file":".trellis/tasks/09-29-external-lib-protocol/design.md","reason":"SDK 必须实现的唯一协议"} diff --git a/.trellis/tasks/09-29-external-lib-ecosystems/implement.md b/.trellis/tasks/09-29-external-lib-ecosystems/implement.md new file mode 100644 index 0000000..0419246 --- /dev/null +++ b/.trellis/tasks/09-29-external-lib-ecosystems/implement.md @@ -0,0 +1,42 @@ +# Implementation Plan + +- [x] 定义 requested/locked/artifact/runtime-pack 内部模型和 canonical lock codec。 +- [x] 实现共享 resolver orchestration、SHA-256、归档安全、atomic lock/cache publish 和 doctor。 +- [x] 增加 `dever lib add/list/update/remove/doctor` 并复用统一项目根/诊断处理。 +- [x] 提供 `ArtifactStore` 适配边界和 test-owned 内容寻址 fixture；正式 `deverd` IPC 仍由共享工具链任务提供。 +- [x] 实现 PyPI wheel/PEP 508/440、npm packument/semver/SRI 和 Go proxy/MVS resolver，接入已验证安装版本及签名 runtime 描述。 +- [ ] 发行正式 CPython/Node/Go runtime/build pack 并运行无宿主验收。 +- [x] 完成锁定 host-target build pack 的受管 Go compile/link、build tags、module ZIP、go:embed 与同名操作绑定；源码移除和清空环境独立 Worker 执行通过。 +- [x] 完成 Python extras/wheel/sdist/PEP517、npm 高级依赖图/安装钩子/native addon 与 Go sumdb；2026-10-02 已运行官方第三方依赖隔离构建和离线 Worker 验收，证据见父生产化任务。 +- [x] Python、JavaScript、Go SDK 协议库、checked manifest typed 校验及显式宿主验收；Python/JS 自动同名 entry 绑定已接入打包器。 +- [x] 使用本地 fixture registry 覆盖 deterministic lock、摘要、归档路径、冲突、offline 和双 target cache 隔离。 +- [x] ArtifactStore 提供 verified `get`，并拒绝摘要不匹配；npm scoped name 和 remove 依赖关系有明确诊断。 +- [x] 增加 `LibResolver`/`ProviderManifest` 边界和真实 provider 接线；仅显式 Lib 命令解析/下载，不读取 PATH 或环境变量。 +- [x] `doctor` 按各生态校验精确依赖图、可达闭包和构建收据；npm 允许合法依赖环。ArtifactStore 在 Worker 启动前验证目标和摘要。 +- [x] 编译器/CLI 接通 external Adapter 的精确 Lib 请求：支持 `lib "pip:name@version"`、`lib "npm:name@version"`、`lib "go:name@version"`，请求进入 schema identity 并参与 lock 校验。 +- [x] 更新 LANGUAGE、README、IMPLEMENTATION 与 compiler/toolchain specs，保留尚未实现和缺发行资产的边界。 +- [x] 运行 CLI/lib resolver/SDK 定向测试；只使用注入 transport、自有 loopback 镜像和显式宿主 fixture，不访问公网 registry、不运行全量。 + +## Current implementation boundary + +2026-10-02：当前锁格式为 v5，Python extras/原生 wheel/sdist 构建、npm 嵌套/peer/optional/bundled 图、安装钩子与 native 分类裁剪、Go sumdb 已实现。最终默认解析 **71/71**，Worker **7/7**，严格 CLI Clippy 通过；simplejson3.20.1、bufferutil4.0.9 和 google/uuid1.6.0 的显式真实构建与离线 Worker 均通过。命令、作者资产与运行日志统一记录于 `../09-29-language-production-closure/implement.md`，不复制历史进度为当前缺口。正式公开及其他平台资产尚未交付，任务暂不归档。以下为历史基线。 + +本阶段已完成 compiler-owned lock/resolver boundary、真实 registry resolver、机器共享资产缓存及三 SDK typed manifest。显式宿主协议验收不是正式受管 pack/无宿主发行验收；未安装签名 runtime 描述的开发入口仍明确失败，不使用系统包管理器或宿主语言环境。 + +2026-09-30：Lib 声明统一到 `config/setting.json.lib`，拒绝旧独立文件；canonical v2 lock 绑定 checked Worker contract 与 runtime（零 Lib managed Worker 也锁 runtime）。每个 Worker 独立 resolve/doctor，跨 Worker 版本可以不同。嵌入精确比对完整 provider 元数据。CLI `external_libs` 最终 27/27 通过，真实 provider 测试注入协议 transport，Go ZIP 的受限跳转使用自有 loopback HTTP 镜像；不使用 FixtureRegistry 冒充产品解析，也不声称已访问公网/发行 pack。Linux 不同 UID 的 artifact put/get、token 隔离和 owner-only clean 通过。CLI Clippy `-D warnings` 与所属文件私有 rustfmt check 通过。 + +## Review Gates + +- 不让各 resolver 重复 lock/cache/path/HTTP 验证。 +- Go ZIP 重定向仅允许固定 GCS bucket、最多两跳、总 30 秒；URL 规范化、userinfo/fragment/端口/路径检查与循环/Location 检查都在后续请求前，诊断不回显签名 URL。 +- 不在 run/build 中解析版本或联网。 +- 不允许项目 registry URL、runtime path 或 token 来自环境变量。 +- 不把生态依赖模型暴露成 Dever 业务类型。 + +## Verification + +- `cargo check --offline -p dever-cli`：通过。 +- `cargo test --offline -p dever-cli --lib libs::tests`：4/4 通过。 +- `cargo test --offline -p dever-cli --test external_libs`：3/3 通过。 +- 手工 `deverc lib add/list/doctor`：通过。 +- 未运行公网 registry 下载、正式签名 runtime 或全量测试；真实 resolver 的定向证据来自注入 transport 的协议元数据，不能等同于发行验收。 diff --git a/.trellis/tasks/09-29-external-lib-ecosystems/prd.md b/.trellis/tasks/09-29-external-lib-ecosystems/prd.md new file mode 100644 index 0000000..1d63b7c --- /dev/null +++ b/.trellis/tasks/09-29-external-lib-ecosystems/prd.md @@ -0,0 +1,37 @@ +# Dever Lib 生态解析 + +## Goal + +在统一 external Adapter/Worker 协议之上，实现确定性的 `dever lib`、`dever.lock`、机器级内容缓存以及首批 pip/npm/go resolver 和官方 SDK。 + +## Dependencies + +- `external-lib-protocol` 已完成并冻结协议版本和 external Adapter HIR。 +- `shared-toolchain` 提供机器级版本目录、权限和共享 cache API；本任务不复制该实现。 + +## Requirements + +- 命令固定为 `lib add/list/update/remove/doctor <project-root> ...`，保持 CLI 的显式项目根合同。 +- spec 使用 `pip:name@version`、`npm:name@version`、`go:module@version`；禁止无版本请求进入 lock。 +- `add/update` 解析完整传递依赖、平台 artifact、runtime pack 与摘要，写 canonical `dever.lock`；写入采用 prepared + atomic replace。 +- `list/doctor/remove` 默认不联网；`doctor` 校验声明、lock、缓存和 Adapter schema identity 一致。 +- 共享 cache 按 verified bytes 和 target identity 去重；项目之间不共享可写 dependency state。 +- pip/npm/go 各自使用固定 resolver adapter，不执行项目提供的 shell；需要构建的生态 hook 必须在声明 capability 的隔离构建 sandbox 内运行。 +- Python、JavaScript、Go SDK 实现同一个协议和生成的类型合同；SDK 不定义第二套业务 schema。 +- 所有项目命令离线消费 lock；缺失 artifact 给出需要执行的精确 `dever lib` 命令，不隐式下载。 + +## Acceptance Criteria + +- [ ] 三个生态的本地 fixture registry 解析出确定性 lock；重复解析字节一致。 +- [ ] 版本冲突、摘要错误、篡改 cache、非法归档路径、缺平台 artifact 和 schema 漂移均明确失败。 +- [ ] 两个项目共享相同 artifact bytes，但 lock、Adapter setting 和项目文件保持隔离。 +- [ ] Python/Node/Go SDK fixture 完成 handshake、success、declared error、cancel 和 shutdown。 +- [ ] PATH 中放置伪造 python/node/go 不影响选择；不读取语言相关环境变量。 +- [ ] 断网且 cache 完整时项目命令前置验证成功；cache 缺失时不联网。 +- [ ] 新 resolver 可通过固定内部接口加入，不修改 CLI 命令、lock 核心 schema 或 Worker 协议。 + +## Out Of Scope + +- Maven、NuGet、Composer、Gem 的正式 resolver。 +- 公共 registry 服务建设和真实互联网端到端测试。 +- 最终应用资源嵌入与目标机器运行。 diff --git a/.trellis/tasks/09-29-external-lib-ecosystems/task.json b/.trellis/tasks/09-29-external-lib-ecosystems/task.json new file mode 100644 index 0000000..9c92018 --- /dev/null +++ b/.trellis/tasks/09-29-external-lib-ecosystems/task.json @@ -0,0 +1,26 @@ +{ + "id": "external-lib-ecosystems", + "name": "external-lib-ecosystems", + "title": "Dever Lib 生态解析", + "description": "实现 dever lib、dever.lock、机器缓存及 pip/npm/go 依赖解析和 SDK", + "status": "in_progress", + "dev_type": null, + "scope": null, + "package": null, + "priority": "P1", + "creator": "Antigravity", + "assignee": "Antigravity", + "createdAt": "2026-09-29", + "completedAt": null, + "branch": "main", + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [], + "parent": "09-29-external-libs", + "relatedFiles": [], + "notes": "2026-10-02：高级生态产品实现和Linux实际第三方构建/Worker验收已补齐，最终external_libs71/71、workers7/7，CLI严格Clippy通过。包括extras/原生wheel/sdist/PEP517、npm复杂安装图/hooks/native分类与物理裁剪、Go sumdb及官方UUID。正式公开/其他平台资产仍未交付；保留in_progress，不将私有签名fixture冒充正式发行。准确命令和运行证据统一见09-29-language-production-closure/implement.md。", + "meta": {} +} diff --git a/.trellis/tasks/09-29-external-lib-packaging/check.jsonl b/.trellis/tasks/09-29-external-lib-packaging/check.jsonl new file mode 100644 index 0000000..911bc7c --- /dev/null +++ b/.trellis/tasks/09-29-external-lib-packaging/check.jsonl @@ -0,0 +1,4 @@ +{"file":".trellis/spec/backend/quality-guidelines.md","reason":"actual run/build、filesystem integrity 和 isolated executable tests"} +{"file":".trellis/spec/backend/error-handling.md","reason":"resource/Worker startup failure 必须保持 runtime fault"} +{"file":".trellis/spec/backend/directory-structure.md","reason":"build/runtime ownership 和 root test 位置"} +{"file":".trellis/tasks/09-29-external-lib-packaging/prd.md","reason":"打包子任务验收标准"} diff --git a/.trellis/tasks/09-29-external-lib-packaging/design.md b/.trellis/tasks/09-29-external-lib-packaging/design.md new file mode 100644 index 0000000..237c066 --- /dev/null +++ b/.trellis/tasks/09-29-external-lib-packaging/design.md @@ -0,0 +1,21 @@ +# Design + +## Build Graph + +native specialization 保留所有生产 Adapter successors。构建器从可达 external Adapter 收集 locked environment，按 target 调用 ecosystem packager，得到 `{manifest, executable/runtime files, digests}`。相同 digest 的资源在应用资源表中 intern 一次。 + +## Embedded Resource Format + +应用资源表包含 format/version/target/path/mode/length/SHA-256 和压缩 bytes。entry path 是 manifest 内部标识，不接受部署配置覆盖。生成程序启动时验证 resource table，再按需释放 selected Adapter 的完整 environment。 + +## Extraction + +根固定为可执行文件同级 `data/cache/lib`。每个 digest 使用 `.staging-<pid>-<counter>` 私有目录，写入时拒绝 symlink、父级跳转和重复 path；逐文件 fsync/权限设置/摘要复核后原子 rename。已发布目录必须复核 manifest 和 entry digest，失败即报错，不原地修补。 + +## Package Resolution + +Dever Package 发布内容保留 external declaration 和 requested Lib specs，不携带开发者机器的 lock/cache。应用项目统一解析所有直接与传递请求到自己的 `dever.lock`；每个 Adapter environment 独立，因此版本差异不需要强行全局合并。相同 artifact bytes 仍由机器 cache 和应用资源表去重。 + +## Output Contract + +保持当前单个独立可执行文件合同。外部 runtime 增加体积但不改变命令；`dever build` 输出确定性 size report，避免把 Python/Node runtime 成本误认为 Dever 核心体积。 diff --git a/.trellis/tasks/09-29-external-lib-packaging/implement.jsonl b/.trellis/tasks/09-29-external-lib-packaging/implement.jsonl new file mode 100644 index 0000000..236d1cc --- /dev/null +++ b/.trellis/tasks/09-29-external-lib-packaging/implement.jsonl @@ -0,0 +1,4 @@ +{"file":".trellis/spec/backend/toolchain-and-library.md","reason":"native build、cache、standalone output 和 runtime lifecycle"} +{"file":".trellis/spec/backend/compiler-contracts.md","reason":"closed Adapter reachability、effects 和 test fake"} +{"file":".trellis/tasks/09-06-private-compiler-distribution/prd.md","reason":"正式 target/runtime pack 与公开发行边界"} +{"file":".trellis/tasks/09-29-external-lib-ecosystems/design.md","reason":"locked environment 与 runtime pack 输入合同"} diff --git a/.trellis/tasks/09-29-external-lib-packaging/implement.md b/.trellis/tasks/09-29-external-lib-packaging/implement.md new file mode 100644 index 0000000..c52a289 --- /dev/null +++ b/.trellis/tasks/09-29-external-lib-packaging/implement.md @@ -0,0 +1,49 @@ +# Implementation Plan + +- [x] 扩展 native build graph，按可达 external Adapter 精确收集 locked environment；Adapter 声明 `lib "<ecosystem>:<name>@<version>"`，Program 导出去重后的精确请求，run/build 统一校验同一 lock。 +- [x] 定义 target packager 输出和共享 embedded resource manifest。 +- [x] 实现资源摘要、确定性排序和构建体积报告；当前 fixture 资源不做额外压缩。 +- [x] 在 runtime 实现安全 extraction、复核、atomic publish、并发首次启动和清理。 +- [x] 增加锁定依赖的传递闭包和 ArtifactStore 摘要/尺寸验证边界；Worker 启动前会拒绝缺失、环依赖或不完整 artifact。 +- [x] 让 run/build 共用 Worker preparation owner，不形成开发/生产两套路径。 +- [x] 实现 Python/JavaScript checked manifest、同名操作绑定、依赖/runtime 展开和受控启动；支持 Python 隔离、ESM/CommonJS、可信 SDK 优先加载。 +- [x] 实现锁定 build pack 的受管 Go compile/link、目标文件选择、module ZIP、go:embed 和静态操作绑定；独立 Worker 只携带编译产物与合同。 +- [ ] 交付正式签名语言 pack；自有 host-target Go pack 不代替六平台无宿主发行验收。 +- [x] 把 Dever Package external requests 接入项目 lock resolution、source loading 和 package 更新检查；保留组件级所有权与无本地回退边界。 +- [x] 保持 Adapter runtime selection；全部 checked exec 候选打包，只有 selected Worker 启动。 +- [ ] 增加无系统 runtime 的隔离执行、资源篡改/中断/并发、Package 传递依赖和 test fake 回归（当前已覆盖资源完整性、路径逃逸和重复发布 fixture）。 +- [ ] 更新 LANGUAGE、README、IMPLEMENTATION、toolchain/package/security specs。 +- [ ] 运行 run/build/package 定向测试和二进制体积分项检查；不运行全量或性能压测。 + +## Review Gates + +- 不把 `setting.json` 的当前 Adapter 选择编译进产物。 +- 不在运行时联网、解析版本或调用系统包管理器。 +- 不允许释放目录成为跨应用或跨用户的可写执行共享边界。 +- 不改变无 external Adapter 应用的资源、启动和输出合同。 + +## Current implementation boundary + +`dever lib` 已接真实 registry resolver、传递依赖和 daemon 资产缓存；packaging 把支持的锁定资源编译进 native application，入口经 `dever-runtime::external` 做私有 staging、SHA-256 复核和原子发布，`*.external.json` 输出分项体积报告。Python/JS preparation 生成 checked runner/SDK/launch manifest；`run` 与 `build` 共用同一份 lock/preparation 校验。raw pack/archive 在展开后仅为 config/exec 仍需的消费者保留，原生二进制输入按实际摘要去重；不同 Worker 的释放路径和清单仍隔离。 + +阶段收口补充：external Adapter 在声明体内列出精确 Lib 请求。编译器把请求纳入 external schema identity，CLI 从已检查 Program（含 Package 源码）收集每个 Worker 的请求，与 `config/setting.json.lib` 和 v2 `dever.lock` 精确核对。Dever Package 已接通版本/传递依赖、共享缓存、源码注入和统一 Lib 锁；不能以局部锁测试代替正式语言 pack 的发行验收。 + +真实 resolver、三 SDK typed manifest、Python/JS 自动 Worker、受管 Go compile/link、Dever Package 集成和 Linux 跨用户 artifact IPC 已实现。daemon build IPC、高级生态安装语义仍缺产品代码；正式签名 runtime pack 和无宿主目标验收未交付。宿主工具制作的自有 pack 定向用例不能等同于正式发行验收。 + +2026-09-30：实际 exec Worker 随程序嵌入，资源 identity 包含可执行位，启动绑定当前精确清单并重验字节和私有 Unix 权限。`external_resources` 6/6 通过，包括并发原子提取、路径/权限拒绝、部署目录不含源码 Worker、清空环境两次独立运行和篡改拒绝；`native_entry` 15/15、`port_adapter` 15/15、CLI `external_libs` 14/14 通过。这证明 exec 打包，不证明无 Python/Node/Go 环境的生态包执行。 + +## Verification + +2026-10-02当前状态：上方阶段记录中的v2与“daemon编译/高级生态未实现”已过时。当前使用v5 lock/源码构建收据，Linux daemon LLVM编译、签名pack/完整ELF闭包、真实第三方高级依赖和OS隔离已实现。展开输出在所有raw消费者消失后才裁剪。增强三生态签名安装/受管编译/移除源码与机器目录后的无系统语言独立执行 **1/1通过**（755.81秒），包含Python/Node原生依赖，Go第三方UUID另有独立Worker证据。Linux签名首装/双项目/真实双UID共享 **1/1通过**（231.51秒）。公开和其他平台发行仍未完成；精确限制与日志见`09-29-language-production-closure/implement.md`。 + +以下保留09-29历史检查，不能覆盖上述当前结果： + +- `cargo check --offline --workspace`：通过。 +- `cargo test --offline -p dever-runtime --features external external::tests`：1/1 通过（含路径逃逸、缓存污染和完整性校验）。 +- `cargo test --offline -p dever-cli --lib libs::tests`：4/4 通过。 +- `cargo test --offline -p dever-cli --test external_libs`：2/2 通过。 +- `cargo test --offline -p dever-tests --test native_entry project_entry_can_embed_verified_external_resources`：1/1 通过。 +- `cargo test --offline -p dever-tests --features component --test external_component`：1/1 通过。 +- `cargo test --offline -p dever-tests --test api_declarations`：16/16 通过。 +- 资源 native 完整编译/启动 smoke 曾尝试，但在当前磁盘/链接器约 120 秒限制内未完成，不能记为通过。 +- 未运行全量测试、真实 registry、真实 Python/Node/Go Worker、独立目标机运行和体积/内存压测。 diff --git a/.trellis/tasks/09-29-external-lib-packaging/prd.md b/.trellis/tasks/09-29-external-lib-packaging/prd.md new file mode 100644 index 0000000..18b0944 --- /dev/null +++ b/.trellis/tasks/09-29-external-lib-packaging/prd.md @@ -0,0 +1,36 @@ +# Dever Lib 构建与 Package 集成 + +## Goal + +把已锁定的 external Adapter 接入 `dever run/build`，生成无需目标机器安装第三方语言环境的自包含应用，并允许 Dever Package 声明和传递 Lib 依赖。 + +## Dependencies + +- `external-lib-protocol` 与 `external-lib-ecosystems` 已完成。 +- 私有编译器发行任务提供正式目标 runtime/worker packs；没有正式 pack 的目标不能标记支持。 + +## Requirements + +- `run` 在用户代码前验证 external declarations、lock、cache、runtime pack 和 schema，启动选中 Adapter Worker，并在应用生命周期结束时完整清理。 +- `build` 为目标平台准备所有运行时可选 external Adapter，不按当前 `setting.json` 偷偷裁剪闭合实现集合。 +- Worker、语言 runtime、依赖和 manifest 作为摘要保护的只读资源嵌入现有独立应用输出，不把源码、下载 cache 或构建工具链带入产物。 +- 运行时只释放到应用目录的 `data/cache/lib/<digest>`，使用私有 staging、完整性验证和原子发布；不执行已存在但摘要不匹配的文件。 +- Dever Package 中的 external declaration 是传递 Lib 请求来源；项目解析统一 lock，Package 使用者不需要直接运行 `lib add` 才能理解依赖，但缺 cache 时仍需显式联网命令。 +- Package 更新导致 Lib/schema 变化时必须更新 lock；旧 lock 不回退、猜测或运行。 +- 同一项目的不同 Adapter 可以携带不同 runtime/版本；构建报告分项显示体积，不隐藏大 runtime 成本。 + +## Acceptance Criteria + +- [ ] `run` 对 exec/Python/Node/Go fixture 自动启动、调用和关闭 Worker，现有纯 Dever 项目行为不变。 +- [ ] `build` 输出在没有 Python、Node、Go 和 Dever 编译器的隔离目标环境运行成功。 +- [ ] 资源截断、替换、权限错误、staging 中断和并发首次启动不会执行未验证或半成品 Worker。 +- [ ] 切换 `adapter.<port>.use` 不重新构建即可选择已打包候选；未打包/未知选择启动失败。 +- [ ] 安装一个携带 Lib 的 Dever Package 后，项目 lock 可确定性解析并离线重建。 +- [ ] 构建报告列出每个 runtime/Worker/依赖的压缩与释放体积，重复内容只嵌入一次。 +- [ ] application tests 继续使用 case fake，不释放或启动生产 Lib。 + +## Out Of Scope + +- 动态从生产网络下载缺失 Worker。 +- 把不同应用的运行时释放目录合并为全局可写执行目录。 +- FFI、共享库加载和外语源码调试器集成。 diff --git a/.trellis/tasks/09-29-external-lib-packaging/task.json b/.trellis/tasks/09-29-external-lib-packaging/task.json new file mode 100644 index 0000000..75a32a5 --- /dev/null +++ b/.trellis/tasks/09-29-external-lib-packaging/task.json @@ -0,0 +1,26 @@ +{ + "id": "external-lib-packaging", + "name": "external-lib-packaging", + "title": "Dever Lib 构建与 Package 集成", + "description": "将外部 Lib 接入 run/build 自包含打包并允许 Dever Package 传递 Lib 依赖", + "status": "in_progress", + "dev_type": null, + "scope": null, + "package": null, + "priority": "P1", + "creator": "Antigravity", + "assignee": "Antigravity", + "createdAt": "2026-09-29", + "completedAt": null, + "branch": "main", + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [], + "parent": "09-29-external-libs", + "relatedFiles": [], + "notes": "2026-10-02：LLVM run/build、Package传递Lib、v5源码构建收据、精确runtime/产物身份、二进制资源附件及npm展开输出裁剪已接线。默认消费者71/7/12/11及当前native_release17/17通过。最终三生态签名安装/受管编译/移除源码和机器目录/无系统语言独立执行1/1通过755.81秒，含simplejson C扩展与bufferutil addon；Linux签名首装双项目双UID验收1/1通过231.51秒。正式公开/其他平台资源仍未交付，详细证据见09-29-language-production-closure/implement.md。", + "meta": {} +} diff --git a/.trellis/tasks/09-29-external-lib-protocol/check.jsonl b/.trellis/tasks/09-29-external-lib-protocol/check.jsonl new file mode 100644 index 0000000..cbd48a4 --- /dev/null +++ b/.trellis/tasks/09-29-external-lib-protocol/check.jsonl @@ -0,0 +1,4 @@ +{"file":".trellis/spec/backend/quality-guidelines.md","reason":"parser、native process 和 bounded resource 定向测试"} +{"file":".trellis/spec/backend/error-handling.md","reason":"business error 与 runtime fault 不得混淆"} +{"file":".trellis/spec/backend/directory-structure.md","reason":"compiler/runtime/CLI owner 和测试目录规则"} +{"file":".trellis/tasks/09-29-external-lib-protocol/prd.md","reason":"协议子任务验收标准"} diff --git a/.trellis/tasks/09-29-external-lib-protocol/design.md b/.trellis/tasks/09-29-external-lib-protocol/design.md new file mode 100644 index 0000000..03f7385 --- /dev/null +++ b/.trellis/tasks/09-29-external-lib-protocol/design.md @@ -0,0 +1,23 @@ +# Design + +## Compiler Model + +HIR `Adapter` 增加 closed implementation kind：`Dever` 或 `External`. `External` 保存 ecosystem、entry、capabilities 和 compiler-generated Port wire schema。Port 仍是唯一 Function ID；dependency/effect/failure owners 看到外部实现的显式摘要，不解析外语源码。 + +Parser 仅在 Adapter role 把 `external`、`lib` 和 `allow` 识别为 contextual syntax。首个子任务只接受 `external exec`，但 AST/HIR ecosystem 使用封闭 enum，后续 pip/npm/go 不复制 parser 或 dispatch。 + +## Wire + +每帧为 `u32 big-endian length + UTF-8 JSON`，最大正文复用 runtime wire 16 MiB 限制。消息包含固定 `kind` 与按 kind 严格字段；payload 使用 compiler-generated concrete codec，不把动态 Node 交给业务。 + +启动序列：supervisor 启动进程 -> `hello` -> Worker `ready` -> admission。关闭序列：停止 admission -> cancel pending -> `shutdown` -> join -> bounded kill。协议 EOF、非法 frame 或进程退出关闭全部 pending request，并保留第一个 runtime fault。 + +## Process Ownership + +新增 runtime `component` owner，封装 Child、stdin writer、stdout reader、stderr bounded logger、pending request table 和 Scope guard。所有 task 注册到现有结构化作用域；Drop 只能触发已有 supervisor 清理，不能 detached kill/reap。 + +不使用 shell、PATH 或应用环境变量。entry 由构建器规范化为当前项目/产物拥有的文件；运行器传入专用管道和固定协议参数。初期 sandbox adapter 提供统一拒绝/允许合同，平台强化放在后续生态/打包子任务，不假装纯 Rust 检查等于 OS sandbox。 + +## Failure Semantics + +Worker 返回的业务 error 必须匹配 Port declared error identity。启动、协议、timeout、crash 和 sandbox 失败转为 source-located runtime fault，不伪造业务 variant，不自动重放请求。崩溃后允许下一次调用创建新 Worker，但有固定启动频率上限。 diff --git a/.trellis/tasks/09-29-external-lib-protocol/implement.jsonl b/.trellis/tasks/09-29-external-lib-protocol/implement.jsonl new file mode 100644 index 0000000..2b1644b --- /dev/null +++ b/.trellis/tasks/09-29-external-lib-protocol/implement.jsonl @@ -0,0 +1,4 @@ +{"file":".trellis/spec/backend/compiler-contracts.md","reason":"external Adapter 必须复用 Port、role、failure/effect 和 test fake 合同"} +{"file":".trellis/spec/backend/toolchain-and-library.md","reason":"wire、native emission、settings 和 structured lifecycle 合同"} +{"file":".trellis/tasks/09-20-port-adapter/design.md","reason":"现有 Adapter registration、closed dispatch 和 setting 设计"} +{"file":".trellis/tasks/09-29-external-libs/design.md","reason":"父任务协议、source 和安全边界"} diff --git a/.trellis/tasks/09-29-external-lib-protocol/implement.md b/.trellis/tasks/09-29-external-lib-protocol/implement.md new file mode 100644 index 0000000..7d1daad --- /dev/null +++ b/.trellis/tasks/09-29-external-lib-protocol/implement.md @@ -0,0 +1,40 @@ +# Implementation Plan + +- [x] 扩展 syntax/parser/formatter/Markdown，仅在 Adapter role 接受 external 声明并保持精确诊断。 +- [x] 扩展 HIR/Port 注册、role matrix、dependencies/effects/failures/specialization，复用现有 Adapter owner。 +- [x] 从 Port/setting 类型生成 Component Protocol schema hash 和 native codecs。 +- [x] 在 runtime 增加有界 frame codec、消息状态机和严格请求 ID 管理。 +- [x] 实现结构化 Worker supervisor、stdin/stdout/stderr、timeout/cancel/shutdown 和 crash cleanup。 +- [x] 将 external Port dispatch 接入 native emitter；reference 明确拒绝。 +- [x] 在仓库根 `test/` 增加 parser/contract/protocol/lifecycle/Port 回归，fixture 只使用测试自有可执行程序。 +- [x] 更新 LANGUAGE、IMPLEMENTATION、compiler/toolchain/directory/error specs。 +- [x] 运行受影响 crate check 及 external protocol、port_adapter、application_testing 定向测试；不运行全量或外部服务。 + +## Review Gates + +2026-10-02补充：以下09-29记录为历史切片。当前Rust/LLVM入口共用Scope-owned Worker与Linux namespace/seccomp；kernel能力/资源/后代清理、协议及LLVM定向已通过。GPU/其他平台拒绝，受限AppArmor非root入口尚未收口。准确日志与最终组合状态见`09-29-language-production-closure/implement.md`。 + +- 不新增第二套 JSON parser、effect graph、Adapter registry 或进程作用域。 +- 不允许外语 entry、Lib 或 capability 成为 App 可见类型。 +- 不使用 shell command string、PATH lookup、环境变量配置、detached process 或自动 retry。 + +## Implementation Record (2026-09-29) + +- Adapter-only `external exec` syntax、format 和 Markdown declaration contract 已接通；entry 只接受领域内的可移植相对路径，capability 固定为 `network/file/process/gpu`。 +- checker 从同领域唯一 Port 合成完整实现，复用原有 Adapter 选择、setting、wire、effect、failure、dependency 和 specialization owner；schema checksum 绑定 setting、操作、顶层 wire 字段名、具体类型和业务错误 payload。 +- native 生成具体输入/输出/error codec，正式调用统一进入 runtime Component Protocol；reference 对可达外部实现明确报不支持。 +- runtime Worker 使用显式 canonical entry、清空环境和固定参数启动，具有有界队列、启动频率、frame/JSON 限额、严格消息字段/request id、health、timeout/cancel drain、crash restart 和 bounded shutdown。关闭时先封闭 admission，再取消在途调用并丢弃排队调用；所有协议写入有期限，fault 和 shutdown 都在 supervisor 内显式 kill/wait。协议 fault 先关闭旧 receiver，再允许下一次调用重建 Worker，不继续复用已失配流。 +- 生成应用入口在 success、business error 和 runtime fault 后统一调用 `component::shutdown()`，随后才关闭数据库；主错误不被 cleanup error 覆盖。 +- capability 当前保证为编译期固定集合/effect 汇总以及 `ready` 精确回显校验；没有实现或宣称 OS 级 syscall/network/filesystem/GPU sandbox。 +- 未实现 resolver、`dever.lock`、机器级共享缓存、最终 build 资源嵌入、shell/PATH/env fallback、detached process 或 FFI。 + +## Verification Record (2026-09-29) + +- 初次 native 复验因根盘仅余约 156 MiB 无法暂存 runtime；仅删除仓库内可重建的 `target/debug/incremental` 后释放约 8.7 GiB，随后完成下列复验。未删除 Cargo 依赖缓存、native runtime 缓存、源码或用户数据。 +- PASS: `cargo check --offline -p dever-runtime --features wire`。 +- PASS: `cargo check --offline -p dever-core`。 +- PASS: `external_component` 1/1；覆盖 setting handshake、health、两次调用、业务 error、主动 cancel、timeout、crash/restart、fault cleanup、shutdown、80 个并发/排队调用的 admission closure、重复字段、乱序 ID、未知 kind、超大和截断 frame。 +- PASS: 完整 `port_adapter` target 14/14；覆盖 Adapter-only parser/check/format/Markdown、setting 与顶层 wire 名称的 schema 漂移、effect/failure、test fake、native Worker 端到端 dispatch 和 cleanup。 +- PASS: `application_testing` 8/8；application-test 发现、隔离、错误和 native suite 回归通过。 +- UNAVAILABLE: `cargo fmt --all -- --check` 与 Clippy；stable toolchain 未安装 rustfmt/cargo-fmt 和 cargo-clippy component。 +- NOT RUN: 全量、外部服务、数据库和性能测试；遵守本任务定向验证范围。 diff --git a/.trellis/tasks/09-29-external-lib-protocol/prd.md b/.trellis/tasks/09-29-external-lib-protocol/prd.md new file mode 100644 index 0000000..f0edecc --- /dev/null +++ b/.trellis/tasks/09-29-external-lib-protocol/prd.md @@ -0,0 +1,32 @@ +# Dever Lib 协议与 Worker + +## Goal + +在不引入任何具体语言包管理器的前提下，完成外部 Adapter 的语言合同、统一进程协议、通用 `exec` Worker 和安全生命周期，为所有后续生态提供唯一运行边界。 + +## Requirements + +- 仅 Adapter role 接受一个 contextual `external exec "relative/entry"` 声明；它可与一个现有 `setting` record 共存，不可与 Dever 实现函数混合。 +- 外部 Adapter 自动完整实现其对应 Port；缺 Port、多个 Port identity、非法 entry、Lib/capability 字段或不完整 Worker handshake 均在入口前失败。 +- 初始 capability 固定为 `network`、`file`、`process`、`gpu`；未知、重复或与平台不兼容的 capability 拒绝。 +- 复用 Port 的类型/failure/effect/setting schema，不向 App 暴露 Worker/JSON/进程类型。 +- 实现版本化、长度分帧、预算受限的 Component Protocol，并拒绝重复字段、未知消息、重复/未知请求 ID、错误 schema 和非法 error variant。 +- Worker 由应用 Scope 拥有，使用显式可执行路径和清理后的环境，不通过 shell 或 PATH 查找。 +- Test 继续要求 case-local Port fake，不启动生产 Worker，不读取 Adapter setting。 +- reference backend 对 external Adapter 返回明确不支持；native backend 完成正式执行，不增加模拟 fallback。 + +## Acceptance Criteria + +- [ ] 合法 external Adapter 通过 check/format/Markdown 合同，并参与现有 Port effect/failure/specialization。 +- [ ] 通用 fixture Worker 完成 setting handshake、两次调用、业务 error、取消、健康检查和正常关闭。 +- [ ] schema/Port/Adapter/protocol 不匹配、超大/截断/重复/乱序消息均在有界时间失败。 +- [ ] Worker 启动失败、崩溃、超时、调用取消和主程序退出不遗留子进程或悬挂任务。 +- [ ] stdout 只能承载协议，stderr 日志不会破坏响应；Secret 和输入值不进入协议错误。 +- [ ] 未声明 capability 的受控操作被 supervisor 拒绝；声明 capability 进入 Port 保守 effect。 +- [ ] 现有 Dever Adapter 选择和 application-test fake 回归保持通过。 + +## Out Of Scope + +- pip/npm/go resolver、SDK、联网下载和项目 lock。 +- 把 Worker 资源嵌入最终 build 产物。 +- FFI、动态库、任意 shell 和远程 Worker。 diff --git a/.trellis/tasks/09-29-external-lib-protocol/task.json b/.trellis/tasks/09-29-external-lib-protocol/task.json new file mode 100644 index 0000000..88bb6ab --- /dev/null +++ b/.trellis/tasks/09-29-external-lib-protocol/task.json @@ -0,0 +1,26 @@ +{ + "id": "external-lib-protocol", + "name": "external-lib-protocol", + "title": "Dever Lib 协议与 Worker", + "description": "实现 external Adapter 合同、统一组件协议、通用 exec Worker 与安全生命周期", + "status": "in_progress", + "dev_type": null, + "scope": null, + "package": null, + "priority": "P1", + "creator": "Antigravity", + "assignee": "Antigravity", + "createdAt": "2026-09-29", + "completedAt": null, + "branch": "main", + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [], + "parent": "09-29-external-libs", + "relatedFiles": [], + "notes": "2026-10-02：external Adapter、统一协议/Scope生命周期已接Rust及LLVM。Linux exec/managed共用OS namespace/seccomp边界，真实内核拒绝、线程/子进程/取消回收与协议回归通过；不再只有capability回显。GPU和未支持平台明确拒绝，受限AppArmor非root验收未通过。当前证据统一见09-29-language-production-closure/implement.md，保留未完成平台边界。", + "meta": {} +} diff --git a/.trellis/tasks/09-29-external-libs/check.jsonl b/.trellis/tasks/09-29-external-libs/check.jsonl new file mode 100644 index 0000000..e743f06 --- /dev/null +++ b/.trellis/tasks/09-29-external-libs/check.jsonl @@ -0,0 +1,4 @@ +{"file":".trellis/spec/backend/quality-guidelines.md","reason":"定向测试、真实 CLI 和资源生命周期验证要求"} +{"file":".trellis/spec/backend/error-handling.md","reason":"业务 error、runtime fault 和协议错误边界"} +{"file":".trellis/spec/backend/directory-structure.md","reason":"新增实现必须归属现有 compiler/runtime/CLI owners"} +{"file":".trellis/tasks/09-29-external-libs/prd.md","reason":"父任务跨子任务验收标准"} diff --git a/.trellis/tasks/09-29-external-libs/design.md b/.trellis/tasks/09-29-external-libs/design.md new file mode 100644 index 0000000..8a1c33a --- /dev/null +++ b/.trellis/tasks/09-29-external-libs/design.md @@ -0,0 +1,62 @@ +# Design + +## Architecture + +```text +App -> typed Port -> selected Adapter -> Worker supervisor -> Component Protocol -> Lib +``` + +Port 保持现有 compiler-owned Function ID 和闭合实现集合。外部 Adapter 是新的实现种类，不是新业务调用层；checker 从 Port 合同合成 Worker operation schema，并把显式 capability 纳入既有 effect/dependency 分析。 + +## Source Contract + +建议外部 Adapter 保持现有 topic identity： + +```text +module/ai/embedding/port.dever +module/ai/embedding/adapter/python.dever +module/ai/embedding/adapter/python/main.py +``` + +```dever +setting { + model: Text +} + +external pip "python/main.py" { + lib "sentence-transformers@3.3.1" + allow gpu +} +``` + +一个外部 Adapter 文件恰好有一个 `external` 声明，可选现有 typed `setting`，没有 Dever operation body/helper。它由编译器合成该 Port 的全部实现，Worker handshake 必须提供全部 operation。入口必须是所属 Adapter 目录内的规范化相对路径。 + +## Component Protocol + +传输使用专用双向管道和四字节大端长度分帧 JSON，复用现有 wire codec 的 16 MiB、64 层、65536 值预算。stderr 只承载日志，不能进入协议。 + +消息固定为：`hello/ready/call/result/error/cancel/health/shutdown`。每次握手包含协议版本、Port identity、schema hash、Adapter identity 和 capability 集；不匹配立即结束 Worker。请求 ID 在单 Worker 生命周期内唯一，输出必须属于当前未完成请求。 + +## Worker Ownership + +每个选中外部 Adapter 启动一个长驻 Worker 和有界请求队列。supervisor 属于现有应用 Scope；入口启动完成后才允许用户代码，关闭时停止 admission、取消未完成请求、发送 shutdown 并在期限内 join/kill。崩溃使当前请求 fault；后续调用可以受限重启，但绝不重放已发送请求。 + +## Lib Resolution + +生态 resolver 共用请求/锁模型：requested spec -> exact artifacts -> content hashes -> runtime pack -> target set。resolver 只能在显式 `lib add/update` 联网，结果写 canonical `dever.lock` 并原子替换。项目命令只验证 lock 与缓存，不重新解析版本。 + +机器级 cache 和 runtime pack 消费 `09-28-shared-toolchain` 的全局工具链合同；在该合同完成前，生态子任务只使用测试自有 fixture store，不新增临时用户缓存协议。 + +## Build Packaging + +每个 Adapter 生成目标专用 Worker artifact。`build` 将所有运行时可选 Adapter、runtime pack、lock manifest 和摘要作为只读资源嵌入应用。首次使用时写入 `data/cache/lib/<digest>` 的私有 staging，校验后原子发布；主程序只执行匹配 manifest 的文件。 + +## Extension Boundary + +新增生态只实现四个固定面：spec parser、resolver、runtime/build packager、SDK。它不能扩展语法、Port 协议、业务类型系统、setting 来源或进程生命周期。 + +## Dependencies + +- 通用协议子任务可以先实现和验证。 +- 生态解析依赖机器级共享工具链目录/缓存合同。 +- 生产自包含打包依赖父发行任务提供可分发 runtime/target pack；本地 fixture 可以先验证封装格式。 diff --git a/.trellis/tasks/09-29-external-libs/implement.jsonl b/.trellis/tasks/09-29-external-libs/implement.jsonl new file mode 100644 index 0000000..b307cb0 --- /dev/null +++ b/.trellis/tasks/09-29-external-libs/implement.jsonl @@ -0,0 +1,5 @@ +{"file":".trellis/spec/backend/compiler-contracts.md","reason":"Port/Adapter、role visibility、failure/effect 和 application testing 合同"} +{"file":".trellis/spec/backend/toolchain-and-library.md","reason":"wire、settings、native build、CLI、runtime 生命周期和离线工具链合同"} +{"file":".trellis/tasks/09-20-port-adapter/design.md","reason":"复用现有闭合 Port dispatch、typed setting 与 test fake 设计"} +{"file":".trellis/tasks/09-28-shared-toolchain/design.md","reason":"复用机器级全局命令、版本和安全共享缓存边界"} +{"file":".trellis/tasks/09-06-private-compiler-distribution/prd.md","reason":"约束最终 runtime/target pack 与公开发行依赖"} diff --git a/.trellis/tasks/09-29-external-libs/implement.md b/.trellis/tasks/09-29-external-libs/implement.md new file mode 100644 index 0000000..0285f25 --- /dev/null +++ b/.trellis/tasks/09-29-external-libs/implement.md @@ -0,0 +1,33 @@ +# Implementation Plan + +- [x] 完成并验收 `external-lib-protocol`，冻结语法、wire 和生命周期合同。 +- [x] 完成全局工具链共享缓存的依赖接口，不在 Lib 任务复制机器安装/版本管理。 +- [x] 完成并验收 `external-lib-ecosystems` 的离线确定性边界，以本地 fixture registry 覆盖 lock/doctor/artifact 校验。 +- [x] 完成并验收 `external-lib-packaging` 的 fixture 资源打包、摘要复核和私有 extraction 边界。 +- [x] 跨子任务复核 Port effects/failures、setting/Secret、测试 fake、缓存身份和 `run/build` 行为。 +- [x] 更新 LANGUAGE、README、IMPLEMENTATION 和 backend specs，删除已被正式合同替代的占位说明。 +- [ ] 只运行各子任务列出的定向检查；不默认运行全量、真实外部 registry、服务或性能测试。 + +## 当前收口补充（2026-09-29） + +- [x] 已增加 `ProviderManifest`/`LibResolver` 边界，fixture provider 与真实 pip/npm/Go provider 明确分离。 +- [x] `dever.lock` doctor 现在校验传递依赖闭包、环依赖和缺失依赖；ArtifactStore 在 Worker 使用前可验证目标、摘要和尺寸。 +- [x] 真实三生态 registry resolver、checked manifest 三 SDK、Python/JS 自动 Worker、Linux 不同 UID 的共享 artifact IPC 已接线并定向验收；当前证据见下方和生产化收口任务。 +- [x] Go managed compile/link 和 Dever Package 安装/传递 Lib 已实现并有定向 fixture 证据；具体完整链路与当前计数见生产化收口任务。 +- [x] 高级生态安装/Go sumdb已在2026-10-02完成Linux实现及真实第三方包定向验收。 +- [ ] 正式公开签名语言packs与其他平台发行尚未交付。缺少时明确失败，不回退到PATH、环境变量或宿主包管理器。当前Linux增强签名程序复验见生产化任务，历史fixture不替代新证据。 + +## 2026-09-29 收口复验 + +- `cargo check --offline --locked --workspace`、`external_libs` 6/6、CLI Lib 单测 4/4、Port Adapter Lib 请求定向 1/1 通过。 +- 本阶段已完成仓库内可安全验证的协议、lock、fixture resolver、artifact 校验、embedded resource 和 run/build preparation；真实 pip/npm/Go registry、受管 CPython/Node/Go runtime、正式三语言 SDK、Package 发布元数据和无宿主运行仍为发行资产阻塞。 + +## Integration Gate + +2026-09-30 当前边界：Port/Adapter 16/16、协议 lifecycle 1/1、typed 三 SDK 宿主协议、Python/JS 默认 Worker 6/6、资源 9/9、Linux cache 14/14 与实际不同 UID 1/1 已通过。managed Node 已由 checked App CMD 编进独立 Native 程序；受管 Go Worker 已按锁编译并完成源码移除、清空环境的独立执行；Package 已实际完成安装→check/test/run/build→停止 daemon→独立运行。自有工具包不等于正式签名或无宿主发行；高级生态安装、daemon 编译和正式后端仍有未实现代码，不仅是“发行资产阻塞”。 + +三个子任务完成后，使用同一 Port 的 Dever/exec/Python/Node/Go 实现进行定向端到端验证，并证明目标环境没有系统 Python、Node、Go 时构建产物仍可运行。 + +## Rollback + +外部 Adapter 是新语法且没有旧兼容面。若某子任务未通过，不启用对应生态或打包入口；现有纯 Dever Port/Adapter、run/build 与配置合同保持可用。 diff --git a/.trellis/tasks/09-29-external-libs/prd.md b/.trellis/tasks/09-29-external-libs/prd.md new file mode 100644 index 0000000..08aae82 --- /dev/null +++ b/.trellis/tasks/09-29-external-libs/prd.md @@ -0,0 +1,77 @@ +# Dever 外部 Lib 生态 + +## Goal + +让 Dever Package 通过现有 Port/Adapter 边界安全复用 PyPI、npm、Go Modules 及后续其它语言生态。用户统一使用 `dever lib` 管理依赖，`dever run/build` 自动管理 Worker；业务源码不直接调用外语函数，也不接触语言工具链、环境变量或进程协议。 + +## Definitions + +- **Package**：Dever 组件的发行单位，包含 `.dever` 业务源码并接受完整静态检查。 +- **Lib**：PyPI、npm、Go Modules 等外部生态依赖。 +- **Adapter**：Port 与 Lib 之间唯一合法的桥接层。 +- **Worker**：运行外部 Adapter 的隔离进程，不是用户可调用的业务层。 + +## Requirements + +### Language Boundary + +- 外部 Adapter 继续属于 `module/<component>/<domain>/adapter`，完整实现同领域一个 Port。 +- App 只调用 Port；Domain、Model、API、Job 不直接调用 Lib、Worker 或外部 SDK。 +- `port.dever` 仍是输入、输出、failure、effect 和 schema 的唯一来源；不得从 Python annotation、TypeScript 类型或 Go 接口反向定义业务合同。 +- 外部 Adapter 使用严格的 `external <ecosystem> <entry>` 声明，可包含 Lib 与 capability 声明；禁止任意 shell/build 命令。 +- 普通 Dever Adapter、测试 Port fake 与现有运行时选择合同保持不变。 + +### Protocol And Lifecycle + +- 所有语言共用一个版本化 Dever Component Protocol，而不是在编译器中增加 `python_call`、`go_call`、`js_call` 特例。 +- 协议支持 handshake、schema identity、typed setting、call、result/error、cancel、health 和 shutdown。 +- Worker 长驻并由结构化生命周期拥有；启动失败、崩溃、超时、取消和关闭均有界，不产生 detached 进程。 +- 业务错误只能返回 Port 声明的 error variant；协议破坏、进程崩溃和非法输出成为安全 runtime fault，不自动重试。 + +### Dependency Management + +- 用户命令为 `dever lib add/list/update/remove/doctor`；生态使用 `pip:`、`npm:`、`go:` 等明确前缀。 +- 解析结果写入编译器生成的项目根 `dever.lock`，记录精确版本、传递依赖、内容摘要、runtime、目标平台和 Port schema identity。 +- `dever.lock` 是可提交、不可手改的构建输入，不替代只负责部署配置和 Secret 的 `config/setting.json`。 +- 只有显式 `lib add/update` 可以联网；`check/test/run/build` 默认离线且不得从用户 PATH 或环境变量选择语言 runtime。 +- 首批正式生态是 `exec`、`pip`、`npm` 和 `go`；Maven、NuGet、Composer、Gem 通过同一扩展合同后续增加，不修改语言核心和协议。 + +### Security And Capabilities + +- 外部源码不受 Dever 内部静态检查，因此默认无网络、项目外文件、子进程、设备和 GPU 权限。 +- Adapter 必须显式声明所需 capability；声明参与 Port effect 的保守分析，并由 Worker sandbox 执行。 +- Worker 只接收当前 Adapter 的 typed setting，不读取完整 `setting.json`；Secret 不进入日志、协议诊断或缓存身份明文。 +- 安装脚本、路径、符号链接、归档解包和原生产物必须经过边界、摘要和所有权校验。 + +### Run, Build And Package + +- `dever run` 从机器级 Dever 缓存准备并监督选中的 Worker，不要求用户手动启动服务。 +- `dever build` 把所有可运行时选择的外部 Adapter、runtime 与锁定依赖封装进自包含应用；目标机器不需要 Python、Node、Go 或包管理器。 +- 嵌入资源以内容摘要验证后释放到应用 `data/cache/lib`，不覆盖项目源码、配置或用户文件。 +- Dever Package 可以携带外部 Adapter 声明；普通使用者只执行 `dever package add`，项目解析时把传递 Lib 统一锁定。 +- 不同 Adapter 的依赖环境隔离，允许同一外部包的不同版本共存；机器缓存按内容去重。 + +## Acceptance Criteria + +- [ ] 同一个 Port 可以分别由 Dever、`exec`、Python、JavaScript 和 Go Adapter 实现，App 调用代码不变。 +- [ ] 协议 schema、operation、输入、输出或 error 不匹配在用户业务代码执行前失败。 +- [ ] Worker 崩溃、取消、超时和主程序关闭不会遗留进程或把半条响应当成功结果。 +- [ ] `lib add/update` 产生确定性 lock；断网 `check/run/build` 使用相同 lock 和机器缓存成功，缺失依赖明确失败且不隐式联网。 +- [ ] Python、Node 和 Go 组件在目标机器没有对应系统 runtime/toolchain 时仍可由构建产物运行。 +- [ ] 未声明 capability 的网络、文件、进程和设备访问被拒绝；Secret 与其他项目路径不出现在诊断或状态输出。 +- [ ] 一个 Dever Package 可以封装外部 Lib，另一个项目只安装该 Package 即可构建和运行。 +- [ ] 新增第四种生态只实现 resolver/runtime pack/SDK，不修改 Port、Component Protocol 或 App 调用规则。 + +## Out Of Scope + +- FFI、动态库加载和同进程外语解释器。 +- 自动把任意第三方函数、类或对象模型暴露成 Dever API；普通生态库仍需要薄 Adapter bridge。 +- 云端编译、源码上传或运行时联网安装。 +- 外部 Adapter 热切换、请求级实现选择和无约束自动重试。 +- 修改现有 Dever Package/Component/Domain 的业务命名和调用规则。 + +## Child Deliverables + +1. `09-29-external-lib-protocol`：语言合同、统一协议、通用 `exec` Worker 和生命周期。 +2. `09-29-external-lib-ecosystems`：`dever lib`、`dever.lock`、共享缓存及 pip/npm/go resolver 和 SDK。 +3. `09-29-external-lib-packaging`：`run/build` 自包含打包与 Dever Package 传递依赖。 diff --git a/.trellis/tasks/09-29-external-libs/task.json b/.trellis/tasks/09-29-external-libs/task.json new file mode 100644 index 0000000..1f3d544 --- /dev/null +++ b/.trellis/tasks/09-29-external-libs/task.json @@ -0,0 +1,30 @@ +{ + "id": "external-libs", + "name": "external-libs", + "title": "Dever 外部 Lib 生态", + "description": "让 Dever Package 通过 Port/Adapter 安全复用 PyPI、npm、Go Modules 及其它语言生态，并由统一 Lib 命令、协议和构建链管理", + "status": "in_progress", + "dev_type": null, + "scope": null, + "package": null, + "priority": "P1", + "creator": "Antigravity", + "assignee": "Antigravity", + "createdAt": "2026-09-29", + "completedAt": null, + "branch": null, + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [ + "09-29-external-lib-protocol", + "09-29-external-lib-ecosystems", + "09-29-external-lib-packaging" + ], + "parent": "09-20-production-application-language", + "relatedFiles": [], + "notes": "2026-10-02：协议/Worker、三生态高级解析构建、v5锁与收据、Package传递Lib、LLVM独立打包及Linux OS隔离已实施；原planning状态过时。官方simplejson/bufferutil/UUID构建及离线Worker通过。最终三生态签名安装、受管check/run/build及移除源码/机器目录后的无系统语言独立执行1/1通过755.81秒，Linux首装双项目双UID共享1/1通过231.51秒。正式公开和其他平台发行仍未完成。准确状态及日志统一见09-29-language-production-closure/implement.md。", + "meta": {} +} diff --git a/.trellis/tasks/09-29-language-production-closure/check.jsonl b/.trellis/tasks/09-29-language-production-closure/check.jsonl new file mode 100644 index 0000000..397986f --- /dev/null +++ b/.trellis/tasks/09-29-language-production-closure/check.jsonl @@ -0,0 +1,12 @@ +{"file":".trellis/tasks/09-29-language-production-closure/prd.md","reason":"按总任务验收标准检查范围是否完整"} +{"file":".trellis/tasks/09-29-language-production-closure/design.md","reason":"检查 API、Lib、工具链边界是否重复或越界"} +{"file":"LANGUAGE.md","reason":"核对当前语言已交付和未交付边界"} +{"file":"IMPLEMENTATION.md","reason":"核对开发编译器与正式发行路径的实际差异"} +{"file":".trellis/tasks/09-20-production-api/prd.md","reason":"本批LLVM API/REST/auth/tenant最新业务安全合同与原实现等价"} +{"file":".trellis/spec/backend/compiler-contracts.md","reason":"typed roots/reachability/failure/affine Upload不放宽"} +{"file":".trellis/spec/backend/database-guidelines.md","reason":"原事务/owned Session/数据库隔离和真实fixture"} +{"file":".trellis/spec/backend/toolchain-and-library.md","reason":"canonical ABI/布局/profile/清理"} +{"file":".trellis/spec/backend/logging-guidelines.md","reason":"单logger、配置owner和敏感信息边界"} +{"file":".trellis/tasks/09-19-application-testing/design.md","reason":"LLVM Test使用case-local checked/fake/数据库/虚拟时钟，保持单suite和原失败合同"} +{"file": ".trellis/tasks/09-29-external-lib-protocol/prd.md", "reason": "沿用 external Adapter 协议、capability、超时取消与业务错误边界"} +{"file": ".trellis/tasks/09-29-external-lib-packaging/prd.md", "reason": "沿用独立 Worker 资源校验和无宿主环境合同"} diff --git a/.trellis/tasks/09-29-language-production-closure/design.md b/.trellis/tasks/09-29-language-production-closure/design.md new file mode 100644 index 0000000..ecf9c11 --- /dev/null +++ b/.trellis/tasks/09-29-language-production-closure/design.md @@ -0,0 +1,39 @@ +# Dever 语言生产化收口设计 + +## 2026-10-02 Linux 两项收口 + +当前缺口是完整质量门/持续负载证据，以及 AppArmor 限制下非 root 的 namespace 创建入口。CI 复用根 test 与 performance owners，通过显式设置记录工具路径、作者输入、阶段结果及源码摘要；ignored 内部探针由父测试调用，缺配置不算通过。实际持续测试覆盖双 CMS、HTTP/2 和连接回收的 64/128 MiB 预算，每个关键场景至少 30 分钟。总负载使用自有 cgroup 限制为 2 CPU/4 GiB，压力客户端与服务分别计量。格式门历史差异采用备份后的机械格式化，不混入语言规则变更。 + +AppArmor 审计确认当前私有 loader 才是首个 exec，bwrap 路径规则不会附着。将已签名 bwrap 资产改为静态 ELF 并直接 exec；受限主机只用固定 root-owned 内容地址 helper，精确匹配嵌入摘要并验证无符号链接、不可组/其他用户写、无 setuid/file capability。复用现有签名安装/事务 owner 安装 helper 与专用 ABI 4.0 profile，不新增任意执行代理或信任源。profile 按上游 bwrap 能力分离结构限制其子进程，必须验证 NNP、直接 aa-exec 与 namespace/capability 拒绝。宿主加载操作独立审批，不能关闭系统限制或用 root 通过替代非 root 验收。 + +修改归属：test/ci 与 test/performance 管验收；dever-sandbox 管入口/资产检查；CLI toolchain packaging/bootstrap 管可信资产和固定安装目标；SDK 作者输入及根 test 随合同同步。其他平台、公开发布、语言/API/CMS 业务重构不在当前范围。 + +## 总体边界 + +收口工作分为四个相互独立、最终统一验收的边界：应用运行边界、外部生态边界、工具链/发行边界和验收证据边界。已有编译器和运行时不做平行重写；每个缺口只在实际拥有该行为的层修复。 + +## 1. 应用运行边界 + +API 只负责请求适配、参数解码、上下文读取和响应编码；核心业务继续由现有 Model、领域规则、Port/Adapter 和 Job 承载。真实 PostgreSQL 验收使用项目配置文件提供连接信息，不新增环境变量兼容路径。权限和租户上下文从可信运行时来源进入请求链路，数据库模式租户选择在数据访问边界完成。 + +CMS Dever 与 Markdown 实现共享行为合同和验收矩阵，但不共享会掩盖实现差异的业务源码。每一套实现都必须经过同样的输入、状态和输出断言。 + +## 2. External Lib 边界 + +Dever Package 只声明和组合 Port/Adapter；Lib resolver 负责解析锁定、缓存和运行时准备；Worker 负责统一进程协议。解析、下载、准备和启动是显式阶段，`run/build` 只消费已经锁定并验证的结果。不同生态的差异封装在 resolver/runtime provider 中，不进入 Dever 语言层或业务 API。 + +所有 Worker 通过统一协议传递版本、capability、请求、响应、错误、取消和超时。构建时可嵌入运行时和依赖，运行时只启动锁定选择的 Worker，并验证内容完整性。 + +Go Worker 复用同一个由签名发行物和 `dever.lock` 精确摘要绑定的 `runtime.pack`。包内 `dever-runtime.json` 的 Go 专用 `build` 段固定 compiler、linker、源码分析器与目标标准库 importcfg；工具和标准库仅在构建阶段解包到私有临时目录，使用绝对路径和清空的子进程环境。编译按目标 build tags 选择源码，沿已锁定的 Go module 闭包解析导入，并以包内标准库和本次生成的依赖 archive 形成 importcfg。Go SDK 与生成的 `main` 同编译；生成入口把已检查的 operation 名静态绑定到同名导出的 Go 函数，Go 编译器校验统一签名，业务源码不手写注册表或 `main`。产物只嵌入编译好的 Worker 和 checked manifest，运行时继续复用既有 Worker 启动与协议校验。缺少构建资源、导入、目标或不支持的 cgo/汇编能力均明确失败；私有固定目标 pack maker 只用于自有 fixture，不代表正式签名发行或 Go sumdb 验收。 + +## 3. 工具链与发行边界 + +本地开发仍可使用 Rust 后端，但公开发行物必须将编译/运行所需的私有实现封装在受保护的工具链和 `deverd` 服务边界内。共享缓存不再由普通 CLI 直接读写；服务负责认证、路径解析、租约、发布和清理。平台实现通过适配层隔离 Linux/macOS/Windows 的权限、服务注册和安装差异。 + +## 4. 验收与回滚 + +- 每个阶段先运行最小定向检查，再运行该阶段的集成检查。 +- 真实数据库、跨平台构建和外部网络解析不可用时，不修改测试使其“通过”；记录阻塞原因和重新运行命令。 +- 新实现必须保留 fixture/fake 路径，普通单元测试不强制启动真实外部 Lib。 +- 发行边界先通过离线 fixture 和干净目录检查，再接入真实平台/注册表。 +- 若某阶段失败，回滚到该阶段前的任务边界，不回退已完成的编译器、缓存和协议基础。 diff --git a/.trellis/tasks/09-29-language-production-closure/implement.jsonl b/.trellis/tasks/09-29-language-production-closure/implement.jsonl new file mode 100644 index 0000000..349918e --- /dev/null +++ b/.trellis/tasks/09-29-language-production-closure/implement.jsonl @@ -0,0 +1,14 @@ +{"file":".trellis/tasks/09-29-language-production-closure/prd.md","reason":"总任务需求、范围和验收标准"} +{"file":".trellis/tasks/09-19-application-testing/prd.md","reason":"现有应用Test入口、case-local fake及普通测试隔离合同"} +{"file":".trellis/tasks/09-19-application-testing/design.md","reason":"复用test_program、每case执行/数据库与错误呈现，不新建测试解释器"} +{"file":".trellis/tasks/09-20-production-api/prd.md","reason":"现有API/REST/可信身份/自动权限/database-only租户最新合同；旧权限设计不得复活"} +{"file":".trellis/tasks/09-20-production-api/implement.md","reason":"原native实际语义、复用owner及已有验收证据"} +{"file":".trellis/spec/backend/compiler-contracts.md","reason":"checked HIR、typed LLVM roots/failure/effect/Upload边界"} +{"file":".trellis/spec/backend/toolchain-and-library.md","reason":"独立runtime ABI profiles、私有ffi与资源所有权"} +{"file":".trellis/spec/backend/database-guidelines.md","reason":"共享SQL/driver、owned Session、事务及租户数据库隔离"} +{"file":".trellis/spec/backend/logging-guidelines.md","reason":"复用单一logger及隐私/清理合同"} +{"file":".trellis/spec/backend/directory-structure.md","reason":"compiler/runtime归属和根test约束"} +{"file":".trellis/spec/backend/error-handling.md","reason":"来源级拒绝、标准API错误映射和原始故障保留"} +{"file":".trellis/spec/backend/quality-guidelines.md","reason":"最小定向/原caller等价验收，不把object当完整发行"} +{"file": ".trellis/tasks/09-29-external-lib-protocol/prd.md", "reason": "沿用 external Adapter 协议、capability、超时取消与业务错误边界"} +{"file": ".trellis/tasks/09-29-external-lib-packaging/prd.md", "reason": "沿用独立 Worker 资源校验和无宿主环境合同"} diff --git a/.trellis/tasks/09-29-language-production-closure/implement.md b/.trellis/tasks/09-29-language-production-closure/implement.md new file mode 100644 index 0000000..58d61cb --- /dev/null +++ b/.trellis/tasks/09-29-language-production-closure/implement.md @@ -0,0 +1,890 @@ +# Dever 语言生产化收口执行计划 + +## 2026-10-02 继续：Linux质量门与非root Worker + +最新范围只推进完整CI/长压测和受限AppArmor非root Worker，其他平台及公开发行暂缓。复用现有任务，不新建平行实现或重新设计语言。 + +2026-10-03用户因耗时明确允许不做剩余长测，要求尽快收口。已停止长测队列；此决定替代下文全部18个子场景必须跑完的本轮要求。保留CMS64/128和HTTP2-64正式通过证据，HTTP2-128记主动中止，recovery/live四阶段未执行；不标为通过。用户仍保留完整CI要求，默认/全功能矩阵已经通过，不再追加长测。随后用户明确回复“可以”，授权临时部署可信helper、加载AppArmor策略、启用严格转换限制并在验收后恢复；授权已经取得，不再重复询问。 + +1. [x] 核对workspace/feature/ignored检查清单，复用现有测试与性能owner建立可重复Linux质量门。 +2. [x] 实测定位AppArmor启动边界，完成不放宽隔离的修复和专用部署策略；已在授权的严格宿主策略下完成验收并恢复原状。 +3. [x] 串行完整检查并修复实际失败；磁盘/内存预算、源版本、未运行项与日志一并保留。 +4. [x] 按2026-10-03最新范围完成长测收口：CMS64/128、HTTP2-64通过，其余长测经用户允许停止/免跑，未冒充全矩阵通过。 +5. [x] 独立复核两项交付、清理自有测试资源，更新当前结果；不以旧定向结果替代新质量门。 + +实施边界：CI/长测逻辑归根test与已有performance runner；Worker启动归dever-sandbox，可信发行资产及可选部署接线归现有CLI toolchain owner。不改App/API/CMS业务语法；最初只读宿主的限制已由上述明确授权替代，仅允许本轮临时部署与严格策略验收，结束后恢复。 + +### 2026-10-03 最终结果：本轮 Linux CI 与受限 AppArmor 非 root Worker 已收口 + +- **最终源码与证据边界**：源码 `7310a568ebe3fbb3cc747e343ab2fd17ff1eada8da32d54e98bb143c3dca6cba`，固定完整输入 `e40b7f16cf6f7a84d256210024a3569143d5716f3cc6b34152162f4d4049e873`。本次接续只修改测试 owner：修复嵌套 libtest 汇总重复计数及标准库 TLS 缓存造成的分配账本误报，新增确定性回归。恢复 `target/ci-harness-before-37kQnY` 的四个原测试文件并排除两个新增回归文件，精确重现旧源码指纹 `90245dd9737bfed7ebe13240a05958e5eba9b088457ba1e90de093d8fa3fa379`；编译器、runtime、SDK 和产品输入均未改变。因此保留原矩阵证据，不重写其指纹、不重复长时间矩阵，也不声称所有阶段在同一新指纹下重跑。 +- **最终专项**：`target/linux-ci/native/1791040913590327459/result.json` **185/185**，`target/linux-ci/sandbox/1791040885539823828/result.json` **9/9**，`target/linux-ci/python/1791040504277745773/result.json` **92/92** 全部通过。inventory `1791040538867110700` 覆盖108目标/217项分类；原生原失败用例固定重复 `target/linux-ci-parallel-repro/1791040439295737989/result.json` **20/20**。真实泄漏和错误返回的负面回归均仍被拒绝；没有放宽断言或重试到通过。 +- **完整矩阵及其他检查**：上述产品字节一致的旧指纹下，default `1791029064673984868` **651/651**、all-features `1791031757319396825` **873/873** 通过；全局格式/默认与全功能全target Clippy、19 feature组合与依赖树、三语言SDK、真实PG **7/7**、有界性能 **3/3** 均保留原正式通过证据。阶段报告在 `target/linux-ci/` 对应目录；不把用户免跑的长测或不同指纹报告拼成严格 `run.py summary` 全阶段通过。 +- **严格宿主与真实非 root**：`target/linux-ci-apparmor/1791040870463419164/result.json` 验收通过且回滚完成。四种普通/立即 `aa-exec`、有/无 `--no-new-privs` 组合均实际运行探针并由内核拒绝 `uid_map`（errno1），未以启动失败代替隔离证据。`target/closure-sandbox-ecosystem-release-acceptance.json` 验证签名安装、三生态受管check/run/build、隔离独立程序和 **UID65534 的 pip/npm/go 全部通过**，包含源码原生依赖与离线构建收据重放。 +- **预算与清理**：最终队列日志 `target/linux-ci-final-ledger.log`，用时1小时3分28秒；父cgroup峰值3,565,158,400字节，另预留32MiB，总计小于4GiB，OOM/oom_kill均0。自有unit已停止，cgroup和PG已退出；临时helper目录 `/opt/dever`、策略文件及两个已加载profile均移除，无残余附着进程。sysctl恢复原值 `apparmor_restrict_unprivileged_unconfined=0`、`apparmor_restrict_unprivileged_userns=1`，后者始终未关闭。根临时 `config/setting.json` 校验SHA `4758568903fd1233d565cd381372353b598f994517fede3d117c6508c948e2bb` 后移至私有 `target/linux-ci-session-BrZ5t3/setting-final.json`（0600），恢复根配置原先不存在的状态；日志、固定输入与备份可恢复保留。 +- **长测与父任务**：CMS64/128及HTTP2-64正式长测通过，均0错误/0 OOM；HTTP2-128主动中止，recovery/live的64/128四阶段未运行，按用户最新要求免跑且不记通过。其他平台适配与正式公开发行继续延期，父任务保持 `in_progress`；本轮两项 Linux 交付无剩余阻塞，不提交、不归档。 + +### 以下为历史执行记录 + +记录中的“进行中”“未授权”“尚未通过”均描述当时状态，最终结论以上节为准；失败和中止证据原样保留。 + +- **分配账本误报定位与修复**：native `1791038231061444951` 的前137项通过，在 `native::synchronous_file_stream_parallel_failure_releases_workers_and_rows` 报+1；宿主报告 `target/linux-ci-apparmor/1791038179604022584/result.json` 确认恢复完成。固定20次复现 `target/linux-ci-parallel-repro/1791039680829850445` 为14通过/6失败。临时指针追踪六次均余留48字节；保存ELF `target/parallel-trace-kernel`，`addr2line -f -i -C ... 0xb0ff6` 明确为 `Arc<std::sync::mpmc::context::Inner>::new -> Context::new:69`，不是业务owner丢失。根因类别E（隐含假设）：一次warmup不保证触发满队列producer的TLS缓存，主线程未退出时缓存仍正常存活。共享managed-driver改为每次入口经独立pthread并join后计数，保持进程初始化、warmup1+重复64和严格delta=0；不改产品队列/ABI、不增加容忍值或概率预热。新增确定性C fixture覆盖warmup后TLS初始化、故意泄漏和错误返回；19项CI单元通过，原driver对合法late TLS稳定失败，修复后通过且两种真错误仍被拒绝。临时trace已移除，备份原driver SHA `3050becbd464c1640739dd1db5df7dcdbe1cb41095e28f7faf4053089a7ca46b`。只读并发专家复核通过；观测边界是入口及宿主线程退出，未声称证明长寿命宿主TLS无增长。当前 `7310a568...` 正进行固定20次复验以及Python/inventory/sandbox/native完整接续；原生门尚未宣布通过。 + +- **2026-10-03已授权宿主验收续跑**：首次部署报告 `target/linux-ci-apparmor/1791037194864425474/result.json`，四种aa-exec转换组合均实际执行探针并被内核拒绝；真实UID65534的 `enforces_unprivileged_namespace` 及子探针均成功退出。沙箱gate误将父/子两条libtest结果相加为2，导致精确单用例门禁失败；宿主已完整恢复（strict=0、userns=1、helper/策略移除、无残余附着进程）。本次只修改CI Python owner：直接单harness验收取最终父汇总，仍要求1通过/0失败/0忽略；保留普通Cargo多汇总计数。17项定向回归通过，包含子测试成功但父测试空跑/忽略/失败的拒绝。当前串行重跑Python、当前inventory、完整sandbox/native；不重跑已通过的Rust默认/全功能矩阵，不制备新作者输入、不追加长测。其旧证据保留原源码指纹，不伪装为这次Python改动后的新证据。 + +- **2026-10-03本轮最终状态**：当前源码 `90245dd9737bfed7ebe13240a05958e5eba9b088457ba1e90de093d8fa3fa379` / 验收输入 `e40b7f16cf6f7a84d256210024a3569143d5716f3cc6b34152162f4d4049e873` 的default `1791029064673984868` **651/651**、all-features `1791031757319396825` **873/873**正式通过，覆盖78/113目标。合计1小时56分17秒，峰值3,565,158,400字节（另预留32MiB），memory.max回收2489次、OOM/oom_kill均0。当前静态、19组合+依赖树、Python88、三SDK、PG7、性能3和3项长测均有同版正式通过证据。完整native185/sandbox9门仍缺受限AppArmor非root授权验收，不称整个CI全绿。 +- **清理/交付**：压测与CI unit已退出，owned cgroup和PG进程不存在，最后复核无自有验收进程；测试可执行文件按既有gate规则回收。原先不存在的根 `config/setting.json` 已确认仍为本轮SHA `4758568903fd1233d565cd381372353b598f994517fede3d117c6508c948e2bb`，移动到私有 `target/linux-ci-session-BrZ5t3/setting.json` 留作复现，恢复根目录无临时测试配置的状态。日志/固定输入/编译缓存保留；将来复验须显式恢复该配置后再运行。宿主AppArmor未改，profile/helper未安装。父任务保持in_progress，无commit/发布/归档。 +- 取消剩余长测后的当前版本 default `1791029064673984868` **651/651通过**，source `90245dd9...` / inputs `e40b7f16...` 起止稳定，覆盖78个目标；all-features `1791031757319396825` 已接续运行。此轮不重制author输入、不追加长测，也不将缺授权的native/sandbox完整正式门标为通过。 +- 已精确向本轮 `http2-128` gate PID1085911发送SIGINT，运行器保存 `KeyboardInterrupt`（命令3002.8秒），自有服务、cgroup、PG均已退出，无残留压测进程。整轮4小时13分钟，峰值3,052,568,576字节、OOM/max事件0；保留 `target/linux-ci-cms-recovery-final.log` 和原始中断报告。宿主strict仍0/userns限制1，helper/profile均未安装。 +- 只读独立审计确认旧651/873完整矩阵仍是历史通过证据，当前source `90245dd9...` / inputs `e40b7f16...` 不可冒用其正式标签。当前88项Python、静态/features/SDK/PG7/bounded3及3项正式长测已通过；`target/linux-ci-final-matrix-without-soaks.log` 现只安排default→all-features。native185/sandbox9完整正式门受非root宿主授权阻塞；旧184/8诊断保留其对应版本，不做无效重跑。 +- HTTP2-64 `1791022159074052799` **通过**：明文/TLS各1800秒、计划各9,000,000请求，成功8,996,761 / 8,996,074（4998.20 / 4997.82次/秒），均0错误、0 OOM、断连FD差0；请求P99 1.328 / 1.376ms，含客户端调度的P99 2.848 / 2.912ms，RSS峰值4.90 / 5.67MiB。3/8长测阶段已完成，HTTP2-128 `1791025811330571248` 进行中；不把固定5000目标负载当饱和吞吐。 +- CMS128 `1791018511743113541` **通过**：双源码各1800秒，Dever/MD为1,760,530 / 1,744,559次成功鉴权读、978.05 / 969.18次/秒，0错误/0 OOM；RSS峰值29.10 / 29.04MiB、后半段增量8 / 12KiB，FD合同通过。64/128MiB四个CMS场景合计7,020,593次成功请求。HTTP2-64 `1791022159074052799` 正在执行，CMS之后其余6阶段长测及最终矩阵未完成。 +- 正式长测首阶段 CMS64 `1791014862780230485` **通过**（source `90245dd9...` / inputs `e40b7f16...`）：Dever/MD各1800秒，1,754,220 / 1,761,284次逐回复校验的鉴权读，974.56 / 978.46次/秒，均0错误、0 OOM，RSS峰值29.37 / 29.22MiB，后半段增量16 / 28KiB，非DB句柄恢复且数据库句柄在池上限内。吞吐来自2CPU总预算下8并发闭环，并非服务饱和上限。CMS128 `1791018511743113541` 继续；其余长测与最终矩阵仍待完成。 +- 最终CMS修订版源码 `90245dd9...` 已完成 inventory `1791013634029656866`、author `1791013959047970369`（6/6）、static `1791014156347447546`、features `1791014186612471905`、Python `1791014213163412921`、SDK `1791014240370055980`、PG `1791014295204024541`（7/7）及bounded-perf `1791014699485900727`（3/3）。作者完成后完整输入固定为 `e40b7f16cf6f7a84d256210024a3569143d5716f3cc6b34152162f4d4049e873`；5内核比0.707/1.004/0.538/1.014/1.069，原1.15门槛不变。`target/linux-ci-cms-recovery-final.log` 中正式CMS64 `1791014862780230485` 已启动，后接其余7个长测阶段、完整default/all-features及独立native/sandbox诊断；当前仅记进行中，owned PG已停止，非root宿主策略授权仍待回复。 +- CMS句柄检查修订后的完整短矩阵 `target/linux-ci-soak-smoke/1791013117430967380` **8/8通过**：64/128MiB各覆盖双源码CMS、HTTP/2明文/TLS、两种HTTP/2连接恢复及TCP/WS/SSE。源码 `90245dd9737bfed7ebe13240a05958e5eba9b088457ba1e90de093d8fa3fa379`、输入 `1649db21a5bd7fb2224f4120d191fad7c034edba9d28bc8072907035ba380b75` 起止一致，6分47秒、峰值126,705,664字节、OOM0；这是短诊断，正式30分钟各场景仍全部待重新执行。下一轮先完成author再冻结输入，串行重跑完整质量门与长矩阵。以下保留历史尝试及其对应版本，不能把其中的进行中描述当作当前状态。 +- 首个真实30分钟CMS64长测 `1791010360837130398` 在1801.7秒后因总FD差断言失败，后续阶段自动停止；运行中RSS约28.9→29.1MiB、OOM0，但失败不能记通过。75秒诊断 `target/linux-ci-cms-fd-diagnostics/1791012321412052347` 确认17→28 FD由SQLite双池各扩3连接（12个主/WAL句柄）与控制socket空闲关闭（-1）组成，压测8连接均关闭；原报告丢失case详细结果。 +- 只修测试owner：CMS关闭控制client后取完整FD目标快照，非DB目标Counter精确恢复；已知SQLite主/WAL各≤配置4、SHM≤1（与锁定bundled SQLite实现及实际句柄一致），CI复用同一校验，不忽略任意文件增长；发布/基线/读负载逐步写入case结果，失败也保存raw/预算/错误并进入总报告。50项runner及15项gate单测通过，覆盖同数socket替换/dup、超池上限及两类失败证据保留；真实75秒复验 `1791012808869143017` 通过，16→28仅为受限数据库池增长，非DB全部恢复、OOM0。当前测试源码已变，准备先完成短协议检查，再冻结新版本重跑正式长测；不把75秒当30分钟。 +- `be785533...` / `1649db21...` SDK `1791009652252648482`、真实PG `1791009708083087094` **7/7**、bounded-perf `1791010181020595614` **3/3正式通过**，材料起止一致。固定25进程×9样本的5内核比值依次0.710/0.998/0.546/1.014/1.053，保留原1.15阈值和全部分组日志，不把参考Rust后端的微基准当LLVM服务吞吐。长测已实际开始：CMS64阶段 `1791010360837130398`，Dever服务PID944306，当前只记running，完成后还需MD与其余7阶段。owned PG已停止。 +- 固定25组采样版本源码 `be785533e4ee767df5d3500570868533c40f2489a0e42c4ff3790fa80d226f8f` 已通过inventory `1791009039923269723`（217项ignored分类）、author `1791009367171058428`（6/6）、static `1791009565839581267`、features `1791009598346719330` 和Python `1791009624921947548`；完整输入为 `1649db21a5bd7fb2224f4120d191fad7c034edba9d28bc8072907035ba380b75`。`target/linux-ci-stable-sampling-closure.log` 串行安排SDK、PG、固定采样性能门、8个长测阶段、default/all-features及不依赖待授权宿主策略的native/sandbox诊断。先完成author再冻结长测输入，后续不重制材料使长测证据失效。长测尚未开始，非root宿主授权仍待回复。 +- 正式PG `1791007358946731625` **7/7通过**。随后bounded-perf `1791007764657474746` **2/3**：Map比值1.421超过1.15，其他4内核及allocation/任务粒度通过，后续长测未启动。固定同一对ELF的诊断保存在 `target/linux-ci-map-diagnostic`、`target/linux-ci-map-diagnostic.log` 和 `target/linux-ci-map-affinity.log`；实际机器码两侧都执行SipHash/Map覆盖写，handler已内联，没有每步Result调用或Map克隆。相同ELF在各CPU仍出现约1.6/2.2–2.3ms成簇耗时，具体宿主/种子根因未定；CPU0前5组比1.370、后5组1.072，合10组1.073，说明原5进程门有采样偏差风险。只将固定进程数5→25并永久保留每组9个原始样本，算法/优化参数/1.15阈值不变，不动态补样直到通过；临时Rust诊断测试已移除。无绑定25组诊断比1.069只是证据，不能替代新正式门；长压测、最终矩阵和待授权非root验收继续。 +- 最新源码仍为 `d39a1108...`。正式inventory `1791006147903077573`、author `1791006502867605785`（6/6）、static `1791006705602191313`（fmt及默认/全功能全target Clippy）、features `1791007065359258253`（19组合+依赖树）、Python和SDK `1791007302833243256` 已全部通过。author重建材料后完整输入更新为 `1acd35250dbdb0872e68744cf9c434ae160a3caf60a5328e65bfcc965009d566`；旧 `c6aebfab...` 的184/8/3诊断不冒充这批新材料的正式native/sandbox证据。`target/linux-ci-final-acceptance-soak.log` 已串行排入新版本PG/有界性能、8阶段约9小时持续测试，以及最后default/all-features；当前PG `1791007358946731625` 进行中。压力阶段不并行Cargo。 +- `d39a1108...` / `c6aebfab...` 独立验收全部完成：原生 `1791003377055500069` **184/184**、沙箱 `1791005949517983914` **8/8**、有界性能 `1791003223649311165` **3/3**，source/input起止一致。三轮在同一2CPU/3400MiB unit共45分52秒，峰值2,988,834,816字节，无max/OOM事件。覆盖此前SIGPIPE、Node大程序链接、Package全链、真实180秒deadline与PID清理、跨UID共享、四profile签名/exec移源、bootstrap228秒双项目双UID和root真实沙箱。**明确排除**待宿主授权的 `enforces_unprivileged_namespace` 及签名三生态非root独立执行；不把这三份诊断记录替代完整native/sandbox正式门。宿主strict仍0/userns限制1，profile/helper均未安装。 +- 新测试源码 `d39a1108526fc593ea192e17f80191ecad7aaec5b26b083195f81bedf24acd54` 的有界性能诊断 `1791003223649311165` **3/3通过**，source/input起止一致（input仍 `c6aebfab...`）。含任务粒度、分配/记录/列表/JSON/Map原生3进程和5内核对匹配runtime基线；不是正式bounded-perf门或LLVM长测。随后 `1791003377055500069` 执行184项不依赖待授权宿主规则的原生用例；三生态签名独立非root链明确未选，待AppArmor策略授权后单独完成，不将184项当185项全通过。 +- 分配基准夹具已迁移为 `main.dever` → `benchmark/allocation/app.dever` 的run入口，删除旧package/exposes和strict Main不允许的public标记，并将7处错误Choice消费改为显式result。保留14轮倍增/16,384字节、每进程text/list/json各6次和map1次、原计时边界及3进程断言。诊断 `1791002981413191937` **2/3通过**，allocation首轮仍被public标记拒绝（已修）；热路径5内核原阈值1.15下实际比值0.710/0.998/0.537/1.012/1.062通过。这是Rust参考后端对匹配runtime基线，不能冒充LLVM HTTP/CMS长测。当前再次复验分配基准；测试源码已变化，之前正式门仍保留对应版本身份。 +- 根盘可用空间另行回升至约12GiB（超出本轮明确清理155MiB，不归功于本任务）。在owned 2CPU/4GiB unit内持Cargo锁，将确切RAM缓存逐文件SHA/类型/权限验证后恢复为普通 `target/native-runtime` 目录，删除本轮 `/dev/shm/dever-ci-reference-cache-7Qn1JD`，释放962,711,552字节内存盘；3971项校验及报告见 `target/linux-ci-reference-restoration.json`，无dangling链接，OOM为0。源码、固定输入及验证摘要未因此改变。 +- 当前 `93971614...` / `c6aebfab...` 的正式PostgreSQL `1791002131949151822` **7/7通过**：私有REST身份/权限/事务、Model组合/stream/schema升级、两类错误回滚、权限目录、CMS双物理租户和ORM池/取消/关闭/TLS拒绝。临时PG34889已停止。紧接的bounded-perf `1791002657426882136` 1/3通过后失败：allocation fixture的根app.dever仍沿用旧package exposes/目录规则，计时前C003拒绝；热路径基准尚未运行，不记性能通过。正在只迁移测试夹具，保留算法、样本规模和断言。该unit共9分15秒，峰值2,433,675,264字节加参考缓存预留996,265,984字节低于4GiB，max/OOM事件均0。 +- 最新完整矩阵正式完成：default `1790994992208925145` **651/651**（78目标）、all-features `1790997779069364253` **873/873**（113目标），源码均为 `93971614df839969267bce14e98a2400c3a68fee80ad78b9ec9fcbc061e7a507`、完整输入 `c6aebfabf6103f85758aa9d1c8145dbd7b019738df4cc541c68616a260ca0bc1`，起止摘要一致。两阶段共1小时57分19秒；OOM/oom_kill为0，memory.max回收2817次，进程峰值3,358,203,904字节加参考缓存预留936,763,392字节达到总4GiB限额。静态与最小feature等正式准备门沿用同版通过证据；当前版本真实PG、显式native/sandbox、长压测继续，尚非完整CI全部收口。 +- 矩阵完全退出后，独立artifact联合图与当前工具fingerprint依赖闭包核对11个旧编译缓存；主线程持Cargo锁重查最新artifact、单链接、固定输入排除及进程maps/fd后回收162,013,184字节。明细 `target/linux-ci-reviewed-libraries-cleanup.json`；仅可重建库/元数据，源码、当前工具、固定归档和报告保留。临时目录只读审计没有其他确定可回收的大型残留，归属不明的旧SQLite目录保留。 +- all-features正式尝试 `1790997779069364253` 中，runtime_entry_cleanup 4/4、完整durable_jobs 18/18均通过；后者277秒，覆盖先前栈膨胀相关的SQLite状态机、信号退出和api/worker/all模式。全矩阵仍执行中，不能以这两组结果代替整组通过。 +- 正式default `1790994992208925145` **651项全部通过**，source `93971614...`，输入摘要由正式门验证稳定；同一受限unit已接续all-features `1790997779069364253`。default末段出现memory.max回收事件（当前202次），OOM/oom_kill仍0；不称“未触顶”。全feature、显式native/PG/sandbox和真实长测仍未完成。 +- 冻结测试源码 `93971614df839969267bce14e98a2400c3a68fee80ad78b9ec9fcbc061e7a507` 后，正式inventory `1790993660614204359`、author `1790993991491934824`（6/6）、static `1790994189999992691`（fmt+两组全target Clippy）、features `1790994532647168020`（19组合+依赖树）、Python `1790994744317023527`（85项）、SDK `1790994771348473444`（三语言）全部通过。19分27秒，进程峰值3,077,398,528字节+参考缓存预留936,763,392字节低于4GiB，OOM/max事件均0。完整default/all-feature矩阵继续，不以准备门替代。 +- 准备门停止且确认无编译/测试活动后，复用元数据回收规则删除555个无配套rlib/a/so的单链接检查元数据，323,555,328字节；`target/linux-ci-fixture-metadata-cleanup.json`保留明细。仅可重建缓存，固定输入/源码/报告保留。全矩阵的native-artifacts使用unit私有768MiB tmpfs并计入总内存限额，退出自动释放；根盘已有缓存保留，测试在独立挂载中使用新缓存，不更改产品配置。 +- `1790993368772328336` 两项新fixture exact均通过：取消/断连/真实180秒期限回收，以及CMD/App缓存与接口基线拒绝。源码/输入摘要前后一致；开始最终fixture版本正式inventory/author/static/features/Python/SDK门。受限AppArmor宿主策略仍未获用户回复，不修改sysctl或安装profile/helper；尚不能记非root通过。 +- 补验 `1790992602334552404` 9/12通过，包括Package check/test/run/build/独立执行、真实跨UID共享编译、官方Python wheel/source、签名四profile及新增exec移源独立运行。三个失败分别是共用wait超时、Node参考backend链接Bus error和native_cache旧main入口。取消fixture已拒绝空/非法PID并区分mode/phase（保留10秒）；cache fixture已迁移CMD/App并保留篡改基线拒绝check/run/build与不生成产物。Node用自有/tmp内存盘复测 `1790993196374065418` exact通过48.4秒；同轮测试源码有其他fixture编辑，因此整个诊断记录仍为failed，不能记正式门通过。两处新fixture独立复验进行中。 +- 剩余native诊断 `1790990565345235019` 的TLS exact及共享suite均通过；累计89项通过，未完成整组。两个接线失败已修：external_workers参考backend fixture遗漏sandbox资产，复用prepare_sandboxed_worker与真实pip用例；llvm_cli旧unsigned真实Lib正向场景迁至既有签名四profile/daemon验收，保留run/build/删整个源项目后独立data7，原私有路径改精确拒绝，新增第5条编译缓存断言。三生态signed在npm run因ENOSPC退出，未到非root；Package完整链在test阶段失败但旧断言丢失stdout/status，现补诊断。随后根盘仅12MiB，主代理SIGINT停止全部自有unit，当前managed_compile exact记中断，不把systemd的success终止摘要算CI通过。 +- 完全idle复核后清理精确中断目录 `/tmp/dever-core-test-699057-0`（内容匹配managed_compile source_project/签名sleeper）及121个无活动maps/fd、单链接、完整三文件native-artifacts条目，273,723,392字节缓存；`target/linux-ci-idle-cache-cleanup.json`保留明细。源码/固定原料/报告未删除，缓存可重编。根盘约109→643MiB。运行期间提出的旧PID缓存回收在PID退出检查时被取消，没有执行。下一轮先做Package/三处fixture迁移与此前未执行的native定向验收。 +- 正式native `1790989136780529193` 86项通过后在TLS HTTP2上传下载被SIGPIPE终止，18分12秒、无OOM；不是完整通过。根因为公共C测试驱动绕过产品process main却漏调用已有 `dever_rt_v1_process_init`。产品进程入口已有正确初始化，可调用root依法不改嵌入宿主信号策略。仅在 `test/native-runtime-abi/managed-driver.c` 首次执行前复用canonical header/初始化并检查返回值，indexed suite自动共用；独立复核通过。既有ABI驱动已有SIG_DFL→init→raise的确定性断言，保留TLS真实网络和allocation ledger复验。继续收集剩余native失败，再冻结测试源码重跑正式门；产品runtime/CLI/压力产物字节未变。 +- 正式 PostgreSQL `1790988576909213458` **7/7通过**，source `6515b6d8...` / input `6a7b4ee8...` 前后一致；涵盖原生Model组合、两类失败回滚、私有REST权限事务、站点角色目录、CMS双租户数据库和ORM连接池/取消/断开/TLS拒绝。耗时8分47秒，进程峰值1,407,135,744字节，无OOM/预算触顶；owned PG端口34889已停止，postmaster.pid不存在，server日志已保存。全矩阵、原生发行和长测继续，非root宿主策略授权仍待用户回复。 +- 2026-10-03 source `6515b6d8...` / input `6a7b4ee8...` 正式准备门通过：inventory `1790987099128193601`（108目标/217分类），author `1790987506507180413`（6/6），static `1790987706540255061`（fmt及两组全target Clippy），features `1790988062781869337`（19组合+依赖树），Python `1790988269722882336`（85项），SDK `1790988297034468817`（三语言）。20分54秒，无OOM；memory.events max=336是限额内回收，不能称未触顶。正式PG、全矩阵、native/sandbox和长测仍待运行。 +- 准备检查结束根盘仅196MiB；持Cargo锁确认无编译器后，回收568个无配套rlib/a/so的单链接检查元数据，323,489,792字节，排除固定inputs。明细 `target/linux-ci-check-metadata-cleanup.json`；可重建，未删除源文件/报告/固定资产。随后以独立loopback端口34889启动本轮owned PostgreSQL，日志 `target/linux-ci-runtime-postgres.log`，退出由托管器停止。 +- 2026-10-03 四runtime、CLI/guard/maker/init、3 peer和双CMS/协议产物全部重建成功；`target/linux-ci-author-runtime-closure/provenance.json` 绑定 source `6515b6d8...`。耗时29分10秒，进程峰值3,414,069,248字节+已预留参考缓存876,298,240字节，合计低于4GiB，无OOM。双CMS为8,325,896/8,328,136字节，sha分别1bdabcf5.../e528684e...。component-fixture固定副本同步刷新为f89f6af7...；临时setting指向新产物（自身SHA47585689...）。本轮临时runtime编译缓存已回收。 +- 重建后磁盘一度仅35MiB，独立审查确认只回收早期Rust backend完整缓存，不能仅凭源码mtime宣称全部失效。主代理持Cargo锁、排除活跃writer/maps/fd和固定inputs，逐项复核mtime/inode/单链接/完整三文件后原子移出并删除287条目，397,664,256字节；`target/linux-ci-native-cache-cleanup.json`保留明细。较新118条目、CLI LLVM cache和所有报告/固定原料保留，删除项可重编恢复。空闲现534MiB，准备启动新源码/新资产的正式门。 +- `1790981698051341418` 最终67目标/605项实际通过，source前后同为 `6515b6d8243fa1ac9fb925510ab2e0b191a644243c9075ec88b745737bdabeeb`，耗时50分34秒、进程峰值1,213,997,056字节、无OOM。包括入口清理4、完整Job18、借用/non-Send新回归及剩余all-feature目标。这是修复充分性诊断，不替代完整正式门；下一步重编四runtime/CLI/guard/maker/peer以及双CMS和协议程序，再冻结资产完成正式验收。 +- 入口/Job scope 修复后 `target/linux-ci-job-probe/1790981398350130546` 三 exact 全通过：SQLite状态机、signal三种退出场景、API/worker/all模式；无OOM，进程峰值933,769,216字节。独立只读复核未见新增类型限制或清理/错误语义回归。新增借用Rc跨await的根入口及cleanup回归；`target/linux-ci-diagnostics/1790981698051341418` 入口清理4/4已通过，继续完整Job及剩余all-feature目标，尚非正式全矩阵通过。 +- 最新 all-features `1790978249784275058` 在 durable_jobs 失败：此前271项通过，本组15通过/3失败，完整门未通过。两个真实问题是根 Future 在入口/task-local 包装间按值搬运导致栈膨胀：信号用例根 Future 294,544字节，入口 block_on poll 单帧3,535,480字节。入口改为首次 Box 后进入共享执行owner，SQLite状态机 exact 已通过（`1790980718162628429`）；信号用例仍溢出，继续修正 Job scope 的同源包装，保持默认栈、身份惰性、清理与错误语义。模式用例旧API body迁移为声明+App+CMD，使用全局Model保持原非租户三模式检查。第二轮复验进行中，不记通过；临时尺寸输出已删除。运行时改动意味着四profile归档和验收程序必须重新生成，之前静态/feature/SDK通过不能算新源码最终证据。 +- 源码备份 `/tmp/dever-linux-quality-backup-hI3zdG/source.tar.gz`（SHA256 `096aa0755e79b88acd5d1acc19f008b7b81feb38460ba750eae86c97d5c6ce20`）；全仓机械格式化完成。初始格式门1174处Diff位置/132文件。初查Clippy修复CLI旧测试多余clone、network peer手工clamp；完整正式门尚未通过。 +- 静态bwrap 0.11.0制作并验证无INTERP/动态依赖；只替换自有作者输入，未安装宿主helper/profile。root沙箱7项实际隔离、2项默认合同、helper不安全路径拒绝1项通过；日志 `target/linux-ci-sandbox-*.log`。非root旧fixture在root构造Launch后才降UID导致仍走私有路径，已修为先降UID清groups再构造Launch，待重编验收。 +- CLI native_release默认18/18通过（6项明确ignored），日志 `target/linux-ci-bootstrap-default.log`；这是增加安全宿主前置检查之前的定向结果，不代表新的签名安装/非root通过。 +- CI入口与持续CMS/FD/RSS门已落地并完成独立复核；gate13、CMS12、runner48项Python定向通过。补齐最新尝试、完整ignored owner和实际长测预算核验。实际完整Cargo矩阵、作者输入、真实PG/SDK和约9小时长测矩阵待完成。 +- 安全独立复核发现 `aa-exec` 可直接进入新父profile；已在产品启动和实际宿主安装增加严格profile切换前置检查，并通过代码复审与定向回归。已发起异步授权询问，待用户批准临时strict=1、安装指定摘要helper和加载专用规则后真实测试；验收后先卸载新规则再恢复strict原值。当前没有获得该操作授权，也未更改任何宿主策略。 +- 完整all-target/all-feature Clippy `-D warnings`通过（`target/linux-ci-all-features-clippy-fixed.log`）。修复旧测试遗漏External声明分支和lint，更新live peer装箱后的关闭调用。默认全workspace矩阵开始预检；最终阶段仍须绑定准备后的输入摘要重跑。磁盘不足已清除三个过时编译快照（bootstrap-cli-snapshot-s5i92v、final-cli-snapshot-jiqbfz、final-native-fixture-snapshot-xw7bGT），约回收300MiB；源码备份、日志、当前工具和固定原料保留，删去的编译快照可重建。 +- 默认预检 `target/linux-ci/default/1790958354110646233` 在 Port target 失败（15通过、1失败），不是完整通过。此前双 CMS 6/6、native_entry 16/16、REST 12/12及CLI常规目标已通过；进程峰值约2.8GiB、无OOM。失败原因为旧测试直接编译无资源的external入口，未启用受验证bundle运行时。保留Port生成/清理检查，真实typed调用与每次shutdown验证合入既有external_resources打包用例，不增加邻接Worker回退。修正后的Port exact 1/1通过（`target/linux-ci-port-fix.log`）；实际打包用例待复验。 +- 当前源码四profile归档、CLI/guard/静态init/压力客户端重建中，作者过程与源码摘要写 `target/linux-ci-author-build/`；临时内存盘仅缓存编译产物。自有PG16集群 `/tmp/dever-linux-ci-pg-AqF2Cz/cluster` 初始化通过，尚未启动；根临时setting声明loopback端口34889与专用库模板，结束后恢复原先不存在状态。 +- 为保留本次编译空间，进一步移除过时closure-current-compiler的runtime/cache/可执行文件、closure-verified-tools-ksA2Hn快照、未引用native-runtime/runtime-sandbox.a，以及回收站内六个旧ledger/LLVM诊断可执行文件。仅删除可重建编译产物，保留源码备份、固定原料、历史报告和摘要；回收站内这六项为永久删除，需重编恢复。 +- 修正CI配置接线：运行时严格拒绝未知顶层键，因此验收工具改用既有 `performance.linux_ci`，只把 `performance.network_peer/live_peer` 解析为peer路径，未放宽产品配置。gate Python14/14、Rust application_config6/6通过。此前失败清理疑点经metadata入口和文件时间核对属重跑生成文件，现有清理正常，没有额外修改。 +- 四优化runtime归档、当前CLI/guard/静态init及3个压力客户端最终构建确认通过，源码SHA `8c699b703e5ef689c076678f2d5ea9ed0cca5c986a5aa8ac245fb567d70e9716`，完整归档/编译器/peer摘要在 `target/linux-ci-author-final/provenance.json`。首轮因CI源码中途修正而拒绝最终通过，旧日志保留。`target/performance/cms-linux-ci-10-02`双CMS检查/构建成功，`protocols-linux-ci-10-02`协议产物构建成功；尚无本轮长测结果。 +- `target/linux-ci-preflight-closure/result.json` 定向通过：配置6/6、external_resources常规8/8及真实打包exact1/1（107.37秒），覆盖清空环境、两次typed返回7、每次新的shutdown确认和资源篡改拒绝。不是完整CI通过，也不是受限非root验收。构建后删除自有1.3GiB内存盘临时编译缓存；磁盘紧张继续删除未使用的 `target/go-cache` 与 `target/go-managed/cache` 旧可重建Go缓存，原runtime.pack/依赖/报告保留。 +- 正式inventory首轮因backend完整静态归档写满磁盘失败，日志 `target/linux-ci/inventory/1790962709158372077`。将整个可重建参考编译缓存逐文件核对后迁到 `/dev/shm/dever-ci-reference-cache-7Qn1JD/native-runtime`，原 `target/native-runtime` 暂时为指向它的链接，不修改编译器。每阶段从总4GiB额度扣除已有缓存页并保留32MiB开销，进程额度最高3400MiB。结束时须回收该链接/临时缓存。重跑inventory `1790963141083952733` 通过：108个实际target、217个ignored逐一分类，无缺项；期间无OOM。 +- 正式author首轮5/6通过，npm第6项因临时构建目录写满磁盘失败（`target/linux-ci/author/1790963610753874259`）。重跑只给本轮systemd临时unit配置私有 `/tmp` tmpfs（2GiB上限、仍计入总内存额度），只读映入托管脚本与PG原料；主机 `/tmp` 未变。`1790963918501049513` 6/6通过，含真实Go/Python依赖验证、工具pack制作、隔离PEP517及npm原生addon构建，输入/输出摘要一致，进程内存峰值1,963,769,856字节、无OOM。作者结果不是非root AppArmor验收；宿主策略仍未获得授权且未改动。 +- 正式static通过（`1790964267132478061`）：全仓fmt及default/all-feature全target Clippy零警告。features通过（`1790964373302707758`）：19个最小组合加CLI依赖树；Python85项（gate14/performance69/ecosystem2）通过；SDK `1790964628129189173` 三语言真实协议全部通过。新default矩阵 `1790964733321613886` 正在执行，随后all-features；这两个完整矩阵及真实PG/native/sandbox/长测仍未记为通过。 +- 上述default正式尝试最终在 `035.log` 的async_composition失败（6通过、1失败）：旧fixture把零输出sleep赋给变量，当前checker正确报C005；此前Port16/16、REST12/12及CMS6/6已通过。进程峰值1,719,320,576字节、无OOM。仅将两处无用赋值改为action后，实际原生exact1/1通过（22.60秒，`target/linux-ci-async-composition-fix.log`）。 +- 同源测试漂移扫描修正async_stream/http_library/live_library/pooled_library/performance/differential/structured_concurrency七文件：按当前成功值与显式result捕获合同迁移调用，Read/Receive使用Done(state)，保留独立行为及拒绝断言。七文件rustfmt通过；`target/linux-ci-diagnostics/1790966028853786940` 串行执行剩余all-feature目标并收集失败，仅为诊断。测试源码已改变，之前正式门不能视为当前版本全部通过；产品和固定runtime/peer字节未改。 +- 上述62目标诊断完成：20组失败、无OOM，进程峰值1,138,159,616字节。修复一处真实checker崩溃：结构类型深度超限时contract阶段未生成summaries，auth仍继续索引；改为返回致命C015，主检查和Port case均停止依赖阶段，普通合同诊断仍聚合。独立只读复核通过，256层原回归未修改，实际复验待完成。 +- 其余失败fixture迁移到当前App/Model目录、公开类型、显式result及main命名；差分仍精确比较原生/参考输出和完整错误JSON，ORM保留查询/迁移/回滚断言，HTTP非法response按现行请求边界500并验证后续请求健康，H2/时间codec补正式请求scope/metadata提交。sqlite_orm有一次磁盘不足，不修改产品规避。当前 `target/linux-ci-diagnostics/1790968164204229608` 复跑20失败组及Port，使用独立2GiB临时/tmp，仍受总4GiB预算约束；尚不记为正式CI通过。 +- 该定向复验已确认Port16/16、contract_semantics14/14（含原256层崩溃用例）、contract_execution11/11、原生SQL7/7通过；当前唯一再次失败组secure_contracts是fixture缺部署site绑定，已恢复原source-only合同检查并独立验证platform数据库，保留global及原匿名/事务断言，待再次执行。复验期间移除5份本次core修复前的过时rlib/rmeta（约238MiB）；均为可重建缓存，保留报告/原料。 +- `1790968164204229608` 最终19/21组通过、无OOM（进程峰值973,602,816字节）；含SQLite5/5、迁移10/10、差分13/13、HTTP/H2/live/入口/time codec。`1790969758624751556` 安全合同14/14通过，并发12/13失败。进一步保留精确liveness断言定位真实遗漏：CaptureResult未参与挂起点判定，且同源inline-blocking分支也只匹配Call。`1790970195428604455` 修改前11/13，实际证明捕获async sleep漏liveness、捕获sync sleep可绕过worker边界。现两处共用Call|CaptureResult原判定，加入同步capture正向及诊断span；独立只读复核确认根因，实际修复后复验正在进行。未重建验收程序或重跑正式质量门，前述正式结果仍为旧版证据。 +- CaptureResult修复后 `1790970325577979930` structured_concurrency 13/13实际通过，source前后一致、无OOM（峰值704,716,800字节）；完整保留三个挂起点的精确liveness span、同步capture正向及捕获同步sleep的worker边界负向。继续评估直接受影响的既有fixture后冻结源码重跑正式门。 +- CaptureResult调用方独立有界审查未发现需迁移的正向用例；async_native及LLVM async/stream checker由后续正式门覆盖。当前 `/tmp/dever-linux-ci-refresh.py` 托管重建CLI/maker与双CMS/协议产物；四runtime归档和三个压力客户端源码未改，逐字节摘要核对后复用原 `linux-ci-author-final/provenance.json` 来源，新的provenance会明确记录复用，不冒充归档重编。 +- 新编译器/打包工具及双CMS/协议产物刷新4/4通过、无OOM（峰值1,046,528,000字节）。冻结源码 `900043ab2c819372ac22b1c8c44bf905d23a115c2f7a936eab0e38dbb04c86d0`，CLI SHA `c22b9fc622322d0b2456cd929533b9f0414c2a9ce48cb1a547fdf6582c078b61`；`target/linux-ci-author-current/provenance.json` 明确保留runtime/peer原构建来源。新 `cms-linux-ci-closure` 两源码分别47.71/47.90秒构建，8,324,776/8,327,016字节，SHA与原CMS一致。临时setting已指向新产物，准备后input SHA `cbdb39c8284c4a3685567c7417fb54eb0dadfe3ff92d7603f3d00b05347b9fc3`。`target/linux-ci-frozen-matrix.log` 串行启动inventory/author/static/features/python/sdk/default/all-features；仍未记为全部通过。宿主strict仍0/userns限制1，未安装profile/helper。 + +- 冻结版inventory `1790970867807152470` 在backend静态归档生成阶段因ENOSPC失败，尚未执行完整测试；峰值785,174,528字节、无OOM。停机后清理4个遗留测试可执行文件、3个旧CLI可执行文件及失败静态归档，约406MiB，均为可重建产物；当前CLI/输入原料/报告未删除。 + +- 冻结版inventory重跑 `1790971446868474214` 通过（108目标、217分类用例）；author `1790971823437290546` 6/6通过，准备后完整input为 `d8657e37f743d4758bbcc6fc5c045dac032df62eedeea58556f824798b0eae87`；static `1790972018127138063` 格式/default与all-feature Clippy均通过，随后继续features/Python/SDK及完整矩阵。 +- 在同一受限control cgroup内逐字节核对旧performance ELF，将58个完全一致的单链接程序合并为硬链接，释放311,181,312字节；历史路径/字节/报告全部保留，当前两个closure输入目录未改。明细 `target/linux-ci-cache-dedup.json`。未删除其他项目或旧数据库。 + +- 冻结版features `1790972235115826233` 20命令通过（19最小组合及CLI树），Python `1790972375205452463` 85/85、SDK `1790972401999175653` 三生态协议通过。default `1790972457820456527` 在CMS5/6后因测试工具hard_link遇私有/tmp跨文件系统失败；CMS真实check/test/run/build/package链已通过，非产品错误。后续default/all-feature改用原盘临时目录，不改源码规避。另复核当前artifact后回收6个过期runtime/embedded bridge文件约141MiB；原候选中的fe5c5dcd75301ecf已被default重新使用，明确保留。 + +- 缓存清理纠正：上述bridge `9735e2773af0aacd` 虽旧时间戳，default仍复用，不应算过期；其约51MiB被误回收后由Cargo自动重建，源码和固定输入未受影响。其余4项约90MiB才是本次实际旧缓存回收，不将暂时消失的有效缓存计为净节省。 + +- 同盘/tmp的完整矩阵 `target/linux-ci-full-tests.log` 已启动；当前default目录 `1790972808956136476`，已通过CMS6/6、原生入口16/16、REST12/12、Port/异步/连接池/合同语义，仍在运行，尚不记为全门通过。 +- 根据实际inventory/default artifact联合图、当前作者工具fingerprint依赖闭包及固定inputs排除，独立核对377个单链接rmeta。主代理持有 `target/debug/.cargo-lock`、确认无writer，再复核最新artifact与dev/inode/size/mtime/blocks后回收361,652,224字节；明细 `target/linux-ci-rmeta-cleanup.json`。都是可重建元数据（含旧rlib配套metadata），不把它们一概称为check-only；未删除任何当前登记库、固定输入或源码。 + +- default `1790972808956136476` 全78目标/650测试实际通过，但阶段结束因固定input漂移拒绝通过；source仍为900043...、无OOM（进程峰值1,167,564,800字节）。精确重算确认唯一变化是 `target/debug/component-fixture`：其Cargo功能组合重链接由66743d...变为089a2d...；只替换该摘要即重现原input d8657e...。未放宽质量门、未将失效证据标pass。 +- 修正验收输入边界：两处显式LLVM Worker fixture改读 `target/native-release-inputs/component-fixture`，普通Worker测试仍读Cargo各自构建的bin；README/spec说明先准备并记录独立副本。临时setting同步固定路径；产品代码/编译器/runtime/已建CMS与协议程序未改。`target/linux-ci-immutable-fixture-matrix.log` 托管重新准备fixture、inventory/author/static/features/Python/SDK，随后须用同盘/tmp重跑default/all-feature。原650结果保留为诊断，未冒充新正式证据。 +- 新冻结源码 `fbe4a92970850e29b3561cc22af437bc1dbe2819d173af5ad32a4848145d4461` 的固定fixture准备通过，程序SHA `58a5a120104baabac29cc712358174b0a1573ca726744fab14168edf5e9bb802`，独立副本/provenance写在 `target/native-release-inputs`。inventory `1790975682052927294`（108目标/217分类）、author `1790976010020603166`（6用例）、static `1790976202296015153`、features `1790976561899757466`（20命令）、Python `1790976771445471337`（85项）、SDK `1790976798389126261` 均通过；准备后input为 `7d5ef87ea08c9ed8803b8c32733578d87b5b2fcc054c9f527fe27aa106077ce3`。受限进程峰值1,638,834,176字节、无OOM。`target/linux-ci-immutable-full-tests.log` 已按all-features→default启动，同盘/tmp；完整矩阵、PG/native/长测仍未记为通过。 +- 发行测试空间预审确认 optimized 安装时根盘同时保留两份约603MiB payload，当前余量不足。为避免必然ENOSPC，主动SIGINT结束all-features `1790976909411656631`，报告如实为KeyboardInterrupt，未记完成。复用已有 `machine_temporary_root` 配置，将 optimized machine、signed_core release/isolated、bootstrap downloaded 四处统一接入既有临时目录owner；author仍留同盘支持原料hardlink。独立审查确认所有hardlink同文件系统、安装仍真实copy、回收边界不变；两文件rustfmt通过，未改产品或验收断言。新源码 `3ca3f74e4696973e5dbde34efa188dda6ad103861c98a8576def3d6d31223254` 正在 `target/linux-ci-release-storage-matrix.log` 重跑inventory/author/static/features/Python/SDK，前版证据不算新门通过。 +- 3ca3版准备门全部通过：inventory `1790977493517800171`、author `1790977818442118586`、static `1790978011370964455`、features `1790978044513313008`、Python `1790978070742789549`、SDK `1790978097831556704`；准备后input为 `02c363bd03e50772acfefe2b59d2493a82665faa74616ef2d502b7d0bd4700da`，无OOM。再次基于新artifact/作者闭包清单、持锁复核368个旧rmeta并回收334,819,328字节（`target/linux-ci-rmeta-storage-cleanup.log`），根盘余量约702MiB。正式all-features→default运行日志为 `target/linux-ci-release-storage-full-tests.log`；此次复用未变的fixture/runtime/CMS及其原provenance，不将测试目录接线改动冒充产品重编。 + +## 2026-09-30 用户批准的最终交付顺序 + +范围固定为以下六个闭环，不再重新设计已稳定的语法、API、目录角色或 CMS 业务。单个模块或 fixture 通过不等于整个闭环完成；代码、正式资产、真实平台验收分别记录。 + +1. [x] Dever Package：版本/传递依赖、共享摘要缓存、安全安装/更新/移除/doctor、源码加载和传递 Lib 统一锁已完成。2026-10-02 已通过签名安装 → LLVM daemon 编译 → check/test/run/build → 停止 daemon 后独立执行的自有完整 fixture，没有私有缓存旁路。公开 registry 仍未发布，归入第 5 项正式发行。 +2. [ ] 正式 runtime/build packs 与 OS sandbox：可重复制作/签名/版本绑定，完整标准库与构建资源，以及真实拒绝未授权文件/网络/子进程操作的验收；不把 capability 回显当沙箱。 +3. [x] 完整 Lib（当前 Linux x86_64 实现）：Go 自动 Worker 编译/链接与 sumdb、Python extras/sdist/PEP517 原生构建、npm 高级依赖与受控 hooks；官方 UUID、simplejson C 扩展及 bufferutil 原生 addon 已通过实际隔离构建、锁定重放与离线 Worker 运行。正式多平台包仍归第 2/5 项。 +4. [ ] 正式原生后端：完整 HIR 语义到 LLVM/LLD、稳定运行时 ABI、六目标资源，干净机器离线 check/run/build；不以隐藏 rustc 或子进程重命名替代。 +5. [ ] 共享编译与发行：deverd 编译 IPC、跨用户产物、签名目录/下载、平台权限与服务、升级回滚卸载，六宿主到六目标的真实验收。 +6. [ ] 最终质量：完整测试/CI、双源码与 PG/HTTP2/Job/上传组合、持续并发/故障恢复、体积/冷启动/延迟/RSS/64/128 MiB 容量报告。 + +验证继续从必要的最小定向检查开始。全量、服务级和正式发布步骤先说明并遵守用户授权；未经授权不提交、发布、安装全局服务或使用真实签名身份。每阶段完成后复核本节及原始验收合同，未完成项不得归档。 + +已接通 Package 配置/解析/锁/cache/source/exec、受管 Go 构建与 LLVM/LLD 连接桥。后续实现了同步强类型函数、受管值/集合、业务失败/Result、Bytes/File/同步 Stream 与真实协程的 Task/Group/Channel/sleep/blocking/parallel、AsyncStream/TCP/timeout/race/ParallelEach HIR→LLVM 和 runtime ABI；2026-10-01 又完成 CMD/wire/Dever Adapter 应用根，以及非租户 Model/双数据库 SQL/事务/DbError/Related/RowStream/schema/Seed 切片，证据分别见对应复验节。2026-09-30 用户批准的 unsafe 例外仍只在独立 `dever-backend-bridge` 的私有 `ffi.rs`，workspace 继续 forbid；runtime 本体、core、CLI 和 tests 不扩大例外，不开放 Dever 语言 FFI。私有默认 CLI 与 Linux 可信 daemon 已接 LLVM/LLD，不再调用 rustc；正式目标 runtime packs 和发行验收仍未完成。Go 保留环境变量禁令，采用固定目标受管工具路线，不引入 GOOS/GOARCH/GOCACHE/GOPATH 运行配置。 + +当前已完成 Linux 可信 daemon 编译、跨用户编译缓存及取消/断连/真实期限回收，真实三生态高级依赖和 Linux root 下的 OS sandbox 已通过定向验收。四个优化 runtime archive 已刷新；双 CMS/真实 PG/Job/API 与 LLVM 性能、64/128 MiB、HTTP/2 恢复矩阵均有独立报告。Linux签名首装、双项目/真实双UID共享使用、升级/篡改拒绝和移源执行通过。三生态最终签名安装、受管check/run/build、移除源码及机器目录后的无系统语言独立执行亦已通过，含Python/Node源码构建原生依赖及离线收据重放。2026-10-03已补齐受限AppArmor真实非root入口、三生态非root独立执行与本轮Linux质量门，证据及版本边界见顶部最终结果。仍缺其他平台产品实现和资产及正式公开发行，按用户要求暂缓；剩余长测免跑，尚未公开发布或安装全局服务。 + +以下保留原始任务的历史阶段编号；当前交付顺序以上述六个闭环为准。 + +## 阶段 0：任务基线与旧记录收口 + +- [x] 核对既有子任务实现与最新源码，标出“已实现但任务未归档”和“文档中已过时的未完成项”。 +- [x] 为每个子任务补充真实验证命令和阻塞项，不把历史记录当作当前缺口。 +- [x] 保持源码配置只走 `config/setting.json`，清理旧环境变量说明。 + +## 阶段 1：Production API 与真实 PostgreSQL + +- [x] 补真实 PostgreSQL fixture/配置入口，默认无配置不连接数据库。 +- [x] 完成 PostgreSQL 授权 Store 的权限目录、角色、跨站点、多角色和撤销验收，以及 CMS Owner HTTP/两个物理租户数据库验收。 +- [x] 完成同一真实 PostgreSQL HTTP 链路的非 Owner 多角色、跨站同名角色及撤销立即失效验收；使用仅注入临时 CMS 副本的成员 fixture，生产 CMS/Owner 保留角色规则未改。 +- [x] 完成请求、响应、Cookie/Header/Context、上传、取消、事务、HTTP/2 的已有定向回归；不代表所有 PostgreSQL/双源码组合已运行。 +- [x] 更新 `09-20-production-api` 和父任务；共享 ORM fixture 已迁移并通过真实 PostgreSQL，未完成的任务不归档。 + +## 阶段 2：大型 CMS + +- [x] 按当前组件目录约束实现账户/会话/权限、内容/版本/状态、媒体、发布、审计和清理 Job。 +- [x] 完成 Dever 与 Markdown 双源码行为合同和等价测试。 +- [x] 完成 `check/test/run/build` 与 `api/worker/all` 验收；CMS isolated `check/test/run/build/package/run` 双源码长链路已通过。 +- [x] 删除旧 demo-only 路径，更新 CMS 文档和示例；新增 `system.health.ping` 作为无数据库运行烟囱，不恢复旧 `news.article.demo` 兼容入口。 + +## 阶段 3：External Lib 生态 + +- [x] 完成离线 lock/resolver/cache/artifact 边界、Adapter 精确 Lib 请求接线，以及真实 PyPI wheel/npm registry/Go proxy 解析器；显式联网入口依赖已验证的安装版本及签名 runtime 描述，不访问宿主语言工具。 +- [x] 完成 Python、JavaScript、Go SDK 协议库和显式宿主验收。 +- [x] 支持 external pip/npm/go，三 SDK 共用 checked wire manifest；Python/JS 自动同名 entry 绑定、离线运行树和受控启动已有实现及定向 fixture 验收。 +- [x] 完成锁定 host-target build pack 的 Go compile/link、build tags、module ZIP、go:embed 和静态操作绑定，并验收源码移除后的独立 Worker。 +- [x] 完成 Dever Package 版本/传递依赖、源码加载、共用缓存与 Lib 锁、离线移除、失败回滚，以及真实安装到独立应用运行的自有 fixture 链路。 +- [x] 完成 Python extras/wheel/sdist/PEP517 原生构建、npm 高级安装/受控生命周期/native 输出和 Go sumdb；默认合同及真实第三方包定向验收通过，见 10-02 记录。 +- [ ] 交付正式签名语言 runtime/build packs，并在无宿主语言环境验收；复制宿主工具的自有 fixture 不代替发行验收。 +- [x] 完成 fixture run/build preparation、Worker 资源完整性、取消/超时/中断和并发协议回归。 +- [x] 完成 Linux x86_64 三生态程序的无宿主语言环境独立执行，包含Python/Node源码构建原生依赖和离线收据重放；Go官方UUID另有隔离构建/移源Worker证据。正式公开与跨平台交付仍属于上述未完成项。 + +## 阶段 4：共享工具链与发行版 + +- [x] 完成 Linux 开发态 `deverd` 认证 IPC、租约、并发发布、完整性校验和缓存状态/清理命令；跨平台 peer credential 与系统服务注册仍阻塞于发行层。 +- [ ] 完成 Linux/macOS/Windows 权限、服务注册、安装、升级、卸载和安全检查（跨平台发行层阻塞）。 +- [ ] 实现公开 HIR→LLVM/LLD 后端与六 target runtime ABI packs，再验证干净机器无 cargo/rustc；Rust bootstrap 不代替此项。 +- [x] 实现隔离的 LLVM 18/LLD 连接桥，验收六目标 object/link、输入拒绝、失败恢复、并发独占和私有输出目录；不是完整 HIR 后端或发行 SDK。 +- [x] 实现同步强类型函数 HIR→LLVM 内核及源位置/调用链，验证真实对象、本机执行、静态 handler、复合值和数值边界；不代表资源/异步/应用语义完成。 +- [x] 实现独立 runtime-abi staticlib 的数值/Text/Bytes 缓冲边界，并通过 canonical C header 和 LLVM object 的实际调用验收；复合资源/异步 ABI 和正式 target packs 未完成。 +- [x] 实现 Linux 内核 peer credential、跨用户 Lib artifact IPC、配额与有界 admission；真实不同 UID 的自有 fixture 已通过。 +- [x] 实现 Linux 可信 daemon build IPC、签名核心/pack 校验、跨用户编译缓存与取消/超时回收，恢复机器管理的 Package 完整执行合同。 +- [ ] 实现 macOS/Windows peer adapters、系统服务与正式发行；Linux 自有 fixture 不代替全平台验收。 + +## 阶段 5：最终质量门 + +- [x] 完成本轮 Linux 全量 workspace 矩阵和阶段定向测试；版本边界见顶部最终结果，其他平台延期。 +- [x] 完成本轮真实 PostgreSQL、CMS 服务级、External Lib 真实运行时和 Linux 签名发行验收；公开发行另行延期。 +- [x] 完成 LLVM 四 profile/双 CMS 体积、启动、并发、RSS、64/128 MiB 与共享编译缓存报告；HTTP/2 含一分钟稳态及 100 轮恢复，不把短测或私有 debug 编译耗时当生产饱和上限。 +- [x] 在私有临时目录准备同版本 rustfmt/clippy；core/runtime 与 CLI libs/bins 的 Clippy `-D warnings` 通过，不安装或替换全局命令。 +- [x] 完成源码全局格式及本轮 Linux CI 质量门；剩余长测按用户要求免跑，不计通过。 +- [ ] 更新规范、归档完成子任务和总任务（本轮规范和执行记录已更新；总任务因跨平台/正式公开发行延期继续in_progress，不归档未完成任务）。 + +## 2026-09-29 收口复验 + +- 通过：`cargo check --offline --locked --workspace`；API declarations 16/16；SQLite authorization 1/1；CMS parity 1/1；External Lib 6/6；shared toolchain 7/7；CLI Lib 单测 4/4；Dever CMS `check`（Dever/Markdown）及 Dever native build/package/run。 +- 通过：真实本机 PostgreSQL authorization ignored 1/1（记录在 `09-20-production-api/implement.md`）。 +- 修复：无租户 Job 的 tenant migration 未定义符号；CMS 打包验收的工作目录配置缺口；恢复 CMS 双源码测试入口。 +- 追加修复：无数据库的 application-test native 编译不再生成引用 `dever_runtime::database` 的 tenant Job 空实现；`application_testing` 8/8 已通过。 +- 追加修复：`deverc run` 在启动临时 native 程序前将项目 `config/` 安全复制到受保护的 build staging 目录，并拒绝配置目录符号链接；因此运行配置仍只来自 `config/setting.json`，不引入环境变量。 +- 阻塞/未运行：旧 `postgres_orm` fixture 在连接前因当前 Model 规则失败；真实 pip/npm/Go registry、受管 runtime/正式 SDK、跨平台服务注册与 peer credential、全量 workspace、64/128 MiB 压测、真实持久服务；rustfmt/clippy 组件未安装。 + +## 阶段 5 定向质量门记录 + +- 通过：workspace `cargo check --offline --locked`；`api_declarations` 16/16、`application_config` 6/6、`application_testing` 8/8、CMS parity 1/1、External Lib 6/6、shared toolchain 7/7、SQLite authorization 1/1。 +- 通过：CMS isolated 长链路 `cms_project_checks_runs_and_builds_from_an_isolated_copy` 1/1（Dever/Markdown 的 check、test、run、build、package、package-run）；native stripped 产物约 6.3 MiB。 +- 通过：项目 formatter 对 Dever/Markdown CMS 的 `fmt --check`；`cargo fmt`/Clippy 仍因组件未安装不可用。 +- 保留阻塞：全量 workspace 测试、真实 registry/runtime、跨平台发行、旧 ORM fixture 迁移及性能/RSS/64/128 MiB 压测未纳入本地完成证据。 + +## 首个实施目标 + +优先启动阶段 1，使用现有 production-api 代码和测试作为基线，只补真实 PostgreSQL fixture/配置及最终组合验收；不先扩展新语法。 + +## 2026-09-30 实施与当前源码证据 + +任务仍为 `in_progress`，没有把五个阶段记为完成。上面的 09-29 数据是历史定向证据，不能代替当前源码的运行级/发行验收。 + +已实现并复验： + +- 共享 SQLite/PostgreSQL ORM fixture 迁移到当前 App/Model 结构，保留事务、CRUD、分页/游标/流、错误恢复及回滚断言。 +- Lib 配置统一为 `config/setting.json.lib`，拒绝旧独立配置文件；canonical v2 lock 绑定 checked Worker schema/操作/capability/Adapter/依赖闭包，允许不同 Worker 独立锁版本，拒绝同一环境冲突和完整 provider 元数据漂移。 +- run/build 自动嵌入全部 checked exec 候选，原子提取、Unix 私有权限/所有权和每次 Worker 启动的精确清单/摘要复验；不扫描其他 bundle。清空环境、不带源码 Worker 的重复运行与篡改拒绝实际通过。 +- 三语言 SDK 的实际协议、i64/JSON 预算、错误/取消/关闭；不合作取消一秒终止。显式宿主验收通过，不代表无宿主发行。 +- `deverd` 同所有者 token 校验、OS singleton 锁、断连/坏帧容错和固定线程/队列；缺 token、256 字节长度差、慢/坏客户端及重复实例/进程退出恢复回归通过。 +- base-only CMD/Job 不再引用未启用 database/tenant 模块；当前四 profile 均实际编译通过。 +- CMS 性能入口改为 stock 双源码的真实 bootstrap、tenant migrate/Owner、admin/front 登录、REST 创建/替换、发布、前台完整查询和重复发布 409；删除旧 `READY|3` fixture，不用无关记录数作为业务验收。 + +当前通过的定向检查： + +- `cargo check --offline --locked -p dever-core -p dever-cli -p dever-runtime --features 'dever-runtime/external dever-runtime/postgres dever-runtime/api'`。 +- `external_resources` 6/6；`native_entry` 15/15；`port_adapter` 15/15；CLI `external_libs` 14/14；`shared_toolchain` 10/10；`application_config` 6/6；`application_testing` 8/8；`cms_acceptance` 1/1。 +- `test/external-sdk/check.py` 的显式 Python/Node/Go 宿主协议验收通过。 +- 真实 PostgreSQL 16.15：`postgres_authorization_preserves_site_scoped_roles_and_catalog` 1/1、`postgres_orm_uses_the_configured_isolated_database` 1/1，后者约 398 秒。服务端 deb 仅解压到自有临时目录，使用现有低权限账号和专用 loopback 端口，不安装系统服务。配置写入临时根 `config/setting.json` 的 `postgres_test` URL 模板，全部结束后恢复原先文件不存在状态，关闭自有实例并删除测试库/下载包；服务日志保留在 `target/performance/postgres-acceptance-09-30-server.log`。此证据不是 PostgreSQL HTTP/租户完整组合验收。 +- CMS stock 双源码实际 `check/build`；产物 Dever 6,594,936 字节、Markdown 6,595,576 字节；每项 SHA 与构建日志在 `target/performance/cms-publish-09-30-v3/manifest.json`。 +- CMS 两套源码在 64 和 128 MiB 自有 cgroup 内各完成 16 篇顺序发布及完整输出核对，无 OOM；专属 cgroup 和服务进程全部清理。当前报告 `target/performance/cms-publish-64-09-30-v1/report.json`、`target/performance/cms-publish-128-09-30-v2/report.json`。峰值 RSS 约 33.3–33.7 MiB；64 MiB 下创建中位约 1.9–2.1 ms、发布约 2.2–2.4 ms，启动约 45–51 ms。这是 loopback/暖页、小规模顺序业务基线，非并发上限、冷缓存总内存或生产 p99。 +- 四 profile 体积：base 919,720、SQLite 2,901,400、PostgreSQL 2,920,488、both 4,884,760 字节，生产 fat LTO/strip 未放宽。报告 `target/performance/profile-cmd-09-30-v2/manifest.json` 与 `profile-report.json`，只构建不连接数据库。 + +失败及后续修复证据：首次 native 检查的开发 rustc PATH 缺失与旧测试合同已修正；发现真正的 base CMD database/tenant 引用缺陷并修复后复验通过。首次临时 PG 配置缺少必需 database.default，补正完整配置后两个实际测试通过。CMS v2 构建的 180 秒等待超时，v3 使用有界 900 秒构建并保留超时输出/关闭自有进程组，实际双源码构建通过。旧报告和失败目录未覆盖。 + +仍未完成的产品代码与验收，不混为“缺环境”： + +- Go 受管 compile/link、Python extras/sdist、npm 高级安装语义、Go sumdb、正式签名语言 runtime/build packs、无宿主发行验收，以及 Dever Package 安装与传递 Lib 集成；真实三生态 resolver、三 SDK typed manifest 和 Python/JS 自动 Worker 代码已补，见本轮追加。 +- OS capability sandbox；当前仅编译 effect 与握手 admission。 +- daemon build 服务、macOS/Windows peer adapters 和三平台正式安装/服务注册/升级/卸载；Linux 跨用户 artifact IPC 已补并通过实际不同 UID 验收。 +- HIR→LLVM/LLD 公开编译器和六 target ABI packs；无 Cargo/Rust 的公开项目构建仍未交付。 +- Markdown PostgreSQL/HTTP2 全组合、持续并发 CMS/全性能矩阵、全量质量门。当前相关的定向测试不代替这些；Dever CMS 的 PostgreSQL 非 Owner HTTP 授权组合已追加通过。 +- 全局工具未安装/替换；私有 rustfmt/Clippy 已可用且以下定向静态门通过。没有默认运行全量 workspace 测试。 + +### 2026-09-30 追加运行与质量复验 + +- PostgreSQL 16.15 的 `postgres_cms_http_isolates_two_tenant_databases` ignored 定向验收 1/1 通过：真实 CMS 原生程序、两个租户独立数据库、admin/front 登录、每租户两篇创建/替换/发布/查询、相同 slug 不同标题、重复发布 409、跨站 Cookie 和跨租户登录 401。报告 `target/performance/postgres-http-09-30-v1/report.json`；不包含非 Owner 授权撤销。Rust fixture 只复制 stock CMS 到自有临时项目，数据库清理失败也会使验收失败，配置 URL/密钥不写摘要。 +- 清理独立复核：PG 仅残留原有验收基库和 postgres，测试控制库/两个租户库均删除；停止本轮自有实例，临时根 `config/setting.json` 恢复为原先不存在，专属 cgroup 和服务进程回收。服务器日志 `target/performance/postgres-http-09-30-v1/server.log`；本轮 PG 临时目录移入回收站，可恢复，未注册系统服务。 +- PG 首次失败为工具在原位私有配置上使用独占创建，已修复为同目录原子替换并验证序列化失败不破坏原文件；第二次为 Rust wrapper 对含 `..` 的 workspace 路径判定错误，已 canonicalize 后第三次通过。独立复核进一步指出公开工具入口原位修改调用者配置，现已改为工具自身复制到 TemporaryDirectory，构建失败也保持源配置字节不变，副本自动清理。 +- SQLite ORM 实际 HTTP：64 MiB 自有 cgroup，128 行、短窗口 50 目标 QPS，25/25 正确响应、无错误/丢弃，实际约 49.91 QPS、请求 p50 约 1.056 ms、峰值 RSS 约 7.54 MiB；单连接事务断连后约 5.965 ms 回收、计数仍为 0。`target/performance/orm-runtime-64-09-30-v1/report.json`。仅短时合同基线，不是饱和吞吐或 p99。 +- typed App→Port→Adapter live TCP：64 MiB 下 8 次实际 echo 全部正确、RSS 约 4.77 MiB、无 OOM，`target/performance/live-tcp-64-09-30-v1/report.json`。不声称 TLS/WebSocket/SSE 压力已复验。 +- rustfmt/Clippy 使用 Rust 1.98.0 同版本的校验下载包，临时解压，不安装全局工具。core/runtime 与 api/postgres/external 组合 Clippy `-D warnings` 通过；CLI libs/bins 同样通过，日志 `target/performance/quality-09-30-v1/clippy-core-runtime-v3.log` 与 `clippy-cli-v3.log`。运行时数据库 open 改为具名 ConnectionOptions 并迁移所有调用点；不改变 setting.json 格式或连接语义。 +- 当前追加定向通过：`external_resources` 6/6、formatter 13/13、ORM performance fixture 4/4、ORM query 9/9、Port performance fixture 3/3、syntax/format 22/22、CLI external_libs 14/14、shared_toolchain 10/10、Python CMS/runner 55/55。 +- ORM query 的过时断言已迁移：事务内密码哈希保留 C014 拒绝，合法路径先哈希再进入短事务；空 create_many 从 owning-domain App 调用，公开名称不带 app 段；where 外 contains 仍拒绝，断言按当前 C004 诊断。没有放宽 checker 合同。formatter 测试补齐新 AST 声明，完整 13/13 通过。 + +### PostgreSQL 非 Owner HTTP 最终补验 + +- `postgres_api` 同一 exact ignored 目标当前 1/1 通过，约 41.29 秒；新摘要 `target/performance/postgres-http-09-30-v1/report-1968019-1790739069361983512.json`，日志 `test-rbac-final.log`，旧 Owner-only 报告保留。stock CMS 副本只追加 `test/dever-tests/fixtures/postgres_api/` 的普通成员 CMD/Reader membership，不修改生产源码或 Owner 不可撤销规则。 +- 真实会话成员 user_id=3，与两个 Owner 不同，三个权限来自各站点实际目录而非硬编码 key：授权前三个 GET 均 403；两个 admin 角色授权形成并集，front 同名角色独立；撤销首个 admin 角色后原会话对应 GET 立即 403，另一 admin GET 及 front GET 仍 200；继续撤销剩余角色后均 403。Rust 独立核对 ID/互异权限键/状态，且在数据库清理成功后才保存通过摘要，不记录密码、邮箱或 Cookie。 +- 两轮工具失败保留：首轮误从 admin 目录找 front 权限，现按当前 site 过滤合同分别读取；次轮 DELETE 发 JSON body，返回 400，现统一在 CmsClient 的 GET/DELETE 路径编码 query 且不发 body。没有放宽目录隔离或 API 输入规则。新增负向/编码回归；Python CMS/runner 最终 57/57，Rust 非 ignored 3/3，私有 rustfmt 对 postgres_api.rs 检查通过，CLI Clippy 最终复验 exit0（`clippy-cli-final.log`）。 +- 最终清理独立复核：仅保留本来就有的验收基库/postgres，client backend 数量 1 是核对自身；已停止本轮 PG，临时根配置再次恢复不存在，PG 临时目录再次移入回收站。`server-rbac.log` 保留完整服务器日志。 +- 真实 provider 诊断也同步事实：当前 resolver 只接 FixtureRegistry，错误明确说 provider integration 未实现，不再暗示安装一个尚未接线的 pack 就能解决。该诊断修正不算 pip/npm/Go resolver 完成；CLI external_libs 再次 14/14。 +- 私有 rustfmt/Clippy 下载与解压目录在复验后也移入回收站，可恢复；未改全局 Rust 工具链。PG 临时目录本次回收名称为 `dever-pg-http-JbyaSX.2`，最终根配置不存在、42209 无监听、自有 CMS/PG 进程及 cgroup 无残留。工作区原有 untracked 源码和用户删除未回退，未创建 commit。 + +### 2026-09-30 缺实现补齐：真实 resolver、typed Worker 与 Linux 共享资产 + +本轮实际产品代码： + +- 三生态共享 `RegistryResolver`/transport/lock/cache owner，实现 PyPI PEP 508/440 wheel、npm semver/SRI、Go proxy/MVS；只从当前已验证发行 core 的签名 runtime 元数据取 pack。拒绝代理、非协议允许的重定向、网络错误回退旧版本、坏归档、身份/工具链漂移，解析预算有界。Go ZIP 最多两跳手工转入固定 HTTPS GCS bucket，规范化目标检查在请求前完成，总 30 秒 deadline、循环/异常 Location 拒绝、签名 URL 日志脱敏；自有 loopback mirror 不成为项目配置。 +- Adapter contextual pip/npm/go 语法、生态/零 Lib runtime 锁定与 checked SDK manifest 接线。三 SDK 复用 setting/input/output/error 的 wire DAG；Python/JS runner 自动绑定 Port 同名导出，不增加业务注册表或第二套动态 App 调用。 +- Python/JS 离线展开 runtime/dependencies/owned source，生成 launch manifest，启动只接受精确资源树中的解释器/argv/cwd。Python 固定 `-I -S -B` 且先导入可信 SDK；Node 兼容 ESM/CommonJS。普通 Dever tests 继续 fake，不启动 Lib。 +- Linux SO_PEERCRED 跨用户 artifact 上传/下载、owner-only token/clean、metadata-only status (`verified:false`)；64 MiB 单资产、2 GiB/4096 条总缓存配额，外部 UID 两连接/总量六连接，保留管理员容量。流式 exact-length/SHA staging、原子发布、操作租约和传输 deadline。 +- 原生资源使用独立二进制编译输入并按实际 SHA 共享一个静态 payload，编译前拒绝伪摘要，缓存身份仍含路径/摘要/长度/可执行位。移除 managed-only raw pack/archive 的重复嵌入；资源校验按 64 KiB 分块，不再额外读取整份解释器到堆。 + +已验收：CLI external_libs 最终 27/27（新增 Go ZIP 正常跳转、非法目标/异常 Location/循环/跳数、其他端点禁止跳转三项；主进程另独立复验 redirect 两项 2/2）；shared_toolchain 14/14，另 actual UID=65534 ignored 1/1（含 token、clean、二连接 admission 和未校验摘要）；external_resources 9/9（含独立 exec、重复资源、篡改和大文件）；native_entry 资源生成定向 2/2；Port/Adapter 16/16、Worker protocol/lifecycle 1/1。Python/JS 默认 Worker 6 项通过/6 个宿主专项默认 ignored，其中最终 Native managed Node exact 用例单独执行 1/1（285.45 秒测试总墙钟，包含打包/编译，不是启动延迟）。三语言 SDK 协议已分别通过显式宿主路径验收，不等同于正式无宿主环境。最终 core/runtime api+postgres+external 与 CLI libs/bins/三项变更测试的 Clippy `-D warnings` 通过；Go 追加变更后的 CLI Clippy 也再次通过，修改的 Rust 文件私有 rustfmt check 通过。未安装全局工具、未默认全量测试。 + +失败及修复：独立资源生成测试首轮固定 32 KiB 总源码上限错误，改为相对不含资源的基线增量，1 MiB payload 不增长为数组且伪摘要拒绝通过。Python 隔离参数下的 SDK 导入顺序和业务同名模块遮蔽已修；Native managed npm 初次已编译但临时 fixture 缺 setting.json，补自有 Adapter 配置后实际独立产物返回 code=0/data=8（已移除原 entry、lock、raw pack 并清空继承环境）。复制宿主 Node 的 pack 只证明 bootstrap 链路，复制 Python 仍依赖宿主 stdlib；不能记成正式发行。最终协议复核发现全面禁跳会阻断 Go ZIP 的官方存储跳转，已按精确 bucket 增加受限跳转；规范化 URL 会抹掉空 userinfo，故同时检查原始 URI authority，负向测试确认不发第二次请求。 + +根因复盘（Trellis break-loop）： + +1. 类别 E/D：小 exec/fake runtime 夹具隐含假设了大 runtime 也能作为数字数组生成，以及 Python script 目录会出现在隔离 sys.path；默认测试缺少真正的隔离启动和 SDK 遮蔽证据。 +2. 表面修复风险：仅减少 emitter 临时字符串不会消除重复 binary payload；只调整 Python source 路径而不先导入受信 SDK仍可能被同名模块替代。 +3. P0 防复发已实施：唯一内容输入/静态引用、差分源码大小/伪摘要回归、真实独立资源执行、固定隔离参数/SDK 导入优先级/同名模块用例，以及 actual UID 与缓存 quota/summary 测试。 +4. 扩展范围只覆盖同一资源打包/启动和 daemon 入口；没有改业务 API、真实配置或其他项目服务。 +5. 已同步 README、LANGUAGE、IMPLEMENTATION、compiler/toolchain specs 与三个相关任务的当前边界，不新增平行文档、commit 或归档；总任务保持 in_progress。 + +该轮结束时待做的 Package 集成和 Go compile/link 已在后续实现并复验，见下一节。当前仍未完成：高级生态安装规则/Go sumdb、OS sandbox、daemon build IPC、macOS/Windows peer/service adapters、完整 HIR→LLVM 与六 target runtime ABI packs、正式签名语言包/无宿主发行、持续并发/完整性能矩阵和全量质量门。连接桥已经有产品代码，不能再把它记为完全未实现，也不能把它记为完整编译器。 + +## 2026-09-30 Package、Go 与编译器连接桥复验 + +本轮已完成的实现边界： + +- `packages.rs` 复用现有 semver、ArtifactStore、Lib resolver 和原子锁：单版本传递闭包、精确归档/manifest 摘要、组件所有权、Package 源码注入普通 check/test/run/build、统一 Lib 锁、离线 remove 保留剩余版本、配置失败恢复原始字节。无本地 module 目录的 Package-only 项目也能加载。 +- exec 和受管 Worker 共用由 setting.json + canonical lock 决定的 Package 归属，不根据归档中是否有同名目录决定是否回退。真实 checker 的入口是 `module/<component>/<domain>/<entry>`；原有组件碰撞已阻止本地同路径替换。新增实际 prepare 回归验证锁定入口缺失拒绝，以及本地组件碰撞拒绝，不声称证明了可利用漏洞。 +- Go 仅使用锁定 build pack 的 compile/link/analyzer/stdlib/importcfg，工具以绝对路径和空环境运行；生成 main 静态检查同名 handler 的签名，并复用 SDK typed manifest。编译后仅嵌入 Worker 与合同，删除展开后的 raw build pack/module archive。显式作者工具制作的 host-target fixture 压缩 55,785,046 字节，展开 203,288,946 字节/358 项，未放宽原有 64 MiB 单资产与 256 MiB 展开预算。 +- 新增 `dever-backend-bridge`，安全 API 显式选择 Linux/macOS/Windows 的 x86_64/aarch64 目标；LLVM 在进程内 parse/verify/O2/object，LLD 在进程内串行链接。所有 foreign buffer 同一 allocator 释放，输出只在当前用户私有且祖先可信的目录独占预留；失败清理不覆盖已有文件。不可恢复的 LLD 状态终止私有 compiler worker。私有 LLVM 18 开发 SDK 不做全局安装，当前 Linux 作者 SDK仍使用宿主 LLVM 动态库，不能记成自包含发行包。 + +主代理最终验证证据（不是仅采用子代理报告）： + +- bridge `--features embedded --test backend_bridge`：5/5；六种真实对象和链接输出、IR/SSA/target/layout 拒绝、失败后重试、并发同输出唯一胜者、共享目录/可写祖先拒绝，Linux freestanding LLVM→LLD 程序在空环境实际退出 42。日志 `target/backend-bridge-test.log`。 +- Go ignored 专项单独执行：1/1，44.46 秒（整项编译+执行，不是启动延迟）；包含 module ZIP、目标 tags、go:embed、typed call 与源码移除/空环境独立执行。日志 `target/closure-go-managed-final.log`。 +- 最终 Lib 27/27、默认 Worker 6 项通过/7 项 ignored、Package 10 项通过/1 个 signed child 入口按设计 ignored/1 个既有慢链路过滤；signed child 由父测试实际执行，覆盖真实 `workers::prepare`。日志 `target/closure-package-worker-final.log`。 +- 本轮之前已由主代理独立执行完整 Package 集合 9/9，其中安装→check/test/run/build→停止私有 daemon→独立运行链路通过，总 221.17 秒；最后的锁归属加强只重跑相关集合，没有重复这个慢链路。日志 `target/closure-package-go-regression.log`。 +- 配置/Port/源码回归：application_config 6/6、Port/Adapter 16/16、source_architecture 12/12、source_visibility 10/10。日志 `target/closure-source-package-regression.log`。 +- 最终 CLI/core/runtime/bridge offline locked check（runtime external/postgres/api）通过；CLI libs/bins 与三项变更测试、bridge embedded all-targets Clippy `-D warnings` 通过；本轮修改的 Rust 文件私有 rustfmt check 通过。日志分别为 `target/closure-current-cargo-check.log`、`target/closure-cli-final-clippy.log`、`target/backend-bridge-clippy.log`、`target/closure-final-rustfmt.log`。 + +没有新增应用环境变量、默认联网或宿主语言回退；未运行全量、真实数据库/CMS 服务级或新性能压测，未使用真实发行签名身份，未提交或归档。自有测试 daemon 已退出，无遗留服务；磁盘约 7.3 GiB 可用。总任务继续 `in_progress`：完整 HIR lowerer/稳定 runtime ABI、正式 packs/OS sandbox、高级生态安装、daemon build/跨平台发行和最终质量门仍需完成。 + +## 2026-09-30 强类型 LLVM 内核与首段 runtime ABI 复验 + +实现：`dever-core::llvm::emit_kernel` 复用 checked HIR、分句 Domain 和静态 specialization，直接产生 typed LLVM IR。当前覆盖 Int/Float/Bool/Nullable、普通 Record/Choice、多输出、分句绑定投影、值拷贝/字段修改、直接调用/静态 handler、左到右求值、短路和 checked 数值错误；故障保留 origin span 和逐层 caller spans，容量由无递归的可达实例图界定。所有可达但未支持的类型、资源、异步、失败、API/CMD/REST/Job 或 intrinsic 都明确带源码位置拒绝，不改默认 CLI 后端、不引入动态 Value 或 Rust fallback。 + +首段 runtime ABI 采用固定宽度标量/u32 status/显式输出指针，Decimal 使用原 runtime 的 LE 双 u64 位表示；Text/Bytes 返回带所有者的 Buffer，错误保留原数值/UTF-8 消息。`runtime-abi` feature 和 `embedded` 分离，前者独立构建的 staticlib 不含 LLVM/C++ SDK；所有 C 导出/原始指针操作仍在唯一私有 ffi.rs，runtime 本体保持 forbid。canonical 签名、借用、对齐、非重叠和释放合同位于 include/runtime.h。C++ compiler Reply 和 Rust Box ABI Buffer 分配器不混用。 + +主代理最终证据： + +- `llvm_lowering` 4/4：typed IR、源位置拒绝、完整调用链，以及仅嵌在表达式中的 nominal type 定义。日志 `target/closure-llvm-lowering-tests.log`。 +- `llvm_native` 9/9：简单内核六目标 object/link、复杂聚合六目标 object、本机分句/Nullable/Choice/record 值语义、静态 handler、Float NaN、多输出、短路、Int 极值/除零/溢出及实际 origin/caller span；其中 Windows 未提供 target runtime 时的显式拒绝是负向验收，不是 Windows 复杂链接成功。日志 `target/closure-llvm-runtime-tests.log`。 +- 原 bridge 回归 5/5、ABI safe 合同 2/2；同上日志。 +- 独立 `--no-default-features --features runtime-abi` archive 在 `target/native-runtime-abi` 离线制作；C header→实际库和 LLVM object→实际库 opt-in 2/2，owned 子进程 env_clear 并按 5 秒期限回收。开发 C/Rust 作者工具只用于夹具，不是产品启动依赖。日志 `target/closure-runtime-link-tests.log`。 +- core/runtime/CLI/bridge（runtime external/api/postgres）offline locked check、bridge/new tests 严格 Clippy（runtime/api feature）、core/IR tests 严格 Clippy及私有 rustfmt check通过。ABI-only 构建成功，但它单独的严格 Clippy 仍遇到旧 HTTP client_address 的 dead_code 警告；未改无关 HTTP 或添加抑制。 +- 双方独立只读审查没有剩余的当前 kernel/ABI milestone 阻断；不代替六目标运行或完整后端验收。 + +实测发现并处理：错误帧数组 GEP 原多一个索引，已在 owning emitter 修复；收集类型只看签名/locals 会漏临时 nominal 表达式，已复用 `check::visit` 全遍历并补最小回归。类别 D/E（IR 字符串通过不证明 LLVM 验证/执行、隐含类型会落地到 local 的假设），P0 防复发是当前真实对象/执行链及嵌套表达式回归。扩到 Windows 复杂聚合时，实际链接暴露 `__chkstk`/`_fltused` target runtime 缺失；没有提供 fixture 替代符号，此项仍归正式 target packs 未完成，已有负向失败/自有输出清理证据。已同步 canonical compiler/toolchain specs，仓库没有模板副本，无 commit/归档。 + +剩余：完整资源/集合/Decimal/Text/Bytes 的 HIR lowering、业务错误/async/Model/Port/Adapter/API/Job、全部 runtime ABI、六目标 CRT/stdlib/runtime assets和正式签名/无宿主发行；总任务及六阶段未完成项继续 in_progress。未运行全量、真实 PostgreSQL、CMS 服务级或新性能压测，未安装全局工具、改应用环境变量或使用真实发行身份。 + +## 2026-09-30 下一段：同步受管值与集合 + +范围：沿用当前任务推进 Decimal/Text/Id、基础 Bytes、List/Map 的 checked HIR→LLVM 与真实 runtime ABI 调用。不修改语法、默认 CLI 后端或配置方式；业务失败/捕获、Bytes Result-choice、资源、异步、Model/Port/Adapter/API/Job、正式目标 packs 仍未完成。 + +- [x] 复用现有 Text/Bytes/数值及 List/Map COW；固定 typed element 的布局、静态生命周期回调与消费式更新合同，原始指针操作仍只在私有 ffi.rs。 +- [x] LLVM 为受管局部、临时、嵌套 Record/Choice/Nullable 建立具体类型所有权；正常、短路和故障出口统一清理，成功输出移交调用者。 +- [x] runtime 原始错误 Buffer 移交 fault 尾槽，保留 source origin/caller stack，并提供调用者释放；不混用 C++ Reply 分配器。 +- [x] 从真实 checked Dever 源码生成对象、链接实际 runtime archive，定向验收集合/静态 handler、别名与字段替换、提前失败、根输出及重复执行分配平衡。 +- [x] 最小定向检查、格式/Clippy、独立审查并同步 canonical 合同；分别记录支持范围和仍明确拒绝的 HIR，不把该里程碑称为完整后端。 + +实现边界： + +- `runtime.h` 的 Text/Bytes/List/Map opaque owner、process-static DeverRtType 回调及消费式 cursor 接入现有 runtime。元素存储用对齐的 MaybeUninit，不复制/读取 padding；普通 Element 保留 PartialEq，只有合法 MapKey 包装实现 Eq/Hash。Text split 使用调用者的同一 Text descriptor。 +- LLVM 拆分到 abi/ownership/intrinsics/collections/traversal，复用 native last-use/字段替换分析。typed guard 在入口初始化，正常/失败统一清理；根输出与原始错误分别通过 `dever_outputs_release`/`dever_fault_release` 释放。无动态 Value/解释器或 Rust fallback。 +- 支持 List/Bytes 的静态 each/filter/find/reduce/reduce_until/sum；C ABI 已有 Bytes 转换/切片，但其 BytesToText/BytesFromInts/BytesSlice Result-choice 源码 lowering 仍明确拒绝，不能记为完整 Bytes 已交付。 +- 本轮所有永久测试都在根 test/。共享 checked-source→IR 辅助函数复用同一 SourceMap/check/emit 路径，不另建测试编译器。 + +主线程独立复验（全部离线，2026-09-30）： + +- runtime-only staticlib 明确构建：`cargo build --offline --locked -p dever-backend-bridge --no-default-features --features runtime-abi --lib --target-dir target/native-runtime-abi`，通过；不含 embedded LLVM/C++ SDK。 +- `cargo test --offline --locked -p dever-backend-bridge --features embedded,runtime-abi --test llvm_managed_native --test llvm_native --test native_runtime_abi --test native_runtime_link -- --include-ignored`：**25/25**。其中 managed 10/10（一个六目标 object 用例，九个 Linux 实际执行用例）、原有 LLVM 9/9、运行时 ABI 4/4、真实 C/LLVM→archive 2/2。日志 `target/closure-managed-final-tests.log`。 +- `cargo test --offline --locked -p dever-tests --test llvm_lowering`：**5/5**，日志 `target/closure-managed-final-ir.log`。 +- core/runtime/CLI/bridge 的 offline cargo check（runtime api/external/postgres、bridge embedded/runtime-abi）通过，仅编译、不连接数据库；日志 `target/closure-managed-final-check.log`。 +- core/bridge 及上述四个 bridge 测试 target 的私有严格 Clippy `-D warnings` 通过，日志 `target/closure-managed-final-clippy.log`；root `llvm_lowering` 的严格 Clippy 也通过，日志 `target/closure-managed-final-ir-clippy.log`；本次改动 Rust 文件私有 rustfmt check 通过，日志 `target/closure-managed-final-fmt.log`。没有安装/替换全局组件。 +- canonical C driver 与 allocation driver 的语法检查通过。九个 managed 执行用例预热后各重复 64 次，结果与存活分配计数保持一致；这只是有界所有权回归，不是性能/RSS/64/128 MiB 容量报告。 +- 独立只读审查确认当前修复范围无剩余阻断；主线程验证关键执行证据，不以审查报告替代测试。 + +本轮实际失败与防复发：新增 `List<Text?>`/`Map<Text,Text?>` 边界执行发现 First/Get 原按 nullable 结果的内层值分配 ABI out，而 ABI 写入完整 nullable 行；同时外层 present 覆盖内部 null。此前只读审查和普通集合用例漏掉该合同。修复统一在 `llvm/abi.rs::runtime_optional_row`：按真实 row_type 分配，status 成功后才读取 present，无行不读 out，有可空行整行转交而非双包装。First/Get 共用此 owner，Find 同型路径也覆盖；新增 IR 与真实 native 回归后整组通过。根因是把“行存在”与“值非 null”混为一层的布局假设（类别 D/E），防复发合同已写入现有 compiler/toolchain specs，不创建重复规范或模板。 + +剩余先补业务失败/Result-choice，再接资源/异步及应用语义；正式 CRT/stdlib/runtime assets、签名 packs、无宿主发行与父任务其他未勾选闭环继续 `in_progress`。测试仅执行自有有限生命周期进程，不启动数据库/CMS 服务，不增加应用环境变量或默认联网；没有运行全量测试、新性能压测或六目标实际运行，没有 commit/归档。 + +## 2026-09-30 下一段：业务失败与 Bytes 结果 + +沿用已有 Rust 后端合同补同步 LLVM 缺口，不修改语言语法、Checker、默认 CLI 后端或应用配置。业务失败保持 nominal identity 和 typed payload，`result` 仅捕获已检查的业务错误；溢出、除零和 ABI 故障继续传播。复用 specialize/capture 的失败集合及匹配规则、已有 runtime Bytes 接口和受管值清理，不建第二套错误系统。 + +- [x] 具体类型的业务失败载荷、默认传播、根释放和源位置/调用链；捕获前后不泄漏或重复释放。 +- [x] `result(named_call)` 的零/单/多输出成功、direct/wrapped error、显式 outer identity 及 static handler 传播。 +- [x] BytesToText/BytesFromInts/BytesSlice 的成功/Failed choice；普通标准库调用和显式 result 共用同一通道，非法 ABI 不伪装为业务错误。 +- [x] 真实 checked-source→LLVM→实际 archive 执行，包含受管载荷/提前失败、捕获后再次失败、运行故障不可捕获、Bytes 边界和重复分配平衡；保留上轮回归。 +- [x] 最小 offline check、私有 fmt/strict Clippy、独立检查及 canonical specs/任务证据。 + +实现代理负责 core LLVM 与 IR 回归；主线程负责根 test 的实际执行验收、整合及规范。资源/异步/应用入口/正式 packs 仍不在此段完成范围；仅执行自有有限生命周期进程，不启动数据库/CMS/HTTP 服务或进行全量/性能测试。 + +实现与复用：`llvm/failures.rs` 统一 ordinary invoke/fail/capture/write_variant，复用 `specialize::failures`、`capture::targets` 与已有 TypeOwner；不改 Checker、runtime/bridge ABI 源码或默认 CLI。fault 前五字段不变，追加 type_id+1（0 为无载荷）、variant、按可达错误 Choice 最大布局对齐的内联 typed payload；Business code=4，匹配捕获后转交并清空整个 fault，根释放静态分派原类型。Bytes 三项只将 ABI status1 转为标准 Failed(Text)，status2 与数值/其他 runtime fault 继续传播。没有 Value/Any、错误字符串匹配、Rust fallback 或新 unsafe 边界。 + +主线程最终独立复验（离线，2026-09-30）： + +- `cargo test --offline --locked -p dever-backend-bridge --features embedded,runtime-abi --test llvm_result_native --test llvm_managed_native --test llvm_native --test native_runtime_abi --test native_runtime_link -- --include-ignored`：**37/37**。新 Result target 12/12（1 checked-source/IR 合同检查、11 actual Linux LLVM native），旧 managed 10/10、原 LLVM 9/9、runtime ABI 4/4、真实 C/LLVM 链接 2/2。日志 `target/closure-results-final-native.log`。复用此前显式准备的 runtime-only archive，未在测试中递归构建。 +- `cargo test --offline --locked -p dever-tests --test llvm_lowering`：**7/7**，日志 `target/closure-results-final-ir.log`。 +- core/runtime/CLI/bridge offline check（api/external/postgres、embedded/runtime-abi）通过，日志 `target/closure-results-final-check.log`，仅编译不连接数据库。 +- core/bridge 及上述 targets、root llvm_lowering 的私有严格 Clippy `-D warnings` 通过，日志 `target/closure-results-final-clippy.log`；改动与相关 LLVM 文件私有 rustfmt check 通过，日志 `target/closure-results-final-fmt.log`。未安装或修改全局工具。 +- 新旧 managed native 共用 `test/dever-tests/tests/support/llvm_managed.rs` 原有 runner 与 C allocation driver：每次实际执行保持明确 output/fault 释放，预热后 64 次结果与存活分配平衡。新测试覆盖错误载荷 COW、原始 Bytes 文案、捕获后重失败、第二 variant 的根释放及精确 origin/两 caller spans。这是有界所有权回归，不是性能或容量报告。 +- 独立 trellis-check 只读对照 Rust native/capture 合同，无当前 milestone 阻断；主线程重新核验关键运行证据，不以子代理报告替代验收。README、LANGUAGE、IMPLEMENTATION 与现有 compiler/toolchain specs 已同步，不建重复设计文档。 + +质量修正：原验收 helper 曾同时认可“任意成功或失败”分支，零输出捕获可能被错误 Done 误判为通过。主线程读用例后加入逐次 wanted_success/预期内层错误标签，成功/失败及 Missing/Conflict 互斥；加强后整组复验通过。记录的是测试判定缺口，不是已观测到的 LLVM 实现故障。中途其余失败是 fixture 语法/类型上下文错误，均按现有合同修正，没有放宽 Checker。防复发要求写入 compiler-contracts 的 Required Tests/Wrong vs Correct。 + +下一段继续资源生命周期与异步语义，再接 Model/Port/Adapter/API/Job 应用入口和正式目标资源。父任务与六个闭环的未完成项仍 `in_progress`；没有全量/数据库服务/CMS/性能/跨平台运行验收，没有 commit 或归档。 + +## 2026-09-30 下一段：资源生命周期与真实协程 + +当前用户已确认继续；沿用父任务，不切换默认 CLI 后端，也不改变语言或配置语法。 + +- [x] File 的 create_new/open/read/write/close、共享别名关闭与最后 owner 释放；原始 I/O 失败进入标准 choice/fail 通道。 +- [x] 同步 Stream 的共享游标、文件 chunks、内部 pull/close 与静态 handler 遍历，EOF、最后错误项和提前释放保持 runtime 合同；不新增源码 pull/first(Stream)。 +- [x] LLVM18 switched-resume coroutine + runtime ForeignFuture：真实 Pending/恢复、owned 输入/局部量/输出与 typed fault，初始/pending/final destroy 配对清理；不是每函数 block_on。 +- [x] 基于现有单 runtime Scope/supervisor 的 Task、Group、Channel、sleep、checked 同步 blocking/parallel；取消和失败先排空后代，保留业务错误身份与来源。 +- [x] checked-source → LLVM object → 实际 runtime archive 的有界自有资源/进程验收、旧定向回归、offline check、私有 fmt/Clippy、独立检查与 canonical 规范同步。 + +归属：bridge 只在原 private ffi.rs/header 增加资源和 future ABI；core 在 llvm/ 中完成 typed lowering，复用现有 specialization、liveness、TypeOwner 与失败通道；测试仍在根 test。异步输入、输出和 fault 通过静态描述符的 typed move/drop 转交，禁止解释器/Any/Value、错误 String 化、raw padding copy 或同步线程冒充 async。LLVM 自有协程帧使用配对 runtime allocator，借用的 poll Context 只在当次 resume 有效，final suspend 后绝不再次 resume。 + +此段不启动数据库/CMS/HTTP 服务，不做全量或压力测试，不安装全局工具；AsyncStream、网络/系统族、Model/Port/Adapter/API/Job 入口及正式跨平台 packs 仍需后续接入，不能以 Task/Group/Channel 的定向验收代替完整应用后端。 + +### 2026-10-01 最终复验 + +- 显式离线构建 runtime-only archive（`target/native-runtime-abi/debug/libdever_backend_bridge.a`），含 File/Stream、异步与同步 time_sleep ABI；与 embedded LLVM/LLD 分离。测试没有递归 cargo 构建、宿主语言回退、联网或数据库连接。 +- `cargo test --offline --locked -p dever-backend-bridge --features embedded,runtime-abi --test llvm_async_source --test llvm_resource_native --test llvm_async_native --test llvm_result_native --test llvm_managed_native --test llvm_native --test native_runtime_abi --test native_runtime_link -- --include-ignored`：**70/70**。async_source 21/21（17 actual archive 执行、六目标 object、源码 Checker/两个原规则拒绝），resource 10/10、async ABI 2/2、旧 Result/managed/kernel/ABI/link 37/37。日志 `target/closure-resources-async-final-native.log`。 +- 源码覆盖大记录及 Text/List/Map/多输出跨 Pending、异步 each/reduce 静态 handler、同步/异步 Task、Group 容量和 Unit child、Channel 背压/Nullable/close/End、stop 与嵌套后代释放、blocking/parallel 的 owned 值和业务错误、capture/refail、默认业务错误的精确 origin/caller、负数 sleep 原始消息、数字 code2 与 ABI code3 原始 Buffer 不可捕获。受管运行 runner 用并发安全的原子 allocator ledger，一次预热后 64 次重复核验；不是 RSS/体积/性能测试。 +- 资源验证覆盖同次第二次 create_new 必须失败、不同内容不能覆盖、File 别名 close、read EOF、Stream 共享进度/提前关闭、reduce_until 未访问行、一次末尾 Failed 后 End、each 的 owned 业务错误、运行故障清理及 root File owner 移交。C 层无效输出在 I/O/cursor advance 前拒绝。 +- `llvm_lowering` **7/7**，日志 `target/closure-resources-async-ir.log`；原 runtime Stream 定向 **3/3**、File 定向 **1/1**，分别见 `target/closure-resource-reference-stream.log`、`target/closure-resource-reference-file.log`。 +- core/runtime/CLI/bridge 的 offline locked check（runtime api/external/postgres，bridge embedded/runtime-abi）通过，日志 `target/closure-resources-async-final-check.log`。postgres 仅编译，没有真实 PostgreSQL。 +- 私有工具对 core lib 与 bridge lib/上述 8 targets 的严格 `Clippy --no-deps -D warnings` 通过，日志 `target/closure-resources-async-core-clippy.log`、`target/closure-resources-async-bridge-clippy.log`。改动和相关 LLVM/ffi/根测试的 rustfmt check 通过，日志 `target/closure-resources-async-final-fmt.log`；没有安装全局工具，不代表全仓库质量门。 +- 独立只读检查已完成：发现并修正内部 Unit/Outputs ABI transferability、同步 sleep 符号、可达 async 临时值的 Send 验证；fresh review 未发现 typed fault、initial/pending/final cleanup 的剩余阻塞。主线程用实际最终日志确认关键证据。 +- 已同步 README/IMPLEMENTATION/LANGUAGE 与 compiler/toolchain canonical specs；父任务仍 `in_progress`，未 commit、未归档。磁盘约 4.9 GiB 可用，此段没有清理用户数据或旧目录。 + +质量复盘：内部 Unit/Outputs 的源码属性不可直接充当 ABI row 属性（跨层合同/隐含假设）；修复归属在唯一 descriptor 生成器，不改 Checker，真实 sync Task/多输出/blocking 用例防回归。SyncFunction.invoke 只借用 runtime-owned input，避免双 drop。测试判定另有两处缺口：create_new 相同内容掩盖错误覆盖，以及以 RuntimeAbi 字符串 oracle 验证数字除零；已改互斥结果/不同内容、精确 code2/origin，并另验 code3 原始 Buffer。规范已记录字段转移、Scope/Send、状态映射和 exact oracle 要求；没有新增重复框架/错误系统或伪造成功分支。 + +后续仍需 AsyncStream/网络及其余系统 intrinsic（含 timeout/race）、Model/Port/Adapter/API/Job 的 LLVM 应用入口、默认 CLI 后端整合、正式六目标 runtime packs，以及父任务其余真实生态/发行/性能质量门。资源/协程这一段已验收，不以此标记整个阶段四或全部语言完成。 + +## 2026-10-01 下一段：AsyncStream、TCP 与 timeout/race + +用户已确认继续，沿用原 LLVM 后端阶段；本段不改变语法、Checker、默认 Rust bootstrap CLI 或配置协议。 + +- [x] AsyncStream 的 List/Bytes/Channel 来源、ticks、共享游标/close、静态 handler 遍历及当前合同允许的并行分派。 +- [x] Socket/Listener typed owner 和 TCP connect/connect_timeout/listen/accept/port/read/write/timeout/close/chunks/connections；复用 runtime net 的方向锁、期限及取消合同。 +- [x] timeout/race 消费同签名 Task，超时/胜出结果返回前排空被取消的工作；fallback 只在真正超时后运行，业务和 runtime fault 保持原始 typed 身份与来源。 +- [x] checked-source → LLVM object → runtime-only archive 的实际运行、六目标 object、旧定向回归、offline check、私有 fmt/Clippy 与独立合同检查。 + +改动归属在 core llvm/ 和 bridge 私有 ffi.rs/canonical runtime.h；永久用例仍在根 test。复用现有协程、typed move/drop、recoverable choice/fault 和单 runtime Scope，不增加第二套异步引擎、动态值或 fallback。验证只使用自有短生命周期 loopback、系统分配端口和有界等待，不接触真实服务或外部网络。HTTP/TLS/WebSocket、其余系统族及 Model/Port/Adapter/API/Job 应用入口不计为此段已完成；正式 packs、共享编译和最终质量门仍按父任务推进。 + +### 本段最终复验 + +- 主线程显式 offline/locked 构建 runtime-only staticlib，最终 archive 晚于最终 ffi 修改，不含 embedded LLVM SDK；日志 `target/closure-stream-network-final-archive.log`。 +- 9 个相关 LLVM/ABI targets 首次最终集合 **92/92**（旧 70 + 新 22），日志 `target/closure-stream-network-final-native.log`。随后追加「并行 handler 故障打断空闲 Channel producer，同时只关闭流别名而不关闭独立 Channel」精确回归；最新新 target **23/23**（21 个实际 archive 执行项、Checker、六目标 object），日志 `target/closure-stream-network-final-new-native.log`。当前独立覆盖共 **93 项通过**，不是将重复执行相加。 +- 新源码执行覆盖 nullable 流、Channel 背压/挂起 handler、ticks 和提前关闭别名、四类序列及同步/挂起 handler 的合法并行路径、同步 File Stream worker 的 owned 业务错误、timeout/race 受管记录及多输出/null/Unit、真正超时的 async fallback、业务错误不走 fallback、数值 code2 的原始来源、ticks/timeout 非法期限的原始 code3 文案。TCP 自有 loopback 验收 short read/完整 chunks/EOF/共享关闭、原始输入错误、chunks/connections 一次末尾 Failed 后 End；超时返回后对端 EOF 证明 pending 后代 Socket 已释放。 +- 沿用同一 atomic allocator ledger，实际 native 每个程序预热一次后重复 64 次，输出/fault 与资源释放平衡；这不是体积/RSS/延迟/64或128 MiB 性能报告。六目标只证明对象生成，本机 Linux x86_64 证明实际运行，没有六 OS 执行验收。 +- 原 runtime/IR 定向复验：`llvm_lowering` **7/7**、`async_network` **9/9**、`async_composition` **6/6**（明确过滤原 Rust native 慢链路）；日志 `target/closure-stream-network-reference-ir.log`。覆盖原背压/空闲 producer 故障、读超时后继续使用、部分写取消先关闭、阻塞后代排空和规则拒绝；不启动现有服务。 +- 异步 poll 的非法输入状态最小修正为既有 canonical `4`，不再与 typed fault `2` 冲突；C driver 新断言实际 archive 执行 **2/2**，日志 `target/closure-stream-network-final-async-abi.log`，严格 C11 语法检查通过。 +- core/runtime/CLI/bridge offline locked check（runtime api/external/postgres、bridge embedded/runtime-abi）通过，日志 `target/closure-stream-network-final-check.log`；postgres 仅编译。core lib 与 bridge lib/9 targets 私有严格 `Clippy --no-deps -D warnings` 通过，日志 `target/closure-stream-network-core-clippy.log`、`target/closure-stream-network-bridge-clippy.log`；追加新 fixture 后再次 Clippy 通过，日志 `target/closure-stream-network-final-new-clippy.log`。相关 LLVM/ffi/新根测试的 rustfmt check 通过，日志 `target/closure-stream-network-final-fmt.log`。没有安装全局工具。 +- 按 Trellis 2.2 进行双向非本人代码审查：core 作者独立对照 bridge ABI，bridge 作者独立对照 core 与原 native/runtime；新 reviewer 线程额度不足时复用已完成线程，没有启动额外进程。最终未发现本段剩余阻断；主线程独立执行关键验收，不以作者 compile 或审查报告代替运行证据。 +- 已同步 README/IMPLEMENTATION/LANGUAGE 和现有 compiler/toolchain/directory specs。父任务仍 `in_progress`，六个闭环的未完成项不勾选；无全量测试、真实 PostgreSQL、CMS 服务、压力测试、正式签名发布、全局安装、commit 或归档。磁盘约 4.7 GiB 可用，未清理用户数据。 + +首轮失败均明确保留为未通过的迭代证据：fixture 的空比较缺类型上下文、named Outputs 跨 suspension、保留词 `other` 当局部名、Result 的 timer failure 集合不匹配；均按现有语言合同改 fixture，没有改 Checker 或降级错误类型。桥接中途一次 Result error 类型推断失败已加具体类型，最终 check/Clippy 通过。质量收尾还保证 race null 创建路径保留 caller owners、遍历关闭责任在 state/context 求值后建立，以及 ParallelHandler 只校验布局而不额外分配临时三份存储。 + +下一段仍需 HTTP/TLS/WebSocket 与其余系统 intrinsic，再接 Model/Port/Adapter/API/Job 应用入口、默认 CLI 后端与正式六目标 runtime packs；不得将这 93 项内核/ABI 定向证据记为整个阶段四或完整语言交付。 + +## 2026-10-01 LLVM HTTP/TLS/WebSocket 与系统 intrinsic（本段已验收） + +沿用当前 checked HIR、静态 specialization、强类型 fault/owner/coroutine 描述符及既有 runtime；不改语言语法、配置或默认 CLI 后端，不把本段内核验收计为完整应用/正式目标包验收。 + +- [x] 补 HTTP 单次/池化/流式请求与静态 server handler 的 typed ABI/lowering，复用 Hyper、原 HTTP/2 和连接/任务 Scope。 +- [x] 补 TLS client/server、live reply/SSE/WebSocket 与对应资源 retain/drop，保持背压、原始可恢复错误、取消和有界排空。 +- [x] 补剩余非应用系统 intrinsic（时间、进程参数、日志/stdout、UUID/Secret/crypto），保持敏感类型不可观察与既有阻塞工作边界；鉴权/上传/Job/Model 归属后续应用 lowering，不擅自合并。 +- [x] 根目录 checked-source→LLVM→真实 runtime archive 定向执行、无网络 Checker/六目标对象生成及必要旧覆盖复验；网络仅使用自有临时文件/有界 loopback peer，TLS 复用根目录 fixture。 +- [x] offline locked check、相关 fmt/Clippy、独立边界复核，并更新现有 canonical 合同与本记录。记录精确已验收范围和剩余项，父任务保持 in_progress。 + +实现所有权：bridge 工作线仅修改私有 `ffi.rs`、canonical `runtime.h` 及必要 runtime feature 接线；core 工作线仅修改 `llvm.rs`/`llvm/**`。主线程负责根测试、target 注册、验收和合同/任务记录；不互相覆盖，不运行全量或现有服务。 + +### 实现与复用 + +- 新 `llvm/http.rs`、`http_values.rs`、`http_handlers.rs` 复用原 Hyper/rustls/tungstenite 与 Scope，使用固定命名字段、opaque owner、静态 typed sync/async handler/context；不经过 JSON/Value。响应 owned 字段先注册再做可能失败的 header 转换。 +- `llvm/system.rs` 接日历/时钟、argv、stdout/log、Uuid/Secret/crypto，密码计算保留 blocking。Uuid 使用真实 UUID owner；敏感类型不新增 Render。`llvm/render.rs` 只在请求边界渲染可观察的终端 fault，保留原有日志+500/中止回复+服务器继续处理的行为。 +- 协议 async poll 在 I/O 前检查 output/presence/align；非法 metadata 返回 4，不改 sentinels 或操作状态，修正后可重试。原非协议 async 合同不扩大。 +- 纯日志探针定位到 runtime Logger 后，主线程明确扩展 bridge 的 owner 范围到 `log.rs`：单一预分配 128 槽队列配合 Mutex/Condvar，保留原过滤、截断、JSON、FIFO、低级别丢弃/高级别背压；串行固定 ACK，异常退出释放积压并唤醒等待者。没有新依赖或第二套 logger/队列。 +- 根 test 保留源码执行、C ABI 与队列回归；实际 native helper 共用原严格 allocation ledger，只增加明确的昂贵用例次数/期限与文件输出捕获，不建另一个执行器。 + +### 最终证据 + +- 显式 offline/locked 重建 runtime-only archive，最终 `2026-10-01 03:44:10 UTC` 晚于最终 ffi 与 log 修改，不含 embedded LLVM/C++ SDK；`target/closure-http-system-final-archive.log`。 +- 新 `llvm_http_system_source` 唯一覆盖 **25/25**：23 项 Linux checked-source→LLVM→实际 archive 执行；21 份源码 Checker；21 份源码×六目标 object。末批输出 fixture 的 Map 分隔修正后单独复验，未重复相加：`closure-http-system-native-final.log` 的 22 项已通过，修正后的三个检查见 `closure-http-system-output-final.log`、`closure-http-system-checker-final.log`、`closure-http-system-objects-final.log`。六目标 object 不等于六 OS 运行。 +- 实际执行覆盖独立 chunked wire/repeated headers、H1/H2/TLS 池化与流式上下行、live Reply/SSE、WS/WSS 文本/二进制/控制帧与共享关闭、owned context 最初挂起前克隆、owned handler 失败后 500 再正常处理、原始诊断、时间/UUID/Secret/密码/argv、stdout 与四级精确 JSON 日志。一般 ledger 每进程 warm 1+64 次；密码是 warm 1+2 次，45 秒边界。 +- C/header/archive **4/4**，含非法协议目的槽→原样保留→有效重试→完成后拒绝 repoll，以及纯 log flush 的严格 ledger；`closure-http-system-c-abi-final.log`。C 探针不假造成功的 WsReceive row，真实消息路径由源码执行覆盖。另同一纯 flush 探针在 **40 个独立进程**中全部通过。 +- 原 8 个相关资源/异步/集合/Result/ABI targets **91/91**；`closure-http-system-old-regression.log`。root IR **8/8**；`closure-http-system-ir-final.log`。原 Rust bootstrap 的 source logging **1/1**；`closure-http-system-bootstrap-log.log`。Logger 的 FIFO/dropcount、Warn/Error 背压、并发 flush、writer panic 四项 **4/4**；`closure-http-system-log-unit.log`。 +- core/runtime/CLI/bridge offline locked check（runtime api/external/postgres，bridge embedded/runtime-abi）通过；`closure-http-system-check-final.log`，postgres 仅编译。core lib、bridge lib/10 定向 targets 与 root IR 的私有严格 `Clippy --no-deps -D warnings` 通过；`closure-http-system-core-clippy.log`、`closure-http-system-bridge-clippy.log`、`closure-http-system-ir-clippy.log`。实际 `log.rs` 与根队列测试的独立 strict Clippy 通过；`closure-http-system-log-clippy.log`。相关文件私有 rustfmt check 与 C11 strict syntax 通过；`closure-http-system-fmt-final.log`。没有安装全局工具；不是全仓库 Clippy 门。 + +### 根因与防复发 + +首轮 native 为 15 通过、6 项 ledger 失败（+2/+1），返回值 oracle 没失败。只调用 flush 的 C 探针同样出现 +2，证明 owner 在 Logger，不能据此断言 LLVM/HTTP 泄漏。固定 ACK 通道仍失败；分配栈最终确认是 std mpsc receiver 首次阻塞的 48B Context 与 96B waiter Vec 持久初始化，发生在 ACK 后。最终修正整个单一队列等待 owner，而不是增加 warm、sleep 或放宽 ledger。此前关于「仅临时 ACK 析构 race 就解释全部失败」的初判过强，已纠正。纯探针、全部协议源码和旧回归共同复验通过；独立检查确认无丢唤醒、串 ACK、队列 owner 遗漏或新阻断。约束已写入 logging/toolchain/compiler canonical specs。 + +另修正 C async poll 的边界检查时机：先 poll 后检查目的槽会推进有副作用的操作，必须先验证 metadata，根 C fixture 固化可重试合同。测试维护同步迁移原「stdout 尚未支持」断言为真实支持检查，保留合法 App 内鉴权 intrinsic 的源码定位拒绝；没有放宽 Checker。Clippy 曾命中失败的 rustc 探测缓存，保留到 `target/closure-http-system-rustc-probe-failed.json` 后只重建该元数据，最终通过。 + +README/LANGUAGE/IMPLEMENTATION 与 backend compiler/toolchain/directory/logging/index 已同步。未运行全量、数据库连接、CMS 服务、新性能/RSS/64–128 MiB 压测或跨 OS 实际执行；没有 commit/归档。下一段继续 Model/Port/Adapter/API/CMD/Job 与鉴权/上传等 LLVM 应用语义，再接默认 CLI 和正式六目标 runtime packs；父任务其余生态/发行/最终质量门未因此变为完成。 + +## 2026-10-01 LLVM CMD/App/Dever Adapter 应用执行链（本批已验收） + +继续原生产化任务，不重定语言语法，不切换默认 CLI。应用合同不能只靠解除 kernel 拒绝：原 Rust 生成入口还拥有配置、事务、权限、租户、服务与清理。 + +本批先交付独立 `llvm::emit_application`：真实 CMD 参数 → shared checked wire schema → App → 配置选中的 Dever Port/Adapter → typed 输出 → 固定 JSON envelope。Wire 的 JSON 只留在输入/配置/输出边界，复用 runtime `wire::parse/Node/Encoder`；业务仍是具体 LLVM 类型和静态函数。选中 Setting 在应用 entry 生命周期内初始化并清理；未选中 Adapter 不要求其配置。同步/挂起 CMD 都必须保持原 source fault、typed business failure 和所有权。 + +主线程负责根测试、实际 archive 链接、文档和串行定向验证；原 core/bridge implement 线程分持 LLVM lowerer 和 canonical C ABI。普通 `emit_kernel` 不替代生成应用根。实际执行使用测试自有临时目录、明确离线 runtime archive 和严格 allocation ledger；不启动数据库、CMS、外部 Worker 或服务。 + +已完成： + +- [x] 独立 `llvm::emit_application` 与 compiler-owned CMD 根，标准 `dever_application_entry`/output/fault release；内部 run 不公开。完整 CMD 名与单个 JSON 对象使用原参数合同，同步/推导挂起 App 共用已有 Scope 和 typed fault。 +- [x] `llvm/wire.rs` 消费原 checked schema DAG，具体 scalar/record/List/nullable codec 复用 runtime Node/Encoder，不把 JSON/Value 引入业务。CMD 顶层复用原 `Inputs`，保留缺失/未知字段文案、重复键检测和字段错误优先级；嵌套 wire policy 不混用。 +- [x] `llvm/ports.rs` 的 closed Dever Adapter 选择与 selected Setting 初始化。配置仍只读 `config/setting.json`，未选中的 Adapter 不要求配置，单个无 Setting 的 Adapter 不要求配置文件。选中槽为模块私有，非阻塞原子 gate 拒绝重入/重叠，根 Scope 排空后清理/复位并释放 gate；部分初始化失败走同一清理。 +- [x] 私有 canonical wire/CMD/settings ABI，回调借用不越过同步调用，输出 metadata 校验在解析/取字段/选择之前。成功解码后若 trailing CMD input 或 Buffer 不合法，桥接 drop 完整 owner；失败回调由编译后的部分初始化 guard 清理。复用原 runtime wire/config owner，`runtime-abi` 只增加 wire feature，不依赖 LLVM/C++ SDK。 +- [x] 新应用根拒绝尚未接通的 Model/API/REST/Job/auth/permission/tenant 和 external Adapter，不按某个 CMD 当前未调用就跳过启动合同；普通 kernel 仍拒绝应用入口。 +- [x] 公共 native runner 下移到根测试 support，kernel/application 共用实际 archive、超时、owned process 和原 allocation ledger；没有平行测试执行器或放宽既有 fault oracle。 + +实际验收： + +- runtime-only archive 显式 offline/locked 重建通过，最终文件时间 `2026-10-01 05:12:48 UTC`，晚于 ffi/header 修改；不含 embedded SDK。core 后续 failure metadata 修正只改变 generated object，不要求重新构建 runtime archive。 +- 新 `llvm_application_source` **19/19**：15 项 Linux x86_64 checked-source→LLVM object→实际 archive 执行，另含原 Checker、六目标 object、kernel/app 边界及 source-located unsupported metadata 检查。实际执行保持 warm 1 + 64 次、严格原分配平衡与每次精确 JSON/fault oracle；覆盖 owned Unicode/大 Int/全部受支持 wire scalar/原始 Json、可空顶层/列表、异步输入、业务失败/result/数值错误、配置选择、部分 Secret/多 Adapter 初始化失败及 Task 读取 selected Setting。日志 `target/closure-llvm-application-source-final.log`。六目标 object 不是六 OS 运行。 +- 旧受影响回归 **43/43**：managed 10、Result 12、async source 21；runtime ABI **4/4**，C/LLVM 实际链接与 ledger **5/5**（含 wire/CMD callback/非法槽/成功后 finish 失败清理），日志 `target/closure-llvm-application-regressions.log`。IR **8/8**，日志 `target/closure-llvm-application-lowering.log`；没有重复相加早期运行。 +- core/runtime/CLI/bridge offline locked check（runtime api/external/postgres、bridge embedded/runtime-abi）通过，日志 `target/closure-llvm-application-check.log`；PostgreSQL 仅编译，未连接。 +- core lib 与 bridge lib/本批六个相关 test targets 私有 `Clippy --no-deps -D warnings` 通过，日志 `target/closure-llvm-application-core-clippy.log`、`target/closure-llvm-application-bridge-clippy.log`。相关文件私有 rustfmt check 通过，日志 `target/closure-llvm-application-fmt.log`；两个 C fixture 的 C11 `-Wall -Wextra -Werror` 检查通过。没有全局工具安装。 +- 本批独立 ABI/所有权交叉检查及最后的 failure metadata/runner 有界复核无阻断。主要风险分别由实际 fault、partial owner、重复 gate 复位和严格 ledger 用例覆盖；不把审查声明当作执行证据。 + +发现并修复的真实缺陷：扩展到 19 项时首轮为 18 通过、1 项 lowering panic。无配置同步 Adapter 声明了 `SleepResult`，实现并未实际构造它，但 fault 布局/drop 仍消费 `specialize::failures`；旧统一收集点遗漏仅声明的 failure nominal/owned 字段。现于 `validate_function`/`Module::new` 对同一 reachable failure 集做校验及递归类型收集，不注册无关类型、不加 fallback。单用例与最终 19 项、43 项旧回归均通过，canonical compiler spec 已记录此 invariant。最初关于「CMD 输入行缺描述符」的猜测已纠正。此前源 fixture 的字段换行、Int `//` 与 Port fails 合同，以及 C fixture 的 mandatory equal callback 修正是测试 fixture 修正，不是放宽语言/ABI。 + +README/LANGUAGE/IMPLEMENTATION 与 backend compiler/toolchain/directory specs 已同步。没有全量测试、数据库/CMS 服务、真实外部 Worker、新性能/RSS/64–128 MiB 压测或跨 OS 执行；没有 commit/归档。父任务仍为 `in_progress`。 + +完整 LLVM 应用阶段仍按依赖收口：① 本批入口/wire/CMD/Dever Adapter；② Model/双数据库 SQL plan/事务/DbError/Related/RowStream；③ API/auth/tenant/component/permission/REST/Upload；④ Job 与 compiler-owned Test；⑤ external Adapter 与准备好的资源集成。后续未支持族继续 source-located 硬拒绝，不用 Rust fallback、成功 stub 或放宽 checker。只有这些全部通过，才进入默认 CLI 和正式六目标 packs。 + +## 2026-10-01 LLVM Model/ORM 与事务（实现与定向验收通过） + +承接已验收的 CMD 应用根，本批实现静态 Model 操作、SQLite/PostgreSQL SQL/bind/row codecs、nested transaction 与精确 DbError、Related 批量加载、typed SQL、RowStream，以及 schema/migration/Seed 启动。沿用 checked QueryPlan 和原 driver，不修改语言语法、业务 CMS、默认 CLI，API/auth/tenant/Job/external 仍为后续边界。 + +代码归属：core implement 负责 `llvm/**` 和必要纯 SQL/schema helper 共享提取；bridge implement 负责私有 `ffi.rs`/`runtime.h`、独立数据库 ABI profiles，以及 runtime 数据库生命周期薄适配；主线程负责根测试、所有 Cargo/实际验收、canonical specs 和本记录。两个 implement 不运行 Cargo/真实数据库，不回滚对方或既有修改。 + +已经确认的结构风险：原 `config::bootstrap_scoped`、SQLite/PostgreSQL registry 都是 OnceLock，shutdown 后不能重复初始化或复用已关闭池。因此新 LLVM 根需要 owned `database::Session`，原 Rust native 仍保留现有 bootstrap；两个路径共享实际 driver 构造。事务普通/顺序调用继承隐藏 context，并发边界显式 detached；迁移和 SQL 仍使用原 owner。Session 在根 Scope 排空后、同一 runtime 销毁前关闭；必要 `task::run_entry_with_typed_cleanup` 只扩展这个生命周期，原入口委托它保持原语义。禁止新建池栈/SQL builder/反射业务 Value、成功 stub、tenant-to-global fallback，禁止新增环境变量配置。 + +验收先离线 Checker/objects/最小新根 target，再实际自有 SQLite 与严格所有权检查；PostgreSQL 只使用显式隔离 fixture，任何临时服务启动先说明。共享 helper/driver/task 改动必须复验受影响的原 Rust native/ORM/async 合同，不能只测新实现。未通过前不标记本批或父任务完成。 + +本批实现： + +- checked Model 的 CRUD、bulk/upsert、offset/cursor、nullable/scalar/private/choice codecs、typed SQL 和 bounded RowStream 均由 LLVM 降级到真实 runtime archive；ToOne nullable join 与 ToMany 按 998 个父键分批，保留每父条数限制。schema/migration/data migration/Seed 按实际数据库分组，先建全表再建外键。SQL/DDL 和 SQLite/PostgreSQL driver 复用原 owner，不复制 ORM 引擎、不通过 Rust fallback 执行。 +- 根 `database::Session` 持有本次 Settings/pools，根 Scope 排空后在同一 runtime 关闭，支持重复进入；没有给全局 OnceLock 重绑新池。数据库配置仍只来自程序旁 `config/setting.json`。本切片只支持普通非租户 Model，真实 tenant 配置硬拒绝，不回落平台数据库。 +- 顺序调用/handler 继承隐藏事务 context，Task/并发边界 detached；嵌套 transaction 共享 owner，不引入 savepoint。CMD 顺序为输入解码→开启事务→App→输出编码→commit→stdout；数值、业务、数据库或输出编码失败均 rollback，commit 前已初始化的输出有明确 typed drop。清理失败只追加 cause，不覆盖原故障。 +- `runtime-abi`、`runtime-sqlite`、`runtime-postgres` profiles 独立选择实际 driver，canonical C header 中的 DbError/Row/Rows/Related/RowStream ABI 只使用固定持久化边界值，业务字段保持具体类型。非法输出槽在 I/O、pull/next 或 clone 前拒绝。 + +实际验收（未将重复复验相加）： + +- 最终双 driver runtime-only archive 重建通过，时间 `2026-10-01 08:56:03 UTC`，晚于最终 ffi/运行时机械修正；日志 `target/closure-llvm-database-archive-final.log`。 +- 最终 `llvm_database_source` **23/23**：19 项自有 SQLite 原生执行、1 项全部 fixture Checker、1 项六目标 object，以及共享 PostgreSQL fixture 的 2 项纯校验。每个原生 probe 为 checked source→LLVM object→实际 archive，warm 1 + 64 次，清空子进程环境，检查精确 stdout/fault 与原分配平衡；覆盖两版迁移、nullable/所有存储 scalar、cursor、ToOne/ToMany、stream full/early-close/cancel、单连接嵌套/顺序事务和 constraint/numeric/encoding rollback。日志 `target/closure-llvm-database-sqlite-final.log`；六目标 object 不等于六 OS 运行。 +- 真实自有 PostgreSQL **3/3 组**（共 19 个逻辑场景，迁移包含两版）：CRUD/codec/query/relations/stream/transaction/schema/Seed，以及 encoding 和 uncaught constraint 回滚后重试/读取。日志 `target/closure-llvm-database-postgres-final.log`，运行 271.58 秒；这是依赖升级与最后 MSRV 机械修正前的实际验收，不声称之后又启动服务。仅连接自有 loopback fixture、随机 schema，未接触部署数据库。检查残留 schema 为 0 后停止服务，临时 `config/setting.json` 删除恢复原先不存在状态；PG fixture 移入可恢复目录 `/root/.local/share/Trash/files/dever-llvm-pg-ZQQmWO`，日志保留 `target/closure-llvm-database-postgres-server.log`。 +- 依赖修复后受影响旧应用/async/managed/Result 回归 **62/62**（19 + 21 + 10 + 12），日志 `target/closure-llvm-database-native-regressions.log`；最后 MSRV/ffi 机械修正后重跑 task-group 取消及后代 frame 释放关键用例 **1/1**，日志 `target/closure-llvm-database-async-critical-final.log`。 +- 最终原 runtime 定向回归 **24/24**：async composition 5、ORM migration 7、Seed 3、entry cleanup 3、SQLite runtime 6，日志 `target/closure-llvm-database-runtime-regressions-final.log`。覆盖 deferred constraint commit rollback、pool 未知状态丢弃、全部四种 decoder 的非法 UTF-8 列索引及后续连接替换。 +- 最终 C ABI probe **1/1**，十种 DbError/borrowed Text cause、Related Loaded(null)/Unloaded、非法/错位槽和不消费原 fault 的 rollback 校验通过；日志 `target/closure-llvm-database-c-abi-final.log`。C11 `-Wall -Wextra -Werror` 语法检查通过，日志 `target/closure-llvm-database-c-format-final.log`。 +- base ABI / SQLite / PostgreSQL / 双 driver 四 profile offline locked lib check 通过；core/runtime/CLI libs/bins、bridge lib/本批根 tests 与 cleanup/SQLite runtime tests 的严格 Clippy `-D warnings` 全通过，日志 `target/closure-llvm-database-profile-*.log`、`target/closure-llvm-database-clippy-{product,bridge,tests}-final.log`。本批 LLVM/bridge/database/task/config/driver/根测试相关文件私有 rustfmt check 通过，日志 `target/closure-llvm-database-rustfmt-final.log`；未声称原 `native/orm.rs` 等历史基线的全局格式门通过。工具私有运行，不安装/替换全局命令。 + +真实故障与根因修复：首轮 SQLite 21 项中 stream 用例超时，之后单用例重复暴露。两个 blocking-worker 栈定位到旧 bundled SQLite 3.51.1 的 WAL close/replacement-open VFS 锁反转（global→inode / inode→global），不是 Dever Scope 未清理。已升级既有精确依赖 `deadpool-sqlite` 0.13.0→0.14.0，锁定 `rusqlite` 0.40.2 / `libsqlite3-sys` 0.38.2 / `hashlink` 0.12.2，使用 bundled SQLite 3.53.2；没有改 registry 缓存、私有 SQLite fork、超时阈值或未知连接复用规则。四条 SQLite decoder 适配真实列索引，新增非法 UTF-8 回归。依赖要求作者 workspace 最低 Rust **1.95**（本机实际使用 1.98.0，未单独运行 1.95 toolchain）；对应新 MSRV 触发的 let-chain/is_multiple_of/as_chunks 写法仅机械修正，独立复核短路/资源 drain/对齐/偶数长度 invariant 后严格 lint 与上述关键复验通过。临时栈探针和 thread-trace 源已移除，原 `managed-driver.c` 恢复，诊断日志保留。 + +另外纠正了测试 oracle：cursor `next = null` 传作 after 表示重新开始，不表示终点；PostgreSQL constraint 会使当前事务 aborted，不能沿用 SQLite 捕获 constraint 后继续 SQL 的测试预期。现 PostgreSQL 捕获 NotFound 独立验证错误身份，uncaught constraint 则验证整笔回滚和下一次入口读取。没有放宽 checker 或改变既有 driver 事务语义。 + +README/LANGUAGE/IMPLEMENTATION 与 compiler/toolchain/directory/database canonical specs 已同步，数据库 spec 保存锁顺序回归及清理 invariant。未运行全量 workspace、CMS/HTTP 服务、性能/RSS/64–128 MiB、其他 OS 或正式 packs。ToMany 的 998 分批静态路径已复核，实际 fixture 只覆盖少量父键；新 LLVM commit/Session-close 人为失败注入未做，原 Seed deferred-commit 回归不冒充新 ABI 注入。完整 log 配置、API/REST/auth/tenant/component/permission/Upload、Job/Test、external Adapter 与默认 CLI/正式发行仍在后续闭环，本批不重新设计这些已约定合同。父任务保持 `in_progress`，没有 commit 或归档。 + +## 2026-10-01 LLVM API/REST 与可信请求上下文（本切片已完成） + +用户在上一切片验收后批准继续本阶段。范围为已有 API/REST、站点目录映射、可信身份、自动权限、database-only 租户/组件状态、Cookie/Header/Upload 与 log 配置的 LLVM 应用接线；不改语法、权限 key 或 CMS 业务，不提前切换默认 CLI，也不加入 Job/Test/external/正式 packs。 + +最小行为差距：原 native 已生成完整 HTTP 路由与认证/授权链，LLVM 只生成 CMD root 并拒绝这些元数据。compiler 需统一所有 API/CMD/REST/verify/字段绑定执行根的 reachability、typed input/output/failure layouts；runtime 需让已存在的 config/auth/store/tenant/API/log owner 服务 invocation-owned Session，而不是重绑 OnceLock 或实现第二套 HTTP/RBAC/数据库。 + +分工：`llvm_api_core` 只改 core `llvm/**` 与确需复用的 native 纯 helper；`llvm_api_runtime` 只改 runtime 及 bridge ffi/header/features；主线程拥有根 tests、文档、所有串行 Cargo/实际验收。代理按项目 `.codex/agents/trellis-implement.toml` 的可用 `gpt-6-astra/high` 配置运行，因本环境内建角色的 `gpt-6.1-sol` 无法启动，显式上下文加载作为 native injection 的 fallback。双方不回滚彼此或既有修改。 + +实现约束：静态 route descriptor + 具体输入 decoder + compiled typed wrapper；runtime 只复用既有 request/auth/tenant scope、dispatch、envelope/metadata，App 无动态业务 Value。verify 使用固定 Claims/Identity pack/unpack，business failure 按精确已编译身份映射。REST 复用原 Model SQL/codecs；POST/PUT/DELETE 仍输入解码后开始事务，输出编码成功后提交。每 root 只加载一次 Settings；身份/租户/响应 metadata 的并发传播按原 caller 合同，不能把事务 detached 规则泛化到全部隐式 context。部分启动失败和最终清理保留原 fault，根任务与资源在同一 runtime 排空。 + +主要风险为：route-only/mixed root 选择、REST 字段绑定漏执行根、global settings/tenant OnceLock 串 owner、请求间身份或响应 metadata 串用、权限目录/多角色/跨站及禁用组件绕过、跨租户 fallback、Upload affine owner 取消泄漏，以及 shutdown 时先销毁 runtime。首轮先以现有 native 合同做 oracle，之后静态检查与本机 LLVM 实际对象/owned loopback 验收；服务或数据库启动在执行前说明，不使用部署服务。 + +资源准备：改动前已备份 `crates/`、`library/`、Cargo manifest/lock 到 `/tmp/dever-llvm-api-backup-AC4QeV`。磁盘约 430 MiB 不足后续构建，已核对无 Cargo/rustc 进程后仅删除 `target/debug/deps/` 内 9 个旧 bridge 静态 archive（旧 profile/hash）；保留最后两版及 `target/native-runtime-abi/debug/libdever_backend_bridge.a`，后者摘要 `994ed15beb13da2856723ce73cc988bd688e931cd7857dfae8baca7111652703` 未改。可重建缓存回收约 0.9 GiB，剩余约 1.3 GiB；未删除源码、数据库、日志、SDK 或配置。 + +本批实现与复用边界： + +- `llvm/api.rs`、`api_context.rs`、`rest.rs` 和 `application_commands.rs` 接通 HTTP/CMD 混合根、REST 字段来源与 owner 过滤、Claims/Identity、Cookie/Header/Upload 及管理命令。仍由 checked HIR 生成具体输入、输出与业务失败布局，API/CMD 不经过 Rust fallback，也不引入动态业务 Value。SQL assignments、tenant fingerprint 与 wire query kind 从既有 owner 共享，原 Rust 后端继续使用同一合同。 +- runtime `application::Session` 统一持有本次配置、生命周期、平台库、tenant manager 和迁移池；Settings 每根加载一次，log-only CMD 也消费配置并在返回前 flush。根任务排空后在同一 runtime 关闭资源；部分迁移与租户池先登记 owner、验证成功才 ready，取消不遗失 pool。 +- `runtime-api` profile 与 canonical C ABI 接通真实 request/auth/store/tenant/upload owner。public 接口允许匿名并保留原 Bearer 校验规则；受保护接口按原配置选择站点/provider，执行可信 verify→租户/组件检查→精确角色权限→输入及业务。权限键仍为 component.domain.site.action，REST 保留 read/create/replace/delete；缺失认证或权限元数据在 lowerer/ABI 拒绝。禁用组件对 Tenant Owner 同样生效，不回落控制库。 +- POST/PUT/DELETE 保持输入解码→事务→App/Model→输出编码→commit→响应 metadata。public/private 错误映射、隐藏 500、跨站/过期会话、多角色撤销、租户物理隔离均沿用原合同。配置只在 `config/setting.json`，无环境变量入口。 + +实际发现并修复的根因： + +- 私有 HTTP 首轮 SIGSEGV 来自调试构建的嵌套 future poll 栈过大，发生在 verify 已成功之后。只在 compiled handler 的共同边界 `Box::pin(self.call(...))`，保留原 auth/tenant/cancel scope；没有扩大线程栈、到处装箱或重写 Identity ABI。 +- borrowed receiver clone 原先保存了新句柄却返回原 SSA 值。现在从真正 clone guard 取值后再释放来源，REST search 复用同一路径；新增 Channel 记录跨 Task 返回的实际回归，覆盖 clone 与原 owner 销毁顺序。 +- REST DELETE 的 `null` 响应原先由两个 guard 同时持有。改成只交给响应 wrapper 的唯一 owner，跨 commit 保留并正确转交。 +- 同时纠正测试 oracle：嵌套输入仍需 payload 对象、owner 不可见行是 404、禁用角色不会由 save 重新启用、max_page_size=4 时必须显式 size、PostgreSQL TLS 使用 disabled 且必须配置 default。没有为测试放宽语言或数据库合同。 +- 原 Rust `rest_model` 的三个编译 fixture 仍对无 settings 的 HIR 生成应用，权限注册为空却调用组件辅助函数，导致 E0425；相关 product 逻辑与本批备份一致。现复用一个本地 fixture，向实际编译阶段补齐 config/setting.json、站点/provider 和名义身份，通过 `check_with_settings` 绑定。原 SQL/owner/字段来源断言保留；没有禁用组件授权或补匿名 fallback。 + +验收证据（不把重复复验累计为不同用例）： + +- `target/closure-llvm-api-final-native.log`：API 根 **14/14**，其中 6 项 Checker/六目标 objects/fixture 校验，8 项实际原生执行；HTTP、私有 REST、禁用组件、上传、混合 CMD、无认证 metadata、启动失败、log-only CMD 均检查 warm 1 + 64 次资源平衡。六目标 objects 不代表六 OS 实际运行。 +- `target/closure-llvm-api-postgres-native.log`：真实 PostgreSQL **1/1 组**，复用同一 private REST 场景，执行 3 次根；隔离的控制库与两个租户物理库覆盖可信身份、多角色/跨站权限、租户隔离、创建/更新/删除、字段绑定、owner 行过滤、并发身份、Cookie 与失败回滚。该验收在最后无行为变化的 Clippy 修正前执行，未声称修正后再次启动 PG。 +- `target/closure-llvm-api-upload-final.log`：最后 runtime-only archive 重建后，包含新增上传中途断开的用例 **1/1**，65 次根均确认文件实际开始写入，再断开连接并等待临时文件清零；正常存储与解码失败同时覆盖。 +- 原应用入口 **19/19**（`target/closure-llvm-api-owner-regressions.log`），最终异步 **22/22**、受管值 **10/10**（`target/closure-llvm-api-owner-regressions-final.log`）。首轮后 9 个异步用例在启动 cc/ELF 时遭遇 ENOENT，未进入业务断言；/bin/bash 同期也暂时无法启动。只读确认工具恢复后，22 项全量定向重跑通过，未更改全局工具或掩盖失败记录。 +- 原配置/请求上下文/鉴权/权限/声明 **36/36**，日志 `target/closure-llvm-api-runtime-regressions-final.log`；该日志保留三个旧 REST fixture 的失败。修正 fixture 后，REST **12/12**（含三条实际 Rust AOT 编译）、entry cleanup **3/3**、tenant database **3/3** 均通过，日志 `target/closure-llvm-api-runtime-rest-final.log`。共 54 项定向结果包含各 target 自带的 PostgreSQL fixture 纯校验，不代表这些 target 启动了 PG。 +- 最终双 driver/API archive 下，原 LLVM Model bulk/upsert 与单连接 RowStream 取消 **2/2**、canonical C header 实际链接运行 **1/1** 通过，日志 `target/closure-llvm-api-database-abi-final.log`。验证共享 SQL 及 API profile 没有破坏先前 Model 根和基础 ABI。 +- 最终 archive 重建通过：`target/closure-llvm-api-archive-final.log`；runtime-api base/SQLite/PostgreSQL/双 driver 的四 profile check 见 `target/closure-llvm-api-profiles.log`。本批文件 rustfmt、C11 `-Wall -Wextra -Werror` 与相对源码备份的 whitespace 检查通过，见 `target/closure-llvm-api-{rustfmt,c-header}-final.log`、`target/closure-llvm-api-diff-check.log`；历史 native 源码整体格式差异未全库重排。 +- 最终 core/runtime/CLI/bridge libs、三个 CLI bins、本批 API/async/REST tests 严格 Clippy `-D warnings` 通过，包含原 runtime external feature 的编译检查，见 `target/closure-llvm-api-clippy-final.log`。旧 REST diagnostic 的 `.err().expect()` 改为等价 `expect_err()` 后，9 项非 AOT 合同再验通过（`target/closure-llvm-api-rest-diagnostics-final.log`）；三个 AOT 不为此机械测试写法重复构建。新 fixture 块 rustfmt check 通过，保留文件其余历史格式。 + +资源清理：真实 PostgreSQL 验收结束后核对随机库已由共享 fixture 清理、无其他连接，只删除本批创建的 `dever_llvm_llvm_api` 基础库，保留此前 `dever_llvm_llvm_database`；停止自有 loopback PostgreSQL，资产回到 `/root/.local/share/Trash/files/dever-llvm-pg-ZQQmWO`，服务日志在 `target/closure-llvm-api-postgres-server.log`。临时仓库 `config/setting.json` 已移除，恢复原先不存在；诊断用失败 fixture、临时 signal/backtrace/debug 标记已移除，原 C allocation ledger 恢复。源码备份保留。磁盘清理只针对已核实的旧 target archive，可重新构建；未删除源码、业务数据或 SDK。 + +最终复核没有本切片未解决的功能失败。README/LANGUAGE/IMPLEMENTATION 和 compiler/database/toolchain/directory/logging specs 已同步；共享 clone 值、DELETE 单 owner、请求 Future 栈边界、pool 取消归属及配置式 REST fixture 的规则已记录。没有运行全量 workspace、CMS 服务、性能/RSS/64–128 MiB 或其他 OS；新 LLVM commit/Session-close 人为故障注入、完整协议组合和容量仍属于最终验收矩阵，不由重复入口分配平衡代替。 + +父任务继续 `in_progress`，不提交、不归档。下一步按原合同完成 LLVM Job/Test 执行根及持久任务的身份重验、取消/重试/事务资源清理，再接 external Adapter、默认 CLI 和正式发行；不重设计语法或缩减既定外部生态/平台范围。 + +## 2026-10-01 LLVM Job/Test 执行根（本切片已完成） + +用户在 API/REST 切片完成后明确继续。当前边界是把现有持久 Job 与应用 Test 合同接到 LLVM；语法、默认 CLI、external Adapter 和发行工作保持各自原定阶段。复用 checked Job/Payload/Test metadata、case-local Port fake、队列 store/worker、Clock、可信身份和 application Session，不另建任务队列、测试解释器或全局可重绑状态。 + +编译器接通入队、调度、typed Job dispatch、User 每次执行的身份/精确权限重验、System 编译器入口、Job 数据库初始化/租户迁移及 api/worker/all 组合；Test 接通 filename entry、断言、每 case fake/数据库/虚拟时钟与资源清理。所有业务值保持具体类型，序列化仅用于既有持久 payload 和 ABI 边界；失败、取消、重试和 lease fencing 由现有 owner 承担。 + +分工：core implement 负责 `crates/dever-core/src/llvm*` 与必要纯 helper 共享；runtime implement 负责 runtime、bridge ffi/header/features。两者先协调共同 ABI，再分别修改；主线程负责根 tests、文档和串行 Cargo/实际运行，第三方独立复核后再收口。不得回滚彼此或既有用户改动。没有新的产品决策或任务批准等待。 + +改动前源码备份：`/tmp/dever-llvm-job-test-backup-spJueT`（crates、library、Cargo manifest/lock）。磁盘起始约 381 MiB，先核对无编译进程后清理必要的旧可重建产物；不清业务数据库、源码或私有工具。 + +验收：最小静态/六目标 object → 实际 archive + 隔离文件/SQLite → 持久 Job、Test fake/虚拟时钟、重复根/取消/错误清理 → 受影响旧 API/Model/task/Job/Test 合同。真实 PG/自有 worker/HTTP 的运行前说明；不访问部署服务，不用环境变量配置，不默认全量/性能测试。 + +### 实现与根因修复 + +- LLVM `jobs.rs` 接入具体类型 payload、入队/定时、handler 事务、Job-only 与 API/worker/all、租户队列迁移及每次执行的 User 身份/精确权限/组件重验。Session 持有本次 bindings/Clock，复用 store/worker/cron/lease fencing；同步 handler 复用 bounded blocking，异步 handler 复用既有协程。 +- `llvm::emit_test_suite` 复用 case-local `test_program`，单 IR/可执行文件按稳定 index 派发；导出 indexed fault size/render/release。只给已声明的 LLVM 内部符号加 case 前缀，不改字符串和 runtime ABI；不同 fake 的效果、失败类型、数据库与时钟互不串用。普通测试不启动真实 Lib 或服务；无数据库 case 不创建无关 Model。 +- 修复服务配置无条件校验：只有 `service_start` 才检查 Worker/HTTP 启动策略，普通入队 CMD/管理命令无需这些配置。修复无 tenant 配置的普通 Job 被要求 tenant ID：两后端共享 `job::require_components`，由实际部署配置决定是否隔离。 +- 实际 API User 入队首次报 500，根因是 LLVM 复用的 `auth::authorize_permission` 成功后未绑定权限，而原 native 已有 `job::bind_permission`。现在授权成功后由该 runtime owner 绑定精确 key，native 也委托同一实现；拒绝/错误不绑定，不伪造 System 身份。 +- 校正身份测试 oracle：空 subject、篡改物理归属会触发 store 的早期 fail-closed 错误，不是 dispatch-time blocked。现使用格式合法的失效 subject/session、已删除权限、与执行租户不符的 tenant claim 来验证重新授权，保持原存储合同。Job 不允许调用 HTTP/auth getter，fixture 的业务写入只读取 typed payload。 +- 更新原 LLVM 应用测试中过期的“Job 必须拒绝”断言，使用新 Job-only/CMD/服务回归证明支持；沿用原 C allocation ledger（warmup + 64 次），仅修正零次重复时的无符号循环警告。新增 case driver 和 SQLite 检查均在根 `test/`，复用原链接/子进程工具。 + +### 最终证据与边界 + +- 新增 Job/Test target **9/9**：2 个静态/六目标 object 项，7 个显式实际 archive 执行组。覆盖 4 个纯断言/业务失败 case、7 个 Job fake/虚拟时钟/重试/cron/事务 case、2 个同步 blocking/异步 fake case，以及入队 CMD、启动前策略拒绝、schema/payload/policy 阻断、超时子任务排空和回滚。纯用例与重复入口沿用 warmup + 64 次分配平衡，未降低 oracle。日志 `target/closure-llvm-job-suite-static.log`、`target/closure-llvm-job-test-native-final.log`；最终纯 suite 加入业务失败和字符串不被 namespace 改写的断言后，六目标复验见 `target/closure-llvm-job-api-suite-final.log`。 +- 新增 API User Job **2/2**：六目标 object（`target/closure-llvm-job-api-static.log`）及真实自有 loopback/两租户 SQLite。实际验证 api 入队、all 执行、worker-only、精确权限引用、撤权/会话失效/权限删除/账号失效/租户不匹配的 `blocked + identity_rejected + attempt=1`、组件禁用的 `component_disabled`，正常业务记录只落入各自租户库。最后与既有私有 REST（65 次根）共同复验 **2/2**，见 `target/closure-llvm-job-api-suite-final.log`。 +- 原后端 `durable_jobs` 选定 **4/4**：case-local SQLite/Clock/fake、撤权后禁止业务派发、SQLite durable state machine、租户业务/队列同库与仅恢复 tenant scope。日志 `target/closure-llvm-job-original-regression.log`。 +- 受影响 LLVM 应用/异步/受管值 **19 + 22 + 10 全通过**，日志 `target/closure-llvm-job-shared-regression.log`。基础 ABI **4/4**，canonical C header 与实际 archive 链接执行 **1/1**，日志 `target/closure-llvm-job-abi-final.log`。重复执行不重复累计新覆盖。 +- 最终 runtime archive 重建通过（`target/closure-llvm-job-permission-archive.log`）；runtime-api base/SQLite/PostgreSQL/双 driver 四 profile offline/locked lib check 全通过（`target/closure-llvm-job-profiles.log`）。core/runtime/CLI libs/bins 与 bridge/lib/本批 tests 严格 Clippy `--no-deps -D warnings` 通过（`target/closure-llvm-job-clippy-product.log`、`target/closure-llvm-job-clippy-bridge.log`）。修改范围私有 rustfmt、C11 `-Wall -Wextra -Werror`、相对源码备份的 whitespace 检查通过（`target/closure-llvm-job-rustfmt.log`、`target/closure-llvm-job-c-driver.log`、`target/closure-llvm-job-diff-check.log`）；没有全局安装工具或重排历史 native 源文件。 +- 独立 review 复核了 typed ABI、资源清理、原 backend 鉴权等价性与两次 shared-owner 修复，没有剩余 P1/P2。早期失败日志保留，最终通过证据如上；测试子进程和临时 SQLite 由 fixture 清理，仓库未新增 `config/setting.json`，源码备份保留。README/LANGUAGE/IMPLEMENTATION、相关 specs 和日志已同步。 + +本轮未运行真实 PostgreSQL Job、全量 workspace/CMS、性能/RSS/64–128 MiB 压测或其他 OS 实际运行；四 profile 和六目标 object 不代表这些已验收。默认 CLI 仍使用 Rust 引导后端。下一切片接现有 external Adapter/Worker 的 LLVM 执行链，再完成默认 CLI、正式 packs/全平台发行与最终容量门。父任务保持 `in_progress`，不提交、不归档。 + +## 2026-10-01 LLVM external Adapter/Worker 与内嵌资源（本切片已完成） + +- 本轮从现有 Worker 协议、生态解析和打包实现接入 LLVM；缺口是 LLVM 当前拒绝 external 函数，尚无 typed wire 调用及内嵌资源入口。备份：`/tmp/dever-llvm-external-backup-kI9I74`。 +- Core 负责 checked Adapter 候选、Setting 校验后启动、typed 输入/输出/声明业务错误、资源元数据与应用入口；runtime/bridge 复用现有 component supervisor、external 安全提取和 Scope，补 canonical ABI。原 Rust backend 的活跃调用者同步迁移，不能另写协议、解析器或进程管理器。 +- Worker 与资源归属于一次应用调用，重复进入不能复用已关闭实例或泄漏元数据。启动失败、取消、超时、未声明错误及清理失败保留原来源/故障语义；普通 Test 仍只走 fake。共享资源 hash/dedupe 算法只有一个 owner。 +- 验证覆盖六目标 IR/object、真实自有 fixture Worker 成功/业务错误/畸形响应/启动失败、连续 root 清理、独立可执行程序内嵌 Worker 与受管 launch 资源，并复验原 Worker、资源及受影响 LLVM 入口。主 agent 串行运行定向 Cargo/链接测试；实现 agent 不执行 Cargo，不清理缓存。当前磁盘约 733 MiB，构建前持续核对容量。 +- 本轮不切换默认 CLI 后端，不重做 pip/npm/go resolver，不访问生产服务、不下载语言环境、不增加配置环境变量；最终发行/容量门仍在父任务内。 + +### 实现与复用 + +- 新增 `llvm/external.rs` 和 `llvm::emit_application_with_resources`：复用 checked Port/Adapter、具名 wire codec、声明错误身份、原 native 摘要/资源去重和 runtime 安全提取。选中的 Setting 校验成功后才启动 Worker；缺失字段（包括可空字段）、额外字段、错类型与未声明错误保持来源位置并作为运行故障，不能伪装成可恢复业务错误。 +- `runtime-external` 独立于编译器 SDK。canonical ABI 明确元数据借用、输入复制、Reply 转移和资源清理；无新的协议、生态解析器、进程管理器或宿主环境查找。 +- 将进程级 Worker 注册表/资源缓存迁入 invocation-owned `component::Session`；显式传播到 Task、blocking 与 API。supervisor 属于根 Scope，短命子任务结束不关闭共享 Worker；启动前注册 owner，取消启动或 shutdown 仍由入口继续收尾。根先关闭 Worker，再排空子任务及应用/数据库 owner；保留原故障并附加清理原因。 +- 普通 Test 沿用 case fake 图，不启动真实 Lib；原 Rust backend 同步使用 owned Definition 和同一 Session。源码备份保留，未提交或归档父任务。 + +### 根因修复 + +- 共享 C 分配账本补计 `realpath(NULL)`：glibc 内部 malloc 绕过链接器包装，Rust 随后的 free 会被计数，原计数因此出现 -2/-1。独立探针验证修正前 -1、修正后 0（`target/closure-llvm-external-ledger-probe.log`）；未放宽 warmup + 64 根的精确余额断言。临时探针与诊断源码已删除。 +- 原 Worker 生命周期测试暴露新嵌套 Future 的调试栈溢出。没有挂起点的 `task::run_in` 改为同步返回 Task，`component::scope_resources` 直接返回 task-local scope Future，去掉重复状态机；不增加线程栈或 Box，不吞错。最终原测试和共享异步回归均通过。 +- 修正新测试中 CMD 绑定多个/零 App 输出的非法 fixture：Port 仍测多/零输出，App 按既有规则返回单个记录/Bool。未修改语言合同。内嵌资源验收保留每根 SHA 校验，仅将外层进程限时调整到能覆盖 65 根的 120 秒；测试墙钟包含编译/链接/反复校验，不作为性能数据。 + +### 最终证据与边界 + +- 新 LLVM external 共 8 项均已有通过证据。四种 exec/pip/npm/go 内嵌 launch 组在 `target/closure-llvm-external-verified.log` 通过：只迁移可执行程序和 setting.json、删除构建目录、清空继承环境，各跑 65 根。该轮另外一项 CMD fixture 失败已修复，见 `target/closure-llvm-external-typed-final.log`。最后两个无挂起 helper 修正后重建 archive（`target/closure-llvm-external-archive-final.log`），其余 7 项全部复验通过（`target/closure-llvm-external-final.log`）；四生态内嵌大组未为等价 helper 修正重复运行。 +- 最终原 Worker 生命周期/资源/入口清理 1+9+3 全通过，含真实启动/关闭取消检查点、短命 Task 调用和原 Rust backend 独立 exec 打包运行（`target/closure-llvm-external-original-final.log`）。 +- 最终共享 application 19、async 22、ABI 4 全通过（`target/closure-llvm-external-shared.log`）；fake-only Test、HTTP 请求连续入口、canonical C 实际链接分别 1/1（`target/closure-llvm-external-{fake,http,c-link}-final.log`）。删除旧 external 拒绝断言后的 application 元数据测试 1/1（`target/closure-llvm-external-application-static-final.log`）。 +- 六 runtime profile 的 offline/locked lib check 通过（`target/closure-llvm-external-profile-*.log`）；base ABI 保留备份中已存在的 `append_fault_cause` dead_code warning，不声称全 profile 无警告。修改范围 rustfmt、C11 strict syntax、相对备份 whitespace 检查通过（`target/closure-llvm-external-{rustfmt,header,diff-check}-final.log`）。 +- core/runtime/CLI libs/bins、bridge lib/受影响 LLVM tests 与原 Worker/资源/清理 tests、fixture binary 的私有严格 Clippy `--no-deps -D warnings` 全通过（`target/closure-llvm-external-clippy-{product,bridge,tests}-final.log`）。未安装或替换全局工具，未将定向检查称为全仓库质量门。 +- 本批原失败日志保留供追溯，以上述最后证据为准。受管 launch 使用自有合成 fixture，不能代替真实 Python/Node/Go SDK 的正式发行验收；六目标 object 也不等于六 OS/架构实际执行。本轮没有全量 workspace/CMS、真实 PostgreSQL、性能/RSS/64–128 MiB 或其他 OS 验收。 +- 独立 trellis-check 最终确认无剩余 P1/P2；复核了资源账本与 Future 层数修正，结论与上述准确测试范围一致。测试子进程无残留，父任务和用户原有工作区变更保留。 + +下一切片接默认 CLI 的 LLVM 编译/链接与 runtime pack 选择，再完成正式 packs、真实生态/共享发行及最终容量门；父任务保持 `in_progress`。 + +## 2026-10-01 LLVM 私有默认 CLI 与 runtime pack 接入（本切片已完成） + +- 差距：`deverc run/build/test` 仍读取 RUSTC 并生成 Rust；已有 LLVM 只导出 callable application/suite，还缺真实进程入口、pack 选择及编译/链接编排。继续现有父任务，不重新设计语言或 API。 +- 改动 owner：core 的 LLVM 入口/故障呈现与 native artifact 临时目录/cache；CLI 的 run/build/test、runtime pack 验证/选择；已有 LLVM/LLD bridge 只补实际链接所需的封闭选项。复用 checked HIR、测试逐case隔离、现有资源pack与输出create-new规则，不另写协议/进程管理或文件清理实现。 +- 默认应用命令不能调用 Rust/Cargo/cc 或通过环境变量/PATH选择后端；依赖显式准备且校验过的目标 runtime/CRT/library，缺失明确失败，不增加 Rust fallback。私有作者准备工具与产品执行严格区分；签名发行继续复用现有 release v1。 +- 源码核对发现 deverd 只有 artifact IPC，没有受信任编译入口。按既有共享缓存安全合同，本切片完成私有 `deverc` 默认 LLVM 并保留其作者缓存；managed `dever-core` 编译命令明确报告缺少共享编译入口，不能把失败降级成本地/无缓存编译。该部分留在已排定的共享编译/发行切片，不新增隐式旁路或用户环境变量。 +- 备份：`/tmp/dever-llvm-cli-backup-6mojiN`（crates/library/Cargo manifests）。磁盘低至153MiB，已删除逐个确认的旧 first-party rlib/LLVM test binary；所有源码、备份、设置、数据、SDK及当前runtime archive保留。主线程独占所有Cargo/链接验证，agent不运行构建或清缓存。 +- 本切片验证实际离线 CLI check/run/build/test、错误/失败继续、pack缺失/篡改、输出防覆盖/资源清理、cache身份与必要旧调用者；包括plain/Markdown、fake和最小数据库/外部Worker路径。六目标metadata/object与当前Linux实际执行分开记录，不将自有pack fixture声称为正式签名多平台发行。 +- 之后仍需正式packs、真实生态/共享编译发行、跨平台和最终性能容量；不在本次默认跑全量测试、CMS长链路或生产服务，不安装全局命令，不提交。 + +### 实现与根因修复 + +- `deverc` 私有 `compile.rs`/`compile/runtime.rs` 接通 checked HIR→LLVM→object→进程内 LLD。复用 `native::compile_artifact` 的临时目录、缓存完整性和 create-new 发布；旧 Rust 作者回归也委托同一 owner，不留产品 fallback。 +- pack 使用编译器同级 `runtime/<platform>/manifest.json`，绑定 compiler version/ABI/target 与 base/sqlite/postgres/both。路径、大小、SHA 和 profile 引用均校验；缓存命中也复核选中输入。链接前复制并重验，仅接受显式 CRT object/普通静态 archive，拒绝 symlink、脚本、thin archive。缓存身份含编译器字节、完整 manifest/profile、IR 与资源。 +- `llvm/process_entry.rs` 在 callable application/suite 外生成 C main，复用 typed 故障呈现、原 logger 和资源释放。suite 参数严格检查单个十进制 index，size/run/render/drop 共用同一 index。实际 closed-pipe 探针发现原入口缺 Rust 启动的 SIGPIPE 策略（退出 -13、stderr 空），现在仅进程 main 调用统一 ABI 初始化；callable 根保持宿主策略，实际回归要求 code=1 与源码错误。 +- 新编译模块最初在共享 CLI lib，readelf 发现 launcher/daemon 因此误链接 LLVM。将 compile/pack 收回 binary 私有模块，最终 `dever`/`deverd` 不再有 LLVM/C++ DT_NEEDED，空环境真实入口回归通过。`deverc` 使用自身 `lib/` 的 LLVM；未安装或修改全局命令/服务。 +- debug 大 archive 的 SHA 校验造成每次 CLI 数十秒等待。共享 `sha256_file` 复用已有 ring 流式 SHA-256，测试 pack 删除重复摘要实现；每次完整校验仍保留，既有签名发行测试用独立 sha2 oracle 复验。没有新增密码库、缓存信任豁免或性能达标声称。 +- 首轮 CLI 5/6 通过，Test fixture 的损坏 JSON 在 Package 元数据读取时被拒绝。改用合法项目 JSON + 无法作为数据库打开的目录，验证测试执行只使用生成的临时 SQLite；缺失真实 Worker 仍由 fake 隔离。旧 RUSTC 失败注入用例已迁移为无 pack 的独占编译器，保留全部用例失败汇总和清理断言。 + +### 最终证据与边界 + +- 实际 CLI **7/7**：`target/closure-llvm-cli-execution-final.log`。覆盖无宿主编译器调用、Unicode/异步、typed 与运行故障/来源、build 防覆盖/搬迁、closed stdout、源码/manifest 缓存身份、坏缓存/命中后的 pack 篡改、Markdown CMD、SQLite/fake 逐 case 隔离、失败继续、非法 suite index，以及内嵌 exec Worker 在源码删除后独立运行。真实 Worker 只是协议 fixture，不代替 Python/npm/Go 生态发行。 +- 进程入口 **3/3** 与既有 bridge **5/5**：`target/closure-llvm-cli-entry-link-final.log`，含六目标 object、原简单内核六目标 link、失败后恢复/并发输出保护。当前实际独立应用仅验收 Linux x86_64。 +- canonical C header 与最终 runtime archive 实际链接 **1/1**：`target/closure-llvm-cli-c-abi-pack.log`，含新增进程参数/错误 ABI 与 SIGPIPE 初始化。最终 ABI version 改为复用同一个常量后 archive 再建、此项重验；其余实际 CLI 已在相同语义的 archive 上通过，没有重复计数。 +- artifact cache 新用例 **2/2**，原 Rust cache 编译回归 **1/1**：`target/closure-llvm-cli-cache.log`、`target/closure-llvm-cli-bootstrap-cache.log`。旧 CLI 编译失败汇总 **1/1**；签名版本安装/回滚、空环境多项目 launcher、自有 daemon IPC 各 **1/1**：`target/closure-llvm-cli-{test-failure,release,launcher,daemon}-final.log`。本切片合计 23 项不同定向用例通过。 +- CLI lib/bins/相关 tests/example、core lib、完整 bridge profile/相关 tests 和 cache test 严格 Clippy 均通过：`target/closure-llvm-cli-clippy-{product,core,bridge,cache}.log`。修改范围 rustfmt/C11/whitespace 检查通过；历史 native 文件未整库重排。base ABI 编译仍有此前已有的 `append_fault_cause` unused warning，不声称所有 profile 无警告。 +- 私有作者 pack 已准备于 `target/debug/runtime/linux-x86_64`，复用 root `test/native-runtime-pack.rs`/共享 fixture，日志 `target/closure-llvm-cli-author-pack.log`；重复准备明确拒绝，原 manifest 摘要未变。四 profile 在此夹具中复用完整 archive，不是四套正式裁剪包。最终动态依赖见 `target/closure-llvm-cli-dynamic-dependencies.log`。两套 CMS 仅在空环境做 check，通过并保留原 W001 建议，日志 `target/closure-llvm-cli-cms-check.log`。 +- 清理仅涉及逐个确认的旧可重建 target 产物，源码备份保留；临时 Worker/SQLite/daemon 由 fixture 回收，仓库未增加应用 setting.json。没有运行全量、完整 CMS、真实 PG、其他 OS、体积/性能/RSS/64–128 MiB 压测；没有提交、发布或安装全局工具。 + +父任务仍为 `in_progress`。下一步接可信 `deverd` 编译与正式签名 packs；managed `dever-core` 目前明确拒绝编译，因此此前 Package 的受管完整执行合同也需在该阶段复验，不能用私有旁路把它标为完成。真实生态高级能力、OS sandbox、跨平台发行及最终容量门仍按父计划实施。旧全 CMS mixed-suite fixture 的坏 JSON/历史用例计数未作为本切片通过证据，留在完整 CMS 验收中同步。 + +## 2026-10-02 可信共享编译与签名 pack 引用闭包（本切片完成） + +- 原始差距：managed core 拒绝 run/build/test，deverd 只有不透明 artifact IPC，不能把上传字节当作可信编译结果。本轮复用现有前端、LLVM、签名发行和 CacheStore 补齐了可信编译链路。 +- caller 保留 Package/Lib 准备、完整部署配置、API baseline 及最终程序运行；提交规范逻辑源码、无秘密编译绑定和资源字节。daemon 自行计算身份，只启动已验签 core 的编译 worker，不执行 App/Test/Lib Worker，不接受客户端 IR、cache key 或输出路径。 +- 改动 owner：core source/check/model 及 runtime config 的无秘密编译投影；binary-private compile/worker 与 NativeProgram 接收产物；toolchain service/cache/release 的 IPC、生命周期、版本共享锁及签名 runtime 引用闭包。LLVM 动态依赖仍不得进入 launcher/daemon。 +- 请求、响应、并发和编译时间有界；断连/取消/失败回收 child 与 staging。缓存命中仍验证签名资源，完整请求才可读编译产物，不能通过 artifact digest 读取编译缓存。复用现有 operation/read lease，安装/卸载持久独占锁与编译共享锁互斥。 +- 主线程独占 Cargo/native 验证。采用自有临时 daemon/签名 fixture，恢复受管 Package 最小完整链路；不启动现有服务，不做全量/CMS压测、全局安装、正式签名发布或其他平台执行。 +- 备份：`/tmp/dever-shared-compile-backup-udVyQC`（crates/library/test/Cargo manifests/父任务）。已确认无构建进程，清理旧可重建 `target/native-artifacts` 和 `target/native-runtime/inputs`，空闲由约417MiB恢复至1.1GiB；未删除源码、备份、SDK、当前LLVM archive/pack或数据库。 +- 本切片签名 fixture 证明安装/引用绑定，不把四 profile 共用 debug archive 当作正式裁剪六平台资产，不把受限编译子进程当作完整 OS sandbox。 + +### 实现与边界 + +- `CompilationBindings` 只携带数据库 driver/选择名、Provider verify 方法及 site path/auth；URL、密码、JWT/Cookie 等部署配置不进 daemon。普通配置和远端编译复用同一验证/数据库选择 owner。Test 不携带部署绑定，继续由 caller 逐例检查并执行隔离 SQLite/fake。 +- 规范请求限制为 64 MiB、4096/16 MiB 源码、4096/48 MiB 资源（单个32MiB），拒绝未知字段、重复/非规范编码、目录逃逸及伪摘要。严格 SourceMap/checker 在已验签核心里重建 HIR，缓存身份绑定完整请求及签名 manifest。 +- daemon 同时最多2个编译，180秒总期限；客户端190秒总接收期限。私有 worker 限制 CPU/地址空间/文件大小/FD/core dump，不运行应用或依赖脚本。取消、断连、失败、超时均 wait/reap 并回收 staging；缓存与安装版本租约持续到传输结束。编译缓存独立限制2GiB/4096条/单产物64MiB，artifact 上传及摘要读取不能访问它。 +- 每次请求包括命中缓存都复验签名、native manifest、全部 profile 的 CRT/archive 与 LLVM 库；安装/卸载用持久独占锁、编译用共享锁。worker 只接收自有0700目录中的0600请求，state/cache/socket祖先仍允许其他用户安全遍历。 +- 共享 runtime pack 校验不引用 bridge，LLVM 只由私有 binary 编译模块调用。修正安装锁误用公开权限验证、worker 对公开祖先误要求0700，以及相对 `--root .` 下切换 cwd 后找不到核心程序的问题；核心路径在验签及持锁后转换为绝对路径。 + +### 定向验收 + +- `packages --ignored --skip installed_managed_package_worker_guard --test-threads=1`：4/4，355.62秒。包含签名安装的 Package check/test/run/build/独立运行、两个实际 UID 跨目录复用、错误源码与缓存恢复、未签名 CRT/篡改 pack 命中拒绝，以及实际180秒超时、取消/断连回收与版本锁互斥。日志 `target/closure-shared-compile-managed-final.log`。 +- 最后将测试 daemon 改为 `--root .` 后，Package 完整执行及跨UID两项复验2/2，88.83秒（`closure-shared-compile-relative-root.log`）。Package Worker 锁定归属/本地碰撞父用例1/1，9秒（`closure-shared-compile-package-worker.log`），由它正确调用签名安装里的 child-only helper。 +- 无秘密源码快照 1/1；`shared_toolchain` 16/16（原独立 artifact 跨 UID 项保持默认 ignored，本轮编译跨 UID 已显式执行）；私有 `llvm_cli` 含 ignored 7/7。日志分别为 `closure-shared-compile-snapshot.log`、`closure-shared-compile-toolchain.log`、`closure-shared-compile-private.log`。 +- `api_declarations` 16/16、`application_config` 6/6，日志 `target/closure-shared-compile-bindings.log`。上述共51个不同定向用例，不把重复复验或嵌套 helper 计入总数。 +- 双源码 CMS 仅清空环境 `check` 通过，既有W001建议保留；没有启动 CMS/数据库或压测。日志 `closure-shared-compile-cms-dever.log`、`closure-shared-compile-cms-md.log`。 +- CLI lib/bins/相关三个测试 target 与 core/runtime lib 的 Clippy `-D warnings` 均通过；本次独立/已格式化 Rust 文件的 rustfmt check、备份对比 whitespace 检查通过。历史大文件未做无关全局格式化。证据 `closure-shared-compile-clippy.log`、`closure-shared-compile-core-clippy.log`、`closure-shared-compile-rustfmt.log`。 +- `readelf -d` 确认 dever/deverd 不依赖 LLVM/libstdc++；deverc 保留固定 LLVM18依赖及 `$ORIGIN/lib`。证据 `target/closure-shared-compile-dynamic-dependencies.log`，不能只凭本机装有LLVM时能启动得出无依赖结论。 +- 首轮失败记录保留：fixture 配置字段/保留命名空间断言修正、祖先权限修复、跨UID fixture外层目录需0755，以及 child-only ignored helper 不应从仓库根独立运行。最终通过日志覆盖对应问题，未放宽语言或权限合同。 + +父任务保持 `in_progress`。下一步制作正式、可重复、裁剪 profile 的 native/runtime/build packs；六目标正式资产、真实生态高级能力、OS sandbox、跨平台发行以及完整性能/容量门仍未完成。本轮未运行全量、真实PG、全CMS、低内存/性能或其他OS验收，未安装全局服务或发布真实签名版本。 + +## 2026-10-02 原生发行包制作（本切片完成） + +- 实施前只有 `test/native-runtime-pack.rs` 的作者 fixture，四 profile 复用 debug archive；签名安装/运行时验证已存在，缺可重复的制作与签名入口。本轮补 Linux native release 的作者制作工具、四个真实优化 archive 和对应安装/LLVM链接验收。 +- 复用 `toolchain/runtime_pack` 的 manifest/引用/格式/摘要校验、`ReleaseManifest` 与 ring Ed25519；作者工具仅从自己根下 `config/setting.json` 读取显式文件/摘要/签名key路径，输出全新发行目录。禁止私钥进入产物、硬链接导致源文件变化污染产物、隐式联网、环境变量/宿主工具查找或覆盖已有发行目录。 +- 作者侧构建与应用 run/build 分离：共用一个 Cargo target 构建 base/sqlite/postgres/both，独立 feature 组合；发布材料记录实际 archive、编译版本/目标/选项和摘要。测试两次制作字节一致、错误输入无半成品、验签安装后实际编译/运行，并检查 profile 裁剪和应用产物体积。重复打包一致不冒充跨机器源码重建一致。 +- 代码 owner：CLI `toolchain` 制作/签名模块及私有 SDK 作者入口，根 Cargo 的 runtime pack profile/作者构建配置，root test 的定向制作/安装验收。产品编译与签名安装协议保持不变；改动原始 fixture 时必须复用同一个制作 owner。 +- 本轮不发布真实密钥签名版本、不安装全局服务；Python/Node/Go正式pack、OS sandbox、其他宿主实际运行与容量门仍按父任务另行验收。Linux本机构建不能充当六平台完成证据。 +- 修改前备份 `/tmp/dever-native-release-backup-u48cQl`，包含 crates/sdk/test/Cargo manifests/父任务。主线程独占 Cargo、实际构建、清理和运行验证；子代理负责边界内实现及检查。 + +### 实现与实际产物 + +- 新增 `toolchain::packaging::create` 和私有 `sdk/native-release.rs` 作者命令；使用现有 v1 manifest/验签合同，无新依赖和 lock 漂移。配置拒绝未知字段/路径逃逸/符号链接/错误摘要，私钥必须 owner-private，拒绝把相同私钥字节作为 payload。先复制再复核摘要，固定清单顺序并签名，以 Linux `RENAME_NOREPLACE` 发布；失败只清理自有 staging，并发不能覆盖输出。 +- 根 `runtime-pack` profile 保留链接符号，四次离线 locked 构建共用 `target/native-runtime-abi`。均启用 `runtime-api,runtime-external`，按 profile 分别启用 SQLite/PostgreSQL；未启用 `embedded`。作者工具与应用 run/build 分离，不引入产品环境变量、联网或宿主编译器查找。 +- 实际保存于 `target/native-release-inputs/`：base 78,568,584 字节，SQLite 93,357,820，PostgreSQL 99,502,804，both 106,926,902；全部 SHA-256 见 `target/closure-native-release-acceptance.json`。`nm` 检查证实 SQLite/PostgreSQL 驱动只在对应 archive 中出现，未包含 LLVMContextCreate。这些是供链接的库体积，不是应用体积。 +- 作者工具链为 rustc 1.98.0（88d9e12ae，内置 LLVM 22.1.8）/cargo 1.98.0（797e8a9bc）；Dever 编译核心使用独立 LLVM 18/LLD，Linux x86_64 输入使用本机 glibc 2.39/GCC 13 CRT。未证明其他 libc/宿主/目标可用或跨机器源码构建一致。 +- 删除旧 `test/native-runtime-pack.rs` 作者入口（上述备份可恢复）。测试共享显式 SDK 输入及 daemon 生命周期，旧私有 pack fixture 保留作回归；没有复制第二套签名/manifest 校验。作者配置、制作与验收用法记录在 `sdk/native-release.md`。 + +### 定向验收 + +- 四个优化 archive 构建全部通过；日志 `target/closure-native-release-{base,sqlite,postgres,both}-build.log`，分别约 2m52s、4m47s、3m40s、3m35s。作者 executable 构建通过，日志 `closure-native-release-maker-build.log`。 +- `cargo test --offline --locked -p dever-cli --test native_release`：默认 5/5，通过重复制作/签名/独立副本、坏输入及 staging 回收、私钥和 symlink 拒绝、已有目录保护和并发单次发布。真实资产用例默认 ignored；日志 `target/closure-native-release-tests.log`。 +- 显式 `native_release acceptance::optimized_profiles_make_reproducible_signed_release_and_run_through_daemon -- --exact --ignored --nocapture`：1/1，97.42 秒。先库调用、再实际作者 CLI 制作，清单和签名字节一致；验签安装后四 profile 的 `check/run/build` 均通过，缓存恰好四条。关闭自有 daemon 并删除源码后，四个程序独立运行通过，SQLite/both 实际创建、计数和删除通过。日志 `target/closure-native-release-acceptance-final.log`，摘要/体积/边界见同前缀 JSON。 +- 该最小 CMD 的程序体积：base 2,088,488 字节，SQLite 5,082,504，PostgreSQL 2,269,832，both 7,592,744。base/postgres 执行异步 sleep，sqlite/both 执行 CRUD，不能直接作为同一业务的四 profile 性能对照；PostgreSQL 配置指向未使用地址，未连接服务，不宣称真实 PG 验收或驱动全部代码都进入最终程序。 +- 验收初轮 fixture 错把单输出 Bool 预期成对象，随后发现 `value/model.dever` 必须声明 `Value`；仅修正这两个测试合同，语言/输出规则未放宽。首次失败日志 `target/closure-native-release-acceptance.log` 保留；最终完整重跑通过。 +- 回归：`llvm_cli` 默认 4/4（3 个历史显式用例保持 ignored），签名版本安装/切换/回滚 1/1；日志 `closure-native-release-llvm-regression.log`、`closure-native-release-signature-regression.log`。连同制作器 5 项及实际发行 1 项，本轮共 11 个不同定向用例通过，不重复计算四 profile 和重复运行。 +- CLI lib/bins、作者 example 及 native_release/llvm_cli/packages/shared_toolchain test targets 的严格 Clippy `-D warnings` 通过，日志 `closure-native-release-clippy-final.log`。初轮暴露测试重复载入 temp 模块及新 Clippy 的定长切片建议，改为复用父级 TemporaryDirectory 和 `as_chunks`，不加 suppress；默认制作测试复验 5/5，日志 `closure-native-release-tests-final.log`。修改范围 rustfmt、备份对比 whitespace 通过，独立审核无确认阻塞。 +- `readelf -d` 复核 dever/deverd/作者制作器均无 LLVM/libstdc++ 依赖；deverc 保留 LLVM18 与 `$ORIGIN/lib`。日志 `target/closure-native-release-dynamic-dependencies.log`。这不证明核心及 LLVM 的其他系统库都已随发行打包，干净机器验收仍未完成。 +- 清理仅使用 Cargo clean 回收已确认的可重建旧 native cache/debug ABI/重复 runtime-pack bridge 输出；当前私有 debug archive 链接、四个优化输入、源码及所有备份保留。测试临时发行目录、密钥、daemon 与 SQLite 由 fixture 回收。收尾磁盘约 1.2 GiB 空闲。 + +父任务保持 `in_progress`。本切片证明 Linux x86_64 原生制作、安装和执行；签名密钥是测试临时身份，核心程序是当前开发构建，没有发布正式版本。Python/Node/Go packs、完整标准库/构建输入、OS sandbox、其他平台发行以及完整性能/容量门仍未完成。本轮未运行全量、真实 PG、CMS 服务、低内存/性能或其他 OS 验收，未安装全局服务或提交。 + +## 2026-10-02 外部生态发行包制作（Linux 定向验收完成） + +- 现有 Registry/Worker 已消费签名 `runtime/<ecosystem>/<target>/manifest.json` 与 `runtime.pack`，缺制作 owner；本轮扩展现有原生作者制作器，按显式文件/摘要生成确定性压缩包及 registry 描述，将 Python/Node/Go 材料统一纳入发行签名。Worker 与作者共用 runtime manifest/闭包验证，不再各写一份字段合同。 +- Python 必须携带标准库与扩展，Node 使用明确发行输入，Go 携带 compile/link/analyzer 与 stdlib importcfg。只在作者准备阶段获取/整理固定输入；产品 run/build 不联网、不读取环境变量或宿主工具。实际脱离源码/输入目录的执行用于证明迁移能力，临时签名不代表公开发行。 +- 已确认 Node 解释器约 118 MiB，超过上一轮共享编译的 32 MiB 单资源、48 MiB 总资源及64 MiB输出限制；本轮在实际编译协议/缓存 owner 修正相互冲突的有界预算，保持不透明 artifact 与可信编译产物隔离、2 GiB 总缓存限制和生命周期校验。避免只让制作器通过、受管 run/build 仍不可用。 +- owner：`toolchain/packaging` 与 Worker/runtime 描述共享校验、registry metadata；共享编译的资源/输出预算；根 test 的三生态签名制作及独立执行验收。语言语法、Provider、API/CMS 业务不变，extras/sdist/npm高级安装/Go sumdb 仍按父计划后续实现。 +- 备份 `/tmp/dever-ecosystem-release-backup-7jv8nj` 包含 crates/sdk/test/Cargo manifests/父任务。实现/审核子代理遵循明确文件边界；主线程独占 Cargo、输入准备和实际验收。 + +### 已落地与验证过程 + +- 继续追踪后确认，旧 LLVM emitter 把每个资源字节编码为三个字符，122,889,056 字节的 Node 会生成约 351.6 MiB 的 IR，不能通过 8 MiB IR 边界。本轮复用 `native::resource_inputs` 摘要去重，把 payload 作为有界二进制附件交给私有 LLVM bridge，在 verify/优化之前填入已声明的常量；IR 仍绑定路径、摘要、长度和执行位，8 MiB IR 限制保留。runtime C ABI 未变，因此不重建上一轮四个优化 archive。 +- 共享编译传输 384 MiB，资源最多 4096 项、单项128 MiB/总256 MiB，可信产物320 MiB；bridge保持对应附件边界。不透明 artifact 上传仍64 MiB，编译与 artifact 缓存分别2 GiB/4096条。canonical编码只排序引用，daemon及worker尽早释放已不需要的请求副本，保留原180/190秒编译与客户端期限。 +- 固定作者输入：python-build-standalone 20260901 CPython3.12.14完整可搬移标准库发行、Node24.15.0官方Linux发行，以及先前保存的Go1.26.3构建fixture。Python/Node下载分别核对GitHub资产摘要和官方SHASUMS；确切SHA及路径保存在 `test/ecosystem-release/prepare.py`，生成材料位于 `target/ecosystem-release-inputs/prepared/`。Go不是独立验证的公开上游pack，临时签名不替代供应链来源证明。 +- 默认 `native_release` 9/9、`external_workers` 6/6；bridge 7/7及LLVM external默认3/3，包括超旧限制的二进制附件实际链接与逐字节核对。共享编译输入/输出配额2/2，canonical无秘密快照1/1。日志前缀 `target/closure-ecosystem-release-`，实际三生态验收仍单独执行，默认ignored不算通过。 +- 实际验收初轮因debug作者准备完整语言材料超过测试夹具190秒而终止，未进入应用编译；日志 `closure-ecosystem-release-acceptance-author-timeout.log`。作者制作步骤独立设为900秒，其他命令及产品编译期限保持不变。 +- 新 fixture 曾遗漏 Port 必须声明 `fails Choice` 的合同，语法/合同检查先后拒绝了缺失body与普通空body；已复用现有Port测试的明确失败Choice声明，并把三生态源码预检查移至制作之前。没有放宽语言合同。失败证据为 `closure-ecosystem-release-acceptance-fixture.log` 与 `closure-ecosystem-release-acceptance-port-contract.log`。 +- 调试版客户端准备完整Python资源另触发一次190秒fixture超时，保留 `closure-ecosystem-release-acceptance-preparation-timeout.log`。只优化压缩依赖后仍慢；对自有客户端3秒CPU采样（297样本）定位约七成开销在未优化sha2软件实现，非共享Worker死锁。诊断后仅终止已确认PID的自有客户端，daemon/项目由fixture回收；证据 `closure-ecosystem-release-preparation-perf.log` 及 `closure-ecosystem-release-acceptance-sha-diagnostic.log`。最终作者构建及显式验收统一使用Cargo命令级 `profile.dev.package.{miniz_oxide,flate2,sha2}.opt-level=3`，不改产品Settings、校验或期限。运行性能/内存仍需另行验收。 +- 仅以 `cargo clean -p dever-tests` 回收确认可重建的旧测试产物约1.0 GiB；保留源码、备份、四个优化archive和本次发行输入。独立审核暂未确认其他阻塞；格式/whitespace及依赖锁不变检查通过。 + +### Go 缓存身份问题 + +- 根因属于跨层合同与覆盖缺口：Go编译器把每次随机stage目录写入调试/源位置，两个相同锁定输入产生不同Worker字节，进而改变Dever编译请求及缓存身份。完整三生态验收第一次跑通全部 `check/run/build` 后，明确得到4条缓存而非3条；日志 `closure-ecosystem-release-acceptance-go-cache.log`，未把它标为通过。 +- 既有Go用例只构建一次并验证移除源码后可运行，不能证明重复构建一致。共享缓存没有误判：不同字节应得到不同身份，因此修正owner为Go `Builder::compile`，SDK/Adapter/依赖共用 `-trimpath` 删除stage前缀，保留逻辑文件名/行号；不按生态绕过哈希或放宽缓存断言。依据为Go官方compile文档的trimpath合同。 +- 将现有真实Go fixture改为在两个新stage重复准备并比较全部资源身份，再执行无源码Worker；定向1/1通过（10.10秒），日志 `closure-ecosystem-release-go-determinism.log`。规范已固定该断言及三生态共享缓存恰好三条要求；最终完整验收已通过，不新增独立构建/缓存实现。 + +### 最终结果与未完成边界 + +- `closure-ecosystem-release-acceptance-final.log`：实际三生态验收1/1通过，573.56秒。签名制作/安装、runtime锁定、三个受管 `check/run/build`、共享缓存恰好三条均通过；停止自有daemon，删除整个机器安装与源码目录后，三程序在只有显式Linux OS ABI库及私有proc的chroot中执行成功，没有系统Python/Node/Go或包管理器。Python另核对全部导入路径在内嵌目录，并执行SQLite、SSL、压缩等标准库能力。 +- 产物证据 `target/closure-ecosystem-release-acceptance.json`：pip/npm/go压缩pack分别41,778,610 / 43,996,315 / 54,494,714字节；测试程序分别94,754,600 / 125,792,424 / 6,960,616字节，完整SHA-256见JSON。Python/Node程序包含解释器与标准库，Go只内嵌编译后的Worker。这是三个探针程序的大小，不代表CMS大小或吞吐/RSS性能。 +- 本轮共30个不同定向用例通过：bridge/LLVM资源10、默认制作/Worker合同15、共享编译配额2、canonical快照1、真实Go重复构建1、完整三生态1。最终默认复验日志 `closure-ecosystem-release-contracts-final.log`；历史ignored未运行项不计入通过。 +- CLI/core/bridge严格Clippy均通过，日志 `closure-ecosystem-release-{clippy-final,core-clippy,bridge-clippy}.log`；修改范围rustfmt、备份对比whitespace、依赖清单/锁不变检查通过。独立审核复核Go统一入口与真实日志，未发现确认阻塞。 +- 最终 `readelf` 日志 `closure-ecosystem-release-dynamic-dependencies-final.log`：launcher/daemon/maker没有LLVM或libstdc++依赖，deverc仍依赖LLVM18及现有系统库。自有进程、签名密钥、安装/项目目录与mount已随fixture回收；磁盘恢复约1.9 GiB，备份和输入保留。 + +父任务仍为 `in_progress`。本切片完成三生态pack制作与Linux无系统语言环境的执行链路；临时签名不等于公开发行，Go输入的正式来源闭环、完整核心动态依赖交付、其他平台资产/执行、sdist/extras/npm高级安装/Go sumdb、OS sandbox及最终性能/低内存门仍未完成。本轮未运行全量、真实PG、CMS服务、第三方包安装、低内存/吞吐或其他OS验收，未安装全局服务或提交。下一步按父任务继续真实生态高级解析/构建能力，之后处理发行平台与容量门。 + +## 2026-10-02 五项剩余收口（实施中） + +用户已明确授权实施高级Lib、OS沙箱、发行/跨平台、最终组合与性能容量五项；沿用本任务，不创建重复任务。源码备份 `/tmp/dever-closure-remaining-backup-EFQlXQ`，保留此前备份和无关删除。主线程串行构建/测试，独立研究与实现遵循不重叠归属。 + +执行依赖：先补不执行第三方代码的精确extras/依赖解析与sumdb；并行建立可复用的Linux隔离启动边界；再接源码包/受控hooks及正式构建输入；之后统一发行验收和质量/性能矩阵。源码包/hook不能靠删掉拒绝分支完成，必须有锁定工具、受控网络/文件/进程及真实包验收。 + +- 研究 `research/ecosystem-closure.md` 明确每生态缺口和锁/Worker隔离合同。Python extras沿用规范化Lib请求身份，不把一个Worker选择的extras泄漏到另一个Worker。主线程负责Worker归档路径与重复wheel去重接线。 +- Linux隔离设计复用显式签名打包的bubblewrap与safe Rust seccomp guard，共享namespace/PID生命周期、路径与固定syscall策略。保持workspace unsafe禁止，不在Tokio线程设置进程级限制，不调用宿主PATH工具，不缺资产裸跑。文件授权由setting.json明确授予，配置内容不发给Worker；未支持的平台必须明确拒绝。非特权用户、真实三生态和后代回收均需执行证据，方案不是完成证据。 +- 最终CMS夹具基线失败于Package元数据读取前的非法JSON。改为合法JSON但不可打开的部署数据库目录，继续证明应用Test使用隔离SQLite；混合用例保留bootstrap，准确要求4项。复用根process owner为所有夹具命令增加180秒期限、文件捕获与空环境，避免输出管道阻塞。`closure-final-cms-project.log` 6/6通过（143.32秒），涵盖双源码check/test/run/build/移除源码后的独立健康入口；服务、PG与性能组合另行验收。 +- Python extras已接解析、精确变体图、同包自激活/回溯、Worker隔离、单归档安装与大小统计。空extra被pep508规范化接受的问题已修；`closure-python-extras-final.log`中external_libs 32/32、`closure-python-extras-install.log`安装7/7与Package11/11通过，显式宿主/发行fixture未被计入。这是extras证据，不包含sdist/native wheel。 +- 沙箱新增`dever-sandbox`共享namespace启动/ELF闭包验证/seccomp helper，发行maker及运行时所有exec/managed启动正在整合；Go工具也改走相同隔离owner，源码只读、build目录独立可写。`closure-sandbox-isolation.log`、`closure-sandbox-missing-library.log`、`closure-sandbox-descendants.log`分别验证文件/网络/进程权限与线程、缺动态库拒绝宿主回退、杀死namespace后无存活后代；`closure-sandbox-component.log`真实协议/生命周期1/1通过（2.28秒，沿用命令级SHA优化）。独立审核指出ioctl参数宽度、DNS输入和动态库回退问题，均已修代码；ioctl高32位实际回归、完整三生态/ABI/发行复验仍待补。 +- `closure-sandbox-unprivileged.log`失败：宿主启用AppArmor非特权user namespace限制，临时签名资产的bwrap不能配置隔离loopback。保留fail-closed；未更改主机sysctl/profile，不能把root下通过说成非root验收通过。GPU/其他平台隔离仍待支持。 +- Go sumdb已接固定官方key、h1、MVS元数据、签名树/收录/一致性与lock v3证据；首轮`closure-sumdb-first.log`6个sumdb测试通过，含官方66321505叶树的离线vector。独立审核的ZIP目录条目、无Go更新锚保留、并发更新锚回退与整锁预算四项正在修复；尚未宣布sumdb收口。新sandbox要求导致的两个exec fixture预期正在同步。 + +本轮后续复验与修正： + +- sumdb四项审查修复和exec准备夹具已完成，`closure-sumdb-final.log` **45/45**通过（1项作者联网准备默认忽略）。真实官方`github.com/google/uuid@1.6.0`及sumdb证明已由显式作者入口获取、验证并存入`target/go-managed/dependency`，日志`closure-official-go-fixture-prepare.log`；这一步不等于实际Go构建/执行，后者正在接线。npm嵌套依赖/peer环境图继续实施，拟直接更新锁格式，不保留旧格式回退。 +- 沙箱独立安全复核的五项已修并复核闭合：执行缓存写授权、目录替换竞争、native配置快照漏接、宿主loader preload、Go输出符号链接。共享mount使用`openat2`和`command-fds`安全传递FD，不新增项目unsafe。`closure-sandbox-kernel-final.log` **6/6**真实kernel通过（含ioctl截断、FD目录替换、读写/网络/进程/线程与后代终止）；`closure-sandbox-config.log` **1/1**；`closure-sandbox-runtime-final.log`协议/生命周期 **1/1**、资源 **8/8**（Rust bootstrap显式fixture1项未运行）；`closure-sandbox-clippy.log`严格lint通过。 +- 当前ABI已重建。LLVM首次9项并发验收因磁盘不足出现5项链接失败，保留失败日志`closure-sandbox-llvm-external.log`；以`--test-threads=1`复验 **9/9通过**（774.66秒），日志`closure-sandbox-llvm-external-serial.log`。真实Python/Node在只读沙箱内完成标准库与线程操作，`closure-sandbox-python-node.log` **1/1通过**；runtime external/api/postgres严格lint通过`closure-sandbox-runtime-clippy-final.log`。这些不是三生态签名release与第三方包完整验收。 +- 已按Cargo包范围清理可恢复编译缓存，保留源码、备份、输入、报告以及`target/native-runtime/runtime-sandbox.a`当前归档；没有清理其他项目。所有大型LLVM夹具后续串行执行，避免低磁盘下并发链接。 +- 官方`google/uuid@1.6.0`已完成真实沙箱内Go编译两次、资源身份相同及移走适配源码后协议执行，`closure-sandbox-official-go-worker.log` **1/1通过**（13.52秒）。工具、依赖源码与importcfg只读，输出独立授权；编译与执行均使用共享隔离入口。 +- npm安装图迁移到lock v4（不保留旧格式回退）。首轮`closure-npm-graph-tests.log` 50项通过、1项失败、1项作者准备忽略；失败是手工夹具漏填精确npm环境，已迁移。bundled父来源、同名依赖优先级、循环孤儿删除、安装膨胀预算四项均修复并经独立复核关闭。`closure-npm-graph-final.log` 解析 **56/56**、Worker安装 **7/7**、Package **11/11**通过，另13项显式环境fixture未在此组运行。npm hooks/native仍另行实施，不混计为依赖图完成。 +- 当前沙箱四profile优化archive串行重建中；编译缓存再次仅按`dever-cli/dever-core/dever-runtime`包范围清理896.9MiB，日志`closure-core-dependency-cache-clean.log`。源码、备份、author SDK/生态输入与验收报告保留。 +- 发行实查确认此前仅复制LLVM不足：Linux编译器仍依赖14个非GNU OS ABI动态库，且核心RUNPATH不会向传递依赖继承。正在将author单`llvm`字段改为`core_libraries`完整声明、核心继承RPATH及制作/签名native发行验证共用闭包检查；追加仅声明GNU OS ABI的自有chroot内真实编译验收，不把原有仅独立程序chroot成功替代核心编译证据。 +- 六平台复核：Linux ARM64作者SDK multiarch路径硬编码已修源码，尚无ARM资产/宿主验收；macOS/Windows仍缺原生可执行链接与runtime协议、发行发布、跨用户认证服务及沙箱实现，不能只称“缺机器”。本轮不把六目标对象文件生成说成六平台产品交付。跨平台实际输入及运行宿主当前均不可用，Linux后续实施继续。 + +后续已完成的定向证据： + +- Linux核心动态库闭包已完成：maker和native发行安装/解析共用校验，核心改用可继承RPATH，严格拒绝unsigned lib/hwcaps目录、缺传递库及宿主preload。`closure-core-contracts-final.log` **12/12**，`closure-core-loader-acceptance.log` 最小OS根真实check/build/移除源码运行 **1/1**（18.62秒）。不是正式公开签名或其他平台交付。 +- wheel三项独立审查修复已闭合：候选阶段使用选定wheel的已验证METADATA，统一Python prefix/安装scheme/脚本，准确模拟RUNPATH和扩展祖先RPATH。`closure-wheel-contracts-reviewed.log` external_libs **63/63**、workers **7/7**、native_release **12/12**、packages **11/11**；18项显式fixture未计入。`closure-official-python-native-wheel-reviewed.log` 官方simplejson3.20.1真实扩展、同prefix资源和独立脚本导入、沙箱执行 **1/1**（16.55秒）。sdist/PEP517/npm生命周期仍在实现。 +- 四个sandbox runtime-pack archive已构建，随后sandbox抽出了已有的preload检查供核心闭包复用；最终构建沙箱修改稳定后仍需刷新四个归档。`target/closure-current-compiler/provenance.json`保存当前私有测量编译器、四archive、系统SDK和14库的准确摘要；不是签名发行。 +- 当前LLVM双CMS构建通过，Dever **8,323,880**字节、Markdown **8,326,120**字节。`cms-llvm-128-10-02`及`cms-llvm-64-10-02`各自双源码、顺序16篇真实登录/租户/CRUD/发布均通过且无OOM，RSS峰值28.20–28.33MiB、cgroup峰值23.48–23.91MiB、启动44.6–51.9ms。顺序发布约53.6–55.6篇/秒是完整登录及发布链基线，不是并发饱和QPS或p99。 +- 发现runtime/http/live性能构建器仍调用旧Rust backend；已迁到公开LLVM项目/CMD入口并移除无其他调用方的native_fixture_builder。每组构建一个多CMD应用，报告明确共享产物和生产默认调度，拒绝旧manifest充当当前证据。5项运行器定向合同已通过，真实协议测量尚未完成。 + +最终矩阵继续： + +- `profiles-llvm-10-02/profile-report.json`四profile同CMD实际大小base **2,083,880**、SQLite **4,139,976**、PostgreSQL **2,265,224**、both **4,238,568**字节；无数据库连接。`protocols-llvm-10-02`已通过公开LLVM构建runtime/http/live，各组共享多CMD产物，三个Rust参考由当前源码runtime-pack优化构建。 +- `memory-llvm-64-10-02`五种空闲/挂起服务和`async-llvm-64-10-02`六种任务/Channel/取消路径完成，均无OOM。`http-llvm-{64,128}-10-02`共24组2秒HTTP/HTTPS测量无请求错误或OOM；目标5000req/s下实际约4889–4980req/s，p99约0.7–3.2ms，少量负载端未派发请求保留在dropped，不能称持续饱和上限。新server owner统一检查OOM，HTTP warmup/measurement复用请求错误与成功计数门；runner合同 **47/47**通过，`closure-performance-runner-contracts.log`。 +- 当前CMS LLVM/PG验收 **1/1**（56.31秒），`closure-postgres-cms-configured.log`：Owner及非Owner多角色、跨站同名角色、原会话即时撤权、两个物理租户库与同slug数据隔离通过。原始首次fixture失败是根Settings缺default，不是产品失败；仅补临时合法default后重跑。专属PG16实例只监听owned 51183端口，结束已删除本轮两基准库、停止并移回原fixture位置，root config恢复原先不存在。测试内部创建的control/tenant库均由共享fixture清理，无其他库被删除。 +- LLVM API首轮17项中16通过，上传取消fixture失败：4KiB正文在头部独立TCP frame时不足以推动16KiB分块，等待写盘不成立。改夹具发送32KiB（声明64KiB，仍不完整），保留真实断连、temp文件与affine owner零残留断言；`closure-upload-cancellation-llvm-final.log` exact **1/1**（65轮生命周期），无运行时限制改动。组合证据覆盖17项，含真实PG权限/事务与排队Job重新鉴权；另9项Job suite单独继续，未因前target失败误记通过。 +- 共享build Stage与signed rootfs边界迁移后，官方Go UUID双构建/移源执行 **1/1**（13.58秒），`closure-shared-build-go-exact.log`。`closure-shared-build-go-regression.log`最初过滤名错误只运行0项，不计通过。独立只读复核固定RO/usr、network=false、普通Worker权限不变、kill/wait与工具依赖闭包，无确认阻塞；实际PEP517/npm工具执行仍待验证。 +- 作者工具叶准备 `test/ecosystem-release/build_inputs.py` 复用已验SHA的Python/Node提取owner，固定GNU工具/sysroot与同版本headers/npm；明确记录两份linker script的受控前缀转换，来源仅用于私有作者fixture，未冒充官方发行。源码包/receipt锁v5正在接线。 +- 又清理已保存archive对应的bridge optimized缓存622MiB、已完CLI验证缓存548.4MiB，并删除四profile manifest已不引用的旧191,253,848字节debug/runtime.a（SHA70022249…，可重建）；当前四archive、私有compiler、SDK、报告与备份均保留。 +- Job独立LLVM定向 **9/9**（185.22秒），`closure-job-cancellation-llvm-final.log`：fake clock/cron、重试、事务、入参策略、CMD enqueue及超时后子任务回收与回滚通过，没有因之前API target失败漏跑却记通过。PEP517作者签名工具pack准备 **1/1**（26.50秒），`closure-pep517-author-tools-fixed.log`；实际sdist/native构建正在验收。 +- CMS增加显式可选并发读取负载，沿用真实登录cookie、查询与资源采样owner，每次校验完整列表，错误立即停止同组客户端并关闭连接；合同 **11/11**，`closure-cms-load-contracts.log`。`cms-read-llvm-{64,128}-10-02/report.json`双源码共四case，8并发、每组10秒，约 **1210.6–1322.2 QPS**、p99 **11.48–13.59ms**、峰值RSS **30.81–31.04MiB**，请求错误和OOM均零。负载端为Python闭环客户端，服务未绑单核，不能称生产容量上限或单核成绩。 +- HTTP/2单核校准 `http2-scan-llvm-64-10-02`：h2c/TLS、4物理连接×16流、3秒/档、1000/5000/15000目标。六case零请求错误/OOM且断连FD差均0；15000目标下h2c完整成功44741/45000，TLS44349/45000，后者不足99%派发成功门，因此不能直接用15k宣称持续容量，后续稳态降至10k。 +- HTTP/2正式一分钟稳态 `http2-steady-llvm-{64,128}-10-02`：单核、4连接×16流、目标10k，四case实际 **9971.6–9979.2 QPS**、成功/计划 **99.718–99.794%**、p99 **2.912–3.424ms**、峰值RSS **4.88–5.72MiB**，错误/OOM/断连FD差全部0，达到99%门。源码/编译器/产物摘要沿用当前protocols manifest；不是历史20k报告或跨主机生产容量。 +- 100轮恢复首次失败于runner每轮复制同一peer导致磁盘耗尽，保留`closure-http2-llvm-recovery.log`和未完成目录，不计通过。`stage_peer`改为同文件系统硬链接（EXDEV才独占复制），配置/日志仍独立；**48/48**合同通过。仅把确认同字节的本轮重复peer替换为共享硬链接，释放约0.8GiB，日志与报告未删；另清理完成后的Cargo缓存261.5+531.9+128.4MiB，已留存四archive/参考程序/SDK。PEP复验的磁盘失败与后续copytree/xattr真实失败均留日志，不能混记为通过。 +- HTTP/2恢复v2两种传输各100轮 **通过**，19,887完整成功、200次FD回落为0、错误/OOM为0；`closure-http2-llvm-recovery-shared.log`。TCP/WS/SSE单核64/128MiB、32/128连接、各10轮共 **12case通过**，120次断开FD差均0、完整消息及关闭计数一致、错误/OOM为0，`closure-live-llvm-capacity.log`。未运行小时级soak或外部网络故障。 +- PEP隔离hook回归 **1/1通过**（`closure-pep517-metadata-execution.log`），官方simplejson仍因原生扩展未生成失败。成功hook现在沿现有Inputs诊断边界输出有界stdout/stderr，定位到作者linker脚本将已有`/usr/lib`二次变成`/usr/usr/lib`；已修token改写且 **2/2**回归通过，已更新两生态工具叶摘要，等待重制pack及真实原生验收。只读xattr与资产执行位共享owner经独立定向复核关闭，实际C syscall probe已重建，kernel复验待运行。 +- 修正工具pack后，官方simplejson3.20.1源码经真实隔离PEP517/GCC构建出 `_speedups.cpython-312-x86_64-linux-gnu.so`，resolver/doctor与同session重放 **1/1通过**（23.55秒），`closure-pep517-native-corrected.log`。作者pack重制1/1通过27.28秒；构建C扩展不是Worker运行证明，后者及npm receipt字段最终冻结后复验仍待完成。Node runtime作者fixture已复用同一已验叶→确定性归档owner生成，`closure-npm-runtime-fixture.log`，未重取上游包或安装系统Node。 +- lock v5新字段接线后，默认四target分别 **68/7/12/11通过**（`closure-source-build-contracts.log`）；显式官方源码解析 **1/1**（25秒）及源码构建产物实际沙箱Worker **1/1**（16.58秒）通过，见`closure-pep517-{lock,worker}-final.log`。只读xattr实际kernel probe **1/1**及资产路径/执行位合同 **2/2**通过，见`closure-sandbox-{metadata-kernel,asset-modes}.log`。 +- 共享编译跨真实UID和并发缓存 **2/2通过**（`closure-managed-compilation-final.log`），两个同步客户端相同新源码收敛为同一产物。取消夹具清空环境后cc找不到ld，在开始产品验证前失败；已补作者显式`-B/usr/bin/`，待重编复验。`closure-shared-compile-measurement.json`记录私有debug核心、热OS缓存下冷请求7.318秒、命中5.564秒、并发新请求7.463/7.838秒；包括签名输入校验和IPC，不能当纯编译速度。两份4,239,368字节产物占缓存8,478,736字节，零损坏条目。 +- npm作者pack **1/1通过**（34.68秒），修正主线程fixture缺空`python_wheel_tags`，见`closure-npm-author-tools-fixed.log`。首次真实bufferutil4.0.9构建在node-gyp配置之后、make运行printf时EPERM，`closure-npm-native-first.log`；尚不算native通过。独立审查另外确认可选依赖环失败传播及预构建addon入口完整性两项，正在同一owner修复。 +- npm失败实际strace确认是make的`POSIX_SPAWN_RESETIDS`重设有效身份被拒绝，clone3原有ENOSYS回退正常。共享filter仅在process授权时允许`setres{u,g}id(-1,当前真实身份,-1)`，三个参数按内核低32位比较；其他有效身份、real/saved修改及无process情况仍拒绝。更新C probe两个capability分支 **1/1通过**（0.36秒），`closure-sandbox-spawn-identity.log`；sandbox严格Clippy通过。作者SDK另补glibc的pthread/dl/rt/util链接占位文件，修复真实链接缺`-lpthread`，没有使用宿主工具回退。 +- node-gyp输出的内部硬链接经同一output owner按(dev,ino)核对完整别名数后转为独立字节，输出外别名仍拒绝；实际正反fixture **1/1通过**（38.27秒），`closure-npm-output-alias-final.log`。可选循环失败固定点、native成功/拒绝完整分区与Worker物理裁剪均经独立复核关闭；不是JS层可绕过的dlopen补丁。Stage增加250ms周期的可见scratch目录512MiB/65536项监控，不声称内核磁盘quota。 +- 最新默认四target **70/7/12/11通过**，`closure-native-build-contracts-final.log`。官方bufferutil4.0.9源码构建/锁定/重放 **1/1**（33.60秒），`closure-npm-native-final.log`：新编译Release及内部obj.target别名、现成Linux-x64 prebuild都经runtime-only实际加载；三个Darwin/Windows变体被分类拒绝。实际hooks/optional环/bin与离线remove **1/1**（38.38秒），scratch超限终止与清理 **1/1**（18.81秒），分别见`closure-npm-{hooks,scratch}-final.log`。 +- 官方npm产物离线Worker与worker_threads **1/1通过**（19.78秒），`closure-npm-worker-final.log`；直接调用生成addon的mask/unmask，移除原adapter后从资源恢复，两次prepare身份一致，所有拒绝项都未进入运行目录。不是纯JS fallback。源码收据到最终LLVM程序的集成仍在追加验收，暂不以Worker测试代替公开run/build。 +- base优化archive刷新完成（5分03秒），通过副本+rename替换作者输入，保留历史测量compiler的原hardlink字节及SHA。仅清理已保存的base桥接缓存与debug下7条已完成测试的原生编译缓存（约91MiB，可重建）；报告/源码/备份/四profile输入未删除。其余三个archive及最终签名集成继续。 +- 最终v5收据下Python作者pack **1/1**（27.26秒）、源码解析/原生构建 **1/1**（23.97秒）、源码Worker **1/1**（16.24秒）通过，`closure-pep517-{final-author-pack,native-lock-v5-final,source-worker-v5-final}.log`。共享Stage新增budget后官方Go UUID双构建一致、移除源码与Go环境运行 **1/1**（13.81秒），`closure-go-final-build-budget.log`。 +- 共享编译取消/断连/版本租约/真实180秒deadline最终 **1/1通过**（196.27秒），`closure-managed-cancellation-final.log`；原夹具ld路径失败已经关闭，不缩短产品期限。结合先前跨UID及并发缓存两项，三项实际共享编译验收全部覆盖。 +- SQLite及PostgreSQL优化archive分别5分09秒、5分01秒刷新，both继续。CLI严格Clippy首次发现pack返回类型复杂度及npm可折叠分支，已在owner修正、等待复验，不使用allow。源码输出归档在managed Worker展开后重复嵌入的问题已在原prune owner修正，保留声明/exec/投影环境共享使用；独立复核通过，默认及最终签名运行证据继续。 +- 最终签名生态夹具已接入simplejson `_speedups.scanstring`和bufferutil原生mask/unmask，仍保留Python标准库路径、Go、三缓存条目及移源chroot。测试作者签名的是实际源码构建绑定的原runtime pack字节，未篡改收据SHA；依赖与npm输出通过公开artifact owner导入，run/build不重建或联网。此时只完成源码/独立复核，尚不计运行通过。 +- both 最终优化 archive 已完成（3分39秒），四份 SHA/大小均已核对；私有 debug runtime 清单同步刷新，历史测量 compiler 的硬链接输入保持原字节。最终默认合同 `closure-final-consumer-contracts-retry.log` **71/7/12/11通过**（101项）；新 npm 收据输出裁剪覆盖 managed、declared、exec 和共享环境，不重复嵌入已展开归档。 +- 严格 CLI Clippy 最终 `closure-final-cli-clippy-clean.log` 通过；前两次暴露测试中三处多余分配和一处多余借用，已直接修复，不加 allow。最终 author example 重新构建通过。保留最终CLI/测试/SDK到 `target/final-cli-snapshot-jiqbfz` 硬链接快照后，仅清理已完成 dev 编译缓存（Cargo报告2.4GiB，实际磁盘可用由1.2增至2.8GiB），恢复原精确可执行路径；源码、日志、历史测量产物、作者输入和备份保留。 +- 最新四profile签名/确定性/受管及移源独立执行 **1/1**（126.43秒），`closure-final-signed-profiles.log`；最小OS核心编译 **1/1**（18.16秒），`closure-final-signed-core.log`。增强三生态最终链首轮 **失败**（752.34秒，`closure-final-signed-native-ecosystems.log`）：managed三组均完成，移源后pip进入Worker前EOF，不能记为无宿主通过。 +- 根因复盘（D/E：组合覆盖缺口/隐含假设）：旧fixture把chroot当成完整namespace根，Linux会拒绝在其中创建CLONE_NEWUSER；与simplejson本身无直接证据关联。静态C最小探针实际确认chroot→EPERM、真正pivot→成功，显式作者bwrap也成功，日志`closure-{owned-chroot,owned-pivot,prepared-pivot}-ns-probe.log`。没有尝试放宽产品seccomp/namespace限制。一次性C源码已删，证据保留。 +- P0防复发：core与生态验收复用`isolated_os_command`，仅挂载自有OS根、proc与synthetic设备，安装器验收也使用同一入口；已更新toolchain/quality规范。全链复验待重编，不以探针成功替代三生态运行。仓库无对应spec模板副本；遵守用户约束，不提交或归档。 +- 正式Linux bootstrap另有实际产品缺口：原maker没有公开launcher/daemon，二者还依赖宿主libgcc_s，原最小OS core验收不覆盖它们。已开始复用既有签名与机器安装owner补独立bootstrap闭包和首次安装事务，备份`/tmp/dever-bootstrap-backup-YRnulM`。只允许owned系统根验收，不安装/重启本机全局服务。 +- Linux bootstrap实现已接入独立签名的launcher/daemon/私有库、固定systemd资产、原子信任锚与五目标恢复journal。五项独立审查修复闭合：首次失败的服务恢复顺序、最终权限/目录树同步、旧签名模板兼容、绝对可信根、公钥中断恢复；额外覆盖通用旧版本`resolve_core`消费者，保留项目固定旧编译器的能力。`closure-bootstrap-contracts-clean.log`默认 **16/16通过**，`closure-bootstrap-trust-recovery-final.log`root信任/恢复 **2/2通过**，`closure-bootstrap-clippy-clean.log`严格Clippy通过。 +- 最终maker已重编；保存精确bin/test/lib/runtime到`target/bootstrap-cli-snapshot-s5i92v`后清理可重建dev缓存，日志`closure-bootstrap-dev-cache-clean.log`（Cargo报告1.5GiB，物理可用1.1→1.8GiB）。首装实际验收`closure-bootstrap-actual-first.log` **失败**（107.92秒）：安装、旧模板升级和篡改包拒绝通过后，独立创建的客户端/daemon PID namespace令SO_PEERCRED的PID不可见。此为新夹具隔离边界不一致，产品认证不放宽；正在让客户端进入同一个owned namespace后复验。三生态pivot复验独立继续。 +- 三生态pivot首轮`closure-final-signed-native-ecosystems-pivot.log` **失败**（526.20秒）：Python受管run/build通过，npm在复制已编译程序进入缓存时遇到ENOSPC；尚未进入最终移源阶段。根盘其后仅约1.5GiB空闲。验收临时机器目录改由显式作者`target/native-release-inputs/config/setting.json`选择`/dev/shm`，输入hardlink仍留在原文件系统，产品无改动/无TMPDIR变量。下一轮复用既有performance进程/cgroup owner，以4GiB上限运行自有测试，避免挤占根盘；结果另记。 +- 临时目录合同 **17/17默认通过**（`closure-bootstrap-fixture-contracts-final.log`），严格Clippy和scoped rustfmt通过；产品dever/deverc/deverd摘要与此前冻结版本完全一致。受4GiB预算的bootstrap实际复验`closure-bootstrap-actual-final/` **失败**（184.97秒）：隔离根双项目check/run/build及缓存恰好1条通过，阶段切换时wrapper已退出而内部daemon尚持锁。峰值1,875,550,208字节、OOM为0；不是容量失败。夹具已用启动时固定的pidfd停止所属namespace，等待内部退出及同一daemon锁释放后再继续；显式停止失败终止验收，Drop不二次panic。产品认证/锁未修改，独立定向复核通过，实际重跑待完成。 + +- 三生态受4GiB预算的复验 `closure-signed-native-ecosystems-bounded/` **失败**（742.77秒）：三种生态的受管check/run/build及3条缓存断言全部通过，移除机器目录和项目源码后，Python独立程序启动Worker收到协议early EOF。预算峰值2,804,113,408字节、OOM/预算触顶均0，磁盘无耗尽；不能把此前chroot修正记为最终通过。正在对真实namespace内层启动做最小探针定位，避免反复整链构建。 +- 最小探针确认上述EOF来自内层bwrap `Can't mount proc ... Operation not permitted`：外层bwrap的只读proc子挂载令内核拒绝嵌套user namespace重建proc。新增root/test下显式静态`native-acceptance-init`，只在已经pivot的owned namespace内覆盖完整private proc后exec；保留外层原能力，内层产品cap-drop/guard不变，不绑定宿主proc。实际Rust init嵌套Python/guard探针 `closure-static-init-nested-probe.log` 通过；所有可见PID的root inode均为owned根的独立探针亦通过。修正rustix信号常量后默认 **17/17** 通过（`closure-final-namespace-contracts.log`），新静态helper实际构建通过。最新fixture Cargo产物hash已改变；误选旧hash的 `closure-bootstrap-actual-private-proc/` 主动中止，不计证据，其5个owned临时目录和cgroup已清理；复验使用当前 `native_release-c3060e189a48ed83`。 + +- Linux首次安装完整验收 `closure-bootstrap-actual-current-init/` **1/1通过**（231.51秒），报告 `target/closure-linux-bootstrap-acceptance.json`：独立信任签名首装、旧签名模板升级、篡改拒绝不损坏原入口、最小OS根双项目check/run/build、单条共享编译缓存、真实UID65533/65534使用和管理拒绝、移除项目/机器目录后的两个独立程序均通过。仅自有image目录的systemd资产，未激活全局服务；4GiB预算峰值1,512,423,424字节，无OOM或预算触顶；所有自有daemon/namespace/cgroup已退出。不是公开发行或其他平台交付。 + +- 三生态最终 `closure-signed-native-ecosystems-current-init/` **1/1通过**（755.81秒），报告 `target/closure-sandbox-ecosystem-release-acceptance.json`：三种签名runtime身份、受管check/run/build、3条编译缓存、离线源码构建收据重放，以及删除全部项目源码/机器目录后的三个最小OS独立程序全部通过。Python实际加载simplejson3.20.1 C扩展，Node加载bufferutil4.0.9 addon；本项Go是标准库协议Worker，官方UUID第三方依赖另有前述真实隔离构建/移源执行证据，不混记。4GiB验收预算峰值3,276,374,016字节，OOM/预算触顶均0；这是编译和临时文件总预算，不是应用RSS。自有进程、namespace、临时机器目录与cgroup已清理。失败日志均保留，没有改动产品sandbox来绕过夹具失败。 + +- 最终静态质量：`closure-final-namespace-clippy.log` CLI lib/bins/native_release及静态test-init **Clippy -D warnings通过**，scoped rustfmt通过。没有运行或冒充全量workspace/完整CI；独立复核确认新init仅属于测试入口，未绑定宿主proc或修改产品sandbox。 +- 结束时将当前已验收bin/test/example、私有lib和四profile runtime以硬链接保存到 `target/closure-verified-tools-ksA2Hn`，清理本轮可重建dev缓存后恢复原路径；当前native_release为 `c3060e189a48ed83`，静态init亦保留。`closure-final-dev-cache-clean.log`记录2822文件/1.8GiB逻辑容量，实际根盘可用约1.1→2.1GiB。首装报告中的launcher/daemon SHA与恢复后完全一致；源码、SDK/作者输入、报告、既有备份和无关项目未清理。 + +## 风险文件/边界 + +- `crates/dever-runtime/**`、`crates/dever-cli/**`：API、数据库和运行时集成，需避免破坏已有协议。 +- `examples/cms/**`：双源码行为一致性，删除旧流程前先保留可恢复验证点。 +- `crates/dever-build/**`、`crates/dever-cli/**`：External Lib 和共享缓存构建边界。 +- `crates/dever-backend-bridge/**`：唯一授权的底层连接 unsafe 边界、foreign buffer 所有权、LLD 串行/恢复与私有输出目录。 +- `config/setting.json` 与测试 fixture：不得引入环境变量秘密或真实凭据。 + +## 验证命令 + +优先使用离线和定向检查，例如： + +```text +cargo test --offline --locked -p dever-tests --test application_testing +cargo test --offline --locked -p dever-tests --test native_cache +cargo test --offline --locked -p dever-cli --test cms_project +deverc check examples/cms/dever +deverc check examples/cms/md +``` + +真实 PostgreSQL、跨平台发行和性能命令必须在条件具备后单独执行并记录结果；不默认运行全量 build 或集成测试。 + +2026-10-04 范围更新：用户另行授权的 Linux x86_64 → ARM64 **应用交叉构建**已在 `linux-arm64-cross-build` 任务完成，最终证据为 `target/arm64-cross/acceptance.json`，包括四profile/缓存/SQLite与完整ARM内核下的三生态Worker及HTTP。此项完成不表示ARM原生编译器、其他平台或公开发行完成，原有延期范围保留。 diff --git a/.trellis/tasks/09-29-language-production-closure/prd.md b/.trellis/tasks/09-29-language-production-closure/prd.md new file mode 100644 index 0000000..7006d7d --- /dev/null +++ b/.trellis/tasks/09-29-language-production-closure/prd.md @@ -0,0 +1,93 @@ +# Dever 语言生产化收口需求 + +## 2026-10-02 当前执行范围 + +用户最新确认先完成Linux的完整CI/长时间压力测试，以及受限AppArmor下非root第三方Worker；其他平台和公开发行暂不推进。下方保留完整产品目标，本次交付不以这些延期项阻塞已授权的Linux工作。 + +- 完整质量门覆盖当前workspace静态检查、默认测试、必要feature组合、Python运行器合同及有明确前提的Linux运行级验收；内部probe、作者准备与不可用条件必须逐项分类，禁止把跳过算通过。 +- 长压测复用现有CMS/HTTP2/连接恢复运行器，每个关键持续场景至少30分钟，覆盖64/128MiB预算、业务响应、OOM、RSS/FD趋势和退出回收；记录实际负载和限制，不把计划时长或短测说成完成。 +- 非root修复必须保留文件/网络/子进程隔离、签名资产和UID边界。不得关闭全局AppArmor限制、引入setuid绕过、依赖环境变量或系统PATH碰巧存在的语言/沙箱程序。若需要安装或加载宿主专用profile，先完成可审查的产品改动及策略，再单独取得操作确认。 +- 用户此次明确授权完整CI和上述长压测；仍只使用自有临时目录、loopback服务/数据库和受限资源，不改动现有业务服务。 + +## 目标 + +把 Dever 从“语言核心和部分生产能力已实现”收口到“可以用来开发、验证、打包并交付大型项目”的状态。收口必须建立在现有实现上，不重复实现已经完成的编译器、运行时、API、Job、Port/Adapter、测试运行器和缓存治理能力。 + +## 背景与已确认事实 + +- Dever 的语言核心、Native 编译链、主要运行时、Model/ORM、Context、API 基础、权限基础、Job、Port/Adapter、应用测试运行器和构建缓存已有定向实现。 +- 生产 API 源码大部分已经实现，但真实 PostgreSQL、真实 HTTP 组合链路和最终任务状态尚未收口。 +- CMS 目前是基础示例和部分能力验证，尚未完成大型项目级双源码验收。 +- External Lib 已有统一协议和 Worker 基础，但 pip/npm/Go 真实生态解析、运行时打包、Package 传递依赖和独立运行尚未完成。 +- 当前发行路径仍主要依赖本地 Rust/Cargo 开发工具链，正式跨平台发行、机器级共享工具链和认证缓存服务尚未完成。 +- 项目配置必须使用 `config/setting.json`；不恢复环境变量配置方案。 +- 当前租户隔离只承诺 database 模式；FFI、front 和表/字段隔离不纳入本轮收口。 + +## 范围 + +### A. 生产 API 与数据库验收 + +1. 完成生产 API 的最终合同检查，清理已被后续实现覆盖的旧任务项。 +2. 使用 `config/setting.json` 提供隔离的真实 PostgreSQL 配置和可重复验收入口。 +3. 验证权限目录、角色、跨站点同名角色、多角色、撤销权限和租户数据库隔离。 +4. 验证请求解析、响应错误映射、Cookie/Header/Context、上传、取消、事务和 HTTP/2 组合行为。 + +### B. 大型 CMS 双源码验收 + +1. 完成 Dever 和 Markdown 两套 CMS 的 Model、业务规则、权限、会话、媒体、发布、审计和 Job。 +2. 两套源码使用同一份行为合同，测试结果和关键输出保持一致。 +3. 验证 `check/test/run/build` 及 `api/worker/all` 配置模式。 +4. 删除或隔离旧 demo-only 流程、旧种子和不符合当前目录约束的代码。 + +### C. External Lib 生态 + +1. 完成统一 external Adapter/Worker 协议的生产错误、取消、超时、capability 和生命周期行为。 +2. 完成 pip、npm、Go Modules 的真实解析、锁定、缓存和运行时准备。 +3. 完成 Python、JavaScript、Go SDK 和 `Adapter -> Lib` 依赖映射。 +4. 完成 Dever Package 的传递 Lib 依赖、run/build 集成和无系统运行时执行。 +5. `run/build` 不隐式联网；所有解析和下载均由显式命令及锁文件驱动。 + +### D. 发行版与共享工具链 + +1. 完成 `deverd` 机器级共享工具链、认证 IPC、并发发布、租约、完整性校验和安全缓存边界。 +2. 支持在无 cargo/rustc 的干净机器上安装、运行和构建公开项目。 +3. 完成 Linux、Windows、macOS 的发行物、安装/升级/卸载边界和平台安全检查。 + +### E. 最终质量验收 + +1. 完成相关定向测试、全量 workspace 测试和真实 PostgreSQL 验收。 +2. 完成 CMS、外部 Lib、发行版的运行级检查。 +3. 完成二进制体积、启动时间、并发、RSS、64/128 MiB 内存压力和缓存占用报告。 +4. 在工具可安装时启用 rustfmt、clippy 和 CI 检查。 + +## 不在本轮范围 + +- FFI。 +- front 产品能力。 +- table/field 租户隔离；只实现 database 模式。 +- 通过环境变量配置 PostgreSQL 或其它运行参数。 +- 重新设计已稳定的语言语法和 API 协议。 + +## 验收标准 + +- 真实 PostgreSQL 下 API、权限、角色、租户和事务链路通过；无配置时不会连接外部数据库。 +- Dever/Markdown CMS 的同一验收矩阵全部通过，并能完成 check、test、run、build。 +- pip/npm/Go Lib 可锁定、准备、打包并通过 Worker 调用；运行不依赖宿主机对应语言环境。 +- 干净机器安装后不需要 cargo/rustc 才能执行公开 Dever 项目。 +- Linux、Windows、macOS 的发行边界和未支持项有可执行检查，不泄漏私有 Rust/Trellis 源码。 +- 全量、定向、性能和安全检查的状态均有明确记录：通过、失败、跳过或阻塞，不以计划代替证据。 + +## 既有任务关系 + +本任务是收口总任务，不复制已有实现，直接复用并收口以下任务： + +- `09-20-production-api` +- `09-20-large-cms-acceptance` +- `09-29-external-libs` 及其三个子任务 +- `09-06-private-compiler-distribution` 与 `09-28-shared-toolchain` +- `09-19-application-testing` +- `09-29-build-cache-storage` + +## 当前无阻塞产品决策 + +本轮按以上范围执行，不再等待新的产品语法或权限设计决定。若平台供应商或真实 PostgreSQL 不可用，只能将对应验收标记为 blocked/skipped，并保留可重复入口，不得伪造通过。 diff --git a/.trellis/tasks/09-29-language-production-closure/research/apparmor-nonroot.md b/.trellis/tasks/09-29-language-production-closure/research/apparmor-nonroot.md new file mode 100644 index 0000000..13de868 --- /dev/null +++ b/.trellis/tasks/09-29-language-production-closure/research/apparmor-nonroot.md @@ -0,0 +1,40 @@ +# 受限 AppArmor 非 root Worker + +2026-10-02，只读安全调查；未加载策略、改 sysctl 或安装全局程序。 + +## 已证实根因 + +旧失败 PID 4087980/4087981 的内核 audit：实际首个 exec 是私有 `ld-linux-x86-64`，创建 user namespace 后进入默认 `unprivileged_userns`，拒绝 capability 8/setpcap 与 12/net_admin，对应 `RTM_NEWADDR: Operation not permitted`。当前 `dever-sandbox/src/lib.rs` 把 bwrap 当 loader 参数，给 bwrap 路径添加 attachment 不会生效。 + +宿主 `apparmor_restrict_unprivileged_userns=1`、`apparmor_restrict_unprivileged_unconfined=0`，没有 Dever/bwrap profile；parser 4.0.1、ABI 4.0 可用。这些只证明环境条件，尚未证明专用 profile 兼容或非 root 验收通过。 + +## 最小实现 + +1. 签名 bwrap 采用静态 ELF（无 PT_INTERP/DT_NEEDED），直接 exec；内部 guard/语言继续使用私有 loader/libs。避免给动态 loader 授予 userns，也防止宿主 preload 先于可信入口运行。 +2. 受限机器 helper 固定 `/opt/dever/sandbox/<bwrap_sha256>/bwrap`，复用现有发行签名安装 owner。整条目录链 root-owned，不可组/其他用户写，无符号链接。普通 0755，不加 setuid/filecaps。独立应用只选择与嵌入签名资产摘要相同者，旧版本并存，无 PATH/env/系统 bwrap 回退。 +3. 专用 AppArmor ABI 4.0 profile 复用上游 bwrap 的 namespace 设置与 exec 子程序 deny-capabilities 堆叠，attachment 只覆盖受信目录。禁止简化为宽泛 unconfined+userns。现有 FD pin、只读挂载、grant、seccomp、后代回收保留。 +4. 作者机缺 libcap.a，需制作固定来源静态 bwrap 资产；不能要求客户机安装编译依赖。规则文件可打包/离线验证，宿主加载须管理员明确确认。 + +## 验收 + +### 安全复核补充：专用 profile 的宿主前提 + +本机 `apparmor_restrict_unprivileged_unconfined=0` 是部署阻塞：Linux 对当前 unconfined 源直接允许显式 change_profile/change_onexec，目标的路径 attachment 不能拒绝这次进入。攻击程序可直接进入允许 capability/userns 的父 profile，尚未执行下一次 exec 时不会叠加 deny-capabilities 子 profile。不能加载当前规则后再补救，也不能仅在目标规则加 deny change_profile。 + +Ubuntu 官方要求管理员启用 `kernel.apparmor_restrict_unprivileged_unconfined=1`。产品非 root 受限启动必须只读检查此前提，实际 `/` 安装在发布 `/etc/apparmor.d` 文件前也必须检查（系统 reload 可能自动加载）。离线镜像可制作，但不是宿主验收。产品不自行改全局设置；主代理完成可审查代码后单独请求临时加强保护及加载新规则的验证授权，其他CI继续。回滚时先卸载本轮 profile，再恢复原严格项，避免留下可进入的宽松窗口。 + +依据：Linux `security/apparmor/domain.c` 的 `change_profile_perms`、`profile_onexec`；https://discourse.ubuntu.com/t/understanding-apparmor-user-namespace-restriction/58007 的 profile changes 节。此结论替代“只要静态 helper+专用规则就足够”的初始假设。 + +- 限制仍为 1 的非零 UID 运行真实三生态 Worker；audit 不再进入默认 unprivileged_userns。 +- 源码/编译服务移除后独立程序仍可运行，只需匹配的受信 helper/profile 和系统 ABI。 +- 默认拒绝未授权文件/网络/fork，声明 grant 各自生效；禁止 namespace/ptrace/capability；取消/超时不留后代。 +- NNP 堆叠及直接 aa-exec 进入规则不能把 capability 带入任意程序。此项必须真实加载后验证，不能凭上游 profile 推断。 +- 缺 helper/hash 错/可写祖先/symlink/profile 缺失失败关闭；多 UID/多版本互不污染；安装恢复保留既有版本。 + +## 上游依据 + +- https://discourse.ubuntu.com/t/understanding-apparmor-user-namespace-restriction/58007 +- https://raw.githubusercontent.com/containers/bubblewrap/main/meson.build +- https://gitlab.com/apparmor/apparmor/-/raw/apparmor-4.0/profiles/apparmor/profiles/extras/bwrap-userns-restrict + +文件 owners：`dever-sandbox/src/assets.rs` 与 `lib.rs`；CLI `toolchain/packaging/sandbox.rs` 与 `bootstrap{,/transaction,/service}.rs`；SDK 作者输入说明；根 `test/dever-sandbox-tests`、`native_release`。不新增自写 namespace helper、通用代理或新签名协议。 diff --git a/.trellis/tasks/09-29-language-production-closure/research/ecosystem-closure.md b/.trellis/tasks/09-29-language-production-closure/research/ecosystem-closure.md new file mode 100644 index 0000000..d1a1b48 --- /dev/null +++ b/.trellis/tasks/09-29-language-production-closure/research/ecosystem-closure.md @@ -0,0 +1,125 @@ +# Research: External ecosystem closure + +- Query: Identify remaining pip/npm/Go implementation owners, contract changes, reusable paths, and independently executable slices. +- Scope: mixed; read-only source and primary protocol documentation; no Cargo, package installation or service tests. +- Date: 2026-10-02 +- Task: parent explicitly supplied this task directory; `task.py current --source` returned none in the child session. No task pointer changed. + +## Findings + +### Existing implementation is substantial + +Signed Python/Node/Go pack authoring, immutable runtime identities, Worker preparation, Go compilation, shared compilation and no-host-language Linux standard-library execution already exist. Latest acceptance is recorded in implement.md:660. These are not new implementation gaps. + +| Owner | Current behavior and relevant source | +| --- | --- | +| `crates/dever-cli/src/libs.rs` | `LibSpec` at 118, canonical lock v2 at 198/244, independent Worker resolution at 487, graph doctor at 1343, explicit commands at 1461. | +| `crates/dever-cli/src/libs/registry.rs` | Bounded official transport at 98; Python/npm solver at 357–628; archive locking at 725; Go MVS at 773; extraction at 1155. | +| `crates/dever-cli/src/workers.rs` | Dependency closure at 460; Python installation at 529; npm installation at 567; shared resource Tree at 620. | +| `crates/dever-cli/src/workers/runtime.rs` | Shared author/consumer runtime manifest and file-closure validation. Interpreter descriptors cannot currently declare build tools. | +| `crates/dever-cli/src/workers/go_build.rs` | Private owned staging, exact packed tools, transitive package compile/link, cancellation/deadline. `Builder::compile` at 346 includes deterministic trimpath. | +| `crates/dever-cli/src/packages.rs` | Package Lib declarations use the same `LibSpec` parser at 274, 601, 672, 953; no second ecosystem resolver is needed. | +| `crates/dever-core/src/check/ports.rs:333` | External Lib declarations receive shape checks; quoted `name[extra]@version` already fits the syntax. Semantic extras checking belongs in the CLI library contract. | +| `test/dever-cli-tests/external_libs.rs` | Injected `LocalRegistry`, real archive fixtures, exact provider metadata, independent Worker versions, canonical lock, offline remove, real loopback redirect rules. | +| `test/dever-cli-tests/external_workers.rs` | Prepared resource inspection, native Python rejection, npm hook rejection, real optional managed execution. | +| `test/go-managed/analyze/main.go` | Go build tag/source analyzer; deliberately rejects cgo/assembly/native source. Not a general Go build frontend. | + +### First independently executable slice: Python extras + +Actual missing behavior: + +- `LibSpec` has only ecosystem/name/version. Parsing name as `PackageName` rejects brackets (`libs.rs:130`). +- `python_dependencies` evaluates markers with an empty extras list and rejects dependency extras (`registry.rs:920–947`). +- The bounded solver chooses versions by base name; extras need monotone dependency activation, including newly activated extras on already-selected nodes and after backtracking (`registry.rs:445–573`). + +Smallest useful design: + +1. Extend the existing Lib identity/parser with sorted, normalized Python extra names using installed `pep508_rs::ExtraName`. Keep `name` the canonical distribution name. Serialize `pip:name[a,b]@version` through the single `key()` owner. Reject extras on npm/go, malformed or empty brackets and noncanonical serialized identities. +2. Use explicit lock format v3 if selected-extra or dependency-edge semantics change; directly migrate repository callers/fixtures. Do not add a permissive v2 compatibility path. Package source manifests can continue storing the same strings. +3. Reuse `Requirement::evaluate_markers(markers, extras)` and existing PEP440 constraints. Maintain extras per requested distribution inside each search branch; activating an extra must re-evaluate that selected distribution's metadata until stable. Do not mutate sibling branches or silently ignore unknown declared extras. +4. Preserve the exact requested extra variant on Worker roots and dependency edges. A single package version may have base and extra variants sharing the same archive. Do not union all Workers' extras globally: `resolve_workers` intentionally solves each Worker separately before merging. +5. During materialization, deduplicate identical distribution/version/artifact installations; conflicting bytes remain an error. Do not change runtime imports or introduce per-extra runtime modules. + +Important graph caveat: storing only one union-extra `LockedLib` per base package is insufficient. A plain Worker and an extras Worker could then share expanded edges, violating isolation or triggering `inconsistent resolved metadata` in `resolve_workers`. Either retain explicitly requested variants as distinct lock nodes (while selecting one version per base name per Worker) or introduce an explicit resolved-context identity. The former is the smaller extension. Existing cross-context version-selection conflicts must remain explicit, never overwritten. A `foo -> foo[feature]` relationship is not a package version cycle; avoid accidental self-edge rejection from treating expansion as an ordinary recursive package edge. + +Acceptance for this slice: + +- Normalize case/separators, duplicate extras, deterministic ordering; round-trip and malformed non-Python rejection. +- Base request excludes extra-only dependencies; extra request includes them. +- Transitive `a -> b[x]`; two parents request disjoint extras on the same b; late activation on an already-selected b; backtracking selects compatible versions without leaking failed-branch extra state. +- Different Workers requesting plain versus extra variants preserve exact roots/edges and installation closure. +- Package-provided extra declarations, lib remove/update, stale/corrupt lock rejection, offline preparation and one physical wheel installation. +- Focused default targets: external_libs, affected external_workers and packages. No interpreter/network required for resolver tests. + +### Python platform wheels and source distributions are separate slices + +`lock_python` only selects `bdist_wheel` matching signed runtime tags (`registry.rs:630–692`). Worker installation then rejects every `.so/.pyd/.dll/.dylib`, `.data/platlib` and `Root-Is-Purelib: false` (`workers.rs:537–557`). Thus tags alone do not establish native-wheel support. + +Platform-wheel slice should first make signed pack tags describe actual ABI/platform compatibility, check WHEEL identity/tag and RECORD/file integrity rules, install purelib/platlib through the existing bounded Tree and verify native dependencies against a packaged OS ABI baseline. Keep `.pth` executable hooks a distinct policy. Test an actual compatible extension and missing/wrong ELF/ABI dependencies in the no-host execution fixture. Merely allowing filename suffixes is not enough. + +Sdist requires a real PEP517 build frontend: acquire and lock source plus build-system requirements; resolve dynamic `get_requires_for_build_wheel`; run backend in a private no-network build sandbox; validate and publish the generated wheel with build provenance. `run/build` then consume only locked outputs, never call an online package manager. Existing `archive_files` supports zip and gzip tar, with bounded no-link extraction; reuse it for admitted sdists instead of unrestricted host extraction. + +The current pack has an interpreter/stdlib only and deliberately strips development inputs. It does not include setuptools/build backends, Python development headers, C/C++/Rust compiler toolchains, sysroot, or arbitrary external system libraries. Source/native builds need signed explicit build packs and locks, not discovery of host cc/cmake/rustc. A deterministic source-build lock must bind source digest, backend+build dependency closure, target/runtime ABI, selected signed toolchain, explicit args/config and output digest. Add that structured provenance to the same lock owner, not a loose per-project cache flag. + +### npm advanced dependency graph + +`npm_dependencies` rejects nonempty optionalDependencies, peerDependencies, bundledDependencies and bundleDependencies (`registry.rs:959`). Existing search and `doctor` enforce one version per name per Worker; installation is flat `node_modules/<name>` (`workers.rs:608`). + +- First add typed metadata/edge kinds and target eligibility. Optional dependencies override same-named regular entries. Optional peers are constraints when present, not auto-installed dependencies. Record omitted optional nodes with a reproducible reason; checksum/signature/protocol failures must not become optional-success fallback. +- Required peers need resolution in their consumer context. A one-version flat graph supports only the compatible subset. Full npm semantics require package-instance identity and installation path ownership, since independent branches legitimately select incompatible versions. Do not claim complete npm support by promoting every peer/optional edge to an ordinary mandatory edge. +- Bundled dependencies are already bytes in the parent tarball. Validate declared bundled package names, package.json identities and nested closures; bind them to the signed parent artifact rather than downloading or flattening them. Reject undeclared/colliding injected trees. +- Keep one extraction/publication owner. Extend lock graph and installer together, with exact instance references and safe relative installation paths if nested graphs are introduced. + +Minimal units: optional/peer metadata+graph tests; bundled archive installation tests; nested package-instance graph+layout if required; then controlled hooks/native builds. Scope claims must distinguish these. + +### npm hooks/native compilation + +`workers.rs:583` rejects lifecycle scripts and gypfile, and 606 rejects `.node`/binding.gyp. Removing these checks would execute or accept unverified install behavior without a build owner. + +Use the same isolated explicit build-artifact owner as Python. npm lifecycle scripts are shell strings, not safely interchangeable with a simplistic `split_whitespace` Node command. Supporting their existing ecosystem semantics requires a signed known shell/tool set, limited filesystem/network/process capabilities, declared Node/ABI/toolchain/headers, bounded capture and complete descendant cleanup. That internal fixed build environment must not read user PATH/env. If only direct JavaScript entry hooks are supported, name it as a restricted contract rather than general npm lifecycle support. Product user configuration stays in setting.json. + +Lock completed outputs and source/script/tool/target provenance during explicit lib preparation. Runtime preparation must not rerun hooks or consult the network. Test hostile output paths/links, timeout descendants, denied network, native ABI mismatch and cached replay. + +### Go sumdb + +Current `go_mod` fetches unverified .mod bytes, and `resolve_go` fetches ZIP then immediately calls `lock_archive` (`registry.rs:831–839`). Local archive SHA verifies cached bytes later but does not authenticate registry content. Both .mod and ZIP must be sumdb-verified before use/publication. + +Required owner: a bounded `libs/registry` sumdb module behind the existing transport, with fixed official key/origin, explicit injected fixture key/transport for tests, not GOSUMDB/GOPRIVATE environment variables. Existing ring SHA/Ed25519, sha2, base64 are available. Either implement the small checked protocol against upstream vectors or supply a signed verifier with a narrow offline API; do not invoke a host Go command. + +Full contract: Go dirhash h1 for the uncompressed logical module file tree (not raw ZIP SHA), separate go.mod h1; lookup record framing/module/version identity; signed checkpoint note verification; inclusion proofs; consistency against previously trusted checkpoints; bounded authenticated tile cache. Verified lock provenance must include enough checkpoint/record evidence or immutable digest binding for offline install. Do not trust HTTPS lookup text alone. Verify every .mod consulted for MVS, including versions not in the final build graph. Retain raw artifact SHA for Dever cache identity alongside upstream h1. + +Tests: official independent dirhash/tlog/note vectors; ZIP order/recompression same h1; changed file/go.mod rejected before publication; invalid key/signature/wrong record/inclusion/consistency/rollback; uppercase path escaping; offline verified replay; transport limits. Then one real public pure-Go package rebuilt and executed with existing signed pack. Existing cgo/assembly exclusion is a separate documented boundary, not a missing sumdb feature. + +### Order and shared contracts + +1. Python extras is executable without adding runtime/build assets or changing sandbox policy. +2. npm metadata and graph semantics can follow independently; preserve claims about nested versions. +3. Go sumdb is a security-critical independent resolver slice and needs dedicated protocol review. +4. Platform-wheel/native-loader acceptance can precede arbitrary source builds. +5. Finish shared OS-enforced build sandbox and signed tool/header/backend assets, then Python sdist/npm hooks through one owned artifact preparation boundary. +6. Finally extend the actual signed-release acceptance with pinned third-party inputs, offline reconstruction and no-host execution; do not reuse standard-library probes as evidence for third-party installation. + +## External references + +Primary sources inspected on 2026-10-02: + +- https://packaging.python.org/en/latest/specifications/pyproject-toml/ — build-system dependencies and dynamic project metadata. Build requirements belong to preparation, not runtime imports. +- https://peps.python.org/pep-0517/ — backend hooks and isolated build requirements; confirms sdist support is a build frontend, not archive copying. +- https://docs.npmjs.com/cli/v11/configuring-npm/package-json/ — npm v11 optional/peer/bundled contracts. Optional peers are not auto-installed; optionalDependencies override regular same-named entries. +- https://go.dev/ref/mod#authenticating — Go checksum database requires inclusion and consistency verification before accepting lookup data; h1 differs from ZIP archive digest. +- https://pkg.go.dev/golang.org/x/mod/sumdb/tlog — canonical transparent-log API and formats useful for independent conformance vectors. + +## Related specs + +- `.trellis/spec/backend/toolchain-and-library.md`: signed packs, no PATH/env/network fallback, resource budgets, independent Worker lifecycle, immutable cache and focused tests. +- `.trellis/spec/backend/error-handling.md`: explicit errors, no secret-bearing protocol diagnostics, owned cleanup and offline preparation failures. +- `.trellis/spec/backend/directory-structure.md`: core/CLI/runtime ownership and root test layout. +- Parent `prd.md` C/D/E and `design.md` External Lib boundary; `implement.md` records existing evidence and retained gaps. + +## Caveats / Not Found + +- Research only; no product code or acceptance was changed or executed. +- Current optional/peer/sdist policies deliberately fail closed. They are missing supported behaviors, not justification to delete validation. +- No signed general native build-tool pack or controlled package-hook executor found. Interpreter runtime packs do not provide those by implication. +- Full npm multi-version context resolution and Python extras cross-Worker identity need explicit lock semantics. A broad lock bump should cover changed semantics coherently, without speculative unused fields. +- Existing author/CLI tests are canonical reusable fixtures; public network package tests should remain explicit rather than become ordinary unit-test dependencies. diff --git a/.trellis/tasks/09-29-language-production-closure/research/ecosystem-pack-inputs.md b/.trellis/tasks/09-29-language-production-closure/research/ecosystem-pack-inputs.md new file mode 100644 index 0000000..f72b555 --- /dev/null +++ b/.trellis/tasks/09-29-language-production-closure/research/ecosystem-pack-inputs.md @@ -0,0 +1,95 @@ +# Research: ecosystem pack author inputs and acceptance reuse + +- Query: Identify available real Python/Node/Go inputs, reusable acceptance, relocation limitations, and disk costs for signed ecosystem packs. +- Scope: internal; read-only source/filesystem inspection and executable version queries; no builds, tests, downloads or services. +- Date: 2026-10-02 + +## Findings + +### Available inputs + +| Ecosystem | Explicit input | Observed identity / size | Evidence limit | +|---|---|---|---| +| Python | `/usr/bin/python3.12`, `/usr/lib/python3.12` | Python 3.12.3 (Aug 31 2026 local build), executable 8,020,928 bytes; stdlib directory 54 MiB | Distribution configured for `/usr`, not verified relocatable. | +| Node | `/root/.nvm/versions/node/v24.15.0/bin/node` | v24.15.0, 122,889,056 bytes | Actual runtime, but needs OS dynamic libraries. Whole nvm version tree is 991 MiB and includes unrelated global dependencies. | +| Go | `target/go-managed/runtime.pack` | 55,785,046 compressed bytes, 203,288,938 extracted bytes, 358 regular entries; manifest says go1.26.3 linux/amd64 | Existing private fixed-target build pack, not an independently verified published release. | + +`df -h /data/project/dever` reported 2.0 GiB free during inspection. `target/go-managed/cache` is 50 MiB; preserve it unless the parent explicitly needs reclaimable build caches. Serial acceptance avoids duplicating multiple 194 MiB extracted Go trees and signed release copies concurrently. + +Python `-I -S -B` currently reports `/usr/lib/python312.zip`, `/usr/lib/python3.12`, `/usr/lib/python3.12/lib-dynload` in `sys.path`. `sysconfig` records `--prefix=/usr`, system expat and system SSL. `readelf` shows executable dependencies on libm, libz, libexpat and libc. Extension modules add libssl/libcrypto, sqlite3, ffi, bz2, lzma, crypt, ncurses/tinfo/panel, db/gdbm and readline. Therefore copying only python3.12 is definitively insufficient for independent Python operation. Copying stdlib alone also does not close the extension native dependency set. + +Python directory contains symlinks: `_sysconfigdata__linux_x86_64-linux-gnu.py` is a sibling alias, `sitecustomize.py` points outside to `/etc/python3.12/sitecustomize.py`, and config libpython points outside the directory. Prepared author inputs must be explicit regular files; do not blindly archive this tree or copy developer sitecustomize. No isolated relocation probe was run in this research; relocating executable plus `lib/python3.12` must be verified rather than assumed. A complete stdlib claim also needs extension loading and filesystem path assertions, not just a JSON echo. + +Node `readelf` NEEDED entries are libdl, libstdc++, libm, libgcc_s, libpthread, libc and ld-linux-x86-64. There is no RPATH/RUNPATH entry. No external JavaScript stdlib directory is referenced by the current SDK/runner; only the runtime binary plus SDK/adapter/dependencies is needed for existing fixtures. This is a practical runtime input, but host-copy execution does not prove the declared OS ABI baseline or a clean machine lacking these shared libraries. + +The Go pack already bundles compiler, linker, analyzer, a complete generated stdlib importcfg and archives. Its manifest does not declare a runtime interpreter. A bounded search did not find an active Go installation at `/usr/local/go`, `/usr/bin/go`, `/usr/local/bin/go`, `/root/go` or `/root/.local`; the retained pack suffices for existing build acceptance without locating/rebuilding host Go. + +### Existing owners and tests + +- `crates/dever-cli/src/libs.rs:631`: installed registry provider reads signed `runtime/<ecosystem>/<platform>/manifest.json` and adjacent runtime.pack; verifies target, metadata requirements, pack SHA and 256 MiB compressed bound. Reuse this chain rather than introducing a second trusted-runtime loader. +- `crates/dever-cli/src/libs/registry.rs:30`: `InstalledRegistryPack` / `RegistryRuntime` own registry metadata and Python markers/wheel tags. +- `crates/dever-cli/src/workers.rs:58`: existing `dever-worker-runtime-v1` parser; `prepare_worker` at 151 validates lock identity, then archive and inner manifest. Tree bound at 17 is 256 MiB uncompressed and 65,536 files. The retained Go pack fits with about 53 MiB remaining for related generated content. +- `crates/dever-cli/src/workers/go_build.rs:27`: fixed-target build descriptor; line 452 uses environment-cleared absolute tool execution. No new Go launcher needed. +- `test/go-managed/make_pack.go:22`: existing private maker builds source selector, lists all stdlib exports, copies tools and writes sorted tar entries. It is explicitly a private fixture maker, not a signed production author tool; its Go build environment is author-side only. +- `test/dever-cli-tests/external_workers.rs:287`: ignored real Go test consumes retained pack, locked module ZIP, target build tags and go:embed, compiles SDK/runner, removes original adapter source, and runs standalone Worker with cleared environment. Asserts generated resources do not retain build/source trees. Reuse this behavioral coverage for produced Go pack acceptance. +- `test/dever-cli-tests/external_workers.rs:647`: real Node fixture run; 653 Python host fixture run; 660/677 CommonJS/ESM export binding; 691 packaged Python SDK shadow protection. Several intentionally override generated executable with a host path. These prove protocol/binding, not release relocation. +- `test/dever-cli-tests/external_workers.rs:706`: relocates an embedded Node Worker after removing source, lock and raw pack. It still compiles via old explicit Rust author helper. New signed-release acceptance should use current LLVM path rather than duplicate this old harness. +- `test/dever-cli-tests/support/daemon.rs`: shared owned-daemon lifecycle helper from native acceptance; use for signed managed execution rather than another child owner. +- `sdk/python/dever_component.py:8`: SDK imports asyncio/json/math/re/struct/sys/threading/uuid/dataclasses/typing and therefore real Python stdlib availability is required before worker startup. +- `sdk/javascript/dever_component.mjs`: checked wire/protocol implementation to embed; current generated runner in workers.rs uses `node:console` and `node:fs`. + +### Concrete acceptance recommendations + +1. Keep maker consumption deterministic and offline: prepare explicit digest-bound regular-file ecosystem inputs separately, then include registry manifest + inner runtime.pack in the same signed release manifest. Do not discover Python/Node/Go through PATH or add application environment configuration. +2. Reuse retained Go pack immediately for signed-install → lock-bound prepare → worker execution. Preserve target/stdlib contract and assert executable output only; source/compiler archives must not survive in final Go Worker resources. +3. For Node, stage only the explicit binary and existing SDK/fixtures, not the 991 MiB nvm tree. Verify CommonJS and ESM, relocation, empty environment, removed original source/pack and tampered signed artifact rejection. +4. For Python, provide an explicit prepared stdlib tree and reject symlink/escaping input; test `sys.prefix`, `sys.path` and imported modules' `__file__` all point inside the relocated runtime where applicable. Exercise asyncio plus SSL/sqlite/ctypes/compression if advertising their availability. A bare executable test can falsely pass by reading `/usr/lib/python3.12`. +5. Distinguish same-host relocation from a no-system-runtime OS image. Neither `env_clear()` nor deleting the project source hides `/usr/lib/python3.12` or host shared libraries. Do not label same-host copied interpreter checks as clean-machine release acceptance. Do not rename/hide global host directories to simulate this. + +## Related specs + +- `.trellis/spec/backend/index.md`: focused offline compiler checks and root test ownership. +- `.trellis/spec/backend/toolchain-and-library.md:173`: verified installed signed registry runtimes and explicit network boundaries. +- `.trellis/spec/backend/toolchain-and-library.md:175`: one lock-bound pack protocol, Python isolation, Go fixed-target tools, no host language lookup. +- `.trellis/tasks/09-29-language-production-closure/{prd,design}.md`: actual ecological independence remains a required outcome, unavailable evidence must be marked honestly. + +## External references + +No external browsing was performed: this topic inventories local prepared inputs and existing code only. Version numbers above came from explicit binaries or retained manifest, not current upstream release claims. + +## Caveats / Not Found + +- Sub-agent `task.py current --source` returned none because its session pointer is absent; parent explicitly supplied this output task directory, so no task selection was guessed or changed. +- No ready signed Python/Node ecosystem release pack was found in the bounded target/tmp paths inspected. +- No clean-machine container/rootfs, official upstream signature, source-build reproducibility, license aggregation or alternate target runtime was verified here. +- No modifications outside this research file; no Cargo, Go compilation, downloads, tests, service startup or destructive action performed. + +## Follow-up: shared compilation size and LLVM representation + +- Query: Are proposed 128 MiB single / 256 MiB total resources, 384 MiB request and 320 MiB compiled output enough for the real Node binary? +- Date: 2026-10-02. Read-only inspection, no measurements or tests. + +### Definite blocker beyond IPC limits + +`crates/dever-core/src/llvm/external.rs:128` emits every resource byte as a three-character `\\XX` escaped LLVM string. The 122,889,056-byte Node executable becomes 368,667,168 textual bytes (351.6 MiB) before other code. `crates/dever-backend-bridge/src/lib.rs:57` rejects IR larger than 8 MiB. Raising only IPC/cache limits is therefore insufficient. + +`crates/dever-backend-bridge/src/bridge.cpp:59` additionally copies all textual IR into an LLVM MemoryBuffer before parsing; parsing then materializes the actual constant. `worker.rs:75` imposes CPU 120 seconds, address space 2 GiB, file size 1 GiB, 128 FDs and no core dump. Raising the IR cap to accept 256 MiB of escaped resources would admit 768 MiB text plus its C++ copy, decoded request/resources, LLVM objects and object-output copies; this conflicts with the existing 2 GiB worker budget before realistic linking overhead. + +Smallest coherent direction: keep bounded textual code IR and move resource payloads to a separate dense bridge input. IR declares named external byte-array constants; the existing private LLVM bridge defines them with dense ConstantDataArray initializers before verification/emission, or emits one separately linked data object. Reuse `native::resource_inputs` digest deduplication and the exact existing symbols/descriptors. Validate missing/extra resources, symbol shape/length/collision and raw SHA at the existing owner. Do not introduce an independent archive/decoder in application runtime merely to solve compile transport. + +Compression of IPC bytes is not necessary to admit this actual Node case: base64 expands it to 163,852,076 bytes, within proposed 384 MiB. It also would not solve the current LLVM expansion. Compression would add another protocol plus decompression limits; defer unless measured transport costs warrant it after dense resource emission. Existing encoded cap remains an independent cap: 256 MiB resources consume about 341.3 MiB base64, leaving about 42.7 MiB for text/metadata; pathological escaping of the permitted 16 MiB source can exceed that, which should remain an explicit encoded-limit rejection rather than an allocation assumption. + +### Limit consumers and preserved ownership + +- `toolchain/compilation.rs:12,117,136,174,194,292`: encoded cap, resource count/total, single decoded cap, canonical encode/decode and base64 deserializer encoded bound must move together; use named constants to avoid mismatched limits. +- `toolchain/service/compile_request.rs:45`: returned executable receipt needs compiled-specific validation before allocating output. +- `toolchain/service/compile.rs:97,144`: output regular-file metadata and outgoing receipt need the same compiled-specific limit. +- `toolchain/cache.rs:293`: `publish` is trusted compiled-output owner; change only this limit, preserve 2 GiB quota and 4096 entries. +- `toolchain/cache.rs:462`: `inspect_entry` currently checks format/key and file length/hash but no compiled byte bound. Apply compiled receipt/size validation before `restore` reads entire file (`cache.rs:356`), including status and quota inspection paths. +- Keep ordinary `ArtifactReceipt::validate` (cache.rs:86), publish_artifact/artifact/artifact_path, service artifact_put/get and ArtifactPut dispatch at 64 MiB. Do not weaken generic user-upload quota to admit trusted compiled output. + +### Memory/lifecycle cleanup worth doing with caps + +`CompileRequest::new` clones resource bytes; `encode` deep-clones the full request to sort; `decode` re-encodes to verify canonical representation. The daemon retains its upload buffer, decoded request and canonical payload while running the worker; `worker.rs` retains encoded input through lowering, while `request.embedded_resources()` clones decoded bytes again. At the proposed limits these create substantial avoidable overlapping allocations. Drop encoded input after canonical decode, release daemon decoded request before long worker execution once identity/version are retained, and avoid cloning payloads merely to sort serialization. Prefer existing ownership moves/borrowed sorted references; do not add compression as a substitute for fixing live owners. + +Daemon admission allows two active compilations (`service/compile.rs:27`); service transfer reader caps upload at 60 seconds (`service.rs:580`), individual socket read/write timeout is 5 seconds, total compilation including upload/link/download is 180 seconds, and client response/download is 190 seconds. Keep these limits initially and measure real Node acceptance. CPU/address-space limits constrain the child only, not daemon/client copies. Raw object output is also copied by bridge reply ownership, so dense inputs remove a major expansion but do not by themselves prove maximum-limit memory acceptance. diff --git a/.trellis/tasks/09-29-language-production-closure/research/linux-ci-scope.md b/.trellis/tasks/09-29-language-production-closure/research/linux-ci-scope.md new file mode 100644 index 0000000..02dc3a7 --- /dev/null +++ b/.trellis/tasks/09-29-language-production-closure/research/linux-ci-scope.md @@ -0,0 +1,147 @@ +# Research: Linux CI 与持续压力验收范围 + +- Query: 根据当前源码设计完整 Linux 质量门；区分默认合同、显式验收、作者准备与内部探针；复用现有长测 runner。 +- Scope: internal；仅调查 `/data/project/dever` Rust 语言仓库，不适用 Go Dever 的 Model/Page/Service 开发流程。 +- Date: 2026-10-02 +- Task: 主代理明确指定本任务；child session 的 `task.py current --source` 返回 none，主代理确认父会话 active=in_progress，因此不创建或切换任务。 + +## Findings + +### 代码与规范入口 + +| 文件 | 用途 | +| --- | --- | +| `Cargo.toml` | 六个 workspace member、Rust 1.95/edition 2024、dev/test 禁用 debug 与 incremental、release/runtime-pack 优化 | +| `crates/*/Cargo.toml`、`test/dever-tests/Cargo.toml` | target 与 feature 真实注册来源；不能按 test 文件名猜测 Cargo target | +| `crates/dever-backend-bridge/build.rs:12` | embedded 需要 `target/backend-sdk/usr/lib/llvm-18` LLD headers/static archives 与系统 LLVM-18 | +| `test/dever-cli-tests/support/llvm_inputs.rs:4` | GNU 作者夹具的 LLVM 18、glibc 2.39、GCC 13 路径与动态依赖闭包 | +| `sdk/native-release.md`、`sdk/native-release.rs` | 四 profile、作者打包工具、签名安装与三生态验收规范 | +| `test/dever-cli-tests/native_release/acceptance.rs:120` | 正式验收读取 `target/native-release-inputs/{base,sqlite,postgres,both}.a` | +| `test/ecosystem-release/{prepare,build_inputs}.py` | 固定摘要的显式作者输入准备；不是默认联网安装器 | +| `test/dever-tests/tests/support/postgres.rs` | `{case}` 数据库模板、唯一 schema/tenant 数据库、失败清理 | +| `test/performance/{run,build,process,live,cms,settings}.py` | 已有构建、隔离进程、cgroup、协议负载、CMS、报告 owners | +| `test/external-sdk/check.py:192` | 三 SDK wire/typed 合同，必须显式给绝对解释器路径 | +| `test/dever-sandbox-tests/isolation.rs` | 正式 sandbox 测试与内部 helper probe；不能全量 `--ignored` | + +相关规范：`.trellis/spec/backend/index.md`、`quality-guidelines.md`、`directory-structure.md`、`error-handling.md`，以及 `toolchain-and-library.md` 的独立 runtime/embedded、作者输入与发行验收合同，`database-guidelines.md` 的隔离数据库合同。任务旧 PRD 仍含跨平台目标与历史“未完成”背景；本轮采用父代理转述的最新授权：完整 Linux CI/长测、AppArmor 非 root Worker；其他 OS 暂缓。历史 TODO 不是本研究的当前实现判断。 + +### 最小复用设计 + +建议只有一个根 `test/` 下的 Linux gate runner 和一个声明式阶段/ignored 用例分类表；`.github/workflows/` 仅负责安装确定工具/启动该 runner/保存日志，不复制测试矩阵。当前 `.github` 不存在。长测继续调用 `test/performance`，不另写负载发生器。每阶段保存命令、开始/结束、退出状态、实际测试数、输入指纹；失败要 fail closed，blocked/skipped 不能计入通过。 + +由 `cargo metadata --offline --locked --no-deps --format-version 1` 获取注册 target/required-features；从编译后的 libtest `--list` / `--ignored --list` 获得精确用例清单。审计分类表与清单的差集：新 ignored 必须明确分类；不能简单 `cargo test --workspace --all-features -- --ignored`。Python `unittest discover` 默认是否递归不能当覆盖证明，应分别指定现有目录。 + +### Cargo 覆盖与顺序 + +workspace 成员：dever-core、dever-runtime、dever-cli、dever-backend-bridge、dever-sandbox、dever-tests。 + +- core 只有 `reference`（启用 runtime crypto）；生产 CLI 不应启用它。 +- runtime 默认空；独立能力 `auth-crypto,crypto,wire,external,api,database,sqlite,postgres`。上层依赖的 feature unification 会掩盖最小 feature 编译错误。 +- bridge 默认空；`embedded` 与 `runtime-abi` 必须独立；其余 `runtime-database,runtime-api,runtime-external,runtime-sqlite,runtime-postgres`。`--all-features` 是合同覆盖，不是可交付 archive 的构建方式。 +- dever-tests 默认空；`reference,sqlite,postgres,api,crypto,component,external`，许多 target 有 required-features。 +- CLI：lib，deverc/dever/deverd 三 bin，llvm_cli/native_release/cms_project/shared_toolchain/external_libs/external_workers/packages 七 integration targets，native-release/native-acceptance-init 两个 `test=false` example。 +- sandbox：lib/guard bin/isolation target。bridge 有 16 个显式 integration targets；dever-tests 设置 `autotests=false`，只测 manifest 注册者。`test/runtime_logging.rs`、安全 cases 等路径还可能被其他 target `#[path]` 引入，不能据未注册就判遗漏。 +- Rust peers `network_bench/live_bench/async_bench` 是 dever-tests 自动 example；`--all-targets` 静态门检查它们，真实运行另由 Python 使用。 + +主代理统一 Cargo `-j1 --offline --locked`。建议阶段命令（并非本研究执行结果）： + +```sh +cargo fmt --all --check +cargo clippy --offline --locked -j1 --workspace --all-targets -- -D warnings +cargo clippy --offline --locked -j1 --workspace --all-targets --all-features -- -D warnings +cargo test --offline --locked -j1 --workspace -- --test-threads=1 +cargo test --offline --locked -j1 --workspace --all-features -- --test-threads=1 +cargo check --offline --locked -j1 -p dever-runtime --lib --no-default-features +cargo check --offline --locked -j1 -p dever-backend-bridge --lib --no-default-features +cargo check --offline --locked -j1 -p dever-backend-bridge --lib --no-default-features --features embedded +cargo tree --offline --locked -p dever-cli --edges features +``` + +还需按上述 runtime 八个能力逐一 `cargo check -p dever-runtime --lib --no-default-features --features <feature>`；bridge 分别 runtime-abi/runtime-database/runtime-api/runtime-external/runtime-sqlite/runtime-postgres 与两驱动组合。不能在这些最小能力命令上加 `--workspace`，否则 workspace feature 合并消除隔离意义。四个正式 profile 构建可以覆盖相应 release 组合，但不能替代 runtime-only 最小能力。生产 CLI 的 feature-tree 检查不得从 all-features 测试解析结果推出 reference 不进入生产。 + +默认与 all-features 会有意重复部分合同，这是覆盖不同配置的证据；无需再对每个 feature 重跑全套功能测试。按 target 串行执行可以降低一次留存的链接产物，仍需记录全 workspace 默认和全特性两个集合的总结果，doctest/lib/bin 不能被 integration-only 循环漏掉。 + +### 作者准备与显式运行 + +SDK 输入必须先检查存在、版本与摘要：Rust>=1.95；LLVM/LLD 18；GNU x86_64 夹具固定 GCC 13/glibc2.39 位置；sandbox bwrap bind-fd/guard/loader 闭包；三生态固定发行与源码构建夹具。`prepare.py` 不负责下载，脚本常量固定 CPython3.12.14、Node24.15.0 下载档及 Go pack 摘要；以源码当前摘要为准。现存文件不证明由当前源码构建，记录 provenance。 + +独立正式 profile 用同一个 `target/native-runtime-abi`： + +```sh +cargo build --offline --locked -j1 -p dever-backend-bridge --lib --profile runtime-pack --no-default-features --features runtime-api,runtime-external --target x86_64-unknown-linux-gnu --target-dir target/native-runtime-abi +``` + +分别增加 `runtime-sqlite`、`runtime-postgres`、两者；每次立即独立保存 archive 到 `target/native-release-inputs/<profile>.a` 后再构建下一个。禁止 all-features/embedded、把 both 复制四份、原地 strip 已硬链文件。许多 LLVM helper 另读取 `target/native-runtime-abi/debug/libdever_backend_bridge.a`，需要显式完整 runtime 测试 archive；不能以正式 profile 名义蒙混。构建 author example（sdk/native-release.md 给 miniz_oxide/flate2/sha2 opt-level3 配置）、三 CLI bin、guard、component-fixture、静态 native-acceptance-init，之后再运行验收。 + +正式 ignored 分类： + +1. bridge native LLVM targets 中依赖 runtime archive 的源码执行、资源/异步/错误/HTTP/Job/API/外部 Worker 全部属于显式运行；`llvm_database_source` 与 `llvm_api_source` 含 PostgreSQL 分支，使用包含 runtime-postgres 的 target build。不能用 object-only/非 ignored 结果替代。 +2. PG：`postgres_orm_uses_the_configured_isolated_database`、`postgres_authorization_preserves_site_scoped_roles_and_catalog`、`postgres_cms_http_isolates_two_tenant_databases`，及 LLVM API/database 的 PG ignored。用 `cargo test -p dever-tests --all-features --test postgres_orm --test postgres_api --test authorization_postgres -- --ignored --test-threads=1`，并运行 bridge 对应 targets 的 ignored。只有目标内全为验收用例时才整体 ignored。 +3. `native_release` 的四个真实链路：`acceptance::optimized_profiles_make_reproducible_signed_release_and_run_through_daemon`、`acceptance::signed_core_builds_without_host_compiler_libraries_or_tools`、`acceptance::bootstrap::signed_bootstrap_installs_and_serves_two_projects_without_host_libraries`、`acceptance::ecosystems::signed_ecosystems_build_and_run_without_system_language_installations`（最终模块名请以 libtest --list 确认）。另外 bootstrap trust/crash-recovery ignored 必须执行。 +4. `packages` 的 installed package check/test/run/build/standalone、managed_compile 的成功/不同 UID/取消清理；shared_toolchain 不同 UID；external_workers 的官方 Go/Python wheel/PEP517/npm、直接 Node/Python adapter 和独立运行；external_libs 的 managed lifecycle/越界/alias/PEP517 合同；native_cache 新鲜 CLI ignored。这些是显式验收，不是可忽略的“可选测试”。 +5. sandbox 的 `enforces_files_network_process_and_threads`、`pins_grants_against_directory_replacement`、`rejects_writable_aliases_of_executable_inputs`、`prepared_python_and_node_run_without_host_files_or_process_creation`、`enforces_unprivileged_namespace`、`terminates_detached_descendants_on_kill`、`rejects_missing_dependency_before_host_loader_can_supply_it`、`kernel_rejects_namespace_ptrace_and_truncated_ioctl_requests` 精确执行。C syscall probe 需提前显式编译至 fixture 指定位置。 +6. `performance` 两个 native kernel/large-map 基准、`concurrency` 调度基准是单独 bounded perf；真实长压测在下节，不互相替代。 + +不要直接运行内部 probe：isolation 的 `pinned_grant_probe/restricted_probe/tree_probe/descendant_probe/granted_probe`，packages 的 `installed_managed_package_worker_guard`。它们由父用例在签名/namespace 环境内 `--exact --ignored` 调用。 + +作者 preparation 用例单独列状态，避免无意联网与重复下载：external_libs wheel 官方 fetch、sumdb 官方 proxy fetch、build.rs/native-build-inputs、npm_build.rs/hashed 输入组装与官方源码构建；它们准备下游执行需要的固定锁定输入。已有内容摘要与合同匹配则复用，但真实 registry 解析/官方源码构建若属于本轮 CI 要求，应在显式 author 阶段执行一次，不能因为下游 fixture 已存在就声称该阶段通过。 + +### Python、SDK 与 PostgreSQL 假通过防线 + +```sh +python3 -B -m unittest discover -s test/performance -p 'test_*.py' -v +python3 -B -m unittest discover -s test/ecosystem-release -p 'test_*.py' -v +python3 -B test/external-sdk/check.py --python /absolute/python --node /absolute/node --go /absolute/go +``` + +performance 的 network/live 测试会启动自有 loopback；必须事先准备 release peers，并在根 setting.json 的 `performance.network_peer/live_peer` 配置路径。SDK checker 没给解释器时打印 unavailable 后继续，甚至三个都没给仍可零退出；CI 必须强制三个路径并断言三生态结果，不只检查退出码。 + +Rust PG helper `load_setting` 强制 `database.postgres_test` 和数据库名中恰好一个 `{case}`，拒绝 query dbname 覆盖；schema 用唯一 PID/time/counter，tenant 用本轮唯一前缀并仅删除规范正整数 ID。schema fixture 不创建基础数据库，需提前准备 `postgres_orm/authorization_postgres/llvm_database` 等每个调用 case 所需数据库；tenant fixture 会创建控制库和租户库，配置角色需要对应权限。全部连接必须指向本次自有 PostgreSQL instance。owned cluster 在 runner finally 停止并清理；失败原始信息与清理错误都要保留。 + +性能 `build.load_postgres_setting` 没配置会返回 None，run orm 自动退化为 SQLite-only；CI PG 阶段先检查配置，报告必须包含 postgres driver。其 Python URL validator 没有 Rust helper 的 query dbname 禁止规则，推荐 fixture 创建者只输出固定安全 URL，不接受任意外部 override。 + +### 长压测：可直接复用的命令与界限 + +构建 `run.py build --selection protocols` 准备 protocols，`--selection cms-profiles` 准备双源码 CMS/四 profile；ORM 需 all 构建并已配置 PG。Rust peers 单次 release 构建后复制指纹，run 阶段不运行 Cargo。记录 `application_configuration=llvm-cmd-v1`、`peer_configuration=setting-json-v1` 与所有实际二进制 hash；旧 manifest 被拒绝。 + +下面每组分别在 memory=64、128 执行，output 用新的独占目录；`CGROUP` 必须是已经委派且有 memory controller 的自有父组。每个子场景是真实持续至少1800秒。CPU affinity 从允许 CPU 集合选，server/client 不重叠；没有隔离不声明独占 CPU。 + +```sh +python3 test/performance/run.py run --artifacts "$ARTIFACTS" --output "$OUT_HTTP" --suite http --implementations dever --duration 1800 --warmup 5 --rates 1000 --paths /plain,/json,/bytes --repeats 1 --concurrency 16 --connection-mode reuse --memory-mib "$MEMORY" --cgroup-parent "$CGROUP" +python3 test/performance/run.py run --artifacts "$ARTIFACTS" --output "$OUT_H2" --suite http2 --implementations dever --duration 1800 --warmup 5 --rates 1000 --paths /plain,/json,/bytes --http2-connections 1 --repeats 1 --memory-mib "$MEMORY" --cgroup-parent "$CGROUP" +python3 test/performance/run.py run --artifacts "$ARTIFACTS" --output "$OUT_RECOVERY" --suite http2-recovery --implementations dever --duration 30 --cycles 60 --recovery-rate 1000 --recovery-connections 1 --repeats 1 --memory-mib "$MEMORY" --cgroup-parent "$CGROUP" +python3 test/performance/run.py run --artifacts "$ARTIFACTS" --output "$OUT_LIVE" --suite live --duration 30 --cycles 60 --connection-tiers 32 --repeats 1 --interval 0.1 --memory-mib "$MEMORY" --cgroup-parent "$CGROUP" +python3 test/performance/cms.py run --manifest "$CMS_MANIFEST" --output "$OUT_CMS" --concurrency 16 --read-seconds 1800 --memory-mib "$MEMORY" --cgroup-parent "$CGROUP" +``` + +1000 req/s 是保守起点而非宣称性能阈值；先独立短时 calibration，长测要求 errors=0、OOM=0、非零成功，建议 >=99% scheduled 成功作为该次目标负载验收，失败需降低目标后重新完整跑并保留失败数据。不要把历史 20k 短测当当前机器长测门槛。 + +runner 当前 profile lifetime=3600000ms（build.py:121），live duration 限制0.3..60、cycles<=100（run.py:1161）；所以 live 不能传1800，可用60×30秒的真实连续同一进程生命周期。HTTP2 recovery 同理，周期总量需小于1h。HTTP/H2 各两种 transport×3 paths，64/128 合计各6小时；recovery 两 transport 两预算约2小时；live 三协议两预算约3小时；CMS 双源码两预算约2小时。以上完整路径矩阵约19小时加准备；若只选一个代表 body 路径，HTTP/H2合计缩为4小时，总约11小时，但必须显式记录未长测另两路径，短合同测试继续覆盖它们。 + +`run --suite cms` 只顺序发布16篇，与 --duration 无关；长测用独立 cms.py 的 read-seconds。`memory` parked fixture 只有3秒窗口，`async` 与 ORM CRUD/list/cursor/stream/pool 是有限操作，给 duration1800也不是长测。ORM 的 HTTP entry 使用 duration 可测30分钟，但有限项需在同一进程内重复 workload 才证明长期资源稳定；单纯外层重复短进程会掩盖泄漏。现有 CMS postgres-case 是验收，不自动成为长压测。若用户要求这些也长测，需在现有 runner owner 上增添持续模式，不冒称现有参数已覆盖。 + +资源/失败判定：run.py server context 最终检查 cgroup OOM；HTTP/2 `observe_disconnected` 要求 FD 精确恢复；live.py 验证 phase 顺序、连接计数、payload 和 heartbeat，run.py:627 当前只记录 FD/RSS delta，不强制 delta=0。必须加报告 gate 或局部 owner 校验 live FD 恢复。RSS 保存每周期基线、断开后值、末段范围/趋势；allocator 稳定保留不等于泄漏，不要求绝对回到首次 RSS，但持续增长、OOM、资源无法回收应判失败。完整进程组 terminate/reap、cgroup 清空/删除、失败也保留 cases.jsonl/report 已完成部分；还应验证控制器缺失明确失败、错误 payload 失败、超时/中断清理的测试。`status=complete` 不单独代表无错误。 + +### 2.1 GiB 空间下的串行策略与耗时 + +只读 df 已确认根分区可用2.1GiB。单个 Cargo -j1 控制编译并发但不限制产物总量；不要承诺完整 all-targets 全量链接可装进2.1GiB。优先共享当前 target/dependency cache，先静态，再默认/全特性按 target 执行并保存结果，只清理本轮可重建的精确输出，不能删作者输入、其他任务缓存或整个 target。任何安全清理计划由主代理解析实际路径/所有权后实施。 + +原生 ABI 四次 release 编译共享一个 target，保存成品后再切 feature；大三生态/安装验收串行且不和 Cargo/PG压力同跑。sdk/native-release.md 已支持 `target/native-release-inputs/config/setting.json` 的 `machine_temporary_root`，只移动 owned image/machine；可选较大且可执行的文件系统。若用 /dev/shm，必须同时预算真实 RAM/该进程 cgroup，不能把磁盘问题变成宿主 OOM。报告只需小量 JSON/日志；live 0.1s采样每30分钟约18000点，多 raw 副本占用需计入预算。run.py stage_case_binary 对 Dever 每case复制，settings.stage_peer 对 immutable peer用硬链；不要假定所有 binaries 都已零复制。 + +耗时只能估计：已准备依赖的静态/合同门数十分钟至数小时；冷四profile fat-LTO每个数分钟到数十分钟，必须实际记录；三生态重打包/源码构建数分钟至数十分钟，临时空间数GiB；19小时长测是按明确场景串行计算的下限，不能用并行负载污染CPU测量或拿超短烟测替代。新增 runner 可提供阶段选择用于重跑失败阶段，但 full gate 必须验证所有必选阶段拥有同一源码/输入版本的通过证据。 + +### 主代理补充限制：总2 CPU / 4GiB + +`process.py:47` 的 Budget 接受任意显式 parent，在其下建唯一子组，并设置 memory.max/swap.max/cpu.max；`Process:105` 只把自有被启动 child迁入所给Budget。现有 server 的预算仅覆盖服务，HTTP/live负载端没有server Budget，默认继承 Python runner 的cgroup。因此支持统一上层限额，但必须由外部CI入口先把 runner/作者准备/Cargo/自有PG都放入共同的4GiB/2 CPU父组，不能只传server的 `--cpu-quota 2` 冒称全局2 CPU。 + +推荐结构：owned parent设置 `memory.max=4294967296`、swap.max=0、`cpu.max=200000 100000`，空父组启用memory/cpu controllers；其下 `control` 放CI/Python/Cargo/负载端，自有 `postgres` 放PG，再传该空父组给performance让其创建server64/128MiB兄弟组。这满足cgroup v2 no-internal-process约束；不要把runner放在已启用domain controller且还要建子组的父组本身。所占CPU可在允许affinity里挑2个，server/client各1个，总quota仍统一2。PG与server受父级内存合计限制，/dev/shm作者验收同样计入4GiB；必须预估并保留headroom，不能假设搬到tmpfs无限可用。 + +主代理报告格式门已实际失败，`target/linux-ci-initial-format.log`有1174处Diff位置（不是1174个文件）。这是待实施的格式修复前置证据；本research未亲自执行格式化，不把其当通过。主代理可将HTTP2正式稳定负载选5000/s并先做短calibration；长测仍需完整30分钟，不能宣称该速率是吞吐上限。 + +## Caveats / Not Found + +- 本研究未运行任何 Cargo、编译、测试、服务、下载或 Git 命令；上述命令是实现建议，不是执行证据。 +- 系统 AppArmor 实际策略与非 root Worker 修改由另一工作流负责；现有 `enforces_unprivileged_namespace` 在root时降为65534，只证明当前 host策略下该夹具的执行，不等价正式部署 profile 的路径覆盖。禁止以 root pass替代非root；禁止通过全局关限制获得绿色。 +- CI 配置当前不存在；作者/fixture paths有明确 Linux GNU x86_64/GCC13/glibc2.39 依赖，不能把普通ubuntu-latest或跨平台可用当事实。 +- ignored 全量精确模块名应由实际 libtest inventory确认；本文件列源码 owners和明确名称，不替代机器可审计清单。 +- PG、SDK缺配置、finite runner duration未生效、内部probe被错误执行、仅all-features验证profiles，是主要 false-pass 风险。 diff --git a/.trellis/tasks/09-29-language-production-closure/research/native-pack-production-inputs.md b/.trellis/tasks/09-29-language-production-closure/research/native-pack-production-inputs.md new file mode 100644 index 0000000..5eb58f6 --- /dev/null +++ b/.trellis/tasks/09-29-language-production-closure/research/native-pack-production-inputs.md @@ -0,0 +1,71 @@ +# Research: native runtime pack 制作与 profile 裁剪 + +- Query: 既有制作入口是否仅 fixture；四 profile 的真实 archive 制作、裁剪、复用及本机资源边界。 +- Scope: internal；只读源码与本机工具元数据，不构建、不测试、不下载、不研究签名/CLI owner。 +- Date: 2026-10-02 + +## Findings + +### 文件与当前证据 + +- `test/native-runtime-pack.rs:1` 明示 private Linux author fixture；main 仅调用 `llvm_pack::prepare`。 +- `test/dever-cli-tests/support/llvm_pack.rs:56` 的 `native_inputs` 固定引用 `target/native-runtime-abi/debug/libdever_backend_bridge.a` 和本机 glibc/GCC CRT/static libs；没有 Cargo 调用。 +- 同文件 `document`（88–101）把 base/sqlite/postgres/both 全映射至同一个 `runtime.a`。`prepare`（107–150）只是检查输入、独占创建目录、硬链接并记录 SHA/大小；不是四 profile 发行构建器。`compiler_library`（153）硬链接本机 LLVM 动态库。 +- `target/debug/runtime/linux-x86_64/manifest.json` 当前 runtime.a 为 **191253848 bytes**，摘要 `7002224967cfbc3609b304f964e911d2f03fe204d5b58c07e90dc35d71c4b84c`。源码 archive 的 stat 相符；本次未重复计算其 SHA。此 archive 为三硬链接之一，严禁原地 strip，否则会修改现有 fixture。 +- `crates/dever-backend-bridge/Cargo.toml:9` 输出 rlib/staticlib；17–25 已有独立 features，无需新 runtime crate。 +- `crates/dever-backend-bridge/build.rs:1` 只有 embedded feature 才编 C++/链接 LLVM；runtime-only archive 必须禁用 embedded。私有 SDK 位于 `target/backend-sdk/usr/lib/llvm-18`（12），LLD 静态库、LLVM18/z 动态库、zstd 静态库是 compiler 侧依赖（39–44）。 +- `crates/dever-runtime/Cargo.toml:11` 已有 api/external/sqlite/postgres 独立 feature owner；SQLite bundled engine 固定于 workspace dependency，不能依赖宿主动态 SQLite。 +- `Cargo.toml:46` release 为 fat LTO、codegen-units=1；dev/test debug=0、incremental=false。现有 debug archive 不能靠 strip 冒充 release 优化。 + +### 四 profile 的最小 feature 映射 + +| profile | bridge features（每次只构建 bridge lib） | +| --- | --- | +| base | runtime-abi,runtime-api,runtime-external | +| sqlite | 上述 + runtime-sqlite | +| postgres | 上述 + runtime-postgres | +| both | 上述 + runtime-sqlite,runtime-postgres | + +profile 的区别只在真实数据库 driver。base 不是仅 runtime-abi：否则缺失 API/external 应用能力。证据：ffi.rs:4319 的 Worker 实现由 runtime-external 门控，ffi.rs:5196/5333 等应用路径由 runtime-api 门控，ffi.rs:5476–5477 按两个数据库 feature 声明 driver 能力。不要在 workspace-wide build 中无意通过 CLI dev-dependencies 合并 embedded/全数据库 features。 + +建议作者构建步骤（未执行）：用显式 Cargo/Rust 工具路径，`cargo build --offline --locked -p dever-backend-bridge --lib --release --no-default-features --features <mapping> --target x86_64-unknown-linux-gnu --target-dir <shared-build-dir>`，每次将结果复制为独立 profile archive 后再构建下一个。四个 profile 共用一个 Cargo target，避免四份依赖树；不用环境变量/PATH作为产品配置。 + +### 裁剪与可重复制作的最小交付 + +1. 作者工具接受显式输入：目标、四份真实 release archive、CRT/static lib 清单及工具路径；记录工具版本、输入摘要与制作参数。将 profile 映射保存在单一配置表，manifest 仍使用既有 v1 协议。 +2. 对**新副本**进行确定性 archive debug 裁剪；本机 GNU strip 支持 `--strip-debug --enable-deterministic-archives -o <new-output>`。不要使用 `--strip-all`，它移除重定位/符号，会损坏可继续链接的 archive。不要凭文件名任意删除 Rust/C ABI object member；真正 driver 裁剪由 Cargo features 完成。 +3. 保留 ABI 导出、链接符号及故障来源字符串；检查普通 archive、目标架构与输入完整性，生成每个 profile 独立 runtime 路径。可共享真正相同的 CRT/libs，但不可拿同一 all-features archive 冒充四包。 +4. 对同一批输入重复制作两次并比对文件/manifest 摘要，证明“pack 制作确定性”。若要声称从源码可重复，还须固定完整 Rust/CRT/SDK 版本、构建路径映射等并实际重建比较；仅重复复制不证明源码 bit-reproducibility。 +5. 将完成后的字节集合交给既有 release 签名 owner（主线程负责），签名之后不得再 strip。定向验收应覆盖四档链接/执行、driver 分离、缺失/篡改及安装后消费；没有实际 PG 服务只能证明 postgres 构建/链接，不能声称真实 PG 执行通过。 + +复用点：`sha256_file` 已为流式摘要 owner（llvm_pack.rs:5）；现有 runtime v1 manifest/校验及 release v1 签名闭包由主线程复用，不新造协议；fixture 的 CRT 顺序与 archive 清单可作为 Linux GNU 输入模板，但其绝对路径与硬链接行为不能直接当通用发布接口。 + +### 对作者工具是否亲自调用 Cargo 的建议 + +**不必让签名/制作器亲自调 Cargo 才算完整交付**。更小且边界清楚的链路是：一个有真实执行证据的统一 `runtime-pack` Cargo profile + 四条固定 feature 构建命令，输出四份真实 archive；作者 maker 只读 `<author-root>/config/setting.json` 的明确输入、验证/裁剪副本、生成 manifest 并交既有签名 owner。如此制作器不依赖源码树、Cargo 缓存或编译工具，不把“签名已准备资产”与“编译源码”混成一个职责。构建配方与制作配置共同构成可重复链，文档应区分两层，不能仅交付 maker 而省略真实四 archive 构建。 + +若当前验收明确要求“一条作者命令从源码生成四包”，才给 maker 增加独立 build 子流程；不要通过扫描 PATH 或环境变量选择工具。现有需求是可重复制作/裁剪/签名，未见必须单命令从源码构建的约束。显式 Cargo/Rust/C 工具输入可属于作者构建配方；避免在产品 setting 中新增这些实现细节。 + +统一 runtime-pack profile 应继承 release 的 fat LTO/单 codegen unit，保持 debug 信息策略明确。先实际构建一次检查 archive/link/尺寸，再决定是否需要额外 strip 步骤；不为缩小时间或磁盘而悄悄降低既有生产优化合同。共享 target + 顺序构建四档即可，不需要泛化跨平台构建框架。 + +### 本机现有资源（只读检查） + +- `/root/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/bin/{cargo,rustc}` 存在；rustlib 只发现 x86_64-unknown-linux-gnu 标准库，没有其他目标 std。 +- `/usr/bin/{ar,strip}` 是 GNU Binutils 2.42；另有 `/usr/bin/x86_64-linux-gnu-objcopy`。其 strip help 列出确定性 archive 支持；未验证对新 release archive 的实际处理结果。 +- `/usr/bin/llvm-ar*`、`llvm-strip*`、`llvm-objcopy*` 和 `/usr/lib/llvm-18/bin` 不存在；私有 SDK 有 LLVM/LLD headers/static libraries，不能由此推定已安装完整跨平台裁剪工具。 +- `/usr/lib/llvm-18/lib/libLLVM.so.1` 为 123215144 bytes；当前 fixture 指定 glibc 2.39、GCC13。正式交付须明确该 Linux GNU 工具链基线，不能声称兼容所有 Linux。 +- `target/native-runtime-abi` 约1.3 GiB；文件系统可用718 MiB。只有主线程可构建/清理；研究未移动或删除任何产物。 + +### 相关规范 / 外部参考 + +- `.trellis/spec/backend/index.md`、`directory-structure.md`、`error-handling.md`、`quality-guidelines.md:5`（共享 Cargo target、release优化、定向验收）。 +- `.trellis/spec/backend/toolchain-and-library.md:7–19`（Rust≥1.95、feature边界、runtime v1与不可伪称六目标发行）。 +- 当前任务 `implement.md` 的 2026-10-02 节明确下一步是正式可重复裁剪 packs，现有签名 fixture 不代表正式资产。 +- 外部网页未访问；工具选项证据直接来自本机 GNU strip 2.42 `--help` 和 ar/strip `--version`。 + +## Caveats / Not Found + +- 子代理运行 task.py current --source 返回 none；输出路径使用父代理明确给出的 Active task，未猜测/改动任务指针。 +- 检查了 shemic-dever SKILL.md；实际仓库 AGENTS.md 明确这是 Rust 语言编译器，不适用 Go Model/Page/Service 规则。 +- 未找到正式四 profile maker；现有入口清楚标记 fixture。本结论不覆盖主线程正在编辑的文件。 +- 本次没有 Cargo/build/test、下载、全局安装、git 操作及真实发布；没有验证 release archive 大小、跨平台运行、签名真实性或源码重建一致性。 diff --git a/.trellis/tasks/09-29-language-production-closure/task.json b/.trellis/tasks/09-29-language-production-closure/task.json new file mode 100644 index 0000000..55ec6c9 --- /dev/null +++ b/.trellis/tasks/09-29-language-production-closure/task.json @@ -0,0 +1,26 @@ +{ + "id": "language-production-closure", + "name": "language-production-closure", + "title": "Dever 语言生产化收口", + "description": "完成 Dever 语言从大型项目可用到可验收、可复用外部生态和可发行产品的收口工作", + "status": "in_progress", + "dev_type": null, + "scope": null, + "package": null, + "priority": "P0", + "creator": "Antigravity", + "assignee": "Antigravity", + "createdAt": "2026-09-29", + "completedAt": null, + "branch": "main", + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [], + "parent": null, + "relatedFiles": [], + "notes": "2026-10-03本轮Linux CI及受限AppArmor非root Worker已收口。source7310a568/e40b7f16：native1791040913590327459为185/185，sandbox1791040885539823828为9/9，Python1791040504277745773为92/92，inventory108目标/217分类通过。宿主报告target/linux-ci-apparmor/1791040870463419164/result.json通过且完整恢复；4种负面探针均实际被内核拒绝，签名三生态pip/npm/go在真实UID65534独立执行全部通过。只修父子libtest计数和std mpmc晚初始化TLS缓存的测试误报，固定20/20复验与真实泄漏/错误返回负面回归通过。恢复4个原测试文件并排除2个新增回归可精确重算90245dd9，产品与固定输入未改；保留默认651/651、全功能873/873、静态、19组合、三SDK、PG7/7、性能3/3的原指纹证据，不冒称同新指纹全阶段重跑。最终队列1小时3分28秒，预算内、OOM0；自有unit/cgroup/PG退出，helper及AppArmor文件/profile移除，sysctl恢复strict=0/userns=1。临时根config/setting.json核验后移到target/linux-ci-session-BrZ5t3/setting-final.json，恢复根配置原先不存在。CMS64/128及HTTP2-64长测0错误/0 OOM；HTTP2-128中止、recovery/live64/128未跑，用户免跑且未标通过。父任务因其他平台/公开发行暂缓继续in_progress，不提交不归档；本轮Linux交付无剩余阻塞。", + "meta": {} +} diff --git a/.trellis/tasks/10-04-developer-onboarding/check.jsonl b/.trellis/tasks/10-04-developer-onboarding/check.jsonl new file mode 100644 index 0000000..2f6fc4f --- /dev/null +++ b/.trellis/tasks/10-04-developer-onboarding/check.jsonl @@ -0,0 +1 @@ +{"file": ".trellis/spec/backend/quality-guidelines.md", "reason": "定向离线验证与临时目录边界"} diff --git a/.trellis/tasks/10-04-developer-onboarding/design.md b/.trellis/tasks/10-04-developer-onboarding/design.md new file mode 100644 index 0000000..99043d1 --- /dev/null +++ b/.trellis/tasks/10-04-developer-onboarding/design.md @@ -0,0 +1,9 @@ +# 设计 + +- `skills/` 为可独立复制/后续建 Git 的 skill 根目录，包含 SKILL.md、按需语言/安装参考、共享模板资源。名称 dever-language，避免与 shemic-dever Go skill 混淆。 +- CLI 复用该目录的模板字节，生成器拥有文件系统发布边界；新项目使用 app + cmd API + 零输入测试，不生成 main/package/expose 或无用业务分层。 +- new 属于非项目命令，机器启动器使用活动核心；模板与发行核心版本一致。 +- skill 纳入发行签名清单并由版本库管理。安装和更新沿用现有机器权限、锁、暂存、校验和恢复机制；可发现路径必须随活动版本切换，不要求每个项目复制或手动更新技能。 +- 明确区分独立 skill 安装、机器 Dever 安装、应用依赖准备。官方发行源内置为 GitHub shemic/dever-main Releases，不读取业务配置或环境变量。Git 仓库和公开二进制资产区分，未发布资产时明确报错。 +- 安装到 AI 的稳定 skill 入口先执行绝对机器启动器的 `skill path`，再加载活动版本的完整 skill。继续保留唯一 active-version journal 提交点，避免核心和 skill 双指针更新撕裂。 +- 实施前保存涉及源码的自有备份；并行任务按文件归属分工，Cargo 验证由主代理串行执行以控制磁盘。 diff --git a/.trellis/tasks/10-04-developer-onboarding/implement.jsonl b/.trellis/tasks/10-04-developer-onboarding/implement.jsonl new file mode 100644 index 0000000..e70eb0d --- /dev/null +++ b/.trellis/tasks/10-04-developer-onboarding/implement.jsonl @@ -0,0 +1 @@ +{"file": ".trellis/spec/backend/toolchain-and-library.md", "reason": "CLI、发行与共享机器契约"} diff --git a/.trellis/tasks/10-04-developer-onboarding/implement.md b/.trellis/tasks/10-04-developer-onboarding/implement.md new file mode 100644 index 0000000..90f8546 --- /dev/null +++ b/.trellis/tasks/10-04-developer-onboarding/implement.md @@ -0,0 +1,34 @@ +# 实施记录 + +状态：实现与定向验证完成，准备提交并推送两个用户指定的仓库。 + +1. CLI 新建项目与共享模板。 +2. 独立开发 skill、安装/更新引导和语言参考。 +3. 发行包携带 skill，机器安装与同步更新集成。 +4. 定向测试、真实模板 CLI 验证、失败保护和独立 skill 校验。 + +当前已确认：机器 update 存在，仅消费本地 downloads/latest；尚无 new 和发布 skill。当前磁盘约 213 MiB 可用，禁止并行 Cargo 或无关构建。 + +## 已完成 + +- CLI `new <new-directory> [--markdown]`、单一模板资源和两种等价源码格式；新目录原子发布，拒绝已有目录/文件/链接，覆盖并发失败保护。 +- 独立 dever-language skill，含语言/架构/接口/数据/权限/租户/库/Markdown/安装参考，项目 AI 指引，官方源安装脚本。SKILL 元数据校验通过。 +- core 主仓库和 skills 独立仓库采用用户指定 SSH remote；主仓库固定 skill submodule。官方更新地址内置 GitHub Releases，已撤销放入配置文件的早期提议。 +- 发行 maker 将完整 skill 加入签名清单；`skill path [project-root]` 按活动/固定版本只读校验；`skill install` 安装永久薄入口，不单独维护可失配的技能版本指针。 +- 官方下载先验签，限制源/重定向/大小/归档成员，校验完整包后发布；更新和安装不读取业务配置或环境变量。损坏包保留旧活动版本。 +- Python 首装助手复用已有 bootstrap，官方资源制作工具发布三个确定性资源，Linux NOREPLACE 防止覆盖并发目标。测试仅在自有目录，首装脚本的最终 bootstrap 为 mock，未触碰宿主安装/服务。 + +## 验证 + +- Rust new_project：4/4；release_source：7/7（含自有 loopback 下载/坏签名/坏内容）；shared_toolchain 定向：3/3（完整版本切换、坏skill、真实降低UID读取）。 +- native_release 两项 maker 定向：2/2（缺skill/超限拒绝，确定性完整签名）。共16项Rust定向通过。 +- Python安装/资产定向：8/8；包含独立临时密钥签名、拒绝缺skill、归档坏尾/重复/链接/unsigned、镜像配置/清理和发布竞态。 +- 独立按skill执行普通/Markdown项目14项new/check/fmt/test/run/build/独立程序命令全部成功；各1个应用测试通过，4次原生执行JSON为code=0、data=Hello, Dever。主代理再次生成Markdown项目并原生run得到相同结果。 +- rustfmt已应用；定向clippy对lib、core/launcher、新项目/共享工具链/发行测试通过 -D warnings；skill validator通过。 +- 未跑全量CI、压测、真实数据库或全局安装；未发布正式二进制资产，不能将源码push当作发行完成。官方GitHub两个仓库初始为空，core仓库已确认public。 + +## 本机保留 + +- 改前源码备份：target/onboarding-before.Vgoy97/source.tar.gz。 +- 新官方发行公钥仅公开pub，私钥留在被忽略的本机目录；已验证PKCS#8 v2符合maker并匹配skill公钥，不打印/提交私钥。 +- 为有限磁盘验证临时移走的旧CLI缓存将归档至同一backup目录，可恢复。未删除业务数据。 diff --git a/.trellis/tasks/10-04-developer-onboarding/prd.md b/.trellis/tasks/10-04-developer-onboarding/prd.md new file mode 100644 index 0000000..3dca818 --- /dev/null +++ b/.trellis/tasks/10-04-developer-onboarding/prd.md @@ -0,0 +1,16 @@ +# Dever 开发入口 + +用户已授权实施：独立 `/data/project/dever/skills`、开发指引、新项目模板、`dever new`、安装引导以及 `dever update` 同步更新语言和 skill。 + +## 验收 + +- skill 根目录可独立分发，名称 `dever-language`，不依赖本机源码绝对路径，不混入 Go 框架规范；指导 AI 安装、编写、检查、测试和构建 Dever 应用。 +- `dever new <new-directory> [--markdown]` 生成最小可运行 CMD 项目、配置、项目 AI 指引和定向应用测试；两种源码格式共享语义，无数据库、鉴权或第三方工具前提。 +- 新项目目录不得覆盖已有内容；机器启动器创建新项目不读取尚不存在的项目配置。 +- 语言和 skill 共用版本发行、完整性校验及更新失败保护；支持一次安装后持续获取活动版本的 skill。 +- 应用配置只来自 `config/setting.json`，不用环境变量。用户已指定官方核心仓库 `git@github.com:shemic/dever-main.git` 和独立 skill 仓库 `git@github.com:shemic/dever-main-skills.git`。发行地址是工具内置元数据，禁止写入业务项目配置。 +- 定向验证模板 check/fmt/test/run/build、安装/更新后 core 与 skill 一致、损坏输入不破坏旧版本。仅使用自有临时目录/安装镜像，不安装全局服务、不替换全局命令。用户已授权两个指定仓库提交并 push；skill 是独立 Git，核心使用 submodule 固定对应提交。 + +## 现有缺口与边界 + +复用 CLI、共享机器版本库、签名发行作者、既有 bootstrap 安装和项目命令分派。当前没有 new；update 仅选择 downloads/latest 的核心，不交付 skill。普通应用配置、语言语义、Go front、其它平台发行、全量 CI 和压测不在本任务范围内。 diff --git a/.trellis/tasks/10-04-developer-onboarding/task.json b/.trellis/tasks/10-04-developer-onboarding/task.json new file mode 100644 index 0000000..ea86e04 --- /dev/null +++ b/.trellis/tasks/10-04-developer-onboarding/task.json @@ -0,0 +1,26 @@ +{ + "id": "developer-onboarding", + "name": "developer-onboarding", + "title": "Dever 开发 skill、新项目模板与统一更新", + "description": "实现独立 Dever language skill、可验证新项目模板、dever new，以及沿用共享机器安装的 core/skill 成套更新和安装指引。", + "status": "in_progress", + "dev_type": null, + "scope": null, + "package": null, + "priority": "P2", + "creator": "Antigravity", + "assignee": "Antigravity", + "createdAt": "2026-10-04", + "completedAt": null, + "branch": "main", + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [], + "parent": null, + "relatedFiles": [], + "notes": "", + "meta": {} +} \ No newline at end of file diff --git a/.trellis/tasks/archive/2026-09/00-bootstrap-guidelines/prd.md b/.trellis/tasks/archive/2026-09/00-bootstrap-guidelines/prd.md new file mode 100644 index 0000000..408d0d1 --- /dev/null +++ b/.trellis/tasks/archive/2026-09/00-bootstrap-guidelines/prd.md @@ -0,0 +1,19 @@ +# Bootstrap project guidelines + +## Scope + +This repository is a Rust workspace for the Dever language compiler, runtime, +CLI and repository-root tests. `AGENTS.md` and the actual implementation are +the source of truth for `.trellis/spec/backend/`. There is no frontend package +or UI source in this repository; frontend scaffolding from `trellis init` is +not applicable and must not prescribe hypothetical conventions. + +## Completion + +- [x] Document compiler/runtime/CLI ownership and real source examples in + `.trellis/spec/backend/directory-structure.md`. +- [x] Document compiler contracts, ORM, errors, logging, toolchain and focused + checks in `.trellis/spec/backend/`, backed by source and tests. +- [x] Remove unused frontend template specs; no frontend rules are claimed. +- [x] Keep the backend index in sync and verify no template placeholders remain + in applicable backend specs. diff --git a/.trellis/tasks/archive/2026-09/00-bootstrap-guidelines/task.json b/.trellis/tasks/archive/2026-09/00-bootstrap-guidelines/task.json new file mode 100644 index 0000000..97d53f0 --- /dev/null +++ b/.trellis/tasks/archive/2026-09/00-bootstrap-guidelines/task.json @@ -0,0 +1,28 @@ +{ + "id": "00-bootstrap-guidelines", + "name": "00-bootstrap-guidelines", + "title": "Bootstrap Guidelines", + "description": "Fill in project development guidelines for AI agents", + "status": "completed", + "dev_type": "docs", + "scope": null, + "package": null, + "priority": "P1", + "creator": "Antigravity", + "assignee": "Antigravity", + "createdAt": "2026-09-04", + "completedAt": "2026-09-18", + "branch": null, + "base_branch": null, + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [], + "parent": null, + "relatedFiles": [ + ".trellis/spec/backend/" + ], + "notes": "Rust compiler workspace; frontend template specs are not applicable.", + "meta": {} +} \ No newline at end of file diff --git a/.trellis/tasks/archive/2026-09/09-09-markdown-source/check.jsonl b/.trellis/tasks/archive/2026-09/09-09-markdown-source/check.jsonl new file mode 100644 index 0000000..f93cbc8 --- /dev/null +++ b/.trellis/tasks/archive/2026-09/09-09-markdown-source/check.jsonl @@ -0,0 +1 @@ +{"file": ".trellis/spec/backend/quality-guidelines.md", "reason": "Focused checks"} diff --git a/.trellis/tasks/archive/2026-09/09-09-markdown-source/design.md b/.trellis/tasks/archive/2026-09/09-09-markdown-source/design.md new file mode 100644 index 0000000..d3b4bf3 --- /dev/null +++ b/.trellis/tasks/archive/2026-09/09-09-markdown-source/design.md @@ -0,0 +1,17 @@ +# 设计 + +## 归属与复用 + +源码格式和 Markdown 内容区间由 core 拥有。使用 CommonMark 解析库识别真实顶层围栏，避免自造解析器误执行列表、HTML 或外层示例内的伪代码。新依赖仅用于编译器，不进入最小运行时。 + +SourceFile 保留原文，词法 token span 使用原字节位置。Markdown 提取不能让缓存、诊断、运行故障指向合成文件。词法/格式化共用程序区间；语义包路径接受 `.dever.md`，重复检测沿用声明注册。 + +## 格式化 + +先解析全包，验证完整声明不跨围栏。复用 AST 格式化方法，按块选择包头/声明/注释生成内容，原位替换内部文本。不能合并代码块或移动说明。CLI 快照检查与 staging 原样复用。 + +## 分工与契约 + +主代理负责 CLI 文案、实际 CLI 验证、示例、文档、规范与任务记录。实现代理负责 core/加载/解析/格式化、必要依赖和 core 定向测试。完成后独立审查，主代理复核关键证据。 + +程序围栏须顶层、闭合，信息字符串为 `dever`；无标记及其他语言只展示。每个类型、函数分句和 package 头完整位于一块内。若不支持缩进围栏，应明确诊断，不静默改变归属。 diff --git a/.trellis/tasks/archive/2026-09/09-09-markdown-source/implement.jsonl b/.trellis/tasks/archive/2026-09/09-09-markdown-source/implement.jsonl new file mode 100644 index 0000000..a710497 --- /dev/null +++ b/.trellis/tasks/archive/2026-09/09-09-markdown-source/implement.jsonl @@ -0,0 +1 @@ +{"file": ".trellis/spec/backend/index.md", "reason": "Compiler conventions"} diff --git a/.trellis/tasks/archive/2026-09/09-09-markdown-source/implement.md b/.trellis/tasks/archive/2026-09/09-09-markdown-source/implement.md new file mode 100644 index 0000000..065c391 --- /dev/null +++ b/.trellis/tasks/archive/2026-09/09-09-markdown-source/implement.md @@ -0,0 +1,24 @@ +# 执行与验证 + +用户已在双格式方案后确认并选择 `.dever.md`，无需重复请求实施授权。 + +- [x] Core：程序块识别、原文位置、加载与包路径、共用解析和格式化。 +- [x] 定向测试：提取/拒绝边界、中文换行定位、混用/重复包/契约、格式化保留说明和幂等。 +- [x] 示例和 CLI：结算示例、空源码提示、实际 check/fmt/run/build 的隔离定向验证。 +- [x] 文档规范：语言说明、实现说明、过时草案、AGENTS 源码规则。 +- [x] 检查：相关 Rust 格式、定向 Clippy/Markdown/formatter 测试及独立复核。 + +`/root/.cargo/bin` 不在 PATH 时只设置命令级 PATH。持久测试放根 `test/`，临时程序归测试所有；无服务或网络操作，无全量/集成测试，无提交。仓库大部分文件原本未跟踪，diff 以本轮前快照为准。 + +## 完成证据（2026-09-09） + +- `cargo test --offline -p dever-tests --test markdown_source --test formatter --test syntax_and_format`：45 项通过（12 + 11 + 22）。主代理已检查原始日志 `/tmp/dever-markdown-checks.log`。 +- 同目标 `cargo-clippy clippy --offline ... -- -D warnings` 通过；CLI 单独 `-p dever-cli --bin deverc -- -D warnings` 通过。10 个修改 Rust 文件的定向 rustfmt 检查通过。 +- 实际 CLI 对 `examples/markdown` 的 check/fmt/run 通过，输出 `customer = 小明`、`subtotal = 47.5`、`discount = 4.75`、`payable = 42.75`。普通 `.dever` 计价包被 Markdown 包调用。 +- 根 test/ 下临时程序验证：无 rustc 时 check 仍可用、两种格式的 API 相同、普通 README 忽略、fmt --check 不写、第二文件错误阻止所有改写、CRLF/说明/围栏保持及幂等、run/build、PATH 不含工具链时独立运行、已有输出不覆盖、重复包 C002、C005 精确到 Markdown 第 11 行第 20 列、运行除零定位 Markdown 第 11 行。临时文件和二进制已清理。 +- 最终 core 的实际 CLI 裸 CR/缩进围栏验证通过：中文原文 C005 第 10 行第 31 列、格式化无新增 LF、说明保持、check 和 fmt --check 通过。首次手写列号预期少算一列，按原文核对修正，非实现缺陷。 +- 独立只读审查无阻塞。补充验证包头前后独立注释块、跨块 Bool 分句、同一文件各块 LF/CRLF/CR、围栏/正文保持和幂等通过。 +- 新编译器依赖 `pulldown-cmark = 0.13.4`（关闭默认特性）及三个传递依赖已获取并锁定；不进入运行时，验证全部离线。复用已有 target/verification-tools 中的 rustfmt/Clippy，未安装全局组件。 +- 更新过时标题语法草案、当前说明和旧主任务源码格式限制。28 个实现/文档变更文件空白检查通过；未修改现有无关删除项。 + +无未解决阻塞。未运行全量、集成或服务测试，未提交 Git；开发工作区改动保留供审查。 diff --git a/.trellis/tasks/archive/2026-09/09-09-markdown-source/prd.md b/.trellis/tasks/archive/2026-09/09-09-markdown-source/prd.md new file mode 100644 index 0000000..a347b29 --- /dev/null +++ b/.trellis/tasks/archive/2026-09/09-09-markdown-source/prd.md @@ -0,0 +1,25 @@ +# Dever Markdown 程序源码支持 + +## 目标与确认决定 + +用户希望用 Markdown 组织易读的 AI 程序，继续由 Dever 类型与契约约束执行。2026-09-09 用户批准同时保留 `.dever` 和 Markdown 源码，并明确选择 `.dever.md` 后缀。 + +- `.dever` 行为兼容；`account/user.dever.md` 对应 `account.user` 包。 +- 标题、说明、表格不参与程序定义；只有标为 `dever` 的顶层围栏代码块属于程序，按顺序处理。代码保留完整 Dever 包、类型和函数声明。 +- 一个文件拥有一个包；格式可跨包混用，同包重复必须报错。 +- 复用现有语法、语义、契约、API 与原生编译路径。 +- 诊断和运行故障定位到原 `.dever.md` 文件及准确行列。 +- `fmt` 只格式化程序代码块，保留其它 Markdown 和围栏；整批检查后逐文件原子写入规则不变。 + +## 验收 + +1. 只加载 `.dever`/`.dever.md`，普通 `.md` 中的示例忽略。 +2. 同一程序两种格式的执行结果一致；混用包可调用，重复包、类型错误、分句缺失、失败义务仍拒绝。 +3. 多代码块、中文、不同换行、注释和原生故障位置正确；普通示例、嵌套围栏、引用、列表与 HTML 内的伪程序不执行。 +4. 程序围栏闭合，各块包含完整顶层声明（允许纯注释或空块）；无程序块、跨块声明报错，避免格式化移动代码跨说明。 +5. 格式化只更新代码块内部、幂等；语法或程序结构错误时无文件改写。 +6. 提供可读的结算 Markdown 示例，更新语言说明和过时草案。 + +## 范围与验证限制 + +不增加新语言语法、浏览器运行时、编辑器扩展或多语言执行。不改无关文件，不提交或安装全局命令。只跑离线定向检查和自有临时程序，不跑全量、集成测试或现有服务。开发依赖获取明确记录，最终程序编译保持离线。 diff --git a/.trellis/tasks/archive/2026-09/09-09-markdown-source/research/source-flow.md b/.trellis/tasks/archive/2026-09/09-09-markdown-source/research/source-flow.md new file mode 100644 index 0000000..10a93c1 --- /dev/null +++ b/.trellis/tasks/archive/2026-09/09-09-markdown-source/research/source-flow.md @@ -0,0 +1,8 @@ +# 源码事实 + +- source.rs 只收集 `.dever`，保存原文、SourceId 和行开始位置；符号链接全部拒绝。 +- lib.rs 的 parse_uncached 共用 lexer→parser；lexer 当前读取 SourceFile.text()。 +- check.rs 的 check_package 固定 `.dever` 路径；注册阶段已有重复包检测。 +- format.rs 从 token 注释和 AST 生成整个包；CLI 先准备全部字符串，再快照核验和逐文件原子替换。 +- check/cache.rs 键含源码全文/路径，文档改动仍需重新建立准确诊断位置。 +- pulldown-cmark 提供带偏移的块事件：https://docs.rs/pulldown-cmark/0.13.4/pulldown_cmark/struct.Parser.html 。当前未缓存，新增这一个编译器依赖用于避免不完整的 Markdown 自制解析器。 diff --git a/.trellis/tasks/archive/2026-09/09-09-markdown-source/task.json b/.trellis/tasks/archive/2026-09/09-09-markdown-source/task.json new file mode 100644 index 0000000..a45f193 --- /dev/null +++ b/.trellis/tasks/archive/2026-09/09-09-markdown-source/task.json @@ -0,0 +1,26 @@ +{ + "id": "markdown-source", + "name": "markdown-source", + "title": "Dever Markdown 程序源码支持", + "description": "保留 .dever，新增 .dever.md 源码加载、原文诊断、保留文档的格式化与定向验证", + "status": "completed", + "dev_type": null, + "scope": null, + "package": null, + "priority": "P2", + "creator": "Antigravity", + "assignee": "Antigravity", + "createdAt": "2026-09-09", + "completedAt": "2026-09-09", + "branch": "main", + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [], + "parent": null, + "relatedFiles": [], + "notes": "", + "meta": {} +} \ No newline at end of file diff --git a/.trellis/tasks/archive/2026-09/09-13-http2-ceiling-soak/check.jsonl b/.trellis/tasks/archive/2026-09/09-13-http2-ceiling-soak/check.jsonl new file mode 100644 index 0000000..9c0f7cd --- /dev/null +++ b/.trellis/tasks/archive/2026-09/09-13-http2-ceiling-soak/check.jsonl @@ -0,0 +1,3 @@ +{"file":".trellis/spec/backend/error-handling.md","reason":"检查错误、OOM、计数和进程退出都显式传播"} +{"file":".trellis/spec/backend/quality-guidelines.md","reason":"核对 CPU 隔离、测量矩阵、最小测试和结果边界"} +{"file":".trellis/tasks/09-13-http2-ceiling-soak/research/client-bottleneck.md","reason":"检查实现没有引入重复负载器或错误分位数聚合"} diff --git a/.trellis/tasks/archive/2026-09/09-13-http2-ceiling-soak/design.md b/.trellis/tasks/archive/2026-09/09-13-http2-ceiling-soak/design.md new file mode 100644 index 0000000..4cba455 --- /dev/null +++ b/.trellis/tasks/archive/2026-09/09-13-http2-ceiling-soak/design.md @@ -0,0 +1,70 @@ +# 设计 + +## 1. 最小改动边界 + +本阶段只扩展 `test/performance` 的负载进程配置和测量文档。现有 Rust `network_bench` 已具备多线程 runtime、固定 HTTP/2 连接拓扑、速率调度、完整响应校验和延迟直方图，因此不修改 Rust 负载器，更不修改生产运行时。 + +多进程客户端会产生多组独立直方图。只合并 p99 会得到错误分位数，输出全部 4096 个桶再聚合又会扩大协议和实现范围。一个多线程进程已经能把请求 worker 分布到多个 CPU，同时保留单一调度和精确的进程内直方图，因此采用该方案。 + +## 2. CPU 与 worker 合同 + +`Process` 的亲和性参数从单个 `cpu` 改为可空的 `cpus` 集合，并在 fork 后、exec 前一次性调用： + +```text +os.sched_setaffinity(0, cpus) +``` + +服务端仍把 `--server-cpu` 转为单元素集合。运行器将 `--client-cpu` 直接迁移为 `--client-cpus`，接受无重复的逗号分隔非负整数；新增 `--client-workers`，默认 1。运行前集中校验： + +- 所有指定 CPU 必须属于当前进程允许集合； +- 服务 CPU 不得出现在客户端 CPU 集合； +- 正式压测显式使用 `--client-workers 4 --client-cpus 1,2,3,4`。 + +不强制 worker 数等于 CPU 数，避免把合理的欠订阅/超订阅变成运行器错误；报告必须同时记录两者，使结果可解释。HTTP `load()` 和 `live.load_live()` 都把 `BENCH_WORKERS` 设置为同一个 `args.client_workers`，并把同一个 CPU 集合交给 `Process`。 + +## 3. 数据流与复用 + +```text +run.py 参数与集中校验 + -> Process(cpus=client_cpus) + -> BENCH_WORKERS=client_workers + -> 现有 network_bench 单进程 Tokio runtime + -> 现有 H2 连接/请求槽、调度、校验、Histogram + -> 现有 load JSON、进程采样、case/report 汇总 +``` + +`Process` 继续只负责子进程生命周期、亲和性和采样；它不解释 server/client 角色。`run.py` 负责参数合同与服务/客户端隔离。`live.py` 只消费已经校验的参数，不新增一套 CPU 解析逻辑。 + +## 4. 测量阶段 + +所有正式场景只测 Dever，服务绑定 CPU 0、1 worker、1 CPU quota、swap 0；客户端绑定 CPU 1–4、4 workers，位于服务 cgroup 之外。使用现存 `typed-orm-build-v2` 指纹产物和 16 流配置；先验证其 HTTP/2 二进制、TLS 文件和清单，不重建生产产物。 + +### 饱和扫描 + +- 预算：128 MiB +- 路径：`/plain` +- 传输：h2c、TLS+h2 +- 拓扑：16x16（256 请求槽） +- offered rate：30000、45000、60000、90000 req/s +- 每 case：1 秒预热、5 秒测量、2 个 repeat + +若 90000 仍未出现平台，则只报告“扫描上界不足”，不继续无界提高负载。观测平台取成功 QPS 不再随 offered rate 近似线性增长的区间，不把容量丢弃计为服务错误。 + +### 60 秒稳态 + +从扫描得到的 h2c/TLS+h2 较低平台值向下取整到不高于 80% 的明确整数 rate，确保两种传输使用相同负载。128 MiB 和 64 MiB 各运行 h2c/TLS+h2 一个 60 秒 case。99% 成功率门槛只用于证明选定 rate 可持续，不用于定义服务极限。 + +### 连接恢复耐久 + +128 MiB 下，h2c/TLS+h2 各在同一服务进程运行 100 轮；每轮 16 条物理连接、每连接 16 流、0.1 秒、1000 req/s，随后使用既有 0.3 秒断开观测。每轮校验握手、请求错误和 FD 回落，整组检查 cgroup OOM 与 RSS 序列。 + +## 5. 风险与解释 + +- 多线程负载器仍有单一速率调度协程。如果扫描中客户端 CPU 未扩展、dispatch expired 显著而服务端未满 1 核，本阶段应如实判定压测器仍受调度器限制；是否实现分片调度另建任务，不能伪称服务端平台。 +- 客户端和服务端仍共享同一虚拟机，只是 CPU 集合隔离；缓存、内存带宽和宿主机噪声仍然共享。 +- 64 MiB cgroup 的文件页计费受热缓存影响，继续同时报告进程 RSS/PSS 和 cgroup peak，不以任一单值替代另一个。 +- 100 轮是分钟级重复回收证据，不是小时级泄漏或生产网络稳定性证明。 + +## 6. 回退 + +改动不涉及生产代码。若 CPU 集合支持造成回归，回退 Python 运行器、测试和文档即可；已有构建产物和历史报告不修改，新测量始终写入新输出目录。 diff --git a/.trellis/tasks/archive/2026-09/09-13-http2-ceiling-soak/implement.jsonl b/.trellis/tasks/archive/2026-09/09-13-http2-ceiling-soak/implement.jsonl new file mode 100644 index 0000000..63e1bee --- /dev/null +++ b/.trellis/tasks/archive/2026-09/09-13-http2-ceiling-soak/implement.jsonl @@ -0,0 +1,3 @@ +{"file":".trellis/spec/backend/directory-structure.md","reason":"保持 Python runner、进程采样和 Rust 协议负载的职责边界"} +{"file":".trellis/spec/backend/quality-guidelines.md","reason":"遵守性能测量公平性、资源预算、报告和最小验证要求"} +{"file":".trellis/tasks/09-13-http2-ceiling-soak/research/client-bottleneck.md","reason":"复用已有多线程能力并避免不可合并的多进程分位数"} diff --git a/.trellis/tasks/archive/2026-09/09-13-http2-ceiling-soak/implement.md b/.trellis/tasks/archive/2026-09/09-13-http2-ceiling-soak/implement.md new file mode 100644 index 0000000..034d3ce --- /dev/null +++ b/.trellis/tasks/archive/2026-09/09-13-http2-ceiling-soak/implement.md @@ -0,0 +1,32 @@ +# 执行计划 + +## 1. 启动与上下文 + +- [x] 用户确认后运行 `task.py start`；启动前不修改基准源码。 +- [x] 使用 `trellis-before-dev` 重新加载 backend 规范，核对当前 diff、磁盘、CPU、cgroup 和正式构建产物；`build-http2-final` 已不在工作树，改用参数满足需求的 `typed-orm-build-v2`，仍须校验指纹。 + +## 2. 客户端并行配置 + +- [x] 将 `Process(cpu=...)` 收敛为 `Process(cpus=...)`，所有调用者一次性迁移；保持亲和性只作用于自有子进程。 +- [x] 在 `run.py` 增加 CPU 集合解析、集中分配校验、`--client-cpus` 和 `--client-workers`，删除旧 `--client-cpu`。 +- [x] 让 HTTP/HTTP2 与 live 客户端共同使用相同 worker 数和 CPU 集合；更新 report environment/arguments。 +- [x] 清理重复解析与分支，确保进程生命周期、运行器合同和协议负载职责仍分离。 + +## 3. 定向验证 + +- [x] 扩展 `test_runner.py`，覆盖 CPU 集合解析、重复/越界/重叠拒绝、子进程实际亲和性和报告配置。 +- [x] Python 语法解析与 runner 34/34 通过；HTTP/2 产物指纹校验通过，未运行全量测试。 +- [x] `http2-ceiling-smoke-09-18`：h2c/TLS+h2 各 30/30 成功、16/16 握手、FD 差 0、OOM 0，报告确认 4 workers、CPU 1–4、客户端 5 线程。 + +## 4. 有界正式测量 + +- [x] `http2-ceiling-scan-09-18`：128 MiB、16x16、30k/45k/60k/90k、h2c/TLS+h2、5 秒、2 repeats 共 16/16 完成，错误/OOM/FD 差为 0。 +- [x] 30k 档 h2c/TLS+h2 吞吐约 29.9k/29.8k；45k 以上 h2c 约 34–40k、TLS+h2 约 37–39k，服务端接近 1 核、客户端 1.4–2.0 核，请求槽丢弃主导，调度过期很少。28k 的 60 秒首轮分别只有 98.68%/97.91% 成功；24k 的 10 秒校准 TLS+h2 只有 99.18%。20k 的 10 秒校准分别为 99.96%/99.92%，据此将正式共同稳态输入定为 20,000 req/s（低于较低观测平台的 80%）。 +- [x] `http2-ceiling-steady-128m-20k-09-18` 与 `http2-ceiling-steady-64m-20k-09-18`：h2c/TLS+h2 各 60 秒，成功率依次为 99.77%/99.92% 和 99.94%/99.80%，吞吐约 19.95k–19.99k req/s；全部零错误/OOM、FD 差 0。28k 首轮两项不足 99%，保留该报告为探索证据。 +- [x] `http2-ceiling-recovery-09-18`：128 MiB 下 h2c/TLS+h2 各 100/100 轮，分别成功 9,944/10,000 请求，余下 56 次/组为调度过期；每轮至少成功 95/98 个，16 次握手、FD 差 0，请求错误/OOM 为 0。首末轮断开后 RSS 为 4,845,568→5,357,568 B / 5,959,680→6,647,808 B，完整逐轮序列保留。 + +## 5. 收尾 + +- [x] 更新 `test/performance/README.md` 与 backend 性能规范中的 CLI、测量结果和限制；HTTP/HTTPS 30/30、TCP/WS/SSE 16/16 最小相邻入口回归通过。 +- [x] 检查了入口到子进程的 CPU/worker 传递、重复、职责、命名和无关改动；未改生产运行时或覆盖历史报告，空 frontend 模板是单独已归档的 bootstrap 任务。 +- [x] `trellis-check`：runner 定向单测 34/34、Python AST、构建清单指纹、扫描/稳态/恢复报告计数与 OOM/FD/握手不变量、空白检查均通过；HTTP/HTTPS 与 TCP/WS/SSE 最小回归通过。未运行全量测试、Cargo build、rustfmt/clippy；未提交 Git。 diff --git a/.trellis/tasks/archive/2026-09/09-13-http2-ceiling-soak/prd.md b/.trellis/tasks/archive/2026-09/09-13-http2-ceiling-soak/prd.md new file mode 100644 index 0000000..71e4082 --- /dev/null +++ b/.trellis/tasks/archive/2026-09/09-13-http2-ceiling-soak/prd.md @@ -0,0 +1,42 @@ +# HTTP/2 饱和边界与耐久压测 + +## 目标 + +解除现有 HTTP/2 压测客户端的单核限制，在服务端固定 1 CPU、128 MiB/64 MiB、无 swap 的条件下，测量 Dever HTTP/2 小响应的本机饱和边界、60 秒稳态表现和重复连接恢复表现。结果用于判断后续是否存在值得单独优化的生产热路径，不把本机 loopback 数字外推为跨机器或生产环境上限。 + +## 已确认事实 + +- `network_bench` 已使用 Tokio 多线程 runtime，并通过 `BENCH_WORKERS` 控制 worker 数；HTTP/2 请求调度、响应校验、计数守恒和延迟直方图已有单一实现。 +- `test/performance/run.py:258` 把负载端 `BENCH_WORKERS` 固定为 1，`test/performance/process.py:104` 只允许子进程绑定一个 CPU。此前 30000 req/s 的 Dever 4x16 场景中，服务端和客户端均约占 0.9 核，因此现有数据不能区分两端上限。 +- 已构建清单允许 32 条物理连接、每连接 16 流、任务容量 272；16x16 需要 272 个服务任务，正好落在已有容量内，无需重新配置生产运行时。 +- Rust 负载结果只有一个进程内的完整直方图。拆成多个客户端进程后，现有 p50/p95/p99 摘要无法无损合并；使用一个多线程负载进程可以保留现有统计合同。 +- 当前机器允许 CPU 0–4；正式测量可将服务绑定 CPU 0，将负载端限制到 CPU 1–4。现有构建产物生命周期为 3600 秒，足以完成单个 60 秒场景和 100 轮恢复场景。 + +## 需求 + +- 将基准进程的单 CPU 亲和性扩展为明确的 CPU 集合；服务端继续使用单 CPU，客户端可使用多个互不重复且不与服务 CPU 重叠的 CPU。 +- 运行器新增明确的客户端 worker 数和客户端 CPU 集合参数，统一用于 HTTP、HTTP/2 和长连接负载；报告记录实际 worker 数和 CPU 集合。 +- 仓库内调用者一次性迁移到新的客户端 CPU 集合参数，不保留旧 `--client-cpu` 别名或兼容分支。 +- 多线程客户端继续复用一个 `network_bench` 进程、一套请求调度和一组原始直方图；不新增协议实现，不增加多进程结果拼接器。 +- 使用现存 `typed-orm-build-v2` 指纹产物（原 `build-http2-final` 已不在工作树，HTTP/2 配置仍为 32 连接、16 流、272 任务），先对 Dever `/plain`、h2c 和 TLS+h2、16x16 拓扑扫描多个 offered rate，区分客户端限制、请求槽限制和服务端饱和。 +- 根据扫描结果选择不高于观测平台 80% 的整数 offered rate，分别完成 h2c 和 TLS+h2 的 60 秒稳态测量；128 MiB 与 64 MiB 服务预算均覆盖，服务 CPU 固定为 1 核，swap 为 0。 +- 在同一 Dever 服务进程上对 h2c 和 TLS+h2 分别完成 100 轮 16x16 建连、请求、断开，逐轮保留计数、FD 和 RSS 回落数据。 +- 所有正式 case 必须保留请求计数、延迟、服务端/客户端 CPU 与内存、cgroup peak/OOM、握手数和断开后 FD 数据；失败时保留已完成结果与日志。 + +## 验收标准 + +- [x] 定向测试证明进程可绑定 CPU 集合，CPU 参数拒绝空值、重复值、越界值以及与服务 CPU 重叠的配置。 +- [x] 报告中的 `client_workers` 与 `client_cpus` 等于实际配置；负载进程采样可报告超过单核的总 CPU 使用率，且服务端仍只绑定一个 CPU。 +- [x] HTTP/1、HTTP/2 和长连接负载共用同一客户端并行配置入口；现有协议调度、响应校验和直方图实现未复制。 +- [x] 16x16 饱和扫描覆盖 h2c/TLS+h2 和至少四个递增 rate；全部请求计数守恒、请求错误为 0、握手数等于物理连接数、无 OOM，报告能识别吞吐平台或确认扫描上界仍不足。 +- [x] 128 MiB 与 64 MiB 下各完成 h2c/TLS+h2 的 60 秒稳态 case；请求错误和 OOM 为 0，成功率不低于计划请求的 99%，结束后 FD 回到 READY 基线。 +- [x] h2c/TLS+h2 各完成 100 轮 16x16 恢复；每轮至少有成功请求、错误为 0、握手数正确、FD 回到 READY 基线，RSS 序列如实记录且不把短时平台误称为无泄漏证明。 +- [x] 文档记录命令、机器约束、扫描结果、稳态/恢复结果和限制；明确区分本机观测饱和点、可持续负载与跨机器性能。 +- [x] 只运行受影响的 Python 单元测试、最小 loopback smoke、现有产物指纹校验和显式有界压测；不运行全量测试，不接触已有服务，不提交 Git。 + +## 排除项 + +- 修改 `crates/dever-runtime`、编译器、语言 API 或 HTTP/2 生产协议路径。 +- 多进程客户端、跨进程直方图合并、新压测依赖或第二套结果格式。 +- 跨主机、公网、丢包/限速注入、HTTP/3、RFC 8441 或小时级 soak。 +- 预设固定 QPS 门槛、与 Go/Fiber 的跨机器数字比较，或根据一次短测直接修改生产热路径。 diff --git a/.trellis/tasks/archive/2026-09/09-13-http2-ceiling-soak/research/client-bottleneck.md b/.trellis/tasks/archive/2026-09/09-13-http2-ceiling-soak/research/client-bottleneck.md new file mode 100644 index 0000000..719e3d8 --- /dev/null +++ b/.trellis/tasks/archive/2026-09/09-13-http2-ceiling-soak/research/client-bottleneck.md @@ -0,0 +1,27 @@ +# 客户端瓶颈证据 + +## 现有实现 + +- `test/dever-tests/examples/network_bench/load.rs` 从 `BENCH_WORKERS` 创建 Tokio multi-thread runtime，范围为 1–64。HTTP/2 的物理连接、请求 worker、速率调度、响应校验和三组 Histogram 均在该进程内完成。 +- `test/performance/run.py` 的 `load()` 无条件写入 `BENCH_WORKERS=1`，并通过 `args.client_cpu` 把负载进程绑定到一个 CPU；`live.py` 使用相同单 worker/单 CPU 方式。 +- `test/performance/process.py` 的 `Process` 已集中拥有 fork 后亲和性设置，只需把单元素集合推广为调用方提供的集合，不需要新增进程封装。 + +## 现有结果 + +`target/performance/http2-plain-128m/report.json` 中，Dever 4x16、30000 req/s 的三轮结果为： + +- h2c 成功 QPS 29506–29804，服务端约 0.87–0.92 CPU，客户端约 0.89–0.91 CPU； +- TLS+h2 成功 QPS 28673–29865，服务端约 0.91–0.93 CPU，客户端约 0.92 CPU； +- 请求错误均为 0，但存在容量丢弃，因此该档位不能确定是哪一端首先饱和。 + +## 方案比较 + +| 方案 | 优点 | 问题 | 结论 | +| --- | --- | --- | --- | +| 多进程客户端 | 调度器天然分片，可跨多个 CPU | 每进程独立直方图，现有分位数无法正确合并；还需拆分 rate、连接和生命周期 | 本阶段不采用 | +| 单进程多线程 | 已有 Tokio 支持；保留单一计数、校验和直方图；改动只在 Python 配置 | 单一调度协程仍可能成为更高档位瓶颈 | 先采用并用结果验证 | +| 引入 h2load 等外部工具 | 成熟、高吞吐 | 新依赖、响应合同和报告格式不同，破坏现有可比性 | 不采用 | + +## 结论 + +当前最小实验不是重写负载器，而是解除 Python 对已有多线程能力的限制。只有在多 CPU 客户端仍显示调度端瓶颈时，才有证据支持后续分片调度或多进程直方图协议。 diff --git a/.trellis/tasks/archive/2026-09/09-13-http2-ceiling-soak/task.json b/.trellis/tasks/archive/2026-09/09-13-http2-ceiling-soak/task.json new file mode 100644 index 0000000..2613f29 --- /dev/null +++ b/.trellis/tasks/archive/2026-09/09-13-http2-ceiling-soak/task.json @@ -0,0 +1,26 @@ +{ + "id": "http2-ceiling-soak", + "name": "http2-ceiling-soak", + "title": "HTTP/2 ceiling and soak", + "description": "解除 HTTP/2 压测客户端单核限制，测量单核小内存服务的饱和边界、稳态表现与连接恢复耐久性。", + "status": "completed", + "dev_type": null, + "scope": "test/performance + performance docs/spec; reuse existing network_bench and production runtime unchanged", + "package": null, + "priority": "P2", + "creator": "Antigravity", + "assignee": "Antigravity", + "createdAt": "2026-09-13", + "completedAt": "2026-09-18", + "branch": "main", + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [], + "parent": "09-04-dever-language-mvp", + "relatedFiles": [], + "notes": "", + "meta": {} +} \ No newline at end of file diff --git a/.trellis/tasks/archive/2026-09/09-18-cms-dual-source-contracts/check.jsonl b/.trellis/tasks/archive/2026-09/09-18-cms-dual-source-contracts/check.jsonl new file mode 100644 index 0000000..34a1998 --- /dev/null +++ b/.trellis/tasks/archive/2026-09/09-18-cms-dual-source-contracts/check.jsonl @@ -0,0 +1,3 @@ +{"file":".trellis/spec/backend/quality-guidelines.md","reason":"定向验收与代码审查"} +{"file":".trellis/spec/backend/database-guidelines.md","reason":"配置、Model、事务和迁移不变量"} +{"file":".trellis/spec/backend/compiler-contracts.md","reason":"双源码和错误合同"} diff --git a/.trellis/tasks/archive/2026-09/09-18-cms-dual-source-contracts/design.md b/.trellis/tasks/archive/2026-09/09-18-cms-dual-source-contracts/design.md new file mode 100644 index 0000000..d8f815f --- /dev/null +++ b/.trellis/tasks/archive/2026-09/09-18-cms-dual-source-contracts/design.md @@ -0,0 +1,26 @@ +# 设计边界 + +- 复用同一 source/parser/check/native 流程；Markdown 只负责抽取及文档合同，不另建 ORM。 +- Model 字面量约束收敛在已有 normalize_value 所有者；覆盖 default/seed 两种调用者。 +- ORM 错误复用现有 choice/error 隐式失败机制；数据库错误身份不得通过文本猜测，非数据库故障仍走 Fault。 +- 标准库 `dever.database.Error` 对应 runtime ErrorKind 的十个 error 分支，每个保留 message。调用方结果类型可用一个 error-only choice 载荷包装原错误，例如 `error Failed(error: dever.database.Error)`；checker/native/reference 共用捕获映射，显式抛出的外层错误仍保留名义身份，重叠包装及不匹配错误集合均拒绝。 +- 数据迁移扩展已有命名 migration 与参数化 SQL 校验/运行流程，不新增迁移框架。 +- CMS 两套源文件具有相同包名和声明，Markdown 补充所需中文文档；共同测试防漂移。普通 CRUD 直接调用 Model，业务状态校验留在 service。 +- CMS 演示管理员 Seed、编辑注册与文章发布；注册规范化邮箱/显示名并检查必填数据，单次 create 不增加空事务包装。发布事务验证作者 Active，再按 slug upsert。入口首次注册编辑、重复运行复用编辑，稳定输出 users=2/news=1；哈希仅为演示值，不实现登录或密码算法。 +- 旧 examples/dever/cms 移至 examples/cms/dever；其余 dever/markdown 示例整体放 examples/old 下。更新维护中的路径，历史日志保留事实。 +- 基准 Python 驱动按受控测试目录写 config/setting.json，Rust peers 使用统一读取规则；工具链配置不混入业务配置。 +- 不触及 HTTP API 设计、路由、认证协议或新增 API 配置。现有编译器 dever.api 合同快照仍是编译器能力，并非用户延期的业务接口。 + +实现分工：Model/Markdown 与基准配置分别独立处理；主线程负责错误链、迁移、CMS 与集成。相邻文件修改前明确交接。 + +## 命名数据迁移 + +`migrate name` 保留 `drop field`，增加可重复的 `before { sqlite = "..."; postgres = "..."; parameters = [...] }` 与 `after { ... }` 块；这里的分号仅表示文档分隔，实际源码使用换行。每块必须显式给出两个方言与参数列表，参数用现有绑定编码，不做字符串插值。 + +- SQL 只接受一条 INSERT、UPDATE 或 DELETE；扫描器跳过引号/注释后检查首关键字及语句终止符，拒绝多语句、事务控制、DDL、PRAGMA。占位符继续复用现有方言扫描与连续参数覆盖检查。 +- `_dever_`、`sqlite_`、`pg_` 前缀及 `information_schema` 是保留对象，普通、引用和 SQLite 单引号名称都不能出现在迁移 SQL 中；同名业务文本使用绑定参数。PostgreSQL Unicode 转义标识符不接受，防止绕开保留名称检查。 +- 无类型上下文的绑定常量支持 Bool、Int、Text、null 和指数形式 Float；无指数小数属于 Decimal，缺少目标精度/scale 时拒绝，不隐式改为 Float。 +- 已有表先校验已应用迁移与实际 catalog，再按源码中迁移/块的顺序执行全部未应用 before，执行自动结构变更，再执行全部未应用 after，最后 Seed/history。所有步骤在现有单 Model 迁移事务中完成；失败整体回滚。 +- 新表直接建立最终结构和 Seed，登记迁移已应用，不重放旧数据转换。无 SQL 的 drop-only revision 保持原算法；包含 SQL 的 revision 纳入名字、drop、方言文本、参数类型/值、phase 与块顺序，以有边界的规范表示计算，避免拼接歧义。修改或删除已应用迁移继续拒绝。 +- 运行语句放 `Model.data_migrations`，历史 Schema 仍保存声明名、revision、drop；复用 `orm::Value` 和数据库绑定，不新增通用值系统。 +- 事务边界是同一数据库中的单 Model 初始化；跨表变换只能引用届时已存在的表。编译器现有 Model 初始化顺序不变，不承诺跨 Model 原子升级、任意对象拓扑或跨数据库迁移。 diff --git a/.trellis/tasks/archive/2026-09/09-18-cms-dual-source-contracts/implement.jsonl b/.trellis/tasks/archive/2026-09/09-18-cms-dual-source-contracts/implement.jsonl new file mode 100644 index 0000000..efafdc9 --- /dev/null +++ b/.trellis/tasks/archive/2026-09/09-18-cms-dual-source-contracts/implement.jsonl @@ -0,0 +1,3 @@ +{"file":".trellis/spec/backend/quality-guidelines.md","reason":"定向离线验证及测试目录"} +{"file":".trellis/spec/backend/database-guidelines.md","reason":"Model、事务、配置与迁移合同"} +{"file":".trellis/spec/backend/compiler-contracts.md","reason":"源码格式、错误和效应一致性"} diff --git a/.trellis/tasks/archive/2026-09/09-18-cms-dual-source-contracts/implement.md b/.trellis/tasks/archive/2026-09/09-18-cms-dual-source-contracts/implement.md new file mode 100644 index 0000000..903b4d7 --- /dev/null +++ b/.trellis/tasks/archive/2026-09/09-18-cms-dual-source-contracts/implement.md @@ -0,0 +1,38 @@ +# 执行记录 + +用户已授权实施；API 明确延期。沿用上一轮审查已验证的缺口。 + +## 顺序 + +- [x] Model 字面量与 Markdown 声明合同、对应定向测试。 +- [x] 基准配置迁移到 config/setting.json，Python/Rust 定向检查。 +- [x] ORM 恢复错误的类型/效应/生成器贯通及事务捕获回归。 +- [x] 命名数据迁移扩展及历史/事务定向回归。 +- [x] examples 重排、CMS 双源码业务实现、路径与文档同步。 +- [x] 双源码 check/run/build 与打包定向验证、独立审查、规范更新。 + +## 验证边界 + +仅当前合同及示例的离线定向测试。SQLite 数据库使用测试临时目录。若需要 loopback 短时 peer 验证，运行前说明；不连接外部数据库、不运行全量集成或压力测试。未安装的 lint 工具如实记录。 + +## 当前证据 + +- Model default/Seed 定向 3/3；Markdown 静态/格式化 18/18（包括新 Model 声明支持）。 +- 基准 runner 37/37；network/live 无网络配置测试 2/2；Rust peers offline check/build 通过。现有根 setting.json 未修改。构建产物协议标记 setting-json-v1，旧产物必须重建。 +- error_effects 原有 8 项通过；新增共享错误包装/native+reference 1 项通过（含显式外层错误身份保持）。 +- SQLite 共享 ORM fixture 通过；新增 NotFound/Constraint 经任务和事务回滚捕获通过，最终记录 0 条。 +- 所有迁移后示例统一合同检查通过。两套 CMS 声明、Model、API、配置 parity 通过。 +- CMS 实际 native 首次运行发现 first() 行解码裸块的 ?/return 错误边界问题，已在共用 decode_included_row 修复；关联查询 native 回归 1/1，通过。 +- CMS CLI/打包验收 1/1（392.43 秒）：两格式各 check、2 次 CLI run、build、2 次无 module 独立打包运行，均 users=2/news=1。业务 probe 确认邮箱/显示名规范化，重复注册 Constraint 后数据不增加，禁用作者后发布 AuthorDisabled 且文章不增加。临时项目和 SQLite 已清理。 +- 两套 CMS 经 deverc fmt 整理后，fmt --check 和 parity 再次通过。CMS 基准 fixture 与当前业务入口的静态检查 1/1；旧 system 示例移除配置环境变量后的 native 检查 1/1。 +- 命名数据迁移 7/7（包括真实 native 双版本 SQLite 升级）；既有 orm_migration 7/7；最后增加 dollar-quote 回归后，静态/SQLite 7/7、共享 typed SQL 3/3 通过。native 升级未因纯扫描器边界修正重复运行。 +- 默认 core 及 core + PostgreSQL runtime cargo check 通过；最终扫描器修正后再次通过。 +- 独立审查确认 1 项 P2：PostgreSQL SQL 以未闭合 $tag 结束时，then_some 急切切片越界。最小回归先复现 panic，再修正共用扫描器；未闭合 delimiter/body 均给出 C014，合法 Unicode tag 和含 $ 的标识符保留。修复已独立复核，无剩余阻塞。 +- rustfmt/clippy 未安装。性能没有重测，旧 CMS 报告对应旧业务入口，不能替代新实现验收。 + +## 收口 + +- examples 仅保留 cms、old 两个目录：cms/dever 与 cms/md 同步；历史示例位于 old/dever、old/markdown。正式测试仍在 test/。 +- README、LANGUAGE、MARKDOWN-SYNTAX、示例说明和 backend specs 已同步。维护中的路径及配置环境变量搜索无旧入口；历史任务/日志保留原始事实。 +- 本次验收完成。业务 HTTP API 等用户重新设计；真实 PostgreSQL、性能/体积/64/128 MiB 压测不属于本轮验收。命名迁移只保证单 Model 事务，不承诺跨 Model 原子升级。 +- 本地任务未关联 PR；按项目要求不创建 commit，归档和日志均使用 no-commit。 diff --git a/.trellis/tasks/archive/2026-09/09-18-cms-dual-source-contracts/prd.md b/.trellis/tasks/archive/2026-09/09-18-cms-dual-source-contracts/prd.md new file mode 100644 index 0000000..f387625 --- /dev/null +++ b/.trellis/tasks/archive/2026-09/09-18-cms-dual-source-contracts/prd.md @@ -0,0 +1,18 @@ +# 双源码 CMS 与语言合同修正 + +## 需求 + +用户要求 examples 最终只有 cms、old 两个目录；cms/md 与 cms/dever 同步实现同一 CMS，旧示例移到 old。API 留待用户重新设计，本任务不实现 HTTP API。 + +承接源码审查：修复 Markdown Model 识别/声明合同、Model 默认值与 Seed 字面量校验、ORM 可恢复错误传播与捕获；补齐命名迁移的数据变换能力；示例与基准配置统一从 config/setting.json 读取；同步过期的语言返回值说明。 + +## 验收 + +- 两套 CMS 执行相同的注册、发布业务，数据库结构和可观察结果等价；重复执行可预测，禁用作者不能发布。 +- 所有旧示例移动到 examples/old；正式测试留在 test；当前代码与文档没有悬空路径。 +- Markdown Model 支持普通 Model 的声明，仍执行文档合同校验。 +- 不合法 Int/Float/Decimal/Text 默认值与 Seed 在编译期拒绝，不推迟到 Rust 编译或生成器 panic。 +- ORM 故障保留错误类别，进入既有隐式失败传播与 result 捕获合同；事务边界不抹掉错误。 +- 命名迁移支持参数化、按数据库方言选择的数据变换；事务、历史一致性约束不降级。 +- 无应用/基准配置环境变量入口；保留构建工具自身环境及通用 process.environment 语言能力。 +- 定向离线检查与最小测试通过，明确未运行的验证。不做全量测试或性能重测，不改全局工具、不提交 Git。 diff --git a/.trellis/tasks/archive/2026-09/09-18-cms-dual-source-contracts/task.json b/.trellis/tasks/archive/2026-09/09-18-cms-dual-source-contracts/task.json new file mode 100644 index 0000000..3ad2c02 --- /dev/null +++ b/.trellis/tasks/archive/2026-09/09-18-cms-dual-source-contracts/task.json @@ -0,0 +1,26 @@ +{ + "id": "cms-dual-source-contracts", + "name": "cms-dual-source-contracts", + "title": "双源码 CMS 与语言合同修正", + "description": "CMS 的 Dever/Markdown 双实现、旧示例归档和已确认语言合同问题；API 延后", + "status": "completed", + "dev_type": null, + "scope": null, + "package": null, + "priority": "P2", + "creator": "Antigravity", + "assignee": "Antigravity", + "createdAt": "2026-09-18", + "completedAt": "2026-09-18", + "branch": "main", + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [], + "parent": "09-04-dever-language-mvp", + "relatedFiles": [], + "notes": "", + "meta": {} +} \ No newline at end of file diff --git a/.trellis/tasks/archive/2026-10/09-20-large-cms-acceptance/check.jsonl b/.trellis/tasks/archive/2026-10/09-20-large-cms-acceptance/check.jsonl new file mode 100644 index 0000000..1cb58ac --- /dev/null +++ b/.trellis/tasks/archive/2026-10/09-20-large-cms-acceptance/check.jsonl @@ -0,0 +1,3 @@ +{"file":".trellis/spec/backend/quality-guidelines.md","reason":"Focused CMS/compiler/runtime verification"} +{"file":".trellis/spec/backend/directory-structure.md","reason":"No junk layers and dual-source ownership"} +{"file":".trellis/spec/backend/error-handling.md","reason":"Business/API/Job failure flow review"} diff --git a/.trellis/tasks/archive/2026-10/09-20-large-cms-acceptance/design.md b/.trellis/tasks/archive/2026-10/09-20-large-cms-acceptance/design.md new file mode 100644 index 0000000..c962ea0 --- /dev/null +++ b/.trellis/tasks/archive/2026-10/09-20-large-cms-acceptance/design.md @@ -0,0 +1,24 @@ +# CMS 业务闭环设计 + +## 复用与依赖 +保留路径角色、生成入口与核心 RBAC。认证走配置站点/account.verify，权限目录自动生成，不新增身份注入。跨领域只调用 App。 +依赖单向：content.revision、publishing.schedule 调用 news.article；news.article 可依赖 content.category、media.asset、audit.operation，不反向调用 revision/schedule。修订拥有编辑编排，文章拥有作者、状态和版本规则。普通 CRUD 保留生成 REST。 + +## 数据与行为 +文章增加可空分类/媒体引用及受保护版本；关联校验复用叶领域 App。修订调用文章受约束更新再保存快照，复用同一租户事务；使用现有条件 update 的 affected rows 判断竞争，不用 count+1 分配版本。 +手动与定时发布共用内聚实现；手动重复保持冲突，任务在已完成状态幂等成功。定时创建验证真实作者，保存必要业务引用并使用 enqueue_at。Worker 仍由核心重新验证可信身份和权限，payload 不能成为认证来源。 +API/Job 保持薄适配。数据库回滚不等于任意文件回滚；区分 Upload 临时资源与已提交存储文件。 + +## 测试分层 +- 应用源码测试验证 App/Domain 和正常存储结果，用例由运行器隔离；错误、权限和会话失效通过真实 HTTP 验证。Test/Job 不能调用身份 getter，不增加后门。 +- 自有 HTTP 验收使用复制的 CMS、临时 SQLite/配置、随机 loopback 端口、真实登录与角色及正常 Worker。 +- 复用 test/performance/cms.py 客户端与现有过程 owner；业务验收留在根 test/，不启动压测。 +- 等价检查覆盖声明、Model/API/Job/Port 和设置，Markdown 保留真实文档合同。 + +## 变更边界 +主要修改 examples/cms 双源码与测试、根 test/ 相关验收和 examples/README.md；复用编译器/runtime，默认不重建发行包或升级依赖。 +改前备份：target/cms-workflows-before.v1FzWO/source.tar.gz。 + +## 已证实的核心阻塞 +- SQL 返回同领域 App View，复用既有行解码；拒绝非存储类型、private 字段和未经解码证明的字段 bounds。仅两个方言均为完整单表简单投影时证明只读，其他 SQL 保持 write 效果。 +- 可信用户/租户 ID 可在明确 nullable ModelId 上下文中保持名义类型，API 与 REST 均按当前 Provider 的 verify 字段验证，不把 Int 任意转换成 ID。CMS 直接读取已验证身份，避免租户写入链重读全局 Session；保留跨库事务禁止规则。 diff --git a/.trellis/tasks/archive/2026-10/09-20-large-cms-acceptance/implement.jsonl b/.trellis/tasks/archive/2026-10/09-20-large-cms-acceptance/implement.jsonl new file mode 100644 index 0000000..2bcf7df --- /dev/null +++ b/.trellis/tasks/archive/2026-10/09-20-large-cms-acceptance/implement.jsonl @@ -0,0 +1,4 @@ +{"file":".trellis/spec/backend/compiler-contracts.md","reason":"Final application role/API/Job/visibility contract"} +{"file":".trellis/spec/backend/database-guidelines.md","reason":"CMS Models, transactions, migrations and test isolation"} +{"file":".trellis/spec/backend/logging-guidelines.md","reason":"Audit and secret-free request logging"} +{"file":".trellis/spec/backend/toolchain-and-library.md","reason":"Runtime modes, Port/Job/API standard packages"} diff --git a/.trellis/tasks/archive/2026-10/09-20-large-cms-acceptance/implement.md b/.trellis/tasks/archive/2026-10/09-20-large-cms-acceptance/implement.md new file mode 100644 index 0000000..8140170 --- /dev/null +++ b/.trellis/tasks/archive/2026-10/09-20-large-cms-acceptance/implement.md @@ -0,0 +1,40 @@ +# CMS 业务闭环实施记录 + +## 2026-10-04 基线 +用户批准本轮实施。旧勾选不代表当前业务验收完成：每套45个生产文件/12领域，3个应用测试中2个固定字符串断言；cms_acceptance只比较API/Model。定时发布用了enqueue而不是enqueue_at，修订版本用count+1。此前真实PG/RBAC/租户和核心Job验收有效，但不代替CMS业务用例。 + +## 执行顺序 +- [x] 更新要求/设计/上下文，保存改前备份。 +- [x] 分类/媒体关联、文章版本竞争、修订事务。 +- [x] 到期入队、共享发布和任务幂等。 +- [x] 替换空测试、同步Markdown、完善等价与CLI预期。 +- [x] 有界HTTP验收：身份权限、编辑冲突、上传、到期/重试/回滚。 +- [x] 两套check/fmt/test、相关Rust/Python检查和真实验收。 +- [x] 独立复核，修复阻塞，更新示例与记录。 + +## 验证约束 +磁盘约462MiB空闲，复用target/debug/dever和runtime pack，Cargo单并发。自有HTTP/Worker前说明；不连真实数据库，不运行全量测试/压测，不变更全局命令/服务。配置只在自有临时项目写config/setting.json；子进程有超时与清理，分别记录通过/失败/未运行。 + +## 结果 +本轮业务闭环已完成。真实HTTP、CLI、核心约束回归、独立复核和静态质量门均通过；未执行范围外的全量CI、真实PG或压测，不将当前结果当作这些验证的替代。所有改动保留在工作区，未创建Git提交。 + +## 已完成的定向证据与直接阻塞 +- 两套源改为46个生产文件+5个真实应用测试；51对源码声明一致，check/fmt及各5/5应用测试已通过。源测试验证正常业务输出，拒绝和失效放在真实HTTP验证，不伪造认证上下文。 +- Python CMS调用迁移单测12/12；SQL View/字段范围/跨域约束4/4；API声明与GET只读SQL规则18/18；CMS语义/配置/测试等价2/2通过。 +- HTTP第1轮在build发现published_detail误删既有业务校验，已恢复；第2轮构建成功，但bootstrap启动拒绝global Session读取与tenant Article写入同一事务。日志保存在target/cms-business-acceptance-{1,2}/，尚未进入HTTP业务断言。 +- 直接语言修复：SQL可返回同领域App View，普通SQL结果record拒绝未经数据库解码证明的bounds；仅完整自有表简单投影可证明只读，两方言都通过才允许GET，其他SQL保留write效果。复用现有类型/行解码/事务，不增依赖和运行时ABI。 +- 已修可信身份的名义ModelId取值，以已验证的auth metadata替代业务重复读取global Session；不放宽跨库事务，不将模型ID改成Int。 +- 第3轮真实HTTP/Worker验收10组全部通过，report在target/cms-business-acceptance-3/report.json：真实登录/角色拒绝、上传清理、关联/非作者/过期版本、修订晚失败回滚、两请求单胜者、发布快照与晚失败回滚、未来到期/同刻任务、完成任务重放、失败重试、撤权阻断任务、会话注销/过期与站点隔离。fixture程序8,465,384 bytes，临时配置/DB和进程已清理；非压测。 +- 独立审查确证REST字段create/replace也可产生typed身份getter，原新增validator只遍历显式API；现已用同一IdentityContext校验API、REST create/replace/search与名义ID owner。新增正反例、跨站点provider错配、native/LLVM生成验证。完整API目标26/26通过，原错误fixture被CLI明确C006拒绝后已清理。普通Int和nullable边界保留。 +- 最终源码等价2/2、typed_sql 4/4、Python CMS调用单测12/12。CLI精确用例2/2：双格式各5个应用测试各运行两次（故意阻断部署DB，证明测试隔离），check/run/build及无源码打包运行；混合6用例验证仅数据库用例初始化临时DB。更新了该混合用例残留的旧3测试预期。 +- HTTP最终运行器显式关闭SQLite连接；复用已构建程序复验10组全部通过，最终报告target/cms-business-acceptance-final/report.json。不改源码/部署配置，不遗留HTTP/Worker/测试进程。此结果不覆盖真实PG、浏览器HTTPS或性能/长压测。 +- 两套最终check/fmt通过，check各有7条W001未读取返回值提示，非错误；语言只允许零输出调用作为动作语句，未用虚假读取或抑制消除提示。Rust改动文件rustfmt --check通过。 +- 定向clippy通过（dever-core/dever-tests/dever-cli，lib、dever bin及api_declarations/orm_native_sql/contract_execution/cms_acceptance/cms_project，sqlite feature，-D warnings）。最终diff检查通过。独立复核指出的SQL bounds和REST身份绑定缺口均修复并保留回归；主线程确认实际测试输出。任务直接在既有main工作区完成、没有PR分支，归档使用skip-branch-validation且no-commit。 +- 磁盘紧张时暂移的可再生成target/native-artifacts已压缩为target/cms-workflows-before.v1FzWO/native-artifacts-cache.tar.xz（约190MiB）。先逐文件比较，再验证重压缩前后tar SHA256一致（675338b0d4af58b76689ae3ec9bc2c7202a1b5aa3e3f117d0df5930f7e88a7d0）；原路径恢复普通空缓存目录，软链和自有内存盘临时副本均已清理。源码、数据库和其他服务未动。 + +## Bug Analysis: SQL投影与业务集成边界 +1. 根因：跨层合同与测试覆盖缺口。私有Model不能穿越App边界，但SQL结果又只能是Model内部record；查询不能直接返回已定义的App View。数据库decoder也不会证明源级bounds。 +2. 失败尝试：只开放同域私有record引用仍会触发公开App签名约束；把Session重新读进文章调用链则实际触发跨库事务限制。不能通过放宽隐私或事务检查解决。 +3. 防止复发：共享SQL result检查同时验证归属、字段类型/可见性和bounds；GET效果采用完整受限语法证明，保留反例；HTTP按正常入口启动、验证完整提交与回滚。 +4. 扩展核对：双源码、旧CMS性能/PG调用方及CLI测试预期同步迁移；原SQL私有record同样拒绝未证明bounds。 +5. 已更新backend/database-guidelines.md和LANGUAGE.md；本仓库无对应spec模板副本，不创建；按用户约束不提交Git。 diff --git a/.trellis/tasks/archive/2026-10/09-20-large-cms-acceptance/prd.md b/.trellis/tasks/archive/2026-10/09-20-large-cms-acceptance/prd.md new file mode 100644 index 0000000..811bb80 --- /dev/null +++ b/.trellis/tasks/archive/2026-10/09-20-large-cms-acceptance/prd.md @@ -0,0 +1,25 @@ +# 双源码 CMS 业务闭环验收 + +## 当前目标与授权 +2026-10-04 用户批准继续完善 CMS：替换空断言，贯通分类、文章、修订、媒体和定时发布，验证权限拒绝、并发冲突、重试与回滚。沿用本任务；旧规划中的手写 main、Principal 入参、业务 RBAC 和旧命令名已被后续语言合同取代，不恢复旧实现。 + +## 要求 +- 两套示例保持业务声明、Model/API/Job/Port 和测试等价。 +- 复用现有 ORM、自动权限、database-only 租户、Upload/Storage Port 和延时 Job；业务规则留在 App/Domain，不绕过真实身份。 +- 分类与媒体可关联文章，创建/编辑时校验。修订快照与文章更新同租户事务；过期版本或并发竞争明确冲突，不丢失更新。 +- 手动发布保持作者校验；定时创建验证作者和文章状态，按真实到期时间执行；重复任务不重复发布/审计，失败不留下部分完成状态。 +- 普通应用测试只用运行器临时 SQLite、结果捕获和 fake，不加身份后门、不读部署配置。认证、租户、上传与 Worker 由自有 HTTP 验收覆盖。 +- 两个固定字符串断言必须删除或改为真实业务测试；check、健康接口、快照不代表业务验收。 +- 配置只用 config/setting.json，不增加环境变量、权限表或框架层。 + +## 验收 +- [x] 两套 check/fmt、应用测试和语义等价通过。 +- [x] 分类/媒体关联、修订编辑、发布快照形成完整链路。 +- [x] 未授权/非作者请求失败且无写入，保留即时撤权。 +- [x] 并发/过期修订只有合法更新成功；失败回滚文章和修订/审计。 +- [x] 任务不提前运行，到期发布、重复执行、失败回滚/重试有真实断言。 +- [x] 上传成功与受限/失败路径验证文件、临时资源归属。 +- [x] 定向 build、自有临时 HTTP/Worker 验收通过，记录范围和限制。 + +## 范围外 +前端、富文本/搜索产品、外部云服务、公开发行、ARM、全量 CI 和长压测。仅确证阻止本任务的缺陷才修改语言核心。 diff --git a/.trellis/tasks/archive/2026-10/09-20-large-cms-acceptance/task.json b/.trellis/tasks/archive/2026-10/09-20-large-cms-acceptance/task.json new file mode 100644 index 0000000..2de2244 --- /dev/null +++ b/.trellis/tasks/archive/2026-10/09-20-large-cms-acceptance/task.json @@ -0,0 +1,26 @@ +{ + "id": "large-cms-acceptance", + "name": "large-cms-acceptance", + "title": "Dever 大型双源码 CMS 验收", + "description": "扩展 Dever 与 Markdown CMS，验证大型项目的安全、Port、Job、API 和测试合同", + "status": "completed", + "dev_type": null, + "scope": null, + "package": null, + "priority": "P1", + "creator": "Antigravity", + "assignee": "Antigravity", + "createdAt": "2026-09-20", + "completedAt": "2026-10-04", + "branch": "main", + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [], + "parent": "09-20-production-application-language", + "relatedFiles": [], + "notes": "", + "meta": {} +} \ No newline at end of file diff --git a/.trellis/tasks/archive/2026-10/10-04-external-command/check.jsonl b/.trellis/tasks/archive/2026-10/10-04-external-command/check.jsonl new file mode 100644 index 0000000..696b7ce --- /dev/null +++ b/.trellis/tasks/archive/2026-10/10-04-external-command/check.jsonl @@ -0,0 +1,3 @@ +{"file":".trellis/spec/backend/quality-guidelines.md","reason":"定向验证、真实执行和资源所有权"} +{"file":".trellis/spec/backend/compiler-contracts.md","reason":"命令合同不得扩散到API和Worker"} +{"file":".trellis/spec/backend/toolchain-and-library.md","reason":"打包闭包与沙箱完整性"} diff --git a/.trellis/tasks/archive/2026-10/10-04-external-command/design.md b/.trellis/tasks/archive/2026-10/10-04-external-command/design.md new file mode 100644 index 0000000..e22fe93 --- /dev/null +++ b/.trellis/tasks/archive/2026-10/10-04-external-command/design.md @@ -0,0 +1,36 @@ +# 普通二进制调用设计 + +## 源码合同 + +Adapter：`external command "bin/tool" { allow file }`，路径沿用领域相对资源路径，allow 沿用现有能力名。不接受 setting/lib；部署配置和授权沿用已有 adapter 节点。 + +每个 command Adapter 对应一个只有一个操作的 Port，操作名按业务自定：`execute(args: List<Text>, stdin: Bytes?) (output: dever.process.Output) fails dever.process.Error`。第二个 stdin 参数允许整项省略（仅 args），传 null 表示立即关闭 stdin。固定字段名防止增加映射 DSL。标准 Output 是 `{code: Int, stdout: Bytes, stderr: Bytes}`；标准 Error 是 `error Failed(message: Text)`，满足现有 Port failure 合同。命令非零退出按沙箱传播的退出码返回，不猜测 signal；启动/超时/超限继续作为源位置 runtime fault。 + +## 编译器与进程边界 + +新增 ExternalEcosystem::Command，保留现有 external 合成函数、Port fake、effect 和 suspends 链路。共享 wire 增加仅 Command policy 可用的 Bytes/base64；API/Job/真实 Worker 原有 wire policy 不变。两个后端仍用同一 external start/call/reply ABI，只给 Bytes wire 增加最小桥接函数。 + +HIR 共用合同构造实现，但 external_worker_contracts 只返回 exec/pip/npm/go，external_command_contracts 返回 command，避免 Lib lock/resolver 散落 command 特判。command 不需要 dever.lock；有真实生态依赖的同项目仍保持原 lock 行为。 + +## 运行时 + +复用 invocation Session、Managed、registry、bounded requests 和 structured Scope。command supervisor 启动不执行二进制或协议握手；收到每个请求才启动一个独立沙箱进程，并发写 stdin、读 stdout/stderr 和等待退出。不追加 --dever-component，不重放命令。超时、调用取消和关闭均 kill/wait，再返回结果；空闲 supervisor 可继续服务后续请求，不为正常多次调用套用 Worker crash restart 次数。 + +复用 verified extraction、launch manifest owner 和 sandbox Launch。内部资源 argv 继续映射，动态调用 argv 作为普通数据追加，不当宿主路径重写。保留只读程序树、grant、namespace/seccomp 与空环境。沙箱 PID namespace 负责后代收束，测试必须观察真实后代清理。 + +限额来自 config/setting.json 的 `adapter.<port>.command`：timeout_ms（默认30000，最大3600000）、output_limit（默认1048576，最大8388608，两路输出合计）。argv 总量最多65536字节，stdin最多8388608字节；拒绝NUL。base64膨胀及JSON包装不得超过已有16MiB wire上限。Worker Adapter 不使用这组命令限额。 + +## 离线打包 + +CLI 专门 command packager 接受一个 ELF 文件及同目录 lib/ 中的普通库文件；只采集所属项目/Package 的显式资源，不扫描宿主或调用 ldd。静态 ELF 可直接运行。动态 ELF 复用签名 sandbox GNU loader/OS库，递归解析 DT_NEEDED、验证目标、SONAME和版本需求，私有库按随附 lib/ 查找，缺少依赖构建期报错。共享已有 Python native ELF 闭包校验算法，不复制第三套解析器。使用固定私有 loader 参数关闭系统 cache，库搜索受打包树与sandbox库限制。 + +资源沿用SHA、私有原子提取、缓存/签名和Package归属验证；每次启动重新校验。具体 launch manifest 字段按既有owner扩展，不能从运行配置接受任意可执行路径。 + +## 改动边界 + +- core syntax/parser/check/ports/hir/wire/native/llvm，library/dever/process.dever，以及 wire runtime/bridge：一条类型与调用合同。 +- runtime component/command/external/config 与 sandbox：一次调用的进程和资源生命周期。 +- CLI workers command/ELF依赖、libs资源准备：离线可执行资源闭包。 +- 根test、LANGUAGE/README和已有规范：真实证据和可用示例。 + +不增加通用执行注册中心、新依赖、第二套SDK/JSON引擎，不改CMS业务。回滚按上述拥有者整体撤销本任务差异，保留开始前源码备份与用户已有改动。 diff --git a/.trellis/tasks/archive/2026-10/10-04-external-command/implement.jsonl b/.trellis/tasks/archive/2026-10/10-04-external-command/implement.jsonl new file mode 100644 index 0000000..3b4915d --- /dev/null +++ b/.trellis/tasks/archive/2026-10/10-04-external-command/implement.jsonl @@ -0,0 +1,4 @@ +{"file":".trellis/spec/backend/index.md","reason":"Rust语言仓库开发边界"} +{"file":".trellis/spec/backend/compiler-contracts.md","reason":"Port/Adapter和两后端的类型与失败合同"} +{"file":".trellis/spec/backend/toolchain-and-library.md","reason":"外部资源、沙箱、目标与生命周期"} +{"file":".trellis/spec/backend/error-handling.md","reason":"源位置错误和资源清理"} diff --git a/.trellis/tasks/archive/2026-10/10-04-external-command/implement.md b/.trellis/tasks/archive/2026-10/10-04-external-command/implement.md new file mode 100644 index 0000000..19138e3 --- /dev/null +++ b/.trellis/tasks/archive/2026-10/10-04-external-command/implement.md @@ -0,0 +1,56 @@ +# 实施与验证 + +## 执行顺序 + +1. [x] 读取规范、调查编译/运行/沙箱/打包，固定用户已确认范围和合同。 +2. [x] 备份当前源码；并行实施编译器合同及runtime进程执行，主线程实现CLI资源打包。 +3. [x] 串行运行必要offline Cargo检查和定向测试；修复真实失败。 +4. [x] 用独立ELF与真实Dever源码验证check/run/build、移除源码运行、动态库、取消和完整性；验证ARM目标。 +5. [x] 独立Trellis复核、清理本任务冗余、更新语言规范及验证证据，归档；不提交Git。 + +## 所有权 + +- 编译代理：core、library/dever/process.dever、runtime/wire.rs、bridge ffi/runtime.h、对应编译/codec测试。 +- runtime代理：runtime component/command/external/config、sandbox及进程测试，不改wire.rs和CLI。 +- 主代理：CLI workers/libs/ELF闭包/Package必要集成及测试、任务/文档、全部Cargo和重资源验证。 + +代理不并行运行Cargo，不覆盖其他代理或用户改动。涉及共同合同先沟通。 + +## 验证原则 + +仅本任务所需定向检查，无全量测试/业务服务/长测；提前说明自有沙箱进程测试。先check与轻量测试，后更新实际runtime ABI再运行真实源码，不能用旧archive验证新codec。开始可用磁盘约3.3GiB，复用现有目标目录，串行构建。 + +预期目标：port_adapter、command专用runtime/CLI测试、external_workers/resources、llvm_external_source中command案例；原Worker回归按实际共享代码影响选择。fmt/clippy只覆盖改动所属目标。错误、跳过和未运行独立记录。 + +## 最终证据 + +- 改动前备份：`target/external-command-before.lBL9wU/source.tar.gz`。 +- CLI/core/runtime 初次 offline check 已通过；command 编译/codec/format/fake 5项通过。 +- external_workers 当前13项通过，10项需特定资产的旧真实生态测试未运行。command 打包4项通过，后续新增越界 RPATH 回归需复验。 +- runtime 首轮3过5失败为未优化debug哈希耗时超过fixture启动等待；作者构建为sha2/miniz_oxide/flate2指定opt-level=3，测试等待与命令deadline分离。随后即时文件锁断言揭示真实namespace清理竞态：管道EOF早于内核后代回收。最终改为info-fd + block-fd门禁，在放行guard前pin namespace pidfd并核验PPid，取消等待pidfd readiness后回收wrapper。9/9定向测试通过（17.86s），包括启动错误、无FD泄露、非零1/127、输入输出、限额、取消/超时/root close/queued/explicit shutdown。 +- 独立复核发现启动失败与非零退出混淆、RPATH越出程序树两项问题；已修复，复核确认无剩余阻塞。新增RPATH回归4/4及共用Python ELF闭包2/2通过。 +- 旧 Port 原生测试先因测试作者 rustc 路径缺失中断；指定作者工具路径后相关测试运行，期间新增sandbox直接依赖导致 --locked 拒绝。Cargo offline check已增量更新锁，不升级第三方包；最终21/21复验已关闭这些失败。 +- 新runtime ABI archive已离线重建（runtime-api/external/sqlite/postgres，3m25s），真实LLVM/CLI验收使用该新档案；未连接任何数据库。改动Rust格式检查及C fixture语法检查通过。 +- LLVM command对象+真实调用2/2通过（102.33s）；六目标对象生成，预热+64轮每轮3次普通程序调用，二进制/EOF/退出7和allocation ledger全通过。初次60s fixture预算仅完成42轮，给command probe单独180s预算后保留全部65轮与相同断言；原Worker预算不变。 +- native_runtime_link 的Bytes wire C ABI验收1/1通过（7.18s）。 +- signed_cli_runs_and_packages_ordinary_command_with_private_library 1/1通过（172.02s）：当前核心和新ABI、真实C动态工具+私有.so、check/run/build、无Lib锁、Package锁定资源、删除项目/作者/机器目录后两种产物独立运行、篡改.so拒绝。所有daemon/程序均为自有临时fixture，未安装全局命令。 +- 最终旧Worker生命周期1/1、external_resources 8/8通过；1项旧Rust bootstrap打包用例维持ignored，本次已有新的真实LLVM/签名CLI独立打包证据。 +- 最终Port/Adapter（包含普通/fake实际原生执行）21/21通过（103.41s），此前作者路径/锁文件导致的失败已全部关闭。 +- Linux ARM64 runtime-api/runtime-external桥接库、runtime、sandbox/guard 交叉cargo check通过（65s）；结合六目标LLVM对象生成与CLI跨目标打包/拒绝错架构验证。未重发四个ARM发行profile、未运行ARM实机/虚拟机验收，不能将这些检查表述为ARM执行通过。 +- 所有改动所属库、guard及6个相关测试target的clippy `-D warnings`通过（45.21s）；Rust定向fmt和C语法检查通过。git diff --check通过但本仓库核心文件为untracked，实际差异还通过改动前tar对比、逐文件检查与独立复核覆盖。 + +## 最终检查入口 + +Cargo均为offline、单并发，不使用全量测试。作者工具路径仅影响构建/验证进程，未加入应用环境配置。作者测试使用`profile.dev.package.{sha2,miniz_oxide,flate2}.opt-level=3`减少大量重复资源校验耗时。 + +- `dever-tests --features external --test port_adapter`：21通过。 +- `dever-tests --features external --test external_command`：9通过。 +- `dever-tests --features external --test external_component --test external_resources`：1+8通过，1旧bootstrap用例ignored。 +- `dever-cli --test external_workers`：13通过、10个旧真实生态fixture ignored；最终command过滤4通过。 +- `dever-cli --test external_libs wheel_tests::native_`：2通过。 +- `dever-backend-bridge --features embedded,runtime-external --test llvm_external_source command_ -- --include-ignored`：2通过。 +- `dever-backend-bridge --features embedded,runtime-external --test native_runtime_link wire_codecs_validate_slots_before_callback_and_release_partial_values -- --ignored --exact`：1通过。 +- `dever-cli --test native_release signed_cli_runs_and_packages_ordinary_command_with_private_library -- --ignored`：1通过。 +- ARM交叉check使用现有`target/arm64-cross`作者sysroot/工具，profile runtime-pack，target aarch64-unknown-linux-gnu；检查sandbox和bridge runtime-api/runtime-external。 + +未运行：全量测试、长压测、真实数据库、ARM目标执行及Windows；未替换全局安装或修改主机策略。当前无已知本任务阻塞。原有两篇tracked文档删除及其他既存任务保持原状。 diff --git a/.trellis/tasks/archive/2026-10/10-04-external-command/prd.md b/.trellis/tasks/archive/2026-10/10-04-external-command/prd.md new file mode 100644 index 0000000..5bddc0d --- /dev/null +++ b/.trellis/tasks/archive/2026-10/10-04-external-command/prd.md @@ -0,0 +1,33 @@ +# Linux 普通二进制调用 + +## Goal + +在 Port/Adapter 下直接执行现成 Linux 二进制，支持参数、标准输入输出、退出状态、超时取消及独立打包 + +## 已确认范围 + +用户已在上一轮审阅普通命令调用、Port/Adapter、参数数组、可选标准输入、两路输出、退出码、超时取消、二进制与动态库打包、Linux x86_64/ARM64、无 shell/env 的方案，并批准创建任务、规划和实施。本任务不再次请求相同授权。 + +## 需求 + +- R1：直接调用现成 Linux ELF，无需程序实现 Dever 协议或用户编写其他语言包装；Port/Adapter 保留唯一外部调用边界。 +- R2：参数按数组原样传递，标准输入可省略；输出不丢失非 UTF-8 字节，正常非零退出码属于结果。 +- R3：启动失败、超时、取消、输出超限明确处理；每次调用均回收自有进程及沙箱后代，不自动重试命令。 +- R4：run/build 离线准备并嵌入程序和显式随附的必要动态库；检查目标架构、依赖闭包和完整性，部署不依赖源码/机器缓存/宿主语言工具。 +- R5：继续使用 config/setting.json 管理部署文件授权和调用限额，不添加环境变量或 PATH/shell 查找。复用现有沙箱，不修改宿主策略。 +- R6：保持现有 exec/pip/npm/go Worker 合同；普通 Dever 测试继续 Port fake，不执行真实命令。兼容 Package 资源与 Markdown 源码。 + +## 验收 + +- [x] 编译检查并格式化合法 command Adapter；拒绝错误 Port 签名、额外操作、setting/lib 和非法路径；其他 wire 边界仍拒绝 Bytes。 +- [x] 真实无协议 ELF 收到空格、引号、shell 元字符和绝对沙箱路径原文；stdin EOF、二进制输入输出、两路并发大输出、正常非零退出正确。 +- [x] 超时/调用取消/根关闭/输出超限均有界完成并回收沙箱进程树；后续调用仍可执行，无自动重放。 +- [x] 打包拒绝错架构、缺库、错误 SONAME/ABI、越界路径和篡改；完整动态程序在移除原始源码后运行。 +- [x] 真实 Dever check/run/build 和独立产物覆盖完整路径；Package 资源与现有 Worker 定向回归通过。 +- [x] Linux ARM64 构建与目标校验覆盖；目标运行证据单独报告，不能由 ELF 标签推导运行通过。 + +ARM64本次证据为Rust交叉检查、LLVM目标对象生成及打包目标校验；实际普通命令与独立产物运行证据来自Linux x86_64。完整验证命令、失败修复及未运行项见implement.md。 + +## 非目标 + +Windows/macOS、FFI、shell/管道语法、后台守护进程/交互式终端、隐式下载或宿主库扫描、全量测试和长压测。 diff --git a/.trellis/tasks/archive/2026-10/10-04-external-command/task.json b/.trellis/tasks/archive/2026-10/10-04-external-command/task.json new file mode 100644 index 0000000..a62fcd1 --- /dev/null +++ b/.trellis/tasks/archive/2026-10/10-04-external-command/task.json @@ -0,0 +1,26 @@ +{ + "id": "external-command", + "name": "external-command", + "title": "Linux 普通二进制调用", + "description": "在 Port/Adapter 下直接执行现成 Linux 二进制，支持参数、标准输入输出、退出状态、超时取消及独立打包", + "status": "completed", + "dev_type": null, + "scope": "external-command", + "package": null, + "priority": "P2", + "creator": "Antigravity", + "assignee": "Antigravity", + "createdAt": "2026-10-04", + "completedAt": "2026-10-04", + "branch": "main", + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [], + "parent": null, + "relatedFiles": [], + "notes": "Linux普通二进制调用已实现并完成定向及真实打包验收；ARM证据为交叉检查/对象生成/目标校验，不含目标执行。备份与完整证据见implement.md。不提交Git、不替换全局安装。", + "meta": {} +} \ No newline at end of file diff --git a/.trellis/tasks/archive/2026-10/10-04-linux-arm64-cross-build/check.jsonl b/.trellis/tasks/archive/2026-10/10-04-linux-arm64-cross-build/check.jsonl new file mode 100644 index 0000000..63d97b4 --- /dev/null +++ b/.trellis/tasks/archive/2026-10/10-04-linux-arm64-cross-build/check.jsonl @@ -0,0 +1,2 @@ +{"file":".trellis/spec/backend/toolchain-and-library.md","reason":"目标贯穿、共享缓存、安全闭包、Worker部署一致性"} +{"file":".trellis/spec/backend/quality-guidelines.md","reason":"真实跨目标执行证据与最小回归边界"} diff --git a/.trellis/tasks/archive/2026-10/10-04-linux-arm64-cross-build/design.md b/.trellis/tasks/archive/2026-10/10-04-linux-arm64-cross-build/design.md new file mode 100644 index 0000000..a5e5a5b --- /dev/null +++ b/.trellis/tasks/archive/2026-10/10-04-linux-arm64-cross-build/design.md @@ -0,0 +1,25 @@ +# Linux ARM64 应用交叉构建设计 + +## 边界与复用 + +只有应用部署目标变化，编译器/daemon/build tools 仍在当前 x86 宿主运行。复用 LLVM bridge 的 Target、现有 pack 清单/签名验证、NativeProgram 发布、Lib resolver/lock/Worker 以及 CacheStore，不建立平行流水线。主机平台身份继续用于 launcher/daemon 验证；新增独立、封闭的构建目标值，不能全局替换 platform_identity。 + +## 目标数据流 + +CLI build 参数 → 明确的 BuildTarget → 目标 Lib 资源准备 → CompileRequest → daemon 验证目标 pack 与缓存 → worker 加载目标 runtime → LLVM object/LLD → 独立程序。私有编译走同一目标与 pack owner。run/test 在入口固定使用 host；应用配置不保存编译器状态。 + +toolchain/target.rs 拥有 BuildTarget（LinuxX86_64、LinuxAarch64），严格解析 linux-x86_64/linux-aarch64，提供 host/platform/triple 映射。共享库不引用 bridge，避免 launcher/deverd 引入 LLVM。原始六目标 bridge 保持独立。 + +每个 runtime/<target>/manifest.json 验证实际目标与 compiler_version/ABI；所选目标的整个输入闭包必须纳入安装签名，缓存键包含目标和该 pack 身份。pack 制作器接受显式准备的目标运行输入，同时只验证/打包 x86 宿主核心，不要求构建 ARM 核心。 + +## Lib 与宿主执行 + +资源路径、runtime/构建收据和 ELF 校验使用部署目标；实际启动的编译工具使用构建宿主。Go 交叉编译工具必须在 x86 执行且输出 ARM，标准库/importcfg 和源码 tags 选择 ARM。Python wheel 与 npm addon 遵守目标 ABI；跨目标源码构建不能把 ARM 工具直接作为 x86 可执行文件启动，也不能无条件接受宿主生成的收据。共用现有显式 Lib 准备入口传递目标，离线 build 只消费完整、已校验的目标材料。lock 模式如需变化直接迁移仓库合同/fixture，不增加旧格式回退。 + +源项目与 Dever Package 的业务声明不增加架构分支。只有明确的部署目标选择和作者工具输入区分架构。缺少目标资产给出明确准备错误，不能退回宿主资源或无沙箱执行。 + +## 验证与运维 + +作者资源/模拟器解包到本任务专用目录，使用绝对工具路径与校验摘要，不全局安装。所有 Cargo/原生构建由主代理串行执行，优先复用现有缓存，开工前检查磁盘与内存预算。跨编译测试验证 ELF、执行结果及坏输入拒绝；共享编译验证目标隔离和命中前完整性校验。模拟 ARM 内核只在任务自有进程/镜像内运行，不修改主机 AppArmor 或已有服务。 + +ARM 真机吞吐和不同硬件/内核兼容性不从模拟结果推导。本次不重新跑既有全量矩阵或长压测。 diff --git a/.trellis/tasks/archive/2026-10/10-04-linux-arm64-cross-build/implement.jsonl b/.trellis/tasks/archive/2026-10/10-04-linux-arm64-cross-build/implement.jsonl new file mode 100644 index 0000000..f80d992 --- /dev/null +++ b/.trellis/tasks/archive/2026-10/10-04-linux-arm64-cross-build/implement.jsonl @@ -0,0 +1,4 @@ +{"file":".trellis/spec/backend/directory-structure.md","reason":"编译/运行库/Lib/资源的唯一owner和根test归属"} +{"file":".trellis/spec/backend/error-handling.md","reason":"显式目标错误与资源/发布清理"} +{"file":".trellis/spec/backend/quality-guidelines.md","reason":"离线定向验证和资源预算"} +{"file":".trellis/spec/backend/toolchain-and-library.md","reason":"运行输入、签名闭包、daemon/cache与生态宿主/目标合同"} diff --git a/.trellis/tasks/archive/2026-10/10-04-linux-arm64-cross-build/implement.md b/.trellis/tasks/archive/2026-10/10-04-linux-arm64-cross-build/implement.md new file mode 100644 index 0000000..4c1a555 --- /dev/null +++ b/.trellis/tasks/archive/2026-10/10-04-linux-arm64-cross-build/implement.md @@ -0,0 +1,61 @@ +# 执行记录 + +## 已确认范围 + +用户已批准 x86 构建 ARM 应用及创建任务继续实施。最终方案沿用已确认目标，没有 ARM 开发工具链/公开发行/长压测。旧生产化父任务的其他平台延期范围，仅为本任务的 Linux ARM 应用部署这一项解除延期。 + +## 收口结果 + +本任务范围全部通过。最终汇总为 `target/arm64-cross/acceptance.json`，逐项绑定构建日志、运行日志和当前产物SHA。最终完整ARM内核模拟日志为 `target/arm64-cross/logs/guest-1791082810005965465.log`：Python、JavaScript、Go各自成功/故障/恢复/清理全部通过，HTTP返回`arm-http`，guest退出0，222.57s，无超时。四profile、私有/共享编译、缓存隔离/篡改拒绝和SQLite真实CRUD由此前独立验收覆盖。 + +本机构建、41项初始定向回归、Lib 73项、Worker 9项、迁移后的真实Go构建运行1项、Python/JavaScript SDK合同、作者Python输入2项及相关Clippy/格式检查通过；这些测试集有交集，不相加成独立总数。最新Python签名构建201.28s通过，包已包含最终reader及completion清理修复。 + +没有ARM真机；未运行ARM真机性能/长测、真实PostgreSQL服务器或全工作区CI。模拟器验证功能，不推导真机吞吐。Python sdist/npm hook跨目标仍需事先准备匹配目标的输出和完整收据；纯Python/npm与Go交叉构建已实际覆盖。产品仍不隐式联网或从环境变量/PATH读取语言工具。未安装/替换全局命令、修改主机AppArmor/sysctl或业务服务，未提交Git。 + +## 实施 + +1. [x] 调查 CLI、runtime pack、daemon/cache、Lib/Worker 目标链路，建立可审查任务。 +2. [x] 编译链路：BuildTarget、build 参数、CompileRequest、runtime/签名/缓存及打包者，配套回归。 +3. [x] Lib 链路：显式目标准备、lock/resource/runtime/Worker/build pack 的宿主与目标分离，错架构拒绝与回归。 +4. [x] 准备可复现 ARM 运行输入与模拟器；构建四 runtime profile 和所需生态资源。 +5. [x] 串行定向检查，执行 ARM 真实应用/三生态模拟验收及 x86 最小回归。 +6. [x] 独立复核、修复实际问题，更新规范/CLI 文档/执行证据，清理自有进程与临时配置，不提交。 + +## 所有权 + +- 编译实施代理：CLI main/compile/worker、toolchain target/runtime_pack/compilation/service/release/cache、packaging.rs 主文件与编译相关根 test。 +- Lib 实施代理：libs.rs 与 libs/**、workers.rs 与 workers/**、packages 的传递 Lib 必需接线、packaging/{ecosystems,builds,sandbox}.rs、生态相关根 test。CLI 参数集成通过主代理/编译代理协调，禁止重写对方文件。 +- 主代理：任务/规范/SDK/README、作者 ARM 资源和模拟验收 owner、所有 Cargo 和重资源命令、最终集成。 + +## 检查原则 + +先参数/目标映射/清单/缓存/锁的定向测试，再用真实 ARM 产物执行。编译命令明确 offline/locked 和单作业；不让子代理并行 Cargo。真实业务服务仅在自有临时项目/端口或模拟机里启动，提前说明。失败保存证据，不放宽断言、不能将未准备目标或模拟器缺失标成通过。 + +初始磁盘可用约8.2GiB；现有 debug 3.1GiB、参考 runtime 948MiB 均保留。准备资源前重新核算，不删除用户不明文件、不恢复旧长测。 + +## 过程与证据（按执行顺序保留） + +- 编译与 Lib 两条接线已实施，新增 BuildTarget、请求/缓存目标绑定、显式目标 Lib 准备及 ELF/Go archive 检查；尚待 Cargo 与真实产物验收，未将代理报告记成通过。 +- 作者工具均在 `target/arm64-cross/`：Rust 1.98 ARM std、GCC 13 ARM cross、QEMU user/system、同版本私有 rustfmt/clippy。未安装全局工具。Ubuntu 包按现有受信 metadata 的 SHA256 验证，ARM OS 库索引另外通过 Ubuntu InRelease 签名验证。 +- ARM base 和 sqlite 已使用 `runtime-pack` 串行构建成功，保存到 `archives/`；postgres/both 仍在构建。初次失败是私有 cross as 找不到同目录 libopcodes，已仅在作者 Cargo 子进程设置私有库路径修复，未加入产品环境变量。 +- Python 3.12.14 ARM standalone 和 Node 24.15.0 ARM 已按上游发布摘要下载并生成目标 pack；真实 six/is-number 包及 Google UUID 现有官方校验证据供后续 Worker 验收。 +- Alpine 3.22.6 ARM 内核完整系统模拟启动成功，日志 `target/arm64-cross/logs/guest-1791077299508695065.log`：`aarch64`、user namespace 存在、`DEVER_ARM_GUEST_EXIT=0`。这只是模拟机准备，不是 Dever/Worker 验收。 +- author native release helper 增加目标参数复用现有 Python/npm选取/manifest；Go author fixture通过私有 Go overlay 固定交叉工具默认目标（源码未改全局SDK），待实际构建与执行。 + +- 四份 ARM runtime 全部通过：base 311.91s/81,418,028B、sqlite 362.84s/96,034,088B、postgres 338.03s/102,287,968B、both 237.84s/109,661,934B。总作者构建20m52s，峰值1.9GiB、无swap/OOM；每份SHA和命令在对应log JSON。`target/debug/runtime/linux-aarch64`已按真实输入准备，未改原x86 pack。 +- 新host工具构建通过；定向测试39项通过：CLI参数3、native maker24、请求目标2、LLVM CLI5、共享缓存1、resolver目标1、exec架构1、managed runtime目标2。未执行的ignored项未算通过。作者Python输入测试2项通过。 +- `trellis-check`独立复核修复1处多余借用，17个生产Rust文件格式检查通过；未发现目标数据流/签名/缓存的功能性阻塞。主代理继续负责实际测试，不以复核替代执行。 +- 实际签名安装/shared+private ARM构建验收1/1通过（177.32s）：4共享profile + 私有base均在删除源码与机器目录后由QEMU执行成功；SQLite真实CRUD、ARM/x86缓存隔离、重复命中、命中前篡改拒绝通过。结果 `target/arm64-cross/compile-acceptance.json`，HTTP应用已导出但尚待完整guest运行。未运行真实PG服务器。 +- ARM guard已按私有sysroot成功链接，sandbox OS assets准备完成；前两次作者链接失败（缺libgcc_s、GNU脚本绝对sysroot）均保留日志，修复只在私有作者输入/链接参数，不改变产品配置。Go交叉pack正在构建。 + +- Go交叉pack已成功（144.86s）；签名安装后的三生态ARM应用离线构建1/1通过（453.51s），各应用导出后删除源码和机器目录。完整guest运行尚未通过，构建成功不算Worker执行成功。 +- 完整模拟定位两个独立问题：fixture直接从initramfs rootfs运行导致bwrap `pivot_root: EINVAL`，已改为switch_root到正常tmpfs根；随后日志 `guest-1791080132703634001.log`明确显示ARM loader为0600，guard启动Permission denied。 +- 根因属隐含宿主假设/跨层传播与测试缺口：`asset_executable`只识别宿主loader，ELF架构校验正确但打包和嵌入权限错误；首轮复核未覆盖这条权限数据流。已在共享owner精确识别两种目标loader，并增加真实pack权限回归（输入0600→loader/guard/bwrap0755、库0644）和负例。目标架构校验、guard和沙箱策略未放宽；规范已补齐。主代理正在重建host工具及全部三生态产物复验，旧产物不能作为修复证据。 +- loader修复后host工具重建通过，定向测试41/41通过（原39项加sandbox默认2项）；三生态应用重新生成通过（461.13s，日志 `ecosystems-1791080644145693595.log`）。旧失败应用保存在 `target/arm64-cross/failed-loader-mode/`，当前guest/apps均是修复后产物。 +- 完整系统验收脚本固化在根test下 `test/ecosystem-release/arm64/{simulate.py,init.sh,bootstrap.sh,guest.sh}`；独立复核补上源码定位的预期错误、每次调用后 `/worker/*` 进程检查、流水线子进程回收，语法检查通过。完整guest正在运行。 +- 四ARM静态runtime/guard沿用前述已验证构建：本次最后修复的`asset_executable`仅由宿主作者/CLI打包与嵌入路径调用，ARM运行期不调用它；重新构建host core并重生成应用资源标记即覆盖实际修复，不重复20分钟runtime构建。 +- `guest-1791081118958747964.log`已得到Python `data:true`，随后Worker以134退出，验收正确判失败。宿主SDK合同用例保持stdin打开后稳定复现 `_enter_buffered_busy`：daemon读线程持有BufferedReader锁，之前测试先关stdin掩盖了退出竞争。SDK已改用asyncio管道与有界队列，finally取消并等待reader、关闭transport；修改后的整个Python SDK握手/调用/业务错误/取消/关闭/拒绝检查通过。只需重嵌入Python SDK，ARM runtime/Node/Go源字节未变；共用原签名构建fixture增加Python定向入口，不复制构建流程。 +- SDK复核补齐有界队列中completion通知任务的取消/等待，`test/external-sdk/check.py`新增Python队列满时关闭、保留stdin等待退出且stderr为空的回归。Python完整合同与JavaScript原合同检查均通过。该保留stdin回归针对CPython退出竞争；JavaScript继续遵循原测试/真实host在shutdown应答后关闭stdin的流程。 +- `guest-1791081776953594061.log`完整通过Python成功/故障/恢复/清理，但整个三生态模拟在180s总上限时被停止，记为未通过。模拟器需反复读取校验大型独立程序，后续有限总预算改为600s并增加每步marker；这是9次调用加HTTP的功能验收，不是长压测。 +- 旧host Go fixture缺少新增的`build.host`，已校验原SHA及358个leaf，保留工具/stdlib原字节和mode，仅重封装descriptor并同步6个作者文件、锁及prepared摘要；原文件完整备份在 `target/arm64-cross/host-go/backup`。新固定SHA `8818a1eccb510948a55f63088c01678391a5e0c2236e6eacadfe45d0a5edbb43`，prepare.py和SDK文档同步，未改其他语言收据和UUID/sumdb。实际host Go重复编译/移除源码后运行回归1/1通过（18.06s）。 +- 静态检查修复沙箱条件嵌套、Go编译参数过多（组合现有构建输入）、共享ELF测试fixture在两个新消费者中的未用辅助方法告警；CLI相关bin/test及sandbox全部target的Clippy `-D warnings`通过。受影响Lib默认73/73、Worker默认9/9通过；各10项ignored未算通过。最后改动Rust格式检查通过，作者Python输入2/2通过。 diff --git a/.trellis/tasks/archive/2026-10/10-04-linux-arm64-cross-build/prd.md b/.trellis/tasks/archive/2026-10/10-04-linux-arm64-cross-build/prd.md new file mode 100644 index 0000000..90eabd1 --- /dev/null +++ b/.trellis/tasks/archive/2026-10/10-04-linux-arm64-cross-build/prd.md @@ -0,0 +1,33 @@ +# Linux ARM64 应用交叉构建 + +## 目标 + +在现有 Linux x86_64 开发机执行 Dever build，得到能在 Linux ARM64 独立运行的应用。部署端不安装 Dever 编译器、Rust、Cargo 或 LLVM。用户已确认上述范围及创建任务、继续实施；不重复要求实施确认。 + +## 实施前基线 + +- LLVM bridge 已有 LinuxAarch64 对象生成和链接；CLI compile.rs、toolchain/runtime_pack.rs 仍按 host 选 pack。 +- toolchain/compilation.rs 的请求未携带产物目标；service/compile.rs 的缓存目标固定为 host。 +- libs.rs、workers.rs 与 Go/build pack 的验证多处把运行目标视为开发机；不能仅改最终主程序而混入 x86 资源。 +- 启动本任务时没有 ARM 真机及模拟器。准备作者工具与 ARM 资源属于本任务，模拟通过不等于真机性能验收。 + +## 要求 + +1. `dever build <root> --target linux-aarch64 --output <new-file>` 选择产物目标；省略目标保持本机构建。run/test 继续本机执行，未知/重复参数明确失败。 +2. 私有 CLI 与机器共享编译均传递同一目标；验证正确版本、ABI、目标 pack 和签名闭包，缓存按目标隔离。不能用宿主 pack 替代缺失 ARM pack。 +3. 制作并验证 ARM64 的 base/sqlite/postgres/both 运行库及必要 CRT/静态库；构建端保持 x86_64。 +4. Lib 准备/锁定/离线打包区分构建宿主与部署目标，Python/Node 解释器、Go/exec Worker、原生扩展和沙箱资产不得混入错误架构。复用原协议、resolver 和资源 owner，不另建跨编译框架。 +5. 产品配置只用 config/setting.json；run/build 不联网、不读取环境变量或系统 PATH 获取语言工具。准备目标依赖必须显式进行。 +6. 用模拟环境运行生成的真实应用与目标 Worker，验证输出、失败和清理；涉及 namespace/seccomp 的验收使用带 ARM 内核的完整系统模拟，不能拿用户态模拟结果替代沙箱证据。 + +## 验收 + +- [x] 默认本机 check/run/build 行为及参数拒绝定向回归通过。 +- [x] x86 构建的应用 ELF 标记 AArch64，模拟器中执行实际 CMD/HTTP/数据库路径并得到预期输出。 +- [x] 四 profile 都使用真实 ARM 输入；私有与共享构建命中正确目标缓存，缺失/错误/篡改目标输入均拒绝。 +- [x] pip/npm/go 与 exec 的目标资源选择、锁定和错架构拒绝有回归；准备的 ARM 三生态应用能在模拟环境独立运行。 +- [x] 产物不依赖 ARM 编译器或源项目，构建不隐式联网；最终日志明确记录通过、未运行和阻塞。 + +## 不在范围 + +ARM 上开发/编译 Dever、macOS/Windows、公开发行/新签名体系、真机性能及长压测。保留现有 x86 验收证据，不重复全量 CI；不改宿主 AppArmor、业务服务或全局工具。 diff --git a/.trellis/tasks/archive/2026-10/10-04-linux-arm64-cross-build/task.json b/.trellis/tasks/archive/2026-10/10-04-linux-arm64-cross-build/task.json new file mode 100644 index 0000000..9a7c9d4 --- /dev/null +++ b/.trellis/tasks/archive/2026-10/10-04-linux-arm64-cross-build/task.json @@ -0,0 +1,26 @@ +{ + "id": "linux-arm64-cross-build", + "name": "linux-arm64-cross-build", + "title": "Linux ARM64 应用交叉构建", + "description": "在Linux x86_64开发机生成可在Linux ARM64运行的应用二进制，覆盖目标运行库、Lib资源、共享缓存和模拟验收；不移植ARM开发工具链。", + "status": "completed", + "dev_type": null, + "scope": null, + "package": null, + "priority": "P2", + "creator": "Antigravity", + "assignee": "Antigravity", + "createdAt": "2026-10-04", + "completedAt": "2026-10-04", + "branch": "main", + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [], + "parent": null, + "relatedFiles": [], + "notes": "", + "meta": {} +} \ No newline at end of file diff --git a/.trellis/workflow.md b/.trellis/workflow.md new file mode 100644 index 0000000..0f81c27 --- /dev/null +++ b/.trellis/workflow.md @@ -0,0 +1,725 @@ +# Development Workflow + +--- + +## Core Principles + +1. **Plan before code** — figure out what to do before you start +2. **Specs injected, not remembered** — guidelines are injected via hook/skill, not recalled from memory +3. **Persist everything** — research, decisions, and lessons all go to files; conversations get compacted, files don't +4. **Incremental development** — one task at a time +5. **Capture learnings** — after each task, review and write new knowledge back to spec + +--- + +## Trellis System + +### Developer Identity + +On first use, initialize your identity: + +```bash +python3 ./.trellis/scripts/init_developer.py <your-name> +``` + +Creates `.trellis/.developer` (gitignored) + `.trellis/workspace/<your-name>/`. + +### Spec System + +`.trellis/spec/` holds coding guidelines organized by package and layer. + +- `.trellis/spec/<package>/<layer>/index.md` — entry point with **Pre-Development Checklist** + **Quality Check**. Actual guidelines live in the `.md` files it points to. +- `.trellis/spec/guides/index.md` — cross-package thinking guides. + +```bash +python3 ./.trellis/scripts/get_context.py --mode packages # list packages / layers +``` + +**When to update spec**: new pattern/convention found · bug-fix prevention to codify · new technical decision. + +### Task System + +Every task has its own directory under `.trellis/tasks/{MM-DD-name}/` holding `task.json`, `prd.md`, optional `design.md`, optional `implement.md`, optional `research/`, and context manifests (`implement.jsonl`, `check.jsonl`) for sub-agent-capable platforms. + +```bash +# Task lifecycle +python3 ./.trellis/scripts/task.py create "<title>" [--slug <name>] [--parent <dir>] +python3 ./.trellis/scripts/task.py start <name> # set active task (session-scoped when available) +python3 ./.trellis/scripts/task.py current --source # show active task and source +python3 ./.trellis/scripts/task.py finish # clear active task (triggers after_finish hooks) +python3 ./.trellis/scripts/task.py archive <name> # move to archive/{year-month}/ +python3 ./.trellis/scripts/task.py list [--mine] [--status <s>] +python3 ./.trellis/scripts/task.py list-archive + +# Code-spec context (injected into implement/check agents via JSONL). +# `implement.jsonl` / `check.jsonl` are seeded (empty) on `task create` for sub-agent-capable +# platforms; the AI curates real spec + research entries during planning. `validate` fails +# and `start` refuses while a seeded manifest is still empty — sub-agents would run with +# zero spec context. Pass `start --allow-empty-context` when that is intentional. +python3 ./.trellis/scripts/task.py add-context <name> <action> <file> <reason> +python3 ./.trellis/scripts/task.py list-context <name> [action] +python3 ./.trellis/scripts/task.py validate <name> + +# Task metadata +python3 ./.trellis/scripts/task.py set-branch <name> <branch> +python3 ./.trellis/scripts/task.py set-base-branch <name> <branch> # PR target +python3 ./.trellis/scripts/task.py set-scope <name> <scope> + +# Hierarchy (parent/child) +python3 ./.trellis/scripts/task.py add-subtask <parent> <child> +python3 ./.trellis/scripts/task.py remove-subtask <parent> <child> + +# PR creation +python3 ./.trellis/scripts/task.py create-pr [name] [--dry-run] +``` + +> Run `python3 ./.trellis/scripts/task.py --help` to see the authoritative, up-to-date list. + +**Current-task mechanism**: `task.py create` creates the task directory and (when session identity is available) auto-sets the per-session active-task pointer so the planning breadcrumb fires immediately. `task.py start` writes the same pointer (idempotent if already set) and flips `task.json.status` from `planning` to `in_progress`. State is stored under `.trellis/.runtime/sessions/`. If no context key is available from hook input, `TRELLIS_CONTEXT_ID`, or a platform-native session environment variable, there is no active task and `task.py start` fails with a session identity hint. `task.py finish` deletes the current session file (status unchanged). `task.py archive <task>` writes `status=completed`, moves the directory to `archive/`, and deletes any runtime session files that still point at the archived task. + +### Workspace System + +Records every AI session for cross-session tracking under `.trellis/workspace/<developer>/`. + +- `journal-N.md` — session log. **Max 2000 lines per file**; a new `journal-(N+1).md` is auto-created when exceeded. +- `index.md` — personal index (total sessions, last active). + +```bash +python3 ./.trellis/scripts/add_session.py --title "Title" --commit "hash" --summary "Summary" +``` + +### Context Script + +```bash +python3 ./.trellis/scripts/get_context.py # full session runtime +python3 ./.trellis/scripts/get_context.py --mode packages # available packages + spec layers +python3 ./.trellis/scripts/get_context.py --mode phase --step <X.Y> # detailed guide for a workflow step +``` + +--- + +<!-- + WORKFLOW-STATE BREADCRUMB CONTRACT (read this before editing the tag blocks below) + + The [workflow-state:STATUS] blocks embedded in the ## Phase Index section + below are the SINGLE source of truth for the per-turn `<workflow-state>` + breadcrumb that every supported AI platform's UserPromptSubmit hook + reads. inject-workflow-state.py (Python platforms) and + inject-workflow-state.js (OpenCode plugin) only parse them — there is no + fallback dict baked into the scripts after v0.5.0-rc.0. + + STATUS charset: [A-Za-z0-9_-]+. When the hook can't find a tag, it + degrades to a generic "Refer to workflow.md for current step." line — + intentionally visible so users notice and fix a broken workflow.md. + + INVARIANT (test/regression.test.ts): + Every workflow-walkthrough step marked `[required · once]` must have a + matching enforcement line in its phase's [workflow-state:*] block. The + breadcrumb is the only per-turn channel; if a mandatory step isn't + mentioned there, the AI silently skips it (Phase 1 planning gate + skip and Phase 3.4 commit skip both manifested via this gap). + + TAG ↔ PHASE scoping: + [workflow-state:no_task] → no active task; before Phase 1 + [workflow-state:task_error] → active task record is unreadable; repair it before continuing + [workflow-state:planning] → all of Phase 1 (status='planning') + [workflow-state:planning-inline] → Codex inline variant of Phase 1 + [workflow-state:in_progress] → Phase 2 + Phase 3.2-3.4 + (status stays 'in_progress' from + task.py start until task.py archive) + [workflow-state:in_progress-inline] → Codex inline variant of Phase 2/3 + [workflow-state:completed] → currently DEAD: cmd_archive flips + status and moves the dir in the same + call, so the resolver loses the + pointer (block kept for a future + explicit in_progress→completed + transition) + + Editing checklist: + - When you change a [workflow-state:STATUS] block, also check the + matching phase's `[required · once]` walkthrough steps for sync + - Run `trellis update` after editing to push the new bodies to + downstream user projects (block-level managed replacement) + - Full runtime contract: + .trellis/spec/cli/backend/workflow-state-contract.md +--> + +## Phase Index + +``` +Phase 1: Plan → classify, get task-creation consent, then write planning artifacts +Phase 2: Execute → implement only after task status is in_progress +Phase 3: Finish → verify, update spec, commit, and wrap up +``` + +### Request Triage + +- Simple conversation or read-only request: handle it directly without creating a Trellis task or asking whether to create one. +- Small, self-contained implementation task: handle it directly without creating a Trellis task or asking whether to create one. +- Complex task: ask whether you may create a Trellis task and enter planning. If the user says no, do not do broad inline implementation; explain, clarify scope, or suggest a smaller split. +- If the user already declined task creation in the current conversation, do not ask again unless the requested scope materially changes. +- User approval to create a task is not approval to start implementation. Planning still happens first. + +### Planning Artifacts + +- `prd.md` — requirements, constraints, and acceptance criteria. Do not put technical design or execution checklists here. +- `design.md` — technical design for complex tasks: boundaries, contracts, data flow, tradeoffs, compatibility, rollout / rollback shape. +- `implement.md` — execution plan for complex tasks: ordered checklist, validation commands, review gates, and rollback points. +- `implement.jsonl` / `check.jsonl` — spec and research manifests for sub-agent context. They do not replace `implement.md`. +- Lightweight tasks may be PRD-only. Complex tasks must have `prd.md`, `design.md`, and `implement.md` before `task.py start`. + +### Parent / Child Task Trees + +Use a parent task when one user request contains several independently verifiable deliverables. The parent task owns the source requirement set, the task map, cross-child acceptance criteria, and final integration review; it normally should not be the implementation target unless it also has direct work. + +Use child tasks for deliverables that can be planned, implemented, checked, and archived independently. Parent/child structure is not a dependency system: if one child must wait for another, write that ordering in the child `prd.md` / `implement.md` and keep each child's acceptance criteria testable. + +Create new children with `task.py create "<title>" --slug <name> --parent <parent-dir>`. Link existing tasks with `task.py add-subtask <parent> <child>`, and unlink mistakes with `task.py remove-subtask <parent> <child>`. + +<!-- Per-turn breadcrumb: shown when there is no active task (before Phase 1) --> + +[workflow-state:no_task] +No active task. Classify the current turn before acting. +Simple conversation or read-only request: respond directly. Do not ask whether to create a Trellis task. +Small, self-contained implementation task: implement and verify it directly without asking whether to create a Trellis task. +Complex task: ask whether you may create a Trellis task and enter planning. If the user says no, explain, clarify scope, or suggest a smaller split. +If the user already declined task creation in the current conversation, do not ask again unless the requested scope materially changes. +[/workflow-state:no_task] + +<!-- Per-turn breadcrumb: shown when the active task record cannot be read. --> + +[workflow-state:task_error] +The active task record could not be read. Do not create or activate another task. +Inspect the task directory named above and repair its task.json. It must be a valid JSON object with a non-empty status. +Preserve existing task fields and artifacts. If the correct status cannot be determined safely, ask the user before reconstructing the record. +[/workflow-state:task_error] + +### Phase 1: Plan +- 1.0 Create task `[required · once]` (only after task-creation consent) +- 1.1 Requirement exploration `[required · repeatable]` (`prd.md`; complex tasks also need `design.md` + `implement.md`) +- 1.2 Research `[optional · repeatable]` +- 1.3 Configure context `[required · once]` — Claude Code, Cursor, OpenCode, Codex, Kiro, Gemini, Qoder, CodeBuddy, Copilot, Droid, Pi, Oh My Pi, ZCode, Snow, Reasonix, Grok, Kimi Code (sub-agent-dispatch platforms only; inline platforms skip) +- 1.4 Activate task `[required · once]` (review gate, then `task.py start`; status → in_progress) +- 1.5 Completion criteria + +<!-- Per-turn breadcrumb: shown throughout Phase 1 (status='planning') --> + +[workflow-state:planning] +Load `trellis-brainstorm`; stay in planning. +Lightweight: `prd.md` can be enough. Complex: finish `prd.md`, `design.md`, and `implement.md`; ask for review before `task.py start`. +Multi-deliverable scope: consider a parent task plus independently verifiable child tasks; dependencies must be written in child artifacts, not implied by tree position. +Sub-agent mode: curate `implement.jsonl` and `check.jsonl` as spec/research manifests before start. +[/workflow-state:planning] + +<!-- Per-turn breadcrumb: shown throughout Phase 1 when codex.dispatch_mode=inline. + Codex-only opt-in alternate to [workflow-state:planning]. The main agent + edits code directly in Phase 2, so jsonl curation is skipped — + the inline workflow loads `trellis-before-dev` instead of injecting JSONL + into a sub-agent. --> + +[workflow-state:planning-inline] +Load `trellis-brainstorm`; stay in planning. +Lightweight: `prd.md` can be enough. Complex: finish `prd.md`, `design.md`, and `implement.md`; ask for review before `task.py start`. +Multi-deliverable scope: consider a parent task plus independently verifiable child tasks; dependencies must be written in child artifacts, not implied by tree position. +Inline mode: skip jsonl curation; Phase 2 reads artifacts/specs via `trellis-before-dev`. +[/workflow-state:planning-inline] + +### Phase 2: Execute +- 2.1 Implement `[required · repeatable]` +- 2.2 Quality check `[required · repeatable]` +- 2.3 Rollback `[on demand]` + +<!-- Per-turn breadcrumb: shown while status='in_progress'. + Scope: all of Phase 2 + Phase 3.2-3.4 (status stays 'in_progress' from + task.py start until task.py archive; only archive flips it). The body + therefore must cover every required step from implementation through + commit, including Phase 3.3 spec update and Phase 3.4 commit. --> + +Sub-agent dispatch protocol applies to all platforms and all sub-agents, including native Codex `SubagentStart` context injection with child-side pull fallback, class-2 Gemini/Qoder/Copilot/Reasonix/Trae/Grok/Kimi Code, hook-backed ZCode/Snow, and `trellis-research`: every dispatch prompt starts with `Active task: <task path from task.py current>` before role-specific instructions. On Grok Build, use `spawn_subagent` with `subagent_type` set to the Trellis agent name (e.g. `trellis-implement`). On Kimi Code, dispatch the built-in `coder` / `explore` sub-agent with the matching `.kimi-code/skills/trellis-<role>/SKILL.md` instructions. + +[workflow-state:in_progress] +Tools: `trellis-implement` / `trellis-research` are sub-agent types only (Task/Agent tool, NOT Skill; there is no skill by these names). `trellis-update-spec` is a skill. `trellis-check` exists as both; prefer the Agent form when verifying after code changes. +Flow: `trellis-implement` -> `trellis-check` -> `trellis-update-spec` -> commit (Phase 3.4) -> `/trellis:finish-work`. +Main-session default: dispatch implement/check sub-agents. Sub-agent self-exemption: if already running as `trellis-implement`, do NOT spawn another `trellis-implement` or `trellis-check`; if already running as `trellis-check`, do NOT spawn another `trellis-check` or `trellis-implement`. Dispatch is main session only. +Dispatch prompt starts with `Active task: <task path from task.py current>`. Read context: jsonl entries -> `prd.md` -> `design.md if present` -> `implement.md if present`. +[/workflow-state:in_progress] + +<!-- Per-turn breadcrumb: shown while status='in_progress' when + codex.dispatch_mode=inline. Codex-only opt-in alternate to + [workflow-state:in_progress]. The main session edits code directly + instead of dispatching sub-agents. --> + +[workflow-state:in_progress-inline] +Flow: `trellis-before-dev` -> edit -> `trellis-check` -> validation -> `trellis-update-spec` -> commit (Phase 3.4) -> `/trellis:finish-work`. +Do not dispatch implement/check sub-agents in inline mode. +Read context: `prd.md` -> `design.md if present` -> `implement.md if present`, plus relevant spec/research loaded by skills. +[/workflow-state:in_progress-inline] + +### Phase 3: Finish +- 3.2 Debug retrospective `[on demand]` +- 3.3 Spec update `[required · once]` +- 3.4 Commit changes `[required · once]` +- 3.5 Wrap-up reminder + +> Note: step 3.1 was folded into 2.2 (last-iteration full-scope check) and 3.4 (commit preamble). Numbering kept stable to avoid breaking external references. + +<!-- Per-turn breadcrumb: shown while status='completed'. + Currently DEAD in normal flow: cmd_archive writes status='completed' in + the same call that moves the task dir to archive/, so the active-task + resolver loses the pointer and the hook never fires on archived tasks. + Block preserved for a future status-transition redesign (e.g. an + explicit in_progress→completed command). Edit through the same spec + channel as the live blocks. --> + +[workflow-state:completed] +Code committed. Run `/trellis:finish-work`; if dirty, return to Phase 3.4 first. +[/workflow-state:completed] + +### Rules + +1. Identify which Phase you're in, then continue from the next step there +2. Run steps in order inside each Phase; `[required]` steps can't be skipped +3. Phases can roll back (e.g., Execute reveals a prd defect → return to Plan to fix, then re-enter Execute) +4. Steps tagged `[once]` are skipped if the output already exists; don't re-run +5. Artifact presence informs the next step; missing `design.md` / `implement.md` is valid for lightweight tasks and incomplete planning for complex tasks. + +### Active Task Routing + +When a user request matches one of these intents inside an active task, route first, then load the detailed phase step if needed. + +[Claude Code, Cursor, OpenCode, codex-sub-agent, Kiro, Gemini, Qoder, CodeBuddy, Copilot, Droid, Pi, Oh My Pi, ZCode, Snow, Reasonix, Trae, Grok, Kimi Code] + +- Planning or unclear requirements -> `trellis-brainstorm`. +- `in_progress` implementation/check -> dispatch `trellis-implement` / `trellis-check`. +- Repeated debugging -> `trellis-break-loop`; spec updates -> `trellis-update-spec`. + +[/Claude Code, Cursor, OpenCode, codex-sub-agent, Kiro, Gemini, Qoder, CodeBuddy, Copilot, Droid, Pi, Oh My Pi, ZCode, Snow, Reasonix, Trae, Grok, Kimi Code] + +[codex-inline, Kilo, Antigravity, Devin, DeepSeek Harness] + +- Planning or unclear requirements -> `trellis-brainstorm`. +- Before editing -> `trellis-before-dev`; after editing -> `trellis-check`. +- Repeated debugging -> `trellis-break-loop`; spec updates -> `trellis-update-spec`. + +[/codex-inline, Kilo, Antigravity, Devin, DeepSeek Harness] + +### Guardrails + +- Task creation approval is not implementation approval; implementation waits for `task.py start` after artifact review. +- PRD-only is valid for lightweight tasks; complex tasks need `design.md` + `implement.md`. +- Planning must be persisted to task artifacts; checks must run before reporting completion. + +### Loading Step Detail + +At each step, run this to fetch detailed guidance: + +```bash +python3 ./.trellis/scripts/get_context.py --mode phase --step <step> +# e.g. python3 ./.trellis/scripts/get_context.py --mode phase --step 1.1 +``` + +--- + +## Phase 1: Plan + +Goal: classify the request, get task-creation consent when a task is needed, and produce the planning artifacts required before implementation. + +#### 1.0 Create task `[required · once]` + +Create the task directory only after task-creation consent. The command sets status to `planning`, writes `task.json`, creates a default `prd.md`, and auto-targets the new task when session identity is available: + +```bash +python3 ./.trellis/scripts/task.py create "<task title>" --slug <name> +``` + +`--slug` is the human-readable name only. Do **not** include the `MM-DD-` date prefix; `task.py create` adds that prefix automatically. + +For task trees, create the parent task first and then create each child with `--parent <parent-dir>`. Do not start the parent just because children exist; start the child that owns the next independently verifiable deliverable. + +After this command succeeds, the per-turn breadcrumb auto-switches to `[workflow-state:planning]`, telling the AI to stay in planning. + +Run only `create` here — do not also run `start`. `start` flips status to `in_progress`, which switches the breadcrumb to the implementation phase before planning artifacts are reviewed. Save `start` for step 1.4. + +Skip when `python3 ./.trellis/scripts/task.py current --source` already points to a task. + +#### 1.1 Requirement exploration `[required · repeatable]` + +Load the `trellis-brainstorm` skill and explore requirements interactively with the user per the skill's guidance. + +The brainstorm skill will guide you to: +- Ask one question at a time +- Prefer researching over asking the user +- Prefer offering options over open-ended questions +- Update `prd.md` immediately after each user answer +- Split large scopes into a parent task plus child tasks when the deliverables can be verified independently +- Keep `prd.md` focused on requirements and acceptance criteria +- For complex tasks, produce `design.md` and `implement.md` before implementation starts + +When considering a parent/child split: +- Use a parent task when one request contains several independently verifiable deliverables. +- Parent tasks own source requirements, child-task mapping, cross-child acceptance criteria, and final integration review. +- Child tasks own actual deliverables that can be planned, implemented, checked, and archived independently. +- Parent/child structure is not a dependency system. If child B depends on child A, write that ordering in child B's `prd.md` / `implement.md`. +- Start the child task that owns the next deliverable. Do not start the parent unless the parent itself has direct implementation work. + +Return to this step whenever requirements change and revise the relevant artifact. + +#### 1.2 Research `[optional · repeatable]` + +Research can happen at any time during requirement exploration. It isn't limited to local code — you can use any available tool (MCP servers, skills, web search, etc.) to look up external information, including third-party library docs, industry practices, API references, etc. + +[Claude Code, Cursor, OpenCode, codex-sub-agent, Kiro, Gemini, Qoder, CodeBuddy, Copilot, Droid, Pi, Oh My Pi, ZCode, Snow, Reasonix, Trae, Grok, Kimi Code] + +Spawn the research sub-agent: + +- **Agent type**: `trellis-research` +- **Task description**: Research <specific question> +- **Key requirement**: Research output MUST be persisted to `{TASK_DIR}/research/` + +[/Claude Code, Cursor, OpenCode, codex-sub-agent, Kiro, Gemini, Qoder, CodeBuddy, Copilot, Droid, Pi, Oh My Pi, ZCode, Snow, Reasonix, Trae, Grok, Kimi Code] + +[codex-inline, Kilo, Antigravity, Devin, DeepSeek Harness] + +Do the research in the main session directly and write findings into `{TASK_DIR}/research/`. `codex-inline` is the explicit mode that keeps work in the main session. + +[/codex-inline, Kilo, Antigravity, Devin, DeepSeek Harness] + +**Research artifact conventions**: +- One file per research topic (e.g. `research/auth-library-comparison.md`) +- Record third-party library usage examples, API references, version constraints in files +- Note relevant spec file paths you discovered for later reference + +Brainstorm and research can interleave freely — pause to research a technical question, then return to talk with the user. + +**Key principle**: Research output must be written to files, not left only in the chat. Conversations get compacted; files don't. + +#### 1.3 Configure context `[required · once]` + +[Claude Code, Cursor, OpenCode, codex-sub-agent, Kiro, Gemini, Qoder, CodeBuddy, Copilot, Droid, Pi, Oh My Pi, ZCode, Snow, Reasonix, Trae, Grok, Kimi Code] + +Curate `implement.jsonl` and `check.jsonl` so the Phase 2 sub-agents get the right spec/research context. These files were seeded on `task create` with a single self-describing `_example` line; your job here is to fill in real entries. + +**Location**: `{TASK_DIR}/implement.jsonl` and `{TASK_DIR}/check.jsonl` (already exist). + +**Format**: one JSON object per line — `{"file": "<path>", "reason": "<why>"}`. Paths are repo-root relative. + +**What to put in**: +- **Spec files** — `.trellis/spec/<package>/<layer>/index.md` and any specific guideline files (`error-handling.md`, `conventions.md`, etc.) relevant to this task +- **Research files** — `{TASK_DIR}/research/*.md` that the sub-agent will need to consult + +**What NOT to put in**: +- Code files (`src/**`, `packages/**/*.ts`, etc.) — those are read by the sub-agent during implementation, not pre-registered here +- Files you're about to modify — same reason + +**Split between the two files**: +- `implement.jsonl` → specs + research the implement sub-agent needs to write code correctly +- `check.jsonl` → specs for the check sub-agent (quality guidelines, check conventions, same research if needed) + +These manifests do not replace `implement.md`. `implement.md` is the human-readable execution plan for a complex task; jsonl files only list context files to inject or load. + +**How to discover relevant specs**: + +```bash +python3 ./.trellis/scripts/get_context.py --mode packages +``` + +Lists every package + its spec layers with paths. Pick the entries that match this task's domain. + +**How to append entries**: + +Either edit the jsonl file directly in your editor, or use: + +```bash +python3 ./.trellis/scripts/task.py add-context "$TASK_DIR" implement "<path>" "<reason>" +python3 ./.trellis/scripts/task.py add-context "$TASK_DIR" check "<path>" "<reason>" +``` + +Delete the seed `_example` line once real entries exist (optional — it's skipped automatically by consumers). + +Ready gate: both `implement.jsonl` and `check.jsonl` must contain at least one real `{"file": "...", "reason": "..."}` entry before `task.py start`. The seed `_example` row alone is not ready. + +Skip this step only when both files already have real curated entries. + +[/Claude Code, Cursor, OpenCode, codex-sub-agent, Kiro, Gemini, Qoder, CodeBuddy, Copilot, Droid, Pi, Oh My Pi, ZCode, Snow, Reasonix, Trae, Grok, Kimi Code] + +[codex-inline, Kilo, Antigravity, Devin, DeepSeek Harness] + +Skip this step. Context is loaded directly by the `trellis-before-dev` skill in Phase 2. + +[/codex-inline, Kilo, Antigravity, Devin, DeepSeek Harness] + +#### 1.4 Activate task `[required · once]` + +After artifact review, flip the task status to `in_progress`: + +```bash +python3 ./.trellis/scripts/task.py start <task-dir> +``` + +For lightweight tasks, `prd.md` can be enough. For complex tasks, `prd.md`, `design.md`, and `implement.md` must exist and be reviewed before start. On sub-agent-dispatch platforms, `implement.jsonl` and `check.jsonl` must both have real curated entries before start. Runtime consumers tolerate missing or seed-only manifests for compatibility, but that tolerance is not a planning-ready state. + +After this command succeeds, the breadcrumb auto-switches to `[workflow-state:in_progress]`, and the rest of Phase 2 / 3 follows. + +If `task.py start` errors with a session-identity message (no context key from hook input, `TRELLIS_CONTEXT_ID`, or platform-native session env), follow the hint in the error to set up session identity, then retry. + +#### 1.5 Completion criteria + +| Condition | Required | +|------|:---:| +| `prd.md` exists | ✅ | +| User confirms task should enter implementation | ✅ | +| `task.py start` has been run (status = in_progress) | ✅ | +| `research/` has artifacts (complex tasks) | recommended | +| `design.md` exists (complex tasks) | ✅ | +| `implement.md` exists (complex tasks) | ✅ | + +[Claude Code, Cursor, OpenCode, codex-sub-agent, Kiro, Gemini, Qoder, CodeBuddy, Copilot, Droid, Pi, Oh My Pi, ZCode, Snow, Reasonix, Trae, Grok, Kimi Code] + +| `implement.jsonl` and `check.jsonl` each contain at least one real curated entry (seed row does not count) | ✅ | + +[/Claude Code, Cursor, OpenCode, codex-sub-agent, Kiro, Gemini, Qoder, CodeBuddy, Copilot, Droid, Pi, Oh My Pi, ZCode, Snow, Reasonix, Trae, Grok, Kimi Code] + +--- + +## Phase 2: Execute + +Goal: turn reviewed planning artifacts into code that passes quality checks. + +#### 2.1 Implement `[required · repeatable]` + +[Claude Code, Cursor, OpenCode, codex-sub-agent, CodeBuddy, Droid, Pi, ZCode, Snow, Oh My Pi] + +Spawn the implement sub-agent: + +- **Agent type**: `trellis-implement` +- **Task description**: Implement the reviewed task artifacts, consulting materials under `{TASK_DIR}/research/`; finish by running project lint and type-check +- **Dispatch prompt guard**: The prompt MUST start with `Active task: <task path>`, then tell the spawned agent it is already the `trellis-implement` sub-agent and must implement directly, not spawn another `trellis-implement` / `trellis-check`. + +The platform hook/plugin auto-handles: +- Reads `implement.jsonl` and injects referenced spec/research files into the agent prompt +- Injects `prd.md`, `design.md` if present, and `implement.md` if present +- For Codex, `SubagentStart` supplies native context injection; the agent profile keeps child-side loading as the fallback + +[/Claude Code, Cursor, OpenCode, codex-sub-agent, CodeBuddy, Droid, Pi, ZCode, Snow, Oh My Pi] + +[Gemini, Qoder, Copilot, Reasonix, Trae, Grok, Kimi Code] + +Spawn the implement sub-agent: + +- **Agent type**: `trellis-implement` +- **Task description**: Implement the reviewed task artifacts, consulting materials under `{TASK_DIR}/research/`; finish by running project lint and type-check +- **Dispatch prompt guard**: The prompt MUST start with `Active task: <task path>`, then explicitly say the spawned agent is already `trellis-implement` and must implement directly without spawning another `trellis-implement` / `trellis-check`. + +The pull-based sub-agent definition auto-handles the context load requirement: +- Resolves the active task with `task.py current --source`, then reads `prd.md`, `design.md` if present, and `implement.md` if present +- Reads `implement.jsonl` and requires the agent to load each referenced spec/research file before coding + +[/Gemini, Qoder, Copilot, Reasonix, Trae, Grok, Kimi Code] + +[Kiro] + +Spawn the implement sub-agent: + +- **Agent type**: `trellis-implement` +- **Task description**: Implement the reviewed task artifacts, consulting materials under `{TASK_DIR}/research/`; finish by running project lint and type-check +- **Dispatch prompt guard**: Tell the spawned agent it is already the `trellis-implement` sub-agent and must implement directly, not spawn another `trellis-implement` / `trellis-check`. + +The platform prelude auto-handles the context load requirement: +- Reads `implement.jsonl` and injects referenced spec/research files into the agent prompt +- Injects `prd.md`, `design.md` if present, and `implement.md` if present + +[/Kiro] + +[codex-inline, Kilo, Antigravity, Devin, DeepSeek Harness] + +1. Load the `trellis-before-dev` skill to read project guidelines +2. Read `{TASK_DIR}/prd.md`, then `design.md` if present, then `implement.md` if present +3. Consult materials under `{TASK_DIR}/research/` +4. Implement the code per reviewed artifacts +5. Run project lint and type-check + +[/codex-inline, Kilo, Antigravity, Devin, DeepSeek Harness] + +#### 2.2 Quality check `[required · repeatable]` + +[Claude Code, Cursor, OpenCode, codex-sub-agent, Kiro, Gemini, Qoder, CodeBuddy, Copilot, Droid, Pi, Oh My Pi, ZCode, Snow, Reasonix, Trae, Grok, Kimi Code] + +Spawn the check sub-agent: + +- **Agent type**: `trellis-check` +- **Task description**: Review all code changes against specs and task artifacts; fix any findings directly; ensure lint and type-check pass +- **Dispatch prompt guard**: The prompt MUST start with `Active task: <task path>`, then tell the spawned agent it is already the `trellis-check` sub-agent and must review/fix directly, not spawn another `trellis-check` / `trellis-implement`. + +The check agent's job: +- Review code changes against specs +- Review code changes against `prd.md`, `design.md` if present, and `implement.md` if present +- Auto-fix issues it finds +- Run lint and typecheck to verify + +[/Claude Code, Cursor, OpenCode, codex-sub-agent, Kiro, Gemini, Qoder, CodeBuddy, Copilot, Droid, Pi, Oh My Pi, ZCode, Snow, Reasonix, Trae, Grok, Kimi Code] + +[codex-inline, Kilo, Antigravity, Devin, DeepSeek Harness] + +Load the `trellis-check` skill and verify the code per its guidance: +- Spec compliance +- lint / type-check / tests +- Cross-layer consistency (when changes span layers) + +If issues are found → fix → re-check, until green. + +[/codex-inline, Kilo, Antigravity, Devin, DeepSeek Harness] + +**Final pass (before Phase 3.4 commit)**: the last 2.2 of a task must run full-scope, not just on the latest implement chunk. List all affected packages with `python3 ./.trellis/scripts/get_context.py --mode packages`, then load each package's spec index Quality Check section. This catches cross-layer / multi-package issues a mid-iteration local 2.2 cannot. + +#### 2.3 Rollback `[on demand]` + +- `check` reveals a prd defect → return to Phase 1, fix `prd.md`, then redo 2.1 +- Implementation went wrong → revert code, redo 2.1 +- Need more research → research (same as Phase 1.2), write findings into `research/` + +--- + +## Phase 3: Finish + +Goal: ensure code quality, capture lessons, record the work. + +#### 3.2 Debug retrospective `[on demand]` + +If this task involved repeated debugging (the same issue was fixed multiple times), load the `trellis-break-loop` skill to: +- Classify the root cause +- Explain why earlier fixes failed +- Propose prevention + +The goal is to capture debugging lessons so the same class of issue doesn't recur. + +#### 3.3 Spec update `[required · once]` + +Load the `trellis-update-spec` skill and review whether this task produced new knowledge worth recording: +- Newly discovered patterns or conventions +- Pitfalls you hit +- New technical decisions + +Update the docs under `.trellis/spec/` accordingly. Even if the conclusion is "nothing to update", walk through the judgment. + +#### 3.4 Commit changes `[required · once]` + +**Spec-sync preamble**: before drafting commits, ask: did this task fix a bug or surface non-obvious knowledge that should land in `.trellis/spec/` so future-you (or future-AI) doesn't repeat the mistake? If yes, return to Phase 3.3 first — spec writes belong in the same task's commit batch, not as a forgotten follow-up. + +The AI drives a batched commit of this task's code changes so `/finish-work` can run cleanly afterwards. Goal: produce work commits FIRST, then bookkeeping (archive + journal) commits land after — never interleaved. + +**Step-by-step**: + +1. **Inspect dirty state**: + ```bash + git status --porcelain + ``` + Snapshot every dirty path. If the working tree is clean, skip to 3.5. + +2. **Learn commit style** from recent history (so drafted messages blend in): + ```bash + git log --oneline -5 + ``` + Note the prefix convention (`feat:` / `fix:` / `chore:` / `docs:` ...), language (中文/English), and length style. + +3. **Classify dirty files into two groups**: + - **AI-edited this session** — files you wrote/edited via Edit/Write/Bash tool calls in this session. You know what changed and why. + - **Unrecognized** — dirty files you did NOT touch this session (could be the user's manual edits, leftover WIP from a previous session, or unrelated work). Do NOT silently include these. + +4. **Draft a commit plan**. Group AI-edited files into logical commits (1 commit per coherent change unit, not 1 commit per file). Each entry: `<commit message>` + file list. List unrecognized files separately at the bottom. + +5. **Present the plan once, ask for one-shot confirmation**. Format: + ``` + Proposed commits (in order): + 1. <message> + - <file> + - <file> + 2. <message> + - <file> + + Unrecognized dirty files (NOT in any commit — confirm include/exclude): + - <file> + - <file> + + Reply 'ok' / '行' to execute. Reply with edits, or '我自己来' / 'manual' to abort. + ``` + +6. **On confirmation**: run `git add <files>` + `git commit -m "<msg>"` for each batch in order. Do not amend. Do not push. + +7. **On rejection** (user replies "不行" / "我自己来" / "manual" / any pushback on the plan): stop. Do not attempt a second plan. The user will commit by hand; you skip ahead to 3.5 once they confirm. + +**Rules**: +- No `git commit --amend` anywhere — three-stage three-commit flow (work commits → archive commit → journal commit). +- Never push to remote in this step. +- If the user wants different message wording but accepts the file grouping, edit the message and re-confirm once — but if they reject the grouping, exit to manual mode. +- The batched plan is one prompt; do not prompt per commit. + +#### 3.5 Wrap-up reminder + +After the above, remind the user they can run `/finish-work` to wrap up (archive the task, record the session). + +--- + +## Customizing Trellis (for forks) + +This section is for developers who want to modify the Trellis workflow itself. All customization is done by editing this file; the scripts are parsers only. + +### Changing what a step means + +Edit the corresponding step's walkthrough body in the Phase 1 / 2 / 3 sections above. Critical invariants: +- No active task must triage first and ask for task-creation consent before creating a Trellis task. +- Planning must distinguish lightweight PRD-only tasks from complex tasks that require `prd.md`, `design.md`, and `implement.md` before start. +- Every required execution path must keep the Phase 3.4 commit reminder reachable before `/trellis:finish-work`. + +All tag blocks live in the `## Phase Index` section above, immediately after each phase summary: + +| Scope | Corresponding tag | +|---|---| +| No active task (before Phase 1) | `[workflow-state:no_task]` (after the Phase Index ASCII art) | +| Active task record unreadable | `[workflow-state:task_error]` (repair the existing task before continuing) | +| All of Phase 1 (task created → ready for implementation) | `[workflow-state:planning]` (after Phase 1 summary) | +| Codex inline Phase 1 | `[workflow-state:planning-inline]` | +| Phase 2 + Phase 3.2–3.4 (implementation + check + wrap-up) | `[workflow-state:in_progress]` (after Phase 2 summary) | +| Codex inline Phase 2 + Phase 3.2–3.4 | `[workflow-state:in_progress-inline]` | +| After Phase 3.5 (archived) | `[workflow-state:completed]` (after Phase 3 summary; **currently DEAD**) | + +### Changing the per-turn prompt text + +Directly edit the body of the corresponding `[workflow-state:STATUS]` block. After editing, run `trellis update` (if you're a template maintainer) or restart your AI session (if you're customizing your own project) — no script changes required. + +### Adding a custom status + +Add a new block: + +``` +[workflow-state:my-status] +your per-turn prompt text +[/workflow-state:my-status] +``` + +Constraints: +- STATUS charset: `[A-Za-z0-9_-]+` (underscores and hyphens allowed, e.g. `in-review`, `blocked-by-team`) +- A lifecycle hook must write `task.json.status` to your custom value, otherwise the tag is never read +- Lifecycle hooks live in `task.json.hooks.after_*` and bind to one of `after_create / after_start / after_finish / after_archive` + +### Adding a lifecycle hook + +Add a `hooks` field to your `task.json`: + +```json +{ + "hooks": { + "after_finish": [ + "your-script-or-command-here" + ] + } +} +``` + +Supported events: `after_create / after_start / after_finish / after_archive`. Note that `after_finish` ≠ a status change (it only clears the active-task pointer); use `after_archive` for "task is done" notifications. + +### Full contract + +For the workflow state machine's runtime contract, the locations of all status writers, pseudo-statuses (`no_task` / `stale_<source_type>`), the hook reachability matrix, and other deep details, see: + +- `.trellis/spec/cli/backend/workflow-state-contract.md` — runtime contract + writer table + test invariants +- `.trellis/scripts/inject-workflow-state.py` — actual parser (reads workflow.md only, no embedded text) diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..bcaf762 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,31 @@ +<!-- TRELLIS:START --> +# Trellis Instructions + +These instructions are for AI assistants working in this project. + +This project is managed by Trellis. The working knowledge you need lives under `.trellis/`: + +- `.trellis/workflow.md` — development phases, when to create tasks, skill routing +- `.trellis/spec/` — package- and layer-scoped coding guidelines (read before writing code in a given layer) +- `.trellis/workspace/` — per-developer journals and session traces +- `.trellis/tasks/` — active and archived tasks (PRDs, research, jsonl context) + +If a Trellis command is available on your platform (e.g. `/trellis:finish-work`, `/trellis:continue`), prefer it over manual steps. Not every platform exposes every command. + +If you're using Codex or another agent-capable tool, additional project-scoped helpers may live in: +- `.agents/skills/` — reusable Trellis skills +- `.codex/agents/` — optional custom subagents + +Managed by Trellis. Edits outside this block are preserved; edits inside may be overwritten by a future `trellis update`. + +<!-- TRELLIS:END --> + +## Dever Language Compiler + +- This repository develops a new Rust bootstrap compiler, not the existing Go Dever application framework. Do not introduce Model, Page JSON or Service scaffolding here. +- This is a new language under active design: change source contracts directly and migrate repository callers together. Do not add legacy aliases, compatibility modes, fallback implementations or speculative defensive branches. Keep required type/protocol validation, explicit errors and resource lifecycle semantics. +- Program source is `.dever` or `.dever.md`; the latter compiles only top-level fenced `dever` blocks and keeps Markdown headings/prose nonsemantic. One file owns one package; both formats share language/contract checks. The language contract is `.trellis/tasks/09-04-dever-language-mvp/prd.md`; Markdown source rules are in `LANGUAGE.md` and `.trellis/spec/backend/toolchain-and-library.md`. +- Compiler phases belong to `crates/dever-core`. Add the native runtime and CLI when their implementation stages need them; avoid empty crates and placeholder commands. +- Read `.trellis/spec/backend/index.md` before editing. All permanent tests belong under repository-root `test/`. +- Parser success means syntax success only. Do not present it as name resolution, type checking or executable language support. +- Use focused offline checks for the changed phase. Do not run full test suites, service checks or install/replace global commands. Do not commit without user authorization. diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..6cc6a8b --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,2664 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + +[[package]] +name = "argon2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072" +dependencies = [ + "base64ct", + "blake2", + "cpufeatures 0.2.17", + "password-hash", + "zeroize", +] + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "boxcar" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36f64beae40a84da1b4b26ff2761a5b895c12adc41dc25aaee1c4f2bbfe97a6e" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytecount" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "005ec2760ca554fae18df7a11195552ec576cd665632a881bc011d5bb2fd4d80" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "rand_core 0.10.1", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "num-traits", +] + +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + +[[package]] +name = "command-fds" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b60b5124979fccd9addd89d8b97a1d6eebb4950694520c75ddd722535ea443f" +dependencies = [ + "nix", + "thiserror 2.0.20", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + +[[package]] +name = "data-encoding" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + +[[package]] +name = "deadpool" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e98a7e119cd347f4201e1159b19831029e203e2d8b790547708e8157b4acf1e" +dependencies = [ + "deadpool-runtime", + "tokio", +] + +[[package]] +name = "deadpool-postgres" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65a536565624b97fc19f758cd01b15d12908d3344425066efc8162236fbd3749" +dependencies = [ + "async-trait", + "deadpool", + "getrandom 0.4.3", + "tokio", + "tokio-postgres", + "tracing", +] + +[[package]] +name = "deadpool-runtime" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2657f61fb1dd8bf37a8d51093cc7cee4e77125b22f7753f49b289f831bec2bae" +dependencies = [ + "tokio", +] + +[[package]] +name = "deadpool-sqlite" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "008b02743f8fad68dc9bb2851c0a4a1d63190d8e3885c5dea20f8a0a2a6bff9a" +dependencies = [ + "deadpool", + "deadpool-sync", + "rusqlite", +] + +[[package]] +name = "deadpool-sync" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e385cc95d3d582c328b36d1ff90feac061102b001894b555e6b465a2e0eaabbf" +dependencies = [ + "deadpool-runtime", +] + +[[package]] +name = "dec" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ae126ef80702bd514a66279b6dac1e58ff1dbcb12024f7110de95608348fd7" +dependencies = [ + "decnumber-sys", + "libc", + "paste", + "static_assertions", +] + +[[package]] +name = "decnumber-sys" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b4bc33814bd5bcd46dd13f9471a29ab1a22c4701ae0c4a182e45e8336d1a5b" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid 0.9.6", + "der_derive", + "flagset", + "zeroize", +] + +[[package]] +name = "der_derive" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "dever-backend-bridge" +version = "0.1.0" +dependencies = [ + "cc", + "dever-core", + "dever-runtime", + "libc", + "rustix", + "serde_json", + "sha2 0.10.9", +] + +[[package]] +name = "dever-cli" +version = "0.1.0" +dependencies = [ + "base64 0.22.1", + "csv", + "dever-backend-bridge", + "dever-core", + "dever-runtime", + "dever-sandbox", + "flate2", + "fs2", + "goblin", + "node-semver", + "pep440_rs", + "pep508_rs", + "ring", + "rustix", + "serde", + "serde_json", + "sha2 0.10.9", + "tar", + "ureq", + "url", + "zip", +] + +[[package]] +name = "dever-core" +version = "0.1.0" +dependencies = [ + "dever-runtime", + "pulldown-cmark", + "serde_json", + "sha2 0.10.9", +] + +[[package]] +name = "dever-runtime" +version = "0.1.0" +dependencies = [ + "argon2", + "base64 0.22.1", + "bytes", + "chrono", + "deadpool-postgres", + "deadpool-sqlite", + "dec", + "dever-sandbox", + "futures-util", + "getrandom 0.2.17", + "hmac 0.12.1", + "http-body-util", + "hyper", + "hyper-util", + "indexmap", + "rustls", + "serde", + "serde_json", + "sha2 0.10.9", + "subtle", + "tokio", + "tokio-postgres", + "tokio-postgres-rustls", + "tokio-rustls", + "tokio-tungstenite", + "webpki-roots", + "zeroize", +] + +[[package]] +name = "dever-sandbox" +version = "0.1.0" +dependencies = [ + "command-fds", + "goblin", + "libc", + "nix", + "rustix", + "seccompiler", + "sha2 0.10.9", +] + +[[package]] +name = "dever-tests" +version = "0.1.0" +dependencies = [ + "bytes", + "dever-core", + "dever-runtime", + "futures-util", + "h2", + "http-body-util", + "hyper", + "hyper-util", + "rustls", + "serde_json", + "sha2 0.10.9", + "tokio", + "tokio-rustls", + "tokio-tungstenite", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common 0.1.7", + "subtle", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", + "ctutils", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "either" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" + +[[package]] +name = "flagset" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7ac824320a75a52197e8f2d787f6a38b6718bb6897a35142d749af3c0e8f4fe" + +[[package]] +name = "flate2" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-sink", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "rand_core 0.10.1", + "wasm-bindgen", +] + +[[package]] +name = "goblin" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17582616a7718cca54cec18e534a76c7c4aec11a8b9a85695712f262fd15a4c8" +dependencies = [ + "log", + "plain", + "scroll", +] + +[[package]] +name = "h2" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashlink" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb" +dependencies = [ + "hashbrown 0.17.1", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + +[[package]] +name = "hyper" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "bytes", + "http", + "http-body", + "hyper", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "icu_collections" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libredox" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6480ccc157a1389bb2e4891b24751b0f798ba640d22386f23143fbcc89da195a" +dependencies = [ + "libc", +] + +[[package]] +name = "libsqlite3-sys" +version = "0.38.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "miette" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7" +dependencies = [ + "cfg-if", + "miette-derive", + "unicode-width 0.1.14", +] + +[[package]] +name = "miette-derive" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys 0.61.2", +] + +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "node-semver" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b1a233ea5dc37d2cfba31cfc87a5a56cc2a9c04e3672c15d179ca118dae40a7" +dependencies = [ + "bytecount", + "miette", + "nom", + "serde", + "thiserror 1.0.69", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags", +] + +[[package]] +name = "objc2-system-configuration" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396" +dependencies = [ + "objc2-core-foundation", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "password-hash" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" +dependencies = [ + "base64ct", + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pep440_rs" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31095ca1f396e3de32745f42b20deef7bc09077f918b085307e8eab6ddd8fb9c" +dependencies = [ + "once_cell", + "serde", + "unicode-width 0.2.2", + "unscanny", + "version-ranges", +] + +[[package]] +name = "pep508_rs" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faee7227064121fcadcd2ff788ea26f0d8f2bd23a0574da11eca23bc935bcc05" +dependencies = [ + "boxcar", + "indexmap", + "itertools", + "once_cell", + "pep440_rs", + "regex", + "rustc-hash", + "serde", + "smallvec", + "thiserror 1.0.69", + "unicode-width 0.2.2", + "url", + "urlencoding", + "version-ranges", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_shared", + "serde", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + +[[package]] +name = "postgres-protocol" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08808e3c483c46e999108051c78334f473d5adb59d78bb80a1268c7e6aa6c514" +dependencies = [ + "base64 0.22.1", + "byteorder", + "bytes", + "fallible-iterator 0.2.0", + "hmac 0.13.0", + "md-5", + "memchr", + "rand", + "sha2 0.11.0", + "stringprep", +] + +[[package]] +name = "postgres-types" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "851ca9db4932932d69f3ea811b1abe63087a0f740a47692619dd40d4899b68be" +dependencies = [ + "bytes", + "fallible-iterator 0.2.0", + "postgres-protocol", +] + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pulldown-cmark" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f068eba8e7071c5f9511831b44f32c740d5adf574e990f946ddb53db2f314e" +dependencies = [ + "bitflags", + "memchr", + "unicase", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rsqlite-vfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" +dependencies = [ + "hashbrown 0.16.1", + "thiserror 2.0.20", +] + +[[package]] +name = "rusqlite" +version = "0.40.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3" +dependencies = [ + "bitflags", + "fallible-iterator 0.3.0", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", + "sqlite-wasm-rs", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustix" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6725596c3f2c3a0aef021139e145d4eafe314a6623e4680ca83852b2c67ab2ba" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "scroll" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1257cd4248b4132760d6524d6dda4e053bc648c9070b960929bf50cfb1e7add" +dependencies = [ + "scroll_derive", +] + +[[package]] +name = "scroll_derive" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1a36a382ed65dbcc0ab47fd5e9a94112417ccd34560a392ef3b7b0f0ec39148" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "seccompiler" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae55de56877481d112a559bbc12667635fdaf5e005712fd4e2b2fa50ffc884" +dependencies = [ + "libc", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "serde_json" +version = "1.0.145" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", + "serde_core", +] + +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba467056f1b547ed52077911161fc86985becbc60e8e1857c8a144dab0def891" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "sqlite-wasm-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" +dependencies = [ + "cc", + "js-sys", + "rsqlite-vfs", + "wasm-bindgen", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "tar" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a" +dependencies = [ + "filetime", + "libc", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl 2.0.20", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + +[[package]] +name = "tls_codec" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de2e01245e2bb89d6f05801c564fa27624dbd7b1846859876c7dad82e90bf6b" +dependencies = [ + "tls_codec_derive", + "zeroize", +] + +[[package]] +name = "tls_codec_derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d2e76690929402faae40aebdda620a2c0e25dd6d3b9afe48867dfd95991f4bd" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "tokio-postgres" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a528f7d280f6d5b9cd149635c8705b0dd049754bc67d81d31fa25169a93809d3" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "fallible-iterator 0.2.0", + "futures-channel", + "futures-util", + "log", + "parking_lot", + "percent-encoding", + "phf", + "pin-project-lite", + "postgres-protocol", + "postgres-types", + "rand", + "socket2", + "tokio", + "tokio-util", + "whoami", +] + +[[package]] +name = "tokio-postgres-rustls" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c2ad44aa0ae96db89c4742212ed41645b2f597311ff6e1945542a4d9fadc2fb" +dependencies = [ + "rustls", + "rustls-native-certs", + "sha2 0.11.0", + "tokio", + "tokio-postgres", + "tokio-rustls", + "x509-cert", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17a073bfed563fa236697a068031408a93cd9522e08abf9933ead3e73411bd71" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "tungstenite" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e48ac77174b19c110a50ab2128b24215ac9cb40e0e12e093fb602d175c569d22" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand", + "sha1", + "thiserror 2.0.20", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unscanny" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9df2af067a7953e9c3831320f35c1cc0600c30d44d9f7a12b01db1cd88d6b47" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "ureq" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7ac20be9b7726e0bbdbf974c059676d9acb1cd414961f570a4e8231cacd7fc" +dependencies = [ + "base64 0.23.1", + "flate2", + "log", + "percent-encoding", + "rustls", + "rustls-pki-types", + "ureq-proto", + "utf8-zero", + "webpki-roots", +] + +[[package]] +name = "ureq-proto" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f86fd172ccca569e458f61b6bdd6220965a9ef36e672a6852953b51a0e1583be" +dependencies = [ + "base64 0.23.1", + "http", + "httparse", + "log", +] + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", + "serde_derive", +] + +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version-ranges" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e9bd4e9c9ff6a2a9b5969462ba26216af3e010df0377dad8320ab515262ef8" +dependencies = [ + "smallvec", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasi" +version = "0.14.7+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" +dependencies = [ + "wasip2", +] + +[[package]] +name = "wasip2" +version = "1.0.1+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasite" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fe902b4a6b8028a753d5424909b764ccf79b7a209eac9bf97e59cda9f71a42" +dependencies = [ + "wasi 0.14.7+wasi-0.2.4", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.4", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fbddc4a036f00ec4f18c83445bd3115cb306a91da554919a099d9222fe4a7f8" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "whoami" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c" +dependencies = [ + "libc", + "libredox", + "objc2-system-configuration", + "wasite", + "web-sys", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "x509-cert" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" +dependencies = [ + "const-oid 0.9.6", + "der", + "spki", + "tls_codec", +] + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33811428bee40dbceb6d545e95754741d17a6aef9a4849f0fd62e2ba4f412a78" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "flate2", + "indexmap", + "memchr", + "thiserror 2.0.20", + "zopfli", +] + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..22880fc --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,63 @@ +[workspace] +members = ["crates/dever-core", "crates/dever-runtime", "crates/dever-cli", "crates/dever-backend-bridge", "crates/dever-sandbox", "test/dever-tests"] +resolver = "3" + +[workspace.package] +version = "0.1.0" +edition = "2024" +rust-version = "1.95" +publish = false + +[workspace.lints.rust] +unsafe_code = "forbid" + +[workspace.dependencies] +dever-core = { path = "crates/dever-core" } +dever-runtime = { path = "crates/dever-runtime" } +dec = "=0.4.11" +indexmap = "=2.14.0" +pulldown-cmark = { version = "=0.13.4", default-features = false } +tokio = { version = "=1.53.1", features = ["rt-multi-thread", "sync", "time", "net", "process", "io-util", "macros", "signal"] } +futures-util = { version = "=0.3.34", default-features = false, features = ["std", "async-await", "sink"] } +bytes = "=1.12.1" +base64 = "=0.22.1" +hyper = { version = "=1.11.1", features = ["http1", "http2", "server", "client"] } +hyper-util = { version = "=0.1.20", features = ["tokio"] } +http-body-util = "=0.1.5" +tokio-tungstenite = { version = "=0.30.0", default-features = false, features = ["handshake"] } +tokio-rustls = { version = "=0.26.5", default-features = false, features = ["ring", "tls12"] } +rustls = { version = "=0.23.44", default-features = false, features = ["ring", "std", "tls12"] } +webpki-roots = "=1.0.8" +serde = { version = "=1.0.228", features = ["derive"] } +serde_json = "=1.0.145" +deadpool-sqlite = { version = "=0.14.0", default-features = false, features = ["rt_tokio_1", "bundled"] } +deadpool-postgres = { version = "=0.14.2", default-features = false, features = ["rt_tokio_1"] } +tokio-postgres = { version = "=0.7.18", default-features = false, features = ["runtime"] } +tokio-postgres-rustls = { version = "=0.14.0", default-features = false, features = ["ring", "native-certs"] } +getrandom = "=0.2.17" +argon2 = { version = "=0.5.3", features = ["zeroize"] } +sha2 = "=0.10.9" +ring = "=0.17.14" +hmac = "=0.12.1" +subtle = "=2.6.1" +zeroize = "=1.9.0" +chrono = { version = "=0.4.45", default-features = false, features = ["std"] } + +[profile.release] +codegen-units = 1 +lto = "fat" + +# Static archives retain linkable symbols; final program stripping belongs to LLD. +[profile.runtime-pack] +inherits = "release" +debug = 0 +strip = "debuginfo" +incremental = false + +[profile.dev] +debug = 0 +incremental = false + +[profile.test] +debug = 0 +incremental = false diff --git a/IMPLEMENTATION.md b/IMPLEMENTATION.md new file mode 100644 index 0000000..0635fc7 --- /dev/null +++ b/IMPLEMENTATION.md @@ -0,0 +1,623 @@ +# Dever 语言实现原理 + +本文解释 Dever 从源码到可执行程序的完整工作方式。语法、标准包接口和应用示例见 [Dever 语言开发指南](LANGUAGE.md)。 + +这里描述的是 Dever 必须长期保持的架构和语义边界，不依赖某一种引导工具链，也不公开内部临时产物格式。只要这些边界保持不变，编译器内部可以替换实现而不影响 Dever 源码。 + +## 1. 设计目标 + +Dever 的实现围绕几个约束展开： + +- 源码中的错误尽量在运行前发现。 +- 条件分句由编译器证明互斥、可达且完整，不依赖书写顺序碰运气。 +- handler 是静态绑定的行为参数，不是运行时函数对象。 +- 普通数据遵循值语义，文件、网络连接和 Stream 等资源遵循共享身份语义。 +- JSON 等算法使用 Dever 源码；HTTP 复用 Hyper 协议引擎。运行时边界不包含应用业务规则。 +- 生产程序走提前编译路径，不携带通用解释器、语法分析器或类型检查器。 +- 默认离线工作，不在检查、运行或构建时隐式下载依赖。 + +这使 Dever 的公开语言契约和编译器内部实现分离：前者应当稳定，后者可以持续优化。 + +## 2. 总体流水线 + +一个 Dever 程序会经过以下阶段： + +```text +用户 .dever / .dever.md 源码 + 官方 .dever 标准包 + | + v + 确定性源码加载与位置记录 + | + v + 词法分析 -> 语法分析 + | + +------+------+ + | | + v v + 源码格式化 全局语义检查 + | + v + 类型化中间表示 HIR + / \ + v v + 规范执行器（测试） 可达性分析与静态特化 + | + v + 原生后端与本机链接 + | + v + 独立程序 + 最小运行时 +``` + +语法分析成功只说明源码结构合法，不表示程序可以执行。名称解析、类型推导、分句证明、确定赋值和调用检查全部通过后，编译器才会生成可执行的类型化中间表示。 + +## 3. 源码加载与位置模型 + +编译器先递归加载指定源码根目录中的 `.dever` 和 `.dever.md` 文件，并把随编译器提供的官方 `.dever` 包加入同一个编译单元。普通 `.md` 不加载。 + +加载阶段遵守以下规则： + +- 文件路径按稳定顺序排序，因此诊断顺序不受文件系统遍历顺序影响。 +- 源码必须是有效 UTF-8，非法内容在进入语法分析前报错。 +- 不跟随源码树中的符号链接，避免同一文件被重复或越界加载。 +- 每段源码都带有来源身份；用户源码不能伪装成官方保留包。 +- 每个 token、语法节点和后续 HIR 节点都保留源文件位置。 + +内部位置使用半开字节区间，展示给开发者时转换成从 1 开始的行号和按 Unicode 字符计算的列号。这样既能高效切片源码，也不会把多字节字符显示成错误列号。 + +Markdown 是带强制说明契约的源码容器。编译器通过 CommonMark 解析库识别顶层 `dever` 或 `typescript dever` 围栏，并把唯一的 H1 映射到 package、把每个 H2 映射到一个 type 或逻辑函数。结构阶段检查固定说明列表、代码块数量和声明归属；函数签名解析后，再核对公开清单、调用方式、类型结构以及函数输入输出。引用、列表、HTML 和展示代码块中的伪围栏不执行；程序围栏未闭合、没有程序块、声明跨块或说明与代码不一致时明确失败。每个包仍由一个文件完整拥有，不能把 `.dever` 和 `.dever.md` 的同名包合并。 + +`SourceFile` 的原文用于诊断、语义缓存和 CLI 格式化快照。程序块的 token、AST、HIR 及原生故障保持该文件的字节位置，不能把临时抽取文本的行列暴露给开发者。Markdown 解析依赖仅在编译器中使用，不进入最终程序的最小运行时。 + +## 4. 词法、语法树与格式化 + +### 4.1 词法分析 + +词法分析器把文本转换为关键字、名称、字面量、标点和注释。字符串转义、数字拼写和注释边界在这一阶段确认，但名称含义和类型仍未判断。 + +每个 token 都保留原始位置。后续阶段不需要重新猜测错误来自哪段文本。 + +### 4.2 语法分析 + +语法分析器只负责回答“源码的结构是什么”，生成保留源代码意图的语法树，例如： + +- 从文件相对路径推导的 package 身份，以及 type/function 声明各自的 `public` 标记； +- type、function 和多个 function 分句； +- pattern、表达式和 function body 中的线性语句； +- 显式括号、注释和 record 字段顺序。 + +语法树不会提前把未解析名称变成具体声明，也不会把表达式强行塞进通用运行时值。名称和类型属于语义检查器的职责。 + +### 4.3 格式化 + +格式化器直接基于语法树工作，不依赖类型检查。它必须保留： + +- 注释内容和相对归属； +- 字面量的实际值； +- 会改变结合关系的括号； +- 源码中 record 表达式的字段顺序。 + +格式化结果要求幂等：连续格式化两次，第二次不能再产生变化。 + +Markdown 先检查 H1/H2 说明结构和完整包语法，再按原代码块边界格式化其中的声明和注释；只替换程序块内部，保留说明、围栏及展示代码块。名称和类型的说明一致性由语义检查负责。格式化与编译使用相同的代码块识别、说明结构和声明边界，不能各自实现一套 Markdown 提取规则。 + +对一个源码根目录执行写入格式化时，编译器先解析全部目标文件，再暂存每个结果，最后逐文件原子替换。任何文件解析失败时都不开始写入。这里保证的是单个文件不会留下半写内容，不承诺多个文件构成操作系统级事务；替换途中发生系统错误时，已经完成的文件不会整体回滚。`fmt --check` 只比较结果，不修改文件。 + +## 5. 全局语义检查 + +Dever 不是逐文件解释语言。编译器先收集整个源码根目录和标准包，再统一检查。因此声明可以跨文件引用，但错误也会在全局检查时一起暴露。 + +语义检查按依赖关系分阶段进行： + +1. **注册声明头**：建立 package、type、function 和公开成员索引。function 以名称和输入数量共同标识。 +2. **解析类型形状**：确认 record、choice、Nullable、List、Map、Stream 等类型组合是否合法，并检查递归类型约束。 +3. **建立 function 签名**：合并同名同输入数量的分句，推导或校验输入输出契约。 +4. **规范化 pattern**：把源码 pattern 转换成可证明的符号域。 +5. **证明分句集合**：检查分句是否重叠、不可达或不完整。 +6. **检查 function body**：完成名称解析、表达式类型检查、确定赋值、稳定局部变量和 handler 调用检查。 +7. **检查依赖环**：拒绝 package 循环、递归 type，以及 function 的直接或间接递归；这条规则同样适用于用户源码和官方 `.dever` 标准包。 +8. **构造 HIR**：只有没有错误时，才形成可交给执行后端的完整程序。 + +这种分阶段结构避免把“尚未收集到声明”误报成“声明不存在”，也使同一套符号和类型信息能够被所有文件复用。 + +## 6. 条件分句如何工作 + +Dever 没有传统的 `if`。一个 function 的多个分句共同描述完整输入空间： + +```dever +type Result { + Ok(value: Int) + error Failed(message: Text) +} + +show(result: Result.Ok(value)) (text: Text) recover("render result as user-visible text") { + text = "value=" + int.to_text(value) +} + +show(result: Result.Failed(message)) (text: Text) recover("render result as user-visible text") { + text = "failed: " + message +} +``` + +编译器不会简单地从上往下寻找第一个匹配项。它先把每个 pattern 转换成符号域： + +- Bool 和不带 payload 的 choice 是有限原子集合； +- Nullable 在原类型域之外增加 `null`； +- 带 payload 的 choice 先选择变体，再为 payload 建立绑定； +- Text 和数值常量代表单点集合； +- Int、Decimal 范围代表区间； +- `other` 代表同一 function 分句组中、同一输入位置全部显式 pattern 的补集，与它写在第几个分句无关。 + +多输入 function 的一个分句是多个输入域的笛卡尔积，可以理解为一个多维矩形。编译器通过矩形相减计算尚未覆盖的区域，不会枚举无限数值域中的每个值。 + +由此可以在编译期证明三件事： + +- **互斥**：任意两个分句的输入域都没有交集。 +- **可达**：每个分句规范化后的输入域都非空；空域分句没有任何输入能够到达。 +- **完整**：全部分句输入域的并集等于 function 的整个合法输入空间。 + +实现上可以逐个从完整输入域中减去分句矩形，以计算交集和剩余区域；这个处理次序只用于算法和诊断，不形成“先写者优先”的语言规则。通过证明后，后端可以自由生成决策树或其他高效控制流。源码顺序不参与语义选择，因此调整互斥分句的顺序不会改变结果。 + +## 7. 类型化中间表示 HIR + +HIR 是语法树和执行后端之间清晰、集中的内部语义契约。它不是公开文件格式，不承诺序列化或跨编译器版本兼容，也不需要在程序运行时保留。 + +与语法树相比，HIR 中已经没有以下不确定性： + +- 名称已经解析成确定的 package、type 或 function 标识； +- 每个表达式都有具体类型； +- 每个局部名称都有确定存储位置和生命周期； +- 每组分句都有规范化输入域； +- 直接调用和静态 handler 调用已经区分； +- 每个节点仍能追溯到源码位置。 + +HIR 同时承担两个职责：一方面为生产后端提供类型完整、无需再次猜测的输入；另一方面供测试使用的规范执行器执行，作为生产后端行为的对照。 + +### 7.1 为什么不从语法树直接生成程序 + +若后端直接读取语法树，它就必须重复名称解析、类型判断、pattern 选择和错误处理。多个后端会逐渐产生不同语义。 + +把这些规则集中在语义检查器并固化为 HIR，可以保证： + +- 一个语言规则只有一个拥有者； +- 执行后端只处理已经证明合法的程序； +- 更换后端不需要改变 Dever 源码； +- 规范执行器和生产程序可以对同一个 HIR 做差分验证。 + +## 8. 从 HIR 到生产程序 + +生产路径采用提前编译，而不是把源码或通用 AST 带到运行时。 + +```text +类型化 HIR + | + v +选定零输入公开入口 + | + v +计算入口可达的 function 与类型 + | + v +展开静态 handler 绑定并生成具体实例 + | + v +降低为原生后端的具体控制流和数据表示 + | + v +本机优化、链接最小运行时 + | + v +独立可执行程序 +``` + +语义检查仍会检查编译单元中的所有声明，避免未使用源码长期积累错误；代码生成则只保留入口可达的实例，避免把无关标准包和编译器能力塞进最终程序。 + +最终程序包含具体 record、choice、循环、调用和必要的系统原语，不包含： +- Dever 词法分析器或语法分析器； +- 类型检查器和分句证明器； +- 规范执行器； +- 通用动态 `Value` 分派层； +- 运行时反射注册表； +- 每个元素调用一次的虚拟 handler 分派。 + +构建期间产生的后端材料只存在于独占临时目录中，成功或失败后都会清理。这里的“独立程序”是指运行时不需要 Dever 源码或编译器继续参与，不表示文件能跨操作系统运行，也不承诺完全静态链接；它仍可依赖目标操作系统正常提供的系统接口和动态库。 + +## 9. 静态 handler 与特化 + +handler 用来把行为作为参数传入，但它不是闭包、函数指针或普通一等值。最外层调用必须把它绑定到编译期可确定的具名 function，中间 function 可以继续转发自己的 handler 参数： + +```dever +total = reduce(add, numbers, 0) +``` + +编译器为每个实际绑定组合生成一个具体实例。可以把实例键理解为： + +```text +原 function + 按声明顺序排列的具体 handler 绑定 +``` + +例如，同一个 `reduce` 分别绑定 `add` 和 `max` 时，会得到两个可独立优化的实例。实例内部直接调用目标 function，不需要运行时查询 handler 是谁。 + +静态特化还会： + +- 沿调用链继续解析 handler 转发； +- 只生成入口可达的绑定组合； +- 复用已经生成的相同实例； +- 在仍处于展开状态时发现同一实例再次依赖自身，并按当前禁止递归的规则报告编译错误。 + +这种模型牺牲了任意运行时闭包的动态性，换来简单的类型系统、可预测的生命周期和直接调用性能。 + +## 10. 值、资源与生命周期 + +### 10.1 普通值 + +Int、Text、Bytes、List、Map、record 和 choice 等普通数据在语言层面遵循值语义。把一个值赋给另一个名称后，后续观察不能因为某处内部修改而意外改变另一个名称。 + +实现不必为每次赋值立刻深拷贝。后端可以使用共享不可变存储、写时复制和最后一次使用分析： + +- 只读值可以共享底层数据； +- 真正需要修改且仍有别名时才分离； +- 一个值最后一次使用时可以直接移动； +- 优化不能改变相等性、遍历顺序或错误行为。 + +Map 的插入顺序属于语言可观察语义。相等性、显示、删除和重新插入都必须遵守同一顺序契约，不能由后端容器的偶然行为决定。 + +具体来说，覆盖已有 key 不改变位置；删除后再次写入同一个 key，则把它作为新条目追加到末尾。两个 key/value 相同但插入顺序不同的 Map 不相等。 + +后端在已检查 HIR 的副本上合并安全的记录更新。只有后续旧记录读取不与已更新字段重叠时，才把旧局部直接转交给新局部；调用后仅剩标量字段读取时，先保留这些标量。字段赋值允许移动右侧最后一次读取的同一字段。旧别名仍需保留时继续复制，资源所在作用域不变。 + +### 10.2 资源值 + +File、Socket、Listener 和 Stream 等资源不是普通可复制数据。它们的多个别名共享同一资源身份。按资源类型不同，共享状态包括： + +- 当前游标、消费位置或耗尽状态； +- 打开和关闭状态； +- 底层系统句柄； +- 尚未消费的输入。 + +任一别名关闭资源后，其他别名也必须观察到关闭状态。资源由作用域清理和显式 `close` 共同管理，不依赖通用垃圾回收器决定何时释放系统资源。 + +资源可以作为 record 字段或 choice payload。复制外层聚合仍遵循值语义，但其中的资源字段只复制对同一资源身份的引用，不复制文件、连接或消费状态。普通数据与资源身份因此保持清晰边界。 + +### 10.3 数值一致性 + +数值规则由语言定义，而不是交给后端默认行为决定。编译期常量计算、规范执行器和生产程序必须经过同一组数值语义： + +- Int 运算检查溢出，除法和余数遵守 Dever 规定的边界行为； +- Decimal 使用固定精度、舍入和非法操作规则； +- Float 保留 IEEE 754 的 NaN、无穷和有符号零行为； +- `sum` 和普通表达式按源码规定的从左到右顺序计算，不能用会改变结果的重排或快速数学优化。 + +后端只有在证明不会改变这些结果时才能省略重复检查。静态可知的故障在检查阶段报告，依赖运行值的故障由生产程序携带源码位置报告。 + +## 11. 序列、Stream 与并发 + +List、Bytes 和 Stream 复用一部分序列操作，但各自允许的行为有明确边界： + +- List 直接遍历已经存在的元素，支持 `each`、`filter`、`find`、`sum`、`reduce` 和 `reduce_until`。 +- Bytes 直接产生 Int 字节值，不先转换成 List，支持 `each`、`reduce` 和 `reduce_until`。 +- Stream 通过拉取接口按需产生下一项，不预先物化全部数据，支持零输出 `each`、`reduce` 和 `reduce_until`。 + +`each` 在 List 或 Bytes 上绑定单输出 handler 时承担映射职责并产生 List；Dever 没有另一套独立的 `map` 操作。`filter`、`find` 和 `sum` 当前只接受 List。 + +编译器内部使用统一的序列分类来复用控制流生成，而不是为每个序列操作复制三套实现。 + +`reduce_until` 在 handler 表示停止后立即结束，不会为了探测结束条件而额外拉取下一项。因此后续操作仍能消费那一项，这对文件和网络流尤其重要。 + +`parallel_each` 的并发边界是显式且有界的： + +- 调用方拥有唯一生产者，Stream 不会被多个 worker 同时拉取； +- 非挂起 handler 的 worker 数量为 1–256，挂起 handler 的任务组容量为 1–65536，另受全局任务上限约束； +- 待处理队列容量与 worker 数量绑定，不会无限积压； +- worker 在调用结束前全部回收，不产生脱离作用域的后台任务； +- 记录第一个程序 fault，并在后续拉取或分派边界停止继续分发； +- 返回前等待全部已经启动的工作结束，不强制取消正在阻塞的系统调用； +- 为避免共享消费状态，context 和单项输入都不能递归包含 Stream。 + +普通序列操作保持确定性顺序。并发 handler 的实际完成顺序不作为语言可移植语义的一部分。 + +独占普通列表通过消费迭代转移元素；共享列表只在拉取时复制当前元素，提前停止不复制剩余部分。直接消费 `text.split` 的遍历使用同一运行时分割规则，避免完整中间列表。相邻单次使用的集合阶段可以融合，但前置 handler 必须同时非挂起、无副作用、无程序故障；仅有 `pure` 不够，整数溢出、错误顺序和 Float 归约顺序仍需保留。挂起 handler 不进入同步迭代器融合。 + +List/Bytes 的并行动作按有限小批次分发，每批最多 256 项；Stream 仍逐项拉取。两者共用有界调度和故障取消状态，批内也检查取消，返回前回收全部线程。 + +### 11.1 异步任务、多线程与 Channel + +挂起能力沿用同一条编译流水线：parser 只建立统一函数语法，checker 通过共享依赖图推断调用和 handler 的挂起 effect，并检查可传递性与 Task/Group 仿射消费。HIR 保存内部 `suspends` 和具体并发操作，原生后端再生成 Rust future。Task 输出类型和静态 handler 目标都在 HIR 中确定，不引入通用 Future、动态 Value 或运行时 handler 查找。 + +完全非挂起的入口直接调用生成函数，不创建 Tokio。推断为挂起的入口调用 `dever_runtime::task::run_entry`，统一创建一个默认有界 runtime；业务源码不能通过 `dever.task.start` 手工创建或嵌套 runtime。宿主嵌入继续使用 `run_entry_with`。运行时集中拥有调度线程、CPU/阻塞 worker 和子任务上限；两个线程数均为 1–64，默认 CPU/阻塞 worker 最多 8，默认活动子任务 4096，任务上限为 65536。每次 `run` 不创建 runtime 或系统线程。 + +执行类别保持分离：普通调用顺序完成，并由后端按目标 effect 生成直接调用或内部 `.await`；`run` 建立受父作用域约束的 Task；`wait` 消费 Task/Group；`parallel` 把 pure 非挂起计算提交到有界 CPU 通道；`blocking` 只承接编译器已识别的阻塞系统调用。effect 检查会穿过普通调用、静态 handler 替换和集合 handler，防止文件、同步睡眠、标准输出或嵌套同步并行留在 Tokio 调度线程。List/Bytes 的命名非挂起 `parallel_each` handler 在挂起路径中进入 blocking 通道；挂起 handler 复用 AsyncStream 的 Group 路径。完全非挂起的 `parallel_each` 保持作用域线程实现。参数先按源码顺序求值，再移动到工作闭包，避免改变故障和副作用顺序。 + +根入口和每个由 `run`/Group 启动的任务都有内部 Scope。业务 future 与不可被业务 Task 直接中止的 supervisor 分离：`stop` 或析构只中止业务 future，supervisor 继续回收其子任务和已经启动的 blocking 工作，完成信号交回父 Scope。入口返回 `Ok` 或 `Err` 前都会取消并等待剩余工作，因此不在 `Drop` 内执行 `block_on`，也不会让阻塞副作用在 `stop` 返回后继续。 + +Group 使用正容量限制和 `JoinSet` 拥有动态零输出 supervisor。容量满时先回收一个已完成任务形成背压；首个故障会取消并清理其余成员。显式 `stop` 执行取消并等待。Group 在异常路径析构时分离 supervisor 句柄，但父 Scope 仍持有完成登记并负责等待；编译器同时拒绝正常分句出口遗留未消费 Task 或 Group。 + +Channel 由共享有界队列、可用槽位/数据量信号量和短临界区组成，不为等待方创建线程。关闭会同时唤醒发送方和接收方，缓存值仍按 FIFO 排空，之后 `receive` 返回 `null`。类型系统不表示嵌套可空值，原生后端复用 List/Map 可空结果的压平规则处理 `Channel<T?>`。容量、元素可传递性和挂起上下文由编译器负责；运行时只拥有队列、唤醒和关闭状态。 + +同步 `dever.time.sleep` 保持阻塞系统原语。官方 `dever.task.sleep` 映射到 Tokio timer。文件和同步 Stream 通过明确的 blocking 边界隔离；TCP 使用相同 runtime 的 I/O driver。 + +`timeout` 和 `race` 复用 Task 的 supervisor 所有权，等待时只借用其句柄，取消路径仍能请求停止并排空。timeout 到期后才调用匹配输出签名的零输入 async fallback；race 收取胜出任务后取消并等待其余任务，程序故障不能被竞争结果掩盖。编译器将它们纳入仿射消费、静态 handler、effect、失败义务和命名输出检查。 + +List/Bytes 的 async 顺序 handler 在原有序列生成路径逐项等待。`stream(Channel/List/Bytes)` 和 `ticks` 复用 AsyncStream 的 producer，不额外预取或创建任务；Channel 流保留实际 null 元素，关闭只释放自己的别名。周期流使用延迟首次触发和跳过错过周期的 Tokio interval。 + +### 11.2 AsyncStream 与 TCP + +`AsyncStream<T>` 保存一个 Send 的 stream producer，使用 futures-util 的 unfold/map 和 Tokio 异步锁；producer 只在创建时装箱，拉取不逐项分配 boxed future。别名共享游标和关闭信号，close 唤醒挂起的 pull；组合器在结束、故障和取消时释放生产者。流不预取，读取缓冲区在同方向读锁可用且 socket 可读后才分配。 + +异步流的 `parallel_each` 复用结构化 Group，先取得组内和全局容量，再拉取下一项；等待生产者期间也收集 handler 故障，随后取消并等待其他成员。List/Bytes 的 CPU 并行仍使用共享 blocking 通道，网络 handler 则生成具体 async future，不创建独立线程池。 + +TCP Socket/Listener 由 net 模块统一拥有资源关闭和唤醒。Socket 的读写锁分开，允许双向同时工作；一次写入保存偏移，写出部分内容后的取消、超时或故障关闭连接，后续写入不能接在残缺内容后继续。DNS 进入已有有界 blocking 通道；数值地址直接使用 Tokio。HTTP 采用 Hyper HTTP/1 和 HTTP/2，连接作用域统一拥有协议任务、流式响应/SSE 和普通 WebSocket。 + +## 12. 标准包与系统原语的边界 + +Dever 的标准能力分成两层： + +```text +应用 .dever 源码 + | + v +官方 .dever 标准包：公开类型、校验规则、算法、状态组织 + | + v +dever.system.*：有类型的最小系统原语契约 + | + v +最小运行时：文件、网络、时钟、进程等原子系统操作 +``` + +官方 `.dever` 包和用户源码经过相同的解析、类型检查、分句证明与代码生成。它们不是绕过语言规则的隐藏实现。 + +只有同时满足下列条件的能力才应进入系统原语： + +- Dever 源码无法自行完成，例如打开文件或创建网络连接； +- 操作能定义成小而稳定的有类型契约； +- 不把高层业务算法、协议状态机或公开数据结构藏进运行时。 + +系统原语由一份穷尽目录统一管理。检查器、规范执行器和生产后端必须同时实现同一条目，避免某个后端悄悄支持额外行为。 + +新增标准能力时，应先问“能否用现有 Dever 语法和原语实现”。答案为可以时，它就属于 `.dever` 标准包，而不是新的隐藏内建函数。 + +## 13. JSON 算法与 HTTP 引擎边界 + +JSON 和 HTTP 是验证语言基础能力是否完整的两类代表性功能。它们需要字节处理、受控重复、choice、record、错误传播、Stream、资源和有界状态，但不应要求专用语法。 + +### 13.1 JSON + +JSON 的公开模型和算法由 Dever 源码定义。解析过程使用序列归约完成扫描和状态推进，写出过程遍历同一公开文档结构。 + +当前文档模型使用扁平、无环的节点表： + +- 节点 ID 连续； +- 子节点总是先于父节点出现； +- 引用只能指向更小的 ID； +- number 可以保留原始文本拼写； +- 解析深度、输出大小和非法引用都有显式失败结果。 + +这种表示不依赖运行时反射或任意递归对象，也便于 Dever 自身验证结构是否合法。 + +### 13.2 HTTP + +N2 将 HTTP 协议边界交给成熟 Hyper 1.x 引擎；公开 Request、Response、Header、Limits 和结果类型仍在 `library/dever/http.dever`。旧 Dever wire/field/transport 解析模块已删除，无备用实现。JSON 仍使用 Dever 源码算法。 + +`dever.system.http_serve` 的 handler 沿用静态绑定和 specialization。HIR Intrinsic 与 Collection 共享 callback 依赖提取；调用环、包依赖、effects/recovery 和 native 可达性都包含该 handler。校验器检查 HTTP 名义记录的字段类型、名称和可见性，native 按字段名生成双向转换；不存在运行期通用 Value 或动态路由解释器。 + +运行时使用 Hyper HTTP/1、HTTP/2 connection API 与 hyper-util 的 TokioIo/Timer。服务端从已有 Listener 接受独占 Tokio TCP 流，复用 AsyncStream 和 Task/Group 先预留容量再接受连接。连接和 handler 子任务归入现有 Scope，停止服务会等待清理。对端错误按协议终止对应流或连接，程序故障沿原位置向父任务传播。 + +单次 send 在同一请求 future 内同时推进 Hyper driver 与响应读取，完成或取消即释放连接；可复用客户端由 13.4 节的池拥有，不自动重试。服务端持久连接、流水线和 chunked 输入由 Hyper 处理；静态业务 handler 仍在编译产物中直接运行。 + +Bytes 使用 bytes::Bytes 共享缓冲区，切片不复制，独占 Text/Bytes 仍可转移存储；别名写入保持值语义。缓冲正文受 body_bytes 限制，单帧共享、多帧增量合并；重复头使用 List，值使用 Bytes。头部、正文、handler 和停滞写入各有期限。N3 提供流式响应、SSE 和 WebSocket，N4 提供 TLS 和客户端池，N5 接通 HTTP/2；精确接口见 LANGUAGE.md。 + +N5 的 Limits.http2 显式选择 HTTP/2，protocol.rs 集中配置 Hyper 的固定流控窗口、流数、16 KiB 帧/发送缓冲和头预算。HTTP 专用 ALPN 配置从共享 TLS 配置派生，HTTPS 必须协商 h2，明文使用 prior knowledge。消息编解码仍共用 message.rs；HTTP/2 authority 规范化到 Host，source target 只保留路径和查询，不暴露协议伪头。 + +http/executor.rs 将 Hyper H2Stream、ConnTask 和发送任务接到 task/scoped.rs。每个 Hyper 任务都有自己的 Dever 子 Scope；连接 owner 的不可取消 supervisor 排空所有后代后才释放物理资源。关闭 admission 和新任务登记使用同一把短锁，故障报告与注销都包含在任务完成通知之前。等待关闭的 future 被取消时，cleanup supervisor 仍由父 Scope 的 completion 登记持有。HTTP/2 handler 错误需要显式报告，因为 Hyper 会在流内部消费该错误。 + +### 13.3 流式响应、SSE 与 WebSocket + +N3 通过 `serve_live` 接入零输出静态 handler 和 HttpReply，共用请求解码、响应校验和 HTTP 连接配置。handler 的挂起 effect 由调用图推断。HTTP/1 每连接以容量 1 的 Group 拥有 handler；HTTP/2 每请求流独立持有 Group，并归该流的 Scope。正文通道只有 1 个槽位。响应体释放信号、Session guard 和 Scope 一起处理 HEAD、提前结束、故障、断开及取消；pending head 被重置时也必须结束该 handler。HTTP/1 升级后 Hyper driver 完成，连接 owner 继续等待 WebSocket handler。 + +SSE 编码按 EventSource 行规则保留空 id 和尾部换行，先计算展开长度再分配。心跳由响应体被拉取时的 timer 产生，不创建定时任务。WebSocket 复用 tokio-tungstenite 0.30.0 的 handshake 功能。Hyper 升级后取回原 TCP/TLS Transport 和预读帧，释放 HTTP 写期限；WebSocket 用独立方向锁、4 KiB 读缓冲和有界消息/写缓冲。Endpoint 与 TCP 共享关闭和唤醒语义；帧写入的取消 guard 只在取得方向锁后生效。 + +HttpReply/WebSocket 纳入资源传递、效果、API/Markdown 类型名称和 Render；静态桥接复用按字段名转换，扩展到 SSE 可空字段和 WebSocket Message 选择项。没有动态 Value、额外解释器或兼容接口。 + +### 13.4 可复用客户端、TLS 与服务生命周期 + +HttpClient 固定一个 origin；`http/client.rs` 拥有请求编码和租约，`client_driver.rs` 拥有连接限额、空闲回收和驱动，`client_body.rs` 拥有流式上传/下载。三者复用 Hyper 低层 HTTP/1、HTTP/2 connection API、消息校验和 AsyncStream，不另写协议解析器或全局主机注册表。 + +请求许可保留到正文完成或丢弃；物理连接许可保留到传输对象及其协议任务释放，空闲、队列内和运行中的连接合计不能超过 connections。每池一个受 Scope 管理的 Task 使用有界队列与 FuturesUnordered 推进连接 driver，空闲 timer 主动回收到期连接。driver owner guard 在创建任务之前构造，父作用域即使在任务首次 poll 前结束，也会关闭池。HTTP/1 Entry 析构中止相应 driver；HTTP/2 单流的租约释放不影响邻流。显式 close_client 等待池任务排空，不能遗留 detached driver。 + +上传用 StreamBody 按需拉取，下载逐帧读取并以共享 Bytes 切出限额内的块。HTTP/1 完整 EOF 才将连接归还空闲池，提前关闭、故障或取消均丢弃连接。HTTP/2 上传和下载共享流租约，双方结束才释放流许可；连接可供邻流继续使用，只有无活跃流时才开始 idle 回收。HTTP/2 握手 job 在池 driver 首次 poll 时创建连接 Scope，避免把长寿命连接登记到临时请求 Scope。GOAWAY 停止新派发，排空中的旧连接仍占物理许可；仅当 Hyper 通过 TrySendError 交回确定未发送的请求时，在原期限内重新排队，已发送请求不重放。缓冲 request 保留总正文和总期限；流式 open/open_stream 只限制头阶段、单块和每次读取，不收集整条 SSE。不跟随重定向。 + +TLS 使用 tokio-rustls 0.26.5、rustls 0.23.44 的 ring 后端和 webpki-roots 1.0.8。ClientTls/ServerTls 是共享配置，支持公共 CA、显式自定义 CA、服务端 PEM 身份，保留链、有效期和主机名校验，客户端与服务端会话缓存的容量参数均设为 32。具体 Transport 枚举只有 TCP 与装箱 TLS 两种，HTTP/WS 共用；普通 TCP 不承担 TLS 大状态或额外装箱。DNS 复用已有有界 blocking 解析路径。 + +四种 http_serve 原语共享可选末尾 Context 的类型检查和静态生成路径。Context 是可传递的具体值，Channel/HttpClient 字段共享资源；不引入一般闭包。Listener.close 对公共 connections 流仍产生终止失败，对 HTTP 内部接收流表示正常结束；连接 driver 收到关闭信号后调用 Hyper graceful_shutdown，等待已开始的请求和升级会话。应用用 timeout(server, grace_ms, fallback) 限制最终排空时间。 + +模块 API 的站点归属由检查器把 API 物理父目录与 `setting.json` 的 `sites.path` 做分段前缀匹配，文件名不参与匹配，站点也不改写 URL。配置引用的 verify 必须解析为一个只读 App 根；它和其数据库读依赖进入生产可达图及原生初始化，不能在运行时按字符串反射调用。 + +原生路由先由 runtime 校验站点绑定的 Bearer 或 Secret Cookie，再把结构化 Claims 交给该静态 verify。verify 输出的 App-owned record 经生成桥接转换为 runtime Identity，随后使用 Tokio task-local 包住真实 handler。源码只能通过 `dever.auth`、`dever.site` 读取复制值，不能构造或保存上下文本体。JWT 固定 HS256，并校验 algorithm、issuer/provider、audience/site、时间边界及签名；public 路由忽略浏览器自动附带的 Cookie，但显式错误 Authorization 仍拒绝。Cookie 写入和 Cookie 认证的非安全方法要求 Origin 与 Host 精确匹配站点配置的外部 HTTP/HTTPS origin，不根据内部监听协议推断，也不信任转发 Header。响应 Header/Cookie 先暂存，只在 handler、wire 编码与事务提交全部成功后发布。含密码 hash/verify 的 HTTP 调用链不生成覆盖整个 handler 的事务；检查器要求全部写入位于显式短 transaction 中，并禁止 transaction 到达密码计算，避免 SQLite writer 锁和 WAL 读快照升级错误。 + +权限目录由编译后的 API 自动生成并同步到核心 RBAC 表，受保护入口按站点、租户、用户和角色校验精确权限，不由 verify 返回权限字符串，也不要求逐个调用 `auth.require`。verify 只提供经验证的用户/租户/会话身份，public 声明显式允许匿名；业务数据所有权是独立边界。官方 `dever.api.Error` 由后端按确切类型与变体映射为固定响应，未知业务失败只记录调用元数据并返回脱敏 500。日志在 runtime 追加站点和已验证身份字段，不记录 token、Cookie 或 JWT payload。 + +外部 Adapter 不增加第二套调用模型。checker 从同领域唯一 Port 合成完整实现，复用既有类型 wire schema、failure/effect 图、Adapter 选择和 setting 解码；HIR 保存封闭的 exec/pip/npm/go 元数据。native 为每个具体操作生成静态输入/输出/业务错误 codec，业务签名中不会出现 JSON、进程句柄或动态值；reference 遇到可达外部实现时明确拒绝。 + +runtime `component` 是唯一 Worker owner。它从当前可执行程序目录解析编译器规范化的相对 entry，以清空环境、固定 `--dever-component` 参数和专用管道直接启动子进程，不使用 shell 或 `PATH`。协议帧为四字节大端长度加 UTF-8 JSON，继续使用 16 MiB、64 层、65536 值以及重复字段拒绝规则。`hello/ready` 绑定版本、Port、Adapter、schema hash、操作、capability 和 setting；随后使用严格递增 request id 的 `call/result/error/cancel`，并以 `health` admission 和 `shutdown` 收口。业务 `error` 必须属于 Port 已声明 variant；协议、超时、崩溃和非法 payload 都是运行 fault，不伪造业务错误，也不自动重放请求。 + +每个 Worker 的有界命令队列、进程、stderr drain 和 supervisor 都注册到现有结构化 Scope。超时或调用任务取消会发送 `cancel`，随后必须消费同一 id 的一个 `result` 或 `error` 才能继续；错误 id、未知 kind、截断/超限 frame 会终止该 Worker 并关闭 pending 调用。崩溃后的下一次新调用可以在固定启动频率内重建，但失败中的调用不重放。生成入口在应用成功、业务失败和运行 fault 后都走同一 cleanup，先完成 Component shutdown，再关闭数据库，并保留主错误及 cleanup cause。allow 集合参与编译期 effect 和握手精确匹配；Linux 启动还统一进入 `dever-sandbox` 的 namespace/seccomp/只读运行树与显式文件授权边界。GPU、其他 OS 和不允许 namespace 的宿主明确失败，不绕过隔离。 + +External Lib 的项目边界由 `dever-cli::libs` 统一拥有：`config/setting.json` 根 `lib` 数组保存显式请求，canonical v5 `dever.lock` 保存精确版本、传递依赖、runtime/schema/artifact 摘要、源码构建收据和每个 checked Worker 的绑定。每个 Worker 独立校验环境闭包与版本冲突，打包精确比对整份 provider 元数据，不允许锁中多写未嵌入资源。`dever lib add/update` 才能解析，`run/build` 只离线消费锁。全部 exec 候选与锁资源形成嵌入清单，bundle identity 包含路径、SHA-256、长度、可执行位；runtime 原子提取到私有 `data/cache/lib/<digest>`，每次 Worker 启动重新核对清单、字节与 Unix 所有权/权限，不扫描别的 bundle。 + +Python、JavaScript、Go SDK 从 `ExternalWorkerContract::sdk_manifest()` 的 checked wire DAG 生成 typed 边界，保留 i64/JSON 预算和不合作取消的一秒有界退出。真实 PyPI wheel/npm registry/Go proxy resolver 共用确定性 lock 和摘要缓存；生产入口只接受安装版本中经过签名验证的 runtime 描述与 pack，不回退到宿主工具。Python/JavaScript 离线打包器生成每个 Adapter 私有的运行树和同名 handler 绑定；Python 固定隔离参数并先加载可信 SDK，Node 支持 ESM/CommonJS。启动清单绑定精确 entry、解释器、资源 argv 和工作目录，仍由唯一 component supervisor 管生命周期。原始依赖归档只在配置根或 exec 仍需它时保留，避免与展开树重复嵌入。资源采用独立二进制编译输入，编译前按实际字节验证摘要，启动校验按 64 KiB 分块执行。 + +Go 受管编译/链接已由 `dever-cli::workers::go_build` 拥有：只启动锁定 pack 内的 compile/link/analyzer，清空环境，按目标 tags、标准库 importcfg、锁定 module ZIP 与 `go:embed` 准备输入；生成的 main 静态绑定同名操作并复用 Go SDK typed manifest。应用只保留编译后的 Worker 与合同，不嵌入构建工具或源归档。Go sumdb 验证签名、包含证明、checkpoint 一致性与 h1；Python extras/wheel/sdist 和 npm 高级安装图也由对应 resolver 校验。PEP517 与 npm 安装钩子共用隔离构建 Stage，只在显式 Lib 准备时执行，收据绑定输入和产物，run/build 离线重放。官方 simplejson C 扩展、bufferutil addon 和 UUID 已有真实构建/Worker 验收；这不等于正式公开或六平台发行。 + +Dever Package 安装由 `dever-cli::packages` 拥有：HTTPS origin、semver、单版本传递闭包、归档 SHA/manifest 校验与所属组件边界，复用现有 ArtifactStore、Lib resolver 和原子锁。`SourceMap::load_with_packages` 在普通项目入口加载 Package 源码，配置变更失败恢复原始字节，remove 离线裁剪且不更新剩余版本。源码与 Worker 字节只来自精确锁定归档；exec 和受管生态共用锁归属判断，不按文件是否存在来决定是否回退本地。 + +`dever-backend-bridge` 已实现安全 Rust API 到进程内 LLVM 18/LLD 的连接。workspace 保留 `unsafe_code=forbid`，只有该 crate 的私有 `ffi.rs` 使用授权例外；编译器返回缓冲仍由原 C++ allocator 释放，LLD 串行调用。输出在当前用户的私有目录中独占预留，失败只清理自有输出。可恢复链接失败允许后续调用，不可恢复的 LLD 状态终止私有编译 worker。 + +`dever-core::llvm::emit_kernel` 直接消费 checked HIR，已有同步强类型函数与推导挂起函数转换，不生成 Rust 或动态 Value。支持 Int/Float/Bool/Decimal、Text/Id、Bytes 标准操作、List/Map、Nullable、普通 Record/Choice、多输出、分句投影、记录值拷贝/字段修改、静态 handler、左到右求值及短路。List/Bytes 的 each/filter/find/reduce/reduce_until/sum 使用静态 handler 与 typed 遍历，支持其现有可挂起 handler 合同；Int 保留 checked overflow/div/rem/neg，Float 保留 IEEE 行为。局部、临时及嵌套受管值通过具体类型的 retain/drop 和所有权 guard 清理；复用原有 last-use/字段替换分析，成功根输出转交调用者。入口初始化输出与 fault，错误保留原始来源与调用链；调用者通过 `dever_outputs_release`/`dever_fault_release` 释放。Model/Port/Adapter/API/Job、鉴权/上传等应用 intrinsic 与其余未支持操作仍带源码位置拒绝，不以默认值或 Rust 回退代替。简单内核六目标 object/link、受管/资源/异步内核六目标 object 和 Linux x86_64 原生执行已验收；Windows 复杂链接缺目标 runtime 的 `__chkstk`/`_fltused`，不能算跨平台执行通过。 + +File 的 create_new/open/read/write/close 与共享别名、同步 Stream 的共享游标/文件 chunks/close/each/reduce/reduce_until 复用原 runtime。原始 I/O 失败进入标准 Choice，文件已关闭的 Stream 产生一次 Failed 后结束，提前遍历不吞掉未访问行。源码没有新增 pull/first(Stream) 接口；pull 仅用于内部遍历 ABI。 + +`llvm/asynchronous.rs` 生成 LLVM18 switched-resume 协程及静态 OwnedType/AsyncFunction/SyncFunction 描述符；`llvm/concurrency.rs` 接通 Task/Group/Channel 与同步 blocking/parallel。构造帧时在 initial suspend 前克隆 borrowed 参数；真实 Pending 保留帧，初始/挂起销毁清理 live guards，正常完成先清理再 final suspend，最终销毁仅释放帧。ForeignFuture 使用现有单 runtime Scope/supervisor，不逐函数 block_on。跨线程边界按 checked 字段、完整可达异步帧及 inferred failure 字段验证 Send；内部 Unit/Outputs ABI row 以字段决定可传递性，不更改语言类型属性。同步 invoke 借用 runtime 拥有的输入，runtime 在完成或取消后释放；typed fault 移交保留 nominal 载荷、代码和来源，只有 Business=4 可被 result 捕获。 + +`llvm/network.rs` 接通 AsyncStream 的 List/Bytes/Channel 来源、ticks 和全部 TCP 操作；Socket/Listener/异步流保持原 runtime 的共享身份、方向锁、期限和取消合同。异步流消费在返回、提前停止、故障或取消时关闭共享 producer；同步 Stream 的提前停止仍保留未访问行。timeout 的独立 presence 区分真正超时与成功输出 null，fallback 在任务及后代排空后调用；race 保持同签名的 typed 多输出，并排空失败/取消的竞争者。`llvm/parallel.rs` 使用静态 typed input pack 与同一运行时的 scoped concurrent、bounded blocking 或 async stream dispatch，不用普通 Group 循环近似替代 pull 前容量预留和空闲 producer 期间的故障竞速。 + +`llvm/http.rs`、`http_values.rs` 和 `http_handlers.rs` 把 HTTP/HTTP2 连接池、TLS、流式 Reply、SSE 与 WebSocket 接入同一 runtime 协议实现。跨 ABI 使用固定命名字段、不透明资源 owner 与静态 typed handler/context；请求或上下文在最初挂起前取得所有权，不经过 JSON/Value 转换。WebSocket 事件保留 kind 与 typed Text/Bytes。HTTP handler 的 typed fault 在请求边界最终渲染：记录错误并返回 500/中止回复，服务器继续处理后续请求，不改成全服务失败。协议异步操作在 poll/I/O 前验证输出槽，非法 metadata 不推进操作，修正后可重试。 + +`llvm/system.rs` 接入时间/日历、参数、stdout/日志、UUID 与 Secret/crypto；Uuid 使用真实 UUID owner 而非 Text 近似，密码运算继续走已有 blocking 边界。仅可达日志或 HTTP server 的根入口在成功/失败返回前刷新日志。runtime logger 的单一固定容量队列保留 FIFO、低级别丢弃和高级别背压；固定确认状态在此前记录释放和输出刷新后唤醒调用者，不依赖临时 ACK 通道或首次阻塞的惰性分配。 + +`llvm::emit_application` 已接通独立 CMD 与 HTTP/REST 应用入口。CMD 保持完整名称和单个 JSON 对象输入；混合 HTTP/CMD 项目执行 CMD 时不绑定监听器。`llvm/api.rs`、`api_context.rs`、`rest.rs` 与 `application_commands.rs` 生成静态路由、具体输入/输出、Claims/Identity 及权限描述符，复用已有 JWT/业务 verify、站点、RBAC、database-only 租户、Cookie/Header、Upload 与日志实现，不另建协议栈或向业务传入动态 Value。非 public 路由缺少认证或精确权限时，源码与私有 ABI 两处均拒绝；权限 key 仍不含 HTTP method。 + +完整应用和可达日志的 CMD 共用 invocation-owned `application::Session`，只读取一次 `config/setting.json`。数据库、迁移临时池、租户 manager 和 lifecycle 均由该 owner 持有；子任务先排空，再在同一 runtime 中关闭连接池、刷新日志。POST/PUT/DELETE 输入解码后开始事务，输出编码后提交，成功才发布响应 metadata。租户管理入口沿用 migrate/owner/component 合同，缺少 ready marker、租户身份或启用组件不能回退平台库。Upload 经 Port/Adapter 消费到存储，部分解码失败清理 affine owner。私有 CLI 和 Linux managed daemon 编译已接 LLVM；正式目标 packs 未交付。 + +`llvm/external.rs` 接通 checked external Adapter；选中的 Setting 成功解码后才启动 Worker，具名输入/输出和声明业务错误复用 typed wire codec。缺失可空输出字段、额外字段、错类型和未声明错误均为运行故障。`llvm::emit_application_with_resources` 消费原打包器的 `EmbeddedResource`，与 Rust backend 共用摘要/去重，运行时复用安全提取和受管 launch；不查找宿主解释器。`runtime-external` 与编译器 SDK 分离。`component::Session` 由每次 task 根持有，传播到子任务、blocking 和 API；supervisor 挂在根 Scope，短命调用者退出不关闭共享 Worker。根退出先停止接收和关闭 Worker，再排空 Scope，typed 原始错误保留并附加清理原因。显式 shutdown 被取消时仍保留 Task owner，根可继续排空；普通 Test 仍只有 fake。 + +`llvm/jobs.rs` 接通 Job-only、API/worker/all、持久 payload、事务内入队、每次 User verify/精确权限/组件重验与租户迁移。`job::resources::Resources` 持有本次入口的 StorageBinding、Spec 和 Clock，复用已有 store/worker/cron/lease fencing；同步 handler 共用既有 bounded blocking，异步 handler 共用 typed coroutine。服务策略只在实际启动服务时校验，入队 CMD 和租户管理无需 HTTP/worker 配置。编译期的租户组件归属不等于部署启用了 tenant；两后端共用 `job::require_components`，仅在启用租户时要求身份并查询组件状态。Job 保持原有 HTTP 请求/身份 getter 禁用规则，不伪造请求上下文。 + +`llvm::emit_test_suite` 复用每个 checked `test_program` 的 fake、effect/failure 和可达闭包，在同一 IR 中按声明符号隔离各 case。稳定 index 选择私有文件名入口；fault 按同一 index 查询大小、渲染和释放，业务载荷保持具体类型。普通测试用独立进程和临时 SQLite；不读取部署配置、启动生产 Adapter/Lib 或服务，无数据库用例也不初始化无关 Model。断言保留源码位置，虚拟时钟和 drain 复用生产队列；根 Scope 排空后关闭 Session，再释放注册表、时钟和入口锁。测试 CLI 切换仍属于后续统一后端阶段。 + +`llvm/database.rs`、`database_rows.rs`、`orm*.rs`、`database_schema.rs` 和 `transaction.rs` 接通 Model；API 应用通过 Session 同时支持平台库与租户物理库。SQL/DDL 与关联查询复用 `native/orm.rs` 的纯构造函数，不生成 Rust、不复制数据库驱动。持久化固定标量在 ABI 边界转换为既有 `orm::Value`，业务记录和错误仍保持具体类型。Row/Rows/RowStream/Related 是不透明所有者；ToOne 保留可空关联，ToMany 按 998 个父键分块并使用既有 per-parent 限额 SQL。启动按实际数据库分组先迁移全部 Model，再建立外键；数据迁移与 Seed 复用原历史校验、事务和顺序规则。 + +`database::Session` 持有本次入口的 Settings 和连接池，复用原配置校验和驱动初始化，不写入全局 OnceLock。`task::run_entry_with_typed_cleanup` 先排空 root Scope，再在同一个 Tokio runtime 内停止所有连接池、等待驱动关闭，最后销毁 runtime。需要租户的应用使用上述 application Session 与 tenant owner，不能回退到平台库。写入 CMD 在输入解码后开事务，普通调用和顺序 handler 继承隐藏事务，Task/parallel 回调使用 detached 上下文；嵌套 transaction 复用外层、不制造 savepoint。输出编码成功后才提交，提交后才发布 stdout；失败/取消清理 typed 输出并回滚或丢弃状态未知的连接。rollback 保留原始 fault，可附加 cleanup cause；DbError 十个变体使用静态 typed 回调，不按消息猜类别。PostgreSQL 的 SQL 约束失败会使当前事务失败，不能承诺捕获后继续使用该事务；NotFound 等非 SQL 失败仍可正常捕获。 + +`llvm/failures.rs` 接通业务 `fail`、默认传播和 `result(named_call)`，直接复用 `specialize::failures` 与 `capture::targets`。fault 原前五字段不变，末尾追加 nominal type（编译期 ID+1，0 表示无载荷）、variant 与按可达错误类型最大布局对齐的内联 Choice；Business code 为 4。载荷用具体类型 store/load 和静态 drop，不是动态 Value 或序列化消息。捕获支持零/单/多输出、直接 error 与一层 error-only Choice 包装；显式 raised outer variant 保留身份。匹配后移交载荷并清空 fault，未匹配的数值/RuntimeAbi 故障继续追加调用帧传播。BytesToText/BytesFromInts/BytesSlice 复用既有 C ABI：status 1 的原始消息转换为标准 Failed(Text)，status 2 保持 RuntimeAbi fault；成功句柄与错误 Buffer 各自准确转交/释放。真实执行覆盖捕获后重失败、非零 variant 根释放、受管 COW 与精确源调用链。 + +`dever_rt_v1_*` ABI 的唯一 canonical 签名在 `crates/dever-backend-bridge/include/runtime.h`。`runtime-abi` feature 构建独立 staticlib，不包含 LLVM/C++ SDK；所有导出仍局限于私有 `ffi.rs`。固定宽度标量传参、u32 状态及显式输出指针避免聚合按值传参；Int/Float/Decimal 复用现有数值/Render，Decimal 使用显式 little-endian 双 u64。Text/Bytes/List/Map 使用不透明 Arc 所有者；集合以 process-static `DeverRtType` 的 clone/drop/equal/hash 回调操作具体类型，复用现有 List/Map COW，不引入动态 Value。普通元素保留 PartialEq（含 NaN），只有合法 Map key 包装提供 Eq/Hash。`_take` 接口即使失败也消费其非空 receiver；消费式 cursor 的提前释放同时清理未访问行。可空元素取值按完整元素类型分配输出槽，外层存在标记不覆盖元素自身的 null；Channel receive 共用相同的 optional-row lowering。输出/错误 Buffer 由 Rust Box 分配并通过同一 ABI 释放，不能与编译器 Reply 的分配器混用。C header→实际库、checked Dever→LLVM object→实际库及重复执行分配平衡夹具已经执行；这不代表完整应用系统 ABI、性能/RSS 容量、六目标 runtime packs 或干净机器发行已完成。 + +私有 `dever run/build/test` 默认由 binary 私有 `compile` 模块执行 HIR→LLVM→object→进程内 LLD；共享 CLI 库与 launcher/daemon 不承担 LLVM 动态依赖。runtime pack 固定在编译器同级，校验版本、ABI、目标、profile、路径、长度和 SHA-256；缓存命中仍校验 pack，链接前复制到独占目录并重新校验，只接受显式 CRT object 与普通静态 archive。缓存和 create-new 输出沿用 `native::compile_artifact`，旧 Rust backend 仅保留作者回归入口。 + +`llvm::emit_executable`/`emit_test_executable` 在原 callable 根外生成 C `main`，复用 typed fault 渲染、输出释放与日志刷新。Unix 进程启动显式忽略 SIGPIPE，让关闭 stdout 返回带来源的 I/O 错误；callable 根不改变嵌入宿主信号策略。Test 入口只接受单个有效十进制 case index，fault size/render/release 使用同一 index。 + +Linux `deverd` 已接可信编译 IPC。managed core 提交逻辑源码、无秘密编译绑定及资源；daemon 验证签名版本与完整 pack 引用后，启动受限核心 worker 重新检查并编译。只由 daemon 发布缓存，不能通过普通 artifact 上传污染它；App/Test/Lib 准备与运行留在 caller。服务缺失明确失败。 + +原生发行包作者入口 `sdk/native-release.rs` 从独立作者根的 `config/setting.json` 读取显式输入及摘要，复用既有 runtime manifest、验证器和 Ed25519 发行协议。制作器复制并复核四 profile、CRT 和核心/LLVM 文件，在私有 staging 中签名，再以不覆盖方式原子发布；私钥不进入产物。`runtime-pack` Cargo profile 为 base/sqlite/postgres/both 分别生成保留链接符号的优化 archive，共享一个构建缓存。相同输入和密钥的重复打包一致，不等同于跨机器源码构建一致。作者流程见 [native release SDK](sdk/native-release.md)。 + +作者配置的可选 `runtimes` 已接 Python/Node/Go 完整运行/构建输入，制作器与 Worker 共用 manifest 和文件闭包校验。排序且固定元数据的 gzip/tar、registry 描述统一进入顶层发行签名；Go SDK、Adapter 和依赖的编译共用 `-trimpath`，避免随机临时目录改变 Worker 与共享缓存身份。LLVM 的二进制资源附件复用既有摘要去重，在 verify 前初始化常量，IR 仍限8 MiB；附件最多4096项、单项128 MiB/总256 MiB，共享编译请求384 MiB、产物320 MiB，不透明 artifact 仍64 MiB。Linux x86_64 三生态签名安装、受管 `check/run/build`、三条共享缓存及移除源码/机器安装后无系统语言环境的独立执行已通过，包含Python/Node源码构建原生依赖与离线收据重放。Linux签名首装、旧签名模板升级、篡改拒绝、双项目及真实双UID共享核心亦已通过自有完整验收。Linux root 沙箱、真实第三方高级依赖、双 CMS/PG 和 LLVM 容量报告已补齐；公开首次安装包、六平台产品代码/资产、受限 AppArmor 非 root 入口与完整质量门仍未完成。各阶段证据及限制见当前生产化任务。 + +database 租户模式把 verify 返回的内部 tenant ModelId 放入同一 task-local 调用域。`global type` 继续使用平台连接，其他 Model、自动 REST 与 Job 通过 `StorageBinding` 选择已显式迁移且 fingerprint 匹配的租户物理库；缺少上下文或 ready marker 直接失败。runtime tenant manager 对 `(connection, tenant_id)` 池设置总量与 idle 回收，生成的 `tenant migrate` 入口按稳定顺序迁移全部 tenant Model/Job 后才写本地和 control marker；PostgreSQL 先通过控制库幂等建租户库。User Job 持久化 Provider/站点/session/租户声明和已通过的能力名，Worker 重新调用静态 verify 并使用最新权限复核，不能把历史任务降级为 System。table/field 模式尚未实现，不能在固定表 SQL 上只补一个配置分支。 + +## 14. 错误与诊断 + +Dever 把失败分成三个层次： + +1. **加载错误**：路径、符号链接、文本编码等问题，发生在可靠源码位置建立之前。 +2. **编译诊断**：词法、语法、名称、类型、分句和生命周期问题，带稳定错误码、主位置和必要的关联位置。 +3. **运行失败**：除零、越界、关闭资源、协议失败等，尽量映射回 Dever 源码位置。 + +多个诊断按确定性规则排序。内部实例名称、后端标识和宿主调用栈不应泄漏到面向 Dever 开发者的错误信息中。 + +可预期的业务和 I/O 失败使用显式 choice 结果，让 Dever 源码能够处理；违反已经建立的运行前提时才产生运行 fault。这两类失败不应混为一个无类型异常通道。 + +CLI 只负责参数解析、退出码和标准输出/错误输出分流。语言规则必须由加载器、检查器、HIR 或执行后端拥有，不能散落在命令分支里。 + +### 14.1 编译期契约分析 + +完成名称、类型、分句和循环检查后，HIR 进入独立的契约阶段。字段范围复用分句的 Int/Decimal 区间归一化；private 由字段访问和构造的原有检查入口拥有，不增加平行的对象系统。 + +事实分析按依赖顺序计算函数输出保证，跟踪输入来源、已知区间、分支值和字段事实。复制创建独立事实快照，赋值使目标旧事实失效。函数输入到输出的直接传递可以在调用处替换；未知运算扩大为未知，不能建立范围。构造、字段写入、返回和 handler 边界共用范围包含判断，诊断保留依据位置。 + +失败分析为显式 error 分支建立义务，跟随值及嵌套载荷流动。消费输入的函数本身也必须检查所有分句，空函数无法洗掉义务。失败输出保证与“可能含错误”分开：未知列表可能为空、可空值可能为 null，不能借它们证明已经返回失败；短路和空遍历不会错误消费未执行路径上的输入。recover 是可审查的显式消费边界，编译器只验证声明，不能验证理由的业务真实性。 + +效果分析从穷尽的系统原语分类出发，经调用和静态 handler 绑定传播到稳定摘要；pure 在此处验证。package 依赖从已经解析的类型、调用和静态 handler 引用自动构图，不在源码重复声明。结构建议复用 HIR 和效果摘要，只处理可证明的结构，保留源码执行行为。 + +类型图验证后一次计算可比较、可移动、可跨线程传递属性，避免反复展开共享类型。事实、效果和失败分析共用依赖图；单调摘要变化只通知受影响的调用方，依赖环仍通过工作队列收敛。 + +API 快照由检查完成的 Program 生成，使用公开名称而非内部编号，排序保证稳定。CLI 只负责读写文件和报告差异；已有基线不会更新，验证先于原生编译。快照和源码的修改授权属于项目治理，不能通过语言内一个新关键字自动建立。 + +## 15. 离线构建与产物生命周期 + +Dever 的默认工作流是本地离线： + +- `check` 只加载和检查源码，不启动原生后端； +- `fmt` 只使用语法前端，不执行程序； +- `run` 和 `build` 使用同一条生产编译路径； +- 编译过程不隐式访问网络或下载 package； +- `run` 在临时位置生成程序、执行并清理； +- `build` 只写入明确指定且尚不存在的输出路径，避免误覆盖文件。 + +`run` 和 `build` 需要当前机器已经具备与编译器匹配的本地原生后端。正式发行工具应随发行包提供或在安装时明确检查它，不能在一次构建过程中临时联网补齐。当前尚未交付面向最终用户的完整发行工具链。 + +内置标准包 AST 在进程内缓存；每线程只保留上一份按有序路径和完整文本精确匹配的检查结果，含错误与警告。新的 CLI 进程仍完整检查源码和 API 基线，没有持久化 HIR 格式。 + +私有 LLVM CLI 使用编译器同级 `cache/native`；旧 Rust 作者回归入口使用仓库 `target/native-artifacts`。两者复用同一产物 owner，SHA-256 身份区分生成内容、编译器、runtime/profile、平台与选项。缓存只持久化可执行文件、身份与完整性摘要，临时生成内容与运行时快照按所有权清理。完整产物原子发布；命中后核验并复制到本次临时目录，损坏时报错，显式输出仍不覆盖已有文件。缓存不替代每次 CLI 的源码/API 与 pack 检查。 + +稳定 `dever` 与版本核心分离。机器根固定拥有 `bin/state/versions/cache`，发行 manifest 以 Ed25519 验证原始字节，并逐项复核平台、长度和 SHA-256；安装先进入 staging，自检成功后才以目录 rename 发布。活动版本使用追加式校验日志：每条完整记录独立带摘要，读取只选择最后一条完整记录，因此写入中断保留上一版本，且不依赖 Windows 不支持的覆盖式 rename。项目锁定只来自 `config/setting.json` 的精确 `dever.version`。 + +`CacheStore` 是 `deverd` 进程内部的不可变条目、摘要复核、操作租约和清理引擎。Linux 普通用户通过经内核 UID 校验的本地 IPC 共享 Lib 资产，不能读取管理员 token 或执行清理；状态只查询元数据并明确返回 `verified:false`，管理员完整校验返回 `verified:true`。上传下载只接受 SHA-256 和精确长度，单资产 64 MiB、总量 2 GiB/4096 条，分块传输到私有 staging 后原子发布；每个外部 UID 最多占两个连接且总量保留管理员处理容量。服务不可用不会回退到 caller-writable store。编译请求与不透明资产传输分离：完整输入及签名 manifest 参与身份，源码仅留在私有 staging，缓存只保留程序和摘要。请求全程持有版本共享锁和 cache operation；取消、断连或180秒期限到达均回收 worker 和 staging。独立编译缓存也受2GiB/4096条预算限制。macOS/Windows peer credential、系统服务注册、提权和完整原生发行仍未完成，不能把开发态 socket 描述为完整跨平台安装服务。 + +离线不等于跨平台。省略构建目标时使用本机平台；Linux x86_64 编译器可通过 `build --target linux-aarch64` 选择已准备的 ARM64 运行库和第三方资源，生成 ARM 应用。编译器核心与 Go 编译工具仍在宿主执行，应用及 Worker 使用目标架构；缓存和签名校验都包含明确目标。其他平台仍需独立适配和验证。模拟执行不等于 ARM 真机性能或所有硬件/内核兼容性验收。 + +源码排序、Map 顺序、诊断排序和归约顺序是确定的。不同平台生成的可执行文件不承诺字节完全相同，但对相同合法程序必须保持相同语言结果；显式并发完成顺序等已声明例外除外。 + +## 16. 规范执行器与验证方法 + +规范执行器只在测试中使用。它直接执行已经检查完成的同步 HIR，目标是清楚地表达语言语义，而不是承担生产性能。异步 HIR 当前由规范执行器明确拒绝，异步语义以原生后端和 runtime 的定向测试为准，不使用不完整解释作为静默回退。 + +生产后端和规范执行器共享 HIR，但不共享最终执行机制。测试可以把同一个程序交给两条路径，比较： + +- 正常输出值和公开 stdout； +- choice、record、List、Map 和 Bytes 语义； +- pattern 选择与 `other` 补集； +- 数值溢出、除零和边界行为； +- 资源别名、关闭和 Stream 消费位置； +- 可预期错误与运行 fault 的分类。 + +这类差分测试能发现后端优化改变语言语义的问题。协议测试则使用独立构造的字节报文验证 JSON/HTTP，不以被测实现自身生成的数据作为唯一正确答案。 + +测试只依赖公开 Dever 行为和检查完成的 HIR 内部契约，不应绑定编译器内部帮助函数。这样重构前端或后端时，测试仍在验证语言，而不是验证某个暂时实现。 + +## 17. 一段源码如何被实现 + +以前面的 `show` 为例，编译器依次完成： + +1. 词法分析得到 `type`、choice 变体、function 名称、pattern 和表达式 token。 +2. 语法分析得到一个 `Result` choice 和两个同名同输入数量的 `show` 分句。 +3. 声明注册把两个分句归入同一个 function 组。 +4. 类型检查确认 `Ok` 绑定 Int，`Failed` 绑定 Text，两个分句都输出 Text。 +5. 分句分析把输入域拆成 `{Ok}` 与 `{Failed}`，证明两者互斥且完整。 +6. HIR 记录确定的变体、payload 局部位置、文本操作和源码位置。 +7. 可达性分析只在入口能调用 `show` 时保留它。 +8. 后端生成对 choice 标签的具体分支和直接的 payload 读取，不进行字符串名称查找或通用 pattern 解释。 + +这条路径体现了 Dever 的核心原则：源码保持声明式和类型化，复杂证明发生在编译期，生产程序只保留已经确定的具体操作。 + +## 18. 扩展语言时的规则 + +新增语法、类型或标准能力时，应遵守以下边界： + +1. 新语法先进入语法树，再由语义检查器建立唯一语义，不能只在某个后端特殊识别。 +2. 新 pattern 必须加入符号域和完整性证明，不能退回源码顺序优先匹配。 +3. 新 handler 使用位置必须保持静态可解析；若要引入运行时闭包，应作为明确的语言模型变更单独设计。 +4. 新标准算法优先写成 `.dever`；只有无法由语言表达的原子系统操作才扩展 `dever.system.*`。 +5. 新系统原语必须同时更新类型契约、规范执行路径、生产路径和一致性测试。 +6. 后端优化必须保持值语义、Map 顺序、数值规则、资源身份和 fault 位置。 +7. CLI 不拥有语言语义，只组合已有编译阶段。 +8. 生产路径不能引入通用解释器或动态 `Value` 作为功能回退。 + +这些规则让每项能力只有一个清晰拥有者，也避免标准包、检查器和执行后端出现三份逐渐分叉的逻辑。 + +## 19. 当前边界 + +本文描述的是当前语言和已经接通的编译架构，不代表所有生态能力都已完成。SQLite/PostgreSQL 类型安全 ORM 已使用编译器拥有的 `ModelSchema`/`QueryPlan`、具体生成代码和薄方言 executor 接入；Model、事务、迁移、Seed、关联和 typed native SQL 的合同见 [语言开发指南](LANGUAGE.md#13-modelorm-与数据库) 与 [Database Guidelines](.trellis/spec/backend/database-guidelines.md)。当前仍未交付的主要部分包括： + +- WebSocket-over-HTTP/2、子协议/扩展、CONNECT、推送及更完整的 HTTP 生态； +- package 注册表和正式安装发行流程； +- 跨平台发行验证； +- 用 Dever 自身实现编译器。 + +这些能力应建立在现有语义和系统原语边界之上，不能通过在 CLI 或运行时中增加隐藏业务逻辑来绕开语言设计。 + +## 20. 核心结论 + +Dever 的实现可以概括为： + +```text +源码负责表达意图 +编译器负责全局证明 +HIR 固化已经证明的语义 +静态特化消除运行时行为分派 +标准 .dever 包负责高层能力 +最小运行时只负责原子系统操作 +生产后端生成独立的本机程序 +``` + +因此，开发者看到的是一门具有明确值语义、穷尽分句和类型化错误的语言；编译器内部则保留更换实现、优化数据表示和扩展原生后端的空间。 diff --git a/LANGUAGE.md b/LANGUAGE.md new file mode 100644 index 0000000..514a9f8 --- /dev/null +++ b/LANGUAGE.md @@ -0,0 +1,1519 @@ +# Dever 语言开发指南 + +本文描述当前已经实现的 Dever 语言。它面向使用 Dever 编写程序的人，不涉及编译器内部实现。 + +编译、检查、静态特化和运行时边界的设计见 [Dever 语言实现原理](IMPLEMENTATION.md)。 + +## 1. 快速开始 + +先按 [安装指引](skills/references/install.md) 安装机器版 Dever。AI 开发使用独立 [dever-language skill](skills/SKILL.md)。已有命令时直接生成项目；目标目录必须不存在，父目录必须已存在： + +```bash +dever new my_app +# Markdown 写法：dever new my_app --markdown +``` + +命令接收项目根。Dever 源码固定放在 `module/`，部署配置固定放在 `config/setting.json`，运行数据放在 `data/`。模板生成： + +```text +my_app/ + config/ + setting.json + AGENTS.md + README.md + module/ + hello/ + greeting/ + app.dever + api.dever + test/hello/greeting/greet.dever +``` + +`my_app/module/hello/greeting/app.dever`： + +```dever +greet(name: Text) (message: Text) { + message = "Hello, " + name +} +``` + +`my_app/module/hello/greeting/api.dever`： + +```dever +cmd greet = app.greet +``` + +`my_app/config/setting.json`： + +```json +{ + "log": { "level": "info" } +} +``` + +模板不需要数据库或 HTTP 配置。添加 HTTP 时再完整配置站点与身份，不能只添加 get 而遗漏 sites/auth。配置不会编译进二进制。`run` 使用项目目录中的配置；独立二进制启动时使用可执行文件同级的 `config/setting.json`，相对数据路径也以该目录为基准。 + +语言在开发和安装后的命令名均为 `dever`，Go 框架使用 `dever-go`。开发仓库生成 `target/debug/dever`；机器版启动器根据 `config/setting.json` 中可选的精确 `dever.version` 选择已安装版本，未锁定项目使用机器活动版本，再把项目命令和退出码原样分派。启动器的内部构建名为 `dever-launcher`，安装后仍叫 `dever`，版本核心为 `dever-core`。Linux 安装制作与机器共享已实现；公开发行资产尚未发布。源码构建不会替换机器上已有的全局命令。 + +以下命令在已经提供 `dever` 的开发环境中执行： + +```bash +dever fmt my_app --check +dever check my_app +dever test my_app +dever run my_app -- hello.greeting.greet '{"name":"Dever"}' +dever build my_app --output my_app/app +``` + +Linux 开发环境运行构建产物： + +```bash +./my_app/app hello.greeting.greet '{"name":"Dever"}' +``` + +- `new`：从随版本验证的模板原子创建新项目，不覆盖已有目录；`--markdown` 生成等价 Markdown 源码。 +- 机器版 `update`：从官方 GitHub Releases 显式下载并校验，统一更新活动核心与对应 skill；不修改业务配置。`skill path [project-root]` 返回经验证的配套指引，`skill install <new-directory>` 安装稳定的 AI 加载入口。项目可锁定 dever.version，普通项目命令不会触发更新。 +- `check`：加载全部源码并完成语法、名称、类型和分句检查。 +- `fmt --check`：只检查格式，不修改文件。 +- `fmt`：统一格式化项目 `module/` 和存在的 `test/`；需要修复格式时单独执行 `dever fmt my_app`。 +- `test`：检查生产源码与应用测试，再按名称串行运行每个隔离用例；没有 `test/` 时报告 0 个测试并成功。 +- `run`：编译器从 API、CMD、Job 声明生成入口；无参数时按 `setting.json` 的 runtime mode 启动服务，`-- <component>.<domain>.<cmd> '<json-object>'` 只执行指定命令。 +- `build`：默认生成本机独立可执行程序；Linux x86_64 可加 `--target linux-aarch64`，使用预先准备的 ARM runtime 和依赖生成 ARM 程序。输出路径必须尚不存在，已有文件不会被覆盖。发布时把二进制放在项目根，使它能从同级 `config/setting.json` 加载部署配置。Windows 和 macOS 发行流程尚未交付。 + +`check`、`fmt`、`run` 和 `build` 默认只使用本地源码及随编译器提供的 package，不会隐式联网下载依赖。 + +## 2. 源文件与领域职责 + +Dever 源文件使用 `.dever` 或 `.dever.md` 后缀和 UTF-8 编码。两种格式共用语言规则；Markdown 写法见下一小节。标识符使用 ASCII；注释和 Text 可以包含任意 UTF-8 字符。 + +```dever +# 单行注释 +type Greeting { message: Text } +greet(name: Text) (response: Greeting) { + response = Greeting { message = "你好，" + name } +} +``` + +规则如下： + +- `#` 开始单行注释，没有块注释。 +- 不写分号，声明和语句由换行或结构边界分隔。 +- 应用 `run/build` 不使用手写 `main.dever`；入口由 API、CMD 与 Job 声明生成。 +- 业务源码固定为 `<component>/<domain>/<role>.dever`，或在确实需要按主题拆分时使用 `<component>/<domain>/<role>/<topic>.dever`。 +- role 只能是 `app`、`domain`、`model`、`port`、`adapter`、`api`、`job`。应用源码不写 `package`、`exposes`、`public` 或 `internal`。 +- `app` 中的全部函数自动成为领域能力，调用名固定为 `<component>.<domain>.<function>`；同组件可省略 component，同领域内部可写 `app.<function>`。`app/<topic>.dever` 的 topic 不进入调用名。 +- App 能力的输入、输出和向外传播的错误都属于公开契约，不能引用 Domain、Adapter 等私有类型。业务错误类型应声明在 App，Domain 可通过 `app.ErrorType.Variant` 使用它。 +- `domain`、`model`、`port`、`adapter`、`api` 和 `job` 都是领域私有实现。跨领域、跨组件只能调用目标 App；API 和 Job 入口不是普通源码调用目标。 +- App 可调用本领域 Domain、Port 及其他领域 App；API 只能调用同领域 App。Domain 只能调用同领域 Domain。Adapter 只能调用同文件 helper 与标准能力，不能反调 App、Domain、Model、Port 或其他 Adapter。Model 操作仍只能由所属领域 App 使用静态 DSL 执行。 +- 同一 role 的单文件与同名目录二选一。非 API 的 topic 目录保持一层；只有 `api/` 可递归。role 目录只有一个源码文件时必须合回 role 文件，避免无意义拆分。 +- 一个文件可放一组内聚类型和函数，不按函数机械拆文件。禁止用 `common`、`shared`、`utils`、`helper`、`base`、`service` 作为业务源码桶。 +- `port` 只声明类型和无函数体的操作合同，必须写 `fails app.ErrorChoice`；错误 choice 至少包含一个显式 error 分支，也可引用公开标准错误 choice。输入必须是无约束的明确值类型，输出同样不加范围约束。当前 Port 不接受 handler 参数。App/Domain 的外部文件、网络 I/O 必须经过 Port；日志、时间、随机数等已有能力保持原约束。 +- 没有 `import`、`use`、通配导入或别名。 +- 不同领域之间不能循环依赖；同领域职责文件共享一个依赖边界，函数递归仍被拒绝。 + +Port 示例：App 声明 `type DeliveryError { error Unavailable(message: Text) }`；同领域 `port.dever` 声明 `send(message: Text) () fails app.DeliveryError`；`adapter.dever` 实现 `port.send(message: Text) () { ... }`。Port topic 使用 `port.mail.send`。qualified 实现不能被普通函数调用。Port/Adapter 只能引用其合同实际使用的 App 类型，不能调用 App。Port 自有 DTO 保持私有，不能泄漏到 App 公开签名。 + +一个 Adapter 文件完整实现一个 Port；单实现自动绑定，多实现由 `config/setting.json` 的 `adapter.<component.domain[.topic]>.use` 选择，根 adapter 名为 `default`，topic 名为实现名。编译产物包含全部候选，运行时选择不改变 failure 全集、effect 或 suspension 合同。每个实现的实际 failure 必须是声明全集的子集。 + +Adapter 可声明 `setting { endpoint: Text token: Secret }`，通过只读 `setting.endpoint` 访问。配置形如 `{"adapter":{"notification.mail":{"use":"smtp","setting":{"endpoint":"...","token":"..."}}}}`。仅选中实现要求 setting；未知 Port/实现/字段、重复键、缺失或错误类型在入口前失败。无 setting 的单实现允许省略配置文件。Secret 不可观察，配置错误不回显值；不使用环境变量。 + +外部程序同样只能实现 Port。需要调用现成 Linux 二进制时用 `external command`；需要类型化多操作 Worker 协议时用 `external exec` 或生态 Adapter。 + +普通二进制不必实现 Dever 协议，也不必另写 Python/Go 包装。例如将程序放在 `module/media/tool/bin/tool`： + +```dever +# module/media/tool/port.dever +execute(args: List<Text>, stdin: Bytes?) (output: dever.process.Output) fails dever.process.Error +``` + +```dever +# module/media/tool/adapter.dever +external command "bin/tool" {} +``` + +```dever +# module/media/tool/app.dever +version() (text: Text) { + output = port.execute(["--version"], null) + text = dever.bytes.to_text(output.stdout) +} +``` + +```dever +# module/media/tool/api.dever +cmd version = app.version +``` + +执行 `dever run . -- media.tool.version '{}'`，或先 `dever build . --output program` 再执行 `./program media.tool.version '{}'`。 + +command Port 只有一个操作，操作名由业务决定；输入固定为 `args: List<Text>`，可选第二项 `stdin: Bytes?`，输出固定为 `output: dever.process.Output`，错误为 `dever.process.Error`。只需 argv 时可省略整项 stdin 参数。Output 有 `code: Int`、`stdout: Bytes`、`stderr: Bytes`；二进制输出不做隐式文本转换，正常非零退出码也是结果。每次调用启动一次进程，不重试，不使用 shell；空格、引号、`$()` 等按参数原样传递。未提供 stdin 时立即发送 EOF。启动失败、超时和输出超限产生源位置 runtime fault。 + +调用限额只从 `config/setting.json` 读取：`{"adapter":{"media.tool":{"command":{"timeout_ms":30000,"output_limit":1048576}}}}`。示例值就是默认值；timeout_ms 最大 3600000，output_limit 为 stdout/stderr 合计字节数、最大 8388608，均必须为正数。argv 含终止符总计最多 65536 字节且不含 NUL，stdin 最多 8388608 字节。超时、取消和退出会回收本次沙箱进程树。需要文件、网络或子进程时仍在声明中写 `allow file/network/process` 并使用已有部署授权；command 不接受 `setting` 或 `lib` 声明。 + +`run/build` 离线嵌入 ELF 和相邻 `bin/lib/` 下显式随附的普通动态库文件；静态程序可直接使用，动态程序使用签名发行包的 GNU loader 和 OS 库。不扫描宿主库，不接受符号链接或 lib 子目录；缺库、错架构、SONAME/版本不匹配、越界搜索路径在构建期失败。Linux x86_64 与 ARM64 必须提供各自架构的二进制和库，Dever 不转换第三方二进制。单独使用 command 不需要 `dever lib` 或 `dever.lock`；资源可随 Dever Package 分发，Package 自身仍须锁定。测试继续用 Port fake，不执行真实命令。 + +在 Adapter 文件中写一个 `external exec` 声明，编译器会用对应 Port 的全部操作、输入、输出、业务错误和可选 `setting` 生成唯一协议合同，不需要再逐个重复函数： + +```dever +setting { + endpoint: Text + token: Secret +} + +external exec "worker/mail" { + allow network +} +``` + +entry 是所属领域目录内的可移植相对路径；上例构建时收集 `module/notification/mail/worker/mail`，将全部候选 exec Worker 的字节、SHA-256 和可执行位嵌入程序。入口原子提取到私有 `data/cache/lib/<bundle>`，每次启动 Worker 重新验证精确资源清单、摘要和 Unix 所有权/权限；不扫描其他 bundle，不依赖部署目录保留源码 Worker。路径不接受绝对形式、`.`、`..`、反斜杠或盘符，也不通过 shell、`PATH`、环境变量或外部命令字符串查找。capability 名称为 `network`、`file`、`process`、`gpu`；重复值和未知值在检查期拒绝。Worker 握手必须精确回显已编译的 Port、Adapter、schema、操作和 capability 集合。Linux 的文件、网络、进程权限还由统一 OS 沙箱执行；文件授权仅来自 `config/setting.json` 的 Adapter 配置。GPU 与其他 OS 沙箱尚不支持，明确失败。宿主必须允许所需 namespace 和 seccomp；不会自动放宽 AppArmor 等主机策略。 + +`external` 只在 Adapter role 中有意义，且一个文件只能包含该声明和至多一个 `setting`。它完整实现且只能实现同领域一个 Port identity；不能同时写 Dever Adapter 函数或 helper。应用测试仍必须提供 case-local Dever Port fake，不启动生产 Worker，也不读取生产 Adapter setting。reference 后端明确不执行外部程序，正式执行只走 native 后端。 + +生态 Adapter 复用同一 Port 合同，声明中的 Lib 不带重复的生态前缀： + +```dever +external pip "worker/mail.py" { + lib "httpx@0.28.1" + allow network +} +``` + +`pip/npm/go` 与 `exec` 使用相同的生命周期和 typed wire schema；各 Worker 的依赖与 runtime 独立锁在 `dever.lock`。Lib 解析是显式 `dever lib` 操作；`run/build` 只读取已锁定的资产。Python/JavaScript 打包器生成 SDK、合同和 runner，自动绑定入口中与 Port 操作同名的函数，不要求业务再写 handlers 注册表。运行树只从锁定 pack、依赖和当前 Adapter 源码生成；Python 固定使用 `-I -S -B`，Node 支持 ESM 和 CommonJS。启动解释器、argv 和工作目录必须匹配编译嵌入的资源清单，既不寻找系统语言环境，也不从部署配置接受任意命令。 + +Go Worker 使用锁定的目标专用 build pack 内的 compiler、linker 和标准库离线编译。入口源码声明 `package main`，按操作名导出 `func(context.Context, any, any) (any, error)`：`send` 对应 `Send`，`text.render` 对应 `TextRender`；生成入口静态绑定函数并由 Go 编译器检查签名，不要求业务手写 `main` 或注册表。构建按目标 build tags 选文件，支持 `go:embed`，导入只能来自标准库、当前 Adapter 源码、SDK 或锁定的 Go module 闭包。构建产物只携带 Worker 可执行文件与 checked 合同，启动不寻找系统 Go。参与解析的 `go.mod` 和最终 module ZIP 均验证官方 sumdb 签名、包含证明与 Go `h1`；项目现有锁保存 checkpoint，一致性和反回滚以该锁为锚，主动删除锁会重新建立锚。 + +`dever-lock-v5` 保存每个 Worker 的精确生态依赖。Python extras 使用规范化、排序后的 `pip:name[a,b]@version`，不同 Worker 不共享隐式 extras；实际选中 wheel 的 METADATA 决定依赖，原生 wheel 还必须符合签名解释器的 tags、扩展 ABI 和封闭动态库搜索规则。npm 保留每个安装实例的嵌套路径、peer 上下文、optional 省略原因和 bundled 归属，不能把不同依赖上下文压成一份全局版本表。 + +显式 `dever lib add/update` 可用签名 build pack 构建 PEP 517 sdist、执行 npm registry 包的 `preinstall/install/postinstall` 和原生 addon 编译。build pack 必须提供目标运行时、头文件、编译器、sysroot、固定 shell 和工具闭包；缺少的工具或库明确报错，不寻找宿主环境。Python 的静态与动态 build requirements 使用独立精确锁，console scripts 安装到隔离构建 prefix；metadata 与 wheel hook 分进程执行，最终 wheel 必须匹配父进程保存的 metadata。npm publishing hooks 不作为 registry 安装钩子执行，可选依赖构建失败会裁剪其不可用闭包，必需依赖失败则整个准备失败。 + +源码构建收据绑定源码摘要、运行时、工具包、固定 frontend、构建依赖与输出摘要；npm 还绑定完整安装图。`run/build` 离线消费已锁定输出，不再次执行上述安装钩子或源码构建。Node addon 在仅有目标 Node 和运行库的独立沙箱中逐个验证；所有 `.node` 文件必须有成功或明确拒绝记录，运行树只安装成功项，其他平台/ABI 的变体保留在原锁定归档中。本次源码生成的 addon 验证失败会使准备失败，不能改用纯 JavaScript 回退并声称原生构建成功。Linux 构建使用只读工具与输入、独占可写工作区并禁止网络。构建期限为五分钟，stdout/stderr 各限 16 MiB；工作区每 250 ms 检查 512 MiB/65,536 项预算并在超限时回收子进程，这不是内核磁盘 quota，检查间隔内可暂时超限。最终安装树仍限 256 MiB/65,536 项。普通 Worker 的能力由同一 OS 沙箱执行，其他平台不能借宿主工具绕过缺失的发行支持。 + +Dever Package 的请求放在 `config/setting.json.package`：`registry` 是显式 HTTPS registry origin，`use` 是 `name@version-range` 数组。`dever package add/update` 解析版本和传递依赖；Package 与 Lib 共用 `dever.lock` 和机器摘要缓存。Package 的源码归属于 `module/<name>/`，Worker 资源归属于同组件路径或 `worker/<name>/`；其他组件路径不能混入归档，也不能与项目本地组件重名。`check/test/run/build` 自动加载锁定源码，无须增加 import 或注册方法；缺失的锁定 Worker 必须失败，不能使用项目同名文件替代。`remove` 离线裁剪已验证锁，不重新选择剩余依赖版本。 + +`test/<component>/<domain>/<topic>.dever` 可在本文件写 `port.send(...) (...) { ... }` fake；每个可达 Port 必须完整提供 fake，不读取生产 Adapter 配置或回退生产实现。同一套件只编译一次，每个 case 在独立进程绑定自己的 fake。编译检查与代码生成也按 case 独立推断 effect、failure 和挂起行为，不合并不同测试的 fake；生产合同独立检查。 + +持久任务使用同领域 `job.dever`，复杂时按主题拆成平铺 `job/*.dever`。Job 只适配同领域 App，不直接调用 Model、Domain、Port、Adapter 或并发原语： + +```dever +database default +job publish(input: app.Publish) () retry(5) timeout(30000) { + app.apply(input) +} +job cleanup() () retry(3) timeout(1000) { + app.clean() +} +schedule cleanup = "0 * * * *" +``` + +Job 必须声明数据库、`retry` 和 `timeout`。输入为零个或一个同领域 App record，输出为零；payload 使用共享 wire codec，允许保留 Model 身份的 ModelId，不允许 Secret、private 字段、资源或 Choice。`retry(1..100)` 包含首次执行，`timeout(1..3600000)` 单位为毫秒。入口不能作为普通函数或 handler 传递。 + +App 使用 `dever.job.enqueue(job.publish, input, key)`；无 payload 时省略 input。`enqueue_at` 在 key 后增加 DateTime。返回 Id；key 为 1..1024 字节的业务幂等键。同物理租户、执行租户、Job identity 与 key 的 pending/running 任务返回同一 Id，终结后允许重新使用 key。transaction App 内业务写入与入队共享物理事务；跨连接事务被拒绝。入队不会同步调用 handler 或继承其 effect/failure。 + +入队必须处于编译器建立的可信执行作用域。受保护 HTTP 入口在自动权限校验成功后保存 User 身份的 Provider、站点、subject、session、租户声明和该入口唯一的 permission key；不保存 JWT、Cookie、角色或权限集合。Worker 每次执行都重新调用配置绑定的 verify，确认该 key 仍在当前编译产物的权限目录，并查询当前角色；session、租户、组件或权限已失效时任务进入 blocked，业务 handler 不执行。public HTTP 调用链不能入队；CMD、schedule 和测试入口使用明确的 System 身份。Job payload 内的用户或租户字段始终只是业务数据，不能成为可信执行身份。 + +`schedule` 仅绑定本文件零输入 Job。cron 固定为 UTC 五字段（分、时、日、月、星期），支持数字、`*`、列表、范围、步长，星期为 0..6；日和星期均受限时按 OR 匹配。首次启用从当前分钟建立 cursor，不回放历史；后续每轮最多扫描并推进 256 个分钟槽，仅匹配 cron 的槽生成任务。cursor、occurrence 和推进同事务，多 scheduler 与已完成任务都不会重建同一窗口。修改已持久化的 cron fingerprint 会失败，需要明确迁移。 + +应用入口由编译器根据 Job 与 API 声明生成；两者同时存在时，共用运行时并由唯一配置文件 `config/setting.json` 选择启动模式： + +```json +{ + "runtime": { "mode": "all", "shutdown_ms": 30000 }, + "job": { "workers": 2, "poll_ms": 250, "lease_ms": 60000, "retry_base_ms": 1000, "retry_max_ms": 300000 } +} +``` + +这里还需配置实际 `database`，API 模式需配置 `http`。`mode` 为 `api`、`worker` 或 `all`；省略 runtime 时编译产物按实际 API/Job 能力推导 api、worker 或 all，关闭期限默认 30000 ms。显式 mode 仍必须与源码能力和所需配置一致。worker 连接池至少比 worker 数量多一个连接，留给 claim/heartbeat。SQLite 的写事务会阻塞续租，因此还要求 `lease_ms` 大于每个 Job 的 `timeout`；PostgreSQL 支持较短租约并续租。未知字段、无效范围、所选 mode 缺少源码能力或设置均在监听/claim 前失败，不支持环境变量或命令行覆盖。 + +任务是至少一次执行；外部副作用仍需业务幂等。每次 claim 增加 attempt 并生成独立 token，续租和结果提交要求 token 与有效 lease 匹配。崩溃、取消后的任务保留 lease，到期才能重领，达到次数上限进入 dead。失败按有界指数退避；未知 target、schema 不兼容、坏 payload、无可信执行身份、身份/能力复核失败，或不满足当前租约约束的历史任务策略进入 blocked，错误摘要不保存业务明文。私有队列 schema 升级时，无法证明执行身份或能力的历史活跃 User 任务也会阻断，不会降级为 System。SIGTERM/Ctrl-C 停止接收新请求和 claim，排空到共同期限后取消在途工作，汇合后关闭数据库并 flush 日志。 + +应用测试可用 `dever.test.advance_clock(ms)` 推进 case 独立时钟，`dever.test.drain_jobs(limit)` 执行到期任务并返回处理数（limit 为 1..10000）。测试复用生产队列状态机、case 的 SQLite 与 Port fake，不启动服务或读取部署配置；没有可达 Job 时 drain 返回 0，也不因此要求数据库。 + +`job.lease_ms` 范围为 100..7200000 毫秒，Job timeout 上限仍为 3600000 毫秒。测试使用最大租约，保证所有合法 timeout 都能在隔离 SQLite 中运行。 + +路径示例： + +```text +my_app/module/user/account/api.dever +my_app/module/user/account/app.dever +my_app/module/user/account/domain.dever +my_app/module/user/account/model.dever +my_app/module/news/article/app/publish.dever +my_app/module/news/article/app/query.dever +``` + +最后两项表示文章 App 已有至少两个内聚主题；对外仍调用 `news.article.publish(...)` 和 `news.article.count()`。只有一个主题文件时应改回 `news/article/app.dever`。 + +命名规则： + +| 对象 | 格式 | 示例 | +| --- | --- | --- | +| component、domain、role、topic、function、字段和局部值 | `lower_snake_case` | `user.account`、`can_sign_in` | +| type、选择项 | `UpperCamelCase` | `User`、`Created` | +| 缩写 | 按普通单词处理 | `Id`、`HttpClient`、`user_id` | + +### 2.1 Markdown 程序源码 + +使用 `.dever.md` 可以把程序和面向人的用途说明放在一起。Dever 代码仍写在顶层 `dever` 或 `typescript dever` 围栏中；`typescript dever` 让编辑器尝试使用 TypeScript 高亮，第二个标记用于确认其中仍是 Dever 源码。普通 `.md` 文档不参与编译。 + +Markdown 结构也是编译契约：一个一级标题说明包，一个二级标题说明一个 type 或一个逻辑函数。标题使用自然语言，技术名称、公开清单、输入输出和类型结构使用固定列表，并由编译器与代码核对。 + +例如 `user/account/api.dever.md`： + +````markdown +# 问候接口 + +向调用方返回一条问候语。 + +- 包：`user.account.api` +- 公开类型：无 +- 公开方法：无 +- 使用：无 + +## 获取问候 + +绑定本领域的问候能力。 + +- 声明：`get greeting` + +```typescript dever +get greeting = app.greet +``` +```` + +规则如下： + +- 文件必须有且只有一个一级标题。一级标题下先写非空包说明，再依次写 `包`、`公开类型`、`公开方法`、`使用`；一级标题下不能有 Dever 代码块。 +- 每个 type、逻辑函数或 API 声明必须分别使用一个二级标题、非空用途说明和一个 Dever 代码块。一个逻辑函数包括同一名称、同一参数数量的全部条件分句；不同声明不能共用说明，分句也不能拆到多个说明中。 +- type 说明使用 `类型` 以及 `字段` 或 `分支`。函数说明使用 `函数`、`输入`、`输出`；API 说明使用 `声明`。空清单显式写 `无`；非空输入、输出、字段和分支均须带自然语言说明。 +- 源码身份必须与路径一致。App 的公开名称和调用方式由 role 自动得到；Model 的记录、choice 和 ID 类型由 Model role 自动提供；API 声明不列入公开方法。字段、分支、函数输入和输出的名称、类型、数量及顺序必须与代码一致；同一函数各分句的输入参数名必须相同。函数模式按解析后的统一类型记录，例如 `true`/`false` 分句写 `Bool`。 +- 路径规则与普通源码一致；`user/account/app.dever.md` 的源码身份是 `user.account.app`，其中 `register` 的调用方式是 `user.account.register(...)`。 +- 两种源码格式可以在一个项目中混用；Model/API 主文件可与 topic 目录并存，其他 role 二选一。 +- 程序围栏使用至少三个反引号或波浪号，允许 0–3 个空格缩进，信息字符串须是字面的 `dever` 或 `typescript dever`；关闭围栏使用相同字符且长度不短于开始围栏，尾部只允许空格，不允许 Tab。程序围栏必须闭合，不能依靠文件结束隐式关闭。 +- 三级及更深标题、表格、普通补充说明、其它语言及无语言标记的代码块不执行。引用、列表、HTML 或展示代码块内的围栏不作为程序。补充内容放在所属 Dever 代码块之后。 +- 函数输入输出、条件分句、类型、失败义务、`pure` 和依赖边界都按现有 Dever 规则检查，说明文字不能代替正式契约。 +- `check/api/run/build` 命令不变，并强制检查 Markdown 契约；错误显示 `.dever.md` 原文中的行列并关联对应代码。`fmt` 检查结构并仅更新程序块内部，保留说明、围栏和展示代码块。 + +固定列表的完整格式见 [Dever Markdown 源码规范](MARKDOWN-SYNTAX.md)，完整接口示例见 [Markdown CMS](examples/cms/md/module/news/article/api/admin/manage.dever.md)。`typescript dever` 只复用现有高亮规则；精确的 Dever 高亮、补全和语言服务仍需编辑器扩展。 + +### 2.2 应用测试 + +应用测试位于项目根的 `test/<component>/<domain>/<topic>.dever` 或 `.dever.md`。文件名 `<topic>` 选择同名的普通 `topic() ()` 作为唯一入口；入口不能有参数或输出，也不能声明为 transaction。文件中的其他函数和类型只供本文件使用，不会自动执行或向生产源码、其他测试公开。路径只允许 component、domain、topic 三层，不建立 `common`、`shared` 或全局 fixture 目录。 + +测试是所属领域的受控友元：可调用同领域 App 和 Domain，跨领域只能调用目标 App。测试可以读取 App 返回的 Model 值，但不能直接执行 Model CRUD，也不能调用 Port、Adapter、API 或其他测试。生产源码同样不能反向依赖测试。测试不能启动 `dever.api.serve()`。 + +首版断言只有： + +```dever +register() () { + account = user.account.register("New@Example.com", "New", "hash") + assert(account.email == "new@example.com") + assert_eq(account.display_name, "New") +} +``` + +`assert` 只接受 Bool；`assert_eq` 要求两侧类型完全相同且可比较，不做隐式数值转换。失败报告测试身份、原始源码位置和实际/期望值。业务失败继续用 `result(call)` 与 choice 分句验证，不增加异常断言语法。 + +`dever test <project-root>` 在执行任何用例前检查全部生产和测试源码，并把所有测试入口编译成一个可缓存的原生测试程序；每个测试仍在独立原生进程与临时项目中运行。不使用 Model 的测试不读取数据库配置或初始化数据库；使用 Model 的测试走正式迁移、Seed、事务和关闭流程，但配置由运行器在临时目录生成且只使用 SQLite。测试命令不会读取项目 `config/setting.json` 或项目 `data/`，也不从环境变量接收数据库覆盖；成功和失败都会清理临时文件。首版不提供共享 setup、筛选、并行、重试或自动超时。 + +## 3. 类型与值 + +### 3.1 基础类型 + +| 类型 | 含义 | +| --- | --- | +| `Bool` | `true` 或 `false` | +| `Int` | 有符号 64 位整数，溢出不会环绕 | +| `Decimal` | decimal128，最多 34 位有效数字，银行家舍入 | +| `Float` | IEEE 754 binary64 近似数 | +| `Text` | UTF-8 文本，不做隐式 Unicode 规范化 | +| `Id` | 不透明稳定标识 | +| `Bytes` | 二进制字节序列 | +| `Secret` | 不透明敏感值；仅受控密码学及输入边界使用 | + +literal 规则： + +```dever +enabled = true +count = 42 +price = 19.90 +ratio = 1.25e6 +name = "中文和 Unicode 都可以" +missing = null +``` + +- 整数 literal 默认是 `Int`。 +- 带小数点的 literal 默认是 `Decimal`；明确的 Float 上下文可将其作为 `Float`。 +- 科学计数 literal 是 `Float`。 +- Text 支持 `\\`、`\"`、`\n`、`\r`、`\t` 和 `\u{HEX}`。 +- `Decimal` 与 `Float` 不隐式混合；Int 转 Float 使用 `float.from_int`。 + +### 3.2 字段型 type（record） + +字段型 type 是固定结构的数据： + +```dever +type User { + id: Id + name: Text + phone: Text? +} + +create_user(id: Id, name: Text) (user: User) { + user = User { + id = id + name = name + phone = null + } +} + +rename(user: User, name: Text) (result: User) { + result = user + result.name = name +} +``` + +构造时必须一次写全全部字段。未知、重复或遗漏字段都是编译错误。字段只能在完整值上读取或修改，使用 `value.field = expression` 语法；不能靠逐字段赋值创建一个半初始化值。 + +### 3.3 选择型 type（choice） + +选择型 type 表示一个值当前只能处于其中一种状态： + +```dever +type CreateResult { + Created(user: User) + InvalidName(reason: Text) + EmailAlreadyUsed(email: Text) +} + +result = CreateResult.InvalidName("name is required") +``` + +选择项必须通过所属 type 限定。选择项可以不携带数据，也可以声明一个或多个有名称、有类型的携带值。携带值的名称属于 type 契约；构造和匹配仍按声明位置书写，不是命名参数调用。 + +### 3.4 可空、List 与 Map + +`Value?` 表示 `Value` 或 `null`。没有 `Value??`、隐式默认值、`?.` 或强制解包。 + +```dever +names = ["Ada", "Lin"] +labels = { + "env" = "production" + "region" = "china" +} +``` + +- List 的元素必须具有相同静态类型。 +- Map 的类型是 `Map<Key, Value>`，保留插入顺序。 +- Map key 可以是 Text、Int、Bool、Id 或不携带数据的选择型 type。 +- Decimal、Float、可空值、字段型值、List 和 Map 不能作为 Map key。 +- 空 `[]` 和 `{}` 必须能从参数或输出等上下文推断类型。 +- `get`、`first` 和 `find` 返回可空值；可空化是幂等的。 +- `entries(map)` 返回保持插入顺序的 `List<MapEntry<Key, Value>>`；每个 `MapEntry` 有 `key` 和 `value` 两个字段。修改局部 MapEntry 仍遵守值语义，不会回写原 Map。 + +### 3.5 Stream 与资源 + +`Stream<Item>` 是按需拉取、顺序、一次消费的流。文件、Socket、Listener 和 Stream 是资源值；复制它们会建立指向同一资源状态的别名。Stream 的所有别名共享一个读取位置，任何别名消费或关闭后，其他别名都会看到相同状态。 + +`AsyncStream<Item>` 是异步拉取的资源流，等待下一项时挂起当前协程。它的别名共享读取位置和关闭状态；元素可跨线程传递时，流也可跨线程传递。它不与同步 Stream 隐式转换，不预取或自动收集为 List。 + +普通字段型值、选择型值、List 和 Map 使用值语义：修改一个局部名称不会改变调用方或其他名称持有的值。 + +## 4. function 与命名输出 + +Dever 的所有行为都由 function 定义： + +```dever +line_total(price: Decimal, quantity: Int) (total: Decimal) { + total = price * quantity +} +``` + +- 第一组括号是输入，第二组括号是命名输出。 +- 没有输入或输出时仍写 `()`。 +- 不使用 `function` 或 `return` 关键字。 +- 调用按位置传参，不支持命名参数调用。 +- 零输出调用是一条动作语句。 +- 单输出调用直接得到值。 +- 多输出调用得到只能按输出名称访问的结果。 +- 用户 function 不能直接或间接递归。 +- function 签名由 package、名称和输入数量确定；同名但输入数量不同的 function 是不同签名。 + +多输出示例： + +```dever +name_parts(value: Text) (original: Text, normalized: Text) { + original = text.trim(value) + normalized = text.lower(original) +} + +display(value: Text) (result: Text) { + parts = name_parts(value) + result = parts.original + ":" + parts.normalized +} +``` + +普通应用没有手写 `main()`；命名输出保留在 App 函数中，API/CMD 绑定一个可序列化输出并封装为 JSON。 + +## 5. 分句是唯一条件机制 + +Dever 没有 `if`、`else`、`switch`、`match` 或条件表达式。所有业务分支都使用同名 function 分句： + +```dever +type OrderStatus { + Draft + Paid + Rejected(reason: Text) +} + +status_text(status: OrderStatus.Paid) (result: Text) { + result = "paid" +} + +status_text(status: OrderStatus.Rejected(reason)) (result: Text) { + result = reason +} + +status_text(status: other) (result: Text) { + result = "pending" +} +``` + +可空值也通过分句处理： + +```dever +phone_text(phone: Text) (result: Text) { + result = phone +} + +phone_text(phone: null) (result: Text) { + result = "not provided" +} +``` + +数值范围可以直接写在输入模式中： + +```dever +grade(score: Int < 60) (result: Text) { + result = "fail" +} + +grade(score: Int >= 60 and < 90) (result: Text) { + result = "pass" +} + +grade(score: Int >= 90) (result: Text) { + result = "excellent" +} +``` + +同一签名的全部分句必须紧邻书写。中间插入 type 声明、其他 function 或另一输入数量的同名 function 后，再次书写原签名会直接报错，不会形成第二组规则。 + +分句输入支持：完整类型、选择项、`null`、Bool/Text/数值常量、Int/Decimal 范围和 `other`。选择项必须为每个携带值提供一个位置绑定；不需要使用的绑定可以写 `_`。编译器把一组分句作为规则检查： + +- 输入和输出契约一致； +- 分句互不重叠； +- 所有可能值都被覆盖； +- 没有不可达分句。 + +分句没有源码顺序优先级。多输入分句按所有输入组合检查覆盖和重叠；`other` 是该输入位置在整组明确模式之外的补集，只能用于分句输入。 + +## 6. 线性 body 与运算 + +function body 只包含顺序赋值和 function 调用： + +```dever +final_price(price: Decimal, rate: Decimal) (result: Decimal) { + discounted = price * (1 - rate) + result = decimal.round(discounted, 2) +} +``` + +`=` 用于初始化，也用于修改当前 function 内的 input、局部值或 output。编译器保证读取前已初始化、字段宿主完整、局部类型稳定，并要求退出前给所有输出赋值。 + +运算规则： + +| 运算 | 适用范围 | +| --- | --- | +| `+ - *` | 同类 Int、Decimal、Float；Int 可无损提升为 Decimal | +| `/` | Int/Int 与 Decimal 运算返回 Decimal；Float/Float 返回 Float | +| `//`、`%` | 仅 Int，整数除法向零截断 | +| `+` | 也用于两个 Text 的连接 | +| `== !=` | 相同的可比较类型 | +| `< <= > >=` | Int、Decimal、Float | +| `and or not` | Bool，`and` 和 `or` 短路 | + +相等比较也适用于成员本身可比较的字段型、选择型、可空值、List、Map 和 MapEntry。List 按位置比较；Map 按插入顺序比较，所以键值相同但插入顺序不同的两个 Map 不相等。Float 遵守 IEEE 754 规则，`NaN == NaN` 为 `false`，包括它嵌套在集合或字段中时。 + +优先级从高到低为：调用和字段访问、一元 `-`/`not`、`* / // %`、`+ -`、比较、`and`、`or`。同级二元运算从左到右，括号可以改变顺序。 + +Int 溢出、Decimal 除零等静态可知故障在检查阶段报错；运行时才知道的数值故障会终止当前执行并报告 `.dever` 文件、行和列。 + +## 7. 集合与受控重复 + +Dever 没有 `for`、`while` 或用户递归。重复处理使用具名 handler 和标准组合器： + +```dever +views = each(to_view, users) +active = filter(is_active, users) +user = find(has_id, users, expected_id) +total = sum(prices) +total = sum(line_total, lines, tax_rate) +state = reduce(update_state, values, initial_state) +state = reduce_until(update_until_done, stream, initial_state) +parallel_each(send_one, jobs, 4, context) +``` + +| 操作 | 可处理序列 | handler / 结果 | +| --- | --- | --- | +| `each` | List、Bytes；Stream、AsyncStream 仅限零输出 handler | 零输出时执行动作；单输出时生成 List | +| `filter` | 仅 List | handler 输出 Bool，返回保留原顺序的 List | +| `find` | 仅 List | handler 输出 Bool，返回第一个匹配项或 `null` | +| `sum` | 仅数值 List | 直接求和，或用单输出数值 handler 投影后求和 | +| `reduce` | List、Bytes、Stream、AsyncStream | handler 为 `(Item, State) -> State`，返回最终状态 | +| `reduce_until` | List、Bytes、Stream、AsyncStream | handler 输出下一状态和 `stop: Bool`，处理当前元素后可停止 | +| `parallel_each` | List、Bytes、Stream | 非挂起零输出 handler；第三个参数是请求并发数，范围为 1–256 | +| `parallel_each` | List、Bytes、AsyncStream | 挂起零输出 handler；第三个参数是任务组容量，范围为 1–65536，同时受全局任务容量约束 | + +Bytes 的元素类型是 Int，值域为 0–255。List、Bytes 可使用非挂起或挂起 handler；同步 Stream 只使用非挂起 handler。AsyncStream 的 `each`、`reduce`、`reduce_until` 可使用两类 handler，具体类别由编译器推断。非挂起 handler 不能在挂起执行路径中隐藏阻塞或未限定的 handler effect。`each`、`filter`、`find` 和投影式 `sum` 最多可以带一个末尾 context，handler 的输入顺序是元素、context。`parallel_each` 的可选第四个参数也是 context。需要多项 context 时，先组合成一个字段型值。`reduce` 和 `reduce_until` 不接收 context；共享数据应放入 State。 + +空 List 的 `sum` 返回相应数值类型的零；空序列的 `reduce` 和 `reduce_until` 原样返回初始状态。`reduce_until` 在处理当前元素并得到 `stop = true` 后停止；如果输入是 Stream，下一个元素保持未消费，可以通过任一 Stream 别名继续读取。 + +AsyncStream 消费会使当前函数被推断为挂起。组合器本身顺序等待拉取和 handler 完成，不需要额外包装。消费返回、提前停止、故障或取消都会关闭这个异步流，其他别名不能继续消费。`parallel_each` 先等待空位再拉取，并在等待下一项时观察子任务故障；返回前清理已启动的工作。`close(stream)` 本身不挂起，会唤醒等待中的拉取。 + +`reduce` 示例： + +```dever +add(value: Int, total: Int) (next: Int) { + next = total + value +} + +total(values: List<Int>) (result: Int) { + result = reduce(add, values, 0) +} +``` + +`reduce_until` 的 handler 必须输出下一状态和名为 `stop` 的 Bool： + +```dever +take_until_limit(value: Int, total: Int) (next: Int, stop: Bool) { + next = total + value + stop = next >= 100 +} +``` + +除 `parallel_each` 外，List 和 Bytes 按位置顺序处理，Stream 按拉取顺序处理，挂起 handler 逐项等待完成。完全非挂起的 `parallel_each` 创建不超过请求并发数的作用域线程。处于挂起调用链时，命名的非挂起 handler 复用 runtime 的 blocking worker，实际并发受该 worker 上限约束，且不能再次触发并发；挂起 handler 则复用有界 Group，先等待任务容量再取下一项。`parallel_each` 的 Stream 仍由单一生产端顺序拉取，不会先把整个流收集到内存。handler 的业务错误保留原类型并由调用方继续传播，不转换成普通字符串。handler 副作用的完成顺序不保证，调用返回前会等待所有已启动工作结束。handler 输入和 context 不能包含 Stream。 + +## 8. 静态 handler + +function 可以接收受限的静态 handler： + +```dever +transform_all( + transform: handler(value: Int) (result: Text), + values: List<Int> +) (results: List<Text>) { + results = each(transform, values) +} +``` + +handler 只能是签名匹配的具名 function，或者当前 function 收到的 handler 参数。它不是普通运行时值，不能存入局部值、字段、List、Map、Stream，不能比较，也不能作为输出返回。 + +## 9. 挂起推断与结构化并发 + +所有 function 使用同一种声明语法。普通调用保持顺序完成；如果调用链包含网络、非阻塞定时器、AsyncStream 或其他挂起操作，编译器会沿静态调用图推断挂起 effect，并在生成的 Rust 内部插入 `.await`。Dever 源码不声明 `async`，也不写 `await`： + +```dever +sleep_ok(result: dever.time.SleepResult.Done) () recover("this operation may continue after timer failure") {} + +sleep_ok(result: dever.time.SleepResult.Failed(message)) () recover("this operation may continue after timer failure") {} + +delayed_value(value: Int) (result: Int) { + sleep_ok(dever.task.sleep(10)) + result = value +} + +main() (result: Int) { + task = run(delayed_value(42)) + result = wait(task) +} +``` + +直接调用始终等待结果；只有 `run(call(...))` 显式并发启动。`run` 同时支持挂起函数和普通函数，普通函数会进入受限 blocking worker，不会堵塞调度线程。它返回局部仿射 `Task`：Task 不能复制、嵌套进其它值、跨线程传递或作为输出返回，并且必须恰好由一次 `wait(task)`、`stop(task)`、`timeout` 或 `race` 消费。`stop` 仅接受零输出 Task，语义是请求取消并等待任务停止；取消不会回滚已经发生的外部副作用。Task 离开异常路径时也会请求取消，不会成为脱离父作用域的后台任务。 + +动态数量的零输出任务使用 `Group`： + +```dever +workers = group(8) +run(workers, write_one(message)) +wait(workers) +``` + +`group(limit)` 的 limit 范围是 1–65536。达到 limit 后，新的 `run(group, call(...))` 会等待已有任务完成，不会无限排队。Group 也是局部仿射值，最终必须由 `wait(group)` 或 `stop(group)` 消费；第一个子任务失败时，其余子任务会被取消并等待结束。 + +CPU 计算和阻塞调用必须显式分类：`parallel(pure_call(...))` 只接受 pure 非挂起 function；`blocking(blocking_call(...))` 只接受 effect 分析已识别为阻塞系统边界且不含未知 handler effect 的非挂起 function。文件、同步睡眠和标准输出属于阻塞边界；TCP 调用会被推断为挂起。两种边界的参数都在提交前按源码顺序求值，工作由有界线程池承担。挂起函数不能原地执行阻塞调用、隐藏并发的集合 handler 或 effect 未知的非挂起 handler 参数；必须使用 `blocking` 或调整调用结构。 + +有界 Channel 用于任务通信： + +```dever +messages = channel(Text, 64) +send(messages, "ready") +message = receive(messages) +close(messages) +``` + +Channel 容量范围是 1–65536。`send`、`receive` 会使当前调用链被推断为挂起；队列满时发送方等待，队列空时接收方等待。`receive` 返回 `T?`：收到值时返回该值，Channel 关闭且缓存值耗尽后返回 `null`。Dever 不产生嵌套可空类型，因此 `Channel<T?>` 的接收结果仍是 `T?`，发送的 `null` 与关闭后的空结果都表现为 `null`。`close(channel)` 幂等，并唤醒等待中的发送方和接收方；关闭前已经缓存的值仍可读完。`Channel<T>` 和 `AsyncStream<T>` 的 T 必须可跨线程传递。 + +跨挂起点仍存活或进入 `run`、Channel、`parallel`、`blocking` 的普通值必须可跨线程传递。Task 和 Group 是受编译器跟踪的局部仿射值，不能嵌套、传参或逃逸。完全非挂起的入口不初始化 Tokio；推断为挂起的入口创建唯一的有界 Tokio 多线程运行时。默认调度线程数按 CPU 数量确定且最多 64，CPU/阻塞 worker 默认最多 8，同时存活的 `run`/Group 子任务默认最多 4096，硬上限为 65536。 + +`dever.task.sleep(milliseconds)` 是非阻塞定时等待，要求非负毫秒数，成功无输出，失败默认传播；显式捕获使用 `dever.time.SleepResult`。已有 `dever.time.sleep` 保持同步阻塞语义；挂起调用链需要同步睡眠时必须使用 `blocking(...)` 隔离。运行时由生成入口统一启动，业务源码不手工创建或嵌套 runtime。 + +`timeout(task, millis, fallback)` 等待任务；millis 必须为正。到期先停止并排空任务，再执行零输入的静态 fallback。fallback 必须与任务的命名输出签名完全相同，支持零个、一个或多个输出。程序故障继续传播，不触发 fallback。`race(first_task, second_task, ...)` 至少接收两个输出签名相同的 Task，等待第一个完成，再停止并排空其余任务；它也不吞掉程序故障。两者会使调用链挂起，并消费传入的全部 Task。 + +```dever +unavailable() (value: Text) { + value = "请求超时" +} + +# lookup 的输出声明为 (value: Text)。 +pending = run(lookup()) +value = timeout(pending, 1000, unavailable) + +first_task = run(lookup_primary()) +second_task = run(lookup_secondary()) +value = race(first_task, second_task) +``` + +`stream(channel)`、`stream(list)`、`stream(bytes)` 创建按需拉取的 AsyncStream；元素必须可传递，Bytes 产生 Int。消费流时由编译器推断挂起。Channel 流先读完关闭前的缓存再结束，发送的 null 仍是一个实际元素。关闭流释放其 Channel 别名，不关闭调用方另持有的 Channel。 + +`dever.task.ticks(millis)` 创建 `AsyncStream<Int>`，周期必须为正；第一个值在一个周期后产生，序号从 1 开始。消费过慢时跳过错过的周期，不积压 tick 或预先创建任务。用 `each`/`reduce_until` 消费，关闭流释放定时器。 + +## 10. 失败处理 + +可恢复失败使用 choice 中显式标记的 `error` 变体。编译器不根据 `Failed` 等名称猜测错误： + +```dever +type MessageResult { + Ready(text: Text) + error Failed(message: Text) +} + +read_message(bytes: Bytes) (message: Text) { + message = dever.bytes.to_text(bytes) +} + +required_port(value: Int < 1) (port: Int) { + fail(MessageResult.Failed("port must be positive")) +} + +required_port(value: Int >= 1) (port: Int) { + port = value +} +``` + +普通调用默认传播失败，因此 `message = dever.bytes.to_text(bytes)` 不需要 `?`、`try` 或 `await`。调用链的错误集合由编译器推导；未处理的失败到达 HTTP、Task 或 CLI 根边界时，由运行时统一回滚事务、记录一次并终止对应执行边界。 + +需要主动检查成功和全部错误变体时，使用 `result(call)` 捕获，然后交给穷尽分句： + +```dever +decode(result: dever.bytes.DecodeResult.Decoded(text)) (message: Text) recover("invalid input is shown to the user") { + message = text +} + +decode(result: dever.bytes.DecodeResult.Failed(reason)) (message: Text) recover("invalid input is shown to the user") { + message = reason +} + +main() (message: Text) { + message = decode(result(dever.bytes.to_text(dever.bytes.from_text("hello")))) +} +``` + +`fail(error_variant)` 主动产生一个领域失败并立即结束当前调用；只能传入带 `error` 标记的变体。`result(call)` 只关闭该次调用的默认传播，捕获结果仍必须被穷尽处理、继续返回或显式恢复。Dever 不提供 exception、`throw` 或 `catch`。 + +错误必须传播、传给同样受检查的处理函数，或转换成明确错误。读取消息、打印日志、覆盖变量或交给空函数都不算处理。主动降级需要声明 `recover("具体原因")`，同一函数所有分句必须声明相同理由；例如可选本地配置缺失时采用固定默认值。理由必须非空，编译器不判断这项业务策略是否合理。详见第 18 节。 + +编程错误或不可恢复运行时故障会终止当前执行，并带源码位置；普通 function 不能捕获这类故障。 + +## 11. 常用核心操作 + +无需 package 前缀的核心名称： + +```text +Bool Int Decimal Float Text Id Bytes Secret List Map MapEntry Stream AsyncStream Channel +true false null other +each parallel_each reduce reduce_until filter find sum +append first get put remove entries length close +run wait stop group parallel blocking channel send receive timeout race stream +``` + +List 与 Map 操作： + +```dever +extended = append(values, value) +first_value = first(values) +label = get(labels, "env") +labels = put(labels, "env", "development") +labels = remove(labels, "env") +pairs = entries(labels) +size = length(values) +``` + +`append`、`put` 和 `remove` 返回新值，不修改原集合。`first(List)` 和 `get(Map, key)` 返回可空值。`entries(Map)` 返回按插入顺序排列的 MapEntry List。`length` 接受 List、Map、Text 或 Bytes；Text 按 Unicode codepoint 计数。核心 `close` 接受 Stream、AsyncStream 或 Channel，且没有输出。 + +## 12. 标准 package + +标准 package 本身也使用 Dever 编写。下表链接到当前公开源码；其中的 `public` 是编译器随附标准库的内部边界，不是应用源码语法范本： + +| package | 已提供的主要接口 | +| --- | --- | +| [`text`](library/text.dever) | `trim`、`lower`、`upper`、`at`、`slice`、`index_of`、`contains`、`starts_with`、`ends_with`、`split`、`replace`、`join`、`is_empty`、Unicode codepoint 转换 | +| [`int`](library/int.dever) | `parse`、`to_text` | +| [`decimal`](library/decimal.dever) | `parse`、`to_text`、`from_int`、`round` | +| [`float`](library/float.dever) | `parse`、`to_text`、`from_int` | +| [`math`](library/math.dever) | `sqrt`、`sin`、`cos`、`log`、`pow`、Float 分类 | +| [`dever.id`](library/dever/id.dever) | `from_text`、`to_text` | +| [`dever.bytes`](library/dever/bytes.dever) | Text/Bytes 转换、`from_ints`、`length`、`at`、`slice`、`concat` | +| [`dever.process`](library/dever/process.dever) | `arguments` | +| [`dever.crypto`](library/dever/crypto.dever) | `token`、`password_hash`、`password_verify`、`sha256`、`hmac_sha256`、`constant_time_eq` | +| [`dever.time`](library/dever/time.dever) | `now`、`parse_datetime`/`format_datetime`、`parse_date`/`format_date`、`parse_time`/`format_time`、`duration`、`add`、`subtract`、`difference`、底层时钟与 `sleep` | +| [`dever.task`](library/dever/task.dever) | 非阻塞 `sleep` 和周期 `ticks` | +| [`dever.io`](library/dever/io.dever) | 标准输出、文件创建/打开/读写/关闭、分块 Stream | +| [`dever.net`](library/dever/net.dever) | TCP 连接、监听、读写、超时、连接与数据 Stream | +| [`dever.json`](library/dever/json.dever) | JSON `parse`、`stringify`、节点/属性/数组元素访问 | +| [`dever.http`](library/dever/http.dever) | Hyper HTTP/1、HTTP/2 多路复用、流式客户端和静态 handler | + +数值或索引解析失败通常返回可空值；文件、网络、Bytes、JSON、HTTP 和 ORM 的可恢复错误默认沿调用链传播。普通调用返回成功值，`result(call)` 才产生显式选择型结果；具体以 function 输出类型为准。 + +### UTC 时间和类型化 wire + +`DateTime` 内部是 Unix epoch 有符号整数毫秒；`Date` 是 UTC 午夜对应的 epoch 毫秒，`Time` 是午夜起 `[0, 86400000)` 毫秒，`Duration` 是有符号毫秒。数据库表示、排序和 Model fingerprint 不随外部格式变化。 + +`dever.time.now()` 返回 UTC `DateTime`。`parse_datetime(Text)` 接受 RFC3339 及显式 offset，`format_datetime(DateTime)` 输出 UTC `YYYY-MM-DDTHH:MM:SS[.sss]Z`；零毫秒省略小数，非零固定三位。输入最多三位小数，拒绝闰秒、子毫秒精度和 UTC 转换后超出四位年份的日期。`parse_date`/`format_date` 使用 `YYYY-MM-DD`；`parse_time`/`format_time` 使用 `HH:MM:SS[.sss]`。日历无效或格式化值不在可表示范围内会失败。 + +`duration(millis: Int)` 构造 Duration；`add(DateTime, Duration)`、`subtract(DateTime, Duration)` 和 `difference(later: DateTime, earlier: DateTime)` 使用 checked i64 算术，最后一个返回 Duration。算术溢出显式失败；合法 i64 结果仍可能超出四位日历格式化范围。确定性转换与算术是 `pure`，读取当前时间有 time effect。普通调用传播失败，显式 `result(call)` 使用 `DateTimeResult`、`DateResult`、`TimeResult`、`DurationResult` 或 `TextResult`，成功分支均为 `Read(value)`，错误分支为 `Failed(message)`。 + +共享 wire schema 从已检查类型生成具体编解码器。Bool/Int/Float 使用 JSON 标量；Decimal/Id/Uuid 使用字符串，不经过 Float；ModelId 使用 i64 整数且 schema 保留 Model 的 nominal identity；DateTime/Date/Time 使用上述字符串格式，Duration 使用整数毫秒。支持 List、nullable 和没有 private 字段或字段范围约束的 record。缺失 nullable 字段得到 null，缺失必填字段、未知字段、重复键、错误标量和非有限 Float 均失败；Int 只接受 i64 范围内的整数字面量。Choice、Map、Related 和资源不属于当前 wire 合同。 + +边界 JSON 最多 16 MiB、64 层容器、65536 个值（含根和容器）。Json 保留验证后的原始 JSON 和数字拼写，不经过动态浮点转换。Secret 仅允许 API/setting 输入策略生成，普通输出和 Job payload 禁止；不同策略具有不同 schema identity。业务函数接收具体类型，边界节点不会传入业务代码。API response、Adapter setting 和 Job payload 复用此 codec；API 输入的后续扩展也必须复用这个 owner。 + +### JSON + +JSON 解析使用扁平、带类型的 `Document`，不产生动态对象。`dever.json.parse` 直接返回 `Document`；`result(dever.json.parse(...))` 使用 `ParseResult` 捕获失败。这些公开 type 位于 `dever.json.value`，契约是： + +```dever +type Value { + Null + Boolean(value: Bool) + Number(raw: Text) + String(text: Text) + Array(children: List<Int>) + Object(members: Map<Text, Int>) +} + +type Document { + root: Int + nodes: Map<Int, Value> +} + +type ParseResult { + Parsed(document: Document) + error Failed(position: Int, message: Text) +} + +type WriteResult { + Written(text: Text) + error Failed(node: Int, message: Text) +} +``` + +调用签名： + +```text +dever.json.parse(Text) -> dever.json.value.Document +dever.json.stringify(dever.json.value.Document) -> Text +dever.json.node(dever.json.value.Document, node_id: Int) -> dever.json.value.Value? +dever.json.property(dever.json.value.Document, object_id: Int, name: Text) -> Int? +dever.json.element(dever.json.value.Document, array_id: Int, index: Int) -> Int? +``` + +`Document` 用整数 ID 保存一棵 JSON 树：`root` 是根节点 ID；`nodes` 的 ID 从 0 连续递增；子节点 ID 小于父节点 ID。Object 将属性名映射到子节点 ID，Array 保存有序子节点 ID。`property` 和 `element` 返回节点 ID，再通过 `node` 取得 Value；缺失、类型不符或越界时返回 `null`。JSON 的 `null` 则是实际存在的 `Value.Null`，两者不同。 + +数字保留原始 JSON 文本，避免先丢失精度。解析失败位置是从 0 开始的 Unicode codepoint 偏移，包括输入末尾位置。重复键、非法转义、未配对代理项、多余输入和超过 256 层的嵌套会传播 `ParseResult.Failed`。手工构造 Document 时，`stringify` 会检查根节点、连续 ID、子节点引用和输出规模，失败时传播 `WriteResult.Failed`；两者都可用 `result` 显式捕获。完整的解析和写回分句见 [标准库示例](examples/old/dever/library/module/main.dever)。 + +### HTTP + +`dever.http` 使用 Hyper HTTP/1 和 HTTP/2 引擎，业务 handler 由 Dever 编译为静态 Rust async 调用。HTTP/1.1 支持持久连接、流水线、Content-Length 和 chunked；HTTP/2 在同一连接上并发处理多个请求流。 + +```text +dever.http.default_limits() -> Limits +dever.http.default_http2_limits() -> Http2Limits +dever.http.header(name: Text, value: Text) -> Header +dever.http.header_value(List<Header>, name: Text) -> Bytes? +dever.http.serve(route, dever.system.Listener, Limits) -> () +dever.http.send(address: Text, port: Int, Request, Limits) -> Response +``` + +`Request` 包含 `method: Text`、`target: Text`、`headers: List<Header>`、`body: Bytes`；`Response` 包含 `status: Int`、`headers`、`body`。`Header` 包含 `name: Text` 和 `value: Bytes`，因此重复头和非 UTF-8 头值可以保留。接收到的名称为小写，列表保留同名头的值顺序，不承诺不同名称之间的原始排列。`header` 构造文本头；`header_value` 忽略名称大小写并返回第一项，找不到返回 null。 + +| Limits 字段 | 默认值 | 约束 | +| --- | --- | --- | +| header_bytes | 8192 | 至少 8192；协议解析缓冲区和头部预算 | +| body_bytes | 1048576 | 非负；单条请求/响应正文的上限 | +| timeout_ms | 5000 | 正数；分阶段服务端期限，客户端单次请求总期限 | +| connections | 32 | 1–65536；服务端并发连接数或每个客户端池的连接上限，另受任务资源约束 | +| http2 | null | null 使用 HTTP/1；Http2Limits 使用 HTTP/2 | + +开启 HTTP/2 沿用原来的服务和客户端函数： + +```dever +http_limits() (limits: dever.http.Limits) { + limits = dever.http.default_limits() + limits.connections = 2 + limits.http2 = dever.http.default_http2_limits() +} +``` + +| Http2Limits 字段 | 默认值 | 约束 | +| --- | --- | --- | +| streams | 16 | 每连接并发流上限；connections × streams 不超过 65536，另受运行时任务容量约束 | +| stream_window_bytes | 65535 | 单流接收窗口，1–2147483647 | +| connection_window_bytes | 262144 | 连接总接收窗口，65535–2147483647 | + +HTTPS 的双方配置必须协商 ALPN `h2`；明文双方使用 HTTP/2 prior knowledge，不发送 h2c Upgrade，不自动降级协议。窗口固定，不自动扩大；帧和引擎发送缓冲设置为 16 KiB。窗口限制协议接收流量，不是整个进程内存上限：已收集的请求正文、每流待发块、TLS/系统缓冲和应用持有的值还需要内存。小机器应一起设置 connections、streams、body_bytes 和 chunk_bytes。 + +HTTP/2 的 `Request.target` 仍为 `/path?query` 或 OPTIONS `*`。引擎负责伪头；入站 authority 在没有 Host 时规范化为 Host，已有 Host 必须与 authority 一致。应用不能传入 Connection、Keep-Alive、Proxy-Connection 等逐连接头字段；请求 TE 只允许 trailers，响应不允许 TE。单流中止不关闭相邻流；连接级故障影响该连接的全部流。 + +`serve` 和 `send` 的正文在限额内完整缓冲；流式服务使用下节的 `serve_live`。单帧正文共享字节存储，多帧逐步合并，不为每个微小帧积累元数据队列。HTTP 引擎负责报文定界；应用不得传入 Content-Length、Transfer-Encoding 或 Upgrade。响应状态为 200–599，204/205/304 的正文必须为空；HEAD 由引擎抑制响应正文。HTTP/1 接收头数量沿用 Hyper 默认上限 100，HTTP/2 使用解压后的头列表字节预算；发送头预算为引擎生成的头预留 80 字节。 + +route 的签名为 `handler(request: dever.http.Request) (response: dever.http.Response)`；是否挂起由 route 的调用图推断： + +```dever +route(request: dever.http.Request) (response: dever.http.Response) { + response = dever.http.Response { + status = 200 + headers = [dever.http.header("Content-Type", "text/plain; charset=utf-8")] + body = dever.bytes.from_text(request.target) + } +} + +start(listener: dever.system.Listener) () { + dever.http.serve(route, listener, dever.http.default_limits()) +} +``` + +先用 `dever.net.listen(...)` 创建 Listener，再调用 serve；编译器会自动等待网络结果。完整启动分句见 [HTTP 示例](examples/old/dever/http/module/main.dever)。serve 持续接收连接，可作为 `server = run(dever.http.serve(...))` 启动，再用 `stop(server)` 取消并等待子任务清理。调用者持有的 Listener 别名仍由调用者关闭。 + +- 服务端先预留任务容量，再接受连接；没有无界应用队列或每连接独占线程。每条 HTTP/1 连接按顺序执行 handler。 +- 握手超时或底层 I/O 错误关闭该连接；正文超时返回 408，handler 超时返回 504，正文超限返回 413。HTTP/1 随后关闭连接，HTTP/2 结束对应流。写入停滞超过期限关闭连接。 +- 期限和取消在异步挂起处生效；已开始的 blocking 操作仍遵守既有“等待完成”的契约，需要操作自身的超时。 +- 对端协议错误影响对应流或连接。无效服务配置、accept 故障、handler 程序故障及无效响应沿 serve 的任务作用域传播，并保留 Dever 故障位置。 +- send 为单次请求，驱动 future 随请求一起完成或取消；需要复用连接时使用下文的 client/request。成功返回 Response，网络/协议/输入失败默认传播；通过 `result` 捕获时为 `ResponseResult.Read(response)` 或 `Failed(message)`，不自动重试。 +- send 的 address 是数值 IPv4/IPv6，port 为 0–65535；请求必须有一个 Host 头。target 使用 `/path?query`，或 OPTIONS 的 `*`，不接受代理 absolute-form 或 CONNECT。 +- 普通 `serve/send` 不接受升级；WebSocket 使用下节的 HTTP/1 专用入口。HTTPS 使用下文的 TLS 服务和客户端；当前不支持 CONNECT、非空 trailer、服务器推送或 WebSocket-over-HTTP/2。 + +### 流式 HTTP 与 SSE + +`serve_live` 的 handler 是 `handler(request: Request, reply: dever.system.HttpReply) ()`。它可以发送普通响应、流式正文和 SSE；HTTP/1 还可升级为 WebSocket。handler 内部是否挂起由编译器推断。HTTP/1 每连接最多一个活跃 handler；HTTP/2 每请求流独立拥有 handler 和清理范围，并受 streams 限额约束。 + +```text +dever.http.default_live_limits() -> LiveLimits +dever.http.serve_live(route, Listener, Limits, LiveLimits) -> () +dever.http.respond(HttpReply, Response) -> () +dever.http.start(HttpReply, status: Int, List<Header>) -> () +dever.http.write(HttpReply, Bytes) -> () +dever.http.finish(HttpReply) -> () +dever.sse.start(HttpReply, List<Header>) -> () +dever.sse.send(HttpReply, Event) -> () +``` + +`LiveLimits` 默认 `chunk_bytes = 65536`、`idle_ms = 60000`、`heartbeat_ms = 15000`；块大小必须为正，`0 < heartbeat_ms < idle_ms`。`Limits.body_bytes` 仍约束请求正文和 `respond` 的完整正文，流式输出按块限制，不累计整条长连接的字节数。响应通道仅容纳 1 块；还需计入当前写入、协议引擎、操作系统及应用自身的缓冲，不代表每条连接总共只用一块内存。 + +响应只能开始一次。`respond` 提交完整响应；`start` 提交流式响应头，随后 `write` 按调用顺序发送，`finish` 结束正文。204/205/304 使用 `respond`。重复开始、超大块、关闭后写入、向 SSE 通道直接写普通正文均传播 `Failed`，可用 `result` 捕获。写入等待受 `Limits.timeout_ms` 限制；对端停止消费时不会无限排队。 + +handler 返回会结束正文；正文被消费完或丢弃、HEAD 抑制正文、连接断开或 `stop(server)` 会取消相应 handler 并等待其子任务清理。已经交给通道的块不因取消而自动重发。`HttpReply` 是线程安全资源别名，不能在所属会话结束后继续使用。handler 没有提交响应就返回属于程序故障；提交响应头之后的程序故障会截断正文并沿服务任务传播。 + +首个响应头沿用普通请求期限；提交流式头之后不再使用整段 handler 的 5 秒期限。普通流在 `idle_ms` 内没有输出会关闭；SSE 在空闲 `heartbeat_ms` 后输出注释心跳。心跳不占用额外任务；客户端不读取时仍受 HTTP 停滞写入期限约束。 + +`dever.sse.Event` 字段为 `event: Text`、`data: Text`、`id: Text?`、`retry_ms: Int?`。空 event 使用浏览器默认事件名；data 按 CR/LF/CRLF 分行且保留尾部换行；空 id 会重置浏览器的事件编号。event 禁止换行，id 禁止换行和 NUL，retry_ms 非负。编码后的完整事件必须放入一块，心跳要求块上限至少 3 字节。`sse.start` 持有 Content-Type，并在应用未指定时添加 Cache-Control: no-cache。应用从请求 `Last-Event-ID` 头读取重连位置，负责历史保存和重放；库不缓存历史事件。 + +### WebSocket + +`dever.websocket` 复用 tokio-tungstenite 的 RFC 6455 握手、掩码、分片重组、UTF-8 校验和关闭协议，不使用自写帧解析器。 + +```text +dever.websocket.default_limits() -> Limits +dever.websocket.accept(HttpReply, Limits) -> dever.system.WebSocket +dever.websocket.connect(address: Text, port: Int, target: Text, Limits) -> dever.system.WebSocket +dever.websocket.open(url: Text, ClientTls, Limits) -> dever.system.WebSocket +dever.websocket.send(WebSocket, Message) -> () +dever.websocket.receive(WebSocket) -> ReceiveState +dever.websocket.messages(WebSocket) -> AsyncStream<MessageEvent> +dever.websocket.close(WebSocket, code: Int, reason: Text) -> () +``` + +连接成功直接得到 WebSocket；用 `result` 捕获时使用 `ConnectResult`，分支为 `Connected(socket)` 和 `Failed(message)`。Message 包含 `Text(value: Text)`、`Binary(value: Bytes)`、`Ping(value: Bytes)` 和 `Pong(value: Bytes)`。receive 直接返回 `ReceiveState.Read(message)` 或正常关闭的 `End`，错误默认传播，捕获时使用 `ReceiveResult`。messages 在正常关闭时结束，故障产生最后一个 `MessageEvent.Failed`。持续接收可使用 `each` 或 `reduce_until`，无需循环或递归语法。 + +WebSocket Limits 默认 `message_bytes = 1048576`、`idle_ms = 60000`、`write_ms = 5000`，均为正数。message_bytes 限制完整数据消息及分片累计；控制帧最多 125 字节。每连接读取缓冲区为 4 KiB，协议写缓冲有上限。方向锁允许同时读写；idle_ms 是一次 receive（含等待方向锁）的期限，write_ms 约束一次写入、客户端握手或关闭握手。超时读取会关闭资源；已开始写帧时取消会关闭连接，尚在等待写锁的取消不影响正在写入的另一个任务。 + +客户端 connect 的 address 是数字 IP，target 为 `/path?query`。open 接受完整的 `ws://` 或 `wss://` URL，支持 DNS，WSS 使用传入的 ClientTls 校验证书和目标名称；不接受 URL 用户信息或 fragment。握手期限包含域名解析、TCP 和 TLS，不自动重试。服务端 accept 从当前 HTTP 请求升级，失败时应用仍可返回普通错误响应。升级后连接仍占服务端名额，并由原 handler 作用域拥有；handler 结束或父任务停止都会关闭它，即使其他地方还持有别名。 + +应用需持续调用 receive/messages 才能读取协议帧；收到 Ping 时引擎先发送 Pong，再交给应用。库不建立额外的定时 Ping 任务，可通过 send(Ping(...)) 实现应用保活。close 校验合法状态码和最多 123 UTF-8 字节的原因，等待对端关闭握手；关闭会唤醒等待中的读写。当前不提供关闭帧元数据、子协议协商或压缩扩展。 + +完整示例见 [live_http](examples/old/dever/live_http/module/main.dever)：`/stream` 分段输出，`/events` 推送三条 SSE 事件，`/ws` 回显文本和二进制消息。 + +### HTTP 连接池与流式客户端 + +```text +dever.http.default_pool_limits() -> PoolLimits +dever.http.client(origin: Text, ClientTls, Limits, PoolLimits) -> dever.system.HttpClient +dever.http.request(HttpClient, Request) -> Response +dever.http.open(HttpClient, Request) -> StreamResponse +dever.http.open_stream(HttpClient, StreamRequest) -> StreamResponse +dever.http.close_client(HttpClient) -> () +``` + +client 返回固定 origin 的共享连接池；需要捕获时使用 `ClientResult`，成功项为 `Ready(client)`。origin 如 `https://example.com:443`，不能含用户信息、业务路径、query 或 fragment；支持 DNS。请求 target 仍用 `/path?query`，没有 Host 时由 origin 补上。每个目标建立自己的池，不保留不断增长的全局主机表；不自动重放已发送请求或跟随重定向。HTTP/2 连接停止接收新流时，协议引擎确认尚未发送的请求会在原期限内重新排队。 + +| PoolLimits 字段 | 默认值 | 约束 | +| --- | --- | --- | +| idle_ms | 10000 | 正数；归还的连接闲置多久后关闭，即使没有下一个请求也会回收 | +| chunk_bytes | 65536 | 正数；流式上传块上限及每次下载返回块上限 | +| read_ms | 60000 | 正数；一次上传取块或下载拉取的等待期限 | + +`Limits.connections` 限制物理连接数，空闲和正在排空的连接也计入上限。HTTP/1 的请求占用上限等于 connections；HTTP/2 的请求上限为 connections × streams，每连接另限 streams。每池一个作用域拥有驱动和有界队列；HTTP/2 的协议子任务也归连接作用域。池属于创建它的任务作用域，作用域结束后，即使外部还持有别名，也不能继续使用。直接调用 `create_client(...)` 会在当前任务内完成；不要通过一个立即结束的 `run` 子任务创建长期共享的池。 + +request 完整收集响应，正文受 `body_bytes` 限制，排队、建连、发送、接收合计受 `timeout_ms` 限制。open 发送普通 Request 并返回响应头与流；open_stream 的 StreamRequest 把 body 改为 `AsyncStream<dever.io.ReadEvent>`，其余字段与 Request 相同。成功均为 `Read(response: StreamResponse)`，其中 body 也是该异步流；`Chunk(bytes)` 代表数据，终止错误最多产生一次 `Failed(message)`，正常 EOF 直接结束。 + +流式客户端只按需读取，不收集整段上传或下载正文；总长度不限于 `body_bytes`，单块及等待时间受 PoolLimits 限制。等待请求槽位、DNS/TCP/TLS 和响应头受 `timeout_ms` 限制，响应流随后按 `read_ms` 读取，适用于 SSE 等长连接。HTTP 出站长度和分块编码仍由引擎拥有，转发入站响应时须移除 Content-Length、Transfer-Encoding 等受保留的头。 + +HTTP/1 响应体完全读完才归还连接，提前 `close(response.body)`、故障或取消会关闭该连接。HTTP/2 的连接可同时服务多个请求；完成或关闭正文释放该流，不中止邻流。仅持有未消费的正文会继续占用请求名额，活跃连接不会被空闲回收。close_client 关闭所有池别名、唤醒等待者并等待驱动清理；下载块共享字节存储，应用保留的块仍计入应用自身内存。 + +### TLS、共享上下文与优雅停机 + +```text +dever.tls.system() -> dever.system.ClientTls +dever.tls.client(ca_pem: Bytes) -> dever.system.ClientTls +dever.tls.server(cert_pem: Bytes, key_pem: Bytes) -> dever.system.ServerTls +dever.http.serve_tls(route, Listener, Limits, ServerTls) -> () +dever.http.serve_live_tls(route, Listener, Limits, LiveLimits, ServerTls) -> () +``` + +system 使用随二进制提供的公共 CA；client 仅信任指定 PEM CA，返回 Ready(config) 或 Failed(message)。server 读取应用提供的证书链和私钥 PEM，返回同样形状的结果。TLS 复用 rustls/tokio-rustls，校验证书链、有效期和目标名称；客户端和服务端配置可跨任务共享，并使用有界会话缓存。TLS 服务沿用 HTTP 的握手期限和连接上限；serve_live_tls 提供 HTTPS 流式响应和 SSE，HTTP/1 模式还支持 WSS 升级。HTTP 单独派生 ALPN 配置，不改变其他 TLS 使用者。 + +需要共享应用状态时，声明具体 Context，并传入系统 serve 原语的可选末尾参数。编译器要求 handler 的最后输入与 Context 类型一致且可跨任务传递；Context 按普通值语义复制，Channel、HttpClient 等资源字段共享同一资源。 + +```dever +type Context { + prefix: Text + requests: Channel<Int> +} + +serve(listener: dever.system.Listener, context: Context) () { + dever.system.http_serve(route, listener, dever.http.default_limits(), context) +} + +route(request: dever.http.Request, context: Context) (response: dever.http.Response) { + send(context.requests, 1) + response = dever.http.Response { + status = 200 + headers = [] + body = dever.bytes.from_text(context.prefix + request.target) + } +} +``` + +http_serve_live、http_serve_tls 和 http_serve_live_tls 也接受这个末尾参数；live handler 的输入顺序为 request、reply、context，TLS 配置位于 context 之前。这里直接绑定静态 handler，不引入闭包或动态路由解释器。 + +优雅停机先调用 `dever.net.close_listener(listener)` 并处理结果，再等待 server Task。HTTP 停止接收新连接，HTTP/2 同时发送 GOAWAY，已开始的响应继续完成；WebSocket/SSE 会话仍由现有 handler 拥有。客户端收到 GOAWAY 后不再向该连接派发新请求，也不自动重放已发送请求。使用 `timeout(server, grace_ms, expired)` 给排空设置上限，expired 是零输入、零输出的静态 handler；到期先停止剩余任务并等待清理。需要立即停止时使用 `stop(server)`。完整可运行用法见 [backend 示例](examples/old/dever/backend/module/main.dever)。 + +### 模块 API + +普通应用由编译器从 API、CMD 和 Job 声明生成入口。没有命令参数时，`run` 和编译产物按 `config/setting.json` 的 runtime mode 启动可用服务；CMD 必须显式传名称和一个 JSON 对象，不能借 CMD 参数改变部署模式。`run/build` 拒绝手写 `module/main.dever`。 + +`module/user/account/app.dever`： + +```dever +type Greeting { message: Text } + +greet(name: Text) (greeting: Greeting) { + greeting = Greeting { message = "你好，" + name } +} +``` + +`module/user/account/api.dever`： + +```dever +get greeting = app.greet +public post greeting = app.greet +cmd greeting = app.greet +``` + +前两项对应 `GET`、`POST /user/account/greeting`；CMD 名称为 `user.account.greeting`，输入是单个 JSON 对象，输出同样使用 `{"code":0,"message":"ok","data":...}`。HTTP 声明默认要求认证，只有明确的登录、健康检查等单条 HTTP 声明可加 `public`；`public cmd` 和 `public rest` 非法。public 入口忽略浏览器自动携带的旧 Cookie，使过期会话不会阻止重新登录；客户端显式提交的无效 Authorization 仍返回 401，不能降级成匿名请求。`get/post/put/delete <action> = app.<function>` 只绑定本领域无歧义的 App 函数，继承其输入输出签名，输出名不必叫 `response`。GET 的完整调用链不能写数据库或入队 Job；普通写 HTTP/CMD/Job 调用链在入口建立事务。含 password hash/verify 的 HTTP 调用链不建立覆盖密码计算的外层事务，其写入必须全部进入显式短 `transaction`，且事务内禁止密码计算；短事务必须重查写入依赖的账户、租户或成员状态。API role 不写函数体或旧 `get_` 前缀。GET/DELETE 从 query 读取标量输入，POST/PUT 从 `Content-Type: application/json` 的对象正文读取；命令也读取严格 JSON 对象。缺少必填值、额外或重复字段均拒绝。输入输出使用同一静态 wire codec，当前不支持 Choice、Bytes 等类型。 + +简单领域使用 `api.dever`；复杂领域可同时使用 `api.dever` 与 `api/<scope>/<resource>.dever`，topic 依次进入 URL，例如 `module/user/account/api/admin/session.dever` 中的 `get status = app.status` 对应 `/user/account/admin/session/status`。`sites.<key>.path` 匹配这里的 API 父目录：`admin` 匹配 `api/admin/**`，不匹配同级文件 `api/admin.dever`，也不改写 URL。每条 HTTP API 必须且只能归属一个站点；目录遗漏、path 前缀重叠和同一路径同方法重复都会在运行前报错。`rest` 可为同领域唯一 Model 生成 GET 列表/详情、POST、PUT、DELETE；多 Model 时写 `rest model` 或 `rest model.<topic>` 消歧。列表只接受有界 `page`、`size` 和至多一个显式 `search` 条件，不开放任意查询。自动 REST 只序列化公开字段；`private owner user_id: ... = dever.auth.user_id()` 在创建时写入可信身份，并把同一 owner 条件放入全部读写 SQL。客户端不能提交 owner 或无 `input` 的服务端字段。需要不同资源策略或跨 Model 编排时仍使用 App，不把业务规则塞进 REST 声明。 + +HTTP 绑定的 App 调用链可用 `dever.api.request_id()`、`method()`、`path()`、`client_address()`、`header(name)`、`cookie(name)`、`secret_cookie(name)` 读取请求元数据，并用 `set_header(name, value)`、`set_cookie(name, value, options)`、`set_secret_cookie(name, secret, options)` 设置响应元数据；无需传递 Context 参数。`client_address()` 只返回直接 TCP peer 的地址，不信任转发 Header。`secure_cookie_options()` 提供 `Path=/`、`SameSite=Lax`、`Secure`、`HttpOnly` 的起点，业务可构造 `CookieOptions` 调整 Domain 和 Max-Age。Secret Cookie 使用 `v1.` 加无填充 base64url 存储原始 Secret 字节，读写不会把 Secret 转成业务 Text；非此格式的值读作缺席。敏感请求头不能经 `header` 读为 Text，Secret Cookie 必须使用 Secure 与 HttpOnly；响应 Header 不允许覆盖 Set-Cookie、Content-Type、Content-Length 等运行时字段或注入换行。Header/Cookie 先暂存在请求作用域，只有 handler 成功、响应序列化成功且数据库提交成功后才发布；失败和回滚会丢弃它们。Cookie 写请求和使用 Cookie 认证的非安全方法必须携带 Origin，并要求 Origin 与 Host 都精确匹配 `sites.<key>.origin` 配置的外部 HTTP/HTTPS scheme、规范化主机和有效端口；运行时不从内部明文监听协议猜外部 HTTPS，也不信任转发 Header。CMD/Job 调用链不能使用 HTTP 请求能力，脱离请求的并发任务也不能调用这些能力。 + +POST App 可以直接接收一个顶层 `Upload` 及普通标量，此时请求必须是 `multipart/form-data`。Runtime 按 `http.upload` 的总大小、单文件、字段、part、header 和 filename 上限流式写入 `data/tmp`；filename 仅作展示值，不能成为路径。`Upload` 是 affine 请求资源，不能嵌套、复制、作为输出或进入 GET/CMD/helper；App 可读取 metadata 后显式 `close_upload`，或一次性传给同领域 Port，再由 Adapter 调用 `dever.storage.put`。未消费资源及解析失败、handler 失败和取消路径会清理临时文件。 + +监听、站点和认证 Provider 都只在 `config/setting.json` 配置： + +```json +{ + "http": { "host": "127.0.0.1", "port": 8181 }, + "auth": { + "providers": { + "session": { + "verify": "user.account.verify", + "jwtSecret": "replace-with-a-deployment-secret-of-at-least-32-bytes", + "ttlSeconds": 3600, + "cookie": "cms_session" + } + } + }, + "sites": { + "admin": { + "path": "admin", + "auth": "session", + "hosts": ["admin.example.com"], + "origin": "https://admin.example.com" + }, + "front": { + "path": "front", + "auth": "session", + "hosts": ["www.example.com"], + "origin": "https://www.example.com" + } + }, + "log": { "level": "info" } +} +``` + +Provider 使用固定 HS256 token，校验算法、签名、有效期、Provider、站点、subject 和 session；密钥只放部署配置，不进入源码。Bearer 和配置指定的 Secure Cookie 二选一，不能同时提交；使用 Cookie 认证的非安全方法以及任何 Cookie 写入都必须通过上述精确同源检查。需要浏览器 Cookie 的站点必须配置 `origin`；它是部署入口的外部 origin，不是内部监听地址。Provider 的 `verify` 必须引用一个公开普通 App 函数：输入为 `dever.auth.Claims`，输出为该 App 自己定义的公开身份 record，字段严格且只能是 `id: Text`、`user_id: <ModelId>?`、`tenant_id: <ModelId>?`。verify 只负责验证账号、session、成员和租户关系；它只能读 global Model，不能返回权限列表、读取原始 HTTP 请求、再次读取身份、写数据库或入队 Job。 + +身份建立后，App 可读取 `dever.auth.id()`、`session()`、`user_id()`、`tenant_id()` 与 `dever.site.key()`；`dever.auth.owns_user(model_id)` 用于显式资源所有权比较。所有非 `public` HTTP API 都在读取业务输入和调用 App 前自动校验精确权限 `component.domain.site.action`；REST 固定生成 `read/create/replace/delete`，HTTP method 只是目录元数据，同一权限键不能重复声明。源码没有 `auth.require`，也不能自报权限。登录使用 `dever.auth.issue_cookie(subject, session, tenant)`，登出使用 `dever.auth.clear_cookie()`；两者自动采用当前 Provider 在 `setting.json` 中配置的 Cookie 名和 TTL。 + +`user_id()`、`tenant_id()` 默认返回 `Int?`；当赋值或函数参数明确期待某个 `model.id?` 时，返回对应的可空名义 ID，不需要再次查询会话表。编译器逐条检查受保护 API 的调用链和自动 REST 字段绑定：该 ID 必须与当前站点 `verify` 身份 record 的对应字段类型一致，`owns_user` 的参数也必须匹配其 `user_id`。这不是通用的 Int 到 ModelId 转换，非空 ID 仍须通过 `null` 分支处理取得。只有源码、没有部署配置的检查延后 Provider 匹配；配置检查和构建必须完成匹配。public、CMD、Job 和普通应用测试不会因此获得身份读取能力。 + +核心在控制库维护全局 `_dever_permission` 目录，服务启动时同步当前编译产物并把已删除权限标为 inactive。角色、角色权限和用户角色存放在当前身份对应的授权库：租户用户使用租户库，平台用户使用控制库。App 可通过 `dever.auth.permissions()`、`save_role(...)`、`grant_role(...)`、`revoke_role(...)`、`disable_role(...)` 实现很薄的后台管理适配；当前站点和租户由隐藏上下文确定。这些管理能力只能由受保护的写 API 到达，`public` 和 GET 调用链会被静态拒绝。新租户迁移完成后使用 `dever tenant owner <root> <tenant-id> <site> <user-id>` 建立保留的全权限 Owner，普通 save/grant/revoke/disable 均不能修改该保留角色。底层 `issue` 与通用 Cookie API 保留给非浏览器协议组合。身份、租户和权限都不接受客户端参数或 Header 自报，Domain 保持纯业务规则。`dever.api.Error` 的 `Invalid`、`Unauthorized`、`Forbidden`、`NotFound`、`Conflict`、`TooManyRequests` 使用固定安全消息映射到 400/401/403/404/409/429；其它失败只记录脱敏日志并返回通用 500。 + +只有 CMD、没有 HTTP 声明的应用无需 `http`、`sites` 或 `auth` 配置。正常响应使用 HTTP 200，正文为 `{"code":0,"message":"ok","data":...}`；错误时 `code` 与 HTTP 状态一致且 `data` 为 `null`。多租户只有 database 隔离，不存在 mode 开关：`tenant.database` 选择控制连接，`global type` 使用平台库，其他持久化 Model 和 Job 使用可信身份或显式 `dever tenant migrate` 选择的租户物理库。缺少租户、租户未迁移、schema fingerprint 过期都会失败，绝不回退平台库。租户组件从 tenant Model 和 tenant Job 自动推导；混合组件整体按租户组件处理，它拥有的全部 Job 使用租户队列。public API 不能触达租户组件，API/CMD/Job 执行前统一检查组件状态。运维通过 `dever tenant component disable|enable <root> <tenant-id> <component>` 控制租户功能，平台组件或未知组件不能切换。 + +## 13. Model、ORM 与数据库 + +### 13.1 Model 文件与字段 + +Model 同时支持 `.dever` 和 `.dever.md`，复合扩展名不影响类型名或表结构。Markdown 中的 Seed、索引、关系、迁移和 SQL 声明按 [Markdown 规范](MARKDOWN-SYNTAX.md)分别说明；CMS 的两套实现位于 `examples/cms/dever` 与 `examples/cms/md`。 + +简单领域在 `<component>/<domain>/model.dever` 中定义一个与 domain 同名的 Model；复杂领域可使用 `model/<topic>.dever`，记录名与 topic 对应。文件继续使用普通 record `type`，不增加 `model` 或 `public` 关键字；主记录、choice 和生成的 ID 类型由 Model role 自动提供。以下 `user/account/model.dever` 对应 `user.account.model.Account`： + +```dever +type AccountStatus { + Active = "启用" + Disabled = "禁用" +} + +type Account { + uuid: Uuid generated unique + email: Text(254) unique + display_name: Text(1, 64) + status: AccountStatus default AccountStatus.Active index + organization_id: organization.team.model.id? +} + +index(status, created_at) +``` + +每个 Model 自动增加只读 `id` 和 `created_at: DateTime`。`id` 的类型是该 Model 专属的 `<model-role>.id`，例如 `user.account.model.id`；不同 Model 的 ID 不能混用。`created_at` 和 generated 字段不出现在创建输入中。 + +Model 字段支持 `Bool`、`Int`、`Float`、`Decimal(P, S)`、`Text`、`Bytes`、`Uuid`、`DateTime`、`Date`、`Time`、`Duration`、`Json`、无载荷 choice、Model ID，以及它们的可空形式。`Text(max)`/`Bytes(max)` 指定最大长度，`Text(min, max)`/`Bytes(min, max)` 同时指定最小和最大长度。结构化值不会自动序列化为列，需要时显式使用 `Json`。 + +默认值和 Seed 共用编译期存储校验：Int 必须是 i64 范围内的整数，Float 必须有限，Decimal 精度和小数位不能溢出，Text 长度按 Unicode 标量计数。非法常量不会等到生成 Rust 或访问数据库后才报错。 + +- `generated` 当前用于由 ORM 生成 UUIDv7 的非空 `Uuid`。 +- `default` 接受与字段类型匹配的静态常量或无载荷 choice 变体。 +- `index` 和 `unique` 定义单字段索引；顶层 `index(a, b)`、`unique(a, b)` 定义复合索引。 +- `target.model.name.id` 字段自动成为外键并自动建立普通索引，但不会自动加载目标记录。 +- choice 的右侧 Text 是显示名称；数据库保存左侧稳定变体名，`Choice.options` 按声明顺序提供选项 Map。 + +自动 REST 的字段来源直接写在 Model 字段同行，不改变 App 中的普通 ORM：`create = expr` 只处理 POST，`replace = expr` 只处理 PUT，`search = expr` 处理可选 GET 列表筛选。表达式引用 `input` 时读取客户端同名值并做纯转换；不引用 `input` 时由服务端上下文提供，客户端同名字段会被拒绝。表达式只能调用同领域 `pure Domain`、无 effect 标准函数或精确白名单的只读 request/auth/site 能力，不能访问 Model、Job、时间、响应修改或并发。`search` 最多一个，必须是公开、非空、可索引 wire 标量；owner 与 search 同时存在时自动生成复合索引。 + +### 13.2 CRUD、条件和分页 + +编译器为 Model 生成唯一一套短函数，不需要手写 Repository 或第二套 CRUD： + +```dever +created = model.create({ + email = "reader@example.com" + display_name = "Reader" +}) + +page = model.list({ + where = status == model.AccountStatus.Active + and (display_name == "Reader" or display_name == "Admin") + order = [created_at.desc, id.desc] + page = 1 + size = 20 +}) + +changed = model.update(created.id, { + status = model.AccountStatus.Disabled +}) +``` + +可用操作为 `create`、`create_many`、`get`、`first`、`list`、`cursor`、`count`、`exists`、`stream`、`update`、`delete` 和 `upsert`。这些操作只能在所属领域 App 内以 `model.<operation>`（多 Model 时为 `model.<topic>.<operation>`）调用；main、API 和其他领域必须调用该领域 App。创建、更新和查询块只能直接出现在对应调用位置，由目标 Model 提供字段命名空间；它们不是运行时 Map 或可保存的查询对象。 + +`where` 支持 `and`、`or`、`not`、括号、比较、null，以及 `in`、`between`、`contains`、`starts_with`、`ends_with`。查询结构、字段和排序全部在编译期确定，运行值始终通过数据库参数绑定。 + +`list` 默认第一页、默认 20 条，返回包含 `items/page/size/total/pages` 的 Model `Page`；`cursor` 返回 `items/next/has_more` 且不执行 count；`stream` 按行拉取并保持有界内存。配置中的 `max_page_size` 同时限制分页、关系子列表和原生 SQL 多行结果，不提供无界 `all()`。 + +### 13.3 关联 + +外键字段会合成同名去掉 `_id` 的 to-one 关系。反向 to-many 在目标 Model 文件中显式声明： + +```dever +relation articles = news.article.model.author_id +``` + +查询只有写入 `with` 才加载关系： + +```dever +accounts = model.list({ + order = [id.asc] + with = [organization, articles] +}) +``` + +关系字段使用 `Related<T>` 的 `Unloaded`/`Loaded(value)` 两态，业务代码通过分句穷尽处理。to-one 使用受控 JOIN；to-many 按当前父结果批量查询并归组，不逐条查询。`with` 只接受一层静态关系名，并只用于 `first`、`list` 和 `cursor`；`stream` 不加载关系。多对多关系使用显式中间 Model。 + +### 13.4 事务和数据库绑定 + +事务是专用 function 声明，不向业务代码暴露 Database、Transaction、begin、commit 或 rollback： + +```dever +transaction register(email: Text, display_name: Text) (account: model.Account) { + account = model.create({ + email = email + display_name = display_name + }) +} +``` + +编译器沿完整调用图推导数据库 effect。最外层 transaction 成功时提交；error、fault 或取消时先回滚再传播原失败。同一连接的嵌套 transaction 复用外层事务。无数据库操作、跨连接，或在事务中通过 `run`、`parallel`、流式游标、Channel 启动可逃逸工作都会在构建前被拒绝。`blocking(call())` 可以在事务中使用，但仍须满足其目标非挂起、无数据库和无并发的既有检查，调用者会等待它完成。 + +数据库错误默认传播，保留 `dever.database.Error` 的类别和消息：Pool、PoolExhausted、Connection、Timeout、Cancelled、Database、Constraint、NotFound、InvalidData、Migration。要主动捕获一个返回 Account 的 App 调用，可以声明下面的结果类型；`result(load_account(...))` 的成功载荷和完整错误集合必须与调用一致。 + +```dever +type ReadResult { + Found(value: user.account.model.Account) + error Failed(error: dever.database.Error) +} +``` + +`Failed` 包装原始错误 choice，事务回滚和任务等待不会把它改成普通 fault。捕获后仍须按语言规则传播、返回或通过明确的 recover 策略处理；错误不会因读取或记录日志而被视为已处理。启动阶段的配置与迁移失败发生在业务入口之前，仍会阻止进程启动。 + +Model 默认按 component 名选择连接，例如 `user.account.model` 优先使用 `database.user`；该键不存在时使用 `database.default`。Model 文件需要固定连接时，在类型声明前写： + +```dever +database report +``` + +显式连接不存在会直接失败，不回退 default。普通 CRUD 不接收数据库句柄，也不能按运行时字符串切换连接。 + +配置顶层存在 `tenant` 时，普通 Model 默认为租户 Model；平台账号、租户目录等共享数据在声明前写 `global type Account { ... }`。`tenant` 只接受 `database`、`max_pools` 和 `idle_timeout_ms`；`database` 指向保存租户就绪状态、组件开关和全局权限目录的控制连接。SQLite 连接为租户 Model 配置 `tenant_directory`，PostgreSQL 配置最多 43 个 lower_snake_case 字节的 `tenant_database_prefix`。业务请求不会自动建库，必须先执行 `dever tenant migrate <project-root> <positive-id>`。PostgreSQL migration 会通过该连接的配置 URL 幂等执行 `CREATE DATABASE <prefix>_<tenant-id>`，部署角色必须具有建库权限；随后才连接租户库并迁移，失败不会写 ready marker。一个事务不能混用 global 与 tenant Model。 + +### 13.5 Schema、迁移和 Seed + +当前 Model 是目标 schema。启动时 ORM 比较数据库 catalog、内部 history 和二进制中的规范化 schema：空库直接建立最终表；安全且意图唯一的变化自动应用；收紧长度、精度或约束前先检查已有数据。失败不会截断、舍入或删除数据。 + +简单字段重命名使用 `from`： + +```dever +type User { + display_name: Text(64) from nickname +} +``` + +已有数据库把 `nickname` 改名并保留数据；全新数据库只创建 `display_name`。复杂或破坏性变化使用稳定命名的 `migrate`，例如： + +```dever +migrate remove_legacy_code { + drop legacy_code +} +``` + +命名迁移成功后只执行一次，已执行内容被修改时拒绝启动。SQLite 需要重建表的变化和 PostgreSQL DDL 都由同一逻辑 schema 计划产生。 + +需要在结构变化前清理旧数据，或在结构变化后回填数据时，在命名迁移中使用 `before`、`after`： + +```dever +migrate normalize_display_name { + before { + sqlite = "UPDATE user SET display_name = ?1 WHERE display_name = ?2" + postgres = "UPDATE user SET display_name = $1 WHERE display_name = $2" + parameters = ["Editor", "Legacy editor"] + } + after { + sqlite = "UPDATE user SET display_name = ?1 WHERE display_name = ?2" + postgres = "UPDATE user SET display_name = $1 WHERE display_name = $2" + parameters = ["Administrator", "Admin"] + } +} +``` + +每块必须声明两个方言和参数列表，只接受一条 `INSERT`、`UPDATE` 或 `DELETE`，不接受多语句、DDL、事务控制和保留的内部表名。值通过参数绑定；参数支持 Bool、Int、Text、null 和指数形式 Float，无类型上下文的小数不隐式转换成 Float。参数序号必须连续且覆盖参数列表。 + +已有表先验证 catalog 和历史，再按源码顺序执行未应用迁移的全部 `before`、自动结构变更、全部 `after`，最后执行 Seed 并记录历史。任一步失败都会回滚该 Model 的数据、结构和历史。全新数据库直接建立最终结构，不执行历史数据变换。已应用迁移的内容和顺序不能改变；变换只能引用当时已存在的表，不保证跨 Model 原子升级。 + +初始化数据直接写在 Model 文件的无版本 `seed` 块中： + +```dever +seed { + { + email = "admin@example.com" + display_name = "Administrator" + status = UserStatus.Active + } +} +``` + +每条 Seed 必须完整提供一个唯一键。ORM 只插入缺失记录，不覆盖或删除线上数据；修改既有初始化数据使用命名迁移。schema、迁移和 Seed 在同一连接的事务与启动锁内执行。 + +### 13.6 Typed native SQL + +内置查询无法清楚表达的复杂读取可以在 Model 文件中声明静态、参数化的原生 SQL： + +```dever +type UserSummary { + status: UserStatus + total: Int +} + +sql summary(status: UserStatus) (summary: UserSummary?) { + sqlite = "SELECT status, COUNT(*) FROM user WHERE status = ?1 GROUP BY status" + postgres = "SELECT status, COUNT(*) FROM user WHERE status = $1 GROUP BY status" +} +``` + +两种方言都必须是 Text 字面量。参数按声明顺序绑定，禁止字符串插值、动态列名和动态行 Map。结果可以是具体 Model、同文件中仅供 SQL 使用的私有 record，或同领域 App 定义的 View，并支持可空值和 List。App View 用 `app.ArticleView` 引用，字段必须公开且为支持的存储标量；不能包含 Secret、嵌套 Model，也不能引用其他领域的 View。SQL 结果普通 record 不接受字段范围约束，数据库解码不能证明这些约束。调用形式与内置操作一致，例如 `model.summary(status)`，并复用当前事务、连接池、statement cache 和具体行 decoder。 + +GET 允许两种方言均被证明只读的简单自有表投影：`SELECT 字段 FROM 自有表`，可带字段与编号参数的等值 `WHERE`（以 `AND` 连接）、自有字段 `ORDER BY` 和编号参数 `LIMIT`。函数、CTE、子查询、其他表或未识别的 SQL 仍按可能写入处理，不能绑定 GET；不会仅凭 `SELECT` 前缀放行。 + +### 13.7 `setting.json` + +数据库配置直接放在 `database` 对象中： + +```json +{ + "database": { + "default": { + "type": "sqlite", + "path": "data/db/app.db", + "max_connections": 1, + "max_page_size": 100 + }, + "report": { + "type": "postgres", + "url": "postgres://user:password@db.example.com/report", + "tls": "system", + "min_connections": 0, + "max_connections": 4, + "max_page_size": 100, + "wait_timeout_ms": 5000, + "io_timeout_ms": 30000 + } + } +} +``` + +`database.default` 必须存在。PostgreSQL 的 `tls` 必须明确为 `system` 或 `disabled`，证书校验失败不会自动降级明文；`wait_timeout_ms` 限制连接池等待，`io_timeout_ms` 独立限制 SQL、事务和迁移 I/O。两者省略时分别为 5000 和 30000 毫秒，可配置范围都是 1–300000。启动时创建 `data/db`、`data/upload`、`data/log`、`data/cache` 和 `data/tmp`；配置缺失或无效时直接失败，不生成默认配置。 + +## 14. 文件、TCP 与 Stream + +文件和 TCP 操作成功时返回资源或数据状态，错误默认传播。`dever.io.open/create` 返回 File；`read` 返回 `ReadState.Read(bytes)` 或 `End`；write 和资源 close 成功无输出。用 `result(call)` 捕获时，分别使用 OpenResult、ReadResult、WriteResult、CloseResult。读取可能短读；文件 `create` 只创建新文件，不覆盖已有文件。 + +```dever +type CountResult { + Counted(total: Int) + error Failed(message: Text) +} + +sum_byte(value: Int, total: Int) (next: Int) { + next = total + value +} + +consume(event: dever.io.ReadEvent.Chunk(bytes), total: CountResult) (next: CountResult) { + next = add_chunk(total, bytes) +} + +consume(event: dever.io.ReadEvent.Failed(message), total: CountResult) (next: CountResult) { + next = CountResult.Failed(message) +} + +add_chunk(total: CountResult.Counted(value), bytes: Bytes) (next: CountResult) { + next = CountResult.Counted(reduce(sum_byte, bytes, value)) +} + +add_chunk(total: CountResult.Failed(message), bytes: Bytes) (next: CountResult) { + next = CountResult.Failed(message) +} + +streaming(result: dever.io.ReadStreamResult.Streaming(stream)) (total: CountResult) { + total = reduce(consume, stream, CountResult.Counted(0)) + close(stream) +} + +streaming(result: dever.io.ReadStreamResult.Failed(message)) (total: CountResult) { + total = CountResult.Failed(message) +} + +opened(result: dever.io.OpenResult.Opened(file)) (total: CountResult) { + total = streaming(dever.io.chunks(file, 4096)) +} + +opened(result: dever.io.OpenResult.Failed(message)) (total: CountResult) { + total = CountResult.Failed(message) +} +``` + +`dever.io.chunks(file, limit)` 返回同步 Stream；`dever.net.chunks(socket, limit)` 返回 AsyncStream。显式捕获创建失败时分别使用 `dever.io.ReadStreamResult`、`dever.net.ReadStreamResult`。两种流都产生 `dever.io.ReadEvent.Chunk`；创建时 limit 非正数或源已关闭会直接失败。`dever.net.connections(listener)` 直接返回 `AsyncStream<ConnectResult>`。 + +关闭 File、Socket 或 Listener 的任一别名后，其他别名上的操作会返回失败。`close(stream)` 是幂等的零输出核心操作：它关闭该 Stream 的全部别名并释放 Stream 持有的源别名，但不会关闭调用方另外持有的 File 或 Socket。Stream 正常结束后所有别名都只会继续得到结束；终止性读取错误最多产生一个 `ReadEvent.Failed`，然后结束。资源最后一个别名离开作用域时会被释放；需要确定关闭结果时仍应调用对应 package 的 close 并处理它的选择型结果。 + +TCP 的 `connect`、`connect_timeout`、`listen`、`accept`、`read`、`write` 都会被推断为挂起；直接调用时顺序等待，需要并发时使用 `run(...)`。`port`、`timeout`、`close`、`close_listener`、`chunks`、`connections` 不挂起。所有网络等待复用程序的 Tokio runtime；Socket 支持同时读写，每个方向按操作串行，关闭会唤醒等待方。 + +`connect` 支持主机名，DNS 使用共享有界 blocking 通道，没有连接超时承诺。`connect_timeout` 要求数值 IPv4/IPv6 和正毫秒数。`timeout(socket, milliseconds)` 设置随后开始的每次读写操作的期限，包括等待同方向锁及全部写入；读超时不消费后续数据。一次写入已经写出部分字节后，超时、故障或取消会关闭 Socket，绝不自动重发整条消息。尚未写出字节的取消不关闭 Socket。文件操作仍是同步边界，在挂起调用链中使用 `blocking` 隔离。 + +关闭 AsyncStream 会结束所有流别名并释放生产者。它不会关闭调用方另外持有的 Socket 或 Listener；需要同时结束这些别名时显式调用 `dever.net.close` 或 `close_listener`。停止持有唯一资源别名的消费任务后，该资源随任务清理释放。 + +## 15. 完整可运行示例 + +[示例索引](examples/README.md) 列出可运行示例及编译拒绝反例，提供每个项目的入口、预期结果和运行边界。每个示例目录都是独立项目根，源码位于其 `module/`。 + +当前完整示例位于 [examples/old/dever/library/module/main.dever](examples/old/dever/library/module/main.dever)。它把以下能力放在同一个程序中： + +- `package` 与公开零输入入口； +- 命名输出； +- `List<Int>` 和 `reduce`； +- JSON 解析、选择型失败分句与重新编码； +- Text、Int 和 Unicode。 + +运行： + +```bash +dever check examples/old/dever/library +dever fmt examples/old/dever/library --check +dever run examples/old/dever/library +``` + +输出： + +```text +json = {"message":"你好 Dever","total":6} +``` + +其他针对性示例： + +- [examples/old/dever/core/module/main.dever](examples/old/dever/core/module/main.dever)：用户 type、选择型、可空值、数值范围分句、List、Map、值语义和数值规则。 +- [examples/old/dever/stream/module/main.dever](examples/old/dever/stream/module/main.dever)：文件分块读取、Stream 和 `reduce`。 +- [examples/old/dever/contracts/module/main.dever](examples/old/dever/contracts/module/main.dever)：受约束的端口、明确失败、主动恢复、纯函数和包依赖。 +- [examples/old/dever/http/module/main.dever](examples/old/dever/http/module/main.dever)：Hyper HTTP/1 服务及静态 route。 +- [examples/old/dever/async/module/main.dever](examples/old/dever/async/module/main.dever)：Task、Group、Channel、非阻塞定时、`parallel` 和 `blocking`。 +- [examples/old/dever/backend/module/main.dever](examples/old/dever/backend/module/main.dever)：源码运行参数、共享 Context、HTTP 连接池和优雅停机；请求一次后自动退出。 +- [examples/cms/dever/module/news/article/api/admin/manage.dever](examples/cms/dever/module/news/article/api/admin/manage.dever)：生成的 admin REST/业务入口、站点认证和租户文章事务。 + +## 16. 当前未实现 + +以下能力不能写进当前 Dever 程序： + +- `if`/`else`、`switch`、`match` 和条件表达式；条件永久使用 function 分句。 +- `for`、`while`、`break`、`continue` 和用户递归；重复使用集合/Stream 组合器。 +- class、继承、interface、method、annotation、macro、unsafe 和 FFI。 +- lambda、闭包、一般化一等 function、用户泛型 type/function。 +- exception、`throw` 和 `catch`；可恢复错误使用推导式默认传播、`result(call)` 和 `fail(...)`。 +- 通用动态 Json 值、Set 和 BigInt；持久化 JSON 使用 Model 的 `Json` 字段，普通 JSON 处理使用 `dever.json.Document`。 +- CONNECT、非空 trailer、服务器推送、WebSocket-over-HTTP/2，以及 WebSocket 子协议、压缩扩展和关闭帧元数据。 +- 正式签名 runtime/build packs、共享 daemon 编译及跨平台发行验收。现有 LLVM 内核覆盖受管值、静态 handler、Result、文件/流、真实协程与并发组合、TCP/HTTP/TLS/SSE/WebSocket 及系统能力；独立应用入口已接通 CMD→App→Dever/external Port/Adapter、SQLite/PostgreSQL Model/事务，以及 HTTP/REST、可信身份、自动权限、database-only 租户与组件状态、Cookie/Header、Upload 存储和日志配置。持久 Job 和应用 Test 复用现有语义，Test 保持一次编译及每用例进程/fake/数据库/时钟隔离。external 复用统一 Worker 协议、类型化数据与声明业务错误，以及校验后提取的内嵌资源。Worker 与其他资源归属每次入口：退出先关闭 Worker，再排空子任务并关闭数据库等资源。私有 `dever run/build/test` 默认使用 LLVM 与显式 runtime pack，不读取后端环境变量或调用宿主 Rust/Cargo/cc；managed `dever-core` 仍等待可信 daemon 编译入口，不能把本机定向验收当作正式或全平台发行。 +- table/field 多租户隔离模式，以及按租户业务配置动态选择已编译 Adapter；当前只支持 database 隔离和进程级 Adapter 选择。 +- 编译器自举。 + +编译器会拒绝尚未支持的语法，不会把它们静默转换成默认行为。 + +## 17. 开发建议 + +1. 先把领域状态建模为字段型和选择型 type。 +2. 用小型具名 function 组合业务逻辑；把每个分支写成穷尽、互斥的分句。 +3. 需要重复时选择 `each`、`filter`、`find`、`sum`、`reduce` 或 `reduce_until`；只有副作用可并行且顺序不重要时使用 `parallel_each`。 +4. 用可空值表达简单缺失；需要区分多种失败原因时定义选择型结果。 +5. 在系统边界保留 Bytes、资源和显式失败，不隐式吞掉 I/O、JSON 或 HTTP 错误。 +6. 每次修改后先执行 `fmt --check` 和 `check`，再运行或构建入口。 + +## 18. 编译器约束与公开接口 + +普通字段型 type 可以直接构造，无需为每个数据类型增加工厂。需要数值不变量时，把范围写在字段上： + +```dever +type Port { + value: Int >= 1 and <= 65535 +} + +default_port() (port: Port) pure { + port = Port { value = 8080 } +} +``` + +`Port { value = 0 }` 编译失败。未知 `Int` 也不能直接放入该字段，先用已有范围分句验证；验证成功分句中可以直接构造。范围同样适用于 choice 载荷、命名输出和 handler 签名中的输入/输出。仅支持 Int、Decimal 的数值范围，不支持任意用户谓词。赋值、字段修改和返回必须继续满足范围。编译器保留复制前的事实，修改原值不会改变副本的事实；遇到不能证明的运算不会猜测安全或自动插入默认值。 + +真正需要隐藏表示时，使用 `private` 字段： + +```dever +type Session { + private key: Text + label: Text +} +``` + +当 Session 声明在 App 中时，其他领域可以使用它，但不能读取、修改 key，也不能构造含私有字段的 Session；普通 record 的 private 仍属于声明 package。`private` 不适用于 choice 载荷。不要给普通数据全部加 private，否则会人为增加构造和读取包装函数。它是源码访问边界，不是敏感数据加密机制。 + +Model 可以声明 `private password_hash: Text(255)`。只有同 component/domain 的 App 能读写私有字段，生成 CRUD 正常持久化；增加 private 本身不改变数据库列或迁移 fingerprint。含 private 字段的完整 Model、Page、Cursor、Related 及其嵌套容器不能穿过 App 参数/输出、API JSON、错误载荷或入口渲染。App 返回显式 View；循环 Model 关联不会绕过检查。普通 record 的 private 语义保持不变。 + +`Secret` 是独立内置标量，可以传参、赋值和存入 record/容器，但不可比较、拼接、渲染、普通 JSON 输出、作为 Map key、Model 列或错误载荷。App 可接收 Secret，不能输出它。检查器拒绝嵌套泄漏；原生运行时 Debug 固定显示脱敏占位，持有的字节在释放时清零。不提供 Secret-to-Text/Bytes 通用转换。 + +`dever.crypto.token(bytes)` 从操作系统生成 16–1024 字节的 Secret；`password_hash(Secret)` 返回 Argon2id v19 PHC Text，固定内存 19456 KiB、2 轮、1 lane、32 字节结果；`password_verify(Secret, Text)` 对错误密码返回 false，对损坏/超限 PHC 返回类型化失败。验证先检查资源上限，再执行哈希。token/hash/verify 失败分别使用 TokenResult/HashResult/VerifyResult，成功分支 Ready，失败分支 Failed(message: Text)，错误不携带输入。哈希和验证是阻塞操作，挂起调用链使用已有 `blocking(...)` 边界。 + +`dever.crypto.sha256(Bytes)` 与 `hmac_sha256(Secret, Bytes)` 返回 Bytes；`constant_time_eq(Bytes, Bytes)` 对相同长度内容进行常量时间比较，长度不是秘密。仅 compiler-owned Test source 提供 `secret(Text)` 固定输入构造，生产源码没有该入口。所有部署配置仍只来自 `config/setting.json`。 + +`pure` 写在函数输出声明后，禁止文件、网络、时钟、进程参数读取、安全随机源、标准输出、Stream 操作和并发等效果。检查沿函数调用和具名 handler 传递；调用具体纯 handler 可以通过，声明为 pure 的泛化函数不能调用未限定效果的 handler。纯函数仍可能产生算术 fault，`pure` 不承诺不会失败。 + +领域之间不声明依赖清单。源码通过 App 名称引用跨领域类型、函数和 handler；编译器根据这些真实引用自动建立依赖图并检查循环。新增或删除引用只修改实际使用位置，不需要同步维护另一份声明。 + +主动恢复的例子： + +```dever +configured_or_default(result: domain.PortResult.Ready(port)) (value: domain.Port) pure recover("optional local configuration uses port 8080") { + value = port +} + +configured_or_default(result: domain.PortResult.Failed(message)) (value: domain.Port) pure recover("optional local configuration uses port 8080") { + value = domain.Port { value = 8080 } +} +``` + +完整的 domain 定义见 [contracts 示例](examples/old/dever/contracts/module/domain.dever)。`recover` 允许这个函数结束时消费失败义务，因此应该放在实际决定恢复策略的小函数上，不能为通过编译给全部函数加同一个声明。 + +失败义务会跟随 record、choice 载荷和集合。未知集合的局部查找不能证明剩余元素已处理；可能覆盖旧值的 Map 更新也不能静默丢掉旧失败。可用完整遍历处理元素、继续返回原集合，或在真实恢复边界声明策略。空遍历不能消费没有传给 handler 的错误上下文，短路跳过的调用也不能算处理。分析是保守的，未实现任意程序的语义等价证明。 + +应用接口快照自动收录 App 能力、App 合同类型以及 Model 合同；topic 文件名和其他角色的私有实现不会进入快照。通过 API 基线约束后续变化： + +```bash +dever api my_app --output my_app/dever.api +dever check my_app +``` + +首次生成需要新路径；已存在的文件不会覆盖。`check`、`run`、`build` 自动检查项目根已有的 `dever.api`，变化时打印增删项并失败。快照记录公开名称、类型、可见字段、构造边界、约束、效果和恢复策略。正常修改私有实现不扩大公开接口。更新基线需要单独生成新文件并审查差异，不会自动“修复”基线。基线必须由项目审查或权限保护；如果允许 AI 同时任意修改源码和基线，它就不是授权边界。未提供基线时不会限制接口变化。 + +编译器还会建议清理有证据的重复范围判断（W002）、相同私有实现（W003）、可复用的纯调用（W004）和没有增加契约的私有转发（W005）。未使用赋值沿用 W001，不叠加第二套规则。指定应用入口时，W007 提醒没有外部使用需求的公开函数；普通库检查不推测未知调用方。建议不改变执行，不作为硬错误，也不按行数或函数个数强迫抽象。官方源码执行同样的契约检查，CLI 默认只展示当前用户源码的建议；官方硬错误不会隐藏。 + +契约分析有明确的工作上限（当前每轮相应证明最多 16,384 个工作单元，名义类型嵌套最多 128 层）。超大嵌套失败结构或事实证明超过上限时报 C015，不会跳过检查后放行。共享结构使用缓存与写时复制，普通重复引用不会直接展开成指数数量的副本。 + +这些约束能拒绝确定的违规，并提示部分冗余，不能证明一个功能有业务价值，也不能完全杜绝过度设计。 diff --git a/MARKDOWN-SYNTAX.md b/MARKDOWN-SYNTAX.md new file mode 100644 index 0000000..6e4d22e --- /dev/null +++ b/MARKDOWN-SYNTAX.md @@ -0,0 +1,145 @@ +# Dever Markdown 源码规范 + +安全数据合同与 `.dever` 完全一致：Model 私有字段在字段说明和代码中均写 `private password_hash: Text(255)`；App 通过显式 View 隐藏完整 Model。`Secret` 使用同一个内置标量和静态非观察检查，不能通过 Markdown 注释或文档元信息获得额外权限。Test source 的 `secret(Text)` 同样由测试源身份授权。 + +`.dever.md` 用 Markdown 保存可读说明，用顶层 `dever` 或 `typescript dever` 围栏保存 Dever 代码。代码仍由 Dever 编译器检查和执行；`typescript dever` 只用于请求编辑器采用 TypeScript 语法高亮。 + +完整项目示例见 [CMS](examples/cms/md/module/news/article/api/admin/manage.dever.md)；旧的[购物车结算](examples/old/markdown/module/main.dever.md)仍保留供语言回归使用。 + +## 源码说明 + +文件必须有且只有一个一级标题。标题使用自然语言名称，标题下依次写源码用途、路径推导的源码身份、公开类型、公开方法和调用方式： + +````markdown +# 账户接口 + +向调用方返回一条问候语。 + +- 包：`user.account.api` +- 公开类型：无 +- 公开方法：无 +- 使用：无 +```` + +源码身份由文件相对源码根的路径确定：`module/user/account/api.dever.md` 是 `user.account.api`，`module/user/account/app.dever.md` 是 `user.account.app`，`test/user/account/ensure_editor.dever.md` 是 `user.account.ensure_editor`。App 中的类型和方法自动列入公开清单，调用方式隐藏 role，写作 `user.account.function(input_name, ...)`；`app/<topic>.dever.md` 的 topic 也不进入调用名。Model 主记录和 choice 自动列入公开类型。Domain、Adapter、API 和测试没有公开方法。没有内容时必须写 `无`。同名函数有多个参数数量时，每个签名各写一条调用方式。 + +Markdown 应用测试固定使用 `test/<component>/<domain>/<topic>.dever.md`。文件必须用一个函数章节记录与 `<topic>` 同名的零输入、零输出普通函数；可在其他章节记录只供本文件使用的私有辅助声明。代码块内可使用测试专用的 `assert(...)` 和 `assert_eq(actual, expected)`，文档结构、函数输入输出和公开清单仍按本规范校验，不存在测试专用的简化格式。 + +Markdown 与普通源码使用同一目录约束：Model/API 主文件可与其 topic 目录共存；其他 role 二选一且单 topic 应并回主文件。非 API topic 目录只有一层，只有 `api/` 可递归。应用代码块不写 `package`、`exposes` 或 `internal`；`public` 只能修饰需要匿名访问的单条 HTTP API 声明。 + +一级标题下不能放 Dever 代码块；每个 Dever 声明放在自己的二级标题下。 + +API 章节用 `- 声明：` 标出 `[public] get <action>`、`[public] post <action>`、`[public] put <action>`、`[public] delete <action>`、`cmd <action>` 或 `rest [model.topic]`，代码块中的声明与说明须一致。`public` 不适用于 CMD 或 REST。API 只绑定本领域 App 或声明自动 REST，不写处理函数体。 + +## 类型说明 + +每个 type 使用一个二级标题。标题使用自然语言名称，随后写用途、技术名称、字段或分支，以及唯一的 type 代码块： + +````markdown +## 用户 + +保存用户的显示名称和年龄。 + +- 类型：`User` +- 字段： + - `name: Text`：显示名称 + - `age: Int >= 0`：年龄 + +```typescript dever +type User { + name: Text + age: Int >= 0 +} +``` +```` + +record 使用 `字段`；choice 使用 `分支`，例如： + +```markdown +- 分支： + - `Found(user: User)`：找到用户 + - `error Failed(message: Text)`：查询失败 +``` + +空 record 写 `- 字段：无`。字段或分支的名称、类型、约束、可见性、顺序和数量必须与代码一致。每项在全角冒号 `：` 后写非空的自然语言说明。 + +## 函数说明 + +每个逻辑函数使用一个二级标题。标题使用自然语言名称，随后写用途、技术名称、输入、输出，以及唯一的函数代码块： + +````markdown +## 生成问候语 + +根据姓名生成问候语。 + +- 函数：`greet` +- 输入： + - `name: Text`：需要问候的人 +- 输出： + - `message: Text`：生成的问候语 + +```typescript dever +greet(name: Text) (message: Text) { + message = "你好，" + name +} +``` +```` + +没有输入或输出时必须写 `- 输入：无` 或 `- 输出：无`。输入和输出使用编译器解析后的统一函数签名，名称、类型、顺序和数量必须一致。条件分句中的字面量和分支模式仍记录为统一类型，例如 `true` 和 `false` 分句的输入都写 `value: Bool`。 + +同一函数、同一参数数量的所有分句必须放在同一个二级标题和同一个代码块内，并且各分句的输入参数名必须一致。不同函数、不同参数数量的重载或 type 必须分别使用二级标题。 + +## Model 声明 + +`module/<component>/<domain>/model.dever.md` 定义简单领域 Model；复杂领域使用 `model/<topic>.dever.md`，记录名分别与 domain 或 topic 对应。类型字段说明保留存储参数，例如 `Text(1, 64)`、`Decimal(12, 2)`；`global`、默认值、索引以及 REST 的 `owner/create/replace/search` 修饰符由代码检查，不在字段清单中重复。 + +database、index、unique、relation、seed、migrate、sql 每个声明使用独立二级标题、用途说明和 `- 声明：` 元数据。例如： + +````markdown +## 初始化管理员 + +按唯一邮箱创建初始用户。 + +- 声明：`seed` + +```dever +seed { + { email = "admin@example.com" } +} +``` +```` + +声明标识分别为 `database primary`、`index(status, created_at)`、`unique(email, name)`、`relation articles = news.article.model.author_id`、`seed`、`migrate remove_legacy`、`sql lookup`。SQL 声明还需要按函数说明的格式列出输入、输出。编译器核对声明标识、参数顺序与类型；说明不能替代代码的存储校验。 + +## Job 声明 + +`job.dever.md` 或 `job/<topic>.dever.md` 使用同一 Job 语法，公开类型和方法清单均为空。Job 段仍写 `- 函数：`、输入和输出元信息；代码中的 `job`、`retry(n)`、`timeout(ms)` 由语言检查，不在文档中重复配置。payload 类型属于同领域 App。 + +`database` 与 `schedule` 各占一个二级标题，声明标识分别为 `database default`、`schedule cleanup`： + +````markdown +## 定时清理 + +每个 UTC 整点调度一次。 + +- 声明：`schedule cleanup` + +```dever +schedule cleanup = "0 * * * *" +``` +```` + +Job 的字段、调用权限、持久化和测试控制均与 `.dever` 一致；Markdown 不扩大公开性或 effect 权限。 + +## 通用规则 + +Port 的函数段在输出之后增加 `- 允许失败：\`app.DeliveryError\``，必须与 bodyless 签名的 `fails` 类型一致。Adapter/fake 函数名完整写为 `port.send` 或 `port.mail.send`；公开方法清单仍为空。Adapter 的 `setting { ... }` 使用普通类型段格式：`- 类型：\`Setting\`` 和完整字段清单；编译器按该 Adapter 私有记录核对字段。 + +- 源码和每个声明的用途说明不能为空。输入、输出、字段和分支的逐项说明也不能为空。 +- 一级标题不能有顶层 Dever 代码块，每个二级标题恰好对应一个；声明不能跨代码块，也不能重复出现在多个标题下。 +- `dever` 与 `typescript dever` 的含义相同。围栏必须符合 CommonMark、显式闭合，并且被 CommonMark 解析为顶层代码块。 +- 三级及更深标题、表格、普通段落和其它语言代码块可用于补充说明，不参与程序执行。补充内容放在所属 Dever 代码块之后。 +- `.dever` 文件继续使用原语法，不要求 Markdown 说明。 +- `fmt` 只改 Dever 代码块内部；`check`、`api`、`run` 和 `build` 同时检查代码及文档契约。 + +结构错误使用 M004–M005；源码身份、公开清单或声明名不一致使用 M006；函数输入不一致使用 M007；函数输出、类型字段或分支不一致使用 M008。签名错误会同时标出 Markdown 条目和对应 Dever 声明。 diff --git a/README.md b/README.md new file mode 100644 index 0000000..616a64b --- /dev/null +++ b/README.md @@ -0,0 +1,164 @@ +# Dever + +Dever 语言编译器。基础类型、表达式、命名输出、条件分句、集合、Stream 和值/资源语义已经接通语义检查与原生执行。提供源码格式化、基础标准库、JSON、HTTP/1.1 连接池、HTTP/2 多路复用、流式传输、SSE、WebSocket、TLS、可配置的有界并发，以及 SQLite/PostgreSQL 类型安全 ORM。Linux 机器安装、共享工具链与 ARM64 应用交叉构建已有实现；公开二进制发行资产、其它平台发行和自举仍需单独交付。 + +面向 Dever 程序开发者的完整语法、标准包和示例说明见 [Dever 语言开发指南](LANGUAGE.md)；编译器的长期架构和语义实现见 [Dever 语言实现原理](IMPLEMENTATION.md)。 + +应用开发从 [安装指引](skills/references/install.md) 和独立 [dever-language skill](skills/SKILL.md) 开始。Skill 仓库为 `shemic/dever-main-skills`，本仓库的 `skills/` submodule 固定配套提交；源码维护者使用 `git clone --recurse-submodules git@github.com:shemic/dever-main.git`。 + +```sh +dever new my_app +dever check my_app +dever test my_app +dever run my_app -- hello.greeting.greet '{"name":"Dever"}' +dever build my_app --output my_app/program +``` + +`new my_app --markdown` 生成等价 Markdown 项目，包含配置、AI 指引和应用测试。机器版 `dever update` 显式从官方 Releases 更新核心与配套 skill；`dever skill path [project-root]` 读取对应版本，`dever skill install <new-directory>` 安装持续跟随机器版本的 AI 入口。地址不写入业务 `config/setting.json`，普通 run/build 不联网。多个项目共享 `/opt/dever`，无需在每个项目复制核心；`crates/` 与 `library/` 是编译器和标准库维护目录。 + +[示例索引](examples/README.md) 以 CMS 为当前业务示例：[Dever 实现](examples/cms/dever/module/news/article/api/admin/manage.dever)和 [Markdown 实现](examples/cms/md/module/news/article/api/admin/manage.dever.md)同步维护。旧示例及编译拒绝反例归档在 `examples/old/`，正式回归测试保留在根目录 `test/`。 + +源码同时支持 `.dever` 和 `.dever.md`。应用固定使用 `module/<component>/<domain>/<role>` 结构，role 为 App、Domain、Model、Port、Adapter、API 或 Job；源码不写 package 或 import，`public` 只允许修饰单条匿名 HTTP 声明。跨领域只能调用 `<component>.<domain>.<function>` 形式的 App 能力，其他 role 保持领域私有。编译器从 API、CMD 和 Job 声明生成入口，不再使用 `module/main.dever`。应用测试放在 `test/<component>/<domain>/<topic>`，由文件名选择同名零输入、零输出函数。Markdown 合同见 [Markdown 源码规范](MARKDOWN-SYNTAX.md)。CMS 展示账号、成员、会话、admin/front 站点以及平台库与租户库分离。 + +例如 `module/news/article/api/admin/manage.dever` 只声明入口，业务组合留在 App；`examples/old/` 保存旧合同回归材料，不是当前应用目录示例。 + +```dever +rest model +post publish = app.publish +``` + +## 运行 + +这是编译器开发仓库，构建编译器自身需要 Rust 1.95 或以上、锁定的 Cargo 依赖和显式准备的 LLVM 18/LLD 作者 SDK。私有 `dever run/build/test` 默认将 checked HIR 编译为 LLVM object，再通过内置 LLD 链接；不调用 Rust/Cargo/cc，不通过环境变量选择后端，也不隐式下载依赖。CLI 参数是项目根，源码固定放在项目根的 `module/` 中，运行配置固定为 `config/setting.json`。 + +编译器同级必须有 `runtime/<platform>/manifest.json` 声明的 runtime、CRT 和静态库。作者侧使用独立的 base、SQLite、PostgreSQL、双数据库优化 archive，通过 [原生发行包制作流程](sdk/native-release.md) 生成带签名清单的新目录；输入固定写在作者根下的 `config/setting.json`。 + +```bash +cargo build --offline --locked -p dever-cli --bin dever +cargo build --offline --locked -p dever-cli --example native-release +target/debug/examples/native-release <author-root> --output <new-release-directory> +``` + +制作器离线工作，不覆盖已有目录；它复用安装端的格式/摘要校验，复制完成后签名，私钥不进入产物。私有调试可按制作说明将该 `runtime/`、`lib/` 放到匹配的 `dever` 同级。六目标正式发行仍需各自资产和平台验收。缺失或损坏的 pack 会明确报错；`check/fmt` 和零用例 `test` 不需要 runtime pack。 + +现成 Linux 二进制可通过 Port 的 `external command "bin/tool" {}` 直接调用，传参数数组及可选 Bytes 输入，得到退出码与两路 Bytes 输出；不用实现 Worker 协议或编写包装脚本。`run/build` 自动嵌入程序和显式随附的动态库，限额与文件授权仍在 `config/setting.json`，不使用 shell、环境变量或 PATH。单独的 command 不需要 Lib 锁；完整写法见 [语言说明](LANGUAGE.md)。 + +外部 Lib 通过显式命令解析，项目声明写入 `config/setting.json` 的根 `lib` 数组，例如 `"lib": ["pip:fixture@1.0.0"]`，锁定结果写入 `dever.lock`；`run/build` 只消费已有锁，不联网、不读取环境变量或系统 PATH。旧 `config/lib.json` 不再接受： + +```bash +dever lib add <project-root> pip:fixture@1.0.0 +dever lib list <project-root> +dever lib doctor <project-root> +dever lib update <project-root> +dever lib remove <project-root> pip:fixture@1.0.0 +``` + +当前仓库已接入 PyPI、npm registry 和 Go proxy 解析、校验下载及机器共享缓存。真实 provider 只从已验证的机器发行目录读取签名 runtime 描述和 pack；开发态缺少发行资产时明确报错，不使用宿主语言环境。v5 锁绑定 Worker 合同、Python extras、npm 嵌套实例和 Go sumdb 证据；不同 Worker 独立解析依赖。显式 Lib 准备可使用签名工具包构建 PEP 517 sdist、执行 npm 安装钩子及原生 addon 编译，源码、工具、构建依赖和产物统一绑定到收据；`run/build` 只复用已锁定输出。SDK 自动绑定 Port 同名操作并校验输入、结果与错误，见 `sdk/README.md` 和 `LANGUAGE.md`。Go Worker 使用锁定工具离线编译/链接，支持目标文件选择与 `go:embed`，启动不需要系统 Go。 + +原生制作器已支持 Python/Node/Go runtime/build pack 的确定性打包与统一签名；Linux x86_64 三生态的签名安装、受管 `check/run/build` 和移除源码/机器目录后的无系统语言独立执行已通过，包含 Python/Node 原生依赖。Python sdist/extras/原生扩展、npm 高级依赖/安装钩子/native addon 和 Go sumdb 均已有实现及真实第三方依赖定向证据。Linux 签名首装、升级保护和两个项目/真实用户共享核心也已通过自有完整验收。Linux Worker 使用 OS 沙箱；当前 root 验收不代表受限 AppArmor 下非 root 部署通过。输入仍是私有作者资产和临时签名，尚未交付公开首次安装包或其他平台发行。具体流程见 [原生发行包制作流程](sdk/native-release.md)。 + +Dever Package 用 `config/setting.json.package` 声明，例如 `"package": {"registry": "https://packages.example.test", "use": ["catalog@^1.0.0"]}`。`dever package add/update/remove/list/doctor <project-root>` 共用机器摘要缓存；版本和传递依赖、Package 摘要及其 Lib 闭包统一写入 `dever.lock`。`check/test/run/build` 加载锁定的 Package 源码，不隐式解析或下载，不允许 Package 覆盖本地同名组件；离线移除保留剩余 Package 的锁定版本。这里的 registry 地址只是配置格式示例，不是已发布服务。 + +编译器底层已有 LLVM 18/LLD 连接桥和强类型 HIR→LLVM 转换，覆盖数值（含 Decimal）、Text/Id、Bytes、集合、嵌套记录/选择型/可空值、静态 handler、遍历、Result 和原始错误调用链。文件/流、真实协程、Task/Group/Channel、blocking/parallel、TCP/HTTP/TLS/SSE/WebSocket 及系统能力复用原 runtime 和同一 typed ABI。受管值通过具体生命周期回调转移，业务不经过解释器或动态 Value。简单内核已验证六目标链接，复杂内核已验证六目标对象及 Linux x86_64 实际执行；Windows 复杂链接仍需正式 runtime 支持。Rust unsafe 例外仅限桥接 crate 的私有 `ffi.rs`，不是语言 FFI。 + +独立 LLVM 应用入口已接通 CMD→App→配置选中的 Dever/external Port/Adapter、SQLite/PostgreSQL Model/事务，以及 HTTP/REST、可信身份、自动权限、database-only 租户/组件状态、Cookie/Header、Upload 存储和日志配置。持久 Job 复用现有队列、重试、调度、权限重验和事务；应用 Test 保持整套一次编译、稳定用例编号、每用例独立进程及 fake/数据库/时钟隔离。external 调用沿用统一 Worker 协议，支持具名类型化输入/输出、声明的业务错误和校验后提取的内嵌资源；只在边界编码 JSON。Worker 属于本次运行，退出时先关闭 Worker，再排空子任务和关闭其他资源；配置仍只读取 `config/setting.json`。私有 CLI 和 Linux 可信 daemon 编译已接 LLVM，正式目标包和跨平台发行仍未交付；本机定向验收不等于性能或正式发行验收。 + +```bash +cargo run --offline --quiet -p dever-cli --bin dever -- check examples/cms/dever +cargo run --offline --quiet -p dever-cli --bin dever -- test examples/cms/dever +cargo run --offline --quiet -p dever-cli --bin dever -- fmt examples/cms/dever --check +cargo run --offline --quiet -p dever-cli --bin dever -- run examples/cms/dever -- user.account.bootstrap '{"tenant_key":"tenant-one","tenant_name":"Tenant One","email":"owner@example.com","display_name":"Owner","password":"change-me-now"}' +cargo run --offline --quiet -p dever-cli --bin dever -- tenant migrate examples/cms/dever 1 +cargo run --offline --quiet -p dever-cli --bin dever -- tenant owner examples/cms/dever 1 admin 1 +cargo run --offline --quiet -p dever-cli --bin dever -- tenant owner examples/cms/dever 1 front 1 +cargo run --offline --quiet -p dever-cli --bin dever -- check examples/cms/md +cargo run --offline --quiet -p dever-cli --bin dever -- test examples/cms/md +cargo run --offline --quiet -p dever-cli --bin dever -- clean examples/cms/dever +``` + +`tenant migrate` 和两条 `tenant owner` 中的租户 `1` 应替换为 bootstrap 响应中的 `tenant_id`，Owner 命令最后的用户 `1` 应替换为响应中的 `user_id`。迁移先建立租户物理库和私有 Job/授权 schema，再分别为 admin/front 站点建立核心 RBAC Owner，完成后才能启动受保护 API/Worker。bootstrap 中的 Owner membership 是业务成员身份，不代替核心接口授权。两套 CMS 源码具有相同的 Model 与 App 合同。编译器开发入口是 `target/debug/dever`，也是 `cargo run -p dever-cli -- ...` 的默认目标。机器版启动器的内部构建名是 `dever-launcher`，发行时安装为 `dever`，版本核心仍安装为 `dever-core`。Go 框架使用独立的 `dever-go`；本仓库构建不会安装或替换全局命令。 + +正式工具链只公开一个安装在机器级 `bin/` 下的 `dever`。它从签名版本仓库选择版本：项目可在 `config/setting.json` 写精确的 `{"dever":{"version":"0.1.0"}}`，未写时使用机器活动版本。`install/update/use/uninstall/version/cache status/cache clean` 属于机器管理命令。`dever clean <project-root>` 只回收当前用户超过 24 小时且所属进程已经退出的 Dever 临时构建目录和旧 `.dever-run-*` 文件，不删除 build 输出、源码、配置、data、Lib 或 Cargo 缓存。Linux `deverd` 已用内核 peer credential 接受其他机器用户的状态查询和按 SHA-256 传输的共享 Lib 资产，非服务所有者不能清缓存或读取管理 token；普通查询返回 `verified:false` 的元数据摘要，完整校验与清理由管理员执行。服务不可用明确失败，不会本地回退。开发 fixture 可用 `deverd --root <machine-root>` 启动服务。编译器贡献者清理本仓库 `target/` 时使用 Cargo 自己的 `cargo clean`。Linux daemon 编译提交规范源码和无秘密配置绑定，签名核心重新检查并编译；完整相同输入可跨项目、跨用户复用产物。App/Test 仍由调用者运行。macOS/Windows 跨用户认证、系统服务注册和正式原生发行仍未交付，开发态 socket 不等于完整跨平台安装。 + +保留一个可独立运行的程序： + +```bash +cargo run --offline --quiet -p dever-cli --bin dever -- build examples/cms/dever --output examples/cms/dever/cms-app +./examples/cms/dever/cms-app user.account.bootstrap '{"tenant_key":"tenant-one","tenant_name":"Tenant One","email":"owner@example.com","display_name":"Owner","password":"change-me-now"}' +``` + +`build` 的输出路径必须是新文件，父目录必须存在；已有文件不会被覆盖。独立程序从可执行文件同级的 `config/setting.json` 加载配置，因此正式产物放在项目根或保持相同的部署目录结构。编译和运行使用同一 LLVM 路径；`run` 直接执行编译器持有的临时程序，临时 object、链接输入快照和程序在执行后清理。私有作者缓存位于编译器同级 `cache/native`，绑定编译器、pack/profile 和完整 IR；命中仍检查 pack 和程序完整性。独立程序无需 LLVM、Cargo、Rust 工具链或 `.dever` 源码即可运行。机器管理的 `dever-core` 通过可信 daemon 编译；服务缺失明确失败，不降级到私有缓存。签名版本必须覆盖 native manifest、全部链接输入和编译器动态库，命中缓存也重新校验。 + +## 已支持 + +Linux x86_64 开发机可用 `dever build <项目目录> --target linux-aarch64 --output <新文件>` 生成 ARM64 应用；部署端不需要 Dever 编译器、Rust 或 LLVM。工具链须已包含对应 ARM 运行库，第三方 Lib 用 `dever lib update <项目目录> --target linux-aarch64` 显式准备。省略目标继续本机构建，`run/test` 也保持本机执行。构建不隐式联网、不读取环境变量选择工具；作者输入及 Go 宿主工具与目标标准库的区别见 [发行制作说明](sdk/native-release.md)。 + +- 应用源码使用路径推导的 component/domain/role；App 自动成为跨领域能力，其他 role 仅在所属领域内可见。Model/API 主文件可与同 role 主题目录共存；其余 role 的主文件与目录互斥，单个非 Model/API 主题应并回主文件。 +- Bool、Int、Decimal、Float、Text、Id；字段型、选择型、可空值、List、Map、MapEntry、Bytes 和 `Stream<Item>`。 +- 构造、字段读写、算术、比较、布尔短路、单个及多个命名输出；初始化检查、稳定类型和值语义。 +- Int/Decimal 字段与输出范围、`private` 字段、`error` 失败义务、默认错误传播、`result(call)` 主动捕获、`fail(...)`、显式 `recover("reason")` 和传递式 `pure` 检查；API 基线约束公开契约变化，保守建议提示重复判断和包装。用法见 [指南第 18 节](LANGUAGE.md#18-编译器约束与公开接口)及 `examples/old/dever/contracts/`。 +- 分句是唯一条件分派方式，检查重叠、遗漏、不可达和多输入组合；`other` 是所在输入位置全部明确模式的补集，不依赖书写顺序。 +- `each`、`reduce`、`reduce_until`、`filter`、`find`、`sum`、`append`、`first`、`get`、`put`、`remove`、`entries`、`length`。List、Bytes 和 Stream 共用静态 handler 检查，handler 参数可跨函数转发但不是运行时值。Map 保留插入顺序，可空返回不产生嵌套可空类型。 +- `parallel_each(handler, sequence, workers[, context])` 接受 List、Bytes、Stream 或 AsyncStream，并根据序列与 handler 的挂起 effect 选择作用域线程、共享 blocking worker 或有界 Group。并发数始终有上限，返回前清理已启动工作；副作用完成顺序不保证。 +- 普通调用自动等待其完整调用链；源码不声明 `async`，也不写 `await`。`run`、`wait`、`stop`、`timeout`、`race`、有界 `Group`、`parallel`、`blocking` 和有界 `Channel` 构成结构化并发基础。Task/Group 必须在父作用域消费，超时和竞争都会清理剩余任务。 +- `run/build` 的应用入口由 API、CMD 和 Job 声明生成；CMD 以稳定名称和一个 JSON 对象选中。HTTP API 按 `config/setting.json` 的 `sites.<key>.path` 归属站点，浏览器 Cookie 使用配置的外部 `sites.<key>.origin` 做精确同源校验；默认需要认证，只有单条 `public get/post/put/delete` 可匿名。函数、用户类型和 package 依赖不能形成循环。 +- `dever test <project-root>` 按名称串行执行 `test/` 中的应用测试。`assert(Bool)` 与同类型可比较值的 `assert_eq(actual, expected)` 只在测试源码可用；每个用例使用独立进程，涉及 Model 时使用运行器生成的临时 SQLite 配置，不读取部署数据库配置，也不从环境变量接收数据库覆盖。 +- `library/` 中的核心和官方 package 都是正常检查的 `.dever` 源码。Text 提供 Unicode 标量索引/切片、字符编码转换、查找、切分、替换与连接；Int/Decimal/Float 提供解析与文本转换。解析失败或索引越界返回 `null`，不截断或默认为零。 +- Bytes 与文件/TCP 资源原语；可恢复错误使用选择型结果，数值故障和标准输出失败定位到 `.dever` 源码。 +- Model role 提供隐式 ID/创建时间、静态 CRUD、分页/游标/Stream、关联、迁移、Seed、事务和参数化 typed native SQL；Model 操作只能由所属领域 App 调用，SQLite 与 PostgreSQL 共享 Model、QueryPlan 和 schema 语义。 +- `dever.auth` 通过配置引用的只读 App verify 建立三字段隐藏身份；所有非 `public` HTTP API 自动按 `component.domain.site.action` 查询核心 RBAC，源码不写权限字符串。权限目录、角色、角色权限和用户角色由 Dever 私有表维护；用户 Job 只保存入口权限键，执行前重新验证身份、组件状态和当前角色。多租户只支持 `tenant.database` 选择控制连接；`global type` 留在平台库，其他 Model 使用独立租户数据库，租户组件从 Model/Job 自动推导并可由受控 CLI 禁用或恢复。未迁移、未就绪、组件禁用或缺少租户上下文都直接失败，不回退平台库。 +- 自动 REST 支持 `owner` 行范围及字段级 `create/replace/search` 纯绑定；上传使用有界 multipart spool 和 affine `Upload`，只允许 POST App 直接接收，并经 Port/Adapter 调用 `dever.storage.put` 或显式关闭。 + +## 格式化与官方库 + +`dever fmt <project-root>` 统一规范化 `module/` 和存在的 `test/` 中的两空格缩进、换行和空白，保留注释、表达式分组及 record 字段的求值顺序。`--check` 只列出需格式化的文件并返回非零。格式化不要求语义检查通过；整批源码先通过语法检查和写入准备，才逐文件原子替换，不能把操作系统中途故障视为跨文件事务。 + +`.dever.md` 先检查标题、说明列表和代码块的一对一结构，再只格式化 Dever 代码块内部；说明文字、标题、围栏和其它代码块保持原样。语义检查还会拒绝与代码名称、类型、数量或顺序不一致的说明。 + +- `dever.process.arguments()` 读取应用参数（不含可执行文件路径）；应用配置只读 `config/setting.json`。`dever.time` 提供 Unix 毫秒、单调纳秒和显式等待，均返回源码定义的结果类型。 +- `dever.task.sleep(milliseconds)` 提供非阻塞等待，`ticks(milliseconds)` 提供不积压的周期流；调用链是否挂起由编译器推断。同步 `dever.time.sleep` 保持阻塞语义。 +- `dever.json.parse(text)` / `stringify(document)` 使用普通分句、reduce 和 Map 完成 JSON 解析与编码。`dever.json.value` 定义公共结果、文档和节点类型，节点以整数链接表示嵌套；数字保留原文。重复键、非法转义、未配对代理项和多余输入明确失败。深度最多 256 层；编码最多 16,777,216 个 Unicode 标量，并验证手工构造的文档引用。 +- `dever.http` 复用 Hyper 的 HTTP/1 与 HTTP/2 引擎，支持持久连接、逐块响应和 SSE；HTTP/1 支持流水线、chunked 和 WebSocket 升级。`client/request/open/open_stream` 提供固定目标的有界连接池和流式上传/下载，支持 DNS。设置 `limits.http2 = dever.http.default_http2_limits()` 开启 HTTP/2 多路复用，独立限制连接数、流数和接收窗口。TLS 复用 rustls/tokio-rustls，校验证书并为 HTTP/2 协商 ALPN h2；关闭 Listener 后通过 GOAWAY 排空。静态 handler 可携带类型化 Context。WebSocket-over-HTTP/2、CONNECT、推送与其他协议扩展仍未交付。 + +JSON 算法和官方接口位于 `library/dever/`；HTTP 的协议状态机复用运行时中的 Hyper，业务 handler 直接编译执行。精确限制见 [LANGUAGE.md](LANGUAGE.md)。 + +`examples/old/` 保留旧语言合同、网络和运行时回归材料；它们不代表当前应用目录结构。[CMS 示例](examples/cms/dever/module/news/article/api/admin/manage.dever) 展示 App/Domain/Model 边界、站点认证、平台/租户数据分离和发布事务。 + +## 资源边界 + +普通值复制后独立修改；资源句柄和 Stream 别名共享资源状态。关闭文件或连接任一别名后，其他别名的操作会得到 `Failed`；关闭 Stream 任一别名后，全部别名结束，重复关闭无副作用。`dever.io.create` 只创建新文件，不覆盖已有文件；`open` 只读打开。`read` 和 `chunks` 接受正数大小上限，允许短读；Stream 在自然 EOF 后结束，终止错误只产生一个 `Failed` 项。 + +TCP 的连接、监听、接收、读写会被推断为挂起并复用一个 Tokio runtime；`connect_timeout` 要求数值 IP。连接流和字节流为 `AsyncStream`，按需拉取；`parallel_each` 使用有界任务组，先等待容量再拉取。Socket 支持同时读写，关闭唤醒等待方，取消部分写入会关闭连接。HTTP、SSE、WebSocket 共用现有任务作用域和连接限额；停止服务会排空升级后的会话。文件 I/O 仍需通过 `blocking(...)` 与调度线程隔离。 + +HTTP 池的连接上限包含空闲连接；响应体读完才归还连接，提前关闭则丢弃该连接。流式传输按块限制并复用 Bytes 存储，不收集整条长连接。TLS 配置共享信任材料，并使用小容量的有界会话缓存。线程、任务、连接和块大小上限共同约束运行资源，应用保留的数据和操作系统缓冲仍需单独计入内存。 + +这是全新语言，接口直接迭代，仓库源码同步迁移，不提供旧源码兼容别名或回退后端。TCP 用法见 [异步 TCP 示例](examples/old/dever/async_tcp/module/main.dever)；流的关闭、限流和取消契约见 [LANGUAGE.md](LANGUAGE.md)。 + +编译器通过最后使用分析移动普通值；含文件、Socket、Listener 或 Stream 的值保留到源码作用域结束，避免优化提前关闭外部资源。基准解释器仅通过 `reference` feature 启用，用于与原生结果和故障位置进行差分验证；CLI 的默认 `build/run` 不使用解释器。 + +记录字段更新、独占列表消费、直接字符遍历和可证明安全的集合融合由编译器自动优化。HTTP 复用 Hyper 增量协议处理，Bytes 共享消息缓冲区；List/Bytes 并行动作按有限批次分发。私有 LLVM CLI 复用编译器同级 `cache/native` 的原生产物，命中时仍检查源码、API 基线、pack 和程序完整性；Lib 资产和 Linux 编译产物通过 `deverd` 共享；编译缓存按完整请求复用，不能仅凭产物摘要读取。大型 Worker 资源随原生产物内嵌；启动校验按块读取。前端检查结果仅在同进程内缓存。实现与边界见 [IMPLEMENTATION.md](IMPLEMENTATION.md)。 + +## 定向验证 + +按当前修改选择定向测试，不默认运行全部测试。原生测试会执行本地小程序；资源测试只使用自身临时目录、临时 SQLite 数据库及 `127.0.0.1` 随机端口。真实 PostgreSQL 测试默认忽略，只有仓库根 `config/setting.json` 明确配置 `database.postgres_test` 后才手动运行，不接受环境变量或命令行连接覆盖。可复制 `config/setting.example.json` 后按本机 PostgreSQL 凭据修改；测试 URL 的数据库名必须保留 `{case}` 占位符。 + +```bash +cargo test --offline -p dever-tests --test core_semantics +cargo test --offline -p dever-tests --test core_native +cargo test --offline -p dever-tests --test runtime_foundations +cargo test --offline -p dever-tests --test hello_native +cargo test --offline -p dever-tests --test structured_concurrency +cargo test --offline -p dever-tests --test async_runtime +cargo test --offline -p dever-tests --test async_native +cargo test --offline -p dever-tests --test async_composition +cargo test --offline -p dever-tests --test pooled_network +cargo test --offline -p dever-tests --test pooled_library +``` + +新增工具和库按需选择 `formatter`、`standard_library`、`native_ownership`、`concurrency`、`json_library`、`http_library`、`http_engine`；差分验证使用 `cargo test --offline -p dever-tests --features reference --test differential`。 + +`performance` 默认忽略，只在明确进行有界性能验证时运行：`cargo test --offline -p dever-tests --test performance -- --ignored --nocapture`。它比较 Int、Float、List、Decimal、Map 五个本地计算内核，排除构建、文件读取和进程启动，构建耗时单独记录。双方交错运行五个独立进程，每个进程预热后取九次测量，以各自全部样本的中位数比较，目标不超过对应基线的 1.15 倍。结果只代表该环境和这些内核，不代表 HTTP 吞吐或整个语言的性能。 + +同一目标还包含 `allocation_sensitive_source_scenarios`，运行 `test/dever-tests/fixtures/performance` 的记录文本、列表、JSON 和 16,384 个不同键用例。`concurrency` 的 `parallel_task_granularity_measurement` 也默认忽略，用于有限任务调度测量，不启动服务。耗时、内存与验证范围见任务实施记录。 + +独立二进制的常驻内存、异步调度和 HTTP/HTTPS 基准位于 [test/performance](test/performance/README.md)。构建和运行分开，支持专属 cgroup 预算及 Rust runtime/Hyper 对照，保存原始计数、延迟与资源采样。文档包含首轮 1 worker/128 MiB 短时结果及 64 MiB 空闲验证；这些结果不代表吞吐上限或长期稳定性。基准必须显式运行，不进入默认测试。 diff --git a/config/setting.example.json b/config/setting.example.json new file mode 100644 index 0000000..245b447 --- /dev/null +++ b/config/setting.example.json @@ -0,0 +1,24 @@ +{ + "database": { + "default": { + "type": "postgres", + "url": "postgresql://dever@127.0.0.1:5432/dever_default?sslmode=disable", + "tls": "disabled", + "min_connections": 0, + "max_connections": 2, + "max_page_size": 100, + "wait_timeout_ms": 1000, + "io_timeout_ms": 5000 + }, + "postgres_test": { + "type": "postgres", + "url": "postgresql://dever@127.0.0.1:5432/dever_{case}?sslmode=disable", + "tls": "disabled", + "min_connections": 0, + "max_connections": 2, + "max_page_size": 100, + "wait_timeout_ms": 1000, + "io_timeout_ms": 5000 + } + } +} diff --git a/crates/dever-backend-bridge/Cargo.toml b/crates/dever-backend-bridge/Cargo.toml new file mode 100644 index 0000000..a269d9b --- /dev/null +++ b/crates/dever-backend-bridge/Cargo.toml @@ -0,0 +1,125 @@ +[package] +name = "dever-backend-bridge" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +publish.workspace = true +autotests = false + +[lib] +crate-type = ["rlib", "staticlib"] + +# Deliberately do not inherit workspace forbid: only src/ffi.rs may allow it. +[lints.rust] +unsafe_code = "deny" +unsafe_op_in_unsafe_fn = "deny" + +[features] +default = [] +embedded = ["dep:cc", "dep:rustix"] +runtime-abi = ["dep:dever-runtime", "dever-runtime/crypto", "dever-runtime/wire", "dep:libc"] +runtime-database = ["runtime-abi", "dever-runtime/database"] +runtime-api = ["runtime-abi", "dever-runtime/api"] +runtime-external = ["runtime-abi", "dever-runtime/external"] +runtime-sqlite = ["runtime-database", "dever-runtime/sqlite"] +runtime-postgres = ["runtime-database", "dever-runtime/postgres"] + +[dependencies] +dever-runtime = { workspace = true, optional = true } + +[target.'cfg(unix)'.dependencies] +libc = { version = "=0.2.189", optional = true } +rustix = { version = "=1.1.2", features = ["process"], optional = true } + +[dev-dependencies] +dever-core.workspace = true +serde_json.workspace = true +sha2.workspace = true + +[build-dependencies] +cc = { version = "=1.4.5", optional = true } + +[[test]] +name = "backend_bridge" +path = "../../test/backend_bridge.rs" +required-features = ["embedded"] + +[[test]] +name = "native_runtime_abi" +path = "../../test/native_runtime_abi.rs" +required-features = ["runtime-abi"] + +[[test]] +name = "native_runtime_link" +path = "../../test/native_runtime_link.rs" +required-features = ["runtime-abi"] + +[[test]] +name = "llvm_database_source" +path = "../../test/llvm_database_source.rs" +required-features = ["embedded", "runtime-sqlite"] + +[[test]] +name = "llvm_api_source" +path = "../../test/llvm_api_source.rs" +required-features = ["embedded", "runtime-api"] + +[[test]] +name = "llvm_job_test_source" +path = "../../test/llvm_job_test_source.rs" +required-features = ["embedded", "runtime-api", "runtime-sqlite"] + +[[test]] +name = "llvm_native" +path = "../../test/llvm_native.rs" +required-features = ["embedded"] + +[[test]] +name = "llvm_managed_native" +path = "../../test/llvm_managed_native.rs" +required-features = ["embedded", "runtime-abi"] + +[[test]] +name = "llvm_result_native" +path = "../../test/llvm_result_native.rs" +required-features = ["embedded", "runtime-abi"] + +[[test]] +name = "llvm_resource_native" +path = "../../test/llvm_resource_native.rs" +required-features = ["embedded", "runtime-abi"] + +[[test]] +name = "llvm_async_native" +path = "../../test/llvm_async_native.rs" +required-features = ["embedded", "runtime-abi"] + +[[test]] +name = "llvm_async_source" +path = "../../test/llvm_async_source.rs" +required-features = ["embedded", "runtime-abi"] + +[[test]] +name = "llvm_stream_network_source" +path = "../../test/llvm_stream_network_source.rs" +required-features = ["embedded", "runtime-abi"] + +[[test]] +name = "llvm_http_system_source" +path = "../../test/llvm_http_system_source.rs" +required-features = ["embedded", "runtime-abi"] + +[[test]] +name = "llvm_application_source" +path = "../../test/llvm_application_source.rs" +required-features = ["embedded", "runtime-abi"] + +[[test]] +name = "llvm_external_source" +path = "../../test/llvm_external_source.rs" +required-features = ["embedded", "runtime-external"] + +[[test]] +name = "llvm_process_entry_source" +path = "../../test/llvm_process_entry_source.rs" +required-features = ["embedded"] diff --git a/crates/dever-backend-bridge/build.rs b/crates/dever-backend-bridge/build.rs new file mode 100644 index 0000000..0124fe5 --- /dev/null +++ b/crates/dever-backend-bridge/build.rs @@ -0,0 +1,50 @@ +fn main() { + #[cfg(feature = "embedded")] + build(); +} + +#[cfg(feature = "embedded")] +fn build() { + use std::path::PathBuf; + + // Private authoring SDK, never a runtime PATH lookup or app setting. + // Release builders prepare these headers/libraries before enabling embedded. + let root = + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../target/backend-sdk/usr/lib/llvm-18"); + let include = root.join("include"); + let lib = root.join("lib"); + for input in [include.join("lld/Common/Driver.h"), lib.join("liblldELF.a")] { + assert!( + input.is_file(), + "private LLVM 18 SDK is missing: {}", + input.display() + ); + } + println!("cargo:rerun-if-changed=src/bridge.cpp"); + println!("cargo:rerun-if-changed={}", root.display()); + cc::Build::new() + .cpp(true) + .std("c++17") + // Upstream headers are system headers; warnings remain enabled for ours. + .flag("-isystem") + .flag(include.to_str().expect("SDK include path must be UTF-8")) + .file("src/bridge.cpp") + .flag_if_supported("-fno-rtti") + .compile("dever_backend_bridge"); + println!("cargo:rustc-link-search=native={}", lib.display()); + let multiarch = match std::env::var("TARGET").as_deref() { + Ok("x86_64-unknown-linux-gnu") => "x86_64-linux-gnu", + Ok("aarch64-unknown-linux-gnu") => "aarch64-linux-gnu", + _ => panic!("the embedded author SDK currently requires a Linux GNU target"), + }; + println!( + "cargo:rustc-link-search=native={}", + root.join("..").join(multiarch).display() + ); + for name in ["lldELF", "lldCOFF", "lldMachO", "lldCommon"] { + println!("cargo:rustc-link-lib=static={name}"); + } + println!("cargo:rustc-link-lib=dylib=LLVM-18"); + println!("cargo:rustc-link-lib=dylib=z"); + println!("cargo:rustc-link-lib=static=zstd"); +} diff --git a/crates/dever-backend-bridge/include/runtime.h b/crates/dever-backend-bridge/include/runtime.h new file mode 100644 index 0000000..334062b --- /dev/null +++ b/crates/dever-backend-bridge/include/runtime.h @@ -0,0 +1,909 @@ +#ifndef DEVER_RUNTIME_ABI_V1_H +#define DEVER_RUNTIME_ABI_V1_H + +#include <stdint.h> + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + uint8_t *ptr; + uint64_t len; +} DeverRtBuffer; + +typedef struct { + uint64_t low; + uint64_t high; +} DeverRtDecimal; + +/* Compiler-emitted callbacks have the C ABI and live for the entire process. + * They operate on fully initialized values of exactly this concrete type. + * clone initializes uninitialized destination storage, drop destroys one + * initialized value, and equal/hash are consistent for Map keys. They must + * not fail or unwind. hash is optional except for Map key descriptors. + */ +typedef struct { + uint64_t size; + uint64_t align; + void (*clone)(const void *source, void *destination); + void (*drop)(void *value); + uint8_t (*equal)(const void *left, const void *right); + uint64_t (*hash)(const void *value); +} DeverRtType; + +/* Callbacks initialize one complete ReadEvent in uninitialized storage. + * The Bytes/Text handle is borrowed only during the callback; retain it to + * keep it in the constructed event. All fields live for the process lifetime. + */ +typedef struct { + const DeverRtType *type; + void (*chunk)(const void *borrowed_bytes, void *out_event); + void (*failed)(const void *borrowed_text, void *out_event); +} DeverRtReadEvent; + +enum { + DEVER_RT_OK = 0, + DEVER_RT_RUNTIME_ERROR = 1, + DEVER_RT_INVALID_INPUT = 2, +}; + +/* Internal compiler/runtime ABI, not a Dever source-language FFI. + * All targets are 64-bit. Inputs are borrowed only until the call returns. + * A nonzero input length requires that many live immutable bytes; null is + * accepted only with zero length. Input memory must not overlap an output. + * Each output pointer must be live, writable, correctly aligned, and distinct + * from every other output. An out buffer is reset to {NULL, 0} on every call + * with a valid pointer; release a previous value before reusing that slot. + * Status 0 initializes the numeric out value or owns the output bytes. + * Status 1 leaves numeric out unchanged and returns the original runtime + * error text. Status 2 reports an invalid ABI pointer or length; a null out + * buffer cannot receive an error message. Do not inspect numeric out on error. + * Every non-null returned buffer is allocated by the Rust runtime ABI and + * must be passed once, with its exact length, to dever_rt_v1_buffer_free. + * Do not use C free or the compiler bridge's C++ allocator for these bytes. + */ +uint32_t dever_rt_v1_version(void); +void dever_rt_v1_buffer_free(uint8_t *ptr, uint64_t len); + +uint32_t dever_rt_v1_int_add(int64_t left, int64_t right, int64_t *out, DeverRtBuffer *buffer); +uint32_t dever_rt_v1_int_sub(int64_t left, int64_t right, int64_t *out, DeverRtBuffer *buffer); +uint32_t dever_rt_v1_int_mul(int64_t left, int64_t right, int64_t *out, DeverRtBuffer *buffer); +uint32_t dever_rt_v1_int_div(int64_t left, int64_t right, int64_t *out, DeverRtBuffer *buffer); +uint32_t dever_rt_v1_int_rem(int64_t left, int64_t right, int64_t *out, DeverRtBuffer *buffer); +uint32_t dever_rt_v1_int_neg(int64_t input, int64_t *out, DeverRtBuffer *buffer); + +uint32_t dever_rt_v1_float_add(double left, double right, double *out, DeverRtBuffer *buffer); +uint32_t dever_rt_v1_float_sub(double left, double right, double *out, DeverRtBuffer *buffer); +uint32_t dever_rt_v1_float_mul(double left, double right, double *out, DeverRtBuffer *buffer); +uint32_t dever_rt_v1_float_div(double left, double right, double *out, DeverRtBuffer *buffer); +uint32_t dever_rt_v1_float_rem(double left, double right, double *out, DeverRtBuffer *buffer); +uint32_t dever_rt_v1_float_neg(double input, double *out, DeverRtBuffer *buffer); + +uint32_t dever_rt_v1_decimal_from_int(int64_t input, DeverRtDecimal *out, DeverRtBuffer *buffer); +uint32_t dever_rt_v1_decimal_add(uint64_t left_low, uint64_t left_high, uint64_t right_low, uint64_t right_high, DeverRtDecimal *out, DeverRtBuffer *buffer); +uint32_t dever_rt_v1_decimal_sub(uint64_t left_low, uint64_t left_high, uint64_t right_low, uint64_t right_high, DeverRtDecimal *out, DeverRtBuffer *buffer); +uint32_t dever_rt_v1_decimal_mul(uint64_t left_low, uint64_t left_high, uint64_t right_low, uint64_t right_high, DeverRtDecimal *out, DeverRtBuffer *buffer); +uint32_t dever_rt_v1_decimal_div(uint64_t left_low, uint64_t left_high, uint64_t right_low, uint64_t right_high, DeverRtDecimal *out, DeverRtBuffer *buffer); +uint32_t dever_rt_v1_decimal_neg(uint64_t low, uint64_t high, DeverRtDecimal *out, DeverRtBuffer *buffer); +uint32_t dever_rt_v1_decimal_round(uint64_t low, uint64_t high, int64_t places, DeverRtDecimal *out, DeverRtBuffer *buffer); + +uint32_t dever_rt_v1_int_to_text(int64_t input, DeverRtBuffer *buffer); +uint32_t dever_rt_v1_float_to_text(double input, DeverRtBuffer *buffer); +uint32_t dever_rt_v1_decimal_to_text(uint64_t low, uint64_t high, DeverRtBuffer *buffer); +uint32_t dever_rt_v1_decimal_compare(uint64_t left_low, uint64_t left_high, uint64_t right_low, uint64_t right_high, int32_t *out, DeverRtBuffer *error); +/* Compiler-checked finite Decimal operands only; numeric, not bitwise equality. */ +uint8_t dever_rt_v1_decimal_equal(uint64_t left_low, uint64_t left_high, uint64_t right_low, uint64_t right_high); +uint32_t dever_rt_v1_decimal_parse(const void *text, DeverRtDecimal *out, uint8_t *present, DeverRtBuffer *error); +uint32_t dever_rt_v1_text_copy_utf8(const uint8_t *input, uint64_t len, DeverRtBuffer *buffer); +uint32_t dever_rt_v1_bytes_copy(const uint8_t *input, uint64_t len, DeverRtBuffer *buffer); +uint32_t dever_rt_v1_bytes_to_text(const uint8_t *input, uint64_t len, DeverRtBuffer *buffer); +/* Synchronous checked time.sleep: Unit uses one byte (0) on success. Negative + * duration returns the original runtime error without initializing out. + */ +uint32_t dever_rt_v1_time_sleep(int64_t milliseconds, uint8_t *out, DeverRtBuffer *error); + +/* Managed values are private, typed Arc handles. Each returned handle is owned; + * retain returns another owner and release consumes one. Only retain/release + * accept NULL as a no-op. Id uses the Text representation and owner functions. + * All other handle arguments are borrowed unless the symbol ends in _take. + * A _take call consumes its non-NULL input handle even when it returns error. + * Source element pointers and descriptor pointers are borrowed, not consumed. + * A successful element extraction initializes previously uninitialized, + * aligned output storage with a new owned value; absent output is untouched. + * The caller must destroy extracted values with its concrete drop callback. + * Every fallible call resets a valid error buffer; status 1 contains the + * original runtime error and status 2 reports an invalid ABI argument. + */ +void *dever_rt_v1_text_retain(const void *handle); +void dever_rt_v1_text_release(void *handle); +uint32_t dever_rt_v1_text_new(const uint8_t *input, uint64_t len, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_text_export(const void *handle, DeverRtBuffer *buffer); +uint8_t dever_rt_v1_text_equal(const void *left, const void *right); +uint64_t dever_rt_v1_text_hash(const void *handle); +uint32_t dever_rt_v1_text_compare(const void *left, const void *right, int32_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_text_concat(const void *left, const void *right, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_text_length(const void *handle, int64_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_text_trim(const void *handle, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_text_lower(const void *handle, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_text_upper(const void *handle, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_text_codepoint(const void *handle, int64_t *out, uint8_t *present, DeverRtBuffer *error); +uint32_t dever_rt_v1_text_from_codepoint(int64_t value, void **out, uint8_t *present, DeverRtBuffer *error); +uint32_t dever_rt_v1_text_at(const void *handle, int64_t index, void **out, uint8_t *present, DeverRtBuffer *error); +uint32_t dever_rt_v1_text_slice(const void *handle, int64_t start, int64_t end, void **out, uint8_t *present, DeverRtBuffer *error); +uint32_t dever_rt_v1_text_index_of(const void *handle, const void *pattern, int64_t *out, uint8_t *present, DeverRtBuffer *error); +uint32_t dever_rt_v1_text_contains(const void *handle, const void *pattern, uint8_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_text_starts_with(const void *handle, const void *pattern, uint8_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_text_ends_with(const void *handle, const void *pattern, uint8_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_text_replace(const void *handle, const void *pattern, const void *replacement, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_text_split(const void *handle, const void *separator, const DeverRtType *text_type, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_int_parse(const void *text, int64_t *out, uint8_t *present, DeverRtBuffer *error); +uint32_t dever_rt_v1_float_parse(const void *text, double *out, uint8_t *present, DeverRtBuffer *error); + +void *dever_rt_v1_bytes_retain(const void *handle); +void dever_rt_v1_bytes_release(void *handle); +uint32_t dever_rt_v1_bytes_new(const uint8_t *input, uint64_t len, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_bytes_export(const void *handle, DeverRtBuffer *buffer); +uint8_t dever_rt_v1_bytes_equal(const void *left, const void *right); +uint32_t dever_rt_v1_bytes_length(const void *handle, int64_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_bytes_at(const void *handle, int64_t index, int64_t *out, uint8_t *present, DeverRtBuffer *error); +uint32_t dever_rt_v1_bytes_concat(const void *left, const void *right, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_bytes_slice(const void *handle, int64_t start, int64_t end, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_bytes_from_text(const void *text, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_bytes_to_text_handle(const void *bytes, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_bytes_from_ints(const void *list, void **out, DeverRtBuffer *error); + +/* Arrays of element pointers are borrowed for the duration of construction. + * count=0 accepts NULL arrays. map_new preserves insertion order and rejects + * duplicate keys exactly as the existing Map::new implementation does. + */ +void *dever_rt_v1_list_retain(const void *handle); +void dever_rt_v1_list_release(void *handle); +uint32_t dever_rt_v1_list_new(const DeverRtType *element_type, const void *const *values, uint64_t count, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_list_append_take(void *list, const void *element, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_list_first_take(void *list, void *out_element, uint8_t *present, DeverRtBuffer *error); +uint32_t dever_rt_v1_list_length(const void *list, int64_t *out, DeverRtBuffer *error); +uint8_t dever_rt_v1_list_equal(const void *left, const void *right); +uint32_t dever_rt_v1_list_cursor_take(void *list, void **out_cursor, DeverRtBuffer *error); +uint32_t dever_rt_v1_list_cursor_next(void *cursor, void *out_element, uint8_t *present, DeverRtBuffer *error); +void dever_rt_v1_list_cursor_release(void *cursor); + +void *dever_rt_v1_map_retain(const void *handle); +void dever_rt_v1_map_release(void *handle); +uint32_t dever_rt_v1_map_new(const DeverRtType *key_type, const DeverRtType *value_type, const void *const *keys, const void *const *values, uint64_t count, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_map_put_take(void *map, const void *key, const void *value, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_map_remove_take(void *map, const void *key, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_map_get_take(void *map, const void *key, void *out_value, uint8_t *present, DeverRtBuffer *error); +uint32_t dever_rt_v1_map_length(const void *map, int64_t *out, DeverRtBuffer *error); +uint8_t dever_rt_v1_map_equal(const void *left, const void *right); +uint32_t dever_rt_v1_map_cursor_take(void *map, void **out_cursor, DeverRtBuffer *error); +uint32_t dever_rt_v1_map_cursor_next(void *cursor, void *out_key, void *out_value, uint8_t *present, DeverRtBuffer *error); +void dever_rt_v1_map_cursor_release(void *cursor); + +/* Async values have one concrete compiler-emitted owner. move_init transfers + * an initialized source into uninitialized destination storage; the caller + * disarms the source after the callback. Neither callback may unwind. + */ +typedef struct { + uint64_t size; + uint64_t align; + void (*move_init)(void *source, void *destination); + void (*drop)(void *value); + uint8_t transferable; +} DeverRtOwnedType; + +typedef struct DeverRtPollState DeverRtPollState; +typedef struct DeverRtAsyncOp DeverRtAsyncOp; +typedef struct DeverRtAsyncTask DeverRtAsyncTask; +typedef struct DeverRtAsyncGroup DeverRtAsyncGroup; +typedef struct DeverRtAsyncChannel DeverRtAsyncChannel; + +/* create clones borrowed inputs into the coroutine frame before its initial + * suspension. The frame exclusively owns those copies until destroy. The + * output/fault slots remain live until the future completes or is destroyed. + * resume cannot retain the current poll context between calls. + */ +typedef struct { + const DeverRtOwnedType *output_type; + const DeverRtOwnedType *fault_type; + void *(*create)(DeverRtPollState *state, const void *input, void *output, void *fault); + void (*resume)(void *frame); + void (*destroy)(void *frame); + uint8_t (*done)(void *frame); + uint8_t send_safe; +} DeverRtAsyncFunction; + +/* invoke is a checked synchronous function; it runs only in the bounded + * blocking pool. Non-null operation creation transfers input to the runtime. + * invoke only borrows that runtime-owned input and must not drop it; the + * runtime releases it after invoke returns or the operation is cancelled. + * Status 0 initializes output, 1 initializes fault, and neither callback + * nor owned-value callback may unwind through this C boundary. + */ +typedef struct { + const DeverRtOwnedType *input_type; + const DeverRtOwnedType *output_type; + const DeverRtOwnedType *fault_type; + uint32_t (*invoke)(void *input, void *output, void *fault); + uint8_t send_safe; +} DeverRtSyncFunction; + +enum { + DEVER_RT_ASYNC_PENDING = 0, + DEVER_RT_ASYNC_READY_OK = 1, + DEVER_RT_ASYNC_READY_FAULT = 2, + DEVER_RT_ASYNC_READY_RUNTIME_ERROR = 3, + DEVER_RT_ASYNC_INVALID_INPUT = 4, +}; + +/* root starts the one bounded task Scope. Completion status is reported by + * the coroutine through complete; only the matching ready output is moved. + * The caller owns output/fault after a matching ready status and error bytes + * after a runtime error. No output is initialized on Pending or invalid input. + * op_poll only borrows its operation until op_release. A ready OK moves its + * concrete output into an uninitialized slot; a ready typed fault moves the + * concrete fault into an uninitialized slot. For receive, present separately + * describes Channel exhaustion, even when the element itself is Nullable. + * Pending initializes none of these slots. The error buffer is reset on each + * poll and must be freed with dever_rt_v1_buffer_free when non-null. + * + * Task and Group retain create another wrapper owner, not another task or + * group. wait/stop consume one wrapper owner and take the inner affine value + * exactly once; release consumes one wrapper owner without waiting. Group run + * borrows a wrapper; Channel retain clones an alias to the same queue. A + * non-null Channel send operation moves the input value and disarms its old + * slot before polling, including if delivery later fails or is cancelled. + */ +uint32_t dever_rt_v1_async_root(const DeverRtAsyncFunction *function, + const void *input, void *output, void *fault, + DeverRtBuffer *error); +void dever_rt_v1_async_complete(DeverRtPollState *state, uint32_t status); +DeverRtAsyncOp *dever_rt_v1_async_sleep(int64_t milliseconds); +DeverRtAsyncOp *dever_rt_v1_async_call(const DeverRtAsyncFunction *function, + const void *input, DeverRtBuffer *error); +DeverRtAsyncOp *dever_rt_v1_async_blocking(const DeverRtSyncFunction *function, + void *input, DeverRtBuffer *error); +DeverRtAsyncOp *dever_rt_v1_async_parallel(const DeverRtSyncFunction *function, + void *input, DeverRtBuffer *error); +uint32_t dever_rt_v1_async_op_poll(DeverRtPollState *state, DeverRtAsyncOp *operation, + void *output, uint8_t *present, void *fault, + DeverRtBuffer *error); +void dever_rt_v1_async_op_release(DeverRtAsyncOp *operation); +DeverRtAsyncOp *dever_rt_v1_async_task_run(const DeverRtAsyncFunction *function, + const void *input, DeverRtBuffer *error); +DeverRtAsyncOp *dever_rt_v1_async_task_run_sync(const DeverRtSyncFunction *function, + void *input, DeverRtBuffer *error); +DeverRtAsyncOp *dever_rt_v1_async_task_wait(DeverRtAsyncTask *task); +DeverRtAsyncOp *dever_rt_v1_async_task_stop(DeverRtAsyncTask *task); +DeverRtAsyncTask *dever_rt_v1_async_task_retain(const DeverRtAsyncTask *task); +void dever_rt_v1_async_task_release(DeverRtAsyncTask *task); +uint32_t dever_rt_v1_async_group_new(int64_t limit, DeverRtAsyncGroup **output, + DeverRtBuffer *error); +DeverRtAsyncOp *dever_rt_v1_async_group_run(DeverRtAsyncGroup *group, + const DeverRtAsyncFunction *function, + const void *input, DeverRtBuffer *error); +DeverRtAsyncOp *dever_rt_v1_async_group_run_sync(DeverRtAsyncGroup *group, + const DeverRtSyncFunction *function, + void *input, DeverRtBuffer *error); +DeverRtAsyncOp *dever_rt_v1_async_group_wait(DeverRtAsyncGroup *group); +DeverRtAsyncOp *dever_rt_v1_async_group_stop(DeverRtAsyncGroup *group); +DeverRtAsyncGroup *dever_rt_v1_async_group_retain(const DeverRtAsyncGroup *group); +void dever_rt_v1_async_group_release(DeverRtAsyncGroup *group); +uint32_t dever_rt_v1_async_channel_new(int64_t capacity, DeverRtAsyncChannel **output, + DeverRtBuffer *error); +DeverRtAsyncChannel *dever_rt_v1_async_channel_retain(const DeverRtAsyncChannel *channel); +void dever_rt_v1_async_channel_release(DeverRtAsyncChannel *channel); +DeverRtAsyncOp *dever_rt_v1_async_channel_send(DeverRtAsyncChannel *channel, + const DeverRtOwnedType *element_type, + void *element, DeverRtBuffer *error); +DeverRtAsyncOp *dever_rt_v1_async_channel_receive(DeverRtAsyncChannel *channel); +DeverRtAsyncOp *dever_rt_v1_async_channel_close(DeverRtAsyncChannel *channel); +/* Stream constructors borrow their source. Descriptors describe the exact + * element and must be transferable before a producer may cross a thread. + * All aliases share one cursor and close state; pull uses the separate present + * slot (including Nullable rows), and emits a terminal Failed exactly once. + */ +uint32_t dever_rt_v1_async_stream_from_list(const void *list, const DeverRtOwnedType *element_type, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_async_stream_from_bytes(const void *bytes, const DeverRtOwnedType *int_type, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_async_ticks(int64_t milliseconds, const DeverRtOwnedType *int_type, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_async_channel_stream(const DeverRtAsyncChannel *channel, void **out, DeverRtBuffer *error); +void *dever_rt_v1_async_stream_retain(const void *stream); +void dever_rt_v1_async_stream_release(void *stream); +DeverRtAsyncOp *dever_rt_v1_async_stream_pull(const void *stream); +uint32_t dever_rt_v1_async_stream_close(const void *stream, uint8_t *out, DeverRtBuffer *error); + +/* TCP operations reuse the runtime's independent read/write locks, deadlines + * and cancellation. Async operation output is Socket, Listener, Socket, + * optional Bytes, or Bool respectively. String I/O errors use async status 3. + * chunks/connections callbacks initialize the original full event Choice; + * connections uses chunk for the borrowed accepted Socket handle. + */ +DeverRtAsyncOp *dever_rt_v1_async_net_connect(const void *host, int64_t port); +DeverRtAsyncOp *dever_rt_v1_async_net_connect_timeout(const void *host, int64_t port, int64_t milliseconds); +DeverRtAsyncOp *dever_rt_v1_async_net_listen(const void *host, int64_t port); +DeverRtAsyncOp *dever_rt_v1_async_net_accept(const void *listener); +DeverRtAsyncOp *dever_rt_v1_async_net_read(const void *socket, int64_t limit); +DeverRtAsyncOp *dever_rt_v1_async_net_write(const void *socket, const void *bytes); +void *dever_rt_v1_socket_retain(const void *socket); +void dever_rt_v1_socket_release(void *socket); +void *dever_rt_v1_listener_retain(const void *listener); +void dever_rt_v1_listener_release(void *listener); +uint32_t dever_rt_v1_net_port(const void *listener, int64_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_net_timeout(const void *socket, int64_t milliseconds, uint8_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_socket_close(const void *socket, uint8_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_listener_close(const void *listener, uint8_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_net_chunks(const void *socket, int64_t limit, const DeverRtReadEvent *event, const DeverRtOwnedType *element_type, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_net_connections(const void *listener, const DeverRtReadEvent *event, const DeverRtOwnedType *element_type, void **out, DeverRtBuffer *error); + +/* wait_timeout consumes one task wrapper, returns the entire task output row + * with present=1, or present=0 after cancellation and descendant draining. + * race requires distinct unconsumed tasks of the exact same output type. A + * non-null result consumes every supplied wrapper; NULL consumes none and + * reports a creation error. Typed faults remain intact through both paths. + */ +DeverRtAsyncOp *dever_rt_v1_async_task_wait_timeout(DeverRtAsyncTask *task, int64_t milliseconds); +DeverRtAsyncOp *dever_rt_v1_async_task_race(DeverRtAsyncTask *const *tasks, uint64_t count, DeverRtBuffer *error); + +/* Exactly one function is present. pack clones borrowed element/context into + * one complete input row. A null context_type means no context. Descriptors + * and callbacks are process-static; send_safe covers packed inputs/context. + * Source kinds: 0 List, 1 Bytes, 2 synchronous Stream, 3 AsyncStream. + * The synchronous entry accepts only a synchronous handler and kinds 0..2; + * the async entry accepts kinds 0,1,3. Both preserve runtime dispatch policy. + * Synchronous completion also uses ASYNC_READY_* statuses (no Pending). + */ +typedef struct { + const DeverRtAsyncFunction *async_function; + const DeverRtSyncFunction *sync_function; + const DeverRtOwnedType *input_type; + const DeverRtType *context_type; + void (*pack)(const void *element, const void *context, void *out_input); +} DeverRtParallelFunction; +DeverRtAsyncOp *dever_rt_v1_async_parallel_each(const void *source, uint32_t source_kind, int64_t limit, const DeverRtParallelFunction *function, const void *context, DeverRtBuffer *error); +uint32_t dever_rt_v1_parallel_each(const void *source, uint32_t source_kind, int64_t limit, const DeverRtParallelFunction *function, const void *context, void *fault, DeverRtBuffer *error); +/* Successful allocation is non-null and must be paired with frame_free using + * the exact coro.size/coro.align. Invalid layout or OOM terminates the process + * before coro.begin can observe a null allocation. + */ +void *dever_rt_v1_async_frame_alloc(uint64_t size, uint64_t align); +void dever_rt_v1_async_frame_free(void *frame, uint64_t size, uint64_t align); + +/* File aliases share one close state. create uses create-new and never replaces + * an existing path. File read returns absent at EOF. A stream alias shares one + * cursor; pull returns the terminal Failed event once, then absent. Closing a + * Stream is idempotent; closing a File twice returns the runtime error text. + */ +void *dever_rt_v1_file_retain(const void *handle); +void dever_rt_v1_file_release(void *handle); +uint32_t dever_rt_v1_file_open(const void *path, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_file_create(const void *path, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_file_read(const void *file, int64_t limit, void **out_bytes, uint8_t *present, DeverRtBuffer *error); +uint32_t dever_rt_v1_file_write(const void *file, const void *bytes, uint8_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_file_close(const void *file, uint8_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_file_chunks(const void *file, int64_t limit, const DeverRtReadEvent *event, void **out_stream, DeverRtBuffer *error); +void *dever_rt_v1_stream_retain(const void *handle); +void dever_rt_v1_stream_release(void *handle); +uint32_t dever_rt_v1_stream_pull(const void *stream, void *out_element, uint8_t *present, DeverRtBuffer *error); +uint32_t dever_rt_v1_stream_close(const void *stream, uint8_t *out, DeverRtBuffer *error); + +uint32_t dever_rt_v1_time_unix_millis(int64_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_time_now(int64_t *out, DeverRtBuffer *error); +/* Invocation-owned Clock: virtual time affects Job scheduling and drain only. */ +uint32_t dever_rt_v1_job_clock_new(uint8_t test, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_job_clock_now(const void *clock, int64_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_job_clock_advance(const void *clock, int64_t millis, DeverRtBuffer *error); +void dever_rt_v1_job_clock_release(void *clock); +uint32_t dever_rt_v1_time_monotonic_nanos(int64_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_time_parse_datetime(const void *text, int64_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_time_format_datetime(int64_t value, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_time_parse_date(const void *text, int64_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_time_format_date(int64_t value, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_time_parse_time(const void *text, int64_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_time_format_time(int64_t value, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_time_add(int64_t left, int64_t right, int64_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_time_subtract(int64_t left, int64_t right, int64_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_time_difference(int64_t left, int64_t right, int64_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_crypto_token(int64_t length, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_crypto_password_hash(const void *password, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_crypto_password_verify(const void *password, const void *encoded, uint8_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_crypto_sha256(const void *bytes, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_crypto_hmac_sha256(const void *secret, const void *bytes, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_crypto_constant_time_eq(const void *left, const void *right, uint8_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_secret_from_text(const void *text, void **out, DeverRtBuffer *error); +void *dever_rt_v1_uuid_retain(const void *handle); +void dever_rt_v1_uuid_release(void *handle); +uint8_t dever_rt_v1_uuid_equal(const void *left, const void *right); +uint32_t dever_rt_v1_uuid_parse(const void *text, void **out, uint8_t *present, DeverRtBuffer *error); +uint32_t dever_rt_v1_uuid_to_text(const void *uuid, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_process_arguments(const DeverRtType *text_type, void **out, DeverRtBuffer *error); +/* Terminal entry adapters: stderr borrows bytes; test_index consumes no argv + * storage and writes out only for exactly one unsigned decimal index < count. */ +/* Call once from process main before threads; preserves EPIPE diagnostics on + * Unix. Embedded callable roots must not alter their host's signal policy. */ +uint32_t dever_rt_v1_process_init(void); +uint32_t dever_rt_v1_process_stderr(const uint8_t *bytes, uint64_t len); +uint32_t dever_rt_v1_process_test_index(int32_t argc, const char *const *argv, uint64_t count, uint64_t *out); +/* Borrow Text and report through the same configured logger as application errors. */ +uint32_t dever_rt_v1_process_error(const void *text); +uint32_t dever_rt_v1_stdout_write(const void *text, uint8_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_log_write(uint32_t level, const void *message, const void *fields, uint8_t *out, DeverRtBuffer *error); +void dever_rt_v1_log_flush(void); + +/* Protocol records have canonical layouts independent of source declaration order. + * Inputs borrow handles. Field getters return owned handles. Builders clone inputs. + * All asynchronous protocol operations use async_op_poll; success is one handle + * (response/client/socket), Bool byte for mutations, or Unit for servers. + * Poll validates protocol output slots before advancing I/O. Invalid slots + * return status 4 without completing or polling the operation; retry is allowed. + */ +typedef struct { int64_t streams, stream_window_bytes, connection_window_bytes; } DeverRtHttp2Limits; +typedef struct { int64_t header_bytes, body_bytes, timeout_ms, connections; const DeverRtHttp2Limits *http2; } DeverRtHttpLimits; +typedef struct { int64_t idle_ms, chunk_bytes, read_ms; } DeverRtHttpPoolLimits; +typedef struct { int64_t chunk_bytes, idle_ms, heartbeat_ms; } DeverRtHttpLiveLimits; +typedef struct { int64_t message_bytes, idle_ms, write_ms; } DeverRtWsLimits; +typedef struct { void *method, *target, *headers, *body; } DeverRtHttpRequestFields; +typedef struct { int64_t status; void *headers, *body; } DeverRtHttpResponseFields; +typedef struct { void *name, *value; } DeverRtHttpHeaderFields; +typedef struct { uint32_t kind; void *payload; } DeverRtWsMessageFields; /* Text=0 Binary=1 Ping=2 Pong=3 */ +typedef struct { const void *event, *data, *id; int64_t retry_ms; uint8_t has_retry; } DeverRtSseEvent; +#define DEVER_RT_PROTOCOL_OWNER(name) void *dever_rt_v1_##name##_retain(const void *handle); void dever_rt_v1_##name##_release(void *handle) +DEVER_RT_PROTOCOL_OWNER(http_headers); +DEVER_RT_PROTOCOL_OWNER(http_request); +DEVER_RT_PROTOCOL_OWNER(http_response); +DEVER_RT_PROTOCOL_OWNER(http_stream_request); +DEVER_RT_PROTOCOL_OWNER(http_stream_response); +DEVER_RT_PROTOCOL_OWNER(client_tls); +DEVER_RT_PROTOCOL_OWNER(server_tls); +DEVER_RT_PROTOCOL_OWNER(http_client); +DEVER_RT_PROTOCOL_OWNER(http_reply); +DEVER_RT_PROTOCOL_OWNER(websocket); +DEVER_RT_PROTOCOL_OWNER(ws_message); +DEVER_RT_PROTOCOL_OWNER(secret); +#undef DEVER_RT_PROTOCOL_OWNER +uint32_t dever_rt_v1_http_headers_new(void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_http_headers_push(void *headers, const void *name, const void *value, DeverRtBuffer *error); +uint32_t dever_rt_v1_http_headers_length(const void *headers, int64_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_http_headers_get(const void *headers, int64_t index, DeverRtHttpHeaderFields *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_http_request_new(const DeverRtHttpRequestFields *fields, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_http_request_fields(const void *request, DeverRtHttpRequestFields *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_http_response_new(const DeverRtHttpResponseFields *fields, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_http_response_fields(const void *response, DeverRtHttpResponseFields *out, DeverRtBuffer *error); +/* Stream upload decoder borrows source event; returns 0 Chunk or 1 Failed and + * initializes an owned Bytes/Text handle. Download uses existing ReadEvent callbacks. */ +uint32_t dever_rt_v1_http_stream_request_new(const DeverRtHttpRequestFields *fields, uint32_t (*decode)(const void *, void **), void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_http_stream_response_fields(const void *response, const DeverRtReadEvent *event, const DeverRtOwnedType *type, DeverRtHttpResponseFields *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_tls_system(void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_tls_client(const void *ca, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_tls_server(const void *cert, const void *key, void **out, DeverRtBuffer *error); +DeverRtAsyncOp *dever_rt_v1_async_http_send(const void *host, int64_t port, const void *request, const DeverRtHttpLimits *limits); +DeverRtAsyncOp *dever_rt_v1_async_http_client(const void *origin, const void *tls, const DeverRtHttpLimits *limits, const DeverRtHttpPoolLimits *pool); +DeverRtAsyncOp *dever_rt_v1_async_http_request(const void *client, const void *request); +DeverRtAsyncOp *dever_rt_v1_async_http_open(const void *client, const void *request); +DeverRtAsyncOp *dever_rt_v1_async_http_open_stream(const void *client, const void *request); +DeverRtAsyncOp *dever_rt_v1_async_http_close_client(const void *client); +typedef struct { + const DeverRtAsyncFunction *asynchronous; + const DeverRtSyncFunction *synchronous; + const DeverRtOwnedType *input_type; + const DeverRtType *context_type; + uint32_t (*pack)(const void *request, const void *reply, const void *context, void *out, DeverRtBuffer *error); + uint32_t (*response)(const void *output, void **out, DeverRtBuffer *error); + uint32_t (*fault_message)(const void *fault, void **out, DeverRtBuffer *error); +} DeverRtHttpHandler; +/* live=null means buffered; tls=null means plaintext. Context is cloned before + * returning. Handler faults remain typed until the terminal request-log callback. */ +DeverRtAsyncOp *dever_rt_v1_async_http_serve(const void *listener, const DeverRtHttpLimits *limits, const DeverRtHttpLiveLimits *live, const void *tls, const DeverRtHttpHandler *handler, const void *context, DeverRtBuffer *error); +DeverRtAsyncOp *dever_rt_v1_async_http_respond(const void *reply, const void *response); +DeverRtAsyncOp *dever_rt_v1_async_http_start(const void *reply, int64_t status, const void *headers); +DeverRtAsyncOp *dever_rt_v1_async_http_write(const void *reply, const void *bytes); +DeverRtAsyncOp *dever_rt_v1_async_http_finish(const void *reply); +DeverRtAsyncOp *dever_rt_v1_async_sse_start(const void *reply, const void *headers); +DeverRtAsyncOp *dever_rt_v1_async_sse_send(const void *reply, const DeverRtSseEvent *event); +DeverRtAsyncOp *dever_rt_v1_async_ws_accept(const void *reply, const DeverRtWsLimits *limits); +DeverRtAsyncOp *dever_rt_v1_async_ws_connect(const void *host, int64_t port, const void *target, const DeverRtWsLimits *limits); +DeverRtAsyncOp *dever_rt_v1_async_ws_open(const void *url, const void *tls, const DeverRtWsLimits *limits); +uint32_t dever_rt_v1_ws_message_new(uint32_t kind, const void *payload, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_ws_message_fields(const void *message, DeverRtWsMessageFields *out, DeverRtBuffer *error); +DeverRtAsyncOp *dever_rt_v1_async_ws_send(const void *socket, const void *message); +/* receive success writes a message handle only when present=1. */ +DeverRtAsyncOp *dever_rt_v1_async_ws_receive(const void *socket); +typedef struct { + const DeverRtType *element_type; + void (*read)(uint32_t kind, const void *payload, void *out); + void (*failed)(const void *text, void *out); +} DeverRtWsEvent; +uint32_t dever_rt_v1_ws_messages(const void *socket, const DeverRtWsEvent *event, const DeverRtOwnedType *type, void **out, DeverRtBuffer *error); +DeverRtAsyncOp *dever_rt_v1_async_ws_close(const void *socket, int64_t code, const void *reason); + +/* Synchronous typed wire codecs. Nodes and the encoder are borrowed only for + * the enclosing callback, including child nodes returned by field/list getters. + * They have no retain/release operation and must never enter a business value, + * coroutine frame or other persistent storage. The concrete type is process- + * static; decode validates its destination before parsing or calling the codec. + * A decoder initializes out only on success and cleans every partial owner on + * failure. An encoder only borrows its concrete input. Callbacks return regular + * status 0/1/2 and own any error Buffer they return, following the common ABI. + * Text/Uuid/Secret getters return normal owned handles; raw returns exact JSON. + * Names are borrowed UTF-8 byte slices. Missing optional fields set present=0 + * without initializing out; required fields use the original wire diagnostic. + */ +typedef struct { const uint8_t *ptr; uint64_t len; } DeverRtWireName; + +/* runtime-external: compiler metadata is copied into one invocation owner. + * Resource paths/digests/bytes are immutable for the process lifetime; the + * descriptor arrays themselves are borrowed only during each call. No host + * runtime or PATH search is performed. Every invocation stops Workers before + * cancelling its remaining tasks, then closes application/database resources. + */ +typedef struct { + DeverRtWireName key, ecosystem, entry, port, adapter, schema; + const DeverRtWireName *capabilities; + uint64_t capability_count; + const DeverRtWireName *operations; + uint64_t operation_count; + uint64_t timeout_ms; +} DeverRtExternalDefinition; +typedef struct { + DeverRtWireName path; + const uint8_t *bytes; + uint64_t len; + DeverRtWireName sha256; + uint8_t executable; +} DeverRtExternalResource; +uint32_t dever_rt_v1_external_prepare(const DeverRtWireName *digest, + const DeverRtExternalResource *resources, + uint64_t count, DeverRtBuffer *error); +/* start borrows an optional Text setting. call borrows three Text handles. + * Async poll uses the existing statuses; call success owns one Reply handle. + */ +DeverRtAsyncOp *dever_rt_v1_external_start(const DeverRtExternalDefinition *definition, + const void *setting, DeverRtBuffer *error); +DeverRtAsyncOp *dever_rt_v1_external_call(const void *key, const void *operation, + const void *payload, DeverRtBuffer *error); +/* Consumes Reply even on failure. Success owns identity and payload Texts. + * kind=0 is a result (empty identity), kind=1 is a declared-error candidate; + * the concrete compiler decoder still validates its exact identity/payload. + */ +uint32_t dever_rt_v1_external_reply_take(void *reply, uint8_t *kind, + void **identity, void **payload, + DeverRtBuffer *error); +void dever_rt_v1_external_reply_release(void *reply); +DeverRtAsyncOp *dever_rt_v1_external_shutdown(void); +void dever_rt_v1_external_cause_append(void **cause, const void *message); +uint32_t dever_rt_v1_external_root(const DeverRtAsyncFunction *function, + const void *input, void *output, void *fault, + void (*append_cause)(void *, const void *), + DeverRtBuffer *error); +typedef uint32_t (*DeverRtWireDecode)(const void *node, void *out, DeverRtBuffer *error); +typedef uint32_t (*DeverRtWireEncode)(const void *value, void *encoder, DeverRtBuffer *error); +uint32_t dever_rt_v1_wire_decode(const void *text, const DeverRtType *type, DeverRtWireDecode decode, void *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_wire_encode(const void *value, const DeverRtType *type, DeverRtWireEncode encode, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_wire_node_is_null(const void *node, uint8_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_wire_node_bool(const void *node, uint8_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_wire_node_int(const void *node, int64_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_wire_node_float(const void *node, double *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_wire_node_text(const void *node, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_wire_node_bytes(const void *node, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_wire_node_raw(const void *node, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_wire_node_decimal(const void *node, DeverRtDecimal *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_wire_node_uuid(const void *node, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_wire_node_secret(const void *node, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_wire_node_datetime(const void *node, int64_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_wire_node_date(const void *node, int64_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_wire_node_time(const void *node, int64_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_wire_node_fields(const void *node, const DeverRtWireName *names, uint64_t count, DeverRtBuffer *error); +uint32_t dever_rt_v1_wire_node_field(const void *node, const DeverRtWireName *name, uint8_t required, const void **out, uint8_t *present, DeverRtBuffer *error); +uint32_t dever_rt_v1_wire_node_list_len(const void *node, int64_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_wire_node_list_at(const void *node, int64_t index, const void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_wire_encoder_null(void *encoder, DeverRtBuffer *error); +uint32_t dever_rt_v1_wire_encoder_bool(void *encoder, uint8_t value, DeverRtBuffer *error); +uint32_t dever_rt_v1_wire_encoder_int(void *encoder, int64_t value, DeverRtBuffer *error); +uint32_t dever_rt_v1_wire_encoder_float(void *encoder, double value, DeverRtBuffer *error); +uint32_t dever_rt_v1_wire_encoder_text(void *encoder, const void *text, DeverRtBuffer *error); +uint32_t dever_rt_v1_wire_encoder_bytes(void *encoder, const void *bytes, DeverRtBuffer *error); +uint32_t dever_rt_v1_wire_encoder_json(void *encoder, const void *text, DeverRtBuffer *error); +uint32_t dever_rt_v1_wire_encoder_decimal(void *encoder, uint64_t low, uint64_t high, DeverRtBuffer *error); +uint32_t dever_rt_v1_wire_encoder_uuid(void *encoder, const void *uuid, DeverRtBuffer *error); +uint32_t dever_rt_v1_wire_encoder_datetime(void *encoder, int64_t value, DeverRtBuffer *error); +uint32_t dever_rt_v1_wire_encoder_date(void *encoder, int64_t value, DeverRtBuffer *error); +uint32_t dever_rt_v1_wire_encoder_time(void *encoder, int64_t value, DeverRtBuffer *error); +uint32_t dever_rt_v1_wire_encoder_begin_array(void *encoder, DeverRtBuffer *error); +uint32_t dever_rt_v1_wire_encoder_begin_object(void *encoder, DeverRtBuffer *error); +uint32_t dever_rt_v1_wire_encoder_key(void *encoder, const DeverRtWireName *name, DeverRtBuffer *error); +uint32_t dever_rt_v1_wire_encoder_end(void *encoder, DeverRtBuffer *error); + +/* CMD top-level input uses the existing Inputs contract: fields are consumed + * in checked parameter order, with unknown names rejected after decoding. + * Inputs is an exclusive callback-scoped borrow and has no owner operations. + * cmd_input_field returns owned raw-JSON Text; it validates all output slots + * before removing a field, so a rejected call may be retried. cmd_decode drops + * a fully initialized concrete output if the final unknown-field check fails. + * Partial output cleanup on callback failure remains the decoder's duty. */ +typedef uint32_t (*DeverRtCmdDecode)(void *inputs, void *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_cmd_decode(const void *text, const DeverRtType *type, DeverRtCmdDecode decode, void *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_cmd_input_field(void *inputs, const DeverRtWireName *name, uint8_t required, void **out, uint8_t *present, DeverRtBuffer *error); + +/* Settings are private entry owners, loaded beside the executable through the + * existing setting.json contract. Select validates every output before lookup; + * only present=1 initializes the owned setting Text. No unselected Setting is + * decoded here. The caller releases Settings on every entry exit path. */ +uint32_t dever_rt_v1_adapter_settings_load(void **out, DeverRtBuffer *error); +void dever_rt_v1_adapter_settings_release(const void *settings); +uint32_t dever_rt_v1_adapter_settings_validate_names(const void *settings, const DeverRtWireName *names, uint64_t count, DeverRtBuffer *error); +uint32_t dever_rt_v1_adapter_settings_select(const void *settings, const DeverRtWireName *identity, const DeverRtWireName *candidates, uint64_t count, int64_t *index, void **setting, uint8_t *present, DeverRtBuffer *error); + +/* Database support is independently bundled with runtime-database and the + * selected runtime-sqlite/runtime-postgres drivers. Bind rows below are the + * existing database scalar codec, never source-language business values. + * All arrays/handles are borrowed through operation creation and copied there. + * SQL byte strings alone are immutable process-static compiler output. + * Regular database calls return 0 success, 1 initialized typed fault, 2 ABI + * error. Async operations use the ordinary async 0/1/2/3/4 contract. Every + * destination, including the fault slot, is checked before advancing work. + */ +typedef struct { + const DeverRtOwnedType *fault_type; + void (*pack)(uint32_t kind, const void *message, void *out_fault); + void (*append_cause)(void *fault, const void *message); +} DeverRtDbError; +/* Error kinds: Pool=0, PoolExhausted=1, Connection=2, Timeout=3, Cancelled=4, + * Database=5, Constraint=6, NotFound=7, InvalidData=8, Migration=9. */ +typedef struct { DeverRtWireName sqlite, postgres; } DeverRtDbSql; +/* kinds: Null=0, Bool=1, Int=2, Float=3, Decimal=4, Text=5, Bytes=6, Uuid=7. + * Bool uses integer 0/1; Decimal uses precision/scale plus exact bits; the + * three handle kinds borrow their normal runtime owners. Other fields ignored. */ +typedef struct { + uint32_t kind, precision, scale; + int64_t integer; + double floating; + DeverRtDecimal decimal; + const void *handle; +} DeverRtDbValue; +typedef struct { + const DeverRtWireName *explicit_name; + DeverRtWireName package_root; + uint8_t tenant_scoped; +} DeverRtDbBinding; +typedef struct { const DeverRtDbBinding *bindings; uint64_t count; } DeverRtDbTransactionBindings; +typedef struct { + DeverRtWireName name, type; + uint8_t nullable, generated; + const DeverRtWireName *default_value, *rename_from; +} DeverRtDbField; +typedef struct { + DeverRtWireName name; + const DeverRtWireName *fields; + uint64_t count; + uint8_t unique; +} DeverRtDbIndex; +typedef struct { + DeverRtWireName name, revision; + const DeverRtWireName *drops; + uint64_t count; +} DeverRtDbMigration; +typedef struct { + DeverRtDbSql sql; + const DeverRtDbValue *parameters; + uint64_t count; +} DeverRtDbSeed; +typedef struct { + DeverRtWireName name; + uint32_t phase; /* Before=0, After=1 */ + DeverRtDbSeed statement; +} DeverRtDbDataMigration; +typedef struct { + DeverRtWireName name, definition, sql_type; + const DeverRtWireName *default_value; +} DeverRtDbPostgresColumn; +typedef struct { DeverRtWireName name, definition; uint8_t foreign_key; } DeverRtDbPostgresConstraint; +typedef struct { + DeverRtWireName model, table, revision, seed_revision; + const DeverRtDbField *fields; uint64_t field_count; + const DeverRtDbIndex *indexes; uint64_t index_count; + const DeverRtDbMigration *migrations; uint64_t migration_count; + DeverRtDbSql create_table; + DeverRtWireName create_temporary_table; + const DeverRtDbSql *create_indexes; uint64_t create_index_count; + const DeverRtDbPostgresColumn *postgres_columns; uint64_t postgres_column_count; + const DeverRtDbPostgresConstraint *postgres_constraints; uint64_t postgres_constraint_count; + const DeverRtDbSeed *seeds; uint64_t seed_count; + const DeverRtDbDataMigration *data_migrations; uint64_t data_migration_count; +} DeverRtDbModel; +typedef struct { + const DeverRtOwnedType *row_type; + uint32_t (*decode)(const void *row, void *out, const DeverRtDbError *errors, void *fault, DeverRtBuffer *error); +} DeverRtDbRowDecoder; +/* Application root closes the published session after child Scope drain and + * before runtime destruction. The compiler releases/zeros its session and + * database globals only after this call returns. Cleanup preserves primary + * faults and uses append_cause; callbacks may not fail or unwind. */ +uint32_t dever_rt_v1_db_application_root(const DeverRtAsyncFunction *function, const void *input, void *out, void *fault, void *const *session_slot, const DeverRtDbError *errors, DeverRtBuffer *error); +void dever_rt_v1_db_cause_append(void **cause_text, const void *message); +uint32_t dever_rt_v1_db_session_new(const DeverRtDbBinding *bindings, uint64_t count, const DeverRtDbTransactionBindings *transactions, uint64_t transaction_count, void **out, DeverRtBuffer *error); +void dever_rt_v1_db_session_release(void *session); +DeverRtAsyncOp *dever_rt_v1_db_prepare(const void *session, const DeverRtDbError *errors, DeverRtBuffer *error); +uint32_t dever_rt_v1_db_select(const void *session, const DeverRtDbBinding *binding, void **out, const DeverRtDbError *errors, void *fault, DeverRtBuffer *error); +void *dever_rt_v1_db_retain(const void *database); +void dever_rt_v1_db_release(void *database); +uint32_t dever_rt_v1_db_max_page_size(const void *database, int64_t *out, DeverRtBuffer *error); +/* Startup order matches native: stable sort by resolved database name, migrate + * every Model, then install foreign keys. Per-model descriptors retain spans. */ +DeverRtAsyncOp *dever_rt_v1_db_migrate_models(const void *const *databases, const DeverRtDbModel *models, const DeverRtDbError *const *errors, uint64_t count, DeverRtBuffer *error); +DeverRtAsyncOp *dever_rt_v1_db_begin(const void *database, const DeverRtDbError *errors, DeverRtBuffer *error); +void *dever_rt_v1_db_transaction_retain(const void *transaction); +void dever_rt_v1_db_transaction_release(void *transaction); +/* finish takes the transaction's shared inner owner on its first poll. The + * handle itself remains borrowed; success/error/cancellation cannot reuse it. */ +DeverRtAsyncOp *dever_rt_v1_db_finish(const void *transaction, uint8_t commit, const DeverRtDbError *errors, DeverRtBuffer *error); +/* On non-NULL creation, moves the complete primary fault into the operation. + * NULL creation does not consume it. Completion always returns that typed + * fault, augmented only if rollback fails; cancellation drops it once. */ +DeverRtAsyncOp *dever_rt_v1_db_rollback_take(const void *transaction, void *primary_fault, const DeverRtOwnedType *primary_type, const DeverRtDbError *errors, DeverRtBuffer *error); +DeverRtAsyncOp *dever_rt_v1_db_execute(const void *database, const void *transaction, const DeverRtDbSql *sql, const DeverRtDbValue *parameters, uint64_t count, const DeverRtDbError *errors, DeverRtBuffer *error); +/* maximum=0 selects unbounded query; positive maximum reuses query_bounded. */ +DeverRtAsyncOp *dever_rt_v1_db_query(const void *database, const void *transaction, const DeverRtDbSql *sql, const DeverRtDbValue *parameters, uint64_t count, int64_t maximum, const DeverRtDbError *errors, DeverRtBuffer *error); +/* Prefix/before_limit/suffix are process-static SQL fragments. */ +DeverRtAsyncOp *dever_rt_v1_db_relation_query(const void *database, const void *transaction, const DeverRtWireName *prefix, const DeverRtWireName *before_limit, const DeverRtWireName *suffix, const int64_t *ids, uint64_t count, int64_t limit, const DeverRtDbError *errors, DeverRtBuffer *error); +DeverRtAsyncOp *dever_rt_v1_db_stream(const void *database, const void *transaction, const DeverRtDbSql *sql, const DeverRtDbValue *parameters, uint64_t count, int64_t capacity, const DeverRtDbRowDecoder *decoder, const DeverRtDbError *errors, DeverRtBuffer *error); +void *dever_rt_v1_db_stream_retain(const void *stream); +void dever_rt_v1_db_stream_release(void *stream); +uint32_t dever_rt_v1_db_stream_close(const void *stream, uint8_t *out, DeverRtBuffer *error); +DeverRtAsyncOp *dever_rt_v1_db_stream_pull(const void *stream, const DeverRtDbError *errors, DeverRtBuffer *error); +void dever_rt_v1_db_rows_release(void *rows); +uint32_t dever_rt_v1_db_rows_len(const void *rows, int64_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_db_rows_at(const void *rows, int64_t index, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_db_required_row(const void *rows, const DeverRtWireName *operation, void **out, const DeverRtDbError *errors, void *fault, DeverRtBuffer *error); +uint32_t dever_rt_v1_db_optional_row(const void *rows, const DeverRtWireName *operation, void **out, uint8_t *present, const DeverRtDbError *errors, void *fault, DeverRtBuffer *error); +void dever_rt_v1_db_row_release(void *row); +uint32_t dever_rt_v1_db_row_len(const void *row, int64_t *out, DeverRtBuffer *error); +/* Related uses indirection for recursive Model associations. NULL is Unloaded; + * a non-NULL owner contains one concrete Loaded row, including nullable rows. */ +uint32_t dever_rt_v1_db_related_new(const DeverRtType *type, const void *value, void **out, DeverRtBuffer *error); +void *dever_rt_v1_db_related_retain(const void *related); +void dever_rt_v1_db_related_release(void *related); +uint8_t dever_rt_v1_db_related_equal(const void *left, const void *right); +uint32_t dever_rt_v1_db_related_get(const void *related, void *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_db_row_null(const void *row, int64_t index, uint8_t *out, const DeverRtDbError *errors, void *fault, DeverRtBuffer *error); +uint32_t dever_rt_v1_db_row_bool(const void *row, int64_t index, uint8_t *out, const DeverRtDbError *errors, void *fault, DeverRtBuffer *error); +uint32_t dever_rt_v1_db_row_int(const void *row, int64_t index, int64_t *out, const DeverRtDbError *errors, void *fault, DeverRtBuffer *error); +uint32_t dever_rt_v1_db_row_float(const void *row, int64_t index, double *out, const DeverRtDbError *errors, void *fault, DeverRtBuffer *error); +uint32_t dever_rt_v1_db_row_decimal(const void *row, int64_t index, DeverRtDecimal *out, const DeverRtDbError *errors, void *fault, DeverRtBuffer *error); +uint32_t dever_rt_v1_db_row_text(const void *row, int64_t index, void **out, const DeverRtDbError *errors, void *fault, DeverRtBuffer *error); +uint32_t dever_rt_v1_db_row_bytes(const void *row, int64_t index, void **out, const DeverRtDbError *errors, void *fault, DeverRtBuffer *error); +uint32_t dever_rt_v1_db_row_uuid(const void *row, int64_t index, void **out, const DeverRtDbError *errors, void *fault, DeverRtBuffer *error); +uint32_t dever_rt_v1_db_uuid_new(void **out, const DeverRtDbError *errors, void *fault, DeverRtBuffer *error); +uint32_t dever_rt_v1_db_error(uint32_t kind, const void *message, const DeverRtDbError *errors, void *fault, DeverRtBuffer *error); +uint32_t dever_rt_v1_db_pagination(int64_t page, int64_t size, int64_t maximum, int64_t *out_page, int64_t *out_size, int64_t *out_offset, const DeverRtDbError *errors, void *fault, DeverRtBuffer *error); +uint32_t dever_rt_v1_db_cursor_size(int64_t size, int64_t maximum, int64_t *out, const DeverRtDbError *errors, void *fault, DeverRtBuffer *error); +uint32_t dever_rt_v1_db_stream_capacity(int64_t size, int64_t maximum, int64_t *out, const DeverRtDbError *errors, void *fault, DeverRtBuffer *error); +uint32_t dever_rt_v1_db_relation_limit(int64_t maximum, int64_t *out, const DeverRtDbError *errors, void *fault, DeverRtBuffer *error); + +/* Application descriptors are compiler-owned process-static metadata. Decode + * borrows Inputs only for its synchronous call; success initializes a complete + * typed input, failure drops its own partial input. Runtime finish rejects any + * trailing fields before calling the concrete handler. Handler success returns + * exactly one owned Text containing checked JSON. Typed failures remain owned + * by runtime until the static status callback has inspected them. + */ +typedef struct { + const DeverRtAsyncFunction *asynchronous; + const DeverRtSyncFunction *synchronous; + const DeverRtOwnedType *input_type; + uint32_t (*decode)(void *inputs, int64_t detail_id, void *out, DeverRtBuffer *error); + int64_t (*fault_status)(const void *fault); + /* Optional; only status 400 may expose this borrowed Text as an input error. */ + const void *(*fault_message)(const void *fault); +} DeverRtApiHandler; +typedef struct { + const DeverRtAsyncFunction *asynchronous; + const DeverRtSyncFunction *synchronous; + const DeverRtOwnedType *input_type; + uint32_t (*pack)(const void *subject, const void *session, const void *nullable_tenant, const void *site, void *out, DeverRtBuffer *error); + uint32_t (*unpack)(const void *identity, void **id, int64_t *user_id, uint8_t *user_present, int64_t *tenant_id, uint8_t *tenant_present, DeverRtBuffer *error); + int64_t (*fault_status)(const void *fault); +} DeverRtApiAuth; +typedef struct { + DeverRtWireName key, component, domain, site, action, method; +} DeverRtApiPermission; +typedef struct { + DeverRtWireName method, path; + const DeverRtWireName *directory; + uint64_t directory_count; + const DeverRtWireName *components; + uint64_t component_count; + const DeverRtApiPermission *permission; + const DeverRtApiAuth *auth; + const DeverRtApiHandler *handler; + uint8_t anonymous, multipart, detail; +} DeverRtApiRoute; +typedef struct { + const void *path, *domain; + int64_t same_site; /* 0 Strict, 1 Lax, 2 None */ + uint8_t secure, http_only; + int64_t max_age; + uint8_t max_age_present; +} DeverRtApiCookieOptions; +uint32_t dever_rt_v1_api_application_root(const DeverRtAsyncFunction *function, const void *input, void *out, void *fault, void *const *session_slot, void (*append_cause)(void *fault, const void *text), DeverRtBuffer *error); +uint32_t dever_rt_v1_api_session_new(const DeverRtDbBinding *bindings, uint64_t count, const DeverRtDbTransactionBindings *transactions, uint64_t transaction_count, void **out, DeverRtBuffer *error); +void dever_rt_v1_api_session_release(void *session); +uint32_t dever_rt_v1_api_session_database(const void *session, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_api_has_tenant(const void *session, uint8_t *out, DeverRtBuffer *error); +DeverRtAsyncOp *dever_rt_v1_api_initialize(const void *session, const DeverRtWireName *fingerprint, const DeverRtApiPermission *permissions, uint64_t count, uint8_t service_start, DeverRtBuffer *error); +DeverRtAsyncOp *dever_rt_v1_api_serve(const void *session, const DeverRtApiRoute *routes, uint64_t count, const DeverRtWireName *tenant_components, uint64_t component_count, DeverRtBuffer *error); +DeverRtAsyncOp *dever_rt_v1_api_tenant_scope(int64_t tenant_id, const DeverRtAsyncFunction *function, const void *input, DeverRtBuffer *error); +DeverRtAsyncOp *dever_rt_v1_api_require_components(const void *session, const DeverRtWireName *components, uint64_t count, const DeverRtWireName *manifest, uint64_t manifest_count, DeverRtBuffer *error); +DeverRtAsyncOp *dever_rt_v1_api_migrate_models(const void *session, const DeverRtDbBinding *bindings, const DeverRtDbModel *models, const DeverRtDbError *const *errors, uint64_t count, int64_t tenant_id, DeverRtBuffer *error); +DeverRtAsyncOp *dever_rt_v1_api_migrate_application(const void *session, const DeverRtDbBinding *bindings, const DeverRtDbModel *models, const DeverRtDbError *const *errors, uint64_t count, const DeverRtDbError *fallback_errors, int64_t tenant_id, DeverRtBuffer *error); + +/* Job metadata is immutable compiler output. The handler reuses the typed API + * callable layout, but decode borrows a persisted payload Text and detail_id=0; + * decode status 1 means invalid payload, and success initializes its concrete + * input. Job handler success is Unit (size 0, align 1), never a dynamic value. + * The registry owns its bindings and Clock within one application Session. + */ +typedef struct { + DeverRtWireName target, schema; + uint32_t attempts, timeout_ms; + const DeverRtWireName *schedule; + const DeverRtDbBinding *binding; + const DeverRtWireName *components; + uint64_t component_count; + const DeverRtApiHandler *handler; +} DeverRtJob; +typedef struct { DeverRtWireName provider; const DeverRtApiAuth *auth; } DeverRtJobAuth; +uint32_t dever_rt_v1_job_session_new(const void *session, const void *clock, const DeverRtJob *jobs, uint64_t count, const DeverRtJobAuth *auth, uint64_t auth_count, const DeverRtApiPermission *permissions, uint64_t permission_count, const DeverRtWireName *components, uint64_t component_count, void **out, DeverRtBuffer *error); +void dever_rt_v1_job_session_release(void *jobs); +uint32_t dever_rt_v1_job_configure(const void *session, uint8_t api, uint8_t worker, DeverRtBuffer *error); +uint32_t dever_rt_v1_job_decode_unit(const void *payload, DeverRtBuffer *error); +DeverRtAsyncOp *dever_rt_v1_job_initialize(const void *jobs, const DeverRtDbError *errors, DeverRtBuffer *error); +DeverRtAsyncOp *dever_rt_v1_job_enqueue(const void *jobs, uint64_t index, const void *database, const void *transaction, const void *payload, const void *key, int64_t run_at, const DeverRtDbError *errors, DeverRtBuffer *error); +DeverRtAsyncOp *dever_rt_v1_job_drain(const void *jobs, int64_t limit, DeverRtBuffer *error); +DeverRtAsyncOp *dever_rt_v1_job_system_scope(int64_t tenant, uint8_t has_tenant, const DeverRtAsyncFunction *function, const void *input, DeverRtBuffer *error); +DeverRtAsyncOp *dever_rt_v1_api_serve_with_jobs(const void *session, const DeverRtApiRoute *routes, uint64_t count, const DeverRtWireName *components, uint64_t component_count, const void *jobs, DeverRtBuffer *error); +DeverRtAsyncOp *dever_rt_v1_api_tenant_owner(const void *session, int64_t tenant_id, const void *site, int64_t user_id, DeverRtBuffer *error); +DeverRtAsyncOp *dever_rt_v1_api_tenant_component(const void *session, int64_t tenant_id, const void *component, uint8_t enabled, const DeverRtWireName *manifest, uint64_t count, DeverRtBuffer *error); +DeverRtAsyncOp *dever_rt_v1_db_resolve_scoped(const void *session, const DeverRtDbBinding *binding, const DeverRtDbError *errors, DeverRtBuffer *error); +void dever_rt_v1_api_cause_append(void **cause, const void *message); +uint32_t dever_rt_v1_api_input_field(void *inputs, const DeverRtWireName *name, uint8_t required, uint32_t query_kind, void **out, uint8_t *present, DeverRtBuffer *error); +uint32_t dever_rt_v1_api_input_raw_field(void *inputs, const DeverRtWireName *name, uint8_t required, void **out, uint8_t *present, DeverRtBuffer *error); +uint32_t dever_rt_v1_api_input_upload(void *inputs, const DeverRtWireName *name, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_api_text_bounds(const DeverRtWireName *name, const void *nullable_text, int64_t minimum, int64_t maximum, uint8_t has_maximum, DeverRtBuffer *error); +uint32_t dever_rt_v1_api_pagination(int64_t page, int64_t size, int64_t maximum, int64_t *out_page, int64_t *out_size, int64_t *out_offset, DeverRtBuffer *error); +uint32_t dever_rt_v1_api_request_id(void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_api_method(void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_api_path(void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_api_client_address(void **out, uint8_t *present, DeverRtBuffer *error); +uint32_t dever_rt_v1_api_header(const void *name, void **out, uint8_t *present, DeverRtBuffer *error); +uint32_t dever_rt_v1_api_cookie(const void *name, void **out, uint8_t *present, DeverRtBuffer *error); +uint32_t dever_rt_v1_api_secret_cookie(const void *name, void **out, uint8_t *present, DeverRtBuffer *error); +uint32_t dever_rt_v1_api_response_header(const void *name, const void *value, uint8_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_api_response_cookie(const void *name, const void *value, const DeverRtApiCookieOptions *options, uint8_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_api_response_secret_cookie(const void *name, const void *value, const DeverRtApiCookieOptions *options, uint8_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_auth_site_key(void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_auth_id(void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_auth_session(void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_auth_user_id(int64_t *out, uint8_t *present, DeverRtBuffer *error); +uint32_t dever_rt_v1_auth_tenant_id(int64_t *out, uint8_t *present, DeverRtBuffer *error); +uint32_t dever_rt_v1_auth_owns_user(int64_t user_id, uint8_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_auth_issue(const void *subject, const void *session, const void *nullable_tenant, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_auth_issue_cookie(const void *subject, const void *session, const void *nullable_tenant, uint8_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_auth_clear_cookie(uint8_t *out, DeverRtBuffer *error); +typedef void (*DeverRtPackPermission)(const void *key, const void *component, const void *domain, const void *site, const void *action, const void *method, void *out); +DeverRtAsyncOp *dever_rt_v1_auth_permissions(const DeverRtType *element, DeverRtPackPermission pack, DeverRtBuffer *error); +DeverRtAsyncOp *dever_rt_v1_auth_save_role(const void *id, const void *name, uint8_t all_permissions, const void *permission_keys, DeverRtBuffer *error); +DeverRtAsyncOp *dever_rt_v1_auth_grant_role(int64_t user_id, const void *role, DeverRtBuffer *error); +DeverRtAsyncOp *dever_rt_v1_auth_revoke_role(int64_t user_id, const void *role, DeverRtBuffer *error); +DeverRtAsyncOp *dever_rt_v1_auth_disable_role(const void *role, DeverRtBuffer *error); +void *dever_rt_v1_upload_retain(const void *upload); +void dever_rt_v1_upload_release(void *upload); +uint32_t dever_rt_v1_upload_filename(const void *upload, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_upload_content_type(const void *upload, void **out, DeverRtBuffer *error); +uint32_t dever_rt_v1_upload_size(const void *upload, int64_t *out, DeverRtBuffer *error); +uint32_t dever_rt_v1_upload_close_take(void *upload, uint8_t *out, DeverRtBuffer *error); +DeverRtAsyncOp *dever_rt_v1_upload_store_take(void *upload, DeverRtBuffer *error); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/crates/dever-backend-bridge/src/bridge.cpp b/crates/dever-backend-bridge/src/bridge.cpp new file mode 100644 index 0000000..8a5c4ad --- /dev/null +++ b/crates/dever-backend-bridge/src/bridge.cpp @@ -0,0 +1,154 @@ +// Private LLVM 18 / LLD bridge. Inputs are borrowed only for the call; outputs +// use malloc/free through one ABI and never expose LLVM ownership to Rust. +#include "lld/Common/Driver.h" +#include "lld/Common/ErrorHandler.h" +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/Verifier.h" +#include "llvm/IRReader/IRReader.h" +#include "llvm/MC/TargetRegistry.h" +#include "llvm/Passes/PassBuilder.h" +#include "llvm/Support/SourceMgr.h" +#include "llvm/Support/TargetSelect.h" +#include "llvm/Target/TargetMachine.h" +#include <cstdlib> +#include <cstring> +#include <mutex> + +LLD_HAS_DRIVER(elf) +LLD_HAS_DRIVER(coff) +LLD_HAS_DRIVER(macho) + +struct Reply { + unsigned char *bytes; + size_t len; + uint32_t status; +}; + +struct Resource { + const unsigned char *symbol; + size_t symbol_len; + const unsigned char *bytes; + size_t len; +}; + +static void answer(Reply *reply, llvm::StringRef bytes, uint32_t status) { + constexpr size_t limit = 320 * 1024 * 1024; + if (bytes.size() > limit) { + bytes = "compiler bridge output exceeds 320 MiB"; + status = 1; + } + reply->status = status; + reply->len = bytes.size(); + reply->bytes = static_cast<unsigned char *>(std::malloc(bytes.size())); + if (!reply->bytes && !bytes.empty()) { + reply->len = 0; + reply->status = 1; + return; + } + if (!bytes.empty()) std::memcpy(reply->bytes, bytes.data(), bytes.size()); +} + +extern "C" void dever_backend_free(unsigned char *bytes) { std::free(bytes); } + +extern "C" void dever_backend_object(const unsigned char *ir, size_t len, + const char *triple, const Resource *resources, + size_t resource_count, Reply *reply) { + try { + static std::once_flag initialized; + std::call_once(initialized, [] { + llvm::InitializeAllTargetInfos(); + llvm::InitializeAllTargets(); + llvm::InitializeAllTargetMCs(); + llvm::InitializeAllAsmPrinters(); + llvm::InitializeAllAsmParsers(); + }); + llvm::LLVMContext context; + llvm::SMDiagnostic diagnostic; + auto input = llvm::MemoryBuffer::getMemBufferCopy( + llvm::StringRef(reinterpret_cast<const char *>(ir), len), "dever.ir"); + auto module = llvm::parseIR(input->getMemBufferRef(), diagnostic, context); + std::string error; + llvm::raw_string_ostream errors(error); + if (!module) { + diagnostic.print("dever", errors); + return answer(reply, error, 1); + } + // Do not silently compile a module for a target different from its ABI. + if (!module->getTargetTriple().empty() && module->getTargetTriple() != triple) + return answer(reply, "LLVM module target disagrees with selected target", 1); + for (size_t index = 0; index < resource_count; ++index) { + const auto &resource = resources[index]; + auto *global = module->getNamedGlobal(llvm::StringRef( + reinterpret_cast<const char *>(resource.symbol), resource.symbol_len)); + auto *array = global ? llvm::dyn_cast<llvm::ArrayType>(global->getValueType()) : nullptr; + if (!global || !global->isDeclaration() || !global->isConstant() || + !global->hasExternalLinkage() || global->isThreadLocal() || + global->getAddressSpace() != 0 || !array || + !array->getElementType()->isIntegerTy(8) || + array->getNumElements() != resource.len || global->use_empty()) + return answer(reply, "LLVM resource must match a used external constant byte-array declaration", 1); + global->setInitializer(llvm::ConstantDataArray::get( + context, llvm::ArrayRef<uint8_t>(resource.bytes, resource.len))); + global->setLinkage(llvm::GlobalValue::PrivateLinkage); + } + if (llvm::verifyModule(*module, &errors)) return answer(reply, error, 1); + auto target = llvm::TargetRegistry::lookupTarget(triple, error); + if (!target) return answer(reply, error, 1); + llvm::TargetOptions options; + std::unique_ptr<llvm::TargetMachine> machine(target->createTargetMachine( + triple, "generic", "", options, llvm::Reloc::PIC_, std::nullopt, + llvm::CodeGenOptLevel::Default)); + if (!machine) return answer(reply, "LLVM target machine is unavailable", 1); + auto layout = machine->createDataLayout(); + if (!module->getDataLayoutStr().empty() && module->getDataLayout() != layout) + return answer(reply, "LLVM module layout disagrees with selected target", 1); + module->setTargetTriple(triple); + module->setDataLayout(layout); + llvm::PassBuilder passes(machine.get()); + llvm::LoopAnalysisManager loops; + llvm::FunctionAnalysisManager functions; + llvm::CGSCCAnalysisManager calls; + llvm::ModuleAnalysisManager modules; + passes.registerModuleAnalyses(modules); + passes.registerCGSCCAnalyses(calls); + passes.registerFunctionAnalyses(functions); + passes.registerLoopAnalyses(loops); + passes.crossRegisterProxies(loops, functions, calls, modules); + auto pipeline = passes.buildPerModuleDefaultPipeline(llvm::OptimizationLevel::O2); + pipeline.run(*module, modules); + if (llvm::verifyModule(*module, &errors)) return answer(reply, error, 1); + llvm::SmallVector<char, 0> output; + llvm::raw_svector_ostream stream(output); + llvm::legacy::PassManager emission; + if (machine->addPassesToEmitFile(emission, stream, nullptr, + llvm::CodeGenFileType::ObjectFile)) + return answer(reply, "LLVM target cannot emit native objects", 1); + emission.run(*module); + answer(reply, llvm::StringRef(output.data(), output.size()), 0); + } catch (const std::exception &error) { + answer(reply, error.what(), 1); + } catch (...) { + answer(reply, "LLVM bridge failed", 1); + } +} + +extern "C" void dever_backend_link(const char *const *args, size_t count, + Reply *reply) { + try { + std::string output, error; + llvm::raw_string_ostream out(output), err(error); + const lld::DriverDef drivers[] = {{lld::Gnu, lld::elf::link}, + {lld::WinLink, lld::coff::link}, + {lld::Darwin, lld::macho::link}}; + auto result = lld::lldMain(llvm::ArrayRef(args, count), out, err, drivers); + // LLD explicitly forbids cleanup/re-entry after failed crash recovery. + // This bridge belongs to a compiler worker, never an application process. + if (!result.canRunAgain) lld::exitLld(result.retCode ? result.retCode : 1); + answer(reply, error, result.retCode == 0 ? 0 : 1); + } catch (const std::exception &) { + lld::exitLld(1); + } catch (...) { + lld::exitLld(1); + } +} diff --git a/crates/dever-backend-bridge/src/ffi.rs b/crates/dever-backend-bridge/src/ffi.rs new file mode 100644 index 0000000..88aeb92 --- /dev/null +++ b/crates/dever-backend-bridge/src/ffi.rs @@ -0,0 +1,10726 @@ +//! The only Rust unsafe boundary. Compiler Reply buffers use the C++ bridge's +//! malloc/free pair; runtime ABI buffers use this crate's Rust Box allocator +//! and its separate versioned release function. The owners never interchange. +#![allow(unsafe_code)] + +#[cfg(feature = "embedded")] +mod compiler { + use std::ffi::{CString, c_char}; + use std::sync::Mutex; + + use crate::{BinaryResource, Target}; + + const MAX_REPLY_BYTES: usize = 320 * 1024 * 1024; + + #[repr(C)] + struct Resource { + symbol: *const u8, + symbol_len: usize, + bytes: *const u8, + len: usize, + } + + #[repr(C)] + #[derive(Default)] + struct Reply { + bytes: *mut u8, + len: usize, + status: u32, + } + + unsafe extern "C" { + fn dever_backend_object( + ir: *const u8, + len: usize, + triple: *const c_char, + resources: *const Resource, + resource_count: usize, + reply: *mut Reply, + ); + fn dever_backend_link(args: *const *const c_char, count: usize, reply: *mut Reply); + fn dever_backend_free(bytes: *mut u8); + } + + impl Drop for Reply { + fn drop(&mut self) { + // SAFETY: only our C++ bridge assigns bytes, by malloc; NULL is valid. + // The buffer is freed once, after all reads, by its owning allocator. + unsafe { dever_backend_free(self.bytes) }; + } + } + + impl Reply { + fn take(&self) -> Result<Vec<u8>, String> { + if self.len > MAX_REPLY_BYTES || (self.len != 0 && self.bytes.is_null()) { + return Err("compiler bridge returned an invalid buffer".into()); + } + let bytes = if self.len == 0 { + &[] + } else { + // SAFETY: the bridge returns a live allocation of exactly len bytes. + // Reply owns it until Drop; no foreign mutation occurs after return. + unsafe { std::slice::from_raw_parts(self.bytes, self.len) } + }; + if self.status != 0 && bytes.is_empty() { + return Err("compiler bridge could not allocate a diagnostic".into()); + } + if self.status == 0 { + Ok(bytes.to_vec()) + } else { + Err(String::from_utf8_lossy(bytes).into_owned()) + } + } + } + + pub(crate) fn emit_object( + ir: &str, + target: Target, + resources: &[BinaryResource<'_>], + ) -> Result<Vec<u8>, String> { + let triple = CString::new(target.triple()).expect("static target contains no NUL"); + let resources = resources + .iter() + .map(|resource| Resource { + symbol: resource.symbol.as_ptr(), + symbol_len: resource.symbol.len(), + bytes: resource.bytes.as_ptr(), + len: resource.bytes.len(), + }) + .collect::<Vec<_>>(); + let mut reply = Reply::default(); + // SAFETY: IR and NUL-terminated triple outlive this synchronous call. + // C++ reads the validated borrowed slices only during this call and copies + // their bytes into LLVM-owned constants before returning its owned reply. + unsafe { + dever_backend_object( + ir.as_ptr(), + ir.len(), + triple.as_ptr(), + resources.as_ptr(), + resources.len(), + &mut reply, + ) + }; + reply.take() + } + + // LLD's global context cannot be used concurrently. The C++ owner terminates + // the compiler worker rather than returning an unrecoverable linker context. + static LLD_LOCK: Mutex<()> = Mutex::new(()); + + pub(crate) fn link(args: &[CString]) -> Result<(), String> { + let _guard = LLD_LOCK.lock().map_err(|_| "LLD state is poisoned")?; + let pointers = args.iter().map(|arg| arg.as_ptr()).collect::<Vec<_>>(); + let mut reply = Reply::default(); + // SAFETY: each CString, the pointer array and reply live through the call. + // The mutex serializes calls; C++ returns only after safe LLD recovery. + unsafe { dever_backend_link(pointers.as_ptr(), pointers.len(), &mut reply) }; + reply.take().map(|_| ()) + } +} + +#[cfg(feature = "embedded")] +pub(super) use compiler::{emit_object, link}; + +#[cfg(feature = "runtime-abi")] +mod runtime { + use std::ptr; + + use dever_runtime::{abi, number, time}; + + use crate::{ABI_INVALID_INPUT, ABI_OK, ABI_RUNTIME_ERROR, AbiDecimal}; + + /// # Safety + /// Call only from the generated process main, before starting application + /// threads. Callable roots must preserve the embedding host's signal policy. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_process_init() -> u32 { + // The CRT enters our C main without Rust's lang_start initialization. + // Preserve the Rust backend's EPIPE error/cleanup path on Unix. + #[cfg(unix)] + if unsafe { libc::signal(libc::SIGPIPE, libc::SIG_IGN) } == libc::SIG_ERR { + return ABI_RUNTIME_ERROR; + } + ABI_OK + } + + /// # Safety + /// `bytes` borrows `len` readable bytes until return; null is valid for zero length. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_process_stderr(bytes: *const u8, len: u64) -> u32 { + use std::io::Write; + + if len > isize::MAX as u64 || (len != 0 && bytes.is_null()) { + return ABI_INVALID_INPUT; + } + let bytes = if len == 0 { + &[] + } else { + // SAFETY: the process entry borrows a live text/buffer allocation. + unsafe { std::slice::from_raw_parts(bytes, len as usize) } + }; + let mut stderr = std::io::stderr().lock(); + match stderr.write_all(bytes).and_then(|()| stderr.flush()) { + Ok(()) => ABI_OK, + Err(_) => ABI_RUNTIME_ERROR, + } + } + + /// # Safety + /// `argv` borrows the CRT's argc pointers to NUL-terminated arguments. + /// `out` is a distinct writable u64 slot, initialized only on success. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_process_test_index( + argc: i32, + argv: *const *const std::ffi::c_char, + count: u64, + out: *mut u64, + ) -> u32 { + if argc != 2 || argv.is_null() || !argv.is_aligned() || out.is_null() || !out.is_aligned() { + return ABI_INVALID_INPUT; + } + // SAFETY: argc=2 and the caller supplies the live CRT argument array. + let argument = unsafe { *argv.add(1) }; + if argument.is_null() { + return ABI_INVALID_INPUT; + } + // SAFETY: each CRT argument is NUL terminated and borrowed during main. + let digits = unsafe { std::ffi::CStr::from_ptr(argument) }.to_bytes(); + if digits.is_empty() { + return ABI_INVALID_INPUT; + } + let index = digits.iter().try_fold(0_u64, |value, digit| { + if !digit.is_ascii_digit() { + return None; + } + value.checked_mul(10)?.checked_add(u64::from(*digit - b'0')) + }); + match index { + Some(index) if index < count => { + // SAFETY: the caller supplied a writable, aligned output slot. + unsafe { out.write(index) }; + ABI_OK + } + _ => ABI_INVALID_INPUT, + } + } + + mod asynchronous { + use std::alloc::{Layout, alloc, dealloc, handle_alloc_error}; + use std::ffi::c_void; + use std::future::Future; + use std::pin::Pin; + use std::ptr::{self, NonNull}; + use std::sync::{Arc, Mutex}; + use std::task::{Context, Poll}; + + use dever_runtime::async_stream::AsyncStream; + use dever_runtime::bytes::Bytes; + use dever_runtime::channel::Channel; + use dever_runtime::{net, task}; + + use super::managed::{self, Element}; + use super::{Buffer, empty, error}; + use crate::ABI_INVALID_INPUT; + + type MoveValue = unsafe extern "C" fn(*mut c_void, *mut c_void); + type DropValue = unsafe extern "C" fn(*mut c_void); + + #[repr(C)] + pub(super) struct OwnedType { + size: u64, + align: u64, + move_init: Option<MoveValue>, + drop: Option<DropValue>, + transferable: u8, + } + + #[repr(C)] + pub(super) struct AsyncFunction { + output_type: *const OwnedType, + fault_type: *const OwnedType, + create: Option< + unsafe extern "C" fn( + *mut PollState, + *const c_void, + *mut c_void, + *mut c_void, + ) -> *mut c_void, + >, + resume: Option<unsafe extern "C" fn(*mut c_void)>, + destroy: Option<unsafe extern "C" fn(*mut c_void)>, + done: Option<unsafe extern "C" fn(*mut c_void) -> u8>, + send_safe: u8, + } + + #[repr(C)] + pub(super) struct SyncFunction { + input_type: *const OwnedType, + output_type: *const OwnedType, + fault_type: *const OwnedType, + invoke: Option<unsafe extern "C" fn(*mut c_void, *mut c_void, *mut c_void) -> u32>, + send_safe: u8, + } + + struct PreparedSync { + input: Owned, + output: Owned, + fault: Owned, + invoke: unsafe extern "C" fn(*mut c_void, *mut c_void, *mut c_void) -> u32, + unit_output: bool, + } + + impl PreparedSync { + fn execute(self) -> Result<Owned, ForeignFault> { + let Self { + input, + mut output, + mut fault, + invoke, + .. + } = self; + // SAFETY: the checked synchronous callback receives one + // initialized input and matching uninitialized result slots. + let status = unsafe { invoke(input.pointer(), output.pointer(), fault.pointer()) }; + match status { + 0 => { + output.initialized = true; + Ok(output) + } + 1 => { + fault.initialized = true; + Err(ForeignFault::Typed(fault)) + } + _ => Err(ForeignFault::from( + "synchronous function returned invalid status".to_owned(), + )), + } + } + } + + struct Owned { + pointer: NonNull<u8>, + layout: Layout, + ty: &'static OwnedType, + initialized: bool, + } + + // Compiler descriptors are process-static and generated only for + // checker-approved transferable values before spawning child tasks. + unsafe impl Send for Owned {} + + unsafe fn owned_layout( + ty: *const OwnedType, + ) -> Result<(&'static OwnedType, Layout), String> { + if ty.is_null() { + return Err("missing async type descriptor".into()); + } + if !ty.is_aligned() { + return Err("unaligned async type descriptor".into()); + } + let ty = unsafe { &*ty }; + let size = usize::try_from(ty.size).map_err(|_| "invalid async value size")?; + let align = usize::try_from(ty.align).map_err(|_| "invalid async value alignment")?; + let layout = Layout::from_size_align(size.max(1), align) + .map_err(|_| "invalid async value layout")?; + if ty.move_init.is_none() || ty.drop.is_none() { + return Err("invalid async value descriptor".into()); + } + Ok((ty, layout)) + } + + impl Owned { + unsafe fn new(ty: *const OwnedType) -> Result<Self, String> { + let (ty, layout) = unsafe { owned_layout(ty) }?; + let pointer = + NonNull::new(unsafe { alloc(layout) }).ok_or("cannot allocate async value")?; + Ok(Self { + pointer, + layout, + ty, + initialized: false, + }) + } + + fn pointer(&self) -> *mut c_void { + self.pointer.as_ptr().cast() + } + + unsafe fn take_from(&mut self, source: *mut c_void) { + // SAFETY: the callback moves one initialized value into this + // uninitialized owner, and the caller disarms its old slot. + unsafe { + (self.ty.move_init.expect("validated descriptor"))(source, self.pointer()) + }; + self.initialized = true; + } + + unsafe fn move_to(&mut self, destination: *mut c_void) { + debug_assert!(self.initialized); + // SAFETY: the compiler callback moves one initialized concrete + // value into its matching aligned, uninitialized destination. + unsafe { + (self.ty.move_init.expect("validated descriptor"))(self.pointer(), destination) + }; + self.initialized = false; + } + } + + impl Drop for Owned { + fn drop(&mut self) { + if self.initialized { + // SAFETY: this owner has not transferred its concrete value. + unsafe { (self.ty.drop.expect("validated descriptor"))(self.pointer()) }; + } + // SAFETY: pointer was allocated with this exact layout. + unsafe { dealloc(self.pointer.as_ptr(), self.layout) }; + } + } + + enum ForeignFault { + Typed(Owned), + Runtime(String), + } + + impl From<String> for ForeignFault { + fn from(message: String) -> Self { + Self::Runtime(message) + } + } + + pub(super) struct PollState { + current: *mut c_void, + completion: u32, + } + + struct ForeignFuture { + state: Box<PollState>, + frame: NonNull<c_void>, + output: Option<Owned>, + fault: Option<Owned>, + resume: unsafe extern "C" fn(*mut c_void), + destroy: unsafe extern "C" fn(*mut c_void), + done: unsafe extern "C" fn(*mut c_void) -> u8, + } + + struct SendForeignFuture(ForeignFuture); + + // Only task/group spawn constructs this wrapper, after checking the + // whole child frame and both result descriptors for transferability. + unsafe impl Send for SendForeignFuture {} + + impl Future for SendForeignFuture { + type Output = Result<Owned, ForeignFault>; + + fn poll(self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll<Self::Output> { + Pin::new(&mut self.get_mut().0).poll(context) + } + } + + impl ForeignFuture { + unsafe fn new( + function: *const AsyncFunction, + input: *const c_void, + ) -> Result<Self, String> { + let function = + unsafe { function.as_ref() }.ok_or("missing async function descriptor")?; + let mut output = unsafe { Owned::new(function.output_type) }?; + let mut fault = unsafe { Owned::new(function.fault_type) }?; + let create = function.create.ok_or("missing async constructor")?; + let resume = function.resume.ok_or("missing async resume callback")?; + let destroy = function.destroy.ok_or("missing async destroy callback")?; + let done = function.done.ok_or("missing async completion callback")?; + let mut state = Box::new(PollState { + current: ptr::null_mut(), + completion: 0, + }); + // SAFETY: the callback's process-static ABI clones borrowed + // inputs and initializes its owned frame before returning. + let frame = + unsafe { create(&mut *state, input, output.pointer(), fault.pointer()) }; + let frame = NonNull::new(frame).ok_or("cannot create async frame")?; + output.initialized = false; + fault.initialized = false; + Ok(Self { + state, + frame, + output: Some(output), + fault: Some(fault), + resume, + destroy, + done, + }) + } + } + + impl Future for ForeignFuture { + type Output = Result<Owned, ForeignFault>; + + fn poll(self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll<Self::Output> { + let this = self.get_mut(); + this.state.current = (context as *mut Context<'_>).cast(); + // SAFETY: only this poll owns the frame, and current is live. + unsafe { (this.resume)(this.frame.as_ptr()) }; + this.state.current = ptr::null_mut(); + if unsafe { (this.done)(this.frame.as_ptr()) } == 0 { + return Poll::Pending; + } + match this.state.completion { + 1 => { + let mut output = this.output.take().expect("async output once"); + output.initialized = true; + Poll::Ready(Ok(output)) + } + 2 => { + let mut fault = this.fault.take().expect("async fault once"); + fault.initialized = true; + Poll::Ready(Err(ForeignFault::Typed(fault))) + } + _ => Poll::Ready(Err(ForeignFault::Runtime( + "async frame completed without a result".into(), + ))), + } + } + } + + impl Drop for ForeignFuture { + fn drop(&mut self) { + // SAFETY: destroy runs once, including at the initial and final + // suspend points, then releases the frame's allocator owner. + unsafe { (self.destroy)(self.frame.as_ptr()) }; + } + } + + pub(super) struct AsyncOp { + future: Pin<Box<dyn Future<Output = Result<OpValue, ForeignFault>>>>, + completed: bool, + output_slots: Option<OutputSlots>, + } + + #[derive(Clone, Copy)] + enum OutputSlots { + Unit, + Bool, + #[cfg(all( + feature = "runtime-api", + any(feature = "runtime-sqlite", feature = "runtime-postgres") + ))] + Int, + Handle, + OptionalHandle, + #[cfg(feature = "runtime-api")] + Typed { + align: usize, + fault_align: usize, + }, + #[cfg(any(feature = "runtime-sqlite", feature = "runtime-postgres"))] + Database { + align: usize, + optional: bool, + fault_align: usize, + }, + } + + impl OutputSlots { + fn validate( + self, + output: *mut c_void, + present: *mut u8, + _fault: *mut c_void, + ) -> Result<(), &'static str> { + match self { + #[cfg(feature = "runtime-api")] + Self::Typed { align, fault_align } => { + if output.is_null() + || !(output as usize).is_multiple_of(align) + || _fault.is_null() + || !(_fault as usize).is_multiple_of(fault_align) + { + return Err("invalid application typed output"); + } + Ok(()) + } + #[cfg(any(feature = "runtime-sqlite", feature = "runtime-postgres"))] + Self::Database { + align, + optional, + fault_align, + } => { + if (align != 0 + && (output.is_null() || !(output as usize).is_multiple_of(align))) + || (optional && present.is_null()) + || _fault.is_null() + || !(_fault as usize).is_multiple_of(fault_align) + { + return Err("invalid database operation output"); + } + Ok(()) + } + Self::Unit => Ok(()), + #[cfg(all( + feature = "runtime-api", + any(feature = "runtime-sqlite", feature = "runtime-postgres") + ))] + Self::Int => { + if output.is_null() || !output.cast::<i64>().is_aligned() { + Err("invalid Job count output") + } else { + Ok(()) + } + } + Self::Bool if output.is_null() => Err("missing protocol bool output"), + Self::Bool => Ok(()), + Self::Handle | Self::OptionalHandle => { + if output.is_null() || !output.cast::<*mut c_void>().is_aligned() { + return Err("invalid protocol handle output"); + } + if matches!(self, Self::OptionalHandle) && present.is_null() { + return Err("missing message presence output"); + } + Ok(()) + } + } + } + } + + enum TaskKind { + Value(task::Task<Owned, ForeignFault>, &'static OwnedType), + Unit(task::Task<(), ForeignFault>), + } + + impl TaskKind { + fn output_type(&self) -> Option<&'static OwnedType> { + match self { + Self::Value(_, ty) => Some(ty), + Self::Unit(_) => None, + } + } + } + + pub(super) struct TaskHandle(Arc<Mutex<Option<TaskKind>>>); + + fn take_task(handle: TaskHandle) -> Result<TaskKind, ForeignFault> { + handle + .0 + .lock() + .map_err(|_| ForeignFault::from("async task state lock failed".to_owned()))? + .take() + .ok_or_else(|| ForeignFault::from("async task was already consumed".to_owned())) + } + + pub(super) struct GroupHandle(Arc<Mutex<Option<task::Group<ForeignFault>>>>); + pub(super) struct ChannelHandle(Channel<Owned>); + + enum StreamRow { + Managed(Element), + Moved(Owned), + Int(i64), + } + + struct AsyncStreamHandle(AsyncStream<StreamRow>); + + impl StreamRow { + fn pointer(&self) -> *const c_void { + match self { + Self::Managed(value) => value.pointer(), + Self::Moved(value) => value.pointer(), + Self::Int(value) => std::ptr::from_ref(value).cast(), + } + } + } + + type PackInput = unsafe extern "C" fn(*const c_void, *const c_void, *mut c_void); + + #[repr(C)] + pub(super) struct ParallelFunction { + asynchronous: *const AsyncFunction, + synchronous: *const SyncFunction, + input_type: *const OwnedType, + context_type: *const managed::TypeDescriptor, + pack: Option<PackInput>, + } + + struct ParallelHandler { + asynchronous: Option<&'static AsyncFunction>, + synchronous: Option<&'static SyncFunction>, + input_type: &'static OwnedType, + context: Option<Element>, + pack: PackInput, + } + + // Only new constructs this owner, after checking compiler Send metadata + // for the complete packed inputs, child frame, outputs and typed faults. + // The descriptors are immutable process-static data; context is borrowed + // immutably by pack, which creates a distinct owned input for every child. + unsafe impl Send for ParallelHandler {} + unsafe impl Sync for ParallelHandler {} + + impl ParallelHandler { + unsafe fn new( + function: *const ParallelFunction, + context: *const c_void, + ) -> Result<Self, String> { + let function = unsafe { function.as_ref() }.ok_or("missing parallel function")?; + let input_type = + unsafe { function.input_type.as_ref() }.ok_or("missing parallel input type")?; + let asynchronous = unsafe { function.asynchronous.as_ref() }; + let synchronous = unsafe { function.synchronous.as_ref() }; + let (send_safe, output_type, fault_type) = match (asynchronous, synchronous) { + (Some(function), None) => ( + function.send_safe, + function.output_type, + function.fault_type, + ), + (None, Some(function)) if std::ptr::eq(function.input_type, input_type) => ( + function.send_safe, + function.output_type, + function.fault_type, + ), + _ => { + return Err( + "parallel function requires exactly one matching handler".into() + ); + } + }; + for ty in [function.input_type, output_type, fault_type] { + let (ty, _) = unsafe { owned_layout(ty) }?; + if send_safe == 0 || ty.transferable == 0 { + return Err("parallel handler is not transferable".into()); + } + } + if unsafe { &*output_type }.size != 0 { + return Err("parallel handler must return Unit".into()); + } + if asynchronous.is_some_and(|function| { + function.create.is_none() + || function.resume.is_none() + || function.destroy.is_none() + || function.done.is_none() + }) || synchronous.is_some_and(|function| function.invoke.is_none()) + { + return Err("incomplete parallel handler".into()); + } + let pack = function + .pack + .ok_or("missing parallel input pack callback")?; + let context = if function.context_type.is_null() { + None + } else { + let ops = unsafe { managed::descriptor(function.context_type, false) } + .map_err(|failure| failure.message)?; + Some( + unsafe { Element::from_source(ops, context) } + .map_err(|failure| failure.message)?, + ) + }; + Ok(Self { + asynchronous, + synchronous, + input_type, + context, + pack, + }) + } + + fn input(&self, row: &StreamRow) -> Result<Owned, ForeignFault> { + let mut input = + unsafe { Owned::new(self.input_type) }.map_err(ForeignFault::from)?; + let context = self.context.as_ref().map_or(ptr::null(), Element::pointer); + // SAFETY: pack initializes the matching concrete input type from + // borrowed element/context. The compiler verified their types. + unsafe { (self.pack)(row.pointer(), context, input.pointer()) }; + input.initialized = true; + Ok(input) + } + + fn execute(&self, row: StreamRow) -> Result<(), ForeignFault> { + let function = self.synchronous.expect("validated synchronous dispatch"); + let input = self.input(&row)?; + let prepared = PreparedSync { + input, + output: unsafe { Owned::new(function.output_type) } + .map_err(ForeignFault::from)?, + fault: unsafe { Owned::new(function.fault_type) } + .map_err(ForeignFault::from)?, + invoke: function.invoke.expect("validated handler"), + unit_output: true, + }; + prepared.execute().map(drop) + } + + fn start(&self, row: StreamRow) -> Result<SendForeignFuture, ForeignFault> { + let input = self.input(&row)?; + let function = self.asynchronous.expect("validated async dispatch"); + // ForeignFuture::new clones this borrowed input into the frame + // before initial suspend, so this temporary may now be dropped. + unsafe { ForeignFuture::new(function, input.pointer()) } + .map(SendForeignFuture) + .map_err(ForeignFault::from) + } + } + + struct GroupLease { + shared: Arc<Mutex<Option<task::Group<ForeignFault>>>>, + group: Option<task::Group<ForeignFault>>, + } + + impl GroupLease { + fn take( + shared: Arc<Mutex<Option<task::Group<ForeignFault>>>>, + ) -> Result<Self, ForeignFault> { + let group = shared + .lock() + .map_err(|_| ForeignFault::from("async group state lock failed".to_owned()))? + .take() + .ok_or_else(|| { + ForeignFault::from("async group is already in use".to_owned()) + })?; + Ok(Self { + shared, + group: Some(group), + }) + } + } + + impl Drop for GroupLease { + fn drop(&mut self) { + if let Ok(mut slot) = self.shared.lock() { + *slot = self.group.take(); + } + } + } + + enum OpValue { + Unit, + Value(Owned), + OptionalValue(Option<Owned>), + Task(TaskKind), + StreamRow(Option<StreamRow>), + Socket(net::Socket), + Listener(net::Listener), + Bytes(Option<Bytes>), + Bool(bool), + OptionalUnit(bool), + Protocol(protocol::Value), + #[cfg(feature = "runtime-external")] + External(dever_runtime::component::Reply), + #[cfg(feature = "runtime-api")] + Api(api_application::Output), + #[cfg(any(feature = "runtime-sqlite", feature = "runtime-postgres"))] + Database(database::Output), + } + + fn operation( + future: impl Future<Output = Result<OpValue, ForeignFault>> + 'static, + ) -> *mut AsyncOp { + operation_with_output(future, None) + } + + fn operation_with_output( + future: impl Future<Output = Result<OpValue, ForeignFault>> + 'static, + output_slots: Option<OutputSlots>, + ) -> *mut AsyncOp { + Box::into_raw(Box::new(AsyncOp { + future: Box::pin(future), + completed: false, + output_slots, + })) + } + + /// # Safety + /// The descriptor and callbacks are process-static compiler output; + /// input remains borrowed through create; output, fault and error are + /// distinct writable slots of their declared concrete types. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_root( + function: *const AsyncFunction, + input: *const c_void, + output: *mut c_void, + fault: *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + if !unsafe { empty(buffer) } || output.is_null() || fault.is_null() { + return 4; + } + let future = match unsafe { ForeignFuture::new(function, input) } { + Ok(future) => future, + Err(message) => return unsafe { error(buffer, 4, message) }, + }; + let result = task::run_entry_with_typed(task::RuntimeConfig::default(), future); + unsafe { root_result(result, output, fault, buffer) } + } + + unsafe fn root_result( + result: Result<Owned, ForeignFault>, + output: *mut c_void, + fault: *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + match result { + Ok(mut value) => { + // SAFETY: caller supplied the matching uninitialized slot. + unsafe { value.move_to(output) }; + 1 + } + Err(ForeignFault::Typed(mut value)) => { + // SAFETY: caller supplied the matching uninitialized slot. + unsafe { value.move_to(fault) }; + 2 + } + Err(ForeignFault::Runtime(message)) => unsafe { error(buffer, 3, message) }, + } + } + + #[cfg(any( + feature = "runtime-external", + feature = "runtime-api", + feature = "runtime-sqlite", + feature = "runtime-postgres" + ))] + fn cleanup_result( + result: Result<Owned, ForeignFault>, + cleanup: Result<(), String>, + append: unsafe extern "C" fn(*mut c_void, *const c_void), + ) -> Result<Owned, ForeignFault> { + match (result, cleanup) { + (result, Ok(())) => result, + (Ok(_), Err(message)) => Err(ForeignFault::Runtime(message)), + (Err(ForeignFault::Runtime(primary)), Err(message)) => Err(ForeignFault::Runtime( + format!("{primary}; cleanup: {message}"), + )), + (Err(ForeignFault::Typed(primary)), Err(message)) => { + let message = managed::owned(message); + // SAFETY: the root validated this callback for its concrete fault. + unsafe { + append(primary.pointer(), message); + managed::release::<String>(message); + } + Err(ForeignFault::Typed(primary)) + } + } + } + + /// # Safety + /// `state` is the live frame owner supplied to the constructor. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_complete(state: *mut PollState, status: u32) { + if let Some(state) = unsafe { state.as_mut() } { + state.completion = status; + } + } + + #[unsafe(no_mangle)] + pub extern "C" fn dever_rt_v1_async_sleep(milliseconds: i64) -> *mut AsyncOp { + operation(async move { + task::sleep(milliseconds) + .await + .map_err(|message| ForeignFault::Runtime(message.to_owned()))?; + Ok(OpValue::Unit) + }) + } + + /// # Safety + /// The process-static descriptor borrows input only through create. + /// A same-scope call can poll a non-Send frame on the root thread; + /// child spawning applies the separate send_safe check. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_call( + function: *const AsyncFunction, + input: *const c_void, + buffer: *mut Buffer, + ) -> *mut AsyncOp { + if !unsafe { empty(buffer) } { + return ptr::null_mut(); + } + let future = match unsafe { ForeignFuture::new(function, input) } { + Ok(future) => future, + Err(message) => { + unsafe { error(buffer, ABI_INVALID_INPUT, message) }; + return ptr::null_mut(); + } + }; + operation(async move { future.await.map(OpValue::Value) }) + } + + unsafe fn prepare_sync( + function: *const SyncFunction, + input: *mut c_void, + buffer: *mut Buffer, + ) -> Option<PreparedSync> { + if !unsafe { empty(buffer) } { + return None; + } + let Some(function) = (unsafe { function.as_ref() }) else { + unsafe { + error( + buffer, + ABI_INVALID_INPUT, + "missing synchronous function descriptor", + ) + }; + return None; + }; + let (Some(input_type), Some(output_type), Some(fault_type), Some(invoke)) = ( + unsafe { function.input_type.as_ref() }, + unsafe { function.output_type.as_ref() }, + unsafe { function.fault_type.as_ref() }, + function.invoke, + ) else { + unsafe { + error( + buffer, + ABI_INVALID_INPUT, + "incomplete synchronous function descriptor", + ) + }; + return None; + }; + if input.is_null() + || function.send_safe == 0 + || input_type.transferable == 0 + || output_type.transferable == 0 + || fault_type.transferable == 0 + { + unsafe { + error( + buffer, + ABI_INVALID_INPUT, + "blocking function value is not transferable", + ) + }; + return None; + } + let mut input_owner = match unsafe { Owned::new(function.input_type) } { + Ok(value) => value, + Err(message) => { + unsafe { error(buffer, ABI_INVALID_INPUT, message) }; + return None; + } + }; + let output = match unsafe { Owned::new(function.output_type) } { + Ok(value) => value, + Err(message) => { + unsafe { error(buffer, ABI_INVALID_INPUT, message) }; + return None; + } + }; + let fault = match unsafe { Owned::new(function.fault_type) } { + Ok(value) => value, + Err(message) => { + unsafe { error(buffer, ABI_INVALID_INPUT, message) }; + return None; + } + }; + unsafe { input_owner.take_from(input) }; + Some(PreparedSync { + input: input_owner, + output, + fault, + invoke, + unit_output: output_type.size == 0, + }) + } + + unsafe fn blocking_operation( + function: *const SyncFunction, + input: *mut c_void, + buffer: *mut Buffer, + ) -> *mut AsyncOp { + let Some(prepared) = (unsafe { prepare_sync(function, input, buffer) }) else { + return ptr::null_mut(); + }; + operation(async move { + task::blocking_typed(move || prepared.execute()) + .await + .map(OpValue::Value) + }) + } + + /// # Safety + /// input is one initialized owned value; after a non-null return the + /// caller must disarm it. The checked callback cannot suspend. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_blocking( + function: *const SyncFunction, + input: *mut c_void, + buffer: *mut Buffer, + ) -> *mut AsyncOp { + unsafe { blocking_operation(function, input, buffer) } + } + + /// # Safety + /// Same contract as async_blocking; both use the bounded blocking pool. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_parallel( + function: *const SyncFunction, + input: *mut c_void, + buffer: *mut Buffer, + ) -> *mut AsyncOp { + unsafe { blocking_operation(function, input, buffer) } + } + + /// # Safety + /// state and operation are live exclusive owners; the operation is + /// polled by only one coroutine and the error slot is writable. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_op_poll( + state: *mut PollState, + operation: *mut AsyncOp, + output: *mut c_void, + present: *mut u8, + fault: *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + if !unsafe { empty(buffer) } { + return 4; + } + let (Some(state), Some(operation)) = + (unsafe { state.as_mut() }, unsafe { operation.as_mut() }) + else { + return unsafe { error(buffer, 4, "missing async operation") }; + }; + if operation.completed || state.current.is_null() { + return unsafe { error(buffer, 4, "async operation is not polling") }; + } + // Reject invalid protocol destinations before advancing network I/O, + // consuming a frame, or committing a response. The operation remains + // unpolled, so a caller may retry with valid output slots. + if let Some(slots) = operation.output_slots + && let Err(message) = slots.validate(output, present, fault) + { + return unsafe { error(buffer, 4, message) }; + } + // SAFETY: current points to the Context for this one active poll. + let context = unsafe { &mut *(state.current as *mut Context<'_>) }; + match operation.future.as_mut().poll(context) { + Poll::Pending => 0, + Poll::Ready(Ok(value)) => { + operation.completed = true; + match value { + #[cfg(any(feature = "runtime-sqlite", feature = "runtime-postgres"))] + OpValue::Database(value) => unsafe { value.write(output, present) }, + OpValue::Unit => {} + OpValue::Value(mut value) => { + if output.is_null() { + return unsafe { error(buffer, 4, "missing async output") }; + } + // SAFETY: output is the descriptor's writable slot. + unsafe { value.move_to(output) }; + } + OpValue::OptionalValue(value) => { + if present.is_null() || !present.is_aligned() { + return unsafe { + error(buffer, 4, "missing async presence output") + }; + } + match value { + Some(mut value) => { + if output.is_null() { + return unsafe { error(buffer, 4, "missing async output") }; + } + // SAFETY: presence and value slots are distinct. + unsafe { + value.move_to(output); + present.write(1) + }; + } + None => unsafe { present.write(0) }, + } + } + OpValue::Task(task) => { + if output.is_null() || !output.is_aligned() { + return unsafe { error(buffer, 4, "missing task output") }; + } + // SAFETY: output is one writable pointer slot. + let handle = TaskHandle(Arc::new(Mutex::new(Some(task)))); + unsafe { + (output as *mut *mut c_void) + .write(Box::into_raw(Box::new(handle)).cast()) + }; + } + OpValue::StreamRow(value) => { + if present.is_null() || output.is_null() { + return unsafe { error(buffer, 4, "missing stream output") }; + } + unsafe { present.write(u8::from(value.is_some())) }; + if let Some(value) = value { + match value { + StreamRow::Managed(value) => { + if let Err(failure) = unsafe { value.copy_to(output) } { + return unsafe { error(buffer, 4, failure.message) }; + } + } + StreamRow::Moved(mut value) => unsafe { value.move_to(output) }, + StreamRow::Int(value) => unsafe { + output.cast::<i64>().write(value) + }, + } + } + } + OpValue::Socket(value) => { + if output.is_null() { + return unsafe { error(buffer, 4, "missing socket output") }; + } + unsafe { output.cast::<*mut c_void>().write(managed::owned(value)) }; + } + OpValue::Listener(value) => { + if output.is_null() { + return unsafe { error(buffer, 4, "missing listener output") }; + } + unsafe { output.cast::<*mut c_void>().write(managed::owned(value)) }; + } + OpValue::Bytes(value) => { + if present.is_null() || output.is_null() { + return unsafe { error(buffer, 4, "missing read output") }; + } + unsafe { present.write(u8::from(value.is_some())) }; + if let Some(value) = value { + unsafe { + output.cast::<*mut c_void>().write(managed::owned(value)) + }; + } + } + OpValue::Bool(value) => { + if output.is_null() { + return unsafe { error(buffer, 4, "missing bool output") }; + } + unsafe { output.cast::<u8>().write(u8::from(value)) }; + } + OpValue::OptionalUnit(value) => { + if present.is_null() { + return unsafe { error(buffer, 4, "missing timeout presence") }; + } + unsafe { present.write(u8::from(value)) }; + } + OpValue::Protocol(value) => { + if let Err(message) = unsafe { value.write(output, present) } { + return unsafe { error(buffer, 4, message) }; + } + } + #[cfg(feature = "runtime-api")] + OpValue::Api(value) => unsafe { value.write(output) }, + #[cfg(feature = "runtime-external")] + OpValue::External(value) => unsafe { + output.cast::<*mut c_void>().write(managed::owned(value)) + }, + } + 1 + } + Poll::Ready(Err(ForeignFault::Typed(mut value))) => { + operation.completed = true; + if fault.is_null() { + return unsafe { error(buffer, 4, "missing async fault output") }; + } + // SAFETY: fault is one writable slot of the fault type. + unsafe { value.move_to(fault) }; + 2 + } + Poll::Ready(Err(ForeignFault::Runtime(message))) => { + operation.completed = true; + unsafe { error(buffer, 3, message) } + } + } + } + + /// # Safety + /// operation is one still-owned handle returned by async_sleep. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_op_release(operation: *mut AsyncOp) { + if !operation.is_null() { + drop(unsafe { Box::from_raw(operation) }); + } + } + + /// # Safety + /// The process-static descriptor and borrowed input satisfy create's + /// contract. A child frame, including all suspend-live locals, must + /// have passed the compiler's Send check. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_task_run( + function: *const AsyncFunction, + input: *const c_void, + buffer: *mut Buffer, + ) -> *mut AsyncOp { + if !unsafe { empty(buffer) } { + return ptr::null_mut(); + } + let Some(function) = (unsafe { function.as_ref() }) else { + unsafe { + error( + buffer, + ABI_INVALID_INPUT, + "missing async function descriptor", + ) + }; + return ptr::null_mut(); + }; + let (Some(output), Some(fault)) = (unsafe { function.output_type.as_ref() }, unsafe { + function.fault_type.as_ref() + }) else { + unsafe { error(buffer, ABI_INVALID_INPUT, "missing async result descriptor") }; + return ptr::null_mut(); + }; + if function.send_safe == 0 || output.transferable == 0 || fault.transferable == 0 { + unsafe { + error( + buffer, + ABI_INVALID_INPUT, + "async task frame is not transferable", + ) + }; + return ptr::null_mut(); + } + let future = match unsafe { ForeignFuture::new(function, input) } { + Ok(future) => SendForeignFuture(future), + Err(message) => { + unsafe { error(buffer, ABI_INVALID_INPUT, message) }; + return ptr::null_mut(); + } + }; + if output.size == 0 { + operation(async move { + task::run_typed(async move { future.await.map(drop) }) + .await + .map(TaskKind::Unit) + .map(OpValue::Task) + .map_err(ForeignFault::from) + }) + } else { + operation(async move { + task::run_typed(future) + .await + .map(|task| TaskKind::Value(task, output)) + .map(OpValue::Task) + .map_err(ForeignFault::from) + }) + } + } + + /// # Safety + /// input is one initialized owned row, disarmed after a non-null + /// return. The synchronous callback runs in the bounded blocking pool. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_task_run_sync( + function: *const SyncFunction, + input: *mut c_void, + buffer: *mut Buffer, + ) -> *mut AsyncOp { + let Some(prepared) = (unsafe { prepare_sync(function, input, buffer) }) else { + return ptr::null_mut(); + }; + let output_type = prepared.output.ty; + if prepared.unit_output { + operation(async move { + task::run_typed(async move { + task::blocking_typed(move || prepared.execute()) + .await + .map(drop) + }) + .await + .map(TaskKind::Unit) + .map(OpValue::Task) + .map_err(ForeignFault::from) + }) + } else { + operation(async move { + task::run_typed(async move { + task::blocking_typed(move || prepared.execute()).await + }) + .await + .map(|task| TaskKind::Value(task, output_type)) + .map(OpValue::Task) + .map_err(ForeignFault::from) + }) + } + } + + /// # Safety + /// task is one exclusive owned handle and is consumed by this call. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_task_wait(task: *mut c_void) -> *mut AsyncOp { + if task.is_null() { + return ptr::null_mut(); + } + let task = take_task(*unsafe { Box::from_raw(task.cast::<TaskHandle>()) }); + operation(async move { + match task? { + TaskKind::Value(task, _) => task::wait(task).await.map(OpValue::Value), + TaskKind::Unit(task) => task::wait(task).await.map(|()| OpValue::Unit), + } + }) + } + + /// # Safety + /// task is one exclusive unit-task handle consumed by this call. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_task_stop(task: *mut c_void) -> *mut AsyncOp { + if task.is_null() { + return ptr::null_mut(); + } + let task = take_task(*unsafe { Box::from_raw(task.cast::<TaskHandle>()) }); + operation(async move { + match task? { + TaskKind::Unit(task) => task::stop(task).await.map(|()| OpValue::Unit), + TaskKind::Value(_, _) => Err(ForeignFault::from( + "only a unit task can be stopped".to_owned(), + )), + } + }) + } + + /// # Safety + /// task transfers one live wrapper. The returned operation owns it. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_task_wait_timeout( + task: *mut c_void, + milliseconds: i64, + ) -> *mut AsyncOp { + if task.is_null() { + return ptr::null_mut(); + } + let task = take_task(*unsafe { Box::from_raw(task.cast::<TaskHandle>()) }); + operation(async move { + match task? { + TaskKind::Value(task, _) => task::wait_timeout(task, milliseconds) + .await + .map(OpValue::OptionalValue), + TaskKind::Unit(task) => task::wait_timeout(task, milliseconds) + .await + .map(|value| OpValue::OptionalUnit(value.is_some())), + } + }) + } + + /// # Safety + /// tasks is a borrowed array of distinct live wrappers of one output type. + /// Non-null return consumes every wrapper; null return consumes none. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_task_race( + tasks: *const *mut c_void, + count: u64, + buffer: *mut Buffer, + ) -> *mut AsyncOp { + if !unsafe { empty(buffer) } { + return ptr::null_mut(); + } + let prepared = (|| { + let count = usize::try_from(count).map_err(|_| "invalid race task count")?; + if count < 2 + || count > isize::MAX as usize / size_of::<*mut c_void>() + || tasks.is_null() + || !tasks.is_aligned() + { + return Err("race requires an array of at least two tasks"); + } + let pointers = unsafe { std::slice::from_raw_parts(tasks, count) }; + let mut handles = Vec::with_capacity(count); + let mut identities = std::collections::HashSet::with_capacity(count); + for pointer in pointers { + let handle = unsafe { pointer.cast::<TaskHandle>().as_ref() } + .ok_or("missing race task")?; + if !identities.insert(Arc::as_ptr(&handle.0)) { + return Err("race tasks must be distinct"); + } + handles.push(handle); + } + // Hold all locks until validation completes: errors never partly consume tasks. + let mut slots = handles + .iter() + .map(|handle| handle.0.lock().map_err(|_| "async task state lock failed")) + .collect::<Result<Vec<_>, _>>()?; + let output_type = slots[0] + .as_ref() + .ok_or("async task was already consumed")? + .output_type() + .map(std::ptr::from_ref); + let unit = output_type.is_none(); + for slot in &slots { + let task = slot.as_ref().ok_or("async task was already consumed")?; + if task.output_type().map(std::ptr::from_ref) != output_type { + return Err("race task outputs must agree"); + } + } + let tasks = slots + .iter_mut() + .map(|slot| slot.take().expect("validated race task")) + .collect::<Vec<_>>(); + drop(slots); + for pointer in pointers { + drop(unsafe { Box::from_raw(pointer.cast::<TaskHandle>()) }); + } + Ok((unit, tasks)) + })(); + let (unit, tasks) = match prepared { + Ok(value) => value, + Err(message) => { + unsafe { error(buffer, ABI_INVALID_INPUT, message) }; + return ptr::null_mut(); + } + }; + operation(async move { + if unit { + let tasks = tasks + .into_iter() + .map(|task| match task { + TaskKind::Unit(task) => task, + TaskKind::Value(_, _) => unreachable!("validated race outputs"), + }) + .collect(); + task::race(tasks).await.map(|()| OpValue::Unit) + } else { + let tasks = tasks + .into_iter() + .map(|task| match task { + TaskKind::Value(task, _) => task, + TaskKind::Unit(_) => unreachable!("validated race outputs"), + }) + .collect(); + task::race(tasks).await.map(OpValue::Value) + } + }) + } + + /// # Safety + /// task is a live borrowed wrapper. The inner task remains affine. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_task_retain(task: *const c_void) -> *mut c_void { + unsafe { task.cast::<TaskHandle>().as_ref() } + .map(|task| Box::into_raw(Box::new(TaskHandle(Arc::clone(&task.0)))).cast()) + .unwrap_or(ptr::null_mut()) + } + + /// # Safety + /// task is one exclusive owned handle, released at most once. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_task_release(task: *mut c_void) { + if !task.is_null() { + drop(unsafe { Box::from_raw(task.cast::<TaskHandle>()) }); + } + } + + /// # Safety + /// output is one writable pointer slot; error is a writable buffer. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_group_new( + limit: i64, + output: *mut *mut GroupHandle, + buffer: *mut Buffer, + ) -> u32 { + if !unsafe { empty(buffer) } || output.is_null() || !output.is_aligned() { + return ABI_INVALID_INPUT; + } + match task::Group::<ForeignFault>::new_typed(limit) { + Ok(group) => { + unsafe { + output.write(Box::into_raw(Box::new(GroupHandle(Arc::new(Mutex::new( + Some(group), + )))))) + }; + 0 + } + Err(message) => unsafe { error(buffer, 3, message) }, + } + } + + /// # Safety + /// group is live and function is a process-static checked unit future. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_group_run( + group: *mut GroupHandle, + function: *const AsyncFunction, + input: *const c_void, + buffer: *mut Buffer, + ) -> *mut AsyncOp { + if !unsafe { empty(buffer) } { + return ptr::null_mut(); + } + let (Some(group), Some(function)) = + (unsafe { group.as_ref() }, unsafe { function.as_ref() }) + else { + unsafe { error(buffer, ABI_INVALID_INPUT, "missing async group or function") }; + return ptr::null_mut(); + }; + let (Some(output), Some(fault)) = (unsafe { function.output_type.as_ref() }, unsafe { + function.fault_type.as_ref() + }) else { + unsafe { error(buffer, ABI_INVALID_INPUT, "missing async result descriptor") }; + return ptr::null_mut(); + }; + if output.size != 0 || function.send_safe == 0 || fault.transferable == 0 { + unsafe { + error( + buffer, + ABI_INVALID_INPUT, + "group child must be a transferable unit future", + ) + }; + return ptr::null_mut(); + } + let future = match unsafe { ForeignFuture::new(function, input) } { + Ok(future) => SendForeignFuture(future), + Err(message) => { + unsafe { error(buffer, ABI_INVALID_INPUT, message) }; + return ptr::null_mut(); + } + }; + let shared = Arc::clone(&group.0); + operation(async move { + let mut lease = GroupLease::take(shared)?; + lease + .group + .as_mut() + .expect("leased group") + .run(async move { future.await.map(drop) }) + .await?; + Ok(OpValue::Unit) + }) + } + + /// # Safety + /// group is a live borrowed wrapper; input is one initialized owned + /// row, disarmed after a non-null return. Output must be Unit. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_group_run_sync( + group: *mut GroupHandle, + function: *const SyncFunction, + input: *mut c_void, + buffer: *mut Buffer, + ) -> *mut AsyncOp { + if !unsafe { empty(buffer) } { + return ptr::null_mut(); + } + let Some(group) = (unsafe { group.as_ref() }) else { + unsafe { error(buffer, ABI_INVALID_INPUT, "missing async group") }; + return ptr::null_mut(); + }; + if unsafe { function.as_ref() } + .and_then(|function| unsafe { function.output_type.as_ref() }) + .is_none_or(|output| output.size != 0) + { + unsafe { error(buffer, ABI_INVALID_INPUT, "group child must return Unit") }; + return ptr::null_mut(); + } + let Some(prepared) = (unsafe { prepare_sync(function, input, buffer) }) else { + return ptr::null_mut(); + }; + let shared = Arc::clone(&group.0); + operation(async move { + let mut lease = GroupLease::take(shared)?; + lease + .group + .as_mut() + .expect("leased group") + .run(async move { + task::blocking_typed(move || prepared.execute()) + .await + .map(drop) + }) + .await?; + Ok(OpValue::Unit) + }) + } + + /// # Safety + /// group is consumed and no group operation may still borrow it. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_group_wait( + group: *mut GroupHandle, + ) -> *mut AsyncOp { + if group.is_null() { + return ptr::null_mut(); + } + let group = *unsafe { Box::from_raw(group) }; + operation(async move { + let group = GroupLease::take(group.0)? + .group + .take() + .expect("leased group"); + group.wait().await.map(|()| OpValue::Unit) + }) + } + + /// # Safety + /// group is consumed and no group operation may still borrow it. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_group_stop( + group: *mut GroupHandle, + ) -> *mut AsyncOp { + if group.is_null() { + return ptr::null_mut(); + } + let group = *unsafe { Box::from_raw(group) }; + operation(async move { + let group = GroupLease::take(group.0)? + .group + .take() + .expect("leased group"); + group.stop().await.map(|()| OpValue::Unit) + }) + } + + /// # Safety + /// group is a live borrowed wrapper. The inner group remains affine. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_group_retain( + group: *const GroupHandle, + ) -> *mut GroupHandle { + unsafe { group.as_ref() } + .map(|group| Box::into_raw(Box::new(GroupHandle(Arc::clone(&group.0))))) + .unwrap_or(ptr::null_mut()) + } + + /// # Safety + /// group is one exclusive owned handle. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_group_release(group: *mut GroupHandle) { + if !group.is_null() { + drop(unsafe { Box::from_raw(group) }); + } + } + + /// # Safety + /// output is a writable pointer slot; error is a writable buffer. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_channel_new( + capacity: i64, + output: *mut *mut ChannelHandle, + buffer: *mut Buffer, + ) -> u32 { + if !unsafe { empty(buffer) } || output.is_null() || !output.is_aligned() { + return ABI_INVALID_INPUT; + } + match Channel::new(capacity) { + Ok(channel) => { + unsafe { output.write(Box::into_raw(Box::new(ChannelHandle(channel)))) }; + 0 + } + Err(message) => unsafe { error(buffer, 3, message) }, + } + } + + /// # Safety + /// channel is a live borrowed handle. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_channel_retain( + channel: *const ChannelHandle, + ) -> *mut ChannelHandle { + unsafe { channel.as_ref() } + .map(|channel| Box::into_raw(Box::new(ChannelHandle(channel.0.clone())))) + .unwrap_or(ptr::null_mut()) + } + + /// # Safety + /// channel is one owned handle. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_channel_release(channel: *mut ChannelHandle) { + if !channel.is_null() { + drop(unsafe { Box::from_raw(channel) }); + } + } + + /// # Safety + /// element is one initialized concrete value, consumed on success; + /// descriptor and channel match its compiler-checked element type. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_channel_send( + channel: *mut ChannelHandle, + element_type: *const OwnedType, + element: *mut c_void, + buffer: *mut Buffer, + ) -> *mut AsyncOp { + if !unsafe { empty(buffer) } { + return ptr::null_mut(); + } + let Some(channel) = (unsafe { channel.as_ref() }) else { + unsafe { error(buffer, ABI_INVALID_INPUT, "missing async channel") }; + return ptr::null_mut(); + }; + if element.is_null() + || unsafe { element_type.as_ref() }.is_none_or(|ty| ty.transferable == 0) + { + unsafe { + error( + buffer, + ABI_INVALID_INPUT, + "channel element is not transferable", + ) + }; + return ptr::null_mut(); + } + let mut value = match unsafe { Owned::new(element_type) } { + Ok(value) => value, + Err(message) => { + unsafe { error(buffer, ABI_INVALID_INPUT, message) }; + return ptr::null_mut(); + } + }; + unsafe { value.take_from(element) }; + let channel = channel.0.clone(); + operation(async move { + channel.send(value).await.map_err(ForeignFault::from)?; + Ok(OpValue::Unit) + }) + } + + /// # Safety + /// channel is a live borrowed handle. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_channel_receive( + channel: *mut ChannelHandle, + ) -> *mut AsyncOp { + let Some(channel) = (unsafe { channel.as_ref() }) else { + return ptr::null_mut(); + }; + let channel = channel.0.clone(); + operation(async move { + channel + .receive() + .await + .map(OpValue::OptionalValue) + .map_err(ForeignFault::from) + }) + } + + /// # Safety + /// channel is a live borrowed handle. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_channel_close( + channel: *mut ChannelHandle, + ) -> *mut AsyncOp { + let Some(channel) = (unsafe { channel.as_ref() }) else { + return ptr::null_mut(); + }; + let channel = channel.0.clone(); + operation(async move { + channel + .close() + .await + .map(|()| OpValue::Unit) + .map_err(ForeignFault::from) + }) + } + + unsafe fn stream_element_type( + ty: *const OwnedType, + size: usize, + align: usize, + ) -> Result<(), managed::AbiFailure> { + let ty = + unsafe { ty.as_ref() }.ok_or_else(|| managed::invalid("missing stream type"))?; + if ty.transferable == 0 + || ty.size != size as u64 + || ty.align != align as u64 + || ty.move_init.is_none() + || ty.drop.is_none() + { + return Err(managed::invalid( + "invalid or non-transferable stream element type", + )); + } + Ok(()) + } + + unsafe fn memory_rows( + source: *const c_void, + kind: u32, + ) -> Result<Box<dyn Iterator<Item = StreamRow> + Send>, String> { + match kind { + 0 => { + let list = unsafe { managed::borrowed::<managed::ListHandle>(source) } + .map_err(|failure| failure.message)?; + Ok(Box::new( + list.values.clone().into_values().map(StreamRow::Managed), + )) + } + 1 => { + let bytes = unsafe { managed::borrowed::<Bytes>(source) } + .map_err(|failure| failure.message)?; + Ok(Box::new( + bytes + .clone() + .into_values() + .map(|value| StreamRow::Int(i64::from(value))), + )) + } + _ => Err("parallel memory source must be List or Bytes".into()), + } + } + + /// # Safety + /// source and context are borrowed matching the static descriptor; its + /// Send contract covers the source row and the complete packed inputs. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_parallel_each( + source: *const c_void, + kind: u32, + limit: i64, + function: *const ParallelFunction, + context: *const c_void, + buffer: *mut Buffer, + ) -> *mut AsyncOp { + if !unsafe { empty(buffer) } { + return ptr::null_mut(); + } + let prepared = (|| { + let handler = Arc::new(unsafe { ParallelHandler::new(function, context) }?); + if kind == 3 { + let stream = unsafe { managed::borrowed::<AsyncStreamHandle>(source) } + .map_err(|failure| failure.message)? + .0 + .clone(); + Ok::<_, String>((handler, None, Some(stream))) + } else { + let rows = unsafe { memory_rows(source, kind) }?; + if handler.asynchronous.is_some() { + Ok((handler, None, Some(AsyncStream::from_values(rows)))) + } else { + Ok((handler, Some(rows), None)) + } + } + })(); + let (handler, rows, stream) = match prepared { + Ok(prepared) => prepared, + Err(message) => { + unsafe { error(buffer, ABI_INVALID_INPUT, message) }; + return ptr::null_mut(); + } + }; + operation(async move { + if let Some(rows) = rows { + task::parallel_each_typed(rows, limit, move |row| handler.execute(row)).await?; + } else { + let stream = stream.expect("validated parallel stream"); + if handler.asynchronous.is_some() { + task::parallel_each_stream_typed(stream, limit, move |row| { + let future = handler.start(row); + async move { future?.await.map(drop) } + }) + .await?; + } else { + task::parallel_each_stream_typed(stream, limit, move |row| { + let handler = Arc::clone(&handler); + async move { handler.execute(row) } + }) + .await?; + } + } + Ok(OpValue::Unit) + }) + } + + /// # Safety + /// source/context match the checked handler; fault/error are distinct + /// writable slots. The synchronous Stream producer stays on this thread. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_parallel_each( + source: *const c_void, + kind: u32, + limit: i64, + function: *const ParallelFunction, + context: *const c_void, + fault: *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + if !unsafe { empty(buffer) } || fault.is_null() { + return 4; + } + let handler = match unsafe { ParallelHandler::new(function, context) } { + Ok(handler) if handler.synchronous.is_some() => handler, + Ok(_) => { + return unsafe { + error( + buffer, + 4, + "synchronous parallel_each requires a synchronous handler", + ) + }; + } + Err(message) => return unsafe { error(buffer, 4, message) }, + }; + let result = match kind { + 0 => match unsafe { managed::borrowed::<managed::ListHandle>(source) } { + Ok(list) => dever_runtime::concurrent::each_slice_typed( + list.values.values(), + limit, + |row| handler.execute(StreamRow::Managed(row.clone())), + ), + Err(failure) => return unsafe { error(buffer, 4, failure.message) }, + }, + 1 => match unsafe { managed::borrowed::<Bytes>(source) } { + Ok(bytes) => { + dever_runtime::concurrent::each_slice_typed(bytes.values(), limit, |row| { + handler.execute(StreamRow::Int(i64::from(*row))) + }) + } + Err(failure) => return unsafe { error(buffer, 4, failure.message) }, + }, + 2 => match unsafe { managed::borrowed::<managed::StreamHandle>(source) } { + Ok(stream) => dever_runtime::concurrent::each_typed( + std::iter::from_fn(|| stream.values.pull()).map(StreamRow::Managed), + limit, + |row| handler.execute(row), + ), + Err(failure) => return unsafe { error(buffer, 4, failure.message) }, + }, + _ => return unsafe { error(buffer, 4, "invalid synchronous parallel source") }, + }; + match result { + Ok(()) => 1, + Err(ForeignFault::Typed(mut value)) => { + unsafe { value.move_to(fault) }; + 2 + } + Err(ForeignFault::Runtime(message)) => unsafe { error(buffer, 3, message) }, + } + } + + /// # Safety + /// list is borrowed; descriptor names its exact transferable element type. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_stream_from_list( + list: *const c_void, + ty: *const OwnedType, + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { managed::checked_output(out, buffer) } { + return status; + } + let result = (|| { + let list = unsafe { managed::borrowed::<managed::ListHandle>(list) }?; + unsafe { stream_element_type(ty, list.element.size, list.element.align) }?; + Ok(AsyncStreamHandle(AsyncStream::from_values( + list.values.clone().into_values().map(StreamRow::Managed), + ))) + })(); + unsafe { managed::complete_handle(out, buffer, result) } + } + + /// # Safety + /// bytes is borrowed; ty is the process-static Int descriptor. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_stream_from_bytes( + bytes: *const c_void, + ty: *const OwnedType, + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { managed::checked_output(out, buffer) } { + return status; + } + let result = (|| { + unsafe { stream_element_type(ty, size_of::<i64>(), align_of::<i64>()) }?; + let bytes = unsafe { managed::borrowed::<Bytes>(bytes) }?.clone(); + Ok(AsyncStreamHandle(AsyncStream::from_values( + bytes + .into_values() + .map(|value| StreamRow::Int(i64::from(value))), + ))) + })(); + unsafe { managed::complete_handle(out, buffer, result) } + } + + /// # Safety + /// ty names Int; output/error are distinct writable slots in an active Scope. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_ticks( + milliseconds: i64, + ty: *const OwnedType, + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { managed::checked_output(out, buffer) } { + return status; + } + let result = unsafe { stream_element_type(ty, size_of::<i64>(), align_of::<i64>()) } + .and_then(|()| { + task::ticks(milliseconds) + .map(|stream| AsyncStreamHandle(stream.map(StreamRow::Int))) + .map_err(managed::runtime_error) + }); + unsafe { managed::complete_handle(out, buffer, result) } + } + + /// # Safety + /// channel is borrowed and contains checked transferable typed values. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_channel_stream( + channel: *const ChannelHandle, + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { managed::checked_output(out, buffer) } { + return status; + } + let result = unsafe { channel.as_ref() } + .ok_or_else(|| managed::invalid("missing async channel")) + .map(|channel| AsyncStreamHandle(channel.0.clone().stream().map(StreamRow::Moved))); + unsafe { managed::complete_handle(out, buffer, result) } + } + + /// # Safety + /// stream is a live borrowed AsyncStream handle, or null. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_stream_retain( + stream: *const c_void, + ) -> *mut c_void { + unsafe { managed::retain::<AsyncStreamHandle>(stream) } + } + + /// # Safety + /// stream transfers one AsyncStream handle, or is null. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_stream_release(stream: *mut c_void) { + unsafe { managed::release::<AsyncStreamHandle>(stream) } + } + + /// # Safety + /// stream is a borrowed AsyncStream handle; operation retains its cursor. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_stream_pull( + stream: *const c_void, + ) -> *mut AsyncOp { + let Ok(stream) = (unsafe { managed::borrowed::<AsyncStreamHandle>(stream) }) else { + return ptr::null_mut(); + }; + let stream = stream.0.clone(); + operation(async move { Ok(OpValue::StreamRow(stream.pull().await)) }) + } + + /// # Safety + /// stream is borrowed; output/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_stream_close( + stream: *const c_void, + out: *mut u8, + buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { managed::checked_output(out, buffer) } { + return status; + } + let result = unsafe { managed::borrowed::<AsyncStreamHandle>(stream) }.map(|stream| { + stream.0.close(); + 1 + }); + unsafe { managed::complete(out, buffer, result) } + } + + /// # Safety + /// host is a borrowed Text handle, cloned before the operation suspends. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_net_connect( + host: *const c_void, + port: i64, + ) -> *mut AsyncOp { + let Ok(host) = (unsafe { managed::borrowed::<String>(host) }) else { + return ptr::null_mut(); + }; + let host = host.clone(); + operation(async move { + net::connect(&host, port) + .await + .map(OpValue::Socket) + .map_err(ForeignFault::from) + }) + } + + /// # Safety + /// host is a borrowed Text handle, cloned before the operation suspends. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_net_connect_timeout( + host: *const c_void, + port: i64, + milliseconds: i64, + ) -> *mut AsyncOp { + let Ok(host) = (unsafe { managed::borrowed::<String>(host) }) else { + return ptr::null_mut(); + }; + let host = host.clone(); + operation(async move { + net::connect_timeout(&host, port, milliseconds) + .await + .map(OpValue::Socket) + .map_err(ForeignFault::from) + }) + } + + /// # Safety + /// host is a borrowed Text handle, cloned before the operation suspends. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_net_listen( + host: *const c_void, + port: i64, + ) -> *mut AsyncOp { + let Ok(host) = (unsafe { managed::borrowed::<String>(host) }) else { + return ptr::null_mut(); + }; + let host = host.clone(); + operation(async move { + net::listen(&host, port) + .await + .map(OpValue::Listener) + .map_err(ForeignFault::from) + }) + } + + /// # Safety + /// listener is a borrowed Listener handle. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_net_accept( + listener: *const c_void, + ) -> *mut AsyncOp { + let Ok(listener) = (unsafe { managed::borrowed::<net::Listener>(listener) }) else { + return ptr::null_mut(); + }; + let listener = listener.clone(); + operation(async move { + net::accept(&listener) + .await + .map(OpValue::Socket) + .map_err(ForeignFault::from) + }) + } + + /// # Safety + /// socket is a borrowed Socket handle. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_net_read( + socket: *const c_void, + limit: i64, + ) -> *mut AsyncOp { + let Ok(socket) = (unsafe { managed::borrowed::<net::Socket>(socket) }) else { + return ptr::null_mut(); + }; + let socket = socket.clone(); + operation(async move { + net::read(&socket, limit) + .await + .map(OpValue::Bytes) + .map_err(ForeignFault::from) + }) + } + + /// # Safety + /// socket and bytes are borrowed handles, cloned before suspension. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_net_write( + socket: *const c_void, + bytes: *const c_void, + ) -> *mut AsyncOp { + let (Ok(socket), Ok(bytes)) = ( + unsafe { managed::borrowed::<net::Socket>(socket) }, + unsafe { managed::borrowed::<Bytes>(bytes) }, + ) else { + return ptr::null_mut(); + }; + let (socket, bytes) = (socket.clone(), bytes.clone()); + operation(async move { + net::write(&socket, &bytes) + .await + .map(|()| OpValue::Bool(true)) + .map_err(ForeignFault::from) + }) + } + + struct StreamEvent { + element: managed::ValueOps, + success: unsafe extern "C" fn(*const c_void, *mut c_void), + failed: unsafe extern "C" fn(*const c_void, *mut c_void), + } + + impl StreamEvent { + unsafe fn new( + event: *const managed::ReadEventDescriptor, + ty: *const OwnedType, + ) -> Result<Self, managed::AbiFailure> { + let event = unsafe { event.as_ref() } + .ok_or_else(|| managed::invalid("missing stream event descriptor"))?; + let element = unsafe { managed::descriptor(event.element_type, false) }?; + unsafe { stream_element_type(ty, element.size, element.align) }?; + let success = event + .chunk + .ok_or_else(|| managed::invalid("missing stream success callback"))?; + let failed = event + .failed + .ok_or_else(|| managed::invalid("missing stream failure callback"))?; + Ok(Self { + element, + success, + failed, + }) + } + + fn row<T>(&self, result: Result<T, String>) -> StreamRow { + StreamRow::Managed(match result { + Ok(value) => managed::initialize_event(value, self.element, self.success), + Err(message) => managed::initialize_event(message, self.element, self.failed), + }) + } + } + + /// # Safety + /// socket is borrowed; callbacks initialize the exact transferable event type. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_net_chunks( + socket: *const c_void, + limit: i64, + event: *const managed::ReadEventDescriptor, + ty: *const OwnedType, + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { managed::checked_output(out, buffer) } { + return status; + } + let result = (|| { + let event = unsafe { StreamEvent::new(event, ty) }?; + let socket = unsafe { managed::borrowed::<net::Socket>(socket) }?.clone(); + let stream = net::chunks(socket, limit).map_err(managed::runtime_error)?; + Ok(AsyncStreamHandle(stream.map(move |value| event.row(value)))) + })(); + unsafe { managed::complete_handle(out, buffer, result) } + } + + /// # Safety + /// listener is borrowed; callbacks initialize the exact transferable event type. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_net_connections( + listener: *const c_void, + event: *const managed::ReadEventDescriptor, + ty: *const OwnedType, + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { managed::checked_output(out, buffer) } { + return status; + } + let result = (|| { + let event = unsafe { StreamEvent::new(event, ty) }?; + let listener = unsafe { managed::borrowed::<net::Listener>(listener) }?.clone(); + Ok(AsyncStreamHandle( + net::connections(listener).map(move |value| event.row(value)), + )) + })(); + unsafe { managed::complete_handle(out, buffer, result) } + } + + #[cfg(feature = "runtime-api")] + mod api_application { + use super::*; + use dever_runtime::{api, application, auth, config, http}; + + pub(super) enum Output { + UploadId(dever_runtime::orm::Uuid), + #[cfg(any(feature = "runtime-sqlite", feature = "runtime-postgres"))] + Permissions(managed::ListHandle), + } + impl Output { + pub(super) unsafe fn write(self, output: *mut c_void) { + let handle = match self { + Self::UploadId(value) => managed::owned(value), + #[cfg(any(feature = "runtime-sqlite", feature = "runtime-postgres"))] + Self::Permissions(value) => managed::owned(value), + }; + unsafe { output.cast::<*mut c_void>().write(handle) }; + } + } + + #[repr(C)] + #[derive(Clone, Copy)] + pub struct Name { + pointer: *const u8, + len: u64, + } + #[repr(C)] + pub struct Binding { + explicit: *const Name, + root: Name, + tenant: u8, + } + #[repr(C)] + pub struct TransactionBindings { + bindings: *const Binding, + count: u64, + } + type Decode = unsafe extern "C" fn(*mut c_void, i64, *mut c_void, *mut Buffer) -> u32; + type FaultStatus = unsafe extern "C" fn(*const c_void) -> i64; + type FaultMessage = unsafe extern "C" fn(*const c_void) -> *const c_void; + type PackClaims = unsafe extern "C" fn( + *const c_void, + *const c_void, + *const c_void, + *const c_void, + *mut c_void, + *mut Buffer, + ) -> u32; + type UnpackIdentity = unsafe extern "C" fn( + *const c_void, + *mut *mut c_void, + *mut i64, + *mut u8, + *mut i64, + *mut u8, + *mut Buffer, + ) -> u32; + #[repr(C)] + pub struct ApiHandler { + asynchronous: *const AsyncFunction, + synchronous: *const SyncFunction, + input_type: *const OwnedType, + decode: Option<Decode>, + fault_status: Option<FaultStatus>, + fault_message: Option<FaultMessage>, + } + #[repr(C)] + pub struct ApiAuth { + asynchronous: *const AsyncFunction, + synchronous: *const SyncFunction, + input_type: *const OwnedType, + pack: Option<PackClaims>, + unpack: Option<UnpackIdentity>, + fault_status: Option<FaultStatus>, + } + #[repr(C)] + pub struct Permission { + key: Name, + component: Name, + domain: Name, + site: Name, + action: Name, + method: Name, + } + #[repr(C)] + pub struct Route { + method: Name, + path: Name, + directory: *const Name, + directory_count: u64, + components: *const Name, + component_count: u64, + permission: *const Permission, + auth: *const ApiAuth, + handler: *const ApiHandler, + anonymous: u8, + multipart: u8, + detail: u8, + } + + unsafe fn record<'a, T>(raw: *const T) -> Result<&'a T, managed::AbiFailure> { + unsafe { managed::borrowed(raw.cast()) } + } + unsafe fn array<'a, T>( + raw: *const T, + count: u64, + ) -> Result<&'a [T], managed::AbiFailure> { + let count = usize::try_from(count) + .map_err(|_| managed::invalid("invalid API array count"))?; + if count == 0 { + return Ok(&[]); + } + if raw.is_null() + || !raw.is_aligned() + || count > isize::MAX as usize / size_of::<T>().max(1) + { + return Err(managed::invalid("invalid API array")); + } + Ok(unsafe { std::slice::from_raw_parts(raw, count) }) + } + unsafe fn name<'a>(raw: &Name) -> Result<&'a str, managed::AbiFailure> { + std::str::from_utf8(unsafe { array(raw.pointer, raw.len) }?) + .map_err(|_| managed::invalid("API name is not UTF-8")) + } + unsafe fn names( + raw: *const Name, + count: u64, + ) -> Result<Vec<&'static str>, managed::AbiFailure> { + unsafe { array(raw, count) }? + .iter() + .map(|value| unsafe { name(value) }) + .collect() + } + fn flag(value: u8) -> Result<bool, managed::AbiFailure> { + match value { + 0 => Ok(false), + 1 => Ok(true), + _ => Err(managed::invalid("invalid API boolean")), + } + } + unsafe fn session( + raw: *const c_void, + ) -> Result<Arc<application::Session>, managed::AbiFailure> { + unsafe { managed::borrowed::<Arc<application::Session>>(raw) }.cloned() + } + fn buffer() -> Buffer { + Buffer { + ptr: ptr::null_mut(), + len: 0, + } + } + fn callback(status: u32, buffer: Buffer) -> Result<(), String> { + if status == 0 && buffer.ptr.is_null() && buffer.len == 0 { + return Ok(()); + } + let message = if buffer.ptr.is_null() { + "API callback failed without a diagnostic".to_owned() + } else { + let bytes = + unsafe { std::slice::from_raw_parts(buffer.ptr, buffer.len as usize) }; + let message = String::from_utf8_lossy(bytes).into_owned(); + unsafe { super::super::dever_rt_v1_buffer_free(buffer.ptr, buffer.len) }; + message + }; + Err(message) + } + + struct Callable { + asynchronous: Option<&'static AsyncFunction>, + synchronous: Option<&'static SyncFunction>, + input: &'static OwnedType, + output: &'static OwnedType, + fault_status: FaultStatus, + } + // Descriptors are immutable compiler output. Each call creates its own + // concrete input/frame/result; source transfer rules are checked before emission. + unsafe impl Send for Callable {} + unsafe impl Sync for Callable {} + impl Callable { + unsafe fn new( + asynchronous: *const AsyncFunction, + synchronous: *const SyncFunction, + input: *const OwnedType, + fault_status: Option<FaultStatus>, + ) -> Result<Self, managed::AbiFailure> { + let asynchronous = if asynchronous.is_null() { + None + } else { + Some(unsafe { record(asynchronous) }?) + }; + let synchronous = if synchronous.is_null() { + None + } else { + Some(unsafe { record(synchronous) }?) + }; + let (output, fault, send) = match (asynchronous, synchronous) { + (Some(function), None) + if function.create.is_some() + && function.resume.is_some() + && function.destroy.is_some() + && function.done.is_some() => + { + ( + function.output_type, + function.fault_type, + function.send_safe, + ) + } + (None, Some(function)) + if function.invoke.is_some() && function.input_type == input => + { + ( + function.output_type, + function.fault_type, + function.send_safe, + ) + } + _ => return Err(managed::invalid("invalid API callable")), + }; + for ty in [input, output, fault] { + let (ty, _) = unsafe { owned_layout(ty) }.map_err(managed::invalid)?; + if send == 0 || ty.transferable == 0 { + return Err(managed::invalid("API callable is not transferable")); + } + } + Ok(Self { + asynchronous, + synchronous, + input: unsafe { &*input }, + output: unsafe { &*output }, + fault_status: fault_status + .ok_or_else(|| managed::invalid("missing API failure mapping"))?, + }) + } + async fn invoke(&self, input: Owned) -> Result<Owned, ForeignFault> { + if let Some(function) = self.asynchronous { + let future = unsafe { ForeignFuture::new(function, input.pointer()) } + .map(SendForeignFuture) + .map_err(ForeignFault::from)?; + drop(input); + return future.await; + } + let function = self.synchronous.expect("validated API function"); + PreparedSync { + input, + output: unsafe { Owned::new(function.output_type) } + .map_err(ForeignFault::from)?, + fault: unsafe { Owned::new(function.fault_type) } + .map_err(ForeignFault::from)?, + invoke: function.invoke.expect("validated API function"), + unit_output: false, + } + .execute() + } + fn failure( + &self, + fault: ForeignFault, + message: Option<FaultMessage>, + ) -> http::Response { + let status = match &fault { + ForeignFault::Typed(fault) => unsafe { + (self.fault_status)(fault.pointer()) + }, + ForeignFault::Runtime(_) => 500, + }; + if status == 400 + && let (ForeignFault::Typed(fault), Some(message)) = (&fault, message) + { + let text = unsafe { message(fault.pointer()) }; + if !text.is_null() { + return match unsafe { managed::borrowed::<String>(text) } { + Ok(text) => api::invalid_input(api::InputError(text.clone())), + Err(_) => api::internal_error(), + }; + } + } + match status { + 400 | 401 | 403 | 404 | 409 | 429 => api::standard_error(status), + _ => { + dever_runtime::log::error("API handler failed", Vec::new()); + api::internal_error() + } + } + } + } + + enum Inputs { + Fields(api::Inputs), + Multipart(api::upload::Multipart), + } + impl Inputs { + fn field( + &mut self, + name: &str, + required: bool, + kind: api::QueryValue, + ) -> Result<String, api::InputError> { + match (self, required) { + (Self::Fields(inputs), true) => inputs.json(name, kind), + (Self::Fields(inputs), false) => inputs.optional_json(name, kind), + (Self::Multipart(inputs), true) => inputs.json(name, kind), + (Self::Multipart(inputs), false) => inputs.optional_json(name, kind), + } + } + fn raw( + &mut self, + name: &str, + required: bool, + ) -> Result<Option<String>, api::InputError> { + match (self, required) { + (Self::Fields(inputs), true) => inputs.raw_json(name).map(Some), + (Self::Fields(inputs), false) => inputs.optional_raw_json(name), + (Self::Multipart(inputs), true) => inputs.raw_json(name).map(Some), + (Self::Multipart(inputs), false) => inputs.optional_raw_json(name), + } + } + fn finish(self) -> Result<(), api::InputError> { + match self { + Self::Fields(inputs) => inputs.finish(), + Self::Multipart(inputs) => inputs.finish(), + } + } + } + + struct Authentication { + function: Callable, + pack: PackClaims, + unpack: UnpackIdentity, + } + impl Authentication { + unsafe fn new(raw: *const ApiAuth) -> Result<Self, managed::AbiFailure> { + let auth = unsafe { record(raw) }?; + Ok(Self { + function: unsafe { + Callable::new( + auth.asynchronous, + auth.synchronous, + auth.input_type, + auth.fault_status, + ) + }?, + pack: auth + .pack + .ok_or_else(|| managed::invalid("missing claims pack callback"))?, + unpack: auth + .unpack + .ok_or_else(|| managed::invalid("missing identity unpack callback"))?, + }) + } + + async fn verify( + &self, + claims: &auth::Claims, + ) -> Result<auth::Identity, ForeignFault> { + let mut input = + unsafe { Owned::new(self.function.input) }.map_err(ForeignFault::from)?; + { + let subject = managed::owned(claims.subject.clone()); + let session = managed::owned(claims.session.clone()); + let tenant = claims + .tenant + .clone() + .map_or(ptr::null_mut(), managed::owned); + let site = managed::owned(claims.site.clone()); + let mut diagnostic = buffer(); + let status = unsafe { + (self.pack)( + subject, + session, + tenant, + site, + input.pointer(), + &mut diagnostic, + ) + }; + unsafe { + managed::release::<String>(subject); + managed::release::<String>(session); + managed::release::<String>(tenant); + managed::release::<String>(site); + } + // A successful pack owns a complete row even if its diagnostic is malformed. + input.initialized = status == 0; + callback(status, diagnostic).map_err(ForeignFault::from)?; + } + let output = self.function.invoke(input).await?; + let (mut id, mut user, mut has_user, mut tenant, mut has_tenant) = + (ptr::null_mut(), 0, 0, 0, 0); + let mut diagnostic = buffer(); + let status = unsafe { + (self.unpack)( + output.pointer(), + &mut id, + &mut user, + &mut has_user, + &mut tenant, + &mut has_tenant, + &mut diagnostic, + ) + }; + let result = callback(status, diagnostic).and_then(|()| { + if has_user > 1 || has_tenant > 1 { + return Err("invalid identity presence flag".into()); + } + unsafe { managed::borrowed::<String>(id) } + .map(|id| auth::Identity { + id: id.clone(), + user_id: (has_user == 1).then_some(user), + tenant_id: (has_tenant == 1).then_some(tenant), + }) + .map_err(|error| error.message) + }); + unsafe { managed::release::<String>(id) }; + result.map_err(ForeignFault::from) + } + } + + #[cfg(any(feature = "runtime-sqlite", feature = "runtime-postgres"))] + #[repr(C)] + pub struct JobDescriptor { + target: Name, + schema: Name, + attempts: u32, + timeout_ms: u32, + schedule: *const Name, + binding: *const Binding, + components: *const Name, + component_count: u64, + handler: *const ApiHandler, + } + #[cfg(any(feature = "runtime-sqlite", feature = "runtime-postgres"))] + #[repr(C)] + pub struct JobAuth { + provider: Name, + auth: *const ApiAuth, + } + #[cfg(any(feature = "runtime-sqlite", feature = "runtime-postgres"))] + struct CompiledJob { + function: Callable, + decode: Decode, + components: Vec<&'static str>, + } + #[cfg(any(feature = "runtime-sqlite", feature = "runtime-postgres"))] + pub(super) struct JobRegistry { + pub(super) application: Arc<application::Session>, + pub(super) resources: Arc<dever_runtime::job::resources::Resources>, + handlers: Vec<CompiledJob>, + authentication: std::collections::BTreeMap<&'static str, Authentication>, + permissions: std::collections::BTreeSet<(&'static str, &'static str)>, + components: Vec<&'static str>, + } + #[cfg(any(feature = "runtime-sqlite", feature = "runtime-postgres"))] + pub(super) unsafe fn job_registry( + raw: *const c_void, + ) -> Result<Arc<JobRegistry>, managed::AbiFailure> { + unsafe { managed::borrowed::<Arc<JobRegistry>>(raw) }.cloned() + } + #[cfg(any(feature = "runtime-sqlite", feature = "runtime-postgres"))] + impl JobRegistry { + async fn require_components( + &self, + claim: &dever_runtime::job::store::Claim, + ) -> Result<(), dever_runtime::job::store::Outcome> { + let Some(index) = self + .resources + .bindings + .iter() + .position(|binding| binding.spec.target == claim.target) + else { + return Ok(()); + }; + dever_runtime::job::require_components( + &claim.execution, + &self.handlers[index].components, + &self.components, + ) + .await + } + + async fn authorize( + &self, + claim: &dever_runtime::job::store::Claim, + ) -> Result<(), dever_runtime::job::store::Outcome> { + use dever_runtime::job::{ExecutionIdentity, store::Outcome}; + let ExecutionIdentity::User { + tenant_id, + provider, + site, + subject, + session, + tenant_claim, + permission_key, + } = &claim.execution + else { + return self.require_components(claim).await; + }; + if self.permissions.is_empty() + || auth::validate_job_source(provider, site).is_err() + { + return Err(Outcome::InvalidIdentity); + } + let authentication = self + .authentication + .get(provider.as_str()) + .ok_or(Outcome::InvalidIdentity)?; + let claims = auth::Claims { + subject: subject.clone(), + session: session.clone(), + tenant: tenant_claim.clone(), + site: site.clone(), + }; + let identity = authentication.verify(&claims).await.map_err(|fault| { + let status = match &fault { + ForeignFault::Typed(value) => unsafe { + (authentication.function.fault_status)(value.pointer()) + }, + ForeignFault::Runtime(_) => 500, + }; + if matches!(status, 401 | 403) { + Outcome::IdentityRejected + } else { + dever_runtime::log::error( + "Job identity verification failed", + Vec::new(), + ); + Outcome::Retry + } + })?; + if auth::validate_job_identity(&claims, &identity).is_err() + || identity.tenant_id != *tenant_id + || !self + .permissions + .contains(&(permission_key.as_str(), site.as_str())) + { + return Err(Outcome::IdentityRejected); + } + self.require_components(claim).await?; + let user_id = identity.user_id.ok_or(Outcome::IdentityRejected)?; + let settings = self.application.settings(); + let database = if tenant_id.is_some() { + let tenant = settings.tenant().ok_or(Outcome::InvalidIdentity)?; + dever_runtime::tenant::database(tenant.database()) + .await + .map_err(|_| Outcome::Retry)? + } else { + let connection = settings + .tenant() + .map(|tenant| tenant.database()) + .unwrap_or("default"); + self.application + .databases() + .database(Some(connection), "") + .map_err(|_| Outcome::Retry)? + }; + match auth::store::authorize(database, user_id, site, permission_key).await { + Ok(true) => Ok(()), + Ok(false) => Err(Outcome::IdentityRejected), + Err(_) => Err(Outcome::Retry), + } + } + + async fn dispatch( + &self, + claim: dever_runtime::job::store::Claim, + ) -> dever_runtime::job::store::Outcome { + use dever_runtime::job::store::Outcome; + if let Err(outcome) = self.authorize(&claim).await { + return outcome; + } + let Some(index) = self + .resources + .bindings + .iter() + .position(|binding| binding.spec.target == claim.target) + else { + return Outcome::UnknownTarget; + }; + if self.resources.bindings[index].spec.schema != claim.schema { + return Outcome::SchemaMismatch; + } + let handler = &self.handlers[index]; + let mut input = match unsafe { Owned::new(handler.function.input) } { + Ok(input) => input, + Err(_) => return Outcome::Retry, + }; + { + let payload = managed::owned(claim.payload); + let mut diagnostic = buffer(); + let status = unsafe { + (handler.decode)(payload, 0, input.pointer(), &mut diagnostic) + }; + unsafe { managed::release::<String>(payload) }; + input.initialized = status == 0; + if callback(status, diagnostic).is_err() { + return if status == 1 { + Outcome::InvalidPayload + } else { + Outcome::Retry + }; + } + } + match handler.function.invoke(input).await { + Ok(_) => Outcome::Success, + Err(_) => Outcome::Retry, + } + } + + async fn serve(self: Arc<Self>) -> Result<(), String> { + if !dever_runtime::lifecycle::worker_enabled() { + return Ok(()); + } + let bindings = self.resources.bindings.clone(); + let clock = self.resources.clock.clone(); + let settings = self.application.settings().jobs()?; + dever_runtime::job::worker::serve(bindings, clock, settings, move |claim| { + let registry = self.clone(); + async move { Box::pin(registry.dispatch(claim)).await } + }) + .await + } + } + + #[cfg(any(feature = "runtime-sqlite", feature = "runtime-postgres"))] + /// # Safety + /// Compiler descriptors and their names are immutable and process-static. + /// Session and Clock are borrowed; out receives a registry owned by this root. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_job_session_new( + raw: *const c_void, + clock: *const c_void, + descriptors: *const JobDescriptor, + count: u64, + authentication: *const JobAuth, + auth_count: u64, + permissions: *const Permission, + permission_count: u64, + components: *const Name, + component_count: u64, + out: *mut *mut c_void, + error_buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { managed::checked_output(out, error_buffer) } { + return status; + } + let result = (|| { + let application = unsafe { session(raw) }?; + let clock = + unsafe { managed::borrowed::<dever_runtime::time::Clock>(clock) }?.clone(); + let mut bindings = Vec::new(); + let mut handlers = Vec::new(); + let mut targets = std::collections::BTreeSet::new(); + for descriptor in unsafe { array(descriptors, count) }? { + let target = unsafe { name(&descriptor.target) }?; + if !targets.insert(target) { + return Err(managed::invalid("duplicate compiled Job target")); + } + let binding = unsafe { record(descriptor.binding) }?; + let scope = if flag(binding.tenant)? { + dever_runtime::database::ModelScope::Tenant + } else { + dever_runtime::database::ModelScope::Global + }; + let storage = dever_runtime::database::StorageBinding::new( + if binding.explicit.is_null() { + None + } else { + Some(unsafe { name(record(binding.explicit)?) }?) + }, + unsafe { name(&binding.root) }?, + scope, + ); + let spec = dever_runtime::job::Spec { + target, + schema: unsafe { name(&descriptor.schema) }?, + attempts: descriptor.attempts, + timeout_ms: descriptor.timeout_ms, + }; + let schedule = if descriptor.schedule.is_null() { + None + } else { + Some( + dever_runtime::cron::Cron::parse(unsafe { + name(record(descriptor.schedule)?) + }?) + .map_err(managed::invalid)?, + ) + }; + let handler = unsafe { record(descriptor.handler) }?; + let function = unsafe { + Callable::new( + handler.asynchronous, + handler.synchronous, + handler.input_type, + handler.fault_status, + ) + }?; + if function.output.size != 0 || function.output.align != 1 { + return Err(managed::invalid("Job handler must return Unit")); + } + handlers.push(CompiledJob { + function, + decode: handler + .decode + .ok_or_else(|| managed::invalid("missing Job payload decoder"))?, + components: unsafe { + names(descriptor.components, descriptor.component_count) + }?, + }); + bindings.push(dever_runtime::job::worker::Binding::owned( + storage, scope, spec, schedule, + )); + } + let mut auth = std::collections::BTreeMap::new(); + for descriptor in unsafe { array(authentication, auth_count) }? { + let provider = unsafe { name(&descriptor.provider) }?; + if auth + .insert(provider, unsafe { Authentication::new(descriptor.auth) }?) + .is_some() + { + return Err(managed::invalid("duplicate Job authentication provider")); + } + } + let permissions = unsafe { array(permissions, permission_count) }? + .iter() + .map(|permission| { + Ok((unsafe { name(&permission.key) }?, unsafe { + name(&permission.site) + }?)) + }) + .collect::<Result<_, managed::AbiFailure>>()?; + let components = unsafe { names(components, component_count) }?; + let resources = + Arc::new(dever_runtime::job::resources::Resources { bindings, clock }); + application::with_resources(Some(application.clone()), || { + resources.validate(dever_runtime::lifecycle::worker_enabled()) + }) + .map_err(managed::runtime_error)?; + application + .initialize_jobs(resources.clone()) + .map_err(managed::runtime_error)?; + Ok(Arc::new(JobRegistry { + application, + resources, + handlers, + authentication: auth, + permissions, + components, + })) + })(); + unsafe { managed::complete_handle(out, error_buffer, result) } + } + + #[cfg(any(feature = "runtime-sqlite", feature = "runtime-postgres"))] + /// # Safety + /// Transfers one registry after application tasks have drained. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_job_session_release(raw: *mut c_void) { + unsafe { managed::release::<Arc<JobRegistry>>(raw) }; + } + + #[cfg(any(feature = "runtime-sqlite", feature = "runtime-postgres"))] + /// # Safety + /// Registry is borrowed and retained until this operation completes or is cancelled. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_job_drain( + raw: *const c_void, + limit: i64, + error_buffer: *mut Buffer, + ) -> *mut AsyncOp { + if !unsafe { empty(error_buffer) } { + return ptr::null_mut(); + } + let registry = match unsafe { job_registry(raw) } { + Ok(registry) => registry, + Err(failure) => { + unsafe { error(error_buffer, failure.status, failure.message) }; + return ptr::null_mut(); + } + }; + operation_with_output( + async move { + let application = registry.application.clone(); + application + .scope(async move { + let bindings = registry.resources.bindings.clone(); + let clock = registry.resources.clock.clone(); + dever_runtime::job::worker::drain( + bindings, + clock, + limit, + move |claim| { + let registry = registry.clone(); + async move { Box::pin(registry.dispatch(claim)).await } + }, + ) + .await + }) + .await + .map(|count| OpValue::Database(database::Output::Int(count))) + .map_err(ForeignFault::from) + }, + Some(OutputSlots::Int), + ) + } + + /// # Safety + /// Borrows the installed Session; booleans select compiler-owned service roots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_job_configure( + raw: *const c_void, + api: u8, + worker: u8, + error_buffer: *mut Buffer, + ) -> u32 { + if !unsafe { empty(error_buffer) } { + return 2; + } + let result = (|| { + let session = unsafe { session(raw) }?; + let api = flag(api)?; + let worker = flag(worker)?; + application::with_resources(Some(session), || { + dever_runtime::lifecycle::configure(api, worker) + }) + .map_err(managed::runtime_error) + })(); + match result { + Ok(()) => 0, + Err(failure) => unsafe { error(error_buffer, failure.status, failure.message) }, + } + } + + #[cfg(any(feature = "runtime-sqlite", feature = "runtime-postgres"))] + /// # Safety + /// Process-static callback metadata. The compiler alone selects System roots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_job_system_scope( + tenant: i64, + has_tenant: u8, + function: *const AsyncFunction, + input: *const c_void, + error_buffer: *mut Buffer, + ) -> *mut AsyncOp { + if !unsafe { empty(error_buffer) } { + return ptr::null_mut(); + } + let prepared = (|| { + let tenant = flag(has_tenant)?.then_some(tenant); + if tenant.is_some_and(|tenant| tenant <= 0) { + return Err(managed::invalid("invalid System tenant")); + } + let function = unsafe { record(function) }?; + let (_, output) = + unsafe { owned_layout(function.output_type) }.map_err(managed::invalid)?; + let (_, fault) = + unsafe { owned_layout(function.fault_type) }.map_err(managed::invalid)?; + let future = + unsafe { ForeignFuture::new(function, input) }.map_err(managed::invalid)?; + Ok(( + tenant, + future, + OutputSlots::Typed { + align: output.align(), + fault_align: fault.align(), + }, + )) + })(); + let (tenant, future, slots) = match prepared { + Ok(value) => value, + Err(failure) => { + unsafe { error(error_buffer, failure.status, failure.message) }; + return ptr::null_mut(); + } + }; + operation_with_output( + async move { + let execution = async move { + match tenant { + Some(tenant) => dever_runtime::tenant::scope(tenant, future).await, + None => future.await, + } + }; + dever_runtime::job::scope_system(tenant, execution) + .await + .map(OpValue::Value) + }, + Some(slots), + ) + } + + struct CompiledRoute { + method: &'static str, + path: &'static str, + directory: Vec<&'static str>, + components: Vec<&'static str>, + permission: Option<(&'static str, &'static str)>, + auth: Authentication, + function: Callable, + decode: Decode, + fault_message: Option<FaultMessage>, + anonymous: bool, + multipart: bool, + detail: bool, + } + impl CompiledRoute { + unsafe fn new(row: &Route) -> Result<Self, managed::AbiFailure> { + let anonymous = flag(row.anonymous)?; + if row.auth.is_null() || (!anonymous && row.permission.is_null()) { + return Err(managed::invalid( + "API route is missing authentication or permission metadata", + )); + } + let handler = unsafe { record(row.handler) }?; + let function = unsafe { + Callable::new( + handler.asynchronous, + handler.synchronous, + handler.input_type, + handler.fault_status, + ) + }?; + if function.output.size != size_of::<*mut c_void>() as u64 + || function.output.align != align_of::<*mut c_void>() as u64 + { + return Err(managed::invalid("API handler must return one owned Text")); + } + let auth = unsafe { Authentication::new(row.auth) }?; + let permission = if row.permission.is_null() { + None + } else { + let permission = unsafe { record(row.permission) }?; + Some((unsafe { name(&permission.key) }?, unsafe { + name(&permission.site) + }?)) + }; + Ok(Self { + method: unsafe { name(&row.method) }?, + path: unsafe { name(&row.path) }?, + directory: unsafe { names(row.directory, row.directory_count) }?, + components: unsafe { names(row.components, row.component_count) }?, + permission, + auth, + function, + decode: handler + .decode + .ok_or_else(|| managed::invalid("missing API input decoder"))?, + fault_message: handler.fault_message, + anonymous, + multipart: flag(row.multipart)?, + detail: flag(row.detail)?, + }) + } + + async fn identity( + &self, + prepared: &auth::Prepared, + ) -> Result<Option<auth::Identity>, http::Response> { + let Some(claims) = prepared.claims() else { + return Ok(None); + }; + self.auth + .verify(claims) + .await + .map(Some) + .map_err(|error| self.auth.function.failure(error, None)) + } + + async fn call( + &self, + mut request: http::Request, + detail: i64, + tenant_components: &[&str], + ) -> http::Response { + #[cfg(any(feature = "runtime-sqlite", feature = "runtime-postgres"))] + { + if let Err(error) = + auth::require_components(&self.components, tenant_components).await + { + return api::auth_error(error); + } + if let Some((permission, site)) = self.permission + && let Err(error) = auth::authorize_permission(permission, site).await + { + return api::auth_error(error); + } + } + #[cfg(not(any(feature = "runtime-sqlite", feature = "runtime-postgres")))] + { + let _ = tenant_components; + if self.permission.is_some() + || (!self.components.is_empty() + && config::current_settings().tenant().is_some()) + { + return api::internal_error(); + } + } + if !self.multipart + && let Err(error) = api::upload::read_body(&mut request).await + { + return api::invalid_input(error); + } + let detail = if self.detail { + let path = request.target.split('?').next().unwrap_or(""); + match path + .strip_prefix(self.path) + .and_then(|suffix| suffix.strip_prefix('/')) + .and_then(|id| id.parse::<i64>().ok()) + .filter(|id| *id > 0) + { + Some(id) => id, + None => return api::not_found(), + } + } else { + detail + }; + let inputs = if self.multipart { + api::upload::read_multipart(&request) + .await + .map(Inputs::Multipart) + } else { + api::Inputs::from_request(&request).map(Inputs::Fields) + }; + let mut inputs = match inputs { + Ok(inputs) => inputs, + Err(error) => return api::invalid_input(error), + }; + let mut input = match unsafe { Owned::new(self.function.input) } { + Ok(input) => input, + Err(_) => return api::internal_error(), + }; + { + let mut diagnostic = buffer(); + let status = unsafe { + (self.decode)( + (&mut inputs as *mut Inputs).cast(), + detail, + input.pointer(), + &mut diagnostic, + ) + }; + if let Err(message) = callback(status, diagnostic) { + return if status == 1 { + api::invalid_input(api::InputError(message)) + } else { + api::internal_error() + }; + } + } + input.initialized = true; + if let Err(error) = inputs.finish() { + return api::invalid_input(error); + } + let output = match self.function.invoke(input).await { + Ok(output) => output, + Err(error) => return self.function.failure(error, self.fault_message), + }; + let json = unsafe { *output.pointer().cast::<*mut c_void>() }; + let json = match unsafe { managed::borrowed::<String>(json) } { + Ok(json) => json, + Err(_) => return api::internal_error(), + }; + let mut encoder = dever_runtime::wire::Encoder::default(); + let value = match encoder.json(json).and_then(|()| encoder.finish()) { + Ok(value) => value, + Err(_) => return api::internal_error(), + }; + if api::commit_response_metadata().is_err() { + return api::internal_error(); + } + api::success(value) + } + + async fn dispatch( + &self, + request: http::Request, + detail: i64, + tenant_components: &[&str], + ) -> http::Response { + let prepared = match auth::prepare(&self.directory, &request, self.anonymous) { + Ok(prepared) => prepared, + Err(error) => return api::auth_error(error), + }; + let identity = match self.identity(&prepared).await { + Ok(identity) => identity, + Err(response) => return response, + }; + // Keep nested identity/tenant scopes from repeatedly embedding the request future. + let handler = Box::pin(self.call(request, detail, tenant_components)); + match auth::scope(prepared, identity, handler).await { + Ok(response) => response, + Err(error) => api::auth_error(error), + } + } + } + + struct Routes { + routes: Vec<CompiledRoute>, + tenant_components: Vec<&'static str>, + } + impl Routes { + async fn dispatch(&self, request: http::Request) -> Result<http::Response, String> { + let path = request.target.split('?').next().unwrap_or(""); + let mut allowed = Vec::new(); + for route in &self.routes { + let suffix = if route.detail { + path.strip_prefix(route.path) + .and_then(|suffix| suffix.strip_prefix('/')) + } else if path == route.path { + Some("") + } else { + None + }; + let Some(_) = suffix else { + continue; + }; + if route.method != request.method { + allowed.push(route.method); + continue; + } + return Ok(route.dispatch(request, 0, &self.tenant_components).await); + } + Ok(if allowed.is_empty() { + api::not_found() + } else { + api::method_not_allowed(&allowed) + }) + } + } + + /// # Safety + /// Route descriptors and all callback/type/name metadata are process-static. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_api_serve( + raw: *const c_void, + routes: *const Route, + count: u64, + tenant_components: *const Name, + component_count: u64, + error_buffer: *mut Buffer, + ) -> *mut AsyncOp { + if !unsafe { empty(error_buffer) } { + return ptr::null_mut(); + } + let prepared = (|| { + let session = unsafe { session(raw) }?; + let routes = unsafe { array(routes, count) }? + .iter() + .map(|route| unsafe { CompiledRoute::new(route) }) + .collect::<Result<Vec<_>, _>>()?; + let tenant_components = unsafe { names(tenant_components, component_count) }?; + Ok::<_, managed::AbiFailure>(( + session, + Arc::new(Routes { + routes, + tenant_components, + }), + )) + })(); + let (session, routes) = match prepared { + Ok(value) => value, + Err(failure) => { + unsafe { error(error_buffer, failure.status, failure.message) }; + return ptr::null_mut(); + } + }; + operation_with_output( + async move { + session + .scope(async move { + dever_runtime::lifecycle::configure(true, false)?; + dever_runtime::lifecycle::run(api::serve(move |request| { + let routes = routes.clone(); + async move { routes.dispatch(request).await } + })) + .await + }) + .await + .map(|()| OpValue::Unit) + .map_err(ForeignFault::from) + }, + Some(OutputSlots::Unit), + ) + } + + #[cfg(any(feature = "runtime-sqlite", feature = "runtime-postgres"))] + /// # Safety + /// Static route metadata; both handles belong to the same configured Session. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_api_serve_with_jobs( + raw: *const c_void, + routes: *const Route, + count: u64, + tenant_components: *const Name, + component_count: u64, + jobs: *const c_void, + error_buffer: *mut Buffer, + ) -> *mut AsyncOp { + if !unsafe { empty(error_buffer) } { + return ptr::null_mut(); + } + let prepared = (|| { + let session = unsafe { session(raw) }?; + let jobs = unsafe { job_registry(jobs) }?; + if !Arc::ptr_eq(&session, &jobs.application) { + return Err(managed::invalid("Job and API Sessions differ")); + } + let routes = unsafe { array(routes, count) }? + .iter() + .map(|route| unsafe { CompiledRoute::new(route) }) + .collect::<Result<Vec<_>, _>>()?; + let components = unsafe { names(tenant_components, component_count) }?; + Ok(( + session, + jobs, + Arc::new(Routes { + routes, + tenant_components: components, + }), + )) + })(); + let (session, jobs, routes) = match prepared { + Ok(value) => value, + Err(failure) => { + unsafe { error(error_buffer, failure.status, failure.message) }; + return ptr::null_mut(); + } + }; + operation_with_output( + async move { + session + .scope(dever_runtime::lifecycle::run(async move { + let has_api = !routes.routes.is_empty(); + let api = async move { + if !has_api { + return Ok(()); + } + api::serve(move |request| { + let routes = routes.clone(); + async move { routes.dispatch(request).await } + }) + .await + }; + dever_runtime::lifecycle::serve_both(api, jobs.serve()).await + })) + .await + .map(|()| OpValue::Unit) + .map_err(ForeignFault::from) + }, + Some(OutputSlots::Unit), + ) + } + + /// # Safety + /// Static permission rows and schema name are borrowed; operation owns session. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_api_initialize( + raw: *const c_void, + fingerprint: *const Name, + permissions: *const Permission, + count: u64, + service_start: u8, + error_buffer: *mut Buffer, + ) -> *mut AsyncOp { + if !unsafe { empty(error_buffer) } { + return ptr::null_mut(); + } + let prepared = (|| { + let session = unsafe { session(raw) }?; + let fingerprint = unsafe { name(record(fingerprint)?) }?.to_owned(); + let service_start = flag(service_start)?; + let rows = unsafe { array(permissions, count) }?; + #[cfg(any(feature = "runtime-sqlite", feature = "runtime-postgres"))] + let permissions = rows + .iter() + .map(|row| { + Ok(auth::store::Permission { + key: unsafe { name(&row.key) }?, + component: unsafe { name(&row.component) }?, + domain: unsafe { name(&row.domain) }?, + site: unsafe { name(&row.site) }?, + action: unsafe { name(&row.action) }?, + method: unsafe { name(&row.method) }?, + }) + }) + .collect::<Result<Vec<_>, managed::AbiFailure>>()?; + #[cfg(not(any(feature = "runtime-sqlite", feature = "runtime-postgres")))] + let permissions = rows.len(); + session + .initialize_authorization(!rows.is_empty()) + .map_err(managed::runtime_error)?; + Ok::<_, managed::AbiFailure>((session, fingerprint, permissions, service_start)) + })(); + let (session, fingerprint, permissions, service_start) = match prepared { + Ok(value) => value, + Err(failure) => { + unsafe { error(error_buffer, failure.status, failure.message) }; + return ptr::null_mut(); + } + }; + operation_with_output( + async move { + session + .clone() + .scope(async move { + #[cfg(any( + feature = "runtime-sqlite", + feature = "runtime-postgres" + ))] + { + session + .databases() + .prepare() + .await + .map_err(|error| error.to_string())?; + if session.settings().tenant().is_some() { + dever_runtime::tenant::initialize(&fingerprint) + .await + .map_err(|error| error.to_string())?; + } + if service_start && !permissions.is_empty() { + let connection = session + .settings() + .tenant() + .map(|tenant| tenant.database()) + .unwrap_or("default"); + let database = session + .databases() + .database(Some(connection), "") + .map_err(|error| error.to_string())?; + auth::store::sync_catalog(database.clone(), &permissions) + .await + .map_err(|error| error.to_string())?; + auth::store::initialize_roles(database) + .await + .map_err(|error| error.to_string())?; + } + } + #[cfg(not(any( + feature = "runtime-sqlite", + feature = "runtime-postgres" + )))] + { + let _ = (fingerprint, service_start); + if permissions != 0 || session.settings().tenant().is_some() { + return Err( + "API authorization requires a database runtime" + .to_owned(), + ); + } + } + Ok::<_, String>(()) + }) + .await + .map(|()| OpValue::Unit) + .map_err(ForeignFault::from) + }, + Some(OutputSlots::Unit), + ) + } + + type Upload = Mutex<Option<api::upload::Upload>>; + /// # Safety + /// Name is borrowed static UTF-8; nullable value borrows a Text owner. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_api_text_bounds( + field: *const Name, + value: *const c_void, + minimum: i64, + maximum: i64, + has_maximum: u8, + error_buffer: *mut Buffer, + ) -> u32 { + if !unsafe { empty(error_buffer) } { + return 2; + } + let result = (|| { + let name = unsafe { name(record(field)?) }?; + let value = if value.is_null() { + None + } else { + Some(unsafe { managed::borrowed::<String>(value) }?.as_str()) + }; + let minimum = u32::try_from(minimum) + .map_err(|_| managed::invalid("invalid text minimum"))?; + let maximum = if flag(has_maximum)? { + Some( + u32::try_from(maximum) + .map_err(|_| managed::invalid("invalid text maximum"))?, + ) + } else { + None + }; + api::text_bounds(name, value, minimum, maximum) + .map_err(|error| managed::runtime_error(error.to_string())) + })(); + match result { + Ok(()) => 0, + Err(failure) => unsafe { error(error_buffer, failure.status, failure.message) }, + } + } + /// # Safety + /// All integer outputs and error are distinct aligned writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_api_pagination( + page: i64, + size: i64, + maximum: i64, + out_page: *mut i64, + out_size: *mut i64, + out_offset: *mut i64, + error_buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { managed::checked_output(out_page, error_buffer) } { + return status; + } + if out_size.is_null() + || !out_size.is_aligned() + || out_offset.is_null() + || !out_offset.is_aligned() + { + return unsafe { error(error_buffer, 2, "invalid pagination output") }; + } + let maximum = match usize::try_from(maximum) { + Ok(value) => value, + Err(_) => return unsafe { error(error_buffer, 2, "invalid page maximum") }, + }; + match dever_runtime::orm::pagination(page, size, maximum) { + Ok((page, size, offset)) => { + unsafe { + out_page.write(page); + out_size.write(size); + out_offset.write(offset); + }; + 0 + } + Err(failure) => unsafe { error(error_buffer, 1, failure.to_string()) }, + } + } + /// # Safety + /// Upload is a live matching owner or null. This alias cannot consume twice. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_upload_retain(raw: *const c_void) -> *mut c_void { + unsafe { managed::retain::<Upload>(raw) } + } + /// # Safety + /// Transfers a matching Upload owner or null. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_upload_release(raw: *mut c_void) { + unsafe { managed::release::<Upload>(raw) } + } + unsafe fn with_upload<T>( + raw: *const c_void, + read: impl FnOnce(&api::upload::Upload) -> T, + ) -> Result<T, managed::AbiFailure> { + let upload = unsafe { managed::borrowed::<Upload>(raw) }? + .lock() + .map_err(|_| managed::runtime_error("upload state lock failed"))?; + upload + .as_ref() + .map(read) + .ok_or_else(|| managed::runtime_error("upload was already consumed")) + } + unsafe fn take_upload( + raw: *mut c_void, + ) -> Result<api::upload::Upload, managed::AbiFailure> { + let result = unsafe { managed::borrowed::<Upload>(raw) }.and_then(|owner| { + owner + .lock() + .map_err(|_| managed::runtime_error("upload state lock failed"))? + .take() + .ok_or_else(|| managed::runtime_error("upload was already consumed")) + }); + unsafe { managed::release::<Upload>(raw) }; + result + } + macro_rules! upload_read { + ($name:ident, $method:ident, $out:ty, $complete:ident) => { + /// # Safety + /// Upload is borrowed; output/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn $name( + raw: *const c_void, + out: *mut $out, + error_buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { managed::checked_output(out, error_buffer) } { + return status; + } + let result = unsafe { with_upload(raw, api::upload::Upload::$method) }; + unsafe { managed::$complete(out, error_buffer, result) } + } + }; + } + upload_read!( + dever_rt_v1_upload_filename, + filename, + *mut c_void, + complete_handle + ); + upload_read!( + dever_rt_v1_upload_content_type, + content_type, + *mut c_void, + complete_handle + ); + upload_read!(dever_rt_v1_upload_size, size, i64, complete); + /// # Safety + /// Consumes one Upload owner, including when the output metadata is invalid. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_upload_close_take( + raw: *mut c_void, + out: *mut u8, + error_buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { managed::checked_output(out, error_buffer) } { + unsafe { managed::release::<Upload>(raw) }; + return status; + } + let result = unsafe { take_upload(raw) }.map(|upload| { + upload.close(); + 0 + }); + unsafe { managed::complete(out, error_buffer, result) } + } + /// # Safety + /// Consumes one Upload owner; pending cancellation owns and drops the resource. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_upload_store_take( + raw: *mut c_void, + error_buffer: *mut Buffer, + ) -> *mut AsyncOp { + if !unsafe { empty(error_buffer) } { + unsafe { managed::release::<Upload>(raw) }; + return ptr::null_mut(); + } + let upload = match unsafe { take_upload(raw) } { + Ok(upload) => upload, + Err(failure) => { + unsafe { error(error_buffer, failure.status, failure.message) }; + return ptr::null_mut(); + } + }; + operation_with_output( + async move { + api::upload::store(upload) + .await + .map(|value| OpValue::Api(Output::UploadId(value))) + .map_err(ForeignFault::from) + }, + Some(OutputSlots::Handle), + ) + } + + #[cfg(any(feature = "runtime-sqlite", feature = "runtime-postgres"))] + mod roles { + use super::*; + type PackPermission = unsafe extern "C" fn( + *const c_void, + *const c_void, + *const c_void, + *const c_void, + *const c_void, + *const c_void, + *mut c_void, + ); + /// # Safety + /// Descriptor and callback are static and initialize one complete permission row. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_auth_permissions( + element: *const managed::TypeDescriptor, + pack: Option<PackPermission>, + error_buffer: *mut Buffer, + ) -> *mut AsyncOp { + if !unsafe { empty(error_buffer) } { + return ptr::null_mut(); + } + let prepared = (|| { + Ok::<_, managed::AbiFailure>(( + unsafe { managed::descriptor(element, false) }?, + pack.ok_or_else(|| { + managed::invalid("missing permission pack callback") + })?, + )) + })(); + let (element, pack) = match prepared { + Ok(value) => value, + Err(failure) => { + unsafe { error(error_buffer, failure.status, failure.message) }; + return ptr::null_mut(); + } + }; + operation_with_output( + async move { + let permissions = + auth::permissions().await.map_err(ForeignFault::from)?; + let mut values = Vec::with_capacity(permissions.len()); + for permission in permissions { + let texts = [ + permission.key, + permission.component, + permission.domain, + permission.site, + permission.action, + permission.method, + ] + .map(managed::owned); + let value = Element::from_initializer(element, |out| unsafe { + pack( + texts[0], texts[1], texts[2], texts[3], texts[4], texts[5], + out, + ) + }); + for text in texts { + unsafe { managed::release::<String>(text) }; + } + values.push(value); + } + Ok(OpValue::Api(Output::Permissions(managed::ListHandle { + values: dever_runtime::collections::List::new(values), + element, + }))) + }, + Some(OutputSlots::Handle), + ) + } + macro_rules! role_op { + ($name:ident($($arg:ident:$ty:ty),*), $prepare:expr, $run:expr) => { + /// # Safety + /// Input handles are borrowed matching owners and cloned before return. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn $name($($arg:$ty,)*error_buffer:*mut Buffer)->*mut AsyncOp { + if !unsafe {empty(error_buffer)} {return ptr::null_mut();} + let prepared:Result<_,managed::AbiFailure> = $prepare; + let prepared=match prepared {Ok(value)=>value,Err(failure)=>{unsafe {error(error_buffer,failure.status,failure.message)};return ptr::null_mut();}}; + operation_with_output(async move {($run)(prepared).await.map(|()|OpValue::Unit).map_err(ForeignFault::from)},Some(OutputSlots::Unit)) + } + }; + } + unsafe fn text(raw: *const c_void) -> Result<String, managed::AbiFailure> { + unsafe { managed::borrowed::<String>(raw) }.cloned() + } + role_op!(dever_rt_v1_auth_save_role(id:*const c_void,name:*const c_void,all:u8,permissions:*const c_void),(|| { + let permissions=unsafe {managed::borrowed::<managed::ListHandle>(permissions)}?; + if permissions.element.size!=size_of::<*mut c_void>() || permissions.element.align!=align_of::<*mut c_void>() {return Err(managed::invalid("role permissions require Text rows"));} + let values=permissions.values.values().iter().map(|value|unsafe {text(*value.pointer().cast::<*mut c_void>())}).collect::<Result<Vec<_>,_>>()?; + Ok((unsafe {text(id)}?,unsafe {text(name)}?,flag(all)?,values)) + })(),|(id,name,all,permissions):(String,String,bool,Vec<String>)|async move {auth::save_role(&id,&name,all,permissions).await}); + role_op!(dever_rt_v1_auth_grant_role(user:i64,role:*const c_void),unsafe {text(role)},|role:String|async move {auth::grant_role(user,&role).await}); + role_op!(dever_rt_v1_auth_revoke_role(user:i64,role:*const c_void),unsafe {text(role)},|role:String|async move {auth::revoke_role(user,&role).await}); + role_op!(dever_rt_v1_auth_disable_role(role:*const c_void),unsafe {text(role)},|role:String|async move {auth::disable_role(&role).await}); + role_op!(dever_rt_v1_api_tenant_owner(raw:*const c_void,tenant:i64,site:*const c_void,user:i64),(||Ok((unsafe {session(raw)}?,unsafe {text(site)}?)))(),|(session,site):(Arc<application::Session>,String)|async move { + if tenant<=0 || user<=0 {return Err("tenant id and user id must be positive".into());} + if !session.settings().sites().contains_key(&site) {return Err(format!("unknown site '{site}'"));} + let connection=session.settings().tenant().ok_or("tenant storage is not configured")?.database().to_owned(); + session.scope(dever_runtime::tenant::scope(tenant,async move { + let database=dever_runtime::tenant::database(&connection).await.map_err(|error|error.to_string())?; + auth::store::initialize_roles(database.clone()).await.map_err(|error|error.to_string())?; + auth::store::provision_owner(database,user,&site).await.map_err(|error|error.to_string())?; + Ok(()) + })).await + }); + role_op!(dever_rt_v1_api_tenant_component(raw:*const c_void,tenant:i64,component:*const c_void,enabled:u8,manifest:*const Name,count:u64),(||Ok((unsafe {session(raw)}?,unsafe {text(component)}?,flag(enabled)?,unsafe {names(manifest,count)}?)))(),|(session,component,enabled,manifest):(Arc<application::Session>,String,bool,Vec<&'static str>)|async move { + session.scope(async move { + let result=if enabled {dever_runtime::tenant::enable_component(tenant,&component,&manifest).await} else {dever_runtime::tenant::disable_component(tenant,&component,&manifest).await}; + result.map_err(|error|error.to_string()) + }).await + }); + } + unsafe fn inputs<'a>(raw: *mut c_void) -> Result<&'a mut Inputs, managed::AbiFailure> { + if raw.is_null() || !raw.cast::<Inputs>().is_aligned() { + return Err(managed::invalid("invalid API inputs")); + } + Ok(unsafe { &mut *raw.cast::<Inputs>() }) + } + /// # Safety + /// Inputs is borrowed exclusively during the synchronous decode callback. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_api_input_field( + raw: *mut c_void, + field: *const Name, + required: u8, + kind: u32, + out: *mut *mut c_void, + present: *mut u8, + error_buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { managed::checked_output(out, error_buffer) } { + return status; + } + if present.is_null() { + return unsafe { error(error_buffer, 2, "missing API input presence") }; + } + let result = (|| { + let kind = match kind { + 0 => api::QueryValue::Text, + 1 => api::QueryValue::Int, + 2 => api::QueryValue::Bool, + 3 => api::QueryValue::Float, + _ => return Err(managed::invalid("invalid query value kind")), + }; + let value = unsafe { inputs(raw) }? + .field(unsafe { name(record(field)?) }?, flag(required)?, kind) + .map_err(|error| managed::runtime_error(error.to_string()))?; + unsafe { present.write(1) }; + Ok(value) + })(); + unsafe { managed::complete_handle(out, error_buffer, result) } + } + /// # Safety + /// Same borrowed input and distinct output contract as api_input_field. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_api_input_raw_field( + raw: *mut c_void, + field: *const Name, + required: u8, + out: *mut *mut c_void, + present: *mut u8, + error_buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { managed::checked_output(out, error_buffer) } { + return status; + } + if present.is_null() { + return unsafe { error(error_buffer, 2, "missing API input presence") }; + } + let result = (|| { + unsafe { inputs(raw) }? + .raw(unsafe { name(record(field)?) }?, flag(required)?) + .map_err(|error| managed::runtime_error(error.to_string())) + })(); + match result { + Ok(value) => { + unsafe { + present.write(u8::from(value.is_some())); + if let Some(value) = value { + out.write(managed::owned(value)); + } + }; + 0 + } + Err(failure) => unsafe { error(error_buffer, failure.status, failure.message) }, + } + } + /// # Safety + /// Inputs is borrowed exclusively; out receives one affine Upload owner. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_api_input_upload( + raw: *mut c_void, + field: *const Name, + out: *mut *mut c_void, + error_buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { managed::checked_output(out, error_buffer) } { + return status; + } + let result = (|| { + let name = unsafe { name(record(field)?) }?; + match unsafe { inputs(raw) }? { + Inputs::Multipart(inputs) => inputs + .take(name) + .map(|upload| Mutex::new(Some(upload))) + .map_err(|error| managed::runtime_error(error.to_string())), + Inputs::Fields(_) => { + Err(managed::invalid("Upload requires multipart inputs")) + } + } + })(); + unsafe { managed::complete_handle(out, error_buffer, result) } + } + + /// # Safety + /// Binding metadata is borrowed; out receives one installed application owner. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_api_session_new( + bindings: *const Binding, + count: u64, + transactions: *const TransactionBindings, + transaction_count: u64, + out: *mut *mut c_void, + error_buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { managed::checked_output(out, error_buffer) } { + return status; + } + let result = (|| { + let binding = |row: &Binding| -> Result<_, managed::AbiFailure> { + Ok(( + if row.explicit.is_null() { + None + } else { + Some(unsafe { name(record(row.explicit)?) }?) + }, + unsafe { name(&row.root) }?, + flag(row.tenant)?, + )) + }; + let bindings = unsafe { array(bindings, count) }? + .iter() + .map(binding) + .collect::<Result<Vec<_>, _>>()?; + let transactions = unsafe { array(transactions, transaction_count) }? + .iter() + .map(|row| { + unsafe { array(row.bindings, row.count) }? + .iter() + .map(binding) + .collect::<Result<Vec<_>, _>>() + }) + .collect::<Result<Vec<_>, _>>()?; + let transactions = transactions.iter().map(Vec::as_slice).collect::<Vec<_>>(); + let session = application::Session::load( + config::RuntimeProfile { + sqlite: cfg!(feature = "runtime-sqlite"), + postgres: cfg!(feature = "runtime-postgres"), + }, + &bindings, + &transactions, + ) + .map_err(managed::runtime_error)?; + application::install(session.clone()).map_err(managed::runtime_error)?; + Ok(session) + })(); + unsafe { managed::complete_handle(out, error_buffer, result) } + } + /// # Safety + /// Transfers the application owner after root cleanup. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_api_session_release(raw: *mut c_void) { + unsafe { managed::release::<Arc<application::Session>>(raw) }; + } + + /// # Safety + /// Borrows the installed application and initializes a Bool output. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_api_has_tenant( + raw: *const c_void, + out: *mut u8, + error_buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { managed::checked_output(out, error_buffer) } { + return status; + } + let result = unsafe { session(raw) } + .map(|session| u8::from(session.settings().tenant().is_some())); + unsafe { managed::complete(out, error_buffer, result) } + } + + /// # Safety + /// Static typed callback metadata; borrowed input is cloned by frame creation. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_api_tenant_scope( + tenant: i64, + function: *const AsyncFunction, + input: *const c_void, + error_buffer: *mut Buffer, + ) -> *mut AsyncOp { + if !unsafe { empty(error_buffer) } { + return ptr::null_mut(); + } + let prepared = (|| { + if tenant <= 0 { + return Err(managed::runtime_error("tenant id must be positive")); + } + let function = unsafe { record(function) }?; + let (_, output) = + unsafe { owned_layout(function.output_type) }.map_err(managed::invalid)?; + let (_, fault) = + unsafe { owned_layout(function.fault_type) }.map_err(managed::invalid)?; + let future = + unsafe { ForeignFuture::new(function, input) }.map_err(managed::invalid)?; + Ok(( + future, + OutputSlots::Typed { + align: output.align(), + fault_align: fault.align(), + }, + )) + })(); + let (future, slots) = match prepared { + Ok(value) => value, + Err(failure) => { + unsafe { error(error_buffer, failure.status, failure.message) }; + return ptr::null_mut(); + } + }; + operation_with_output( + async move { + #[cfg(any(feature = "runtime-sqlite", feature = "runtime-postgres"))] + { + dever_runtime::tenant::scope(tenant, future) + .await + .map(OpValue::Value) + } + #[cfg(not(any(feature = "runtime-sqlite", feature = "runtime-postgres")))] + { + drop(future); + Err(ForeignFault::Runtime( + "tenant execution requires a database runtime".into(), + )) + } + }, + Some(slots), + ) + } + + /// # Safety + /// Session is borrowed; compiler-owned component names live for this call. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_api_require_components( + raw: *const c_void, + components: *const Name, + count: u64, + manifest: *const Name, + manifest_count: u64, + error_buffer: *mut Buffer, + ) -> *mut AsyncOp { + if !unsafe { empty(error_buffer) } { + return ptr::null_mut(); + } + let prepared = (|| { + Ok::<_, managed::AbiFailure>(( + unsafe { session(raw) }?, + unsafe { names(components, count) }?, + unsafe { names(manifest, manifest_count) }?, + )) + })(); + let (session, components, manifest) = match prepared { + Ok(value) => value, + Err(failure) => { + unsafe { error(error_buffer, failure.status, failure.message) }; + return ptr::null_mut(); + } + }; + operation_with_output( + async move { + if session.settings().tenant().is_none() || components.is_empty() { + return Ok(OpValue::Unit); + } + #[cfg(any(feature = "runtime-sqlite", feature = "runtime-postgres"))] + { + let id = dever_runtime::tenant::current_id() + .map_err(|error| ForeignFault::Runtime(error.to_string()))?; + session + .scope(dever_runtime::tenant::require_components( + id, + &components, + &manifest, + )) + .await + .map_err(|error| ForeignFault::Runtime(error.to_string()))?; + Ok(OpValue::Unit) + } + #[cfg(not(any(feature = "runtime-sqlite", feature = "runtime-postgres")))] + { + let _ = manifest; + Err(ForeignFault::Runtime( + "tenant components require a database runtime".into(), + )) + } + }, + Some(OutputSlots::Unit), + ) + } + + #[cfg(any(feature = "runtime-sqlite", feature = "runtime-postgres"))] + /// # Safety + /// Borrows the application and returns an owned alias of its database Session. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_api_session_database( + raw: *const c_void, + out: *mut *mut c_void, + error_buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { managed::checked_output(out, error_buffer) } { + return status; + } + let result = unsafe { session(raw) }.map(|session| { + Arc::into_raw(session.databases().clone()) + .cast_mut() + .cast::<c_void>() + }); + unsafe { managed::complete(out, error_buffer, result) } + } + + /// # Safety + /// The callback descriptors are static. All writable outputs are distinct, + /// and session_slot remains live until same-runtime cleanup finishes. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_api_application_root( + function: *const AsyncFunction, + input: *const c_void, + out: *mut c_void, + fault: *mut c_void, + session_slot: *const *mut c_void, + append_cause: Option<unsafe extern "C" fn(*mut c_void, *const c_void)>, + error_buffer: *mut Buffer, + ) -> u32 { + if !unsafe { empty(error_buffer) } { + return 4; + } + let prepared = (|| { + unsafe { record(session_slot) }?; + let function = unsafe { record(function) }?; + let (_, output_layout) = + unsafe { owned_layout(function.output_type) }.map_err(managed::invalid)?; + let (_, fault_layout) = + unsafe { owned_layout(function.fault_type) }.map_err(managed::invalid)?; + if out.is_null() + || !(out as usize).is_multiple_of(output_layout.align()) + || fault.is_null() + || !(fault as usize).is_multiple_of(fault_layout.align()) + { + return Err(managed::invalid("invalid API root outputs")); + } + let append = append_cause.ok_or_else(|| { + managed::invalid("missing application cleanup cause callback") + })?; + let future = + unsafe { ForeignFuture::new(function, input) }.map_err(managed::invalid)?; + Ok((future, append)) + })(); + let (future, append) = match prepared { + Ok(value) => value, + Err(failure) => return unsafe { error(error_buffer, 4, failure.message) }, + }; + let result = task::run_entry_with_typed_resource_cleanup( + task::RuntimeConfig::default(), + application::scope_entry(future), + |result, resources| async move { + let result = cleanup_result(result, resources, append); + let raw = unsafe { *session_slot }; + if raw.is_null() { + return result; + } + let session = unsafe { session(raw) } + .map_err(|error| ForeignFault::Runtime(error.message))?; + let cleanup = session.clone().scope(session.close()).await; + cleanup_result(result, cleanup, append) + }, + ); + unsafe { root_result(result, out, fault, error_buffer) } + } + } + + #[cfg(feature = "runtime-external")] + mod external { + use super::*; + use crate::ABI_OK; + use dever_runtime::{component, external as resources, wire}; + + #[repr(C)] + pub struct Name { + pointer: *const u8, + len: u64, + } + + #[repr(C)] + pub struct Definition { + key: Name, + ecosystem: Name, + entry: Name, + port: Name, + adapter: Name, + schema: Name, + capabilities: *const Name, + capability_count: u64, + operations: *const Name, + operation_count: u64, + timeout_ms: u64, + } + + #[repr(C)] + pub struct Resource { + path: Name, + bytes: *const u8, + len: u64, + sha256: Name, + executable: u8, + } + + unsafe fn array<'a, T>( + raw: *const T, + count: u64, + ) -> Result<&'a [T], managed::AbiFailure> { + let count = usize::try_from(count) + .map_err(|_| managed::invalid("invalid external array count"))?; + if count == 0 { + return Ok(&[]); + } + if raw.is_null() + || !raw.is_aligned() + || count > isize::MAX as usize / size_of::<T>().max(1) + { + return Err(managed::invalid("invalid external array")); + } + Ok(unsafe { std::slice::from_raw_parts(raw, count) }) + } + + unsafe fn name<'a>(raw: &Name) -> Result<&'a str, managed::AbiFailure> { + std::str::from_utf8(unsafe { array(raw.pointer, raw.len) }?) + .map_err(|_| managed::invalid("external name is not UTF-8")) + } + + unsafe fn names( + raw: *const Name, + count: u64, + ) -> Result<Vec<String>, managed::AbiFailure> { + unsafe { array(raw, count) }? + .iter() + .map(|value| unsafe { name(value) }.map(str::to_owned)) + .collect() + } + + /// # Safety + /// Metadata and resource bytes are immutable process-static compiler output. + /// The descriptor array itself is borrowed only during this call. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_external_prepare( + digest: *const Name, + rows: *const Resource, + count: u64, + buffer: *mut Buffer, + ) -> u32 { + if !unsafe { empty(buffer) } { + return ABI_INVALID_INPUT; + } + let result = (|| { + let digest = unsafe { managed::borrowed::<Name>(digest.cast()) }?; + let digest = unsafe { name(digest) }?; + let resources = unsafe { array(rows, count) }? + .iter() + .map(|row| { + Ok(resources::Resource { + path: unsafe { name(&row.path) }?, + bytes: unsafe { array(row.bytes, row.len) }?, + sha256: unsafe { name(&row.sha256) }?, + executable: match row.executable { + 0 => false, + 1 => true, + _ => { + return Err(managed::invalid( + "invalid external executable flag", + )); + } + }, + }) + }) + .collect::<Result<Vec<_>, managed::AbiFailure>>()?; + let root = dever_runtime::config::executable_directory() + .map_err(managed::runtime_error)?; + resources::prepare(&root, digest, &resources).map_err(managed::runtime_error) + })(); + match result { + Ok(()) => ABI_OK, + Err(failure) => unsafe { error(buffer, failure.status, failure.message) }, + } + } + + /// # Safety + /// The definition and optional setting Text are borrowed only during creation. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_external_start( + raw: *const Definition, + setting: *const c_void, + buffer: *mut Buffer, + ) -> *mut AsyncOp { + if !unsafe { empty(buffer) } { + return ptr::null_mut(); + } + let prepared: Result<component::Definition, managed::AbiFailure> = (|| { + let row = unsafe { managed::borrowed::<Definition>(raw.cast()) }?; + Ok(component::Definition { + key: unsafe { name(&row.key) }?.to_owned(), + ecosystem: unsafe { name(&row.ecosystem) }?.to_owned(), + entry: unsafe { name(&row.entry) }?.to_owned(), + port: unsafe { name(&row.port) }?.to_owned(), + adapter: unsafe { name(&row.adapter) }?.to_owned(), + schema: unsafe { name(&row.schema) }?.to_owned(), + capabilities: unsafe { names(row.capabilities, row.capability_count) }?, + operations: unsafe { names(row.operations, row.operation_count) }?, + setting: if setting.is_null() { + None + } else { + Some(unsafe { managed::borrowed::<String>(setting) }?.clone()) + }, + timeout_ms: row.timeout_ms, + }) + })( + ); + let definition = match prepared { + Ok(value) => value, + Err(failure) => { + unsafe { error(buffer, failure.status, failure.message) }; + return ptr::null_mut(); + } + }; + operation_with_output( + async move { + component::start(definition) + .await + .map_err(ForeignFault::from)?; + Ok(OpValue::Unit) + }, + Some(OutputSlots::Unit), + ) + } + + /// # Safety + /// All Text handles are borrowed until return; the operation owns its inputs. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_external_call( + key: *const c_void, + operation_name: *const c_void, + payload: *const c_void, + buffer: *mut Buffer, + ) -> *mut AsyncOp { + if !unsafe { empty(buffer) } { + return ptr::null_mut(); + } + let prepared: Result<_, managed::AbiFailure> = (|| { + let key = unsafe { managed::borrowed::<String>(key) }?.clone(); + let operation = unsafe { managed::borrowed::<String>(operation_name) }?.clone(); + let payload = unsafe { managed::borrowed::<String>(payload) }?; + let mut encoder = wire::Encoder::default(); + encoder.json(payload).map_err(managed::runtime_error)?; + Ok(( + key, + operation, + encoder.finish().map_err(managed::runtime_error)?, + )) + })(); + let (key, operation, payload) = match prepared { + Ok(value) => value, + Err(failure) => { + unsafe { error(buffer, failure.status, failure.message) }; + return ptr::null_mut(); + } + }; + operation_with_output( + async move { + component::call(&key, &operation, &payload) + .await + .map(OpValue::External) + .map_err(ForeignFault::from) + }, + Some(OutputSlots::Handle), + ) + } + + /// # Safety + /// Consumes one Reply on every path. Outputs are distinct writable slots. + /// Success initializes kind (0=result, 1=declared error) and two owned Texts. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_external_reply_take( + raw: *mut c_void, + kind: *mut u8, + identity: *mut *mut c_void, + payload: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + let reply = unsafe { managed::consumed::<component::Reply>(raw) }; + if !unsafe { empty(buffer) } { + return ABI_INVALID_INPUT; + } + if kind.is_null() + || identity.is_null() + || !identity.is_aligned() + || payload.is_null() + || !payload.is_aligned() + { + return unsafe { + error(buffer, ABI_INVALID_INPUT, "invalid external reply output") + }; + } + match reply { + Ok(reply) => { + let (tag, error_identity, body) = match &*reply { + component::Reply::Result(body) => (0, String::new(), body.clone()), + component::Reply::Error { identity, payload } => { + (1, identity.clone(), payload.clone()) + } + }; + unsafe { + kind.write(tag); + identity.write(managed::owned(error_identity)); + payload.write(managed::owned(body)); + } + ABI_OK + } + Err(failure) => unsafe { error(buffer, failure.status, failure.message) }, + } + } + + /// # Safety + /// Transfers one Reply handle returned by external_call. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_external_reply_release(raw: *mut c_void) { + unsafe { managed::release::<component::Reply>(raw) }; + } + + #[unsafe(no_mangle)] + pub extern "C" fn dever_rt_v1_external_shutdown() -> *mut AsyncOp { + operation_with_output( + async { + component::shutdown().await.map_err(ForeignFault::from)?; + Ok(OpValue::Unit) + }, + Some(OutputSlots::Unit), + ) + } + + /// # Safety + /// cause is an initialized nullable Text slot; message is borrowed Text. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_external_cause_append( + cause: *mut *mut c_void, + message: *const c_void, + ) { + unsafe { managed::append_fault_cause(cause, message) }; + } + + /// # Safety + /// Static callbacks and typed output/fault slots follow async_root. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_external_root( + function: *const AsyncFunction, + input: *const c_void, + output: *mut c_void, + fault: *mut c_void, + append_cause: Option<unsafe extern "C" fn(*mut c_void, *const c_void)>, + buffer: *mut Buffer, + ) -> u32 { + if !unsafe { empty(buffer) } { + return 4; + } + let prepared = (|| { + let function = unsafe { managed::borrowed::<AsyncFunction>(function.cast()) } + .map_err(|failure| failure.message)?; + let (_, output_layout) = unsafe { owned_layout(function.output_type) }?; + let (_, fault_layout) = unsafe { owned_layout(function.fault_type) }?; + if output.is_null() + || !(output as usize).is_multiple_of(output_layout.align()) + || fault.is_null() + || !(fault as usize).is_multiple_of(fault_layout.align()) + { + return Err("invalid external root outputs".to_owned()); + } + let append = append_cause.ok_or("missing external cleanup callback")?; + let future = unsafe { ForeignFuture::new(function, input) }?; + Ok((future, append)) + })(); + let (future, append) = match prepared { + Ok(value) => value, + Err(message) => return unsafe { error(buffer, 4, message) }, + }; + let result = task::run_entry_with_typed_resource_cleanup( + task::RuntimeConfig::default(), + future, + |result, resources| { + std::future::ready(cleanup_result(result, resources, append)) + }, + ); + unsafe { root_result(result, output, fault, buffer) } + } + } + + #[cfg(any(feature = "runtime-sqlite", feature = "runtime-postgres"))] + mod database { + use super::*; + use dever_runtime::{database as db, orm}; + + #[repr(C)] + #[derive(Clone, Copy)] + pub struct Name { + pointer: *const u8, + len: u64, + } + #[repr(C)] + pub struct ErrorDescriptor { + fault_type: *const OwnedType, + pack: Option<unsafe extern "C" fn(u32, *const c_void, *mut c_void)>, + append_cause: Option<unsafe extern "C" fn(*mut c_void, *const c_void)>, + } + #[repr(C)] + pub struct Sql { + sqlite: Name, + postgres: Name, + } + #[repr(C)] + pub struct Value { + kind: u32, + precision: u32, + scale: u32, + integer: i64, + floating: f64, + decimal: crate::AbiDecimal, + handle: *const c_void, + } + #[repr(C)] + pub struct Binding { + explicit: *const Name, + root: Name, + tenant: u8, + } + #[repr(C)] + pub struct TransactionBindings { + bindings: *const Binding, + count: u64, + } + #[repr(C)] + pub struct Field { + name: Name, + ty: Name, + nullable: u8, + generated: u8, + default: *const Name, + rename: *const Name, + } + #[repr(C)] + pub struct Index { + name: Name, + fields: *const Name, + count: u64, + unique: u8, + } + #[repr(C)] + pub struct Migration { + name: Name, + revision: Name, + drops: *const Name, + count: u64, + } + #[repr(C)] + pub struct Seed { + sql: Sql, + parameters: *const Value, + count: u64, + } + #[repr(C)] + pub struct DataMigration { + name: Name, + phase: u32, + statement: Seed, + } + #[repr(C)] + pub struct PostgresColumn { + name: Name, + definition: Name, + sql_type: Name, + default: *const Name, + } + #[repr(C)] + pub struct PostgresConstraint { + name: Name, + definition: Name, + foreign_key: u8, + } + #[repr(C)] + pub struct Model { + model: Name, + table: Name, + revision: Name, + seed_revision: Name, + fields: *const Field, + field_count: u64, + indexes: *const Index, + index_count: u64, + migrations: *const Migration, + migration_count: u64, + create_table: Sql, + temporary_table: Name, + create_indexes: *const Sql, + create_index_count: u64, + postgres_columns: *const PostgresColumn, + postgres_column_count: u64, + postgres_constraints: *const PostgresConstraint, + postgres_constraint_count: u64, + seeds: *const Seed, + seed_count: u64, + data_migrations: *const DataMigration, + data_migration_count: u64, + } + #[repr(C)] + pub struct RowDecoder { + row_type: *const OwnedType, + decode: Option< + unsafe extern "C" fn( + *const c_void, + *mut c_void, + *const ErrorDescriptor, + *mut c_void, + *mut Buffer, + ) -> u32, + >, + } + + enum Failure { + Abi(String), + Database(orm::Error), + } + impl From<managed::AbiFailure> for Failure { + fn from(value: managed::AbiFailure) -> Self { + Self::Abi(value.message) + } + } + impl From<orm::Error> for Failure { + fn from(value: orm::Error) -> Self { + Self::Database(value) + } + } + fn invalid(message: impl Into<String>) -> Failure { + Failure::Abi(message.into()) + } + + unsafe fn record<'a, T>(pointer: *const T) -> Result<&'a T, Failure> { + if pointer.is_null() || !pointer.is_aligned() { + return Err(invalid("invalid database ABI record")); + } + Ok(unsafe { &*pointer }) + } + unsafe fn array<'a, T>(pointer: *const T, count: u64) -> Result<&'a [T], Failure> { + let count = + usize::try_from(count).map_err(|_| invalid("invalid database ABI count"))?; + if count == 0 { + return Ok(&[]); + } + if pointer.is_null() + || !pointer.is_aligned() + || count > isize::MAX as usize / size_of::<T>().max(1) + { + return Err(invalid("invalid database ABI array")); + } + Ok(unsafe { std::slice::from_raw_parts(pointer, count) }) + } + unsafe fn name<'a>(value: &Name) -> Result<&'a str, Failure> { + std::str::from_utf8(unsafe { array(value.pointer, value.len) }?) + .map_err(|_| invalid("database ABI name is not UTF-8")) + } + unsafe fn optional_name(value: *const Name) -> Result<Option<String>, Failure> { + if value.is_null() { + Ok(None) + } else { + Ok(Some(unsafe { name(record(value)?) }?.to_owned())) + } + } + fn flag(value: u8) -> Result<bool, Failure> { + match value { + 0 => Ok(false), + 1 => Ok(true), + _ => Err(invalid("invalid database ABI boolean")), + } + } + unsafe fn cloned<T: Clone>(handle: *const c_void) -> Result<T, Failure> { + Ok(unsafe { managed::borrowed::<T>(handle) }?.clone()) + } + unsafe fn shared<T>(handle: *const c_void) -> Result<Arc<T>, Failure> { + unsafe { managed::borrowed::<T>(handle) }?; + // SAFETY: a validated borrowed Arc owner is retained for this operation. + unsafe { + Arc::increment_strong_count(handle.cast::<T>()); + } + Ok(unsafe { Arc::from_raw(handle.cast::<T>()) }) + } + unsafe fn sql(value: &Sql) -> Result<db::Sql, Failure> { + // SQL bytes are immutable process-static compiler literals, as + // required by both existing driver APIs. No caller data is leaked. + Ok(db::Sql { + sqlite: unsafe { name(&value.sqlite) }?, + postgres: unsafe { name(&value.postgres) }?, + }) + } + unsafe fn schema_sql(value: &Sql) -> Result<db::SchemaSql, Failure> { + Ok(db::SchemaSql { + sqlite: unsafe { name(&value.sqlite) }?.into(), + postgres: unsafe { name(&value.postgres) }?.into(), + }) + } + unsafe fn parameters( + values: *const Value, + count: u64, + ) -> Result<Vec<orm::Value>, Failure> { + unsafe { array(values, count) }? + .iter() + .map(|value| { + Ok(match value.kind { + 0 => orm::Value::Null, + 1 => orm::Value::Bool(match value.integer { + 0 => false, + 1 => true, + _ => return Err(invalid("invalid database Bool bind")), + }), + 2 => orm::Value::Int(value.integer), + 3 if value.floating.is_finite() => orm::Value::Float(value.floating), + 3 => { + return Err(orm::Error::invalid_data( + "database Float must be finite", + ) + .into()); + } + 4 => orm::decimal_value( + value.decimal.decode().map_err(invalid)?, + u8::try_from(value.precision) + .map_err(|_| invalid("invalid Decimal precision"))?, + u8::try_from(value.scale) + .map_err(|_| invalid("invalid Decimal scale"))?, + )?, + 5 => orm::Value::Text(unsafe { cloned(value.handle) }?), + 6 => orm::Value::Bytes( + unsafe { managed::borrowed::<Bytes>(value.handle) }? + .values() + .to_vec(), + ), + 7 => orm::Value::Uuid(unsafe { cloned(value.handle) }?), + _ => return Err(invalid("invalid database bind kind")), + }) + }) + .collect() + } + unsafe fn model(value: *const Model) -> Result<db::Model, Failure> { + let value = unsafe { record(value) }?; + let fields = unsafe { array(value.fields, value.field_count) }? + .iter() + .map(|field| { + Ok(db::Field { + name: unsafe { name(&field.name) }?.into(), + ty: unsafe { name(&field.ty) }?.into(), + nullable: flag(field.nullable)?, + generated: flag(field.generated)?, + default: unsafe { optional_name(field.default) }?, + rename_from: unsafe { optional_name(field.rename) }?, + }) + }) + .collect::<Result<_, Failure>>()?; + let indexes = unsafe { array(value.indexes, value.index_count) }? + .iter() + .map(|index| { + Ok(db::Index { + name: unsafe { name(&index.name) }?.into(), + unique: flag(index.unique)?, + fields: unsafe { array(index.fields, index.count) }? + .iter() + .map(|field| unsafe { name(field) }.map(str::to_owned)) + .collect::<Result<_, _>>()?, + }) + }) + .collect::<Result<_, Failure>>()?; + let migrations = unsafe { array(value.migrations, value.migration_count) }? + .iter() + .map(|migration| { + Ok(db::Migration { + name: unsafe { name(&migration.name) }?.into(), + revision: unsafe { name(&migration.revision) }?.into(), + drops: unsafe { array(migration.drops, migration.count) }? + .iter() + .map(|field| unsafe { name(field) }.map(str::to_owned)) + .collect::<Result<_, _>>()?, + }) + }) + .collect::<Result<_, Failure>>()?; + let postgres_columns = + unsafe { array(value.postgres_columns, value.postgres_column_count) }? + .iter() + .map(|column| { + Ok(db::PostgresColumn { + name: unsafe { name(&column.name) }?.into(), + definition: unsafe { name(&column.definition) }?.into(), + sql_type: unsafe { name(&column.sql_type) }?.into(), + default: unsafe { optional_name(column.default) }?, + }) + }) + .collect::<Result<_, Failure>>()?; + let postgres_constraints = unsafe { + constraints(value.postgres_constraints, value.postgres_constraint_count) + }?; + let seeds = unsafe { array(value.seeds, value.seed_count) }? + .iter() + .map(|seed| { + Ok(db::Seed { + sql: unsafe { sql(&seed.sql) }?, + parameters: unsafe { parameters(seed.parameters, seed.count) }?, + }) + }) + .collect::<Result<_, Failure>>()?; + let data_migrations = + unsafe { array(value.data_migrations, value.data_migration_count) }? + .iter() + .map(|migration| { + Ok(db::DataMigration { + name: unsafe { name(&migration.name) }?.into(), + phase: match migration.phase { + 0 => db::MigrationPhase::Before, + 1 => db::MigrationPhase::After, + _ => return Err(invalid("invalid database migration phase")), + }, + sql: unsafe { sql(&migration.statement.sql) }?, + parameters: unsafe { + parameters( + migration.statement.parameters, + migration.statement.count, + ) + }?, + }) + }) + .collect::<Result<_, Failure>>()?; + Ok(db::Model { + schema: db::Schema { + model: unsafe { name(&value.model) }?.into(), + table: unsafe { name(&value.table) }?.into(), + revision: unsafe { name(&value.revision) }?.into(), + seed_revision: unsafe { name(&value.seed_revision) }?.into(), + fields, + indexes, + migrations, + }, + create_table: unsafe { schema_sql(&value.create_table) }?, + create_temporary_table: unsafe { name(&value.temporary_table) }?.into(), + create_indexes: unsafe { + array(value.create_indexes, value.create_index_count) + }? + .iter() + .map(|sql| unsafe { schema_sql(sql) }) + .collect::<Result<_, _>>()?, + postgres_columns, + postgres_constraints, + seeds, + data_migrations, + }) + } + unsafe fn constraints( + pointer: *const PostgresConstraint, + count: u64, + ) -> Result<Vec<db::PostgresConstraint>, Failure> { + unsafe { array(pointer, count) }? + .iter() + .map(|constraint| { + Ok(db::PostgresConstraint { + name: unsafe { name(&constraint.name) }?.into(), + definition: unsafe { name(&constraint.definition) }?.into(), + foreign_key: flag(constraint.foreign_key)?, + }) + }) + .collect() + } + + #[derive(Clone, Copy)] + struct Errors { + descriptor: &'static ErrorDescriptor, + ty: &'static OwnedType, + align: usize, + } + // Static callbacks and immutable descriptors are compiler output. + // new validates the complete fault's transferable owner descriptor. + unsafe impl Send for Errors {} + unsafe impl Sync for Errors {} + impl Errors { + unsafe fn new(pointer: *const ErrorDescriptor) -> Result<Self, Failure> { + let descriptor = unsafe { record(pointer) }?; + let (ty, layout) = + unsafe { owned_layout(descriptor.fault_type) }.map_err(invalid)?; + if descriptor.pack.is_none() + || descriptor.append_cause.is_none() + || ty.transferable == 0 + { + return Err(invalid("invalid database fault descriptor")); + } + Ok(Self { + descriptor, + ty, + align: layout.align(), + }) + } + fn fault(self, error: orm::Error) -> ForeignFault { + let mut value = match unsafe { Owned::new(self.ty) } { + Ok(value) => value, + Err(message) => return ForeignFault::Runtime(message), + }; + let message = Arc::new(error.to_string()); + unsafe { + (self.descriptor.pack.expect("validated callback"))( + kind(error.kind()), + Arc::as_ptr(&message).cast(), + value.pointer(), + ) + }; + value.initialized = true; + ForeignFault::Typed(value) + } + fn failure(self, failure: Failure) -> ForeignFault { + match failure { + Failure::Database(error) => self.fault(error), + Failure::Abi(message) => ForeignFault::Runtime(message), + } + } + fn augment(self, primary: ForeignFault, cleanup: orm::Error) -> ForeignFault { + match primary { + ForeignFault::Typed(value) => { + let message = Arc::new(cleanup.to_string()); + unsafe { + (self.descriptor.append_cause.expect("validated callback"))( + value.pointer(), + Arc::as_ptr(&message).cast(), + ) + }; + ForeignFault::Typed(value) + } + ForeignFault::Runtime(message) => { + ForeignFault::Runtime(format!("{message}; caused by: {cleanup}")) + } + } + } + fn slots(self, align: usize, optional: bool) -> OutputSlots { + OutputSlots::Database { + align, + optional, + fault_align: self.align, + } + } + } + fn kind(value: orm::ErrorKind) -> u32 { + match value { + orm::ErrorKind::Pool => 0, + orm::ErrorKind::PoolExhausted => 1, + orm::ErrorKind::Connection => 2, + orm::ErrorKind::Timeout => 3, + orm::ErrorKind::Cancelled => 4, + orm::ErrorKind::Database => 5, + orm::ErrorKind::Constraint => 6, + orm::ErrorKind::NotFound => 7, + orm::ErrorKind::InvalidData => 8, + orm::ErrorKind::Migration => 9, + } + } + fn error_kind(value: u32) -> Result<orm::ErrorKind, Failure> { + match value { + 0 => Ok(orm::ErrorKind::Pool), + 1 => Ok(orm::ErrorKind::PoolExhausted), + 2 => Ok(orm::ErrorKind::Connection), + 3 => Ok(orm::ErrorKind::Timeout), + 4 => Ok(orm::ErrorKind::Cancelled), + 5 => Ok(orm::ErrorKind::Database), + 6 => Ok(orm::ErrorKind::Constraint), + 7 => Ok(orm::ErrorKind::NotFound), + 8 => Ok(orm::ErrorKind::InvalidData), + 9 => Ok(orm::ErrorKind::Migration), + _ => Err(invalid("invalid database error kind")), + } + } + unsafe fn checked<T>( + out: *mut T, + errors: *const ErrorDescriptor, + fault: *mut c_void, + buffer: *mut Buffer, + ) -> Result<Errors, u32> { + unsafe { managed::checked_output(out, buffer) }?; + let result = unsafe { Errors::new(errors) }.and_then(|errors| { + if fault.is_null() || !(fault as usize).is_multiple_of(errors.align) { + Err(invalid("invalid database fault output")) + } else { + Ok(errors) + } + }); + result.map_err(|failure| unsafe { abi_failure(failure, buffer) }) + } + unsafe fn abi_failure(failure: Failure, buffer: *mut Buffer) -> u32 { + let message = match failure { + Failure::Abi(message) => message, + Failure::Database(error) => error.to_string(), + }; + unsafe { error(buffer, 2, message) } + } + unsafe fn finish<T>( + result: Result<T, Failure>, + out: *mut T, + errors: Errors, + fault: *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + match result { + Ok(value) => { + unsafe { out.write(value) }; + 0 + } + Err(Failure::Database(database_error)) => match errors.fault(database_error) { + ForeignFault::Typed(mut value) => { + unsafe { value.move_to(fault) }; + 1 + } + ForeignFault::Runtime(message) => unsafe { error(buffer, 2, message) }, + }, + Err(failure) => unsafe { abi_failure(failure, buffer) }, + } + } + unsafe fn regular<T>( + out: *mut T, + errors: *const ErrorDescriptor, + fault: *mut c_void, + buffer: *mut Buffer, + apply: impl FnOnce() -> Result<T, Failure>, + ) -> u32 { + let errors = match unsafe { checked(out, errors, fault, buffer) } { + Ok(errors) => errors, + Err(status) => return status, + }; + unsafe { finish(apply(), out, errors, fault, buffer) } + } + + type Transaction = Arc<dever_runtime::database::TransactionState>; + + fn completed<T>( + mut transaction: Option<db::TransactionLease<'_>>, + result: Result<T, orm::Error>, + ) -> Result<T, Failure> { + if let Some(transaction) = &mut transaction { + transaction.complete(); + } + result.map_err(Failure::Database) + } + + pub(super) enum Output { + Unit, + Database(db::Database), + #[cfg(feature = "runtime-api")] + Text(String), + Int(i64), + Transaction(Transaction), + Rows(Vec<orm::Row>), + Stream(Stream), + Row(Option<Owned>), + } + impl Output { + pub(super) unsafe fn write(self, out: *mut c_void, present: *mut u8) { + match self { + Self::Unit => {} + Self::Database(value) => unsafe { + out.cast::<*mut c_void>().write(managed::owned(value)) + }, + #[cfg(feature = "runtime-api")] + Self::Text(value) => unsafe { + out.cast::<*mut c_void>().write(managed::owned(value)) + }, + Self::Int(value) => unsafe { out.cast::<i64>().write(value) }, + Self::Transaction(value) => unsafe { + out.cast::<*mut c_void>().write(managed::owned(value)) + }, + Self::Rows(value) => unsafe { + out.cast::<*mut c_void>().write(managed::owned(value)) + }, + Self::Stream(value) => unsafe { + out.cast::<*mut c_void>().write(managed::owned(value)) + }, + Self::Row(value) => { + unsafe { present.write(u8::from(value.is_some())) }; + if let Some(mut value) = value { + unsafe { value.move_to(out) }; + } + } + } + } + } + unsafe fn operation<P, F>( + errors: *const ErrorDescriptor, + buffer: *mut Buffer, + align: usize, + optional: bool, + prepare: impl FnOnce() -> Result<P, Failure>, + run: impl FnOnce(P) -> F + 'static, + ) -> *mut AsyncOp + where + P: 'static, + F: Future<Output = Result<Output, Failure>> + 'static, + { + if !unsafe { empty(buffer) } { + return ptr::null_mut(); + } + let errors = match unsafe { Errors::new(errors) } { + Ok(errors) => errors, + Err(failure) => { + unsafe { abi_failure(failure, buffer) }; + return ptr::null_mut(); + } + }; + let prepared = prepare(); + if let Err(Failure::Abi(message)) = prepared { + unsafe { error(buffer, 2, message) }; + return ptr::null_mut(); + } + operation_with_output( + async move { + let prepared = prepared.map_err(|failure| errors.failure(failure))?; + run(prepared) + .await + .map(OpValue::Database) + .map_err(|failure| errors.failure(failure)) + }, + Some(errors.slots(align, optional)), + ) + } + + struct Query { + database: db::Database, + transaction: Option<Transaction>, + sql: db::Sql, + parameters: Vec<orm::Value>, + } + impl Query { + unsafe fn new( + database: *const c_void, + transaction: *const c_void, + sql_pointer: *const Sql, + values: *const Value, + count: u64, + ) -> Result<Self, Failure> { + Ok(Self { + database: unsafe { cloned(database) }?, + transaction: if transaction.is_null() { + None + } else { + Some(unsafe { cloned(transaction) }?) + }, + sql: unsafe { sql(record(sql_pointer)?) }?, + parameters: unsafe { parameters(values, count) }?, + }) + } + } + + macro_rules! owner { + ($retain:ident, $release:ident, $ty:ty) => { + /// # Safety + /// The handle is a live borrowed owner of the named database type, or null. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn $retain(handle: *const c_void) -> *mut c_void { + unsafe { managed::retain::<$ty>(handle) } + } + /// # Safety + /// Transfers one matching database owner, or null. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn $release(handle: *mut c_void) { + unsafe { managed::release::<$ty>(handle) } + } + }; + } + owner!(dever_rt_v1_db_retain, dever_rt_v1_db_release, db::Database); + owner!( + dever_rt_v1_db_transaction_retain, + dever_rt_v1_db_transaction_release, + Transaction + ); + owner!( + dever_rt_v1_db_stream_retain, + dever_rt_v1_db_stream_release, + Stream + ); + owner!( + dever_rt_v1_db_related_retain, + dever_rt_v1_db_related_release, + Element + ); + + #[cfg(feature = "runtime-api")] + /// # Safety + /// Borrows the registry and static DbError mapping; output is Unit. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_job_initialize( + jobs: *const c_void, + errors: *const ErrorDescriptor, + buffer: *mut Buffer, + ) -> *mut AsyncOp { + unsafe { + operation( + errors, + buffer, + 0, + false, + || super::api_application::job_registry(jobs).map_err(Failure::from), + |jobs| async move { + jobs.application + .clone() + .scope(jobs.resources.initialize()) + .await?; + Ok(Output::Unit) + }, + ) + } + } + + #[cfg(feature = "runtime-api")] + /// # Safety + /// All handles are borrowed; nullable transaction is the caller's physical owner. + /// The operation retains owners and validates output before queue I/O. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_job_enqueue( + jobs: *const c_void, + index: u64, + database: *const c_void, + transaction: *const c_void, + payload: *const c_void, + key: *const c_void, + run_at: i64, + errors: *const ErrorDescriptor, + buffer: *mut Buffer, + ) -> *mut AsyncOp { + unsafe { + operation( + errors, + buffer, + align_of::<*mut c_void>(), + false, + || { + let jobs = super::api_application::job_registry(jobs)?; + let index = + usize::try_from(index).map_err(|_| invalid("invalid Job index"))?; + let spec = jobs + .resources + .bindings + .get(index) + .ok_or_else(|| invalid("unknown compiled Job index"))? + .spec + .clone(); + let database = cloned::<db::Database>(database)?; + let transaction = if transaction.is_null() { + None + } else { + Some(cloned::<Transaction>(transaction)?) + }; + let mut encoder = dever_runtime::wire::Encoder::default(); + encoder + .json(managed::borrowed::<String>(payload)?) + .map_err(invalid)?; + let payload = encoder.finish().map_err(invalid)?; + let key = cloned::<String>(key)?; + Ok((jobs, spec, database, transaction, payload, key)) + }, + move |(jobs, spec, database, transaction, payload, key)| async move { + jobs.application + .clone() + .scope(async move { + let mut lease = + db::TransactionState::borrow(transaction.as_deref()) + .await?; + let result = dever_runtime::job::enqueue( + database, + lease.as_deref(), + &spec, + payload, + &key, + run_at, + ) + .await; + if let Some(lease) = &mut lease { + lease.complete(); + } + result.map(|id| Output::Text(id.0)).map_err(Failure::from) + }) + .await + }, + ) + } + } + + /// # Safety + /// Bindings are borrowed immutable rows; out is an uninitialized owner slot. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_db_session_new( + bindings: *const Binding, + count: u64, + transactions: *const TransactionBindings, + transaction_count: u64, + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { managed::checked_output(out, buffer) } { + return status; + } + let result = (|| { + let binding = |value: &Binding| -> Result<_, Failure> { + Ok(( + if value.explicit.is_null() { + None + } else { + Some(unsafe { name(record(value.explicit)?) }?) + }, + unsafe { name(&value.root) }?, + flag(value.tenant)?, + )) + }; + let bindings = unsafe { array(bindings, count) }? + .iter() + .map(binding) + .collect::<Result<Vec<_>, _>>()?; + let transactions = unsafe { array(transactions, transaction_count) }? + .iter() + .map(|transaction| { + unsafe { array(transaction.bindings, transaction.count) }? + .iter() + .map(binding) + .collect::<Result<Vec<_>, _>>() + }) + .collect::<Result<Vec<_>, Failure>>()?; + let transactions = transactions.iter().map(Vec::as_slice).collect::<Vec<_>>(); + Ok(db::Session::load( + dever_runtime::config::RuntimeProfile { + sqlite: cfg!(feature = "runtime-sqlite"), + postgres: cfg!(feature = "runtime-postgres"), + }, + &bindings, + &transactions, + )) + })(); + match result { + Ok(Ok(session)) => { + unsafe { out.write(managed::owned(session)) }; + 0 + } + Ok(Err(message)) => unsafe { error(buffer, 1, message) }, + Err(failure) => unsafe { abi_failure(failure, buffer) }, + } + } + /// # Safety + /// Transfers one entry session after the application root has closed it. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_db_session_release(session: *mut c_void) { + unsafe { managed::release::<db::Session>(session) } + } + + /// # Safety + /// Same callback contract as async_root. session_slot is the module's live + /// pointer slot, published by the root and kept through cleanup. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_db_application_root( + function: *const AsyncFunction, + input: *const c_void, + out: *mut c_void, + fault: *mut c_void, + session_slot: *const *mut c_void, + errors: *const ErrorDescriptor, + buffer: *mut Buffer, + ) -> u32 { + if !unsafe { empty(buffer) } { + return 4; + } + let prepared = (|| { + unsafe { record(session_slot) }?; + let errors = unsafe { Errors::new(errors) }?; + let function = unsafe { record(function) }?; + let (_, layout) = + unsafe { owned_layout(function.output_type) }.map_err(invalid)?; + if out.is_null() + || !(out as usize).is_multiple_of(layout.align()) + || fault.is_null() + || !(fault as usize).is_multiple_of(errors.align) + || !std::ptr::eq(function.fault_type, errors.ty) + { + return Err(invalid("invalid database root output descriptor")); + } + let future = unsafe { ForeignFuture::new(function, input) }.map_err(invalid)?; + Ok((future, errors)) + })(); + let (future, errors) = match prepared { + Ok(value) => value, + Err(failure) => { + unsafe { abi_failure(failure, buffer) }; + return 4; + } + }; + let result = task::run_entry_with_typed_resource_cleanup( + task::RuntimeConfig::default(), + future, + |result, resources| async move { + let result = cleanup_result( + result, + resources, + errors.descriptor.append_cause.expect("validated callback"), + ); + let handle = unsafe { *session_slot }; + if handle.is_null() { + return result; + } + let session = unsafe { managed::borrowed::<db::Session>(handle) } + .map_err(|failure| ForeignFault::Runtime(failure.message))?; + match (result, session.close().await) { + (result, Ok(())) => result, + (Ok(_), Err(cleanup)) => Err(errors.fault(cleanup)), + (Err(primary), Err(cleanup)) => Err(errors.augment(primary, cleanup)), + } + }, + ); + unsafe { root_result(result, out, fault, buffer) } + } + /// # Safety + /// cause is an initialized nullable owned Text slot; message is borrowed Text. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_db_cause_append( + cause: *mut *mut c_void, + message: *const c_void, + ) { + unsafe { managed::append_fault_cause(cause, message) }; + } + /// # Safety + /// All inputs are borrowed matching owners and static descriptors. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_db_prepare( + session: *const c_void, + errors: *const ErrorDescriptor, + buffer: *mut Buffer, + ) -> *mut AsyncOp { + unsafe { + operation( + errors, + buffer, + 0, + false, + || shared::<db::Session>(session), + |session| async move { + session.prepare().await?; + Ok(Output::Unit) + }, + ) + } + } + /// # Safety + /// Borrows this entry's Session and a static Model binding/error descriptor. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_db_resolve_scoped( + session: *const c_void, + binding: *const Binding, + errors: *const ErrorDescriptor, + buffer: *mut Buffer, + ) -> *mut AsyncOp { + unsafe { + operation( + errors, + buffer, + align_of::<*mut c_void>(), + false, + || { + let session = shared::<db::Session>(session)?; + let binding = record(binding)?; + let explicit = optional_name(binding.explicit)?; + let root = name(&binding.root)?.to_owned(); + let tenant = flag(binding.tenant)?; + Ok((session, explicit, root, tenant)) + }, + |(session, explicit, root, tenant)| async move { + #[cfg(feature = "runtime-api")] + if tenant + && dever_runtime::config::current_settings().tenant().is_some() + { + let database = session.database(explicit.as_deref(), &root)?; + return Ok(Output::Database( + dever_runtime::tenant::database(database.name()).await?, + )); + } + #[cfg(not(feature = "runtime-api"))] + let _ = tenant; + Ok(Output::Database( + session.database(explicit.as_deref(), &root)?, + )) + }, + ) + } + } + /// # Safety + /// The session/binding are borrowed; out and fault are matching writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_db_select( + session: *const c_void, + binding: *const Binding, + out: *mut *mut c_void, + errors: *const ErrorDescriptor, + fault: *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + unsafe { + regular(out, errors, fault, buffer, || { + let session = managed::borrowed::<db::Session>(session)?; + let binding = record(binding)?; + let explicit = if binding.explicit.is_null() { + None + } else { + Some(name(record(binding.explicit)?)?) + }; + flag(binding.tenant)?; + Ok(managed::owned( + session.database(explicit, name(&binding.root)?)?, + )) + }) + } + } + /// # Safety + /// Database is borrowed and out is a writable Int slot. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_db_max_page_size( + database: *const c_void, + out: *mut i64, + buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { managed::checked_output(out, buffer) } { + return status; + } + let result = + unsafe { managed::borrowed::<db::Database>(database) }.and_then(|database| { + i64::try_from(database.max_page_size()) + .map_err(|_| managed::invalid("database page maximum exceeds Int")) + }); + unsafe { managed::complete(out, buffer, result) } + } + /// # Safety + /// Arrays contain matching borrowed database/model/error rows. SQL and + /// error descriptors are static; all other metadata is copied now. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_db_migrate_models( + databases: *const *const c_void, + models: *const Model, + errors: *const *const ErrorDescriptor, + count: u64, + buffer: *mut Buffer, + ) -> *mut AsyncOp { + if !unsafe { empty(buffer) } { + return ptr::null_mut(); + } + let prepared = (|| { + let databases = unsafe { array(databases, count) }?; + let models = unsafe { array(models, count) }?; + let errors = unsafe { array(errors, count) }?; + let first = unsafe { + Errors::new( + *errors + .first() + .ok_or_else(|| invalid("missing database Models"))?, + ) + }?; + let mut entries = Vec::with_capacity(databases.len()); + for ((database, model_row), errors) in databases.iter().zip(models).zip(errors) + { + let errors = unsafe { Errors::new(*errors) }?; + if !std::ptr::eq(first.ty, errors.ty) { + return Err(invalid("mismatched database migration fault types")); + } + let database = unsafe { cloned::<db::Database>(*database) }?; + let model = match unsafe { model(model_row) } { + Ok(model) => model, + Err(Failure::Database(error)) => { + return Ok((first, Err((errors, error)))); + } + Err(failure) => return Err(failure), + }; + entries.push(migration_entry(database, model, errors)); + } + entries.sort_by(|left, right| left.0.name().cmp(right.0.name())); + Ok((first, Ok(entries))) + })(); + let (errors, entries) = match prepared { + Ok(value) => value, + Err(failure) => { + unsafe { abi_failure(failure, buffer) }; + return ptr::null_mut(); + } + }; + operation_with_output( + async move { + let entries = entries.map_err(|(errors, error)| errors.fault(error))?; + migrate_entries(entries).await?; + Ok(OpValue::Database(Output::Unit)) + }, + Some(errors.slots(0, false)), + ) + } + + type MigrationEntry = ( + db::Database, + db::Model, + (String, String, Vec<db::PostgresConstraint>), + Errors, + ); + fn migration_entry( + database: db::Database, + model: db::Model, + errors: Errors, + ) -> MigrationEntry { + let keys = model + .postgres_constraints + .iter() + .filter(|constraint| constraint.foreign_key) + .map(|constraint| db::PostgresConstraint { + name: constraint.name.clone(), + definition: constraint.definition.clone(), + foreign_key: true, + }) + .collect(); + let keys = (model.schema.model.clone(), model.schema.table.clone(), keys); + (database, model, keys, errors) + } + async fn migrate_entries(mut entries: Vec<MigrationEntry>) -> Result<(), ForeignFault> { + entries.sort_by(|left, right| left.0.name().cmp(right.0.name())); + let mut foreign_keys = Vec::with_capacity(entries.len()); + for (database, model, keys, errors) in entries { + database + .migrate(model) + .await + .map_err(|error| errors.fault(error))?; + foreign_keys.push((database, keys, errors)); + } + for (database, (model, table, keys), errors) in foreign_keys { + if !keys.is_empty() { + database + .migrate_foreign_keys(model, table, keys) + .await + .map_err(|error| errors.fault(error))?; + } + } + Ok(()) + } + + #[cfg(feature = "runtime-api")] + /// # Safety + /// Session is borrowed. Parallel arrays are immutable compiler metadata. + /// Zero tenant_id migrates platform Models; positive ids are explicit migration. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_api_migrate_models( + session: *const c_void, + bindings: *const Binding, + models: *const Model, + errors: *const *const ErrorDescriptor, + count: u64, + tenant_id: i64, + buffer: *mut Buffer, + ) -> *mut AsyncOp { + unsafe { + dever_rt_v1_api_migrate_application( + session, + bindings, + models, + errors, + count, + ptr::null(), + tenant_id, + buffer, + ) + } + } + + #[cfg(feature = "runtime-api")] + /// # Safety + /// Same immutable metadata contract as api_migrate_models. Empty Model arrays + /// require a fallback fault descriptor for applications containing only Jobs. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_api_migrate_application( + session: *const c_void, + bindings: *const Binding, + models: *const Model, + errors: *const *const ErrorDescriptor, + count: u64, + fallback_errors: *const ErrorDescriptor, + tenant_id: i64, + buffer: *mut Buffer, + ) -> *mut AsyncOp { + if !unsafe { empty(buffer) } { + return ptr::null_mut(); + } + let prepared = (|| { + let session = unsafe { + managed::borrowed::<Arc<dever_runtime::application::Session>>(session) + }? + .clone(); + let bindings = unsafe { array(bindings, count) }?; + let models = unsafe { array(models, count) }?; + let error_rows = unsafe { array(errors, count) }?; + let first = unsafe { + Errors::new( + error_rows + .first() + .copied() + .or_else(|| (!fallback_errors.is_null()).then_some(fallback_errors)) + .ok_or_else(|| { + invalid("missing database migration fault mapping") + })?, + ) + }?; + let mut definitions = Vec::with_capacity(models.len()); + for ((binding, model_row), errors) in + bindings.iter().zip(models).zip(error_rows) + { + let errors = unsafe { Errors::new(*errors) }?; + if !std::ptr::eq(first.ty, errors.ty) { + return Err(invalid("mismatched database migration fault types")); + } + let explicit = unsafe { optional_name(binding.explicit) }?; + let root = unsafe { name(&binding.root) }?.to_owned(); + let tenant = flag(binding.tenant)?; + let model = match unsafe { model(model_row) } { + Err(failure @ Failure::Abi(_)) => return Err(failure), + result => result, + }; + definitions.push((explicit, root, tenant, model, errors)); + } + Ok::<_, Failure>((session, definitions, first)) + })(); + let (session, definitions, errors) = match prepared { + Ok(value) => value, + Err(failure) => { + unsafe { abi_failure(failure, buffer) }; + return ptr::null_mut(); + } + }; + operation_with_output( + async move { + session + .clone() + .scope(async move { + let tenant_enabled = session.settings().tenant().is_some(); + if tenant_id < 0 || (tenant_id > 0 && !tenant_enabled) { + return Err(errors.fault(orm::Error::migration( + "tenant storage is not configured or tenant id is invalid", + ))); + } + let mut tenant_databases = + std::collections::BTreeMap::<String, db::Database>::new(); + let fingerprint = if tenant_id > 0 { + dever_runtime::tenant::schema_fingerprint() + .map_err(|error| errors.fault(error))? + } else { + String::new() + }; + if tenant_id > 0 { + if !definitions.iter().any(|(_, _, tenant, _, _)| *tenant) + && !session + .jobs() + .is_some_and(|jobs| jobs.has_tenant_jobs()) + { + return Err(errors.fault(orm::Error::migration( + "application has no tenant Model or Job", + ))); + } + if session.has_authorization() { + let connection = session + .settings() + .tenant() + .expect("validated tenant settings") + .database(); + let database = dever_runtime::tenant::begin_migration( + connection, + tenant_id, + &fingerprint, + ) + .await + .map_err(|error| errors.fault(error))?; + tenant_databases.insert(connection.to_owned(), database); + } + } + let mut entries = Vec::new(); + for (explicit, root, tenant, model, error_mapping) in definitions { + if (tenant_id > 0 && !tenant) + || (tenant_id == 0 && tenant && tenant_enabled) + { + continue; + } + let model = + model.map_err(|error| error_mapping.failure(error))?; + let platform = session + .databases() + .database(explicit.as_deref(), &root) + .map_err(|error| error_mapping.fault(error))?; + let database = if tenant_id == 0 { + platform + } else { + let connection = platform.name().to_owned(); + if !tenant_databases.contains_key(&connection) { + let database = dever_runtime::tenant::begin_migration( + &connection, + tenant_id, + &fingerprint, + ) + .await + .map_err(|error| error_mapping.fault(error))?; + tenant_databases.insert(connection.clone(), database); + } + tenant_databases[&connection].clone() + }; + entries.push(migration_entry(database, model, error_mapping)); + } + migrate_entries(entries).await?; + if tenant_id > 0 { + if let Some(jobs) = session.jobs() { + jobs.migrate_tenant( + tenant_id, + &fingerprint, + &mut tenant_databases, + ) + .await + .map_err(|error| errors.fault(error))?; + } + if session.has_authorization() { + let connection = session + .settings() + .tenant() + .expect("validated tenant settings") + .database(); + dever_runtime::auth::store::initialize_roles( + tenant_databases[connection].clone(), + ) + .await + .map_err(|error| errors.fault(error))?; + } + for database in tenant_databases.values() { + dever_runtime::tenant::mark_database_ready( + database, + tenant_id, + &fingerprint, + ) + .await + .map_err(|error| errors.fault(error))?; + } + dever_runtime::tenant::finish_migration( + tenant_id, + &fingerprint, + ) + .await + .map_err(|error| errors.fault(error))?; + } + Ok(OpValue::Database(Output::Unit)) + }) + .await + }, + Some(errors.slots(0, false)), + ) + } + + /// # Safety + /// Database is a borrowed owner and errors is static. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_db_begin( + database: *const c_void, + errors: *const ErrorDescriptor, + buffer: *mut Buffer, + ) -> *mut AsyncOp { + unsafe { + operation( + errors, + buffer, + align_of::<*mut c_void>(), + false, + || cloned::<db::Database>(database), + |database| async move { + Ok(Output::Transaction(Arc::new(db::TransactionState::new( + database.begin().await?, + )))) + }, + ) + } + } + /// # Safety + /// Transaction is a borrowed shared state; commit is 0 or 1. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_db_finish( + transaction: *const c_void, + commit: u8, + errors: *const ErrorDescriptor, + buffer: *mut Buffer, + ) -> *mut AsyncOp { + unsafe { + operation( + errors, + buffer, + 0, + false, + || Ok((cloned::<Transaction>(transaction)?, flag(commit)?)), + |(transaction, commit)| async move { + transaction.finish(commit).await?; + Ok(Output::Unit) + }, + ) + } + } + /// # Safety + /// Primary is an initialized concrete fault, transferred only on non-null + /// creation. The transaction remains borrowed and is cloned before suspend. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_db_rollback_take( + transaction: *const c_void, + primary: *mut c_void, + primary_type: *const OwnedType, + errors: *const ErrorDescriptor, + buffer: *mut Buffer, + ) -> *mut AsyncOp { + if !unsafe { empty(buffer) } { + return ptr::null_mut(); + } + let prepared = (|| { + let errors = unsafe { Errors::new(errors) }?; + if !std::ptr::eq(primary_type, errors.ty) + || primary.is_null() + || !(primary as usize).is_multiple_of(errors.align) + { + return Err(invalid("invalid rollback primary fault")); + } + let transaction = unsafe { cloned::<Transaction>(transaction) }?; + let mut owner = unsafe { Owned::new(primary_type) }.map_err(invalid)?; + unsafe { owner.take_from(primary) }; + Ok((transaction, owner, errors)) + })(); + let (transaction, primary, errors) = match prepared { + Ok(value) => value, + Err(failure) => { + unsafe { abi_failure(failure, buffer) }; + return ptr::null_mut(); + } + }; + operation_with_output( + async move { + let primary = ForeignFault::Typed(primary); + Err(match transaction.finish(false).await { + Ok(()) => primary, + Err(rollback) => errors.augment(primary, rollback), + }) + }, + Some(errors.slots(0, false)), + ) + } + + /// # Safety + /// SQL is static; parameter handles and database/transaction are borrowed. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_db_execute( + database: *const c_void, + transaction: *const c_void, + sql: *const Sql, + parameters: *const Value, + count: u64, + errors: *const ErrorDescriptor, + buffer: *mut Buffer, + ) -> *mut AsyncOp { + unsafe { + operation( + errors, + buffer, + align_of::<i64>(), + false, + || Query::new(database, transaction, sql, parameters, count), + |query| async move { + let transaction = + db::TransactionState::borrow(query.transaction.as_deref()).await?; + let executor = + db::Executor::new(query.database, transaction.as_deref())?; + let result = executor.execute(query.sql, query.parameters).await; + drop(executor); + Ok(Output::Int(orm::affected(completed(transaction, result)?)?)) + }, + ) + } + } + /// # Safety + /// SQL is static; other inputs are cloned before the operation suspends. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_db_query( + database: *const c_void, + transaction: *const c_void, + sql: *const Sql, + parameters: *const Value, + count: u64, + maximum: i64, + errors: *const ErrorDescriptor, + buffer: *mut Buffer, + ) -> *mut AsyncOp { + unsafe { + operation( + errors, + buffer, + align_of::<*mut c_void>(), + false, + || { + let maximum = usize::try_from(maximum) + .map_err(|_| invalid("invalid database query maximum"))?; + Ok(( + Query::new(database, transaction, sql, parameters, count)?, + maximum, + )) + }, + |(query, maximum)| async move { + let transaction = + db::TransactionState::borrow(query.transaction.as_deref()).await?; + let executor = + db::Executor::new(query.database, transaction.as_deref())?; + let result = if maximum == 0 { + executor.query(query.sql, query.parameters).await + } else { + executor + .query_bounded(query.sql, query.parameters, maximum) + .await + }; + drop(executor); + Ok(Output::Rows(completed(transaction, result)?)) + }, + ) + } + } + /// # Safety + /// SQL fragments are static; IDs and owners are borrowed only until creation. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_db_relation_query( + database: *const c_void, + transaction: *const c_void, + prefix: *const Name, + before_limit: *const Name, + suffix: *const Name, + ids: *const i64, + count: u64, + limit: i64, + errors: *const ErrorDescriptor, + buffer: *mut Buffer, + ) -> *mut AsyncOp { + unsafe { + operation( + errors, + buffer, + align_of::<*mut c_void>(), + false, + || { + let ids = array(ids, count)?; + let sql = db::relation_sql( + name(record(prefix)?)?, + name(record(before_limit)?)?, + name(record(suffix)?)?, + ids.len(), + ); + let parameters = ids + .iter() + .copied() + .chain(std::iter::once(limit)) + .map(orm::Value::Int) + .collect::<Vec<_>>(); + Ok(( + cloned::<db::Database>(database)?, + if transaction.is_null() { + None + } else { + Some(cloned::<Transaction>(transaction)?) + }, + sql, + parameters, + )) + }, + |(database, transaction, sql, parameters)| async move { + let transaction = + db::TransactionState::borrow(transaction.as_deref()).await?; + let executor = db::Executor::new(database, transaction.as_deref())?; + let result = executor.query_owned(sql, parameters).await; + drop(executor); + Ok(Output::Rows(completed(transaction, result)?)) + }, + ) + } + } + + #[derive(Clone)] + pub(super) struct Stream { + rows: orm::RowStream<orm::Row>, + decoder: Decoder, + } + #[derive(Clone, Copy)] + struct Decoder { + row_type: &'static OwnedType, + decode: unsafe extern "C" fn( + *const c_void, + *mut c_void, + *const ErrorDescriptor, + *mut c_void, + *mut Buffer, + ) -> u32, + } + // Decoder contains immutable process-static checked transferable metadata. + unsafe impl Send for Decoder {} + unsafe impl Sync for Decoder {} + impl Decoder { + unsafe fn new(pointer: *const RowDecoder) -> Result<Self, Failure> { + let decoder = unsafe { record(pointer) }?; + let (row_type, _) = + unsafe { owned_layout(decoder.row_type) }.map_err(invalid)?; + if row_type.transferable == 0 { + return Err(invalid("database stream row is not transferable")); + } + Ok(Self { + row_type, + decode: decoder + .decode + .ok_or_else(|| invalid("missing database row decoder"))?, + }) + } + fn row(self, row: orm::Row, errors: Errors) -> Result<Owned, ForeignFault> { + let mut output = + unsafe { Owned::new(self.row_type) }.map_err(ForeignFault::Runtime)?; + let mut fault = + unsafe { Owned::new(errors.ty) }.map_err(ForeignFault::Runtime)?; + let mut buffer = Buffer { + ptr: ptr::null_mut(), + len: 0, + }; + let status = unsafe { + (self.decode)( + std::ptr::from_ref(&row).cast(), + output.pointer(), + errors.descriptor, + fault.pointer(), + &mut buffer, + ) + }; + if status == 0 { + output.initialized = true; + } + if status == 1 { + fault.initialized = true; + } + if status <= 1 && buffer.ptr.is_null() && buffer.len == 0 { + return if status == 0 { + Ok(output) + } else { + Err(ForeignFault::Typed(fault)) + }; + } + let message = if buffer.ptr.is_null() { + "invalid database row decoder result".to_owned() + } else { + let bytes = + unsafe { std::slice::from_raw_parts(buffer.ptr, buffer.len as usize) }; + let message = String::from_utf8_lossy(bytes).into_owned(); + unsafe { super::super::dever_rt_v1_buffer_free(buffer.ptr, buffer.len) }; + message + }; + Err(ForeignFault::Runtime(message)) + } + } + /// # Safety + /// SQL and decoder metadata are static. All borrowed values are cloned now. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_db_stream( + database: *const c_void, + transaction: *const c_void, + sql: *const Sql, + parameters: *const Value, + count: u64, + capacity: i64, + decoder: *const RowDecoder, + errors: *const ErrorDescriptor, + buffer: *mut Buffer, + ) -> *mut AsyncOp { + unsafe { + operation( + errors, + buffer, + align_of::<*mut c_void>(), + false, + || { + Ok(( + Query::new(database, transaction, sql, parameters, count)?, + Decoder::new(decoder)?, + )) + }, + move |(query, decoder)| async move { + let transaction = + db::TransactionState::borrow(query.transaction.as_deref()).await?; + let executor = + db::Executor::new(query.database, transaction.as_deref())?; + let capacity = + orm::stream_capacity(capacity, executor.max_page_size())?; + let result = + executor.stream(query.sql, query.parameters, capacity).await; + drop(executor); + Ok(Output::Stream(Stream { + rows: completed(transaction, result)?, + decoder, + })) + }, + ) + } + } + /// # Safety + /// Stream is borrowed; output is a writable Unit byte. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_db_stream_close( + stream: *const c_void, + out: *mut u8, + buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { managed::checked_output(out, buffer) } { + return status; + } + let result = unsafe { managed::borrowed::<Stream>(stream) }.map(|stream| { + stream.rows.close(); + 0 + }); + unsafe { managed::complete(out, buffer, result) } + } + /// # Safety + /// The stream is borrowed; errors identifies the current consumption site. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_db_stream_pull( + stream: *const c_void, + errors: *const ErrorDescriptor, + buffer: *mut Buffer, + ) -> *mut AsyncOp { + if !unsafe { empty(buffer) } { + return ptr::null_mut(); + } + let prepared = (|| { + Ok((unsafe { cloned::<Stream>(stream) }?, unsafe { + Errors::new(errors) + }?)) + })(); + let (stream, errors) = match prepared { + Ok(value) => value, + Err(failure) => { + unsafe { abi_failure(failure, buffer) }; + return ptr::null_mut(); + } + }; + let align = stream.decoder.row_type.align as usize; + operation_with_output( + async move { + let row = stream + .rows + .pull() + .await + .map_err(|error| errors.fault(error))?; + let decoded = row.map(|row| stream.decoder.row(row, errors)).transpose(); + if decoded.is_err() { + stream.rows.close(); + } + decoded.map(|row| OpValue::Database(Output::Row(row))) + }, + Some(errors.slots(align, true)), + ) + } + + /// # Safety + /// Transfers one owned rows collection, or null. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_db_rows_release(rows: *mut c_void) { + unsafe { managed::release::<Vec<orm::Row>>(rows) } + } + /// # Safety + /// Transfers one owned database row, or null; decoder callback rows are borrowed. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_db_row_release(row: *mut c_void) { + unsafe { managed::release::<orm::Row>(row) } + } + macro_rules! length { + ($function:ident, $ty:ty) => { + /// # Safety + /// Handle is borrowed and out is a writable Int slot. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn $function( + handle: *const c_void, + out: *mut i64, + buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { managed::checked_output(out, buffer) } { + return status; + } + let result = + unsafe { managed::borrowed::<$ty>(handle) }.and_then(|value| { + i64::try_from(value.len()) + .map_err(|_| managed::invalid("database row count exceeds Int")) + }); + unsafe { managed::complete(out, buffer, result) } + } + }; + } + length!(dever_rt_v1_db_rows_len, Vec<orm::Row>); + length!(dever_rt_v1_db_row_len, orm::Row); + /// # Safety + /// Rows is borrowed, index is checked, out receives an independent owned row. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_db_rows_at( + rows: *const c_void, + index: i64, + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { managed::checked_output(out, buffer) } { + return status; + } + let result = unsafe { managed::borrowed::<Vec<orm::Row>>(rows) }.and_then(|rows| { + usize::try_from(index) + .ok() + .and_then(|index| rows.get(index)) + .cloned() + .ok_or_else(|| managed::invalid("invalid database row index")) + }); + unsafe { managed::complete_handle(out, buffer, result) } + } + /// # Safety + /// Rows/operation are borrowed; out and fault are distinct matching destinations. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_db_required_row( + rows: *const c_void, + operation: *const Name, + out: *mut *mut c_void, + errors: *const ErrorDescriptor, + fault: *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + unsafe { + regular(out, errors, fault, buffer, || { + Ok(managed::owned(orm::required_row( + cloned(rows)?, + name(record(operation)?)?, + )?)) + }) + } + } + /// # Safety + /// The presence slot is distinct; absent success leaves out uninitialized. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_db_optional_row( + rows: *const c_void, + operation: *const Name, + out: *mut *mut c_void, + present: *mut u8, + errors: *const ErrorDescriptor, + fault: *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + let errors = match unsafe { checked(out, errors, fault, buffer) } { + Ok(errors) => errors, + Err(status) => return status, + }; + if present.is_null() { + return unsafe { error(buffer, 2, "invalid database presence output") }; + } + let result = (|| { + Ok(orm::optional_row(unsafe { cloned(rows) }?, unsafe { + name(record(operation)?) + }?)?) + })(); + match result { + Ok(value) => { + unsafe { present.write(u8::from(value.is_some())) }; + if let Some(value) = value { + unsafe { out.write(managed::owned(value)) }; + } + 0 + } + Err(failure) => unsafe { + finish::<*mut c_void>(Err(failure), out, errors, fault, buffer) + }, + } + } + unsafe fn column(row: *const c_void, index: i64) -> Result<orm::Value, Failure> { + let row = unsafe { managed::borrowed::<orm::Row>(row) }?; + let index = + usize::try_from(index).map_err(|_| invalid("invalid database column index"))?; + Ok(row.get(index)?.clone()) + } + macro_rules! getter { + ($function:ident, $ty:ty, $convert:expr) => { + /// # Safety + /// Row is borrowed; output/fault slots match their descriptors. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn $function( + row: *const c_void, + index: i64, + out: *mut $ty, + errors: *const ErrorDescriptor, + fault: *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + unsafe { + regular(out, errors, fault, buffer, || { + let value = column(row, index)?; + ($convert)(value).map_err(Failure::from) + }) + } + } + }; + } + getter!(dever_rt_v1_db_row_null, u8, |value| Ok::<_, orm::Error>( + u8::from(matches!(value, orm::Value::Null)) + )); + getter!(dever_rt_v1_db_row_bool, u8, |value| orm::boolean(value) + .map(u8::from)); + getter!(dever_rt_v1_db_row_int, i64, orm::int); + getter!(dever_rt_v1_db_row_float, f64, orm::float); + getter!(dever_rt_v1_db_row_decimal, crate::AbiDecimal, |value| { + orm::decimal(value).map(crate::AbiDecimal::encode) + }); + getter!(dever_rt_v1_db_row_text, *mut c_void, |value| orm::text( + value + ) + .map(managed::owned)); + getter!(dever_rt_v1_db_row_bytes, *mut c_void, |value| orm::bytes( + value + ) + .map(managed::owned)); + getter!(dever_rt_v1_db_row_uuid, *mut c_void, |value| orm::uuid( + value + ) + .map(managed::owned)); + /// # Safety + /// Outputs are distinct matching owned UUID/fault destinations. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_db_uuid_new( + out: *mut *mut c_void, + errors: *const ErrorDescriptor, + fault: *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + unsafe { + regular(out, errors, fault, buffer, || { + Ok(managed::owned(orm::Uuid::new_v7()?)) + }) + } + } + /// # Safety + /// Message is borrowed Text and fault matches the static descriptor. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_db_error( + kind: u32, + message: *const c_void, + errors: *const ErrorDescriptor, + fault: *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + let mut unit = 0u8; + unsafe { + regular(&mut unit, errors, fault, buffer, || { + Err::<u8, _>( + orm::Error::new(error_kind(kind)?, cloned::<String>(message)?).into(), + ) + }) + } + } + /// # Safety + /// All numeric and fault destinations are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_db_pagination( + page: i64, + size: i64, + maximum: i64, + out_page: *mut i64, + out_size: *mut i64, + out_offset: *mut i64, + errors: *const ErrorDescriptor, + fault: *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + let errors = match unsafe { checked(out_page, errors, fault, buffer) } { + Ok(errors) => errors, + Err(status) => return status, + }; + if out_offset.is_null() + || !out_offset.is_aligned() + || out_size.is_null() + || !out_size.is_aligned() + { + return unsafe { error(buffer, 2, "invalid pagination output") }; + } + let result = usize::try_from(maximum) + .map_err(|_| invalid("invalid page maximum")) + .and_then(|maximum| Ok(orm::pagination(page, size, maximum)?)); + match result { + Ok((a, b, c)) => { + unsafe { + out_page.write(a); + out_size.write(b); + out_offset.write(c); + } + 0 + } + Err(failure) => unsafe { + finish::<i64>(Err(failure), out_page, errors, fault, buffer) + }, + } + } + /// # Safety + /// Out/fault destinations match the declared concrete types. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_db_cursor_size( + size: i64, + maximum: i64, + out: *mut i64, + errors: *const ErrorDescriptor, + fault: *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + unsafe { + regular(out, errors, fault, buffer, || { + Ok(orm::cursor_size( + size, + usize::try_from(maximum) + .map_err(|_| invalid("invalid cursor maximum"))?, + )?) + }) + } + } + /// # Safety + /// Out/fault destinations match the declared concrete types. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_db_stream_capacity( + size: i64, + maximum: i64, + out: *mut i64, + errors: *const ErrorDescriptor, + fault: *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + unsafe { + regular(out, errors, fault, buffer, || { + let capacity = orm::stream_capacity( + size, + usize::try_from(maximum) + .map_err(|_| invalid("invalid stream maximum"))?, + )?; + i64::try_from(capacity).map_err(|_| invalid("stream capacity exceeds Int")) + }) + } + } + /// # Safety + /// Out/fault destinations match the declared concrete types. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_db_relation_limit( + maximum: i64, + out: *mut i64, + errors: *const ErrorDescriptor, + fault: *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + unsafe { + regular(out, errors, fault, buffer, || { + Ok(orm::relation_limit( + usize::try_from(maximum) + .map_err(|_| invalid("invalid relation maximum"))?, + )?) + }) + } + } + /// # Safety + /// Type describes the borrowed initialized source; out receives one owner. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_db_related_new( + ty: *const managed::TypeDescriptor, + value: *const c_void, + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { managed::checked_output(out, buffer) } { + return status; + } + let result = unsafe { managed::descriptor(ty, false) } + .and_then(|ops| unsafe { Element::from_source(ops, value) }); + unsafe { managed::complete_handle(out, buffer, result) } + } + /// # Safety + /// Both pointers are matching nullable Related owners. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_db_related_equal( + left: *const c_void, + right: *const c_void, + ) -> u8 { + if left.is_null() || right.is_null() { + return u8::from(left == right); + } + u8::from(unsafe { *left.cast::<Element>() == *right.cast::<Element>() }) + } + /// # Safety + /// Related is Loaded; out is uninitialized storage for its exact payload type. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_db_related_get( + related: *const c_void, + out: *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + if !unsafe { empty(buffer) } { + return 2; + } + match unsafe { managed::borrowed::<Element>(related) } + .and_then(|value| unsafe { value.copy_to(out) }) + { + Ok(()) => 0, + Err(failure) => unsafe { error(buffer, failure.status, failure.message) }, + } + } + } + + mod protocol { + use super::*; + use dever_runtime::{http, sse, tls, websocket}; + + pub(super) enum Value { + Response(http::Response), + StreamResponse(http::StreamResponse), + Client(http::HttpClient), + Socket(websocket::WebSocket), + Message(Option<websocket::Message>), + } + + impl Value { + pub(super) unsafe fn write( + self, + output: *mut c_void, + present: *mut u8, + ) -> Result<(), &'static str> { + let output = output.cast::<*mut c_void>(); + if output.is_null() || !output.is_aligned() { + return Err("missing protocol output"); + } + let handle = match self { + Self::Response(value) => managed::owned(value), + Self::StreamResponse(value) => managed::owned(value), + Self::Client(value) => managed::owned(value), + Self::Socket(value) => managed::owned(value), + Self::Message(value) => { + if present.is_null() { + return Err("missing message presence output"); + } + unsafe { present.write(u8::from(value.is_some())) }; + let Some(value) = value else { + return Ok(()); + }; + managed::owned(value) + } + }; + unsafe { output.write(handle) }; + Ok(()) + } + } + + unsafe fn cloned<T: Clone>(handle: *const c_void) -> Result<T, managed::AbiFailure> { + unsafe { managed::borrowed::<T>(handle) }.cloned() + } + + unsafe fn record<'a, T>(pointer: *const T) -> Result<&'a T, managed::AbiFailure> { + if pointer.is_null() || !pointer.is_aligned() { + return Err(managed::invalid("invalid protocol record")); + } + Ok(unsafe { &*pointer }) + } + + macro_rules! owner { + ($retain:ident, $release:ident, $ty:ty) => { + /// # Safety + /// handle borrows the matching opaque protocol owner. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn $retain(handle: *const c_void) -> *mut c_void { + unsafe { managed::retain::<$ty>(handle) } + } + /// # Safety + /// handle transfers one matching protocol owner, or null. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn $release(handle: *mut c_void) { + unsafe { managed::release::<$ty>(handle) } + } + }; + } + owner!( + dever_rt_v1_http_headers_retain, + dever_rt_v1_http_headers_release, + Mutex<Vec<http::Header>> + ); + owner!( + dever_rt_v1_http_request_retain, + dever_rt_v1_http_request_release, + http::Request + ); + owner!( + dever_rt_v1_http_response_retain, + dever_rt_v1_http_response_release, + http::Response + ); + owner!( + dever_rt_v1_http_stream_request_retain, + dever_rt_v1_http_stream_request_release, + http::StreamRequest + ); + owner!( + dever_rt_v1_http_stream_response_retain, + dever_rt_v1_http_stream_response_release, + http::StreamResponse + ); + owner!( + dever_rt_v1_client_tls_retain, + dever_rt_v1_client_tls_release, + tls::ClientTls + ); + owner!( + dever_rt_v1_server_tls_retain, + dever_rt_v1_server_tls_release, + tls::ServerTls + ); + owner!( + dever_rt_v1_http_client_retain, + dever_rt_v1_http_client_release, + http::HttpClient + ); + owner!( + dever_rt_v1_http_reply_retain, + dever_rt_v1_http_reply_release, + http::HttpReply + ); + owner!( + dever_rt_v1_websocket_retain, + dever_rt_v1_websocket_release, + websocket::WebSocket + ); + owner!( + dever_rt_v1_ws_message_retain, + dever_rt_v1_ws_message_release, + websocket::Message + ); + + #[repr(C)] + #[derive(Clone, Copy)] + pub(super) struct Http2Limits { + streams: i64, + stream_window_bytes: i64, + connection_window_bytes: i64, + } + #[repr(C)] + pub(super) struct HttpLimits { + header_bytes: i64, + body_bytes: i64, + timeout_ms: i64, + connections: i64, + http2: *const Http2Limits, + } + impl HttpLimits { + unsafe fn runtime(&self) -> Result<http::Limits, managed::AbiFailure> { + let http2 = if self.http2.is_null() { + None + } else { + let value = unsafe { record(self.http2) }?; + Some(http::Http2Limits { + streams: value.streams, + stream_window_bytes: value.stream_window_bytes, + connection_window_bytes: value.connection_window_bytes, + }) + }; + Ok(http::Limits { + header_bytes: self.header_bytes, + body_bytes: self.body_bytes, + timeout_ms: self.timeout_ms, + connections: self.connections, + http2, + }) + } + } + #[repr(C)] + pub(super) struct PoolLimits { + idle_ms: i64, + chunk_bytes: i64, + read_ms: i64, + } + #[repr(C)] + pub(super) struct LiveLimits { + chunk_bytes: i64, + idle_ms: i64, + heartbeat_ms: i64, + } + #[repr(C)] + pub(super) struct WsLimits { + message_bytes: i64, + idle_ms: i64, + write_ms: i64, + } + impl WsLimits { + fn runtime(&self) -> websocket::Limits { + websocket::Limits { + message_bytes: self.message_bytes, + idle_ms: self.idle_ms, + write_ms: self.write_ms, + } + } + } + #[repr(C)] + #[derive(Clone, Copy)] + pub(super) struct RequestFields { + method: *mut c_void, + target: *mut c_void, + headers: *mut c_void, + body: *mut c_void, + } + #[repr(C)] + #[derive(Clone, Copy)] + pub(super) struct ResponseFields { + status: i64, + headers: *mut c_void, + body: *mut c_void, + } + #[repr(C)] + #[derive(Clone, Copy)] + pub(super) struct HeaderFields { + name: *mut c_void, + value: *mut c_void, + } + #[repr(C)] + #[derive(Clone, Copy)] + pub(super) struct MessageFields { + kind: u32, + payload: *mut c_void, + } + #[repr(C)] + pub(super) struct SseEvent { + event: *const c_void, + data: *const c_void, + id: *const c_void, + retry_ms: i64, + has_retry: u8, + } + + unsafe fn headers( + handle: *const c_void, + ) -> Result<Vec<http::Header>, managed::AbiFailure> { + unsafe { managed::borrowed::<Mutex<Vec<http::Header>>>(handle) }? + .lock() + .map(|headers| headers.clone()) + .map_err(|_| managed::invalid("HTTP headers lock failed")) + } + + /// # Safety + /// out/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_http_headers_new( + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + unsafe { + managed::complete_handle( + out, + buffer, + Ok(Mutex::new(Vec::<http::Header>::new())), + ) + } + } + /// # Safety + /// Handles borrow matching header builder/Text/Bytes; buffer is writable. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_http_headers_push( + handle: *mut c_void, + name: *const c_void, + value: *const c_void, + buffer: *mut Buffer, + ) -> u32 { + if !unsafe { empty(buffer) } { + return ABI_INVALID_INPUT; + } + let result: Result<(), managed::AbiFailure> = (|| { + let header = http::Header { + name: unsafe { cloned(name) }?, + value: unsafe { cloned(value) }?, + }; + unsafe { managed::borrowed::<Mutex<Vec<http::Header>>>(handle) }? + .lock() + .map_err(|_| managed::invalid("HTTP headers lock failed"))? + .push(header); + Ok(()) + })(); + match result { + Ok(()) => 0, + Err(failure) => unsafe { error(buffer, failure.status, failure.message) }, + } + } + /// # Safety + /// headers is borrowed; output/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_http_headers_length( + handle: *const c_void, + out: *mut i64, + buffer: *mut Buffer, + ) -> u32 { + let result = (|| { + let headers = unsafe { managed::borrowed::<Mutex<Vec<http::Header>>>(handle) }? + .lock() + .map_err(|_| managed::invalid("HTTP headers lock failed"))?; + Ok(headers.len() as i64) + })(); + unsafe { managed::complete(out, buffer, result) } + } + /// # Safety + /// headers is borrowed; output/error are distinct writable slots. Returned fields are owned. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_http_headers_get( + handle: *const c_void, + index: i64, + out: *mut HeaderFields, + buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { managed::checked_output(out, buffer) } { + return status; + } + let result = (|| { + let headers = unsafe { managed::borrowed::<Mutex<Vec<http::Header>>>(handle) }? + .lock() + .map_err(|_| managed::invalid("HTTP headers lock failed"))?; + let header = usize::try_from(index) + .ok() + .and_then(|index| headers.get(index)) + .ok_or_else(|| managed::invalid("invalid HTTP header index"))?; + Ok(HeaderFields { + name: managed::owned(header.name.clone()), + value: managed::owned(header.value.clone()), + }) + })(); + unsafe { managed::complete(out, buffer, result) } + } + + /// # Safety + /// fields borrow canonical Text/header/Bytes handles; outputs are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_http_request_new( + fields: *const RequestFields, + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + let result = (|| { + let fields = unsafe { record(fields) }?; + Ok(http::Request { + method: unsafe { cloned(fields.method) }?, + target: unsafe { cloned(fields.target) }?, + headers: unsafe { headers(fields.headers) }?, + body: unsafe { cloned(fields.body) }?, + }) + })(); + unsafe { managed::complete_handle(out, buffer, result) } + } + /// # Safety + /// request is borrowed; outputs are distinct writable slots. Returned fields are owned. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_http_request_fields( + request: *const c_void, + out: *mut RequestFields, + buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { managed::checked_output(out, buffer) } { + return status; + } + let result = unsafe { managed::borrowed::<http::Request>(request) }.map(|value| { + RequestFields { + method: managed::owned(value.method.clone()), + target: managed::owned(value.target.clone()), + headers: managed::owned(Mutex::new(value.headers.clone())), + body: managed::owned(value.body.clone()), + } + }); + unsafe { managed::complete(out, buffer, result) } + } + /// # Safety + /// fields borrow canonical header/Bytes handles; outputs are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_http_response_new( + fields: *const ResponseFields, + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + let result = (|| { + let fields = unsafe { record(fields) }?; + Ok(http::Response { + status: fields.status, + headers: unsafe { headers(fields.headers) }?, + body: unsafe { cloned(fields.body) }?, + }) + })(); + unsafe { managed::complete_handle(out, buffer, result) } + } + /// # Safety + /// response is borrowed; outputs are distinct writable slots. Returned fields are owned. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_http_response_fields( + response: *const c_void, + out: *mut ResponseFields, + buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { managed::checked_output(out, buffer) } { + return status; + } + let result = + unsafe { managed::borrowed::<http::Response>(response) }.map(|value| { + ResponseFields { + status: value.status, + headers: managed::owned(Mutex::new(value.headers.clone())), + body: managed::owned(value.body.clone()), + } + }); + unsafe { managed::complete(out, buffer, result) } + } + + /// # Safety + /// output/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_tls_system( + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { managed::checked_output(out, buffer) } { + return status; + } + unsafe { managed::complete_handle(out, buffer, Ok(tls::system())) } + } + /// # Safety + /// ca borrows Bytes; output/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_tls_client( + ca: *const c_void, + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + let result = unsafe { managed::borrowed::<Bytes>(ca) } + .and_then(|ca| tls::client(ca).map_err(managed::runtime_error)); + unsafe { managed::complete_handle(out, buffer, result) } + } + /// # Safety + /// cert/key borrow Bytes; output/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_tls_server( + cert: *const c_void, + key: *const c_void, + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + let result = (|| { + tls::server(unsafe { managed::borrowed(cert) }?, unsafe { + managed::borrowed(key) + }?) + .map_err(managed::runtime_error) + })(); + unsafe { managed::complete_handle(out, buffer, result) } + } + + type DecodeChunk = unsafe extern "C" fn(*const c_void, *mut *mut c_void) -> u32; + + /// # Safety + /// fields borrow canonical Text/header/AsyncStream handles; decoder is static + /// and returns one owned Bytes or Text handle for each matching event. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_http_stream_request_new( + fields: *const RequestFields, + decode: Option<DecodeChunk>, + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { managed::checked_output(out, buffer) } { + return status; + } + let result = (|| { + let fields = unsafe { record(fields) }?; + let decode = + decode.ok_or_else(|| managed::invalid("missing upload event decoder"))?; + let stream = unsafe { managed::borrowed::<AsyncStreamHandle>(fields.body) }? + .0 + .clone(); + Ok(http::StreamRequest { + method: unsafe { cloned(fields.method) }?, + target: unsafe { cloned(fields.target) }?, + headers: unsafe { headers(fields.headers) }?, + body: stream.map(move |row| { + let mut handle = ptr::null_mut(); + let status = unsafe { decode(row.pointer(), &mut handle) }; + match status { + 0 => { + let value = unsafe { cloned::<Bytes>(handle) } + .map_err(|failure| failure.message); + unsafe { managed::release::<Bytes>(handle) }; + value + } + 1 => { + let message = unsafe { cloned::<String>(handle) } + .unwrap_or_else(|failure| failure.message); + unsafe { managed::release::<String>(handle) }; + Err(message) + } + _ => Err("invalid upload event decoder status".into()), + } + }), + }) + })(); + unsafe { managed::complete_handle(out, buffer, result) } + } + + /// # Safety + /// callbacks match the checked stream event type; returned fields own handles. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_http_stream_response_fields( + response: *const c_void, + event: *const managed::ReadEventDescriptor, + ty: *const OwnedType, + out: *mut ResponseFields, + buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { managed::checked_output(out, buffer) } { + return status; + } + let result = (|| { + let event = unsafe { StreamEvent::new(event, ty) }?; + let response = unsafe { managed::borrowed::<http::StreamResponse>(response) }?; + let body = + AsyncStreamHandle(response.body.clone().map(move |value| event.row(value))); + Ok(ResponseFields { + status: response.status, + headers: managed::owned(Mutex::new(response.headers.clone())), + body: managed::owned(body), + }) + })(); + unsafe { managed::complete(out, buffer, result) } + } + + /// # Safety + /// payload borrows Text for kind=0, Bytes for kinds=1..3. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_ws_message_new( + kind: u32, + payload: *const c_void, + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + let result = match kind { + 0 => unsafe { cloned(payload) }.map(websocket::Message::Text), + 1 => unsafe { cloned(payload) }.map(websocket::Message::Binary), + 2 => unsafe { cloned(payload) }.map(websocket::Message::Ping), + 3 => unsafe { cloned(payload) }.map(websocket::Message::Pong), + _ => Err(managed::invalid("invalid WebSocket message kind")), + }; + unsafe { managed::complete_handle(out, buffer, result) } + } + /// # Safety + /// message is borrowed; returned payload owns Text or Bytes according to kind. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_ws_message_fields( + message: *const c_void, + out: *mut MessageFields, + buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { managed::checked_output(out, buffer) } { + return status; + } + let result = + unsafe { managed::borrowed::<websocket::Message>(message) }.map(|message| { + match message { + websocket::Message::Text(value) => MessageFields { + kind: 0, + payload: managed::owned(value.clone()), + }, + websocket::Message::Binary(value) => MessageFields { + kind: 1, + payload: managed::owned(value.clone()), + }, + websocket::Message::Ping(value) => MessageFields { + kind: 2, + payload: managed::owned(value.clone()), + }, + websocket::Message::Pong(value) => MessageFields { + kind: 3, + payload: managed::owned(value.clone()), + }, + } + }); + unsafe { managed::complete(out, buffer, result) } + } + #[repr(C)] + pub(super) struct WsEvent { + element_type: *const managed::TypeDescriptor, + read: Option<unsafe extern "C" fn(u32, *const c_void, *mut c_void)>, + failed: Option<unsafe extern "C" fn(*const c_void, *mut c_void)>, + } + fn message_row( + message: websocket::Message, + element: managed::ValueOps, + read: unsafe extern "C" fn(u32, *const c_void, *mut c_void), + ) -> StreamRow { + fn initialize<T>( + kind: u32, + payload: T, + element: managed::ValueOps, + read: unsafe extern "C" fn(u32, *const c_void, *mut c_void), + ) -> StreamRow { + let payload = managed::owned(payload); + let row = Element::from_initializer(element, |out| unsafe { + read(kind, payload, out) + }); + unsafe { managed::release::<T>(payload) }; + StreamRow::Managed(row) + } + match message { + websocket::Message::Text(value) => initialize(0, value, element, read), + websocket::Message::Binary(value) => initialize(1, value, element, read), + websocket::Message::Ping(value) => initialize(2, value, element, read), + websocket::Message::Pong(value) => initialize(3, value, element, read), + } + } + /// # Safety + /// socket is borrowed; callbacks initialize the exact transferable event type. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_ws_messages( + socket: *const c_void, + event: *const WsEvent, + ty: *const OwnedType, + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { managed::checked_output(out, buffer) } { + return status; + } + let result = (|| { + let event = unsafe { record(event) }?; + let element = unsafe { managed::descriptor(event.element_type, false) }?; + unsafe { stream_element_type(ty, element.size, element.align) }?; + let read = event + .read + .ok_or_else(|| managed::invalid("missing WebSocket read callback"))?; + let failed = event + .failed + .ok_or_else(|| managed::invalid("missing WebSocket failed callback"))?; + let socket = unsafe { cloned::<websocket::WebSocket>(socket) }?; + Ok(AsyncStreamHandle(websocket::messages(socket).map( + move |value| match value { + Ok(message) => message_row(message, element, read), + Err(message) => StreamRow::Managed(managed::initialize_event( + message, element, failed, + )), + }, + ))) + })(); + unsafe { managed::complete_handle(out, buffer, result) } + } + + type PackRequest = unsafe extern "C" fn( + *const c_void, + *const c_void, + *const c_void, + *mut c_void, + *mut Buffer, + ) -> u32; + type ConvertHandle = + unsafe extern "C" fn(*const c_void, *mut *mut c_void, *mut Buffer) -> u32; + #[repr(C)] + pub(super) struct HttpHandler { + asynchronous: *const AsyncFunction, + synchronous: *const SyncFunction, + input_type: *const OwnedType, + context_type: *const managed::TypeDescriptor, + pack: Option<PackRequest>, + response: Option<ConvertHandle>, + fault_message: Option<ConvertHandle>, + } + struct Handler { + asynchronous: Option<&'static AsyncFunction>, + synchronous: Option<&'static SyncFunction>, + input_type: &'static OwnedType, + context: Option<Element>, + pack: PackRequest, + response: Option<ConvertHandle>, + fault_message: ConvertHandle, + } + // Checked process-static callbacks only access transferable input/context, + // and create a separate owned frame/result/fault for every request. + unsafe impl Send for Handler {} + unsafe impl Sync for Handler {} + + fn callback_error(status: u32, buffer: Buffer) -> Result<(), String> { + if status == 0 && buffer.ptr.is_null() { + return Ok(()); + } + let message = if buffer.ptr.is_null() { + "protocol callback failed without a diagnostic".to_owned() + } else { + let bytes = + unsafe { std::slice::from_raw_parts(buffer.ptr, buffer.len as usize) }; + let message = String::from_utf8_lossy(bytes).into_owned(); + unsafe { super::super::dever_rt_v1_buffer_free(buffer.ptr, buffer.len) }; + message + }; + Err(message) + } + fn callback_buffer() -> Buffer { + Buffer { + ptr: ptr::null_mut(), + len: 0, + } + } + + impl Handler { + unsafe fn new( + descriptor: *const HttpHandler, + context: *const c_void, + live: bool, + ) -> Result<Self, managed::AbiFailure> { + let descriptor = unsafe { record(descriptor) }?; + let asynchronous = if descriptor.asynchronous.is_null() { + None + } else { + Some(unsafe { record(descriptor.asynchronous) }?) + }; + let synchronous = if descriptor.synchronous.is_null() { + None + } else { + Some(unsafe { record(descriptor.synchronous) }?) + }; + let (send_safe, output_type, fault_type) = match (asynchronous, synchronous) { + (Some(function), None) + if function.create.is_some() + && function.resume.is_some() + && function.destroy.is_some() + && function.done.is_some() => + { + ( + function.send_safe, + function.output_type, + function.fault_type, + ) + } + (None, Some(function)) + if function.input_type == descriptor.input_type + && function.invoke.is_some() => + { + ( + function.send_safe, + function.output_type, + function.fault_type, + ) + } + _ => return Err(managed::invalid("invalid HTTP handler function")), + }; + for ty in [descriptor.input_type, output_type, fault_type] { + let (ty, _) = unsafe { owned_layout(ty) }.map_err(managed::invalid)?; + if send_safe == 0 || ty.transferable == 0 { + return Err(managed::invalid("HTTP handler is not transferable")); + } + } + if live && unsafe { &*output_type }.size != 0 { + return Err(managed::invalid("live HTTP handler must return Unit")); + } + if !live && descriptor.response.is_none() { + return Err(managed::invalid("missing HTTP response callback")); + } + let context = if descriptor.context_type.is_null() { + None + } else { + let ops = unsafe { managed::descriptor(descriptor.context_type, false) }?; + Some(unsafe { Element::from_source(ops, context) }?) + }; + Ok(Self { + asynchronous, + synchronous, + input_type: unsafe { &*descriptor.input_type }, + context, + pack: descriptor + .pack + .ok_or_else(|| managed::invalid("missing HTTP request callback"))?, + response: descriptor.response, + fault_message: descriptor + .fault_message + .ok_or_else(|| managed::invalid("missing HTTP fault callback"))?, + }) + } + + fn input( + &self, + request: http::Request, + reply: Option<http::HttpReply>, + ) -> Result<Owned, String> { + let mut input = unsafe { Owned::new(self.input_type) }?; + let request = managed::owned(request); + let reply = reply.map_or(ptr::null_mut(), managed::owned); + let context = self.context.as_ref().map_or(ptr::null(), Element::pointer); + let mut buffer = callback_buffer(); + let status = unsafe { + (self.pack)(request, reply, context, input.pointer(), &mut buffer) + }; + unsafe { + managed::release::<http::Request>(request); + managed::release::<http::HttpReply>(reply); + } + callback_error(status, buffer)?; + input.initialized = true; + Ok(input) + } + + fn terminal_fault(&self, fault: ForeignFault) -> String { + match fault { + ForeignFault::Runtime(message) => message, + ForeignFault::Typed(fault) => { + let mut handle = ptr::null_mut(); + let mut buffer = callback_buffer(); + let status = unsafe { + (self.fault_message)(fault.pointer(), &mut handle, &mut buffer) + }; + if let Err(message) = callback_error(status, buffer) { + return message; + } + let message = unsafe { cloned::<String>(handle) } + .unwrap_or_else(|failure| failure.message); + unsafe { managed::release::<String>(handle) }; + message + } + } + } + + async fn invoke( + &self, + request: http::Request, + reply: Option<http::HttpReply>, + ) -> Result<Owned, String> { + let input = self.input(request, reply)?; + let result = if let Some(function) = self.asynchronous { + let future = unsafe { ForeignFuture::new(function, input.pointer()) } + .map(SendForeignFuture)?; + drop(input); + future.await + } else { + let function = self.synchronous.expect("validated handler"); + let prepared = PreparedSync { + input, + output: unsafe { Owned::new(function.output_type) }?, + fault: unsafe { Owned::new(function.fault_type) }?, + invoke: function.invoke.expect("validated handler"), + unit_output: false, + }; + prepared.execute() + }; + result.map_err(|fault| self.terminal_fault(fault)) + } + + fn response(&self, output: Owned) -> Result<http::Response, String> { + let mut handle = ptr::null_mut(); + let mut buffer = callback_buffer(); + let status = unsafe { + (self.response.expect("buffered handler"))( + output.pointer(), + &mut handle, + &mut buffer, + ) + }; + callback_error(status, buffer)?; + let response = unsafe { cloned::<http::Response>(handle) } + .map_err(|failure| failure.message); + unsafe { managed::release::<http::Response>(handle) }; + response + } + } + + /// # Safety + /// All inputs borrow canonical owners and static checked handler descriptors. + /// The constructor clones context before return; pending cancel drops all owners. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_http_serve( + listener: *const c_void, + limits: *const HttpLimits, + live: *const LiveLimits, + tls: *const c_void, + descriptor: *const HttpHandler, + context: *const c_void, + buffer: *mut Buffer, + ) -> *mut AsyncOp { + if !unsafe { empty(buffer) } { + return ptr::null_mut(); + } + let prepared = (|| { + let listener = unsafe { cloned::<net::Listener>(listener) }?; + let limits = unsafe { record(limits)?.runtime() }?; + let live = if live.is_null() { + None + } else { + let live = unsafe { record(live) }?; + Some(http::LiveLimits { + chunk_bytes: live.chunk_bytes, + idle_ms: live.idle_ms, + heartbeat_ms: live.heartbeat_ms, + }) + }; + let tls = if tls.is_null() { + None + } else { + Some(unsafe { cloned::<tls::ServerTls>(tls) }?) + }; + let handler = + Arc::new(unsafe { Handler::new(descriptor, context, live.is_some()) }?); + Ok::<_, managed::AbiFailure>((listener, limits, live, tls, handler)) + })(); + let (listener, limits, live, tls, handler) = match prepared { + Ok(value) => value, + Err(failure) => { + unsafe { error(buffer, failure.status, failure.message) }; + return ptr::null_mut(); + } + }; + operation_with_output( + async move { + let result = if let Some(live) = live { + let route = move |request, reply| { + let handler = handler.clone(); + async move { handler.invoke(request, Some(reply)).await.map(drop) } + }; + match tls { + Some(tls) => { + http::serve_live_tls(route, listener, limits, live, tls).await + } + None => http::serve_live(route, listener, limits, live).await, + } + } else { + let route = move |request| { + let handler = handler.clone(); + async move { + let output = handler.invoke(request, None).await?; + handler.response(output) + } + }; + match tls { + Some(tls) => http::serve_tls(route, listener, limits, tls).await, + None => http::serve(route, listener, limits).await, + } + }; + result.map(|()| OpValue::Unit).map_err(ForeignFault::from) + }, + Some(OutputSlots::Unit), + ) + } + + // Constructors copy all borrowed inputs before returning, including + // records nested beneath config pointers. Futures only capture owners. + macro_rules! protocol_op { + ($slots:ident, $name:ident($($arg:ident: $ty:ty),*) $prepare:block => |$input:pat_param| $future:expr) => { + /// # Safety + /// Inputs borrow the canonical types in runtime.h through this call only. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn $name($($arg:$ty),*) -> *mut AsyncOp { + let prepare = || $prepare; + let prepared: Result<_, managed::AbiFailure> = prepare(); + let Ok($input) = prepared else { return ptr::null_mut(); }; + operation_with_output($future, Some(OutputSlots::$slots)) + } + }; + } + protocol_op!(Handle, dever_rt_v1_async_http_send(host: *const c_void, port: i64, request: *const c_void, limits: *const HttpLimits) { + Ok((unsafe { cloned::<String>(host) }?, unsafe { cloned::<http::Request>(request) }?, unsafe { record(limits)?.runtime() }?)) + } => |(host, request, limits)| async move { http::send(&host, port, request, limits).await.map(|value| OpValue::Protocol(Value::Response(value))).map_err(ForeignFault::from) }); + protocol_op!(Handle, dever_rt_v1_async_http_client(origin: *const c_void, tls: *const c_void, limits: *const HttpLimits, pool: *const PoolLimits) { + let pool = unsafe { record(pool) }?; + Ok((unsafe { cloned::<String>(origin) }?, unsafe { cloned::<tls::ClientTls>(tls) }?, unsafe { record(limits)?.runtime() }?, http::PoolLimits { idle_ms: pool.idle_ms, chunk_bytes: pool.chunk_bytes, read_ms: pool.read_ms })) + } => |(origin, tls, limits, pool)| async move { http::client(&origin, tls, limits, pool).await.map(|value| OpValue::Protocol(Value::Client(value))).map_err(ForeignFault::from) }); + protocol_op!(Handle, dever_rt_v1_async_http_request(client: *const c_void, request: *const c_void) { + Ok((unsafe { cloned::<http::HttpClient>(client) }?, unsafe { cloned::<http::Request>(request) }?)) + } => |(client, request)| async move { http::request(&client, request).await.map(|value| OpValue::Protocol(Value::Response(value))).map_err(ForeignFault::from) }); + protocol_op!(Handle, dever_rt_v1_async_http_open(client: *const c_void, request: *const c_void) { + Ok((unsafe { cloned::<http::HttpClient>(client) }?, unsafe { cloned::<http::Request>(request) }?)) + } => |(client, request)| async move { http::open(&client, request).await.map(|value| OpValue::Protocol(Value::StreamResponse(value))).map_err(ForeignFault::from) }); + protocol_op!(Handle, dever_rt_v1_async_http_open_stream(client: *const c_void, request: *const c_void) { + Ok((unsafe { cloned::<http::HttpClient>(client) }?, unsafe { cloned::<http::StreamRequest>(request) }?)) + } => |(client, request)| async move { http::open_stream(&client, request).await.map(|value| OpValue::Protocol(Value::StreamResponse(value))).map_err(ForeignFault::from) }); + protocol_op!(Bool, dever_rt_v1_async_http_close_client(client: *const c_void) { unsafe { cloned::<http::HttpClient>(client) } } + => |client| async move { http::close_client(&client).await.map(|()| OpValue::Bool(true)).map_err(ForeignFault::from) }); + protocol_op!(Bool, dever_rt_v1_async_http_respond(reply: *const c_void, response: *const c_void) { + Ok((unsafe { cloned::<http::HttpReply>(reply) }?, unsafe { cloned::<http::Response>(response) }?)) + } => |(reply, response)| async move { http::respond(&reply, response).await.map(|()| OpValue::Bool(true)).map_err(ForeignFault::from) }); + protocol_op!(Bool, dever_rt_v1_async_http_start(reply: *const c_void, status: i64, header_values: *const c_void) { + Ok((unsafe { cloned::<http::HttpReply>(reply) }?, unsafe { headers(header_values) }?)) + } => |(reply, headers)| async move { http::start(&reply, status, headers).await.map(|()| OpValue::Bool(true)).map_err(ForeignFault::from) }); + protocol_op!(Bool, dever_rt_v1_async_http_write(reply: *const c_void, bytes: *const c_void) { + Ok((unsafe { cloned::<http::HttpReply>(reply) }?, unsafe { cloned::<Bytes>(bytes) }?)) + } => |(reply, bytes)| async move { http::write(&reply, bytes).await.map(|()| OpValue::Bool(true)).map_err(ForeignFault::from) }); + protocol_op!(Bool, dever_rt_v1_async_http_finish(reply: *const c_void) { unsafe { cloned::<http::HttpReply>(reply) } } + => |reply| async move { http::finish(&reply).await.map(|()| OpValue::Bool(true)).map_err(ForeignFault::from) }); + protocol_op!(Bool, dever_rt_v1_async_sse_start(reply: *const c_void, header_values: *const c_void) { + Ok((unsafe { cloned::<http::HttpReply>(reply) }?, unsafe { headers(header_values) }?)) + } => |(reply, headers)| async move { sse::start(&reply, headers).await.map(|()| OpValue::Bool(true)).map_err(ForeignFault::from) }); + protocol_op!(Bool, dever_rt_v1_async_sse_send(reply: *const c_void, event: *const SseEvent) { + let event = unsafe { record(event) }?; + Ok((unsafe { cloned::<http::HttpReply>(reply) }?, sse::Event { event: unsafe { cloned(event.event) }?, data: unsafe { cloned(event.data) }?, id: if event.id.is_null() { None } else { Some(unsafe { cloned(event.id) }?) }, retry_ms: (event.has_retry != 0).then_some(event.retry_ms) })) + } => |(reply, event)| async move { sse::send(&reply, event).await.map(|()| OpValue::Bool(true)).map_err(ForeignFault::from) }); + protocol_op!(Handle, dever_rt_v1_async_ws_accept(reply: *const c_void, limits: *const WsLimits) { + Ok((unsafe { cloned::<http::HttpReply>(reply) }?, unsafe { record(limits) }?.runtime())) + } => |(reply, limits)| async move { websocket::accept(&reply, limits).await.map(|value| OpValue::Protocol(Value::Socket(value))).map_err(ForeignFault::from) }); + protocol_op!(Handle, dever_rt_v1_async_ws_connect(host: *const c_void, port: i64, target: *const c_void, limits: *const WsLimits) { + Ok((unsafe { cloned::<String>(host) }?, unsafe { cloned::<String>(target) }?, unsafe { record(limits) }?.runtime())) + } => |(host, target, limits)| async move { websocket::connect(&host, port, &target, limits).await.map(|value| OpValue::Protocol(Value::Socket(value))).map_err(ForeignFault::from) }); + protocol_op!(Handle, dever_rt_v1_async_ws_open(url: *const c_void, tls: *const c_void, limits: *const WsLimits) { + Ok((unsafe { cloned::<String>(url) }?, unsafe { cloned::<tls::ClientTls>(tls) }?, unsafe { record(limits) }?.runtime())) + } => |(url, tls, limits)| async move { websocket::open(&url, tls, limits).await.map(|value| OpValue::Protocol(Value::Socket(value))).map_err(ForeignFault::from) }); + protocol_op!(Bool, dever_rt_v1_async_ws_send(socket: *const c_void, message: *const c_void) { + Ok((unsafe { cloned::<websocket::WebSocket>(socket) }?, unsafe { cloned::<websocket::Message>(message) }?)) + } => |(socket, message)| async move { websocket::send(&socket, message).await.map(|()| OpValue::Bool(true)).map_err(ForeignFault::from) }); + protocol_op!(OptionalHandle, dever_rt_v1_async_ws_receive(socket: *const c_void) { unsafe { cloned::<websocket::WebSocket>(socket) } } + => |socket| async move { websocket::receive(&socket).await.map(|value| OpValue::Protocol(Value::Message(value))).map_err(ForeignFault::from) }); + protocol_op!(Bool, dever_rt_v1_async_ws_close(socket: *const c_void, code: i64, reason: *const c_void) { + Ok((unsafe { cloned::<websocket::WebSocket>(socket) }?, unsafe { cloned::<String>(reason) }?)) + } => |(socket, reason)| async move { websocket::close(&socket, code, &reason).await.map(|()| OpValue::Bool(true)).map_err(ForeignFault::from) }); + } + + #[unsafe(no_mangle)] + pub extern "C" fn dever_rt_v1_async_frame_alloc(size: u64, align: u64) -> *mut c_void { + let size = usize::try_from(size).unwrap_or_else(|_| std::process::abort()); + let align = usize::try_from(align).unwrap_or_else(|_| std::process::abort()); + let layout = Layout::from_size_align(size.max(1), align) + .unwrap_or_else(|_| std::process::abort()); + // SAFETY: layout is valid and the matching free receives it. + NonNull::new(unsafe { alloc(layout) }) + .unwrap_or_else(|| handle_alloc_error(layout)) + .as_ptr() + .cast() + } + + /// # Safety + /// ptr, size and align are one exact frame_alloc allocation, consumed + /// once. A null ptr is a no-op for allocation-elided coroutines. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_async_frame_free( + ptr: *mut c_void, + size: u64, + align: u64, + ) { + if ptr.is_null() { + return; + } + let Ok(size) = usize::try_from(size) else { + return; + }; + let Ok(align) = usize::try_from(align) else { + return; + }; + let Ok(layout) = Layout::from_size_align(size.max(1), align) else { + return; + }; + // SAFETY: the callback promises the exact allocation identity. + unsafe { dealloc(ptr.cast(), layout) }; + } + } + + #[repr(C)] + pub(super) struct Buffer { + ptr: *mut u8, + len: u64, + } + + unsafe fn empty(buffer: *mut Buffer) -> bool { + if buffer.is_null() || !buffer.is_aligned() { + return false; + } + // SAFETY: the exported caller promises a live, writable, aligned buffer. + // Every call initializes it before returning, including error paths. + unsafe { + buffer.write(Buffer { + ptr: ptr::null_mut(), + len: 0, + }); + } + true + } + + unsafe fn owned_bytes(buffer: *mut Buffer, bytes: Vec<u8>) { + if bytes.is_empty() { + return; + } + let boxed = bytes.into_boxed_slice(); + let len = boxed.len() as u64; + let ptr = Box::into_raw(boxed) as *mut u8; + // SAFETY: empty() validated and initialized this caller-owned out slot. + // The allocation itself remains owned by buffer_free until called once. + unsafe { buffer.write(Buffer { ptr, len }) }; + } + + unsafe fn error(buffer: *mut Buffer, status: u32, message: impl Into<String>) -> u32 { + // SAFETY: this slot was validated by empty() before the call. + unsafe { owned_bytes(buffer, message.into().into_bytes()) }; + status + } + + unsafe fn value<T: Copy>( + out: *mut T, + buffer: *mut Buffer, + result: Result<T, &'static str>, + ) -> u32 { + // SAFETY: the exported caller promises a live Buffer slot. + if !unsafe { empty(buffer) } { + return ABI_INVALID_INPUT; + } + if out.is_null() || !out.is_aligned() { + // SAFETY: empty() validated and initialized the Buffer slot. + return unsafe { error(buffer, ABI_INVALID_INPUT, "invalid ABI output pointer") }; + } + match result { + Ok(value) => { + // SAFETY: the caller promises one live, writable, aligned T slot, + // disjoint from the Buffer slot; it is initialized only on success. + unsafe { out.write(value) }; + ABI_OK + } + // SAFETY: empty() validated and initialized the Buffer slot. + Err(message) => unsafe { error(buffer, ABI_RUNTIME_ERROR, message) }, + } + } + + unsafe fn text(buffer: *mut Buffer, result: Result<Vec<u8>, String>) -> u32 { + // SAFETY: the exported caller promises a live Buffer slot. + if !unsafe { empty(buffer) } { + return ABI_INVALID_INPUT; + } + match result { + Ok(bytes) => { + // SAFETY: empty() validated and initialized the Buffer slot. + unsafe { owned_bytes(buffer, bytes) }; + ABI_OK + } + // SAFETY: empty() validated and initialized the Buffer slot. + Err(message) => unsafe { error(buffer, ABI_RUNTIME_ERROR, message) }, + } + } + + /// # Safety + /// `ptr` must address `len` live bytes for this call when `len > 0`. + unsafe fn input<'a>(ptr: *const u8, len: u64) -> Result<&'a [u8], &'static str> { + let len = usize::try_from(len).map_err(|_| "invalid ABI input length")?; + if len > isize::MAX as usize { + return Err("invalid ABI input length"); + } + if len == 0 { + return Ok(&[]); + } + if ptr.is_null() { + return Err("invalid ABI input pointer"); + } + // SAFETY: the caller owns a live allocation of at least len bytes + // and does not mutate it until the returned slice is no longer used. + Ok(unsafe { std::slice::from_raw_parts(ptr, len) }) + } + + #[unsafe(no_mangle)] + pub extern "C" fn dever_rt_v1_version() -> u32 { + crate::RUNTIME_ABI_VERSION + } + + /// # Safety + /// out and buffer are live, writable, aligned, and disjoint. A returned + /// error buffer is released through dever_rt_v1_buffer_free. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_time_sleep( + milliseconds: i64, + out: *mut u8, + buffer: *mut Buffer, + ) -> u32 { + if !unsafe { empty(buffer) } { + return ABI_INVALID_INPUT; + } + if out.is_null() { + return unsafe { error(buffer, ABI_INVALID_INPUT, "invalid ABI output pointer") }; + } + match time::sleep(milliseconds) { + Ok(()) => { + unsafe { out.write(0) }; + ABI_OK + } + Err(message) => unsafe { error(buffer, ABI_RUNTIME_ERROR, message) }, + } + } + + macro_rules! int_binary { + ($name:ident, $operation:ident) => { + #[doc = "# Safety\n`out` and `buffer` must be writable, aligned, disjoint live slots. The caller must free any returned buffer exactly once with dever_rt_v1_buffer_free."] + #[unsafe(no_mangle)] + pub unsafe extern "C" fn $name( + left: i64, + right: i64, + out: *mut i64, + buffer: *mut Buffer, + ) -> u32 { + // SAFETY: this export's pointer contract covers both out slots. + unsafe { value(out, buffer, number::$operation(left, right)) } + } + }; + } + + int_binary!(dever_rt_v1_int_add, int_add); + int_binary!(dever_rt_v1_int_sub, int_sub); + int_binary!(dever_rt_v1_int_mul, int_mul); + int_binary!(dever_rt_v1_int_div, int_div); + int_binary!(dever_rt_v1_int_rem, int_rem); + + /// # Safety + /// `out` and `buffer` must be writable, aligned, disjoint live slots. + /// Free any returned buffer exactly once with dever_rt_v1_buffer_free. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_int_neg( + input: i64, + out: *mut i64, + buffer: *mut Buffer, + ) -> u32 { + // SAFETY: this export's pointer contract covers both out slots. + unsafe { value(out, buffer, number::int_neg(input)) } + } + + macro_rules! float_binary { + ($name:ident, $operation:tt) => { + #[doc = "# Safety\n`out` and `buffer` must be writable, aligned, disjoint live slots. The caller must free any returned buffer exactly once with dever_rt_v1_buffer_free."] + #[unsafe(no_mangle)] + pub unsafe extern "C" fn $name( + left: f64, + right: f64, + out: *mut f64, + buffer: *mut Buffer, + ) -> u32 { + // SAFETY: this export's pointer contract covers both out slots. + unsafe { value(out, buffer, Ok(left $operation right)) } + } + }; + } + + float_binary!(dever_rt_v1_float_add, +); + float_binary!(dever_rt_v1_float_sub, -); + float_binary!(dever_rt_v1_float_mul, *); + float_binary!(dever_rt_v1_float_div, /); + float_binary!(dever_rt_v1_float_rem, %); + + /// # Safety + /// `out` and `buffer` must be writable, aligned, disjoint live slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_float_neg( + input: f64, + out: *mut f64, + buffer: *mut Buffer, + ) -> u32 { + // SAFETY: this export's pointer contract covers both out slots. + unsafe { value(out, buffer, Ok(-input)) } + } + + fn decimal_binary( + left_low: u64, + left_high: u64, + right_low: u64, + right_high: u64, + operation: fn( + number::DecimalValue, + number::DecimalValue, + ) -> Result<number::DecimalValue, &'static str>, + ) -> Result<AbiDecimal, &'static str> { + let left = AbiDecimal { + low: left_low, + high: left_high, + } + .decode()?; + let right = AbiDecimal { + low: right_low, + high: right_high, + } + .decode()?; + operation(left, right).map(AbiDecimal::encode) + } + + macro_rules! decimal_binary { + ($name:ident, $operation:ident) => { + #[doc = "# Safety\n`out` and `buffer` must be writable, aligned, disjoint live slots. The caller must free any returned buffer exactly once with dever_rt_v1_buffer_free."] + #[unsafe(no_mangle)] + pub unsafe extern "C" fn $name( + left_low: u64, + left_high: u64, + right_low: u64, + right_high: u64, + out: *mut AbiDecimal, + buffer: *mut Buffer, + ) -> u32 { + // SAFETY: this export's pointer contract covers both out slots. + unsafe { value(out, buffer, decimal_binary( + left_low, left_high, right_low, right_high, + number::DecimalValue::$operation, + )) } + } + }; + } + + decimal_binary!(dever_rt_v1_decimal_add, checked_add); + decimal_binary!(dever_rt_v1_decimal_sub, checked_sub); + decimal_binary!(dever_rt_v1_decimal_mul, checked_mul); + decimal_binary!(dever_rt_v1_decimal_div, checked_div); + + /// # Safety + /// `out` and `buffer` must be writable, aligned, disjoint live slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_decimal_from_int( + input: i64, + out: *mut AbiDecimal, + buffer: *mut Buffer, + ) -> u32 { + // SAFETY: this export's pointer contract covers both out slots. + unsafe { + value( + out, + buffer, + Ok(AbiDecimal::encode(number::DecimalValue::from_int(input))), + ) + } + } + + /// # Safety + /// `out` and `buffer` must be writable, aligned, disjoint live slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_decimal_neg( + low: u64, + high: u64, + out: *mut AbiDecimal, + buffer: *mut Buffer, + ) -> u32 { + let result = AbiDecimal { low, high } + .decode() + .and_then(number::DecimalValue::checked_neg) + .map(AbiDecimal::encode); + // SAFETY: this export's pointer contract covers both out slots. + unsafe { value(out, buffer, result) } + } + + /// # Safety + /// `out` and `buffer` must be writable, aligned, disjoint live slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_decimal_round( + low: u64, + high: u64, + places: i64, + out: *mut AbiDecimal, + buffer: *mut Buffer, + ) -> u32 { + let result = AbiDecimal { low, high } + .decode() + .and_then(|value| value.round(places)) + .map(AbiDecimal::encode); + // SAFETY: this export's pointer contract covers both out slots. + unsafe { value(out, buffer, result) } + } + + /// # Safety + /// `buffer` must be a writable, aligned live slot. Free returned bytes once. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_int_to_text(input: i64, buffer: *mut Buffer) -> u32 { + // SAFETY: this export's pointer contract covers the Buffer slot. + unsafe { text(buffer, Ok(abi::int_to_text(input).into_bytes())) } + } + + /// # Safety + /// `buffer` must be a writable, aligned live slot. Free returned bytes once. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_float_to_text(input: f64, buffer: *mut Buffer) -> u32 { + // SAFETY: this export's pointer contract covers the Buffer slot. + unsafe { text(buffer, Ok(abi::float_to_text(input).into_bytes())) } + } + + /// # Safety + /// `buffer` must be a writable, aligned live slot. Free returned bytes once. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_decimal_to_text( + low: u64, + high: u64, + buffer: *mut Buffer, + ) -> u32 { + // SAFETY: this export's pointer contract covers the Buffer slot. + unsafe { + text( + buffer, + abi::decimal_to_text(AbiDecimal { low, high }) + .map(String::into_bytes) + .map_err(str::to_owned), + ) + } + } + + unsafe fn copy_input( + input_ptr: *const u8, + input_len: u64, + buffer: *mut Buffer, + convert: impl FnOnce(&[u8]) -> Result<Vec<u8>, String>, + ) -> u32 { + // SAFETY: the exported caller promises a live Buffer slot. + if !unsafe { empty(buffer) } { + return ABI_INVALID_INPUT; + } + // SAFETY: exported callers promise the input remains live and unchanged + // until this synchronous call returns; input() checks null/length first. + let input = match unsafe { input(input_ptr, input_len) } { + Ok(input) => input, + // SAFETY: empty() validated and initialized the Buffer slot. + Err(message) => return unsafe { error(buffer, ABI_INVALID_INPUT, message) }, + }; + match convert(input) { + Ok(bytes) => { + // SAFETY: empty() validated and initialized the Buffer slot. + unsafe { owned_bytes(buffer, bytes) }; + ABI_OK + } + // SAFETY: empty() validated and initialized the Buffer slot. + Err(message) => unsafe { error(buffer, ABI_RUNTIME_ERROR, message) }, + } + } + + /// # Safety + /// `input_ptr` must address `input_len` live, immutable bytes when nonzero; + /// `buffer` must be writable, aligned, and disjoint from the input. + /// Free returned bytes exactly once. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_text_copy_utf8( + input_ptr: *const u8, + input_len: u64, + buffer: *mut Buffer, + ) -> u32 { + // SAFETY: this export's pointer contract covers input and output. + unsafe { copy_input(input_ptr, input_len, buffer, abi::text_from_utf8) } + } + + /// # Safety + /// `input_ptr` must address `input_len` live, immutable bytes when nonzero; + /// `buffer` must be writable, aligned, and disjoint from the input. + /// Free returned bytes exactly once. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_bytes_copy( + input_ptr: *const u8, + input_len: u64, + buffer: *mut Buffer, + ) -> u32 { + // SAFETY: this export's pointer contract covers input and output. + unsafe { copy_input(input_ptr, input_len, buffer, |bytes| Ok(bytes.to_vec())) } + } + + /// # Safety + /// `input_ptr` must address `input_len` live, immutable bytes when nonzero; + /// `buffer` must be writable, aligned, and disjoint from the input. + /// Free returned bytes exactly once. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_bytes_to_text( + input_ptr: *const u8, + input_len: u64, + buffer: *mut Buffer, + ) -> u32 { + // SAFETY: this export's pointer contract covers input and output. + unsafe { copy_input(input_ptr, input_len, buffer, abi::text_from_utf8) } + } + + /// # Safety + /// `ptr` and `len` must be one exact, still-owned pair returned by this + /// runtime ABI, passed at most once; null with zero length is a no-op. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_buffer_free(ptr: *mut u8, len: u64) { + if ptr.is_null() && len == 0 { + return; + } + let Ok(len) = usize::try_from(len) else { + return; + }; + if ptr.is_null() || len == 0 { + return; + } + // SAFETY: only owned_bytes creates nonempty buffers, using Box<[u8]> + // from the same Rust allocator; the caller transfers that exact pair once. + drop(unsafe { Box::from_raw(ptr::slice_from_raw_parts_mut(ptr, len)) }); + } + + mod managed { + use std::ffi::c_void; + use std::hash::{Hash, Hasher}; + use std::mem::MaybeUninit; + use std::ptr; + use std::sync::Arc; + + use dever_runtime::bytes::Bytes; + use dever_runtime::collections::{IntoValues, List, Map, MapEntry}; + use dever_runtime::{number, resource, text as text_runtime}; + + use super::*; + + #[unsafe(no_mangle)] + pub extern "C" fn dever_rt_v1_log_flush() { + dever_runtime::log::flush(); + } + + // System ABI delegates domain validation and sensitive storage to runtime. + macro_rules! system_call { + ($name:ident($($arg:ident: $ty:ty),*) -> $out:ty, $complete:ident, $body:block) => { + /// # Safety + /// Handles borrow the exact canonical types; output/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn $name($($arg:$ty,)* out: *mut $out, buffer: *mut Buffer) -> u32 { + if let Err(status) = unsafe { checked_output(out, buffer) } { return status; } + let execute = || $body; + let result: Result<_, AbiFailure> = execute(); + unsafe { $complete(out, buffer, result) } + } + }; + } + + #[cfg(feature = "runtime-api")] + mod api_context { + use super::*; + use dever_runtime::{api, auth}; + + fn auth_error(error: auth::Error) -> AbiFailure { + runtime_error(error.to_string()) + } + unsafe fn text<'a>(value: *const c_void) -> Result<&'a str, AbiFailure> { + unsafe { borrowed::<String>(value) }.map(String::as_str) + } + unsafe fn optional_text<'a>( + value: *const c_void, + ) -> Result<Option<&'a str>, AbiFailure> { + if value.is_null() { + Ok(None) + } else { + unsafe { text(value) }.map(Some) + } + } + #[repr(C)] + pub struct CookieOptions { + path: *const c_void, + domain: *const c_void, + same_site: i64, + secure: u8, + http_only: u8, + max_age: i64, + max_age_present: u8, + } + unsafe fn cookie_options( + raw: *const CookieOptions, + ) -> Result<api::CookieOptions, AbiFailure> { + let row = unsafe { borrowed::<CookieOptions>(raw.cast()) }?; + if row.secure > 1 || row.http_only > 1 || row.max_age_present > 1 { + return Err(invalid("invalid Cookie boolean")); + } + Ok(api::CookieOptions { + path: unsafe { text(row.path) }?.to_owned(), + domain: unsafe { optional_text(row.domain) }?.map(str::to_owned), + same_site: match row.same_site { + 0 => api::SameSite::Strict, + 1 => api::SameSite::Lax, + 2 => api::SameSite::None, + _ => return Err(invalid("invalid SameSite")), + }, + secure: row.secure == 1, + http_only: row.http_only == 1, + max_age: (row.max_age_present == 1).then_some(row.max_age), + }) + } + system_call!(dever_rt_v1_api_request_id() -> *mut c_void, complete_handle, { api::request_id().map_err(runtime_error) }); + system_call!(dever_rt_v1_api_method() -> *mut c_void, complete_handle, { api::method().map_err(runtime_error) }); + system_call!(dever_rt_v1_api_path() -> *mut c_void, complete_handle, { api::path().map_err(runtime_error) }); + system_call!(dever_rt_v1_auth_site_key() -> *mut c_void, complete_handle, { auth::site_key().map_err(auth_error) }); + system_call!(dever_rt_v1_auth_id() -> *mut c_void, complete_handle, { auth::id().map_err(auth_error) }); + system_call!(dever_rt_v1_auth_session() -> *mut c_void, complete_handle, { auth::session().map_err(auth_error) }); + system_call!(dever_rt_v1_auth_owns_user(user: i64) -> u8, complete, { auth::owns_user(user).map(u8::from).map_err(auth_error) }); + system_call!(dever_rt_v1_auth_issue(subject: *const c_void, session: *const c_void, tenant: *const c_void) -> *mut c_void, complete_handle, { auth::issue(unsafe { text(subject) }?, unsafe { text(session) }?, unsafe { optional_text(tenant) }?).map_err(runtime_error) }); + system_call!(dever_rt_v1_auth_issue_cookie(subject: *const c_void, session: *const c_void, tenant: *const c_void) -> u8, complete, { auth::issue_cookie(unsafe { text(subject) }?, unsafe { text(session) }?, unsafe { optional_text(tenant) }?).map(|()| 0).map_err(runtime_error) }); + system_call!(dever_rt_v1_auth_clear_cookie() -> u8, complete, { auth::clear_cookie().map(|()| 0).map_err(runtime_error) }); + system_call!(dever_rt_v1_api_response_header(name: *const c_void, value: *const c_void) -> u8, complete, { api::response_header(unsafe { text(name) }?, unsafe { text(value) }?).map(|()| 0).map_err(runtime_error) }); + system_call!(dever_rt_v1_api_response_cookie(name: *const c_void, value: *const c_void, options: *const CookieOptions) -> u8, complete, { api::response_cookie(unsafe { text(name) }?, unsafe { text(value) }?, unsafe { cookie_options(options) }?).map(|()| 0).map_err(runtime_error) }); + system_call!(dever_rt_v1_api_response_secret_cookie(name: *const c_void, value: *const c_void, options: *const CookieOptions) -> u8, complete, { api::response_secret_cookie(unsafe { text(name) }?, unsafe { borrowed::<dever_runtime::secret::Secret>(value) }?.clone(), unsafe { cookie_options(options) }?).map(|()| 0).map_err(runtime_error) }); + + macro_rules! optional { + ($name:ident($($arg:ident:$ty:ty),*) -> $out:ty, $finish:ident, $body:block) => { + /// # Safety + /// Handles borrow canonical owners; outputs are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn $name($($arg:$ty,)* out:*mut $out, present:*mut u8, buffer:*mut Buffer) -> u32 { + if let Err(status) = unsafe { checked_optional_output(out, present, buffer) } { return status; } + let execute = || $body; + unsafe { $finish(out, present, buffer, execute()) } + } + }; + } + optional!(dever_rt_v1_api_client_address() -> *mut c_void, complete_optional_handle, { api::client_address().map_err(runtime_error) }); + optional!(dever_rt_v1_api_header(name:*const c_void) -> *mut c_void, complete_optional_handle, { api::header(unsafe { text(name) }?).map_err(runtime_error) }); + optional!(dever_rt_v1_api_cookie(name:*const c_void) -> *mut c_void, complete_optional_handle, { api::cookie(unsafe { text(name) }?).map_err(runtime_error) }); + optional!(dever_rt_v1_api_secret_cookie(name:*const c_void) -> *mut c_void, complete_optional_handle, { api::secret_cookie(unsafe { text(name) }?).map_err(runtime_error) }); + optional!(dever_rt_v1_auth_user_id() -> i64, complete_optional, { auth::user_id().map_err(auth_error) }); + optional!(dever_rt_v1_auth_tenant_id() -> i64, complete_optional, { auth::tenant_id().map_err(auth_error) }); + + /// # Safety + /// cause is an initialized nullable Text owner slot; message is borrowed. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_api_cause_append( + cause: *mut *mut c_void, + message: *const c_void, + ) { + unsafe { append_fault_cause(cause, message) }; + } + } + + mod application { + use super::*; + use dever_runtime::api::Inputs; + use dever_runtime::config::Settings; + use dever_runtime::wire::{Encoder, Node}; + + #[repr(C)] + pub struct WireName { + ptr: *const u8, + len: u64, + } + + type Decode = unsafe extern "C" fn(*const c_void, *mut c_void, *mut Buffer) -> u32; + type CmdDecode = unsafe extern "C" fn(*mut c_void, *mut c_void, *mut Buffer) -> u32; + type Encode = unsafe extern "C" fn(*const c_void, *mut c_void, *mut Buffer) -> u32; + + // Node lifetimes are confined to wire_decode's synchronous callback. + // This conversion never constructs an owner or extends the parsed tree. + unsafe fn node<'a>(raw: *const c_void) -> Result<&'a Node<'a>, AbiFailure> { + unsafe { borrowed(raw) } + } + + unsafe fn node_text<'a>(raw: *const c_void) -> Result<&'a str, AbiFailure> { + unsafe { node(raw) }?.text().map_err(runtime_error) + } + + unsafe fn encoder<'a>(raw: *mut c_void) -> Result<&'a mut Encoder, AbiFailure> { + if raw.is_null() || !raw.cast::<Encoder>().is_aligned() { + return Err(invalid("invalid wire encoder")); + } + // SAFETY: generated codecs borrow this encoder exclusively and + // synchronously from wire_encode; no reference survives a call. + Ok(unsafe { &mut *raw.cast::<Encoder>() }) + } + + unsafe fn name<'a>(raw: *const WireName) -> Result<&'a str, AbiFailure> { + let name = unsafe { borrowed::<WireName>(raw.cast()) }?; + let bytes = unsafe { input(name.ptr, name.len) }.map_err(invalid)?; + std::str::from_utf8(bytes).map_err(|_| invalid("wire name is not UTF-8")) + } + + unsafe fn names<'a>( + raw: *const WireName, + count: u64, + ) -> Result<Vec<&'a str>, AbiFailure> { + let count = + usize::try_from(count).map_err(|_| invalid("invalid wire name count"))?; + if count > isize::MAX as usize / size_of::<WireName>() + || (count != 0 && (raw.is_null() || !raw.is_aligned())) + { + return Err(invalid("invalid wire names")); + } + (0..count) + .map(|index| unsafe { name(raw.add(index)) }) + .collect() + } + + unsafe fn concrete_slot( + ty: *const TypeDescriptor, + slot: *const c_void, + ) -> Result<ValueOps, AbiFailure> { + let ty = unsafe { descriptor(ty, false) }?; + if slot.is_null() || !(slot as usize).is_multiple_of(ty.align) { + return Err(invalid("invalid wire concrete value slot")); + } + Ok(ty) + } + + unsafe fn status(buffer: *mut Buffer, result: Result<(), AbiFailure>) -> u32 { + match result { + Ok(()) => ABI_OK, + Err(failure) => unsafe { error(buffer, failure.status, failure.message) }, + } + } + + unsafe fn callback_status(status: u32, buffer: *mut Buffer) -> u32 { + // SAFETY: the callback follows the canonical Buffer ownership + // contract even if it returns an unrecognized status code. + let reply = unsafe { &*buffer }; + if matches!(status, ABI_RUNTIME_ERROR | ABI_INVALID_INPUT) + || (status == ABI_OK && reply.ptr.is_null() && reply.len == 0) + { + return status; + } + unsafe { dever_rt_v1_buffer_free(reply.ptr, reply.len) }; + unsafe { + error( + buffer, + ABI_INVALID_INPUT, + "invalid wire codec callback result", + ) + } + } + + unsafe fn finish_decoding( + result: u32, + operations: ValueOps, + out: *mut c_void, + buffer: *mut Buffer, + finish: impl FnOnce() -> Result<(), AbiFailure>, + ) -> u32 { + let mut checked = unsafe { callback_status(result, buffer) }; + if result != ABI_OK { + return checked; + } + if checked == ABI_OK { + checked = unsafe { status(buffer, finish()) }; + } + if checked != ABI_OK { + // The callback completed this concrete owner. A rejected + // Buffer or trailing CMD field must release it exactly once. + unsafe { (operations.drop)(out) }; + } + checked + } + + /// # Safety + /// type and callback are process-static. out is an uninitialized + /// concrete slot; callback obeys the scoped Node and cleanup contract. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_wire_decode( + text: *const c_void, + ty: *const TypeDescriptor, + decode: Option<Decode>, + out: *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + if !unsafe { empty(buffer) } { + return ABI_INVALID_INPUT; + } + let execute = || -> Result<(u32, ValueOps), AbiFailure> { + let operations = unsafe { concrete_slot(ty, out) }?; + let decode = decode.ok_or_else(|| invalid("missing wire decoder"))?; + let text = unsafe { borrowed::<String>(text) }?; + let root = dever_runtime::wire::parse(text).map_err(runtime_error)?; + // SAFETY: root and all of its borrowed raw spans remain + // alive throughout this synchronous generated codec call. + Ok(( + unsafe { decode(ptr::from_ref(&root).cast(), out, buffer) }, + operations, + )) + }; + match execute() { + Ok((result, operations)) => unsafe { + finish_decoding(result, operations, out, buffer, || Ok(())) + }, + Err(failure) => unsafe { error(buffer, failure.status, failure.message) }, + } + } + + /// # Safety + /// value has exactly type's layout; encode only borrows it and the + /// scoped encoder. out and buffer are disjoint writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_wire_encode( + value: *const c_void, + ty: *const TypeDescriptor, + encode: Option<Encode>, + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + if let Err(result) = unsafe { checked_output(out, buffer) } { + return result; + } + let prepare = || -> Result<Encode, AbiFailure> { + unsafe { concrete_slot(ty, value) }?; + encode.ok_or_else(|| invalid("missing wire encoder callback")) + }; + let encode = match prepare() { + Ok(encode) => encode, + Err(failure) => { + return unsafe { error(buffer, failure.status, failure.message) }; + } + }; + let mut writer = Encoder::default(); + let result = unsafe { encode(value, ptr::from_mut(&mut writer).cast(), buffer) }; + let result = unsafe { callback_status(result, buffer) }; + if result != ABI_OK { + return result; + } + let encoded = writer + .finish() + .map(|encoded| encoded.as_str().to_owned()) + .map_err(runtime_error); + unsafe { complete_handle(out, buffer, encoded) } + } + + /// # Safety + /// type is process-static. The callback borrows Inputs exclusively + /// during this call and cleans partial outputs on callback failure. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_cmd_decode( + text: *const c_void, + ty: *const TypeDescriptor, + decode: Option<CmdDecode>, + out: *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + if !unsafe { empty(buffer) } { + return ABI_INVALID_INPUT; + } + let execute = || -> Result<(u32, ValueOps, Inputs), AbiFailure> { + let operations = unsafe { concrete_slot(ty, out) }?; + let decode = decode.ok_or_else(|| invalid("missing CMD decoder"))?; + let text = unsafe { borrowed::<String>(text) }?; + let mut inputs = Inputs::from_json_object(text) + .map_err(|failure| runtime_error(failure.to_string()))?; + let result = unsafe { decode(ptr::from_mut(&mut inputs).cast(), out, buffer) }; + Ok((result, operations, inputs)) + }; + match execute() { + Ok((result, operations, inputs)) => unsafe { + finish_decoding(result, operations, out, buffer, || { + inputs + .finish() + .map_err(|failure| runtime_error(failure.to_string())) + }) + }, + Err(failure) => unsafe { error(buffer, failure.status, failure.message) }, + } + } + + /// # Safety + /// inputs is an exclusive borrow from cmd_decode; name is borrowed + /// UTF-8 and outputs are distinct writable canonical slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_cmd_input_field( + inputs: *mut c_void, + field: *const WireName, + required: u8, + out: *mut *mut c_void, + present: *mut u8, + buffer: *mut Buffer, + ) -> u32 { + if let Err(result) = unsafe { checked_output(out, buffer) } { + return result; + } + if !slot_valid(present) || required > 1 { + return unsafe { + error( + buffer, + ABI_INVALID_INPUT, + "invalid CMD input output or requirement", + ) + }; + } + let execute = || -> Result<Option<String>, AbiFailure> { + if inputs.is_null() || !inputs.cast::<Inputs>().is_aligned() { + return Err(invalid("invalid CMD inputs")); + } + let field = unsafe { name(field) }?; + // SAFETY: cmd_decode owns this Inputs and grants the + // generated synchronous callback its only mutable borrow. + let inputs = unsafe { &mut *inputs.cast::<Inputs>() }; + if required != 0 { + inputs.raw_json(field).map(Some) + } else { + inputs.optional_raw_json(field) + } + .map_err(|failure| runtime_error(failure.to_string())) + }; + unsafe { complete_optional_handle(out, present, buffer, execute()) } + } + + system_call!(dever_rt_v1_wire_node_is_null(raw: *const c_void) -> u8, complete, { + Ok(u8::from(unsafe { node(raw) }?.is_null())) + }); + system_call!(dever_rt_v1_wire_node_bool(raw: *const c_void) -> u8, complete, { + unsafe { node(raw) }?.boolean().map(u8::from).map_err(runtime_error) + }); + system_call!(dever_rt_v1_wire_node_int(raw: *const c_void) -> i64, complete, { + unsafe { node(raw) }?.int().map_err(runtime_error) + }); + system_call!(dever_rt_v1_wire_node_float(raw: *const c_void) -> f64, complete, { + unsafe { node(raw) }?.float().map_err(runtime_error) + }); + system_call!(dever_rt_v1_wire_node_text(raw: *const c_void) -> *mut c_void, complete_handle, { + Ok(unsafe { node_text(raw) }?.to_owned()) + }); + system_call!(dever_rt_v1_wire_node_bytes(raw: *const c_void) -> *mut c_void, complete_handle, { + unsafe { node(raw) }?.bytes().map_err(runtime_error) + }); + system_call!(dever_rt_v1_wire_node_raw(raw: *const c_void) -> *mut c_void, complete_handle, { + Ok(unsafe { node(raw) }?.raw().to_owned()) + }); + system_call!(dever_rt_v1_wire_node_decimal(raw: *const c_void) -> AbiDecimal, complete, { + number::DecimalValue::parse(unsafe { node_text(raw) }?).map(AbiDecimal::encode).map_err(runtime_error) + }); + system_call!(dever_rt_v1_wire_node_uuid(raw: *const c_void) -> *mut c_void, complete_handle, { + dever_runtime::orm::Uuid::parse(unsafe { node_text(raw) }?).map_err(|failure| runtime_error(failure.to_string())) + }); + system_call!(dever_rt_v1_wire_node_secret(raw: *const c_void) -> *mut c_void, complete_handle, { + Ok(dever_runtime::secret::Secret::from_input(unsafe { node_text(raw) }?.as_bytes().to_vec())) + }); + system_call!(dever_rt_v1_wire_node_datetime(raw: *const c_void) -> i64, complete, { + time::parse_datetime(unsafe { node_text(raw) }?).map_err(runtime_error) + }); + system_call!(dever_rt_v1_wire_node_date(raw: *const c_void) -> i64, complete, { + time::parse_date(unsafe { node_text(raw) }?).map_err(runtime_error) + }); + system_call!(dever_rt_v1_wire_node_time(raw: *const c_void) -> i64, complete, { + time::parse_time(unsafe { node_text(raw) }?).map_err(runtime_error) + }); + system_call!(dever_rt_v1_wire_node_list_len(raw: *const c_void) -> i64, complete, { + Ok(unsafe { node(raw) }?.list().map_err(runtime_error)?.len() as i64) + }); + system_call!(dever_rt_v1_wire_node_list_at(raw: *const c_void, index: i64) -> *const c_void, complete, { + let values = unsafe { node(raw) }?.list().map_err(runtime_error)?; + let index = usize::try_from(index).map_err(|_| invalid("invalid wire List index"))?; + values.get(index).map(|value| ptr::from_ref(value).cast()).ok_or_else(|| invalid("invalid wire List index")) + }); + + macro_rules! wire_operation { + ($name:ident($($arg:ident: $ty:ty),*) $body:block) => { + /// # Safety + /// All wire pointers are scoped borrows from the current + /// codec callback; buffer is a writable canonical error slot. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn $name($($arg: $ty,)* buffer: *mut Buffer) -> u32 { + if !unsafe { empty(buffer) } { return ABI_INVALID_INPUT; } + let execute = || -> Result<(), AbiFailure> { $body }; + unsafe { status(buffer, execute()) } + } + }; + } + + wire_operation!(dever_rt_v1_wire_node_fields(raw: *const c_void, allowed: *const WireName, count: u64) { + let allowed = unsafe { names(allowed, count) }?; + unsafe { node(raw) }?.fields(&allowed).map(|_| ()).map_err(runtime_error) + }); + + wire_operation!(dever_rt_v1_job_decode_unit(payload: *const c_void) { + let payload = unsafe { borrowed::<String>(payload) }?; + let value = dever_runtime::wire::parse(payload).map_err(runtime_error)?; + if !value.is_null() { return Err(runtime_error("expected null Job payload")); } + Ok(()) + }); + + /// # Safety + /// Node/name are borrowed from this codec call. out, present and + /// buffer are distinct writable slots; out is initialized if present. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_wire_node_field( + raw: *const c_void, + field: *const WireName, + required: u8, + out: *mut *const c_void, + present: *mut u8, + buffer: *mut Buffer, + ) -> u32 { + if let Err(result) = unsafe { checked_output(out, buffer) } { + return result; + } + if !slot_valid(present) || required > 1 { + return unsafe { + error( + buffer, + ABI_INVALID_INPUT, + "invalid wire field output or requirement", + ) + }; + } + let execute = || -> Result<Option<*const c_void>, AbiFailure> { + let field = unsafe { name(field) }?; + let value = unsafe { node(raw) }? + .object() + .map_err(runtime_error)? + .get(field); + if value.is_none() && required != 0 { + return Err(runtime_error("missing required wire field")); + } + Ok(value.map(|value| ptr::from_ref(value).cast())) + }; + unsafe { complete_optional(out, present, buffer, execute()) } + } + + wire_operation!(dever_rt_v1_wire_encoder_null(raw: *mut c_void) { + unsafe { encoder(raw) }?.null().map_err(runtime_error) + }); + wire_operation!(dever_rt_v1_wire_encoder_bool(raw: *mut c_void, value: u8) { + if value > 1 { return Err(invalid("invalid wire Bool")); } + unsafe { encoder(raw) }?.boolean(value != 0).map_err(runtime_error) + }); + wire_operation!(dever_rt_v1_wire_encoder_int(raw: *mut c_void, value: i64) { + unsafe { encoder(raw) }?.int(value).map_err(runtime_error) + }); + wire_operation!(dever_rt_v1_wire_encoder_float(raw: *mut c_void, value: f64) { + unsafe { encoder(raw) }?.float(value).map_err(runtime_error) + }); + wire_operation!(dever_rt_v1_wire_encoder_text(raw: *mut c_void, text: *const c_void) { + let value = unsafe { borrowed::<String>(text) }?; + unsafe { encoder(raw) }?.text(value).map_err(runtime_error) + }); + wire_operation!(dever_rt_v1_wire_encoder_bytes(raw: *mut c_void, bytes: *const c_void) { + let value = unsafe { borrowed::<Bytes>(bytes) }?; + unsafe { encoder(raw) }?.bytes(value).map_err(runtime_error) + }); + wire_operation!(dever_rt_v1_wire_encoder_json(raw: *mut c_void, text: *const c_void) { + let value = unsafe { borrowed::<String>(text) }?; + unsafe { encoder(raw) }?.json(value).map_err(runtime_error) + }); + wire_operation!(dever_rt_v1_wire_encoder_decimal(raw: *mut c_void, low: u64, high: u64) { + let value = AbiDecimal { low, high }.decode().map_err(runtime_error)?; + unsafe { encoder(raw) }?.text(&value.to_string()).map_err(runtime_error) + }); + wire_operation!(dever_rt_v1_wire_encoder_uuid(raw: *mut c_void, uuid: *const c_void) { + let value = unsafe { borrowed::<dever_runtime::orm::Uuid>(uuid) }?; + unsafe { encoder(raw) }?.text(&value.to_string()).map_err(runtime_error) + }); + wire_operation!(dever_rt_v1_wire_encoder_datetime(raw: *mut c_void, value: i64) { + let text = time::format_datetime(value).map_err(runtime_error)?; + unsafe { encoder(raw) }?.text(&text).map_err(runtime_error) + }); + wire_operation!(dever_rt_v1_wire_encoder_date(raw: *mut c_void, value: i64) { + let text = time::format_date(value).map_err(runtime_error)?; + unsafe { encoder(raw) }?.text(&text).map_err(runtime_error) + }); + wire_operation!(dever_rt_v1_wire_encoder_time(raw: *mut c_void, value: i64) { + let text = time::format_time(value).map_err(runtime_error)?; + unsafe { encoder(raw) }?.text(&text).map_err(runtime_error) + }); + wire_operation!(dever_rt_v1_wire_encoder_begin_array(raw: *mut c_void) { + unsafe { encoder(raw) }?.begin_array().map_err(runtime_error) + }); + wire_operation!(dever_rt_v1_wire_encoder_begin_object(raw: *mut c_void) { + unsafe { encoder(raw) }?.begin_object().map_err(runtime_error) + }); + wire_operation!(dever_rt_v1_wire_encoder_key(raw: *mut c_void, field: *const WireName) { + let field = unsafe { name(field) }?; + unsafe { encoder(raw) }?.key(field).map_err(runtime_error) + }); + wire_operation!(dever_rt_v1_wire_encoder_end(raw: *mut c_void) { + unsafe { encoder(raw) }?.end().map_err(runtime_error) + }); + + system_call!(dever_rt_v1_adapter_settings_load() -> *mut c_void, complete_handle, { + dever_runtime::component::adapter_settings().map_err(runtime_error) + }); + wire_operation!(dever_rt_v1_adapter_settings_validate_names(raw: *const c_void, identities: *const WireName, count: u64) { + let identities = unsafe { names(identities, count) }?; + unsafe { borrowed::<Arc<Settings>>(raw) }?.validate_adapter_names(&identities).map_err(runtime_error) + }); + + /// # Safety + /// settings is a live owner, names are borrowed UTF-8, and every + /// output is distinct, writable and correctly aligned. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_adapter_settings_select( + raw: *const c_void, + identity: *const WireName, + candidates: *const WireName, + count: u64, + index: *mut i64, + setting: *mut *mut c_void, + present: *mut u8, + buffer: *mut Buffer, + ) -> u32 { + if let Err(result) = unsafe { checked_output(index, buffer) } { + return result; + } + if !slot_valid(setting) || !slot_valid(present) { + return unsafe { + error( + buffer, + ABI_INVALID_INPUT, + "invalid Adapter selection output", + ) + }; + } + let execute = || -> Result<(), AbiFailure> { + let identity = unsafe { name(identity) }?; + let candidates = unsafe { names(candidates, count) }?; + let (selected, value) = unsafe { borrowed::<Arc<Settings>>(raw) }? + .select_adapter(identity, &candidates) + .map_err(runtime_error)?; + unsafe { + index.write(selected as i64); + present.write(u8::from(value.is_some())); + if let Some(value) = value { + setting.write(owned(value)); + } + } + Ok(()) + }; + unsafe { status(buffer, execute()) } + } + + /// # Safety + /// settings is null or one live owner returned by settings_load. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_adapter_settings_release(settings: *const c_void) { + unsafe { release::<Arc<Settings>>(settings.cast_mut()) }; + } + } + + system_call!(dever_rt_v1_job_clock_new(test: u8) -> *mut c_void, complete_handle, { + match test { + 0 => Ok(time::Clock::default()), + 1 => Ok(time::Clock::for_test()), + _ => Err(invalid("invalid Job clock mode")), + } + }); + system_call!(dever_rt_v1_job_clock_now(clock: *const c_void) -> i64, complete, { + unsafe { borrowed::<time::Clock>(clock) }?.now().map_err(|error| runtime_error(error.to_string())) + }); + /// # Safety + /// Borrows one Clock. Buffer is a distinct writable diagnostic slot. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_job_clock_advance( + clock: *const c_void, + millis: i64, + buffer: *mut Buffer, + ) -> u32 { + if !unsafe { empty(buffer) } { + return ABI_INVALID_INPUT; + } + let result = unsafe { borrowed::<time::Clock>(clock) } + .and_then(|clock| clock.advance(millis).map_err(runtime_error)); + match result { + Ok(()) => 0, + Err(failure) => unsafe { error(buffer, failure.status, failure.message) }, + } + } + /// # Safety + /// Transfers one live Clock handle, or null. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_job_clock_release(clock: *mut c_void) { + unsafe { release::<time::Clock>(clock) }; + } + + system_call!(dever_rt_v1_time_unix_millis() -> i64, complete, { time::unix_millis().map_err(runtime_error) }); + system_call!(dever_rt_v1_time_now() -> i64, complete, { time::now().map_err(runtime_error) }); + system_call!(dever_rt_v1_time_monotonic_nanos() -> i64, complete, { time::monotonic_nanos().map_err(runtime_error) }); + system_call!(dever_rt_v1_time_parse_datetime(text: *const c_void) -> i64, complete, { time::parse_datetime(unsafe { borrowed::<String>(text) }?).map_err(runtime_error) }); + system_call!(dever_rt_v1_time_format_datetime(value: i64) -> *mut c_void, complete_handle, { time::format_datetime(value).map_err(runtime_error) }); + system_call!(dever_rt_v1_time_parse_date(text: *const c_void) -> i64, complete, { time::parse_date(unsafe { borrowed::<String>(text) }?).map_err(runtime_error) }); + system_call!(dever_rt_v1_time_format_date(value: i64) -> *mut c_void, complete_handle, { time::format_date(value).map_err(runtime_error) }); + system_call!(dever_rt_v1_time_parse_time(text: *const c_void) -> i64, complete, { time::parse_time(unsafe { borrowed::<String>(text) }?).map_err(runtime_error) }); + system_call!(dever_rt_v1_time_format_time(value: i64) -> *mut c_void, complete_handle, { time::format_time(value).map_err(runtime_error) }); + system_call!(dever_rt_v1_time_add(left: i64, right: i64) -> i64, complete, { time::add(left, right).map_err(runtime_error) }); + system_call!(dever_rt_v1_time_subtract(left: i64, right: i64) -> i64, complete, { time::subtract(left, right).map_err(runtime_error) }); + system_call!(dever_rt_v1_time_difference(left: i64, right: i64) -> i64, complete, { time::difference(left, right).map_err(runtime_error) }); + + system_call!(dever_rt_v1_crypto_token(length: i64) -> *mut c_void, complete_handle, { dever_runtime::crypto::token(length).map_err(runtime_error) }); + system_call!(dever_rt_v1_crypto_password_hash(password: *const c_void) -> *mut c_void, complete_handle, { dever_runtime::crypto::password_hash(unsafe { borrowed(password) }?).map_err(runtime_error) }); + system_call!(dever_rt_v1_crypto_password_verify(password: *const c_void, encoded: *const c_void) -> u8, complete, { dever_runtime::crypto::password_verify(unsafe { borrowed(password) }?, unsafe { borrowed::<String>(encoded) }?).map(u8::from).map_err(runtime_error) }); + system_call!(dever_rt_v1_crypto_sha256(bytes: *const c_void) -> *mut c_void, complete_handle, { Ok(dever_runtime::crypto::sha256(unsafe { borrowed(bytes) }?)) }); + system_call!(dever_rt_v1_crypto_hmac_sha256(secret: *const c_void, bytes: *const c_void) -> *mut c_void, complete_handle, { Ok(dever_runtime::crypto::hmac_sha256(unsafe { borrowed(secret) }?, unsafe { borrowed(bytes) }?)) }); + system_call!(dever_rt_v1_crypto_constant_time_eq(left: *const c_void, right: *const c_void) -> u8, complete, { Ok(u8::from(dever_runtime::crypto::constant_time_eq(unsafe { borrowed(left) }?, unsafe { borrowed(right) }?))) }); + system_call!(dever_rt_v1_secret_from_text(text: *const c_void) -> *mut c_void, complete_handle, { Ok(dever_runtime::secret::Secret::from_input(unsafe { borrowed::<String>(text) }?.as_bytes().to_vec())) }); + system_call!(dever_rt_v1_uuid_to_text(uuid: *const c_void) -> *mut c_void, complete_handle, { Ok(unsafe { borrowed::<dever_runtime::orm::Uuid>(uuid) }?.to_string()) }); + system_call!(dever_rt_v1_stdout_write(text: *const c_void) -> u8, complete, { resource::stdout_write(unsafe { borrowed::<String>(text) }?).map(|()| 0).map_err(runtime_error) }); + system_call!(dever_rt_v1_log_write(level: u32, message: *const c_void, fields: *const c_void) -> u8, complete, { + let level = match level { 0 => dever_runtime::log::Level::Debug, 1 => dever_runtime::log::Level::Info, 2 => dever_runtime::log::Level::Warn, 3 => dever_runtime::log::Level::Error, _ => return Err(invalid("invalid log level")) }; + let message = unsafe { borrowed::<String>(message) }?.clone(); + let fields = unsafe { borrowed::<MapHandle>(fields) }?; + if fields.key.size != size_of::<*const c_void>() || fields.value.size != size_of::<*const c_void>() { return Err(invalid("log fields require Text values")); } + let fields = fields.values.pairs().map(|(name, value)| { + let name = unsafe { name.0.pointer().cast::<*const c_void>().read() }; + let value = unsafe { value.pointer().cast::<*const c_void>().read() }; + Ok(dever_runtime::log::Field { name: unsafe { borrowed::<String>(name) }?.clone(), value: unsafe { borrowed::<String>(value) }?.clone() }) + }).collect::<Result<Vec<_>, AbiFailure>>()?; + dever_runtime::log::emit(level, message, fields); + Ok(0) + }); + system_call!(dever_rt_v1_process_arguments(ty: *const TypeDescriptor) -> *mut c_void, complete_handle, { + let element = unsafe { descriptor(ty, false) }?; + if element.size != size_of::<*const c_void>() || element.align != align_of::<*const c_void>() { return Err(invalid("process arguments require Text descriptor")); } + let arguments = dever_runtime::process::arguments().map_err(runtime_error)?; + let mut values = Vec::with_capacity(arguments.values().len()); + for argument in arguments.into_values() { + let text = owned(argument); + let value = unsafe { Element::from_source(element, std::ptr::from_ref(&text).cast()) }; + unsafe { release::<String>(text) }; + values.push(value?); + } + Ok(ListHandle { element, values: List::new(values) }) + }); + + /// # Safety + /// `text` borrows a live Text owner until return; it is not consumed. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_process_error(text: *const c_void) -> u32 { + let Ok(message) = (unsafe { borrowed::<String>(text) }) else { + return ABI_INVALID_INPUT; + }; + dever_runtime::log::error(message.clone(), Vec::new()); + ABI_OK + } + + /// # Safety + /// text is borrowed; output/present/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_uuid_parse( + text: *const c_void, + out: *mut *mut c_void, + present: *mut u8, + buffer: *mut Buffer, + ) -> u32 { + let result = unsafe { borrowed::<String>(text) } + .map(|text| dever_runtime::orm::Uuid::parse(text).ok()); + unsafe { complete_optional_handle(out, present, buffer, result) } + } + /// # Safety + /// Both handles borrow canonical UUID owners. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_uuid_equal( + left: *const c_void, + right: *const c_void, + ) -> u8 { + match ( + unsafe { borrowed::<dever_runtime::orm::Uuid>(left) }, + unsafe { borrowed::<dever_runtime::orm::Uuid>(right) }, + ) { + (Ok(left), Ok(right)) => u8::from(left == right), + _ => 0, + } + } + /// # Safety + /// handle borrows the matching opaque owner. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_secret_retain(handle: *const c_void) -> *mut c_void { + unsafe { retain::<dever_runtime::secret::Secret>(handle) } + } + /// # Safety + /// handle transfers one matching opaque owner, or null. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_secret_release(handle: *mut c_void) { + unsafe { release::<dever_runtime::secret::Secret>(handle) } + } + + /// # Safety + /// handle borrows the matching opaque owner. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_uuid_retain(handle: *const c_void) -> *mut c_void { + unsafe { retain::<dever_runtime::orm::Uuid>(handle) } + } + /// # Safety + /// handle transfers one matching opaque owner, or null. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_uuid_release(handle: *mut c_void) { + unsafe { release::<dever_runtime::orm::Uuid>(handle) } + } + + type CloneValue = unsafe extern "C" fn(*const c_void, *mut c_void); + type DropValue = unsafe extern "C" fn(*mut c_void); + type EqualValue = unsafe extern "C" fn(*const c_void, *const c_void) -> u8; + type HashValue = unsafe extern "C" fn(*const c_void) -> u64; + + #[repr(C)] + pub struct TypeDescriptor { + size: u64, + align: u64, + clone: Option<CloneValue>, + drop: Option<DropValue>, + equal: Option<EqualValue>, + hash: Option<HashValue>, + } + + #[repr(C)] + pub struct ReadEventDescriptor { + pub(super) element_type: *const TypeDescriptor, + pub(super) chunk: Option<unsafe extern "C" fn(*const c_void, *mut c_void)>, + pub(super) failed: Option<unsafe extern "C" fn(*const c_void, *mut c_void)>, + } + + #[derive(Clone, Copy)] + pub(super) struct ValueOps { + identity: usize, + pub(super) size: usize, + words: usize, + pub(super) align: usize, + clone: CloneValue, + drop: DropValue, + equal: EqualValue, + hash: Option<HashValue>, + } + + #[derive(Debug)] + pub(super) struct AbiFailure { + pub(super) status: u32, + pub(super) message: String, + } + + pub(super) fn invalid(message: impl Into<String>) -> AbiFailure { + AbiFailure { + status: ABI_INVALID_INPUT, + message: message.into(), + } + } + + pub(super) fn runtime_error(message: impl Into<String>) -> AbiFailure { + AbiFailure { + status: ABI_RUNTIME_ERROR, + message: message.into(), + } + } + + pub(super) unsafe fn descriptor( + raw: *const TypeDescriptor, + key: bool, + ) -> Result<ValueOps, AbiFailure> { + if raw.is_null() || !raw.is_aligned() { + return Err(invalid("invalid ABI type descriptor")); + } + // SAFETY: the private compiler supplies a live, aligned, process-static descriptor. + let value = unsafe { &*raw }; + let size = + usize::try_from(value.size).map_err(|_| invalid("invalid ABI element size"))?; + let align = usize::try_from(value.align) + .map_err(|_| invalid("invalid ABI element alignment"))?; + if size > isize::MAX as usize || !align.is_power_of_two() || align > align_of::<u64>() { + return Err(invalid("invalid ABI element layout")); + } + let words = size + .checked_add(7) + .ok_or_else(|| invalid("invalid ABI element size"))? + / 8; + let (Some(clone), Some(drop), Some(equal)) = (value.clone, value.drop, value.equal) + else { + return Err(invalid("incomplete ABI type descriptor")); + }; + if key && value.hash.is_none() { + return Err(invalid("Map key descriptor requires hash")); + } + Ok(ValueOps { + identity: raw as usize, + size, + words: words.max(1), + align, + clone, + drop, + equal, + hash: value.hash, + }) + } + + fn slot_valid<T>(slot: *mut T) -> bool { + !slot.is_null() && slot.is_aligned() + } + + pub(super) unsafe fn complete<T: Copy>( + out: *mut T, + buffer: *mut Buffer, + result: Result<T, AbiFailure>, + ) -> u32 { + // SAFETY: each export promises the Buffer slot is live and writable. + if !unsafe { empty(buffer) } { + return ABI_INVALID_INPUT; + } + if !slot_valid(out) { + // SAFETY: empty initialized this valid Buffer slot. + return unsafe { error(buffer, ABI_INVALID_INPUT, "invalid ABI output pointer") }; + } + match result { + Ok(value) => { + // SAFETY: the caller promises a disjoint, writable, aligned output slot. + unsafe { out.write(value) }; + ABI_OK + } + // SAFETY: empty initialized this valid Buffer slot. + Err(failure) => unsafe { error(buffer, failure.status, failure.message) }, + } + } + + unsafe fn complete_optional<T: Copy>( + out: *mut T, + present: *mut u8, + buffer: *mut Buffer, + result: Result<Option<T>, AbiFailure>, + ) -> u32 { + // SAFETY: the caller promises a live Buffer slot. + if !unsafe { empty(buffer) } { + return ABI_INVALID_INPUT; + } + if !slot_valid(out) || !slot_valid(present) { + // SAFETY: empty initialized this valid Buffer slot. + return unsafe { error(buffer, ABI_INVALID_INPUT, "invalid ABI output pointer") }; + } + match result { + Ok(value) => { + // SAFETY: out and present are distinct writable slots by the export contract. + unsafe { present.write(u8::from(value.is_some())) }; + if let Some(value) = value { + // SAFETY: a present value initializes the previously uninitialized slot. + unsafe { out.write(value) }; + } + ABI_OK + } + // SAFETY: empty initialized this valid Buffer slot. + Err(failure) => unsafe { error(buffer, failure.status, failure.message) }, + } + } + + pub(super) unsafe fn complete_handle<T>( + out: *mut *mut c_void, + buffer: *mut Buffer, + result: Result<T, AbiFailure>, + ) -> u32 { + // SAFETY: every managed export promises a live Buffer slot. + if !unsafe { empty(buffer) } { + return ABI_INVALID_INPUT; + } + if !slot_valid(out) { + // SAFETY: empty initialized this valid Buffer slot. + return unsafe { error(buffer, ABI_INVALID_INPUT, "invalid ABI output pointer") }; + } + match result { + Ok(value) => { + // SAFETY: out is a distinct, live pointer slot. + unsafe { out.write(owned(value)) }; + ABI_OK + } + // SAFETY: empty initialized this valid Buffer slot. + Err(failure) => unsafe { error(buffer, failure.status, failure.message) }, + } + } + + unsafe fn complete_optional_handle<T>( + out: *mut *mut c_void, + present: *mut u8, + buffer: *mut Buffer, + result: Result<Option<T>, AbiFailure>, + ) -> u32 { + // SAFETY: every managed export promises a live Buffer slot. + if !unsafe { empty(buffer) } { + return ABI_INVALID_INPUT; + } + if !slot_valid(out) || !slot_valid(present) { + // SAFETY: empty initialized this valid Buffer slot. + return unsafe { error(buffer, ABI_INVALID_INPUT, "invalid ABI output pointer") }; + } + match result { + Ok(value) => { + // SAFETY: the two output slots are distinct and writable. + unsafe { present.write(u8::from(value.is_some())) }; + if let Some(value) = value { + unsafe { out.write(owned(value)) }; + } + ABI_OK + } + // SAFETY: empty initialized this valid Buffer slot. + Err(failure) => unsafe { error(buffer, failure.status, failure.message) }, + } + } + + pub(super) fn owned<T>(value: T) -> *mut c_void { + Arc::into_raw(Arc::new(value)) as *mut c_void + } + + pub(super) unsafe fn append_fault_cause(cause: *mut *mut c_void, message: *const c_void) { + if cause.is_null() || !cause.is_aligned() { + return; + } + let Ok(message) = (unsafe { borrowed::<String>(message) }) else { + return; + }; + let previous = unsafe { *cause }; + let text = if previous.is_null() { + format!("caused by: {message}") + } else { + let Ok(previous) = (unsafe { borrowed::<String>(previous) }) else { + return; + }; + format!("{previous}; caused by: {message}") + }; + unsafe { + release::<String>(previous); + cause.write(owned(text)); + } + } + + pub(super) unsafe fn borrowed<'a, T>(handle: *const c_void) -> Result<&'a T, AbiFailure> { + if handle.is_null() || !(handle as *const T).is_aligned() { + return Err(invalid("invalid ABI handle")); + } + // SAFETY: the private caller guarantees a live handle of this exact owner type. + Ok(unsafe { &*(handle as *const T) }) + } + + pub(super) unsafe fn consumed<T>(handle: *mut c_void) -> Result<Arc<T>, AbiFailure> { + if handle.is_null() || !(handle as *const T).is_aligned() { + return Err(invalid("invalid ABI handle")); + } + // SAFETY: each _take export consumes exactly one Arc owner from into_raw. + Ok(unsafe { Arc::from_raw(handle as *const T) }) + } + + pub(super) unsafe fn retain<T>(handle: *const c_void) -> *mut c_void { + if handle.is_null() { + return ptr::null_mut(); + } + // SAFETY: the caller guarantees this is a live Arc<T> owner. Incrementing + // creates one independent owner represented by the returned same pointer. + unsafe { Arc::increment_strong_count(handle as *const T) }; + handle as *mut c_void + } + + pub(super) unsafe fn release<T>(handle: *mut c_void) { + if !handle.is_null() { + // SAFETY: the caller transfers one live Arc<T> owner exactly once. + drop(unsafe { Arc::from_raw(handle as *const T) }); + } + } + + pub(super) struct Element { + words: Box<[MaybeUninit<u64>]>, + ops: ValueOps, + } + + impl Element { + pub(super) fn from_initializer( + ops: ValueOps, + initialize: impl FnOnce(*mut c_void), + ) -> Self { + let mut words = vec![MaybeUninit::new(0u64); ops.words].into_boxed_slice(); + initialize(words.as_mut_ptr().cast()); + Self { words, ops } + } + + pub(super) unsafe fn from_source( + ops: ValueOps, + source: *const c_void, + ) -> Result<Self, AbiFailure> { + if source.is_null() || !(source as usize).is_multiple_of(ops.align) { + return Err(invalid("invalid ABI element pointer")); + } + let mut words = vec![MaybeUninit::new(0u64); ops.words].into_boxed_slice(); + // SAFETY: source is a live initialized concrete value and words have + // enough target-layout bytes at at least the required alignment. + unsafe { (ops.clone)(source, words.as_mut_ptr().cast()) }; + Ok(Self { words, ops }) + } + + pub(super) fn pointer(&self) -> *const c_void { + self.words.as_ptr().cast() + } + + pub(super) unsafe fn copy_to(&self, out: *mut c_void) -> Result<(), AbiFailure> { + if out.is_null() || !(out as usize).is_multiple_of(self.ops.align) { + return Err(invalid("invalid ABI element output")); + } + // SAFETY: the caller supplies an uninitialized, aligned destination + // for this exact type; callback creates its independent owner. + unsafe { (self.ops.clone)(self.pointer(), out) }; + Ok(()) + } + } + + impl Clone for Element { + fn clone(&self) -> Self { + let mut words = vec![MaybeUninit::new(0u64); self.ops.words].into_boxed_slice(); + // SAFETY: both pointers refer to this descriptor's exact type. + unsafe { (self.ops.clone)(self.pointer(), words.as_mut_ptr().cast()) }; + Self { + words, + ops: self.ops, + } + } + } + + impl Drop for Element { + fn drop(&mut self) { + // SAFETY: the callback initialized this concrete value exactly once. + unsafe { (self.ops.drop)(self.words.as_mut_ptr().cast()) }; + } + } + + impl PartialEq for Element { + fn eq(&self, other: &Self) -> bool { + self.ops.identity == other.ops.identity && + // SAFETY: matching descriptors guarantee both values have the same type. + unsafe { (self.ops.equal)(self.pointer(), other.pointer()) != 0 } + } + } + + #[derive(Clone)] + struct MapKey(Element); + + impl PartialEq for MapKey { + fn eq(&self, other: &Self) -> bool { + self.0 == other.0 + } + } + // The checker permits only reflexive, hashable key types. General + // elements remain PartialEq because Float NaN is not equal to itself. + impl Eq for MapKey {} + + impl Hash for MapKey { + fn hash<H: Hasher>(&self, state: &mut H) { + let hash = self.0.ops.hash.expect("validated Map key descriptor"); + // SAFETY: Map keys are fully initialized values of this descriptor. + state.write_u64(unsafe { hash(self.0.pointer()) }); + } + } + + #[derive(Clone)] + pub(super) struct ListHandle { + pub(super) values: List<Element>, + pub(super) element: ValueOps, + } + #[derive(Clone)] + struct MapHandle { + values: Map<MapKey, Element>, + key: ValueOps, + value: ValueOps, + } + + struct ListCursor { + values: IntoValues<Element>, + } + struct MapCursor { + values: IntoValues<MapEntry<MapKey, Element>>, + key: ValueOps, + value: ValueOps, + } + + pub(super) struct StreamHandle { + pub(super) values: resource::Stream<Element>, + element: ValueOps, + } + + pub(super) fn initialize_event<T>( + source: T, + element: ValueOps, + callback: unsafe extern "C" fn(*const c_void, *mut c_void), + ) -> Element { + let handle = owned(source); + let value = Element::from_initializer(element, |out| { + // SAFETY: the process-static callback initializes one complete event + // from this borrowed handle and retains any stored reference. + unsafe { callback(handle, out) }; + }); + // SAFETY: owned created exactly one temporary Arc<T> owner. + unsafe { release::<T>(handle) }; + value + } + + fn take_unique<T: Clone>(value: Arc<T>) -> T { + Arc::try_unwrap(value).unwrap_or_else(|shared| (*shared).clone()) + } + + unsafe fn inputs<'a>( + values: *const *const c_void, + count: u64, + ) -> Result<&'a [*const c_void], AbiFailure> { + let count = usize::try_from(count).map_err(|_| invalid("invalid ABI element count"))?; + if count == 0 { + return Ok(&[]); + } + if values.is_null() + || !values.is_aligned() + || count > (isize::MAX as usize / size_of::<*const c_void>()) + { + return Err(invalid("invalid ABI element array")); + } + // SAFETY: the private caller guarantees count live pointer slots. + Ok(unsafe { std::slice::from_raw_parts(values, count) }) + } + + unsafe fn write_element_optional( + value: impl FnOnce() -> Result<Option<Element>, AbiFailure>, + out: *mut c_void, + present: *mut u8, + buffer: *mut Buffer, + ) -> u32 { + // SAFETY: each export promises a live Buffer slot. + if !unsafe { empty(buffer) } { + return ABI_INVALID_INPUT; + } + if !slot_valid(present) || out.is_null() { + // SAFETY: empty initialized this valid Buffer slot. + return unsafe { error(buffer, ABI_INVALID_INPUT, "invalid ABI output pointer") }; + } + match value() { + Ok(Some(element)) => { + // SAFETY: the caller promises disjoint output slots, including + // storage large and aligned enough for the concrete element. + if let Err(failure) = unsafe { element.copy_to(out) } { + return unsafe { error(buffer, failure.status, failure.message) }; + } + unsafe { present.write(1) }; + ABI_OK + } + Ok(None) => { + unsafe { present.write(0) }; + ABI_OK + } + // SAFETY: empty initialized this valid Buffer slot. + Err(failure) => unsafe { error(buffer, failure.status, failure.message) }, + } + } + + macro_rules! handle_owner { + ($retain:ident, $release:ident, $ty:ty) => { + /// # Safety + /// A non-null handle must be one live owner of this exact managed type. + /// The returned handle owns one additional reference; null is a no-op. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn $retain(handle: *const c_void) -> *mut c_void { + // SAFETY: this export's private handle contract establishes type and lifetime. + unsafe { retain::<$ty>(handle) } + } + + /// # Safety + /// A non-null handle must be one owned reference of this exact type, + /// transferred exactly once; null is a no-op. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn $release(handle: *mut c_void) { + // SAFETY: this export consumes the caller's one reference. + unsafe { release::<$ty>(handle) } + } + }; + } + + handle_owner!(dever_rt_v1_text_retain, dever_rt_v1_text_release, String); + handle_owner!(dever_rt_v1_bytes_retain, dever_rt_v1_bytes_release, Bytes); + handle_owner!( + dever_rt_v1_list_retain, + dever_rt_v1_list_release, + ListHandle + ); + handle_owner!(dever_rt_v1_map_retain, dever_rt_v1_map_release, MapHandle); + handle_owner!( + dever_rt_v1_file_retain, + dever_rt_v1_file_release, + resource::FileHandle + ); + handle_owner!( + dever_rt_v1_stream_retain, + dever_rt_v1_stream_release, + StreamHandle + ); + handle_owner!( + dever_rt_v1_socket_retain, + dever_rt_v1_socket_release, + dever_runtime::net::Socket + ); + handle_owner!( + dever_rt_v1_listener_retain, + dever_rt_v1_listener_release, + dever_runtime::net::Listener + ); + + /// # Safety + /// listener is borrowed; out/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_net_port( + listener: *const c_void, + out: *mut i64, + buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { checked_output(out, buffer) } { + return status; + } + let result = unsafe { borrowed::<dever_runtime::net::Listener>(listener) } + .and_then(|listener| dever_runtime::net::port(listener).map_err(runtime_error)); + unsafe { complete(out, buffer, result) } + } + + /// # Safety + /// socket is borrowed; out/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_net_timeout( + socket: *const c_void, + milliseconds: i64, + out: *mut u8, + buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { checked_output(out, buffer) } { + return status; + } + let result = + unsafe { borrowed::<dever_runtime::net::Socket>(socket) }.and_then(|socket| { + dever_runtime::net::timeout(socket, milliseconds) + .map(|()| 1) + .map_err(runtime_error) + }); + unsafe { complete(out, buffer, result) } + } + + /// # Safety + /// socket is borrowed; out/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_socket_close( + socket: *const c_void, + out: *mut u8, + buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { checked_output(out, buffer) } { + return status; + } + let result = unsafe { borrowed::<dever_runtime::net::Socket>(socket) } + .and_then(|socket| socket.close().map(|()| 1).map_err(runtime_error)); + unsafe { complete(out, buffer, result) } + } + + /// # Safety + /// listener is borrowed; out/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_listener_close( + listener: *const c_void, + out: *mut u8, + buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { checked_output(out, buffer) } { + return status; + } + let result = unsafe { borrowed::<dever_runtime::net::Listener>(listener) } + .and_then(|listener| listener.close().map(|()| 1).map_err(runtime_error)); + unsafe { complete(out, buffer, result) } + } + + pub(super) unsafe fn checked_output<T>( + out: *mut T, + buffer: *mut Buffer, + ) -> Result<(), u32> { + if !unsafe { empty(buffer) } { + return Err(ABI_INVALID_INPUT); + } + if !slot_valid(out) { + return Err(unsafe { + error(buffer, ABI_INVALID_INPUT, "invalid ABI output pointer") + }); + } + Ok(()) + } + + unsafe fn checked_optional_output<T>( + out: *mut T, + present: *mut u8, + buffer: *mut Buffer, + ) -> Result<(), u32> { + unsafe { checked_output(out, buffer) }?; + if !slot_valid(present) { + return Err(unsafe { + error(buffer, ABI_INVALID_INPUT, "invalid ABI output pointer") + }); + } + Ok(()) + } + + /// # Safety + /// path is a live borrowed Text handle. Output slots are distinct and writable. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_file_open( + path: *const c_void, + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { checked_output(out, buffer) } { + return status; + } + let result = unsafe { borrowed::<String>(path) } + .and_then(|path| resource::file_open(path).map_err(runtime_error)); + unsafe { complete_handle(out, buffer, result) } + } + + /// # Safety + /// path is a live borrowed Text handle. Output slots are distinct and writable. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_file_create( + path: *const c_void, + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { checked_output(out, buffer) } { + return status; + } + let result = unsafe { borrowed::<String>(path) } + .and_then(|path| resource::file_create(path).map_err(runtime_error)); + unsafe { complete_handle(out, buffer, result) } + } + + /// # Safety + /// file is a live borrowed File; out/present/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_file_read( + file: *const c_void, + limit: i64, + out: *mut *mut c_void, + present: *mut u8, + buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { checked_optional_output(out, present, buffer) } { + return status; + } + let result = unsafe { borrowed::<resource::FileHandle>(file) } + .and_then(|file| resource::read(file, limit).map_err(runtime_error)); + unsafe { complete_optional_handle(out, present, buffer, result) } + } + + /// # Safety + /// file and bytes are live borrowed handles; out/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_file_write( + file: *const c_void, + bytes: *const c_void, + out: *mut u8, + buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { checked_output(out, buffer) } { + return status; + } + let result = (|| { + let file = unsafe { borrowed::<resource::FileHandle>(file) }?; + let bytes = unsafe { borrowed::<Bytes>(bytes) }?; + resource::write(file, bytes).map_err(runtime_error)?; + Ok(1) + })(); + unsafe { complete(out, buffer, result) } + } + + /// # Safety + /// file is a live borrowed File; out/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_file_close( + file: *const c_void, + out: *mut u8, + buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { checked_output(out, buffer) } { + return status; + } + let result = unsafe { borrowed::<resource::FileHandle>(file) } + .and_then(|file| file.close().map(|()| 1).map_err(runtime_error)); + unsafe { complete(out, buffer, result) } + } + + /// # Safety + /// file is a live borrowed File. Event callbacks and type descriptor are + /// process-static and initialize complete values without unwinding. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_file_chunks( + file: *const c_void, + limit: i64, + event: *const ReadEventDescriptor, + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { checked_output(out, buffer) } { + return status; + } + let result = (|| { + let file = unsafe { borrowed::<resource::FileHandle>(file) }?; + if event.is_null() || !event.is_aligned() { + return Err(invalid("invalid ABI read event descriptor")); + } + let event = unsafe { &*event }; + let element = unsafe { descriptor(event.element_type, false) }?; + let (Some(chunk), Some(failed)) = (event.chunk, event.failed) else { + return Err(invalid("incomplete ABI read event descriptor")); + }; + let values = resource::read_chunks(file.clone(), limit) + .map_err(runtime_error)? + .map(move |result| match result { + Ok(bytes) => initialize_event(bytes, element, chunk), + Err(message) => initialize_event(message, element, failed), + }); + Ok(StreamHandle { values, element }) + })(); + unsafe { complete_handle(out, buffer, result) } + } + + /// # Safety + /// stream is a live borrowed Stream; out/present/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_stream_pull( + stream: *const c_void, + out: *mut c_void, + present: *mut u8, + buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { checked_optional_output(out, present, buffer) } { + return status; + } + let stream = match unsafe { borrowed::<StreamHandle>(stream) } { + Ok(stream) => stream, + Err(failure) => return unsafe { error(buffer, failure.status, failure.message) }, + }; + if !(out as usize).is_multiple_of(stream.element.align) { + return unsafe { error(buffer, ABI_INVALID_INPUT, "invalid ABI element output") }; + } + match stream.values.pull() { + Some(element) => { + if let Err(failure) = unsafe { element.copy_to(out) } { + return unsafe { error(buffer, failure.status, failure.message) }; + } + unsafe { present.write(1) }; + } + None => unsafe { present.write(0) }, + } + ABI_OK + } + + /// # Safety + /// stream is a live borrowed Stream; out/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_stream_close( + stream: *const c_void, + out: *mut u8, + buffer: *mut Buffer, + ) -> u32 { + if let Err(status) = unsafe { checked_output(out, buffer) } { + return status; + } + let result = unsafe { borrowed::<StreamHandle>(stream) }.map(|stream| { + stream.values.close(); + 1 + }); + unsafe { complete(out, buffer, result) } + } + + /// # Safety + /// Input is borrowed for len bytes (null only for zero length); out and + /// error are distinct writable slots. The returned handle is owned. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_text_new( + input_ptr: *const u8, + len: u64, + out: *mut *mut c_void, + error_buffer: *mut Buffer, + ) -> u32 { + // SAFETY: the caller keeps input live and immutable through this call. + let result = unsafe { input(input_ptr, len) } + .map_err(invalid) + .and_then(|bytes| { + Bytes::new(bytes.to_vec()) + .into_text() + .map_err(runtime_error) + }); + // SAFETY: the caller promises distinct aligned writable output slots. + unsafe { complete_handle(out, error_buffer, result) } + } + + /// # Safety + /// handle is a live borrowed Text handle; buffer is a distinct writable slot. + /// Free non-null output through dever_rt_v1_buffer_free. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_text_export( + handle: *const c_void, + buffer: *mut Buffer, + ) -> u32 { + // SAFETY: the private caller supplies a live Text handle. + let result = + unsafe { borrowed::<String>(handle) }.map(|value| value.as_bytes().to_vec()); + unsafe { export_bytes(buffer, result) } + } + + unsafe fn export_bytes(buffer: *mut Buffer, result: Result<Vec<u8>, AbiFailure>) -> u32 { + // SAFETY: the caller supplies a live output Buffer slot. + if !unsafe { empty(buffer) } { + return ABI_INVALID_INPUT; + } + match result { + Ok(bytes) => { + unsafe { owned_bytes(buffer, bytes) }; + ABI_OK + } + Err(failure) => unsafe { error(buffer, failure.status, failure.message) }, + } + } + + /// # Safety + /// Both non-null handles are borrowed live Text handles. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_text_equal( + left: *const c_void, + right: *const c_void, + ) -> u8 { + // SAFETY: the caller guarantees the exact handle type and lifetime. + u8::from( + unsafe { borrowed::<String>(left).expect("valid Text handle") } + == unsafe { borrowed::<String>(right).expect("valid Text handle") }, + ) + } + + /// # Safety + /// handle is a non-null live borrowed Text handle. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_text_hash(handle: *const c_void) -> u64 { + let value = unsafe { borrowed::<String>(handle).expect("valid Text handle") }; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + value.hash(&mut hasher); + hasher.finish() + } + + /// # Safety + /// Both handles are live borrowed Text; out/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_text_compare( + left: *const c_void, + right: *const c_void, + out: *mut i32, + buffer: *mut Buffer, + ) -> u32 { + let result = (|| { + let left = unsafe { borrowed::<String>(left) }?; + let right = unsafe { borrowed::<String>(right) }?; + Ok(match left.cmp(right) { + std::cmp::Ordering::Less => -1, + std::cmp::Ordering::Equal => 0, + std::cmp::Ordering::Greater => 1, + }) + })(); + unsafe { complete(out, buffer, result) } + } + + /// # Safety + /// Both handles are live borrowed Text; out/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_text_concat( + left: *const c_void, + right: *const c_void, + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + let result = (|| { + let mut result = unsafe { borrowed::<String>(left) }?.clone(); + result.push_str(unsafe { borrowed::<String>(right) }?); + Ok(result) + })(); + unsafe { complete_handle(out, buffer, result) } + } + + /// # Safety + /// handle is live borrowed Text; out/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_text_length( + handle: *const c_void, + out: *mut i64, + buffer: *mut Buffer, + ) -> u32 { + let result = + unsafe { borrowed::<String>(handle) }.map(|value| value.chars().count() as i64); + unsafe { complete(out, buffer, result) } + } + + macro_rules! text_transform { + ($name:ident, $body:expr) => { + /// # Safety + /// handle is live borrowed Text; out/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn $name( + handle: *const c_void, + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + let result = unsafe { borrowed::<String>(handle) }.map($body); + unsafe { complete_handle(out, buffer, result) } + } + }; + } + text_transform!(dever_rt_v1_text_trim, |value: &String| value + .trim() + .to_owned()); + text_transform!(dever_rt_v1_text_lower, |value: &String| value + .to_lowercase()); + text_transform!(dever_rt_v1_text_upper, |value: &String| value + .to_uppercase()); + + /// # Safety + /// handle is live borrowed Text; out/present/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_text_codepoint( + handle: *const c_void, + out: *mut i64, + present: *mut u8, + buffer: *mut Buffer, + ) -> u32 { + let result = + unsafe { borrowed::<String>(handle) }.map(|value| text_runtime::codepoint(value)); + unsafe { complete_optional(out, present, buffer, result) } + } + + /// # Safety + /// out/present/error are distinct writable slots; returned handle is owned. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_text_from_codepoint( + value: i64, + out: *mut *mut c_void, + present: *mut u8, + buffer: *mut Buffer, + ) -> u32 { + unsafe { + complete_optional_handle( + out, + present, + buffer, + Ok(text_runtime::from_codepoint(value)), + ) + } + } + + /// # Safety + /// handle is live borrowed Text; out/present/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_text_at( + handle: *const c_void, + index: i64, + out: *mut *mut c_void, + present: *mut u8, + buffer: *mut Buffer, + ) -> u32 { + let result = + unsafe { borrowed::<String>(handle) }.map(|value| text_runtime::at(value, index)); + unsafe { complete_optional_handle(out, present, buffer, result) } + } + + /// # Safety + /// handle is live borrowed Text; out/present/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_text_slice( + handle: *const c_void, + start: i64, + end: i64, + out: *mut *mut c_void, + present: *mut u8, + buffer: *mut Buffer, + ) -> u32 { + let result = unsafe { borrowed::<String>(handle) } + .map(|value| text_runtime::slice(value, start, end)); + unsafe { complete_optional_handle(out, present, buffer, result) } + } + + /// # Safety + /// handles are live borrowed Text; out/present/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_text_index_of( + handle: *const c_void, + pattern: *const c_void, + out: *mut i64, + present: *mut u8, + buffer: *mut Buffer, + ) -> u32 { + let result = (|| { + Ok(text_runtime::index_of( + unsafe { borrowed::<String>(handle) }?, + unsafe { borrowed::<String>(pattern) }?, + )) + })(); + unsafe { complete_optional(out, present, buffer, result) } + } + + macro_rules! text_predicate { + ($name:ident, $method:ident) => { + /// # Safety + /// handles are live borrowed Text; out/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn $name( + handle: *const c_void, + pattern: *const c_void, + out: *mut u8, + buffer: *mut Buffer, + ) -> u32 { + let result = (|| { + Ok(u8::from( + unsafe { borrowed::<String>(handle) }? + .$method(unsafe { borrowed::<String>(pattern) }?), + )) + })(); + unsafe { complete(out, buffer, result) } + } + }; + } + text_predicate!(dever_rt_v1_text_contains, contains); + text_predicate!(dever_rt_v1_text_starts_with, starts_with); + text_predicate!(dever_rt_v1_text_ends_with, ends_with); + + /// # Safety + /// handles are live borrowed Text; out/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_text_replace( + handle: *const c_void, + pattern: *const c_void, + replacement: *const c_void, + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + let result = (|| { + Ok(unsafe { borrowed::<String>(handle) }? + .replace(unsafe { borrowed::<String>(pattern) }?, unsafe { + borrowed::<String>(replacement) + }?)) + })(); + unsafe { complete_handle(out, buffer, result) } + } + + /// # Safety + /// text is live borrowed Text; out/present/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_int_parse( + text: *const c_void, + out: *mut i64, + present: *mut u8, + buffer: *mut Buffer, + ) -> u32 { + let result = unsafe { borrowed::<String>(text) }.map(|value| number::parse_int(value)); + unsafe { complete_optional(out, present, buffer, result) } + } + + /// # Safety + /// text is live borrowed Text; out/present/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_float_parse( + text: *const c_void, + out: *mut f64, + present: *mut u8, + buffer: *mut Buffer, + ) -> u32 { + let result = + unsafe { borrowed::<String>(text) }.map(|value| number::parse_float(value)); + unsafe { complete_optional(out, present, buffer, result) } + } + + /// # Safety + /// text is live borrowed Text; out/present/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_decimal_parse( + text: *const c_void, + out: *mut AbiDecimal, + present: *mut u8, + buffer: *mut Buffer, + ) -> u32 { + let result = unsafe { borrowed::<String>(text) } + .map(|value| number::parse_decimal(value).map(AbiDecimal::encode)); + unsafe { complete_optional(out, present, buffer, result) } + } + + /// # Safety + /// out/error are distinct writable slots; input words must be finite Decimal values. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_decimal_compare( + left_low: u64, + left_high: u64, + right_low: u64, + right_high: u64, + out: *mut i32, + buffer: *mut Buffer, + ) -> u32 { + let result = (|| { + let left = AbiDecimal { + low: left_low, + high: left_high, + } + .decode() + .map_err(runtime_error)?; + let right = AbiDecimal { + low: right_low, + high: right_high, + } + .decode() + .map_err(runtime_error)?; + Ok(match left.cmp(&right) { + std::cmp::Ordering::Less => -1, + std::cmp::Ordering::Equal => 0, + std::cmp::Ordering::Greater => 1, + }) + })(); + unsafe { complete(out, buffer, result) } + } + + /// # Safety + /// Both word pairs must encode compiler-checked finite Decimal values. + /// This infallible typed callback helper compares numeric values, not bits. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_decimal_equal( + left_low: u64, + left_high: u64, + right_low: u64, + right_high: u64, + ) -> u8 { + let left = AbiDecimal { + low: left_low, + high: left_high, + } + .decode() + .expect("checked finite Decimal"); + let right = AbiDecimal { + low: right_low, + high: right_high, + } + .decode() + .expect("checked finite Decimal"); + u8::from(left == right) + } + + /// # Safety + /// Both handles are live borrowed Text; text_type describes the pointer + /// representation of Text and remains live; out/error are writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_text_split( + handle: *const c_void, + separator: *const c_void, + text_type: *const TypeDescriptor, + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + let result = (|| { + let value = unsafe { borrowed::<String>(handle) }?; + let separator = unsafe { borrowed::<String>(separator) }?; + let ops = unsafe { descriptor(text_type, false) }?; + if ops.size != size_of::<*mut c_void>() { + return Err(invalid("invalid Text element layout")); + } + let mut values = Vec::new(); + for piece in text_runtime::split(value, separator).into_values() { + let handle = owned(piece); + // SAFETY: this stack slot contains a live Text handle of the + // compiler-declared element type until the clone finishes. + let element = unsafe { + Element::from_source(ops, (&handle as *const *mut c_void).cast()) + }; + unsafe { release::<String>(handle) }; + values.push(element?); + } + Ok(ListHandle { + values: List::new(values), + element: ops, + }) + })(); + unsafe { complete_handle(out, buffer, result) } + } + + /// # Safety + /// Input is borrowed for len bytes (null only for zero); out/error are + /// distinct writable slots. The returned Bytes handle is owned. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_bytes_new( + input_ptr: *const u8, + len: u64, + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + let result = unsafe { input(input_ptr, len) } + .map(|bytes| Bytes::new(bytes.to_vec())) + .map_err(invalid); + unsafe { complete_handle(out, buffer, result) } + } + + /// # Safety + /// handle is live borrowed Bytes; buffer is a distinct writable slot. + /// Free non-null output via dever_rt_v1_buffer_free. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_bytes_export( + handle: *const c_void, + buffer: *mut Buffer, + ) -> u32 { + let result = unsafe { borrowed::<Bytes>(handle) }.map(|value| value.values().to_vec()); + unsafe { export_bytes(buffer, result) } + } + + /// # Safety + /// Both handles are live borrowed Bytes handles. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_bytes_equal( + left: *const c_void, + right: *const c_void, + ) -> u8 { + u8::from( + unsafe { borrowed::<Bytes>(left).expect("valid Bytes handle") } + == unsafe { borrowed::<Bytes>(right).expect("valid Bytes handle") }, + ) + } + + /// # Safety + /// handle is live borrowed Bytes; out/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_bytes_length( + handle: *const c_void, + out: *mut i64, + buffer: *mut Buffer, + ) -> u32 { + let result = unsafe { borrowed::<Bytes>(handle) }.map(Bytes::length); + unsafe { complete(out, buffer, result) } + } + + /// # Safety + /// handle is live borrowed Bytes; out/present/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_bytes_at( + handle: *const c_void, + index: i64, + out: *mut i64, + present: *mut u8, + buffer: *mut Buffer, + ) -> u32 { + let result = unsafe { borrowed::<Bytes>(handle) }.map(|value| value.at(index)); + unsafe { complete_optional(out, present, buffer, result) } + } + + /// # Safety + /// Both handles are live borrowed Bytes; out/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_bytes_concat( + left: *const c_void, + right: *const c_void, + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + let result = (|| { + Ok(unsafe { borrowed::<Bytes>(left) }? + .clone() + .concat(unsafe { borrowed::<Bytes>(right) }?)) + })(); + unsafe { complete_handle(out, buffer, result) } + } + + /// # Safety + /// handle is live borrowed Bytes; out/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_bytes_slice( + handle: *const c_void, + start: i64, + end: i64, + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + let result = unsafe { borrowed::<Bytes>(handle) } + .and_then(|value| value.slice(start, end).map_err(runtime_error)); + unsafe { complete_handle(out, buffer, result) } + } + + /// # Safety + /// text is live borrowed Text; out/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_bytes_from_text( + text: *const c_void, + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + let result = unsafe { borrowed::<String>(text) }.map(|value| Bytes::from_text(value)); + unsafe { complete_handle(out, buffer, result) } + } + + /// # Safety + /// bytes is live borrowed Bytes; out/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_bytes_to_text_handle( + bytes: *const c_void, + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + let result = unsafe { borrowed::<Bytes>(bytes) } + .and_then(|value| value.to_text().map_err(runtime_error)); + unsafe { complete_handle(out, buffer, result) } + } + + /// # Safety + /// list is a live borrowed List<Int> handle; out/error are distinct + /// writable slots. The caller's concrete descriptor has eight-byte Ints. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_bytes_from_ints( + list: *const c_void, + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + let result = (|| { + let list = unsafe { borrowed::<ListHandle>(list) }?; + if list.element.size != size_of::<i64>() { + return Err(invalid("invalid Int element layout")); + } + let values = list + .values + .values() + .iter() + .map(|value| { + // SAFETY: the checked Bytes.fromInts call supplies List<Int>; + // Int is exactly one initialized, aligned i64 with no padding. + unsafe { (value.pointer() as *const i64).read() } + }) + .collect::<Vec<_>>(); + Bytes::from_ints(&values).map_err(runtime_error) + })(); + unsafe { complete_handle(out, buffer, result) } + } + + /// # Safety + /// element_type is a process-static descriptor. values points to count + /// live borrowed element pointers (or is null when count is zero). + /// out/error are distinct writable slots; returned handle is owned. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_list_new( + element_type: *const TypeDescriptor, + values: *const *const c_void, + count: u64, + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + let result = (|| { + let element = unsafe { descriptor(element_type, false) }?; + let sources = unsafe { inputs(values, count) }?; + let mut collected = Vec::with_capacity(sources.len()); + for source in sources { + collected.push(unsafe { Element::from_source(element, *source) }?); + } + Ok(ListHandle { + values: List::new(collected), + element, + }) + })(); + unsafe { complete_handle(out, buffer, result) } + } + + /// # Safety + /// list transfers one live owned List handle even on failure; element is + /// borrowed initialized storage of its descriptor's type. out/error are + /// distinct writable slots and a successful out owns the new List. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_list_append_take( + list: *mut c_void, + element: *const c_void, + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + let result = (|| { + let list = unsafe { consumed::<ListHandle>(list) }?; + let mut list = take_unique(list); + let element = unsafe { Element::from_source(list.element, element) }?; + list.values = list.values.append(element); + Ok(list) + })(); + unsafe { complete_handle(out, buffer, result) } + } + + /// # Safety + /// list transfers one live owned List handle even on failure. out_element + /// is uninitialized storage for its concrete type, present/error are + /// distinct writable slots; only a present result initializes the value. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_list_first_take( + list: *mut c_void, + out_element: *mut c_void, + present: *mut u8, + buffer: *mut Buffer, + ) -> u32 { + let list = unsafe { consumed::<ListHandle>(list) }; + unsafe { + write_element_optional( + || list.map(|list| take_unique(list).values.into_first()), + out_element, + present, + buffer, + ) + } + } + + /// # Safety + /// list is a live borrowed List handle; out/error are writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_list_length( + list: *const c_void, + out: *mut i64, + buffer: *mut Buffer, + ) -> u32 { + let result = unsafe { borrowed::<ListHandle>(list) } + .map(|list| list.values.values().len() as i64); + unsafe { complete(out, buffer, result) } + } + + /// # Safety + /// Both handles are live borrowed Lists of the same concrete element type. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_list_equal( + left: *const c_void, + right: *const c_void, + ) -> u8 { + let left = unsafe { borrowed::<ListHandle>(left).expect("valid List handle") }; + let right = unsafe { borrowed::<ListHandle>(right).expect("valid List handle") }; + u8::from(left.element.identity == right.element.identity && left.values == right.values) + } + + /// # Safety + /// list transfers one live owned List handle even on failure. out_cursor + /// and error are distinct writable slots; returned cursor is owned. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_list_cursor_take( + list: *mut c_void, + out_cursor: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + let result = unsafe { consumed::<ListHandle>(list) }.map(|list| ListCursor { + values: take_unique(list).values.into_values(), + }); + // SAFETY: cursor is a Box owner, not an Arc managed value. + unsafe { complete_cursor(out_cursor, buffer, result) } + } + + unsafe fn complete_cursor<T>( + out: *mut *mut c_void, + buffer: *mut Buffer, + result: Result<T, AbiFailure>, + ) -> u32 { + // SAFETY: every cursor export promises a live Buffer slot. + if !unsafe { empty(buffer) } { + return ABI_INVALID_INPUT; + } + if !slot_valid(out) { + return unsafe { error(buffer, ABI_INVALID_INPUT, "invalid ABI output pointer") }; + } + match result { + Ok(cursor) => { + // SAFETY: out is a distinct writable pointer slot. + unsafe { out.write(Box::into_raw(Box::new(cursor)).cast()) }; + ABI_OK + } + Err(failure) => unsafe { error(buffer, failure.status, failure.message) }, + } + } + + unsafe fn cursor_mut<'a, T>(cursor: *mut c_void) -> Result<&'a mut T, AbiFailure> { + if cursor.is_null() || !(cursor as *mut T).is_aligned() { + return Err(invalid("invalid ABI cursor")); + } + // SAFETY: the caller grants this call exclusive access to a live cursor. + Ok(unsafe { &mut *(cursor as *mut T) }) + } + + /// # Safety + /// cursor is a live exclusive List cursor. out_element is uninitialized + /// aligned storage for its element; present/error are distinct slots. + /// A present element is independently owned and must be dropped once. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_list_cursor_next( + cursor: *mut c_void, + out_element: *mut c_void, + present: *mut u8, + buffer: *mut Buffer, + ) -> u32 { + unsafe { + write_element_optional( + || cursor_mut::<ListCursor>(cursor).map(|cursor| cursor.values.next()), + out_element, + present, + buffer, + ) + } + } + + /// # Safety + /// cursor is one owned List cursor returned by list_cursor_take, consumed + /// exactly once; null is a no-op. Remaining values are destroyed. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_list_cursor_release(cursor: *mut c_void) { + if !cursor.is_null() { + // SAFETY: the caller transfers one Box<ListCursor> owner. + drop(unsafe { Box::from_raw(cursor as *mut ListCursor) }); + } + } + + /// # Safety + /// Descriptors are process-static; keys/values point to count borrowed + /// initialized concrete values (null arrays only with count zero). + /// out/error are distinct writable slots and returned Map is owned. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_map_new( + key_type: *const TypeDescriptor, + value_type: *const TypeDescriptor, + keys: *const *const c_void, + values: *const *const c_void, + count: u64, + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + let result = (|| { + let key = unsafe { descriptor(key_type, true) }?; + let value = unsafe { descriptor(value_type, false) }?; + let keys = unsafe { inputs(keys, count) }?; + let values = unsafe { inputs(values, count) }?; + let mut entries = Vec::with_capacity(keys.len()); + for (key_source, value_source) in keys.iter().zip(values) { + entries.push(( + MapKey(unsafe { Element::from_source(key, *key_source) }?), + unsafe { Element::from_source(value, *value_source) }?, + )); + } + let values = Map::new(entries).map_err(runtime_error)?; + Ok(MapHandle { values, key, value }) + })(); + unsafe { complete_handle(out, buffer, result) } + } + + /// # Safety + /// map transfers one live owned Map even on failure. key/value are + /// borrowed initialized values of the stored descriptors; out/error are + /// distinct writable slots. Returned Map is an owned COW value. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_map_put_take( + map: *mut c_void, + key: *const c_void, + value: *const c_void, + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + let result = (|| { + let map = unsafe { consumed::<MapHandle>(map) }?; + let mut map = take_unique(map); + let key = MapKey(unsafe { Element::from_source(map.key, key) }?); + let value = unsafe { Element::from_source(map.value, value) }?; + map.values = map.values.put(key, value); + Ok(map) + })(); + unsafe { complete_handle(out, buffer, result) } + } + + /// # Safety + /// map transfers one live owned Map even on failure. key is a borrowed + /// initialized key value; out/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_map_remove_take( + map: *mut c_void, + key: *const c_void, + out: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + let result = (|| { + let map = unsafe { consumed::<MapHandle>(map) }?; + let mut map = take_unique(map); + let key = MapKey(unsafe { Element::from_source(map.key, key) }?); + map.values = map.values.remove(&key); + Ok(map) + })(); + unsafe { complete_handle(out, buffer, result) } + } + + /// # Safety + /// map transfers one live owned Map even on failure. key is borrowed; + /// out_value is uninitialized storage of the concrete value type. + /// present/error are distinct writable slots; present output is owned. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_map_get_take( + map: *mut c_void, + key: *const c_void, + out_value: *mut c_void, + present: *mut u8, + buffer: *mut Buffer, + ) -> u32 { + let result = (|| { + let map = unsafe { consumed::<MapHandle>(map) }?; + let map = take_unique(map); + let key = MapKey(unsafe { Element::from_source(map.key, key) }?); + Ok(map.values.into_get(&key)) + })(); + unsafe { write_element_optional(|| result, out_value, present, buffer) } + } + + /// # Safety + /// map is a live borrowed Map; out/error are distinct writable slots. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_map_length( + map: *const c_void, + out: *mut i64, + buffer: *mut Buffer, + ) -> u32 { + let result = + unsafe { borrowed::<MapHandle>(map) }.map(|map| map.values.pairs().count() as i64); + unsafe { complete(out, buffer, result) } + } + + /// # Safety + /// Both handles are live borrowed Maps of the same key/value types. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_map_equal( + left: *const c_void, + right: *const c_void, + ) -> u8 { + let left = unsafe { borrowed::<MapHandle>(left).expect("valid Map handle") }; + let right = unsafe { borrowed::<MapHandle>(right).expect("valid Map handle") }; + u8::from( + left.key.identity == right.key.identity + && left.value.identity == right.value.identity + && left.values == right.values, + ) + } + + /// # Safety + /// map transfers one live owned Map even on failure; out_cursor/error + /// are distinct writable slots. Returned cursor is owned. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_map_cursor_take( + map: *mut c_void, + out_cursor: *mut *mut c_void, + buffer: *mut Buffer, + ) -> u32 { + let result = unsafe { consumed::<MapHandle>(map) }.map(|map| { + let map = take_unique(map); + MapCursor { + values: map.values.into_entries().into_values(), + key: map.key, + value: map.value, + } + }); + unsafe { complete_cursor(out_cursor, buffer, result) } + } + + /// # Safety + /// cursor is a live exclusive Map cursor. out_key/out_value are distinct + /// uninitialized concrete value slots, also distinct from present/error. + /// Present outputs each own one independent value and must be dropped. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_map_cursor_next( + cursor: *mut c_void, + out_key: *mut c_void, + out_value: *mut c_void, + present: *mut u8, + buffer: *mut Buffer, + ) -> u32 { + if !unsafe { empty(buffer) } { + return ABI_INVALID_INPUT; + } + if !slot_valid(present) || out_key.is_null() || out_value.is_null() { + return unsafe { error(buffer, ABI_INVALID_INPUT, "invalid ABI output pointer") }; + } + let cursor = match unsafe { cursor_mut::<MapCursor>(cursor) } { + Ok(cursor) => cursor, + Err(failure) => return unsafe { error(buffer, failure.status, failure.message) }, + }; + if !(out_key as usize).is_multiple_of(cursor.key.align) + || !(out_value as usize).is_multiple_of(cursor.value.align) + { + return unsafe { error(buffer, ABI_INVALID_INPUT, "invalid ABI element output") }; + } + match cursor.values.next() { + Some(entry) => { + // SAFETY: both aligned, disjoint slots are uninitialized and + // callbacks establish independent ownership before success. + unsafe { (cursor.key.clone)(entry.key.0.pointer(), out_key) }; + unsafe { (cursor.value.clone)(entry.value.pointer(), out_value) }; + unsafe { present.write(1) }; + } + None => unsafe { present.write(0) }, + } + ABI_OK + } + + /// # Safety + /// cursor is one owned Map cursor returned by map_cursor_take, consumed + /// exactly once; null is a no-op. Remaining entries are destroyed. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn dever_rt_v1_map_cursor_release(cursor: *mut c_void) { + if !cursor.is_null() { + // SAFETY: caller transfers one Box<MapCursor> owner. + drop(unsafe { Box::from_raw(cursor as *mut MapCursor) }); + } + } + } +} diff --git a/crates/dever-backend-bridge/src/lib.rs b/crates/dever-backend-bridge/src/lib.rs new file mode 100644 index 0000000..8b32010 --- /dev/null +++ b/crates/dever-backend-bridge/src/lib.rs @@ -0,0 +1,310 @@ +//! Private compiler bridge. LLVM/LLD handles never escape the safe interface. +//! This crate is not the HIR lowerer and does not expose language FFI. + +/// Runtime packs must match the versioned symbols in include/runtime.h. +pub const RUNTIME_ABI_VERSION: u32 = 1; + +#[cfg(any(feature = "embedded", feature = "runtime-abi"))] +mod ffi; + +#[cfg(feature = "runtime-abi")] +pub use dever_runtime::abi::DecimalBits as AbiDecimal; + +#[cfg(feature = "runtime-abi")] +pub const ABI_OK: u32 = 0; +#[cfg(feature = "runtime-abi")] +pub const ABI_RUNTIME_ERROR: u32 = 1; +#[cfg(feature = "runtime-abi")] +pub const ABI_INVALID_INPUT: u32 = 2; + +/// The six release targets; target selection never consults host defaults. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Target { + LinuxX86_64, + LinuxAarch64, + MacosX86_64, + MacosAarch64, + WindowsX86_64, + WindowsAarch64, +} + +impl Target { + pub const ALL: [Self; 6] = [ + Self::LinuxX86_64, + Self::LinuxAarch64, + Self::MacosX86_64, + Self::MacosAarch64, + Self::WindowsX86_64, + Self::WindowsAarch64, + ]; + + pub fn triple(self) -> &'static str { + match self { + Self::LinuxX86_64 => "x86_64-unknown-linux-gnu", + Self::LinuxAarch64 => "aarch64-unknown-linux-gnu", + Self::MacosX86_64 => "x86_64-apple-macosx11.0.0", + Self::MacosAarch64 => "arm64-apple-macosx11.0.0", + Self::WindowsX86_64 => "x86_64-pc-windows-msvc", + Self::WindowsAarch64 => "aarch64-pc-windows-msvc", + } + } +} + +/// Parse, verify, optimize and emit a position-independent native object. +/// Errors are LLVM diagnostics, not successful empty artifacts. +#[cfg(feature = "embedded")] +pub fn emit_object(ir: &str, target: Target) -> Result<Vec<u8>, String> { + emit_object_with_resources(ir, target, &[]) +} + +/// A borrowed immutable initializer for one declared byte-array global. +#[derive(Clone, Copy, Debug)] +pub struct BinaryResource<'a> { + pub symbol: &'a str, + pub bytes: &'a [u8], +} + +/// Attach binary payloads before verification and optimization, without text encoding. +#[cfg(feature = "embedded")] +pub fn emit_object_with_resources( + ir: &str, + target: Target, + resources: &[BinaryResource<'_>], +) -> Result<Vec<u8>, String> { + if ir.is_empty() || ir.len() > 8 * 1024 * 1024 { + return Err("LLVM IR must contain 1..8388608 bytes".into()); + } + if ir.as_bytes().contains(&0) { + return Err("LLVM IR must escape NUL bytes".into()); + } + if resources.len() > 4096 { + return Err("LLVM resources exceed 4096 attachments".into()); + } + let mut total = 0usize; + let mut symbols = std::collections::BTreeSet::new(); + for resource in resources { + if resource.symbol.is_empty() + || resource.symbol.len() > 256 + || !resource.symbol.bytes().enumerate().all(|(index, byte)| { + byte == b'_' || byte.is_ascii_alphabetic() || (index > 0 && byte.is_ascii_digit()) + }) + { + return Err("LLVM resource requires an unquoted symbol name".into()); + } + if !symbols.insert(resource.symbol) { + return Err("duplicate LLVM resource symbol".into()); + } + if resource.bytes.len() > 128 * 1024 * 1024 { + return Err("LLVM resource exceeds 128 MiB".into()); + } + total = total + .checked_add(resource.bytes.len()) + .ok_or("LLVM resource size overflow")?; + if total > 256 * 1024 * 1024 { + return Err("LLVM resources exceed 256 MiB".into()); + } + } + ffi::emit_object(ir, target, resources) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum LinkKind { + Executable, + /// Standard CRT entry with explicit archives; never search host shared libraries. + StaticExecutable, + Shared, +} + +/// Inputs are explicit object/archive/import-library paths supplied by the +/// compiler's verified pack owner. LLD cannot search host library directories. +/// Output must be new and below a compiler-owned private build directory. +/// An unrecoverable LLD crash terminates the private compiler worker rather +/// than allowing cleanup or another call into potentially corrupted state. +#[cfg(feature = "embedded")] +pub fn link( + target: Target, + kind: LinkKind, + inputs: &[std::path::PathBuf], + output: &std::path::Path, + entry: Option<&str>, +) -> Result<(), String> { + use std::ffi::CString; + + if inputs.is_empty() || inputs.len() > 4096 { + return Err("link requires 1..4096 explicit inputs".into()); + } + let parent = output + .parent() + .ok_or("link output has no parent")? + .canonicalize() + .map_err(|error| format!("link output directory: {error}"))?; + verify_link_directory(&parent)?; + let output = parent.join(output.file_name().ok_or("link output has no filename")?); + let mut args = link_arguments(target, kind, entry)?; + args.extend(output_arguments(target, &output)?); + for input in inputs { + let path = input + .canonicalize() + .map_err(|error| format!("link input: {error}"))?; + if !path.is_file() { + return Err("link input is not a regular file".into()); + } + args.push( + path.to_str() + .ok_or("link input path must be UTF-8")? + .to_owned(), + ); + } + let args = args + .into_iter() + .map(CString::new) + .collect::<Result<Vec<_>, _>>() + .map_err(|_| "link argument contains NUL")?; + // Claim the new output before entering the serialized linker. Concurrent + // callers cannot both pass a check-then-create and overwrite each other. + let mut create = std::fs::OpenOptions::new(); + create.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + create.mode(0o600); + } + drop( + create + .open(&output) + .map_err(|error| format!("cannot reserve new link output: {error}"))?, + ); + let result = ffi::link(&args); + // Only our new output is eligible for failure cleanup, never input packs. + if result.is_err() + && let Err(error) = std::fs::remove_file(&output) + && error.kind() != std::io::ErrorKind::NotFound + { + return Err(format!( + "{}; link output cleanup: {error}", + result.unwrap_err() + )); + } + result +} + +#[cfg(all(feature = "embedded", unix))] +fn verify_link_directory(directory: &std::path::Path) -> Result<(), String> { + use std::os::unix::fs::MetadataExt; + let uid = rustix::process::geteuid().as_raw(); + for (index, ancestor) in directory.ancestors().enumerate() { + let metadata = ancestor + .symlink_metadata() + .map_err(|error| format!("cannot inspect link directory: {error}"))?; + if !metadata.is_dir() || !matches!(metadata.uid(), owner if owner == uid || owner == 0) { + return Err("link directory has an untrusted ancestor".into()); + } + let mode = metadata.mode(); + if index == 0 && (metadata.uid() != uid || mode & 0o077 != 0) { + return Err("link output requires a compiler-owned private directory".into()); + } + // Sticky OS temp roots protect our owned entry. A writable non-sticky + // ancestor would permit another UID to rename an otherwise private leaf. + if mode & 0o022 != 0 && mode & 0o1000 == 0 { + return Err("link directory has a publicly writable ancestor".into()); + } + } + Ok(()) +} + +#[cfg(all(feature = "embedded", not(unix)))] +fn verify_link_directory(_: &std::path::Path) -> Result<(), String> { + Err("compiler-private link directory verification is unavailable on this host".into()) +} + +#[cfg(feature = "embedded")] +fn output_arguments(target: Target, output: &std::path::Path) -> Result<Vec<String>, String> { + let output = output.to_str().ok_or("link output path must be UTF-8")?; + Ok(match target { + Target::WindowsX86_64 | Target::WindowsAarch64 => vec![format!("/out:{output}")], + _ => vec!["-o".into(), output.into()], + }) +} + +#[cfg(feature = "embedded")] +fn link_arguments( + target: Target, + kind: LinkKind, + entry: Option<&str>, +) -> Result<Vec<String>, String> { + if let Some(entry) = entry + && (entry.is_empty() + || !entry + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_')) + { + return Err("link entry must be a symbol name".into()); + } + let args: &[&str] = match target { + Target::LinuxX86_64 => &["ld.lld", "-m", "elf_x86_64", "--no-undefined"], + Target::LinuxAarch64 => &["ld.lld", "-m", "aarch64linux", "--no-undefined"], + Target::MacosX86_64 => &[ + "ld64.lld", + "-arch", + "x86_64", + "-platform_version", + "macos", + "11.0", + "11.0", + ], + Target::MacosAarch64 => &[ + "ld64.lld", + "-arch", + "arm64", + "-platform_version", + "macos", + "11.0", + "11.0", + ], + Target::WindowsX86_64 => &[ + "lld-link", + "/machine:x64", + "/nodefaultlib", + "/subsystem:console", + ], + Target::WindowsAarch64 => &[ + "lld-link", + "/machine:arm64", + "/nodefaultlib", + "/subsystem:console", + ], + }; + let mut args = args + .iter() + .map(|arg| (*arg).into()) + .collect::<Vec<String>>(); + match (target, kind) { + (Target::LinuxX86_64 | Target::LinuxAarch64, LinkKind::StaticExecutable) => { + args.extend([ + "-static".into(), + "--gc-sections".into(), + "--strip-all".into(), + ]); + } + (_, LinkKind::StaticExecutable) => { + return Err("static executable runtime packs currently require a Linux target".into()); + } + (Target::LinuxX86_64 | Target::LinuxAarch64, LinkKind::Shared) => { + args.push("-shared".into()) + } + (Target::MacosX86_64 | Target::MacosAarch64, LinkKind::Shared) => { + args.push("-dylib".into()) + } + (Target::WindowsX86_64 | Target::WindowsAarch64, LinkKind::Shared) => { + args.extend(["/dll".into(), "/noentry".into()]); + } + _ => {} + } + if let Some(entry) = entry { + match target { + Target::WindowsX86_64 | Target::WindowsAarch64 => args.push(format!("/entry:{entry}")), + _ => args.extend(["-e".into(), entry.into()]), + } + } + Ok(args) +} diff --git a/crates/dever-cli/Cargo.toml b/crates/dever-cli/Cargo.toml new file mode 100644 index 0000000..8604bbe --- /dev/null +++ b/crates/dever-cli/Cargo.toml @@ -0,0 +1,99 @@ +[package] +name = "dever-cli" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +publish.workspace = true +default-run = "dever" + +[lints] +workspace = true + +[dependencies] +dever-backend-bridge = { path = "../dever-backend-bridge", features = ["embedded"] } +dever-core.workspace = true +dever-sandbox = { path = "../dever-sandbox" } +dever-runtime = { workspace = true, features = ["database"] } +serde_json.workspace = true +serde.workspace = true +sha2.workspace = true +ring.workspace = true +fs2 = "=0.4.3" +ureq = "=3.4.2" +url = "=2.5.8" +pep440_rs = "=0.7.3" +pep508_rs = "=0.9.2" +node-semver = "=2.2.0" +base64 = "=0.22.1" +zip = { version = "=2.4.2", default-features = false, features = ["deflate"] } +tar = { version = "=0.4.44", default-features = false } +flate2 = "=1.1.5" +csv = "=1.4.0" +goblin = { version = "=0.10.7", default-features = false, features = ["elf32", "elf64", "endian_fd", "std"] } + +[dev-dependencies] +dever-runtime = { workspace = true, features = ["external"] } + +[target.'cfg(target_os = "linux")'.dev-dependencies] +rustix = { version = "=1.1.2", features = ["mount", "thread"] } + +[target.'cfg(unix)'.dependencies] +rustix = { version = "=1.1.2", features = ["fs", "net", "process"] } + +[lib] +name = "dever_cli" +path = "src/lib.rs" + +[[bin]] +name = "dever" +path = "src/main.rs" + +[[bin]] +name = "dever-launcher" +path = "src/bin/dever-launcher.rs" + +[[bin]] +name = "deverd" +path = "src/bin/deverd.rs" + +[[test]] +name = "llvm_cli" +path = "../../test/dever-cli-tests/llvm_cli.rs" + +[[example]] +name = "native-release" +path = "../../sdk/native-release.rs" +test = false + +[[example]] +name = "native-acceptance-init" +path = "../../test/dever-cli-tests/support/native_acceptance_init.rs" +test = false + +[[test]] +name = "native_release" +path = "../../test/dever-cli-tests/native_release.rs" + +[[test]] +name = "cms_project" +path = "../../test/dever-cli-tests/cms_project.rs" + +[[test]] +name = "shared_toolchain" +path = "../../test/dever-cli-tests/shared_toolchain.rs" + +[[test]] +name = "external_libs" +path = "../../test/dever-cli-tests/external_libs.rs" + +[[test]] +name = "external_workers" +path = "../../test/dever-cli-tests/external_workers.rs" + +[[test]] +name = "packages" +path = "../../test/dever-cli-tests/packages.rs" + +[[test]] +name = "new_project" +path = "../../test/dever-cli-tests/new_project.rs" diff --git a/crates/dever-cli/build.rs b/crates/dever-cli/build.rs new file mode 100644 index 0000000..5a0c29f --- /dev/null +++ b/crates/dever-cli/build.rs @@ -0,0 +1,11 @@ +fn main() { + // The private compiler carries LLVM beside itself; project compilation must + // not require an author shell's library-path environment. + if std::env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("linux") { + // RPATH also resolves the packaged LLVM library's transitive imports. + for binary in ["dever", "dever-launcher", "deverd"] { + println!("cargo:rustc-link-arg-bin={binary}=-Wl,--disable-new-dtags"); + println!("cargo:rustc-link-arg-bin={binary}=-Wl,-rpath,$ORIGIN/lib"); + } + } +} diff --git a/crates/dever-cli/src/api.rs b/crates/dever-cli/src/api.rs new file mode 100644 index 0000000..3724e60 --- /dev/null +++ b/crates/dever-cli/src/api.rs @@ -0,0 +1,48 @@ +use std::fs::{self, OpenOptions}; +use std::io::{self, Write}; +use std::path::Path; +use std::process::ExitCode; + +use dever_core::hir::Program; + +pub(super) fn check_baseline(program: &Program, root: &Path) -> Result<(), String> { + let path = root.join("dever.api"); + let baseline = match fs::read_to_string(&path) { + Ok(baseline) => baseline, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(error) => { + return Err(format!( + "cannot read API baseline '{}': {error}", + path.display() + )); + } + }; + program + .check_api(&baseline) + .map_err(|error| format!("{}: {error}", path.display())) +} + +pub(super) fn write(program: &Program, output: Option<&Path>) -> Result<ExitCode, String> { + let snapshot = program.api_snapshot(); + match output { + None => io::stdout() + .lock() + .write_all(snapshot.as_bytes()) + .map_err(|error| error.to_string())?, + Some(path) => { + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .map_err(|error| { + format!("cannot create API snapshot '{}': {error}", path.display()) + })?; + file.write_all(snapshot.as_bytes()) + .and_then(|()| file.sync_all()) + .map_err(|error| { + format!("cannot write API snapshot '{}': {error}", path.display()) + })?; + } + } + Ok(ExitCode::SUCCESS) +} diff --git a/crates/dever-cli/src/bin/dever-launcher.rs b/crates/dever-cli/src/bin/dever-launcher.rs new file mode 100644 index 0000000..6ebca46 --- /dev/null +++ b/crates/dever-cli/src/bin/dever-launcher.rs @@ -0,0 +1,44 @@ +use std::ffi::OsString; +use std::io::{self, Write}; + +fn main() { + std::process::exit(run()); +} + +fn run() -> i32 { + let arguments: Vec<OsString> = std::env::args_os().skip(1).collect(); + if arguments.as_slice() == ["--dever-bootstrap-health"] { + return 0; + } + #[cfg(target_os = "linux")] + if let [flag, root] = arguments.as_slice() + && flag == "--dever-install" + { + return match dever_cli::toolchain::bootstrap::install(std::path::Path::new(root)) { + Ok(version) => { + println!("installed Dever {version}"); + 0 + } + Err(error) => fail(error), + }; + } + let executable = match std::env::current_exe() { + Ok(path) => path, + Err(error) => return fail(format!("cannot locate dever launcher: {error}")), + }; + match dever_cli::toolchain::execute(&executable, &arguments) { + Ok(dever_cli::toolchain::CommandResult::Exited(code)) => code, + Ok(dever_cli::toolchain::CommandResult::Message(message)) => { + match writeln!(io::stdout().lock(), "{message}") { + Ok(()) => 0, + Err(error) => fail(format!("cannot write command output: {error}")), + } + } + Err(error) => fail(error), + } +} + +fn fail(error: String) -> i32 { + let _ = writeln!(io::stderr().lock(), "{error}"); + 1 +} diff --git a/crates/dever-cli/src/bin/deverd.rs b/crates/dever-cli/src/bin/deverd.rs new file mode 100644 index 0000000..1f5caa5 --- /dev/null +++ b/crates/dever-cli/src/bin/deverd.rs @@ -0,0 +1,34 @@ +use std::env; +use std::path::PathBuf; + +fn main() { + if env::args_os().skip(1).collect::<Vec<_>>().as_slice() == ["--dever-bootstrap-health"] { + return; + } + let mut arguments = env::args_os().skip(1); + let Some(flag) = arguments.next() else { + eprintln!("Usage: deverd --root <machine-toolchain-root>"); + std::process::exit(2); + }; + if flag != "--root" { + eprintln!("Usage: deverd --root <machine-toolchain-root>"); + std::process::exit(2); + } + let Some(root) = arguments.next() else { + eprintln!("deverd: --root requires a machine toolchain root"); + std::process::exit(2); + }; + if arguments.next().is_some() { + eprintln!("deverd: unexpected arguments"); + std::process::exit(2); + } + let layout = dever_cli::toolchain::Layout::new(PathBuf::from(root)); + let result = layout + .initialize() + .and_then(|()| layout.validate_machine_permissions()) + .and_then(|()| dever_cli::toolchain::serve(&layout)); + if let Err(error) = result { + eprintln!("deverd: {error}"); + std::process::exit(1); + } +} diff --git a/crates/dever-cli/src/compile.rs b/crates/dever-cli/src/compile.rs new file mode 100644 index 0000000..583c0e5 --- /dev/null +++ b/crates/dever-cli/src/compile.rs @@ -0,0 +1,213 @@ +//! The default compiler pipeline: checked HIR -> LLVM object -> verified LLD +//! inputs. Artifact lifetime, cache integrity and output publication stay in core. + +use std::fs; +use std::io::Write; +use std::path::{Path, PathBuf}; + +use dever_backend_bridge::{BinaryResource, LinkKind, Target, emit_object_with_resources, link}; +use dever_core::hir::Program; +use dever_core::llvm::ResourceModule; +use dever_core::native::{EmbeddedResource, NativeProgram}; +use dever_core::source::SourceMap; +use dever_runtime::config::{CompilationBindings, RuntimeProfile}; +use sha2::{Digest, Sha256}; + +use dever_cli::toolchain::compilation::{CompileKind, CompileRequest}; +use dever_cli::toolchain::runtime_pack::RuntimePack; +use dever_cli::toolchain::{BuildTarget, Layout, Version}; + +pub fn application( + program: &Program, + sources: &SourceMap, + bindings: &CompilationBindings, + resources: &[EmbeddedResource], + target: BuildTarget, +) -> Result<NativeProgram, String> { + if let Some((layout, version)) = managed_compiler()? { + let request = CompileRequest::new( + version, + CompileKind::Application, + sources, + Some(bindings.clone()), + resources, + target, + )?; + return NativeProgram::from_verified_bytes(&dever_cli::toolchain::compile( + &layout, &request, + )?); + } + let profile = bindings.profile(); + compile(profile, target, || { + dever_core::llvm::emit_executable_with_resources(program, sources, resources) + }) +} + +pub fn test_suite(program: &Program, sources: &SourceMap) -> Result<NativeProgram, String> { + let target = BuildTarget::host()?; + if let Some((layout, version)) = managed_compiler()? { + let request = + CompileRequest::new(version, CompileKind::TestSuite, sources, None, &[], target)?; + return NativeProgram::from_verified_bytes(&dever_cli::toolchain::compile( + &layout, &request, + )?); + } + let profile = test_profile(program); + compile(profile, target, || { + dever_core::llvm::emit_test_executable(program, sources).map(|ir| ResourceModule { + ir, + resources: Vec::new(), + }) + }) +} + +pub(super) fn compiler_directory() -> Result<PathBuf, String> { + let executable = + std::env::current_exe().map_err(|error| format!("cannot locate compiler: {error}"))?; + executable + .parent() + .map(Path::to_owned) + .ok_or_else(|| "compiler has no parent directory".into()) +} + +pub(super) fn managed_compiler() -> Result<Option<(Layout, String)>, String> { + let executable = + std::env::current_exe().map_err(|error| format!("cannot locate compiler: {error}"))?; + let directory = executable + .parent() + .ok_or("compiler has no parent directory")?; + let managed = executable.file_stem().and_then(|name| name.to_str()) == Some("dever-core") + || directory + .parent() + .and_then(Path::file_name) + .is_some_and(|name| name == "versions"); + if !managed { + return Ok(None); + } + let versions = directory + .parent() + .filter(|path| path.file_name().is_some_and(|name| name == "versions")) + .ok_or( + "managed compiler must be installed under <toolchain>/versions/<version>/dever-core", + )?; + if executable.file_stem().and_then(|name| name.to_str()) != Some("dever-core") { + return Err("managed compiler must use the installed dever-core executable".into()); + } + let version = directory + .file_name() + .and_then(|name| name.to_str()) + .ok_or("invalid managed compiler version directory")?; + Version::parse(version)?; + if version != env!("CARGO_PKG_VERSION") { + return Err("managed compiler version does not match its installation directory".into()); + } + let root = versions + .parent() + .ok_or("managed compiler has no toolchain root")?; + Ok(Some((Layout::new(root), version.to_owned()))) +} + +fn test_profile(program: &Program) -> RuntimeProfile { + RuntimeProfile { + sqlite: program.tests().iter().any(|test| test.uses_database()), + postgres: false, + } +} + +pub(super) fn worker_compile( + program: &Program, + sources: &SourceMap, + request: &CompileRequest, + build: &Path, + output: &Path, +) -> Result<(), String> { + let resources = request.embedded_resources(); + let (profile, ir) = match request.kind { + CompileKind::Application => { + let profile = program.validate_database_bindings( + request + .bindings + .as_ref() + .ok_or("application bindings are missing")?, + )?; + let ir = + dever_core::llvm::emit_executable_with_resources(program, sources, &resources)?; + (profile, ir) + } + CompileKind::TestSuite => ( + test_profile(program), + ResourceModule { + ir: dever_core::llvm::emit_test_executable(program, sources)?, + resources: Vec::new(), + }, + ), + }; + let pack = RuntimePack::load(&compiler_directory()?, profile, request.target)?; + link_ir(&ir, &pack, build, output) +} + +fn compile<'a>( + profile: RuntimeProfile, + target: BuildTarget, + emit: impl FnOnce() -> Result<ResourceModule<'a>, String>, +) -> Result<NativeProgram, String> { + let executable = + std::env::current_exe().map_err(|error| format!("cannot locate compiler: {error}"))?; + let directory = compiler_directory()?; + let pack = RuntimePack::load(&directory, profile, target)?; + let ir = emit()?; + let compiler = dever_cli::toolchain::sha256_file(&executable)?; + let mut identity = Sha256::new(); + for part in [ + "dever-llvm-executable-v1", + &compiler, + &pack.identity, + target.platform(), + &ir.ir, + ] { + identity.update((part.len() as u64).to_le_bytes()); + identity.update(part.as_bytes()); + } + dever_core::native::compile_artifact( + &directory.join("cache/native"), + &format!("{:x}", identity.finalize()), + |build, output| link_ir(&ir, &pack, build, output), + ) +} + +fn link_ir( + module: &ResourceModule<'_>, + pack: &RuntimePack, + build: &Path, + output: &Path, +) -> Result<(), String> { + let target = Target::ALL + .into_iter() + .find(|target| target.triple() == pack.target.triple()) + .ok_or("native runtime target is not supported by the compiler bridge")?; + let resources = module + .resources + .iter() + .map(|resource| BinaryResource { + symbol: &resource.symbol, + bytes: resource.bytes, + }) + .collect::<Vec<_>>(); + let object = emit_object_with_resources(&module.ir, target, &resources)?; + let path = build.join("application.o"); + fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&path) + .and_then(|mut file| file.write_all(&object)) + .map_err(|error| format!("cannot write application object: {error}"))?; + let snapshot = pack.snapshot(build)?; + let inputs = snapshot + .start + .into_iter() + .chain(std::iter::once(path)) + .chain(snapshot.libraries) + .chain(snapshot.end) + .collect::<Vec<_>>(); + link(target, LinkKind::StaticExecutable, &inputs, output, None) +} diff --git a/crates/dever-cli/src/format.rs b/crates/dever-cli/src/format.rs new file mode 100644 index 0000000..97078b2 --- /dev/null +++ b/crates/dever-cli/src/format.rs @@ -0,0 +1,153 @@ +use std::fs::{self, OpenOptions}; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::ExitCode; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use dever_core::source::SourceMap; + +pub(super) fn execute(root: &Path, sources: &SourceMap, check: bool) -> Result<ExitCode, String> { + let mut changes = Vec::new(); + let mut diagnostics = Vec::new(); + for source in sources.files() { + match dever_core::format::format(source) { + Ok(formatted) if formatted != source.text() => { + changes.push((source.path(), destination(root, source), formatted)); + } + Ok(_) => {} + Err(errors) => diagnostics.extend(errors), + } + } + if !diagnostics.is_empty() { + return Err(diagnostics + .iter() + .map(|error| error.render(sources)) + .collect()); + } + if check { + for (path, _, _) in &changes { + writeln!(std::io::stdout().lock(), "{}", path.display()) + .map_err(|error| error.to_string())?; + } + return Ok(if changes.is_empty() { + ExitCode::SUCCESS + } else { + ExitCode::FAILURE + }); + } + + // Prepare all replacements before changing any source. A syntax, permission or staging + // failure leaves the original files intact; each final replacement is an atomic rename. + validate_snapshot(root, sources)?; + let mut staged = Vec::new(); + for (_, destination, formatted) in changes { + staged.push(Replacement::prepare(destination, &formatted)?); + } + validate_snapshot(root, sources)?; + for replacement in &mut staged { + replacement.commit()?; + } + Ok(ExitCode::SUCCESS) +} + +fn validate_snapshot(root: &Path, sources: &SourceMap) -> Result<(), String> { + // Reuse loading rules so added/deleted paths and new symlinks also invalidate the snapshot. + let current = + SourceMap::load_project(&root.join("module"), &root.join("test")).map_err(|errors| { + errors + .iter() + .map(ToString::to_string) + .collect::<Vec<_>>() + .join("\n") + })?; + if !current + .files() + .iter() + .map(|source| source.path()) + .eq(sources.files().iter().map(|source| source.path())) + { + return Err(format!( + "{}: source file set changed during formatting", + root.display() + )); + } + for (current, source) in current.files().iter().zip(sources.files()) { + if current.text() != source.text() { + return Err(format!( + "{}: source changed during formatting", + destination(root, source).display() + )); + } + } + Ok(()) +} + +fn destination(root: &Path, source: &dever_core::source::SourceFile) -> PathBuf { + if source.is_test() { + root.join(source.path()) + } else { + root.join("module").join(source.path()) + } +} + +struct Replacement { + destination: PathBuf, + temporary: Option<PathBuf>, +} + +impl Replacement { + fn prepare(destination: PathBuf, formatted: &str) -> Result<Self, String> { + static NEXT: AtomicUsize = AtomicUsize::new(0); + let metadata = + fs::metadata(&destination).map_err(|error| file_error(&destination, error))?; + if metadata.permissions().readonly() { + return Err(format!("{}: source is read-only", destination.display())); + } + let parent = destination.parent().expect("source files have a parent"); + let (temporary, mut file) = loop { + let temporary = parent.join(format!( + ".dever-fmt-{}-{}.tmp", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + match OpenOptions::new() + .write(true) + .create_new(true) + .open(&temporary) + { + Ok(file) => break (temporary, file), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => return Err(file_error(&destination, error)), + } + }; + let replacement = Self { + destination, + temporary: Some(temporary), + }; + file.write_all(formatted.as_bytes()) + .and_then(|()| file.set_permissions(metadata.permissions())) + .and_then(|()| file.sync_all()) + .map_err(|error| file_error(&replacement.destination, error))?; + Ok(replacement) + } + + fn commit(&mut self) -> Result<(), String> { + let temporary = self.temporary.as_ref().expect("replacement is pending"); + fs::rename(temporary, &self.destination) + .map_err(|error| file_error(&self.destination, error))?; + self.temporary = None; + Ok(()) + } +} + +impl Drop for Replacement { + fn drop(&mut self) { + if let Some(temporary) = &self.temporary { + let _ = fs::remove_file(temporary); + } + } +} + +fn file_error(path: &Path, error: std::io::Error) -> String { + format!("{}: cannot format source: {error}", path.display()) +} diff --git a/crates/dever-cli/src/lib.rs b/crates/dever-cli/src/lib.rs new file mode 100644 index 0000000..350a036 --- /dev/null +++ b/crates/dever-cli/src/lib.rs @@ -0,0 +1,5 @@ +pub mod libs; +pub mod packages; +pub mod project; +pub mod toolchain; +pub mod workers; diff --git a/crates/dever-cli/src/libs.rs b/crates/dever-cli/src/libs.rs new file mode 100644 index 0000000..3269189 --- /dev/null +++ b/crates/dever-cli/src/libs.rs @@ -0,0 +1,2620 @@ +//! Deterministic, offline-first external library resolution. +//! +//! This module owns the project lock contract and resolver orchestration. It +//! deliberately does not know how a machine cache is authenticated: released +//! clients will supply an implementation of `ArtifactStore` through `deverd`. + +use std::collections::{BTreeMap, BTreeSet, VecDeque}; +use std::fs; +use std::path::{Component, Path, PathBuf}; +use std::str::FromStr; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +pub mod build; +mod mutation; +mod registry; +pub(crate) use mutation::ProjectMutation; +pub use registry::npm; +pub(crate) use registry::python_dependencies; +pub use registry::sumdb; +pub use registry::{HttpRegistry, RegistryResolver, RegistryRuntime, RegistryTransport}; +pub(crate) use registry::{InstalledRegistryPack, archive_files}; + +const LOCK_FORMAT: &str = "dever-lock-v5"; +const TOOLCHAIN: &str = env!("CARGO_PKG_VERSION"); + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Ecosystem { + Pip, + Npm, + Go, +} + +impl Ecosystem { + pub fn as_str(&self) -> &'static str { + match self { + Self::Pip => "pip", + Self::Npm => "npm", + Self::Go => "go", + } + } +} + +impl FromStr for Ecosystem { + type Err = String; + + fn from_str(value: &str) -> Result<Self, Self::Err> { + match value { + "pip" => Ok(Self::Pip), + "npm" => Ok(Self::Npm), + "go" => Ok(Self::Go), + _ => Err(format!("unsupported lib ecosystem '{value}'")), + } + } +} + +/// The provider boundary is deliberately separate from lock-file handling. +/// A released client can install a real registry/runtime provider without +/// teaching the Dever language about pip, npm, or Go details. The current +/// development distribution only ships the deterministic fixture provider. +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct ProviderManifest { + pub ecosystem: Ecosystem, + pub resolver: String, + pub runtime_pack: String, + pub sdk: String, + pub available: bool, +} + +impl ProviderManifest { + pub fn fixture(ecosystem: Ecosystem) -> Self { + let (resolver, runtime_pack, sdk) = match ecosystem { + Ecosystem::Pip => ("fixture-pip", "fixture-cpython", "fixture-python-worker"), + Ecosystem::Npm => ("fixture-npm", "fixture-node", "fixture-javascript-worker"), + Ecosystem::Go => ("fixture-go", "fixture-go", "fixture-go-worker"), + }; + Self { + ecosystem, + resolver: resolver.into(), + runtime_pack: runtime_pack.into(), + sdk: sdk.into(), + available: true, + } + } + + /// Metadata for a production provider whose signed runtime is not installed. + pub fn unavailable(ecosystem: Ecosystem) -> Self { + let (resolver, runtime_pack, sdk) = match ecosystem { + Ecosystem::Pip => ("pip-registry", "managed-cpython", "python-worker-sdk"), + Ecosystem::Npm => ("npm-registry", "managed-node", "javascript-worker-sdk"), + Ecosystem::Go => ("go-module", "managed-go", "go-worker-sdk"), + }; + Self { + ecosystem, + resolver: resolver.into(), + runtime_pack: runtime_pack.into(), + sdk: sdk.into(), + available: false, + } + } + + pub fn blocked_message(&self, spec: &LibSpec) -> String { + format!( + "external Lib provider '{}' for {} requires a signed {} runtime pack and the managed artifact service; run/build never use PATH, environment variables, or host package managers", + self.resolver, + spec.key(), + self.runtime_pack, + ) + } +} + +/// Resolver implementations own registry and runtime-specific behavior. +/// Lock normalization, dependency validation, and artifact verification stay +/// in this module and are shared by every provider. +pub trait LibResolver { + fn manifest(&self) -> ProviderManifest; + fn resolve(&self, requested: &[LibSpec]) -> Result<LockFile, String>; + fn go_verifier(&self) -> sumdb::Verifier { + sumdb::Verifier::official() + } +} + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LibSpec { + pub ecosystem: Ecosystem, + pub name: String, + pub version: String, +} + +impl LibSpec { + pub fn key(&self) -> String { + format!("{}:{}@{}", self.ecosystem.as_str(), self.name, self.version) + } + + /// Extras select dependency edges, not a different installed distribution. + pub fn distribution_name(&self) -> &str { + distribution_name(&self.ecosystem, &self.name) + } +} + +fn distribution_name<'a>(ecosystem: &Ecosystem, name: &'a str) -> &'a str { + if *ecosystem == Ecosystem::Pip { + name.split_once('[').map_or(name, |(base, _)| base) + } else { + name + } +} + +fn python_name(name: &str) -> Result<(String, Vec<pep508_rs::ExtraName>), String> { + let (base, extras) = match name.split_once('[') { + Some((base, suffix)) => { + let extras = suffix + .strip_suffix(']') + .ok_or("invalid pip extras suffix")?; + let extras = extras + .split(',') + .map(python_extra) + .collect::<Result<BTreeSet<_>, _>>()?; + (base, extras.into_iter().collect::<Vec<_>>()) + } + None => (name, Vec::new()), + }; + if base.is_empty() { + return Err("pip distribution name must not be empty".into()); + } + let base = base + .parse::<pep508_rs::PackageName>() + .map_err(|error| format!("invalid pip lib name: {error}"))? + .to_string(); + let canonical = if extras.is_empty() { + base + } else { + format!( + "{base}[{}]", + extras + .iter() + .map(ToString::to_string) + .collect::<Vec<_>>() + .join(",") + ) + }; + Ok((canonical, extras)) +} + +fn python_extra(extra: &str) -> Result<pep508_rs::ExtraName, String> { + // pep508_rs normalization accepts an empty string; selection and metadata + // still require a nonempty declared extra name. + if extra.is_empty() { + return Err("pip extra name must not be empty".into()); + } + extra + .parse() + .map_err(|error| format!("invalid pip extra: {error}")) +} + +impl FromStr for LibSpec { + type Err = String; + + fn from_str(value: &str) -> Result<Self, Self::Err> { + let (ecosystem, rest) = value + .split_once(':') + .ok_or("lib spec must be <pip|npm|go>:<name>@<version>")?; + let ecosystem = ecosystem.parse()?; + let (name, version) = rest + .rsplit_once('@') + .ok_or("lib spec must include an exact @version")?; + let name = if ecosystem == Ecosystem::Pip { + python_name(name)?.0 + } else { + name.to_owned() + }; + validate_name(&ecosystem, &name)?; + validate_version(&ecosystem, version)?; + let version = if ecosystem == Ecosystem::Go { + version.strip_prefix('v').unwrap_or(version) + } else { + version + }; + Ok(Self { + ecosystem, + name, + version: version.to_owned(), + }) + } +} + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LockedDependency { + pub spec: LibSpec, +} + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LockedArtifact { + pub target: String, + pub path: String, + pub bytes: u64, + pub sha256: String, +} + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RuntimePack { + pub name: String, + pub version: String, + pub sha256: String, +} + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LockedLib { + pub spec: LibSpec, + pub dependencies: Vec<LockedDependency>, + pub runtime: RuntimePack, + pub artifacts: Vec<LockedArtifact>, + pub schema: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub build: Option<String>, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LockFile { + pub format: String, + pub toolchain: String, + pub libs: Vec<LockedLib>, + pub workers: Vec<LockedWorker>, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub go_sumdb: Vec<sumdb::Evidence>, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub npm: Vec<npm::Environment>, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub builds: Vec<build::Receipt>, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub packages: Vec<crate::packages::LockedPackage>, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LockedWorker { + pub port: String, + pub adapter: String, + pub ecosystem: String, + pub runtime: Option<RuntimePack>, + pub entry: String, + pub schema: String, + pub capabilities: Vec<String>, + pub operations: Vec<String>, + pub libs: Vec<LibSpec>, +} + +impl LockedWorker { + fn from_contract(contract: dever_core::hir::ExternalWorkerContract) -> Result<Self, String> { + let mut libs = contract + .libs + .iter() + .map(|request| request.parse()) + .collect::<Result<Vec<LibSpec>, _>>()?; + libs.sort(); + libs.dedup(); + Ok(Self { + port: contract.port, + adapter: contract.adapter, + ecosystem: contract.ecosystem, + runtime: None, + entry: contract.entry, + schema: contract.schema, + capabilities: contract.capabilities, + operations: contract.operations, + libs, + }) + } +} + +impl LockFile { + pub fn new(mut libs: Vec<LockedLib>) -> Result<Self, String> { + normalize_libs(&mut libs)?; + Ok(Self { + format: LOCK_FORMAT.into(), + toolchain: TOOLCHAIN.into(), + libs, + workers: Vec::new(), + go_sumdb: Vec::new(), + npm: Vec::new(), + builds: Vec::new(), + packages: Vec::new(), + }) + } + + pub fn encode(&self) -> Result<Vec<u8>, String> { + if self.format != LOCK_FORMAT { + return Err("unsupported dever.lock format".into()); + } + let mut normalized = self.clone(); + normalize_libs(&mut normalized.libs)?; + normalize_workers(&mut normalized.workers)?; + npm::normalize(&mut normalized.npm)?; + build::normalize(&mut normalized.builds)?; + crate::packages::normalize_lock(&mut normalized.packages)?; + let mut bytes = + serde_json::to_vec_pretty(&normalized).map_err(|error| error.to_string())?; + bytes.push(b'\n'); + // Publication and offline decoding must accept exactly the same bounds. + let text = std::str::from_utf8(&bytes).expect("JSON serialization is UTF-8"); + dever_runtime::wire::parse(text).map_err(|error| format!("invalid dever.lock: {error}"))?; + Ok(bytes) + } + + pub fn decode(bytes: &[u8]) -> Result<Self, String> { + let text = std::str::from_utf8(bytes).map_err(|_| "dever.lock must contain UTF-8 JSON")?; + dever_runtime::wire::parse(text).map_err(|error| format!("invalid dever.lock: {error}"))?; + let lock: Self = serde_json::from_slice(bytes) + .map_err(|error| format!("invalid dever.lock: {error}"))?; + if lock.format != LOCK_FORMAT { + return Err(format!( + "unsupported dever.lock format '{}'; expected {LOCK_FORMAT}", + lock.format + )); + } + if lock.toolchain != TOOLCHAIN { + return Err(format!( + "dever.lock was created by toolchain {}, current toolchain is {TOOLCHAIN}", + lock.toolchain + )); + } + let canonical = lock.encode()?; + if canonical != bytes { + return Err( + "dever.lock is not canonical; run 'dever lib update <project-root>'".into(), + ); + } + Ok(lock) + } + + pub fn write_atomic(&self, project_root: &Path) -> Result<(), String> { + let path = project_root.join("dever.lock"); + if path.exists() + && fs::symlink_metadata(&path) + .map_err(|e| e.to_string())? + .file_type() + .is_symlink() + { + return Err("dever.lock must not be a symbolic link".into()); + } + let parent = path.parent().ok_or("project root has no parent")?; + fs::create_dir_all(parent).map_err(|e| format!("cannot create project root: {e}"))?; + let temporary = parent.join(format!( + ".dever.lock.{}-{}.tmp", + std::process::id(), + unique_suffix() + )); + write_new_file(&temporary, &self.encode()?) + .map_err(|e| format!("cannot stage dever.lock: {e}"))?; + fs::rename(&temporary, &path).map_err(|e| { + let _ = fs::remove_file(&temporary); + format!("cannot replace dever.lock: {e}") + }) + } +} + +#[derive(Clone, Debug)] +pub struct FixturePackage { + pub spec: LibSpec, + pub dependencies: Vec<LibSpec>, + pub runtime: RuntimePack, + pub artifacts: Vec<FixtureArtifact>, + pub schema: String, +} + +impl FixturePackage { + fn locked(&self) -> LockedLib { + let mut artifacts = self + .artifacts + .iter() + .map(|artifact| LockedArtifact { + target: artifact.target.clone(), + path: artifact.path.clone(), + bytes: artifact.bytes.len() as u64, + sha256: sha256(&artifact.bytes), + }) + .collect::<Vec<_>>(); + artifacts.sort(); + let mut dependencies = self + .dependencies + .iter() + .cloned() + .map(|spec| LockedDependency { spec }) + .collect::<Vec<_>>(); + dependencies.sort(); + LockedLib { + spec: self.spec.clone(), + dependencies, + runtime: self.runtime.clone(), + artifacts, + schema: self.schema.clone(), + build: None, + } + } +} + +#[derive(Clone, Debug)] +pub struct FixtureArtifact { + pub target: String, + pub path: String, + pub bytes: Vec<u8>, +} + +#[derive(Clone, Debug, Default)] +pub struct FixtureRegistry { + packages: BTreeMap<LibSpec, FixturePackage>, +} + +impl FixtureRegistry { + pub fn register(&mut self, package: FixturePackage) -> Result<(), String> { + if self + .packages + .insert(package.spec.clone(), package) + .is_some() + { + return Err("duplicate fixture lib spec".into()); + } + Ok(()) + } + + pub fn package(&self, spec: &LibSpec) -> Option<&FixturePackage> { + self.packages.get(spec) + } + + pub fn resolve(&self, requested: &[LibSpec]) -> Result<LockFile, String> { + let mut queue = requested.iter().cloned().collect::<VecDeque<_>>(); + let mut resolved = BTreeMap::<LibSpec, FixturePackage>::new(); + let mut selected = BTreeMap::<(Ecosystem, String), String>::new(); + while let Some(spec) = queue.pop_front() { + if resolved.contains_key(&spec) { + continue; + } + let key = (spec.ecosystem.clone(), spec.distribution_name().to_owned()); + if let Some(version) = selected.get(&key) + && version != &spec.version + { + return Err(format!( + "lib version conflict for {}:{}: {version} and {}", + spec.ecosystem.as_str(), + spec.name, + spec.version + )); + } + selected.insert(key, spec.version.clone()); + let package = self + .packages + .get(&spec) + .cloned() + .ok_or_else(|| format!("fixture registry has no {}", spec.key()))?; + queue.extend(package.dependencies.iter().cloned()); + resolved.insert(spec, package); + } + let libs = resolved + .into_values() + .map(|package| package.locked()) + .collect::<Vec<_>>(); + LockFile::new(libs) + } + + pub fn builtin() -> Self { + let mut registry = Self::default(); + // Built-in fixtures are deterministic and intentionally do not access a + // registry, shell, PATH or language-specific environment variables. + for (ecosystem, name) in [ + (Ecosystem::Pip, "fixture"), + (Ecosystem::Npm, "fixture"), + (Ecosystem::Go, "example.com/fixture/lib"), + ] { + let spec = LibSpec { + ecosystem: ecosystem.clone(), + name: name.into(), + version: "1.0.0".into(), + }; + let bytes = format!("dever fixture {}\n", spec.key()).into_bytes(); + let _ = registry.register(FixturePackage { + spec: spec.clone(), + dependencies: Vec::new(), + runtime: RuntimePack { + name: match ecosystem { + Ecosystem::Pip => "cpython", + Ecosystem::Npm => "node", + Ecosystem::Go => "go", + } + .into(), + version: "managed-1".into(), + sha256: sha256(format!("runtime:{}", ecosystem.as_str()).as_bytes()), + }, + artifacts: vec![FixtureArtifact { + target: "host".into(), + path: format!("{}/1.0.0/package.bin", ecosystem.as_str()), + bytes, + }], + schema: format!("fixture-schema-{}-v1", ecosystem.as_str()), + }); + } + registry + } +} + +impl LibResolver for FixtureRegistry { + fn manifest(&self) -> ProviderManifest { + // The fixture registry is intentionally multi-ecosystem, so expose a + // generic manifest only for diagnostics. Individual package specs + // still select their ecosystem provider below. + ProviderManifest { + ecosystem: Ecosystem::Pip, + resolver: "fixture-registry".into(), + runtime_pack: "fixture-managed".into(), + sdk: "fixture-worker".into(), + available: true, + } + } + + fn resolve(&self, requested: &[LibSpec]) -> Result<LockFile, String> { + FixtureRegistry::resolve(self, requested) + } +} + +pub fn provider_manifest(ecosystem: Ecosystem) -> ProviderManifest { + ProviderManifest::unavailable(ecosystem) +} + +pub fn resolve_workers( + resolver: &impl LibResolver, + declared: &[LibSpec], + mut workers: Vec<LockedWorker>, +) -> Result<LockFile, String> { + normalize_workers(&mut workers)?; + let mut resolved = BTreeMap::<LibSpec, LockedLib>::new(); + let mut go_sumdb = Vec::new(); + let mut npm = Vec::new(); + let mut builds = Vec::new(); + let mut roots = vec![declared.to_vec()]; + roots.extend(workers.iter().map(|worker| worker.libs.clone())); + for requested in roots { + let result = resolver.resolve(&requested)?; + go_sumdb.extend(result.go_sumdb); + npm.extend(result.npm); + builds.extend(result.builds); + for lib in result.libs { + if let Some(previous) = resolved.get(&lib.spec) { + if previous != &lib { + return Err(format!( + "inconsistent resolved metadata for {}", + lib.spec.key() + )); + } + } else { + resolved.insert(lib.spec.clone(), lib); + } + } + } + let mut lock = LockFile::new(resolved.into_values().collect())?; + lock.workers = workers; + lock.go_sumdb = go_sumdb; + npm::normalize(&mut npm)?; + lock.npm = npm; + build::normalize(&mut builds)?; + lock.builds = builds; + doctor_with_go_verifier(&lock, &resolver.go_verifier())?; + Ok(lock) +} + +pub(crate) fn resolve_project( + project_root: &Path, + requested: &[LibSpec], + workers: Vec<LockedWorker>, + target: crate::toolchain::BuildTarget, +) -> Result<LockFile, String> { + let prior_evidence = if project_root.join("dever.lock").exists() { + let lock = read_lock(&project_root.join("dever.lock"))?; + doctor(&lock)?; + lock.go_sumdb + } else { + Vec::new() + }; + let registry = FixtureRegistry::builtin(); + let all = requested + .iter() + .chain(workers.iter().flat_map(|worker| &worker.libs)) + .collect::<Vec<_>>(); + if all.iter().all(|spec| registry.package(spec).is_some()) + && workers.iter().all(|worker| worker.ecosystem == "exec") + { + let mut lock = resolve_workers(®istry, requested, workers)?; + lock.go_sumdb = prior_evidence; + lock.encode()?; + return Ok(lock); + } + let managed = InstalledRegistry::current(target)?; + let mut runtimes = BTreeMap::new(); + let mut ecosystems = all + .iter() + .filter(|spec| registry.package(spec).is_none()) + .map(|spec| spec.ecosystem.clone()) + .collect::<BTreeSet<_>>(); + for worker in workers.iter().filter(|worker| worker.ecosystem != "exec") { + ecosystems.insert(worker.ecosystem.parse()?); + } + for ecosystem in ecosystems { + runtimes.insert(ecosystem.clone(), managed.runtime(ecosystem)?.0); + } + for worker in workers.iter().filter(|worker| worker.ecosystem != "exec") { + let ecosystem: Ecosystem = worker.ecosystem.parse()?; + if worker.runtime.as_ref() != runtimes.get(&ecosystem).map(|runtime| &runtime.pack) { + return Err(format!( + "managed Worker '{}.{}' runtime identity differs from signed release", + worker.port, worker.adapter + )); + } + } + let store = ManagedArtifactStore { + layout: managed.layout.clone(), + }; + let transport = HttpRegistry::official(); + let verified = sumdb::verify_chain(&prior_evidence, &sumdb::Verifier::official())?; + let previous = (!verified.checkpoint.is_empty()).then_some(verified.checkpoint); + let build = build::Session::new(&managed); + let resolver = RegistryResolver { + build: Some(&build), + transport: &transport, + store: &store, + runtimes, + go_sumdb: sumdb::ChecksumDatabase::official(previous)?, + }; + let combined = ProjectResolver { + fixtures: ®istry, + registry: &resolver, + }; + let mut lock = resolve_workers(&combined, requested, workers)?; + if lock.go_sumdb.is_empty() { + lock.go_sumdb = prior_evidence; + } + lock.encode()?; + Ok(lock) +} + +struct ProjectResolver<'a> { + fixtures: &'a FixtureRegistry, + registry: &'a RegistryResolver<'a>, +} + +impl LibResolver for ProjectResolver<'_> { + fn manifest(&self) -> ProviderManifest { + self.registry.manifest() + } + + fn resolve(&self, requested: &[LibSpec]) -> Result<LockFile, String> { + let (fixture, real): (Vec<_>, Vec<_>) = requested + .iter() + .cloned() + .partition(|spec| self.fixtures.package(spec).is_some()); + let mut lock = self.registry.resolve(&real)?; + lock.libs.extend(self.fixtures.resolve(&fixture)?.libs); + normalize_libs(&mut lock.libs)?; + Ok(lock) + } +} + +#[derive(Clone)] +struct InstalledRegistry { + layout: crate::toolchain::Layout, + version_root: PathBuf, + target: crate::toolchain::BuildTarget, +} + +impl build::Inputs for InstalledRegistry { + fn diagnostic(&self, output: &[u8]) { + if !output.is_empty() { + eprintln!("{}", String::from_utf8_lossy(output)); + } + } + + fn runtime(&self, ecosystem: Ecosystem) -> Result<(RegistryRuntime, Vec<u8>), String> { + self.runtime(ecosystem) + } + + fn tools(&self, ecosystem: Ecosystem) -> Result<(build::pack::Descriptor, Vec<u8>), String> { + let directory = PathBuf::from("build") + .join(ecosystem.as_str()) + .join(self.target.platform()); + let metadata = self.signed_artifact(&directory.join("manifest.json"))?; + let descriptor: build::pack::Descriptor = serde_json::from_slice(&metadata) + .map_err(|error| format!("invalid signed build descriptor: {error}"))?; + if descriptor.ecosystem != ecosystem { + return Err("signed build pack ecosystem differs".into()); + } + let bytes = self.signed_artifact(&directory.join("build.pack"))?; + Ok((descriptor, bytes)) + } + + fn assets(&self) -> Result<Vec<(String, Vec<u8>)>, String> { + self.sandbox_resources(crate::toolchain::BuildTarget::host()?)? + .into_iter() + .map(|resource| { + let path = resource + .path + .strip_prefix("sandbox/") + .ok_or("invalid sandbox resource path")? + .to_owned(); + Ok((path, resource.bytes)) + }) + .collect() + } +} + +impl InstalledRegistry { + fn current(target: crate::toolchain::BuildTarget) -> Result<Self, String> { + let executable = + fs::canonicalize(std::env::current_exe().map_err(|error| error.to_string())?) + .map_err(|error| format!("cannot resolve Dever core executable: {error}"))?; + let version_root = executable + .parent() + .ok_or("Dever core executable has no version directory")?; + let version = version_root + .file_name() + .and_then(|name| name.to_str()) + .ok_or("Dever core executable is not in a managed version")?; + let version = crate::toolchain::Version::parse(version) + .map_err(|_| "real Lib providers require a signed managed Dever release".to_owned())?; + let versions = version_root + .parent() + .ok_or("Dever version directory has no parent")?; + if versions.file_name().and_then(|name| name.to_str()) != Some("versions") { + return Err("real Lib providers require a signed managed Dever release".into()); + } + let layout = crate::toolchain::Layout::new( + versions.parent().ok_or("Dever machine root is missing")?, + ); + layout.validate_machine_permissions()?; + let installed = + crate::toolchain::MachineManager::new(layout.clone()).resolve_core(&version)?; + if fs::canonicalize(installed).map_err(|error| error.to_string())? != executable { + return Err("running Dever core is not the verified managed release".into()); + } + Ok(Self { + layout, + version_root: version_root.to_path_buf(), + target, + }) + } + + fn runtime(&self, ecosystem: Ecosystem) -> Result<(RegistryRuntime, Vec<u8>), String> { + let directory = PathBuf::from("runtime") + .join(ecosystem.as_str()) + .join(self.target.platform()); + let manifest_bytes = self.signed_artifact(&directory.join("manifest.json"))?; + let manifest: registry::InstalledRegistryPack = serde_json::from_slice(&manifest_bytes) + .map_err(|error| { + format!( + "invalid signed {} runtime manifest: {error}", + ecosystem.as_str() + ) + })?; + manifest.validate(&ecosystem, self.target.platform())?; + let bytes = self.signed_artifact(&directory.join("runtime.pack"))?; + if bytes.is_empty() + || bytes.len() > 256 * 1024 * 1024 + || sha256(&bytes) != manifest.runtime.pack.sha256 + { + return Err(format!( + "signed {} runtime pack digest or size is invalid", + ecosystem.as_str() + )); + } + Ok((manifest.runtime, bytes)) + } + + fn signed_artifact(&self, relative: &Path) -> Result<Vec<u8>, String> { + let manifest: crate::toolchain::ReleaseManifest = serde_json::from_slice( + &fs::read(self.version_root.join("manifest.json")) + .map_err(|error| error.to_string())?, + ) + .map_err(|error| format!("invalid signed release manifest: {error}"))?; + let name = relative.to_str().ok_or("runtime path is not UTF-8")?; + let identity = manifest + .artifacts + .iter() + .find(|artifact| artifact.path == name) + .ok_or_else(|| format!("signed release does not include runtime artifact '{name}'"))?; + let bytes = fs::read(self.version_root.join(relative)) + .map_err(|error| format!("cannot read signed runtime artifact '{name}': {error}"))?; + if bytes.len() as u64 != identity.bytes || sha256(&bytes) != identity.sha256 { + return Err(format!( + "signed runtime artifact '{name}' failed identity verification" + )); + } + Ok(bytes) + } + + fn sandbox_resources( + &self, + target: crate::toolchain::BuildTarget, + ) -> Result<Vec<dever_core::native::EmbeddedResource>, String> { + let manifest: crate::toolchain::ReleaseManifest = serde_json::from_slice( + &fs::read(self.version_root.join("manifest.json")) + .map_err(|error| error.to_string())?, + ) + .map_err(|error| format!("invalid signed release manifest: {error}"))?; + let prefix = format!("sandbox/{}/", target.platform()); + let mut resources = Vec::new(); + let mut files = Vec::new(); + for artifact in &manifest.artifacts { + let Some(relative) = artifact.path.strip_prefix(&prefix) else { + continue; + }; + let bytes = self.signed_artifact(Path::new(&artifact.path))?; + files.push((relative.to_owned(), bytes.clone())); + resources.push(dever_core::native::EmbeddedResource { + path: format!("sandbox/{relative}"), + sha256: artifact.sha256.clone(), + bytes, + executable: dever_sandbox::asset_executable(relative), + }); + } + dever_sandbox::validate_assets_for_target(&files, target.platform())?; + Ok(resources) + } +} + +pub(crate) struct ManagedArtifactStore { + layout: crate::toolchain::Layout, +} + +pub(crate) fn managed_artifact_store() -> Result<ManagedArtifactStore, String> { + Ok(ManagedArtifactStore { + layout: InstalledRegistry::current(crate::toolchain::BuildTarget::host()?)?.layout, + }) +} + +impl ArtifactStore for ManagedArtifactStore { + fn has(&self, _sha256: &str, _target: &str) -> Result<bool, String> { + Err("managed artifact lookup requires the locked byte length".into()) + } + + fn publish(&self, bytes: &[u8], _target: &str) -> Result<String, String> { + let receipt = crate::toolchain::artifact_put(&self.layout, bytes)?; + Ok(receipt.sha256) + } + + fn get(&self, _sha256: &str, _target: &str) -> Result<Option<Vec<u8>>, String> { + Err("managed artifact lookup requires the locked byte length".into()) + } + + fn get_exact( + &self, + sha256: &str, + bytes: u64, + _target: &str, + ) -> Result<Option<Vec<u8>>, String> { + crate::toolchain::artifact_get(&self.layout, sha256, bytes).map(Some) + } +} + +pub trait ArtifactStore { + fn has(&self, sha256: &str, target: &str) -> Result<bool, String>; + fn publish(&self, bytes: &[u8], target: &str) -> Result<String, String>; + fn get(&self, sha256: &str, target: &str) -> Result<Option<Vec<u8>>, String>; + + fn get_exact( + &self, + sha256: &str, + _bytes: u64, + target: &str, + ) -> Result<Option<Vec<u8>>, String> { + self.get(sha256, target) + } + + fn verify_exact( + &self, + sha256: &str, + expected_bytes: u64, + target: &str, + ) -> Result<Vec<u8>, String> { + let bytes = self + .get_exact(sha256, expected_bytes, target)? + .ok_or_else(|| format!("artifact {sha256} for target {target} is missing"))?; + if bytes.len() as u64 != expected_bytes { + return Err(format!( + "artifact {sha256} for target {target} has size drift" + )); + } + if sha256_bytes(&bytes) != sha256 { + return Err(format!( + "artifact {sha256} for target {target} failed SHA-256 verification" + )); + } + Ok(bytes) + } + + fn verify(&self, sha256: &str, target: &str) -> Result<Vec<u8>, String> { + let bytes = self + .get(sha256, target)? + .ok_or_else(|| format!("artifact {sha256} for target {target} is missing"))?; + if sha256_bytes(&bytes) != sha256 { + return Err(format!( + "artifact {sha256} for target {target} failed SHA-256 verification" + )); + } + Ok(bytes) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct VerifiedArtifact { + pub lib: LibSpec, + pub target: String, + pub path: String, + pub bytes: Vec<u8>, + pub sha256: String, +} + +/// Verify the exact immutable bytes selected by a lock file. The caller +/// supplies the authenticated machine-cache implementation; no registry, +/// package manager, PATH lookup, or environment variable is consulted here. +pub fn verify_locked_artifacts( + lock: &LockFile, + store: &dyn ArtifactStore, + target: &str, +) -> Result<Vec<VerifiedArtifact>, String> { + verify_locked_artifacts_with_go_verifier(lock, store, target, &sumdb::Verifier::official()) +} + +pub fn verify_locked_artifacts_with_go_verifier( + lock: &LockFile, + store: &dyn ArtifactStore, + target: &str, + verifier: &sumdb::Verifier, +) -> Result<Vec<VerifiedArtifact>, String> { + doctor_with_go_verifier(lock, verifier)?; + let go = sumdb::verify_chain(&lock.go_sumdb, verifier)?; + let mut verified = Vec::new(); + for lib in &lock.libs { + for artifact in &lib.artifacts { + if artifact.target != target && artifact.target != "host" { + return Err(format!( + "locked artifact for {} targets {}, not {target}", + lib.spec.key(), + artifact.target + )); + } + let bytes = store.verify_exact(&artifact.sha256, artifact.bytes, &artifact.target)?; + if lib.spec.ecosystem == Ecosystem::Go && go.contains(&lib.spec.name, &lib.spec.version) + { + go.verify_zip(&lib.spec.name, &lib.spec.version, &bytes)?; + } + verified.push(VerifiedArtifact { + lib: lib.spec.clone(), + target: artifact.target.clone(), + path: artifact.path.clone(), + bytes, + sha256: artifact.sha256.clone(), + }); + } + } + for receipt in build::npm_outputs(lock) { + let artifact = &receipt.output; + if artifact.target != target && artifact.target != "host" { + return Err(format!( + "locked npm build output targets {}, not {target}", + artifact.target + )); + } + verified.push(VerifiedArtifact { + lib: receipt + .roots + .first() + .ok_or("npm build roots missing")? + .clone(), + target: artifact.target.clone(), + path: artifact.path.clone(), + bytes: store.verify_exact(&artifact.sha256, artifact.bytes, &artifact.target)?, + sha256: artifact.sha256.clone(), + }); + } + verified.sort_by(|left, right| { + left.path + .cmp(&right.path) + .then(left.sha256.cmp(&right.sha256)) + }); + Ok(verified) +} + +#[derive(Default)] +pub struct FixtureArtifactStore { + entries: std::sync::Mutex<BTreeMap<(String, String), Vec<u8>>>, +} + +impl ArtifactStore for FixtureArtifactStore { + fn has(&self, sha256: &str, target: &str) -> Result<bool, String> { + Ok(self + .entries + .lock() + .map_err(|_| "fixture cache poisoned")? + .contains_key(&(sha256.into(), target.into()))) + } + fn publish(&self, bytes: &[u8], target: &str) -> Result<String, String> { + let digest = sha256(bytes); + self.entries + .lock() + .map_err(|_| "fixture cache poisoned")? + .insert((digest.clone(), target.into()), bytes.to_vec()); + Ok(digest) + } + + fn get(&self, sha256: &str, target: &str) -> Result<Option<Vec<u8>>, String> { + Ok(self + .entries + .lock() + .map_err(|_| "fixture cache poisoned")? + .get(&(sha256.into(), target.into())) + .cloned()) + } +} + +/// Metadata-only SDK fixture used by offline tests. Actual language SDKs are +/// Worker binaries and use the frozen Component Protocol; this type only checks +/// that their generated schema identity is the one recorded in `dever.lock`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SdkFixture { + pub ecosystem: Ecosystem, + pub schema: String, +} + +/// Offline preparation result consumed by `run` and `build`. Resolution is +/// deliberately not performed here: project commands only accept a canonical +/// lock produced by an explicit `dever lib add/update`. +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct PreparationReport { + pub libs: usize, + pub artifacts: usize, + pub artifact_bytes: u64, + pub runtimes: usize, +} + +pub fn write_report(output: &Path, report: &PreparationReport) -> Result<(), String> { + let mut path = output.to_path_buf(); + let name = output + .file_name() + .and_then(|value| value.to_str()) + .ok_or("build output must have a file name")?; + path.set_file_name(format!("{name}.external.json")); + let mut bytes = serde_json::to_vec_pretty(report).map_err(|error| error.to_string())?; + bytes.push(b'\n'); + if path.exists() + && fs::symlink_metadata(&path) + .map_err(|error| error.to_string())? + .file_type() + .is_symlink() + { + return Err(format!( + "external build report '{}' must not be a symbolic link", + path.display() + )); + } + let temporary = path.with_file_name(format!( + ".{}.{}-{}.tmp", + name, + std::process::id(), + unique_suffix() + )); + write_new_file(&temporary, &bytes).map_err(|error| { + format!( + "cannot stage external build report '{}': {error}", + path.display() + ) + })?; + fs::rename(&temporary, &path).map_err(|error| { + let _ = fs::remove_file(&temporary); + format!( + "cannot publish external build report '{}': {error}", + path.display() + ) + }) +} + +impl PreparationReport { + pub fn summary(&self) -> String { + format!( + "external libs: {} lib(s), {} artifact(s), {} runtime pack(s), {} artifact bytes", + self.libs, self.artifacts, self.runtimes, self.artifact_bytes, + ) + } +} + +/// Validate the project declarations and lock without contacting a registry, +/// reading PATH, or consulting an environment variable. Missing external +/// artifacts are reported by the authenticated machine cache owner in the +/// packaging build; this stage only proves that the project contract is +/// deterministic and complete. +pub fn prepare( + project_root: &Path, + source_requests: &[String], +) -> Result<Option<PreparationReport>, String> { + let lock_path = project_root.join("dever.lock"); + let mut declarations = read_declarations(project_root)?; + declarations.extend(crate::packages::declared_libs(project_root)?); + if declarations.is_empty() && !lock_path.exists() && source_requests.is_empty() { + return Ok(None); + } + if !lock_path.exists() { + return Err(format!( + "external libraries are declared but '{}' is missing; run 'dever lib add {} <pip|npm|go>:<name>@<version>'", + lock_path.display(), + project_root.display(), + )); + } + let lock = read_lock(&lock_path)?; + doctor(&lock)?; + declarations.extend( + source_requests + .iter() + .map(|request| request.parse()) + .collect::<Result<Vec<LibSpec>, _>>()?, + ); + declarations.sort(); + declarations.dedup(); + for requested in declarations { + if !lock.libs.iter().any(|lib| lib.spec == requested) { + return Err(format!( + "dever.lock does not contain declared {}; run 'dever lib update {}'", + requested.key(), + project_root.display(), + )); + } + } + let artifacts = lock + .libs + .iter() + .flat_map(|lib| &lib.artifacts) + .chain(build::npm_outputs(&lock).map(|receipt| &receipt.output)) + .collect::<BTreeSet<_>>(); + let artifact_bytes = artifacts.iter().map(|artifact| artifact.bytes).sum(); + let runtimes = lock + .libs + .iter() + .map(|lib| (&lib.runtime.name, &lib.runtime.version)) + .chain(lock.workers.iter().filter_map(|worker| { + worker + .runtime + .as_ref() + .map(|runtime| (&runtime.name, &runtime.version)) + })) + .collect::<BTreeSet<_>>() + .len(); + Ok(Some(PreparationReport { + libs: lock.libs.len(), + artifacts: artifacts.len(), + artifact_bytes, + runtimes, + })) +} + +pub fn prepare_program( + project_root: &Path, + program: &dever_core::hir::Program, + target: crate::toolchain::BuildTarget, +) -> Result<Option<PreparationReport>, String> { + if program_workers(program)? + .iter() + .any(|worker| worker.ecosystem != "exec") + && !project_root.join("dever.lock").exists() + { + return Err(format!( + "managed external Worker requires dever.lock; run 'dever lib add {}'", + project_root.display() + )); + } + let report = prepare(project_root, &program.external_lib_requests())?; + if report.is_some() { + validate_worker_bindings( + &read_lock(&project_root.join("dever.lock"))?, + &bound_program_workers(program, target)?, + )?; + } + Ok(report) +} + +fn program_workers(program: &dever_core::hir::Program) -> Result<Vec<LockedWorker>, String> { + program + .external_worker_contracts() + .into_iter() + .filter(|worker| worker.ecosystem != "exec" || !worker.libs.is_empty()) + .map(LockedWorker::from_contract) + .collect() +} + +pub(crate) fn bound_program_workers( + program: &dever_core::hir::Program, + target: crate::toolchain::BuildTarget, +) -> Result<Vec<LockedWorker>, String> { + let mut workers = program_workers(program)?; + if workers.iter().all(|worker| worker.ecosystem == "exec") { + return Ok(workers); + } + let managed = InstalledRegistry::current(target)?; + for worker in &mut workers { + if worker.ecosystem != "exec" { + worker.runtime = Some(managed.runtime(worker.ecosystem.parse()?)?.0.pack); + } + } + Ok(workers) +} + +fn validate_worker_bindings(lock: &LockFile, expected: &[LockedWorker]) -> Result<(), String> { + if lock.workers != expected { + return Err("dever.lock Worker schema, entry, capability or Lib binding changed; run 'dever lib update <project-root>'".into()); + } + Ok(()) +} + +/// Materialize locked bytes into the compiler-owned resource table. Released +/// providers read the authenticated machine cache and signed runtime pack. +pub fn embedded_resources( + project_root: &Path, +) -> Result<Vec<dever_core::native::EmbeddedResource>, String> { + embedded_resources_for_target(project_root, crate::toolchain::BuildTarget::host()?) +} + +pub fn embedded_resources_for_target( + project_root: &Path, + target: crate::toolchain::BuildTarget, +) -> Result<Vec<dever_core::native::EmbeddedResource>, String> { + let lock_path = project_root.join("dever.lock"); + let lock = read_lock(&lock_path)?; + doctor(&lock)?; + let registry = FixtureRegistry::builtin(); + let mut resources = Vec::new(); + let go = sumdb::verify_chain(&lock.go_sumdb, &sumdb::Verifier::official())?; + let mut managed = None; + let mut runtimes = BTreeMap::new(); + for lib in &lock.libs { + if let Some(package) = registry.package(&lib.spec) { + if &package.locked() != lib { + return Err(format!( + "locked metadata for {} changed; run 'dever lib update {}'", + lib.spec.key(), + project_root.display() + )); + } + for artifact in &package.artifacts { + resources.push(dever_core::native::EmbeddedResource { + path: artifact.path.clone(), + bytes: artifact.bytes.clone(), + sha256: sha256(&artifact.bytes), + executable: artifact.path.ends_with(".bin") + || artifact.path.ends_with("/worker"), + }); + } + continue; + } + let installed = match &managed { + Some(installed) => installed, + None => managed.insert(InstalledRegistry::current(target)?), + }; + if let Some(runtime) = runtimes.get(&lib.spec.ecosystem) { + if runtime != &lib.runtime { + return Err(format!( + "locked runtime pack for {} differs from the signed release", + lib.spec.key() + )); + } + } else { + let (runtime, bytes) = installed.runtime(lib.spec.ecosystem.clone())?; + if runtime.pack != lib.runtime { + return Err(format!( + "locked runtime pack for {} differs from the signed release", + lib.spec.key() + )); + } + runtimes.insert(lib.spec.ecosystem.clone(), runtime.pack.clone()); + resources.push(dever_core::native::EmbeddedResource { + path: format!( + "lib/runtime/{}/{}/runtime.pack", + lib.spec.ecosystem.as_str(), + runtime.target + ), + sha256: runtime.pack.sha256, + bytes, + executable: false, + }); + } + let store = ManagedArtifactStore { + layout: installed.layout.clone(), + }; + for artifact in &lib.artifacts { + if artifact.target != target.platform() { + return Err(format!( + "locked artifact for {} targets {}, not the selected build target", + lib.spec.key(), + artifact.target + )); + } + let bytes = store.verify_exact(&artifact.sha256, artifact.bytes, &artifact.target)?; + if lib.spec.ecosystem == Ecosystem::Go { + go.verify_zip(&lib.spec.name, &lib.spec.version, &bytes)?; + } + resources.push(dever_core::native::EmbeddedResource { + path: artifact.path.clone(), + bytes, + sha256: artifact.sha256.clone(), + executable: false, + }); + } + } + for receipt in build::npm_outputs(&lock) { + let installed = match &managed { + Some(installed) => installed, + None => managed.insert(InstalledRegistry::current(target)?), + }; + let artifact = &receipt.output; + if artifact.target != target.platform() { + return Err("locked npm build output differs from the selected build target".into()); + } + let store = ManagedArtifactStore { + layout: installed.layout.clone(), + }; + resources.push(dever_core::native::EmbeddedResource { + path: artifact.path.clone(), + bytes: store.verify_exact(&artifact.sha256, artifact.bytes, &artifact.target)?, + sha256: artifact.sha256.clone(), + executable: false, + }); + } + for worker in lock + .workers + .iter() + .filter(|worker| worker.ecosystem != "exec") + { + let ecosystem: Ecosystem = worker.ecosystem.parse()?; + let installed = match &managed { + Some(installed) => installed, + None => managed.insert(InstalledRegistry::current(target)?), + }; + if !runtimes.contains_key(&ecosystem) { + let (runtime, bytes) = installed.runtime(ecosystem.clone())?; + if worker.runtime.as_ref() != Some(&runtime.pack) { + return Err(format!( + "locked Worker '{}.{}' runtime differs from signed release", + worker.port, worker.adapter + )); + } + runtimes.insert(ecosystem.clone(), runtime.pack.clone()); + resources.push(dever_core::native::EmbeddedResource { + path: format!( + "lib/runtime/{}/{}/runtime.pack", + ecosystem.as_str(), + runtime.target + ), + sha256: runtime.pack.sha256, + bytes, + executable: false, + }); + } else if worker.runtime.as_ref() != runtimes.get(&ecosystem) { + return Err(format!( + "locked Worker '{}.{}' runtime differs from signed release", + worker.port, worker.adapter + )); + } + } + resources.sort_by(|left, right| { + left.path + .cmp(&right.path) + .then(left.sha256.cmp(&right.sha256)) + }); + resources.dedup_by(|left, right| left.path == right.path && left.sha256 == right.sha256); + Ok(resources) +} + +pub fn prepare_resources( + project_root: &Path, + program: &dever_core::hir::Program, + target: crate::toolchain::BuildTarget, +) -> Result<Vec<dever_core::native::EmbeddedResource>, String> { + let mut resources = + if project_root.join("dever.lock").exists() && program.has_external_adapters() { + embedded_resources_for_target(project_root, target)? + } else { + Vec::new() + }; + for entry in program + .external_worker_contracts() + .into_iter() + .filter(|worker| worker.ecosystem == "exec") + .map(|worker| worker.entry) + { + let bytes = match crate::packages::owned_worker_bytes(project_root, &entry)? { + Some(bytes) => bytes, + None => { + let path = owned_worker(project_root, &entry)?; + fs::read(&path) + .map_err(|error| format!("cannot read external Worker '{entry}': {error}"))? + } + }; + if bytes.is_empty() { + return Err(format!("external Worker '{entry}' is empty")); + } + resources.push(dever_core::native::EmbeddedResource { + path: entry, + sha256: sha256(&bytes), + bytes, + executable: true, + }); + } + let mut host_assets = Vec::new(); + if program.has_external_adapters() { + let installed = InstalledRegistry::current(target)?; + resources.extend(installed.sandbox_resources(target)?); + if program + .external_worker_contracts() + .iter() + .any(|worker| worker.ecosystem == "go") + { + host_assets = installed.sandbox_resources(crate::toolchain::BuildTarget::host()?)?; + } + } + let packaged = crate::workers::prepare_for_target( + project_root, + program, + &resources, + target, + &host_assets, + )?; + let exec_entries = program + .external_worker_contracts() + .into_iter() + .filter(|worker| worker.ecosystem == "exec") + .map(|worker| worker.entry) + .collect::<BTreeSet<_>>(); + resources.retain(|resource| !exec_entries.contains(&resource.path)); + let workers = program_workers(program)?; + if workers.iter().any(|worker| worker.ecosystem != "exec") { + let lock = read_lock(&project_root.join("dever.lock"))?; + prune_expanded_resources_for_target( + &lock, + &workers, + &read_declarations(project_root)?, + &mut resources, + target, + )?; + } + resources.extend(packaged); + resources.sort_by(|left, right| left.path.cmp(&right.path)); + for pair in resources.windows(2) { + if pair[0].path == pair[1].path + && (pair[0].sha256 != pair[1].sha256 || pair[0].executable != pair[1].executable) + { + return Err(format!( + "conflicting external resources for '{}'", + pair[0].path + )); + } + } + resources.dedup_by(|left, right| left.path == right.path); + Ok(resources) +} + +/// The managed packager has already expanded these archives into verified +/// execution trees. Keep raw bytes if a declared Lib or exec Worker still uses +/// the same locked dependency closure. +pub fn prune_expanded_resources( + lock: &LockFile, + workers: &[LockedWorker], + declared: &[LibSpec], + resources: &mut Vec<dever_core::native::EmbeddedResource>, +) -> Result<(), String> { + prune_expanded_resources_for_target( + lock, + workers, + declared, + resources, + crate::toolchain::BuildTarget::host()?, + ) +} + +pub fn prune_expanded_resources_for_target( + lock: &LockFile, + workers: &[LockedWorker], + declared: &[LibSpec], + resources: &mut Vec<dever_core::native::EmbeddedResource>, + target: crate::toolchain::BuildTarget, +) -> Result<(), String> { + let by_spec = lock + .libs + .iter() + .map(|lib| (&lib.spec, lib)) + .collect::<BTreeMap<_, _>>(); + let closure = |roots: Vec<LibSpec>| -> Result<BTreeSet<LibSpec>, String> { + let mut selected = BTreeSet::new(); + let npm_archives = npm::archive_roots(lock, &roots)?; + let mut remaining = VecDeque::from(roots); + remaining.extend(npm_archives); + while let Some(spec) = remaining.pop_front() { + if !selected.insert(spec.clone()) { + continue; + } + let lib = by_spec + .get(&spec) + .ok_or_else(|| format!("locked Lib {} is missing", spec.key()))?; + remaining.extend( + lib.dependencies + .iter() + .map(|dependency| dependency.spec.clone()), + ); + } + Ok(selected) + }; + let managed_workers = workers + .iter() + .filter(|worker| worker.ecosystem != "exec") + .collect::<Vec<_>>(); + let npm_output = |roots: &[LibSpec]| -> Result<Option<&str>, String> { + let roots = npm::roots(roots); + let Some(environment) = lock + .npm + .iter() + .find(|environment| environment.roots == roots) + else { + return Ok(None); + }; + build::npm_receipt(lock, environment) + .map(|receipt| receipt.map(|receipt| receipt.output.path.as_str())) + }; + let mut managed = BTreeSet::new(); + let mut expanded_outputs = BTreeSet::new(); + for worker in &managed_workers { + managed.extend(closure(worker.libs.clone())?); + expanded_outputs.extend(npm_output(&worker.libs)?); + } + let mut retained = closure(declared.to_vec())?; + let mut retained_outputs = BTreeSet::new(); + retained_outputs.extend(npm_output(declared)?); + for worker in workers.iter().filter(|worker| worker.ecosystem == "exec") { + retained.extend(closure(worker.libs.clone())?); + retained_outputs.extend(npm_output(&worker.libs)?); + } + let managed_ecosystems = managed_workers + .iter() + .map(|worker| worker.ecosystem.as_str()) + .collect::<BTreeSet<_>>(); + let retained_ecosystems = retained + .iter() + .map(|spec| spec.ecosystem.as_str()) + .collect::<BTreeSet<_>>(); + let retained_artifacts = lock + .libs + .iter() + .filter(|lib| retained.contains(&lib.spec)) + .flat_map(|lib| lib.artifacts.iter().map(|artifact| artifact.path.as_str())) + .chain(retained_outputs) + .collect::<BTreeSet<_>>(); + let expanded_artifacts = lock + .libs + .iter() + .filter(|lib| managed.contains(&lib.spec) && !retained.contains(&lib.spec)) + .flat_map(|lib| lib.artifacts.iter().map(|artifact| artifact.path.as_str())) + .chain(expanded_outputs) + .filter(|path| !retained_artifacts.contains(path)) + .collect::<BTreeSet<_>>(); + let target = target.platform(); + let expanded_runtimes = managed_ecosystems + .difference(&retained_ecosystems) + .map(|ecosystem| format!("lib/runtime/{ecosystem}/{target}/runtime.pack")) + .collect::<BTreeSet<_>>(); + resources.retain(|resource| { + !expanded_artifacts.contains(resource.path.as_str()) + && !expanded_runtimes.contains(&resource.path) + }); + Ok(()) +} + +pub(crate) fn owned_worker(project_root: &Path, entry: &str) -> Result<PathBuf, String> { + let relative = Path::new(entry); + if entry.contains(['\\', ':']) + || relative.is_absolute() + || relative + .components() + .any(|part| !matches!(part, Component::Normal(_))) + { + return Err("external Worker entry must be a portable relative path".into()); + } + let mut path = project_root.to_path_buf(); + if fs::symlink_metadata(&path) + .map_err(|error| error.to_string())? + .file_type() + .is_symlink() + { + return Err("external Worker project root must not be a symbolic link".into()); + } + for part in relative.components() { + path.push(part); + let metadata = fs::symlink_metadata(&path) + .map_err(|error| format!("cannot inspect external Worker '{entry}': {error}"))?; + if metadata.file_type().is_symlink() { + return Err("external Worker entry crosses a symbolic link".into()); + } + } + if !fs::metadata(&path) + .map_err(|error| error.to_string())? + .is_file() + { + return Err("external Worker entry must be a regular file".into()); + } + Ok(path) +} + +impl SdkFixture { + pub fn handshake(&self, expected_schema: &str) -> Result<(), String> { + if self.schema != expected_schema { + return Err("SDK fixture schema identity drift".into()); + } + Ok(()) + } +} + +pub fn doctor(lock: &LockFile) -> Result<(), String> { + doctor_with_go_verifier(lock, &sumdb::Verifier::official()) +} + +pub fn doctor_with_go_verifier(lock: &LockFile, verifier: &sumdb::Verifier) -> Result<(), String> { + lock.encode()?; + crate::packages::doctor_lock(&lock.packages)?; + npm::doctor(lock)?; + build::doctor(lock)?; + let go = sumdb::verify_chain(&lock.go_sumdb, verifier)?; + let fixtures = FixtureRegistry::builtin(); + let mut workers = lock.workers.clone(); + normalize_workers(&mut workers)?; + let mut seen = BTreeSet::new(); + let mut distributions = BTreeMap::<(&str, &str), &LockedLib>::new(); + let locked = lock + .libs + .iter() + .map(|lib| lib.spec.clone()) + .collect::<BTreeSet<_>>(); + for lib in &lock.libs { + if lib.spec.ecosystem == Ecosystem::Npm + && (!lib.dependencies.is_empty() + || !lock.npm.iter().any(|environment| { + environment + .instances + .iter() + .any(|node| node.source.archive == lib.spec) + })) + && !fixtures + .package(&lib.spec) + .is_some_and(|package| package.locked() == *lib) + { + return Err( + "npm archive requires an exact instance graph, not flat dependencies".into(), + ); + } + if lib.spec.ecosystem == Ecosystem::Go + && !go.contains(&lib.spec.name, &lib.spec.version) + && !fixtures + .package(&lib.spec) + .is_some_and(|package| package.locked() == *lib) + { + return Err(format!( + "{} lacks authenticated Go sumdb evidence", + lib.spec.key() + )); + } + validate_name(&lib.spec.ecosystem, &lib.spec.name)?; + validate_version(&lib.spec.ecosystem, &lib.spec.version)?; + if !seen.insert(lib.spec.key()) { + return Err(format!("duplicate locked lib {}", lib.spec.key())); + } + if lib.schema.is_empty() { + return Err(format!( + "locked lib {} has empty adapter schema identity", + lib.spec.key() + )); + } + validate_runtime(&lib.runtime)?; + if lib.spec.ecosystem == Ecosystem::Pip { + let key = (lib.spec.distribution_name(), lib.spec.version.as_str()); + if let Some(previous) = distributions.insert(key, lib) + && (previous.artifacts != lib.artifacts + || previous.runtime != lib.runtime + || previous.schema != lib.schema + || previous.build != lib.build) + { + return Err(format!( + "Python extra variants have inconsistent distribution artifacts for {}@{}", + key.0, key.1 + )); + } + } + for artifact in &lib.artifacts { + validate_artifact(artifact)?; + } + for dependency in &lib.dependencies { + if !locked.contains(&dependency.spec) { + return Err(format!( + "locked lib {} depends on missing {}", + lib.spec.key(), + dependency.spec.key() + )); + } + if dependency.spec == lib.spec { + return Err(format!("locked lib {} depends on itself", lib.spec.key())); + } + } + } + validate_dependency_cycles(&lock.libs)?; + for worker in &workers { + let mut selected = BTreeMap::new(); + let mut visited = BTreeSet::new(); + let mut remaining = worker.libs.iter().cloned().collect::<VecDeque<_>>(); + while let Some(spec) = remaining.pop_front() { + if spec.ecosystem == Ecosystem::Npm { + continue; + } + if !visited.insert(spec.clone()) { + continue; + } + let key = (spec.ecosystem.clone(), spec.distribution_name().to_owned()); + if let Some(version) = selected.get(&key) + && version != &spec.version + { + return Err(format!( + "Worker '{}.{}' has conflicting versions of {}:{}", + worker.port, + worker.adapter, + spec.ecosystem.as_str(), + spec.name + )); + } + selected.insert(key, spec.version.clone()); + let lib = lock + .libs + .iter() + .find(|lib| lib.spec == spec) + .ok_or_else(|| { + format!( + "Worker '{}.{}' references missing {}", + worker.port, + worker.adapter, + spec.key() + ) + })?; + remaining.extend( + lib.dependencies + .iter() + .map(|dependency| dependency.spec.clone()), + ); + } + } + Ok(()) +} + +fn validate_dependency_cycles(libs: &[LockedLib]) -> Result<(), String> { + // Find strongly connected components of the exact request graph. Collapsing + // all variants first would invent cycles between independent Workers. + let mut graph = BTreeMap::<&LibSpec, Vec<&LibSpec>>::new(); + let mut reverse = BTreeMap::<&LibSpec, Vec<&LibSpec>>::new(); + for lib in libs { + let dependencies = lib + .dependencies + .iter() + .map(|dependency| &dependency.spec) + .collect::<Vec<_>>(); + for dependency in &dependencies { + reverse.entry(dependency).or_default().push(&lib.spec); + } + graph.insert(&lib.spec, dependencies); + } + fn visit<'a>( + spec: &'a LibSpec, + graph: &BTreeMap<&'a LibSpec, Vec<&'a LibSpec>>, + visited: &mut BTreeSet<&'a LibSpec>, + order: &mut Vec<&'a LibSpec>, + ) { + if !visited.insert(spec) { + return; + } + for dependency in graph.get(spec).into_iter().flatten() { + visit(dependency, graph, visited, order); + } + order.push(spec); + } + let mut order = Vec::new(); + let mut visited = BTreeSet::new(); + for spec in graph.keys() { + visit(spec, &graph, &mut visited, &mut order); + } + visited.clear(); + for root in order.into_iter().rev() { + let mut component = Vec::new(); + let mut remaining = VecDeque::from([root]); + while let Some(spec) = remaining.pop_front() { + if !visited.insert(spec) { + continue; + } + component.push(spec); + remaining.extend(reverse.get(spec).into_iter().flatten().copied()); + } + if component.len() > 1 + && !component.iter().all(|spec| { + spec.ecosystem == Ecosystem::Pip + && root.ecosystem == Ecosystem::Pip + && spec.distribution_name() == root.distribution_name() + && spec.version == root.version + }) + { + return Err(format!("cyclic locked lib dependency at {}", root.key())); + } + } + Ok(()) +} + +pub fn execute(command: &str, project_root: &Path, specs: &[String]) -> Result<String, String> { + execute_for_target( + command, + project_root, + specs, + crate::toolchain::BuildTarget::host()?, + ) +} + +pub fn execute_for_target( + command: &str, + project_root: &Path, + specs: &[String], + target: crate::toolchain::BuildTarget, +) -> Result<String, String> { + let _mutation = ProjectMutation::for_command(command, project_root)?; + let path = project_root.join("dever.lock"); + match command { + "add" => { + let mut requested = read_declarations(project_root)?; + let workers = source_workers(project_root, target)?; + let package_libs = crate::packages::declared_libs(project_root)?; + if specs.is_empty() + && requested.is_empty() + && workers.is_empty() + && package_libs.is_empty() + { + return Err( + "dever lib add requires an external Adapter lib or at least one exact lib spec" + .into(), + ); + } + requested.extend( + specs + .iter() + .map(|value| value.parse()) + .collect::<Result<Vec<LibSpec>, _>>()?, + ); + requested.sort(); + requested.dedup(); + let mut all = requested.clone(); + all.extend(package_libs); + let mut lock = resolve_project(project_root, &all, workers, target)?; + lock.packages = locked_packages(project_root)?; + lock.encode()?; + write_declarations(project_root, &requested)?; + lock.write_atomic(project_root)?; + Ok(format!( + "resolved {} lib(s) into {}", + lock.libs.len(), + path.display() + )) + } + "update" => { + let requested = if specs.is_empty() { + read_declarations(project_root)? + } else { + specs + .iter() + .map(|value| value.parse()) + .collect::<Result<Vec<LibSpec>, _>>()? + }; + let workers = source_workers(project_root, target)?; + let package_libs = crate::packages::declared_libs(project_root)?; + if requested.is_empty() && workers.is_empty() && package_libs.is_empty() { + return Err("dever lib update requires declared libs or exact lib specs".into()); + } + let mut all = requested.clone(); + all.extend(package_libs); + let mut lock = resolve_project(project_root, &all, workers, target)?; + lock.packages = locked_packages(project_root)?; + lock.encode()?; + if !specs.is_empty() { + write_declarations(project_root, &requested)?; + } + lock.write_atomic(project_root)?; + Ok(format!( + "updated {} lib(s) in {}", + lock.libs.len(), + path.display() + )) + } + "list" => { + let lock = read_lock(&path)?; + Ok(lock + .libs + .iter() + .map(|lib| lib.spec.key()) + .collect::<Vec<_>>() + .join("\n")) + } + "doctor" => { + let lock = read_lock(&path)?; + doctor(&lock)?; + validate_worker_bindings(&lock, &source_workers(project_root, target)?)?; + prepare( + project_root, + &lock + .workers + .iter() + .flat_map(|worker| worker.libs.iter().map(LibSpec::key)) + .collect::<Vec<_>>(), + )?; + embedded_resources_for_target(project_root, target)?; + Ok(format!("{} is valid", path.display())) + } + "remove" => { + if specs.is_empty() { + return Err("dever lib remove requires at least one exact lib spec".into()); + } + let removals = specs + .iter() + .map(|value| value.parse::<LibSpec>()) + .collect::<Result<BTreeSet<_>, _>>()?; + let package_libs = crate::packages::declared_libs(project_root)?; + if removals.iter().any(|spec| package_libs.contains(spec)) { + return Err( + "cannot remove a Lib declared by a Package; remove the Package first".into(), + ); + } + let previous_lock = if path.exists() { + Some(read_lock(&path)?) + } else { + None + }; + let mut requested = read_declarations(project_root)?; + requested.retain(|spec| !removals.contains(spec)); + let workers = source_workers(project_root, target)?; + if workers + .iter() + .any(|worker| worker.libs.iter().any(|spec| removals.contains(spec))) + { + return Err("cannot remove a Lib declared by an external Adapter; remove the source declaration first".into()); + } + if let Some(lock) = &previous_lock { + let mut roots = requested.clone(); + roots.extend(package_libs.iter().cloned()); + let mut groups = vec![roots]; + groups.extend(workers.iter().map(|worker| worker.libs.clone())); + // An orphan cycle is removed together. Only consumers reachable + // from the remaining project/Worker roots can block removal. + for environment in npm::retain(lock, &groups)? { + for node in environment + .instances + .iter() + .filter(|node| !removals.contains(&node.spec)) + { + for target in node.edges.iter().filter_map(|edge| edge.target.as_ref()) { + if let Some(required) = environment + .instances + .iter() + .find(|node| &node.path == target && removals.contains(&node.spec)) + { + return Err(format!( + "cannot remove {}: it is required by {}", + required.spec.key(), + node.spec.key() + )); + } + } + } + } + for lib in &lock.libs { + if !removals.contains(&lib.spec) + && lib + .dependencies + .iter() + .any(|dependency| removals.contains(&dependency.spec)) + { + let dependency = lib + .dependencies + .iter() + .find(|dependency| removals.contains(&dependency.spec)) + .unwrap(); + return Err(format!( + "cannot remove {}: it is required by {}", + dependency.spec.key(), + lib.spec.key() + )); + } + } + } + if requested.is_empty() + && workers.is_empty() + && package_libs.is_empty() + && locked_packages(project_root)?.is_empty() + && previous_lock + .as_ref() + .is_none_or(|lock| lock.go_sumdb.is_empty()) + { + write_declarations(project_root, &requested)?; + if path.exists() { + fs::remove_file(&path) + .map_err(|e| format!("cannot remove '{}': {e}", path.display()))?; + } + } else { + let old = previous_lock + .ok_or("dever.lock is required to remove a Lib without registry access")?; + let mut all = requested.clone(); + all.extend(package_libs); + let lock = retain_locked_libs(&old, &all, workers)?; + lock.encode()?; + write_declarations(project_root, &requested)?; + lock.write_atomic(project_root)?; + } + Ok(format!("removed {} lib(s)", removals.len())) + } + _ => Err("Usage: dever lib add|list|update|remove|doctor <project-root> [spec ...]".into()), + } +} + +pub(crate) fn retain_locked_libs( + old: &LockFile, + requested: &[LibSpec], + workers: Vec<LockedWorker>, +) -> Result<LockFile, String> { + let mut groups = vec![requested.to_vec()]; + groups.extend(workers.iter().map(|worker| worker.libs.clone())); + let npm = npm::retain(old, &groups)?; + let by_spec = old + .libs + .iter() + .map(|lib| (lib.spec.clone(), lib)) + .collect::<BTreeMap<_, _>>(); + let mut queue = requested + .iter() + .cloned() + .chain( + workers + .iter() + .flat_map(|worker| worker.libs.iter().cloned()), + ) + .collect::<VecDeque<_>>(); + queue.extend(npm.iter().flat_map(|environment| { + environment + .instances + .iter() + .map(|node| node.source.archive.clone()) + })); + let mut retained = BTreeMap::new(); + while let Some(spec) = queue.pop_front() { + if retained.contains_key(&spec) { + continue; + } + let lib = by_spec.get(&spec).ok_or_else(|| { + format!( + "dever.lock lacks {}; run 'dever lib update <project-root>'", + spec.key() + ) + })?; + queue.extend( + lib.dependencies + .iter() + .map(|dependency| dependency.spec.clone()), + ); + retained.insert(spec, (*lib).clone()); + } + let mut lock = LockFile::new(retained.into_values().collect())?; + lock.workers = workers; + lock.packages = old.packages.clone(); + lock.go_sumdb = old.go_sumdb.clone(); + lock.npm = npm; + let retained_builds = lock + .libs + .iter() + .filter_map(|lib| lib.build.as_deref()) + .chain( + lock.npm + .iter() + .filter_map(|environment| environment.build.as_deref()), + ) + .collect::<BTreeSet<_>>(); + lock.builds = old + .builds + .iter() + .filter_map(|receipt| match receipt.identity() { + Ok(identity) if retained_builds.contains(identity.as_str()) => { + Some(Ok(receipt.clone())) + } + Ok(_) => None, + Err(error) => Some(Err(error)), + }) + .collect::<Result<Vec<_>, String>>()?; + doctor(&lock)?; + Ok(lock) +} + +fn locked_packages(project_root: &Path) -> Result<Vec<crate::packages::LockedPackage>, String> { + crate::packages::owned_files(project_root)?; + let path = project_root.join("dever.lock"); + if path.exists() { + Ok(read_lock(&path)?.packages) + } else { + Ok(Vec::new()) + } +} + +pub(crate) fn read_lock(path: &Path) -> Result<LockFile, String> { + LockFile::decode(&fs::read(path).map_err(|e| format!("cannot read '{}': {e}", path.display()))?) +} + +fn source_workers( + project_root: &Path, + target: crate::toolchain::BuildTarget, +) -> Result<Vec<LockedWorker>, String> { + source_workers_with_packages( + project_root, + &crate::packages::sources(project_root)?, + target, + ) +} + +pub(crate) fn source_workers_with_packages( + project_root: &Path, + packages: &[dever_core::source::PackageSource], + target: crate::toolchain::BuildTarget, +) -> Result<Vec<LockedWorker>, String> { + let source_root = project_root.join("module"); + if !source_root.exists() && packages.is_empty() { + return Ok(Vec::new()); + } + let sources = dever_core::source::SourceMap::load_with_packages(&source_root, None, packages) + .map_err(|errors| { + errors + .iter() + .map(ToString::to_string) + .collect::<Vec<_>>() + .join("\n") + })?; + let program = dever_core::check(&sources).map_err(|errors| { + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + })?; + bound_program_workers(&program, target) +} + +fn declaration_path(project_root: &Path) -> PathBuf { + project_root.join("config/setting.json") +} + +pub(crate) fn read_declarations(project_root: &Path) -> Result<Vec<LibSpec>, String> { + if project_root.join("config/lib.json").exists() { + return Err( + "config/lib.json is not supported; declare Libs in config/setting.json 'lib'".into(), + ); + } + let path = declaration_path(project_root); + if !path.exists() { + return Ok(Vec::new()); + } + reject_declaration_symlinks(project_root)?; + let settings = dever_runtime::config::Settings::load_project(project_root)?; + let mut specs = settings + .external_libs() + .iter() + .map(|value| value.parse::<LibSpec>()) + .collect::<Result<Vec<_>, _>>()?; + specs.sort(); + specs.dedup(); + for spec in &specs { + validate_name(&spec.ecosystem, &spec.name)?; + validate_version(&spec.ecosystem, &spec.version)?; + } + Ok(specs) +} + +fn write_declarations(project_root: &Path, specs: &[LibSpec]) -> Result<(), String> { + reject_declaration_symlinks(project_root)?; + let path = declaration_path(project_root); + let parent = path + .parent() + .ok_or("project root has no config directory")?; + fs::create_dir_all(parent).map_err(|e| format!("cannot create config directory: {e}"))?; + let mut specs = specs.to_vec(); + specs.sort(); + specs.dedup(); + let mut document = if path.exists() { + let text = fs::read_to_string(&path).map_err(|error| error.to_string())?; + dever_runtime::wire::parse(&text) + .map_err(|error| format!("invalid setting.json: {error}"))?; + serde_json::from_str::<serde_json::Map<String, serde_json::Value>>(&text) + .map_err(|error| error.to_string())? + } else { + serde_json::Map::new() + }; + if specs.is_empty() { + document.remove("lib"); + } else { + document.insert( + "lib".into(), + serde_json::json!(specs.iter().map(LibSpec::key).collect::<Vec<_>>()), + ); + } + let mut bytes = serde_json::to_vec_pretty(&document).map_err(|e| e.to_string())?; + bytes.push(b'\n'); + let temporary = parent.join(format!( + ".lib.json.{}-{}.tmp", + std::process::id(), + unique_suffix() + )); + write_new_file(&temporary, &bytes) + .map_err(|e| format!("cannot stage '{}': {e}", path.display()))?; + fs::rename(&temporary, &path).map_err(|e| { + let _ = fs::remove_file(&temporary); + format!("cannot replace '{}': {e}", path.display()) + }) +} + +pub(crate) fn reject_declaration_symlinks(project_root: &Path) -> Result<(), String> { + for path in [ + project_root.to_path_buf(), + project_root.join("config"), + declaration_path(project_root), + ] { + match fs::symlink_metadata(&path) { + Ok(metadata) if metadata.file_type().is_symlink() => { + return Err(format!( + "Lib configuration '{}' must not be a symbolic link", + path.display() + )); + } + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(format!( + "cannot inspect Lib configuration '{}': {error}", + path.display() + )); + } + } + } + Ok(()) +} + +fn normalize_libs(libs: &mut [LockedLib]) -> Result<(), String> { + for lib in libs.iter_mut() { + lib.dependencies.sort(); + lib.dependencies.dedup(); + for dependency in &lib.dependencies { + validate_name(&dependency.spec.ecosystem, &dependency.spec.name)?; + validate_version(&dependency.spec.ecosystem, &dependency.spec.version)?; + } + lib.artifacts.sort(); + validate_name(&lib.spec.ecosystem, &lib.spec.name)?; + validate_version(&lib.spec.ecosystem, &lib.spec.version)?; + validate_runtime(&lib.runtime)?; + for artifact in &lib.artifacts { + validate_artifact(artifact)?; + } + } + libs.sort_by(|a, b| a.spec.cmp(&b.spec)); + for pair in libs.windows(2) { + if pair[0].spec == pair[1].spec { + return Err(format!("duplicate locked lib {}", pair[0].spec.key())); + } + } + Ok(()) +} + +fn normalize_workers(workers: &mut [LockedWorker]) -> Result<(), String> { + for worker in workers.iter_mut() { + if worker.port.is_empty() + || worker.adapter.is_empty() + || worker.schema.is_empty() + || worker.operations.is_empty() + { + return Err("locked Worker identity and operations must be complete".into()); + } + let entry = Path::new(&worker.entry); + if worker.entry.is_empty() + || worker.entry.contains(['\\', ':']) + || entry.is_absolute() + || entry + .components() + .any(|part| !matches!(part, Component::Normal(_))) + { + return Err("locked Worker entry must be a portable relative path".into()); + } + worker.libs.sort(); + worker.libs.dedup(); + for spec in &worker.libs { + validate_name(&spec.ecosystem, &spec.name)?; + validate_version(&spec.ecosystem, &spec.version)?; + } + worker.operations.sort(); + worker.capabilities.sort(); + if !["exec", "pip", "npm", "go"].contains(&worker.ecosystem.as_str()) { + return Err("locked Worker ecosystem is unknown".into()); + } + if worker.ecosystem == "exec" { + if worker.runtime.is_some() { + return Err("exec Worker must not select a managed runtime".into()); + } + } else { + let runtime = worker + .runtime + .as_ref() + .ok_or("managed Worker lacks a runtime pack identity")?; + validate_runtime(runtime)?; + } + if worker.operations.iter().any(|name| name.is_empty()) + || worker.operations.windows(2).any(|pair| pair[0] == pair[1]) + { + return Err("locked Worker operations must be unique and nonempty".into()); + } + if worker + .capabilities + .iter() + .any(|name| !["network", "file", "process", "gpu"].contains(&name.as_str())) + || worker + .capabilities + .windows(2) + .any(|pair| pair[0] == pair[1]) + { + return Err("locked Worker capabilities are unknown or duplicated".into()); + } + } + workers.sort_by(|left, right| { + left.port + .cmp(&right.port) + .then(left.adapter.cmp(&right.adapter)) + }); + if workers + .windows(2) + .any(|pair| pair[0].port == pair[1].port && pair[0].adapter == pair[1].adapter) + { + return Err("duplicate locked Worker identity".into()); + } + Ok(()) +} + +fn validate_name(ecosystem: &Ecosystem, name: &str) -> Result<(), String> { + if name.is_empty() + || name.len() > 240 + || name + .chars() + .any(|c| c.is_control() || c.is_whitespace() || c == '\\') + { + return Err(format!("invalid {} lib name", ecosystem.as_str())); + } + match ecosystem { + Ecosystem::Pip => { + let canonical = python_name(name)?.0; + if canonical != name { + return Err(format!( + "pip lib name '{name}' is not canonical; use '{canonical}'" + )); + } + Ok(()) + } + Ecosystem::Npm => { + let valid_atom = |atom: &str| { + !atom.is_empty() + && atom != "." + && atom != ".." + && atom + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.' | '~')) + }; + if let Some((scope, package)) = name + .strip_prefix('@') + .and_then(|value| value.split_once('/')) + { + if !valid_atom(scope) || !valid_atom(package) { + return Err("invalid npm scoped lib name".into()); + } + } else if !valid_atom(name) { + return Err("invalid npm lib name".into()); + } + Ok(()) + } + Ecosystem::Go => { + if name.split('/').any(|part| { + part.is_empty() + || part == "." + || part == ".." + || !part + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '_' | '~')) + }) { + return Err("invalid go module name".into()); + } + Ok(()) + } + } +} + +fn validate_version(ecosystem: &Ecosystem, value: &str) -> Result<(), String> { + if value.is_empty() + || value.len() > 128 + || !value.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'+' | b'-' | b'_' | b'!') + }) + { + return Err(format!("lib version '{value}' must be exact and path-safe")); + } + let valid = match ecosystem { + Ecosystem::Pip => value.parse::<pep440_rs::Version>().is_ok(), + Ecosystem::Npm => value.parse::<node_semver::Version>().is_ok(), + Ecosystem::Go => value + .trim_start_matches('v') + .parse::<node_semver::Version>() + .is_ok(), + }; + if !valid { + return Err(format!( + "invalid exact {} version '{value}'", + ecosystem.as_str() + )); + } + Ok(()) +} + +fn validate_runtime(runtime: &RuntimePack) -> Result<(), String> { + if runtime.name.is_empty() || runtime.version.is_empty() { + return Err("runtime pack identity is incomplete".into()); + } + validate_digest(&runtime.sha256) +} + +fn validate_artifact(artifact: &LockedArtifact) -> Result<(), String> { + if artifact.target.is_empty() || artifact.bytes == 0 { + return Err("artifact target/size is invalid".into()); + } + let path = Path::new(&artifact.path); + if path.is_absolute() + || artifact.path.contains(['\\', ':']) + || artifact + .path + .split('/') + .any(|part| part.is_empty() || part == "." || part == "..") + || path + .components() + .any(|component| !matches!(component, Component::Normal(_))) + { + return Err(format!( + "artifact path '{}' is not a safe relative archive path", + artifact.path + )); + } + validate_digest(&artifact.sha256) +} + +fn validate_digest(value: &str) -> Result<(), String> { + if value.len() != 64 + || !value + .bytes() + .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) + { + return Err("artifact digest must be lowercase SHA-256".into()); + } + Ok(()) +} + +fn sha256(bytes: &[u8]) -> String { + let mut digest = Sha256::new(); + digest.update(bytes); + digest + .finalize() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +fn sha256_bytes(bytes: &[u8]) -> String { + sha256(bytes) +} + +fn write_new_file(path: &Path, bytes: &[u8]) -> std::io::Result<()> { + use std::io::Write; + let mut options = std::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options.open(path)?; + file.write_all(bytes)?; + file.sync_all() +} + +fn unique_suffix() -> u64 { + use std::sync::atomic::{AtomicU64, Ordering}; + static NEXT: AtomicU64 = AtomicU64::new(0); + NEXT.fetch_add(1, Ordering::Relaxed) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn spec_requires_exact_version_and_known_ecosystem() { + assert!("pip:fixture@1.0.0".parse::<LibSpec>().is_ok()); + assert!("pip:fixture@latest".parse::<LibSpec>().is_err()); + assert!("ruby:fixture@1.0.0".parse::<LibSpec>().is_err()); + } + + #[test] + fn builtin_resolution_is_deterministic() { + let requested = vec![ + "npm:fixture@1.0.0".parse().unwrap(), + "pip:fixture@1.0.0".parse().unwrap(), + ]; + let first = FixtureRegistry::builtin() + .resolve(&requested) + .unwrap() + .encode() + .unwrap(); + let second = FixtureRegistry::builtin() + .resolve(&requested) + .unwrap() + .encode() + .unwrap(); + assert_eq!(first, second); + } + + #[test] + fn lock_rejects_noncanonical_or_unsafe_artifact() { + let mut lock = FixtureRegistry::builtin() + .resolve(&["pip:fixture@1.0.0".parse().unwrap()]) + .unwrap(); + let bytes = lock.encode().unwrap(); + assert!(LockFile::decode(&bytes).is_ok()); + lock.libs[0].artifacts[0].path = "../escape".into(); + assert!(lock.encode().is_err()); + } + + #[test] + fn conflicts_are_explicit() { + let mut registry = FixtureRegistry::default(); + let a: LibSpec = "pip:a@1.0.0".parse().unwrap(); + let b: LibSpec = "pip:b@1.0.0".parse().unwrap(); + let dep: LibSpec = "pip:a@2.0.0".parse().unwrap(); + for spec in [a.clone(), dep.clone(), b.clone()] { + registry + .register(FixturePackage { + spec: spec.clone(), + dependencies: if spec == b { vec![dep.clone()] } else { vec![] }, + runtime: RuntimePack { + name: "cpython".into(), + version: "1".into(), + sha256: "00".repeat(32), + }, + artifacts: vec![FixtureArtifact { + target: "host".into(), + path: "a.bin".into(), + bytes: vec![1], + }], + schema: "s".into(), + }) + .unwrap(); + } + assert!(registry.resolve(std::slice::from_ref(&a)).is_ok()); + assert!(registry.resolve(&[a, b]).is_err()); + } +} diff --git a/crates/dever-cli/src/libs/build.rs b/crates/dever-cli/src/libs/build.rs new file mode 100644 index 0000000..561d5a6 --- /dev/null +++ b/crates/dever-cli/src/libs/build.rs @@ -0,0 +1,526 @@ +//! Explicit Lib preparation: shared owned staging and bounded tool execution. + +mod npm; +pub mod pack; +pub(crate) mod process; +mod python; + +use super::{Ecosystem, LibSpec, LockFile, LockedArtifact, RegistryRuntime, RuntimePack}; +use serde::{Deserialize, Serialize}; +use std::cell::RefCell; +use std::collections::{BTreeMap, BTreeSet}; + +fn require_execution_target(target: &str, operation: &str) -> Result<(), String> { + if target != crate::toolchain::BuildTarget::host()?.platform() { + return Err(format!( + "{operation} for {target} requires prepared target output and a complete build receipt; prepare the signed target inputs in a matching isolated execution environment before the offline build" + )); + } + Ok(()) +} + +/// Production supplies only authenticated release resources. Explicit author +/// tests inject the same immutable inputs without creating a host-tool fallback. +pub trait Inputs { + fn runtime(&self, ecosystem: Ecosystem) -> Result<(RegistryRuntime, Vec<u8>), String>; + fn tools(&self, ecosystem: Ecosystem) -> Result<(pack::Descriptor, Vec<u8>), String>; + fn assets(&self) -> Result<Vec<(String, Vec<u8>)>, String>; + /// Author tools may display bounded hook output without retaining temporary + /// staging trees or putting diagnostics in reproducible build identities. + fn diagnostic(&self, _output: &[u8]) {} +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PythonSystem { + pub requires: Vec<String>, + pub backend: String, + pub backend_path: Vec<String>, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum NativeRejection { + Format, + Target, + Closure, + Unloadable, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Receipt { + pub ecosystem: Ecosystem, + pub roots: Vec<LibSpec>, + pub sources: Vec<LockedArtifact>, + pub graph: Option<super::npm::Environment>, + pub runtime: RuntimePack, + pub auxiliary_runtime: Option<RuntimePack>, + pub native_entries: BTreeSet<String>, + pub native_rejections: BTreeMap<String, NativeRejection>, + pub python_markers: Option<pep508_rs::MarkerEnvironment>, + pub tools: pack::Descriptor, + pub frontend: String, + pub python: Option<PythonSystem>, + pub dynamic_requires: Vec<String>, + pub dependencies: Box<LockFile>, + pub config_settings: BTreeMap<String, serde_json::Value>, + pub output: LockedArtifact, +} + +impl Receipt { + pub fn identity(&self) -> Result<String, String> { + serde_json::to_vec(self) + .map(|bytes| super::sha256(&bytes)) + .map_err(|error| error.to_string()) + } +} + +#[derive(Clone)] +pub(crate) struct Product { + pub bytes: Vec<u8>, + pub filename: String, + pub receipt: Receipt, +} + +pub struct Session<'a> { + inputs: &'a dyn Inputs, + active: RefCell<BTreeSet<String>>, + products: RefCell<BTreeMap<String, Product>>, +} + +impl<'a> Session<'a> { + pub fn new(inputs: &'a dyn Inputs) -> Self { + Self { + inputs, + active: RefCell::new(BTreeSet::new()), + products: RefCell::new(BTreeMap::new()), + } + } + + pub(crate) fn receipts(&self, identities: &BTreeSet<String>) -> Result<Vec<Receipt>, String> { + let receipts = self + .products + .borrow() + .values() + .map(|product| Ok((product.receipt.identity()?, product.receipt.clone()))) + .collect::<Result<BTreeMap<_, _>, String>>()?; + identities + .iter() + .map(|identity| { + receipts + .get(identity) + .cloned() + .ok_or("prepared build receipt is missing".into()) + }) + .collect() + } + + fn remember(&self, key: String, product: Product) -> Result<(), String> { + let total = self + .products + .borrow() + .values() + .try_fold(product.bytes.len(), |total, cached| { + total.checked_add(cached.bytes.len()) + }) + .ok_or("build candidate cache byte overflow")?; + if total > crate::workers::MAX_TREE_BYTES { + return Err("build candidate cache exceeds byte budget".into()); + } + self.products.borrow_mut().insert(key, product); + Ok(()) + } +} + +pub fn normalize(receipts: &mut Vec<Receipt>) -> Result<(), String> { + let mut indexed = BTreeMap::new(); + for receipt in receipts.drain(..) { + indexed.insert(receipt.identity()?, receipt); + } + *receipts = indexed.into_values().collect(); + Ok(()) +} + +pub fn npm_outputs(lock: &LockFile) -> impl Iterator<Item = &Receipt> { + lock.builds + .iter() + .filter(|receipt| receipt.ecosystem == Ecosystem::Npm) +} + +pub(crate) fn npm_receipt<'a>( + lock: &'a LockFile, + environment: &super::npm::Environment, +) -> Result<Option<&'a Receipt>, String> { + let Some(identity) = &environment.build else { + return Ok(None); + }; + lock.builds + .iter() + .find(|receipt| receipt.identity().as_ref() == Ok(identity)) + .map(Some) + .ok_or("npm installation receipt is missing".into()) +} + +/// Offline consumption never invokes a frontend. Removing roots projects the +/// already built graph and output; it does not rerun lifecycle side effects. +pub fn npm_installation( + lock: &LockFile, + environment: &super::npm::Environment, + bytes: &[u8], +) -> Result<Vec<(String, Vec<u8>, bool)>, String> { + let receipt = npm_receipt(lock, environment)?.ok_or("npm installation has no build receipt")?; + if bytes.len() as u64 != receipt.output.bytes || super::sha256(bytes) != receipt.output.sha256 { + return Err("npm built installation differs from its locked artifact".into()); + } + let graph = receipt + .graph + .as_ref() + .ok_or("npm receipt lacks its exact graph")?; + let files = super::archive_files("tgz", bytes)?; + super::npm::validate_built_files(graph, &files)?; + let candidates = files + .iter() + .filter(|(path, _)| path.ends_with(".node")) + .map(|(path, _)| path.clone()) + .collect::<BTreeSet<_>>(); + let accounted = receipt + .native_entries + .iter() + .chain(receipt.native_rejections.keys()) + .cloned() + .collect::<BTreeSet<_>>(); + if candidates != accounted + || receipt + .native_rejections + .keys() + .any(|path| receipt.native_entries.contains(path)) + { + return Err("npm native admission does not cover its exact addon files".into()); + } + for path in &receipt.native_entries { + if !files.iter().any(|(name, bytes)| { + name == path && path.ends_with(".node") && bytes.starts_with(b"\x7fELF") + }) { + return Err("npm locked native entry point is absent from output".into()); + } + } + let executable = super::registry::archive_executables("tgz", bytes)?; + let retained = environment + .instances + .iter() + .map(|node| node.path.as_str()) + .collect::<BTreeSet<_>>(); + let mut output = Vec::new(); + for (path, bytes) in files { + if path.ends_with(".node") && !receipt.native_entries.contains(&path) { + continue; + } + if retained.contains(super::npm::installed_owner(&path)?.as_str()) { + let mode = executable.contains(&path); + output.push((path, bytes, mode)); + } + } + Ok(output) +} + +pub fn doctor(lock: &LockFile) -> Result<(), String> { + doctor_at(lock, 0) +} + +fn doctor_at(lock: &LockFile, depth: usize) -> Result<(), String> { + if depth > 16 { + return Err("build dependency depth exceeds limit".into()); + } + let records = lock + .builds + .iter() + .map(|receipt| Ok((receipt.identity()?, receipt))) + .collect::<Result<BTreeMap<_, _>, String>>()?; + if records.len() != lock.builds.len() { + return Err("duplicate build receipt".into()); + } + let mut used = BTreeSet::new(); + for lib in &lock.libs { + if let Some(identity) = &lib.build { + let receipt = records + .get(identity) + .ok_or("Python Lib has no exact build receipt")?; + if lib.spec.ecosystem != Ecosystem::Pip + || receipt.ecosystem != Ecosystem::Pip + || receipt.runtime != lib.runtime + || !lib.artifacts.contains(&receipt.output) + || receipt.roots.len() != 1 + || receipt.roots[0].distribution_name() != lib.spec.distribution_name() + || receipt.roots[0].version != lib.spec.version + { + return Err( + "Python Lib build receipt differs from locked artifact identity".into(), + ); + } + used.insert(identity.clone()); + } + } + for environment in &lock.npm { + if let Some(identity) = &environment.build { + let receipt = records + .get(identity) + .ok_or("npm environment has no exact build receipt")?; + let graph = receipt + .graph + .as_ref() + .ok_or("npm build receipt lacks its graph")?; + if receipt.ecosystem != Ecosystem::Npm + || receipt.roots != graph.roots + || graph.build.is_some() + || !environment + .roots + .iter() + .all(|root| graph.roots.contains(root)) + || !environment + .instances + .iter() + .all(|node| graph.instances.contains(node)) + || !environment.instances.iter().all(|node| { + lock.libs.iter().any(|lib| { + lib.spec == node.source.archive + && lib.runtime == receipt.runtime + && lib + .artifacts + .iter() + .all(|artifact| receipt.sources.contains(artifact)) + }) + }) + { + return Err("npm installation build receipt differs from locked graph".into()); + } + used.insert(identity.clone()); + } + } + if used.len() != records.len() { + return Err("lock contains an unreferenced build receipt".into()); + } + for receipt in records.values() { + super::validate_runtime(&receipt.runtime)?; + if receipt.tools.runtime_sha256 != receipt.runtime.sha256 + || receipt.tools.ecosystem != receipt.ecosystem + || receipt.sources.is_empty() + || receipt.output.target != receipt.tools.target + || receipt.roots.is_empty() + || receipt.frontend != frontend(&receipt.ecosystem)? + { + return Err("build receipt has inconsistent tools, frontend, or target".into()); + } + for artifact in receipt.sources.iter().chain([&receipt.output]) { + super::validate_artifact(artifact)?; + if artifact.target != receipt.tools.target { + return Err("build artifact target differs from its tools".into()); + } + } + if receipt.ecosystem == Ecosystem::Pip + && (receipt.python.is_none() + || receipt.graph.is_some() + || receipt.auxiliary_runtime.is_some()) + { + return Err("Python build receipt lacks its backend configuration".into()); + } + if receipt.ecosystem == Ecosystem::Pip + && (!receipt.native_entries.is_empty() || !receipt.native_rejections.is_empty()) + { + return Err("Python receipt cannot declare Node native entry points".into()); + } + for path in receipt + .native_entries + .iter() + .chain(receipt.native_rejections.keys()) + { + crate::workers::valid_path(path)?; + } + let policy = RegistryRuntime { + pack: receipt.runtime.clone(), + target: receipt.tools.target.clone(), + python_markers: receipt.python_markers.clone(), + python_wheel_tags: vec![], + npm_libc: None, + }; + let auxiliary = receipt + .auxiliary_runtime + .as_ref() + .map(|runtime| RegistryRuntime { + pack: runtime.clone(), + target: receipt.tools.target.clone(), + python_markers: None, + python_wheel_tags: vec![], + npm_libc: None, + }); + if let Some(runtime) = &receipt.auxiliary_runtime { + super::validate_runtime(runtime)?; + } + receipt.tools.validate(&policy, auxiliary.as_ref())?; + if receipt.ecosystem == Ecosystem::Npm + && (receipt.python.is_some() + || receipt.python_markers.is_some() + || !receipt.dynamic_requires.is_empty() + || !receipt.dependencies.libs.is_empty() + || receipt.graph.is_none()) + { + return Err("npm build receipt contains Python backend state".into()); + } + if !receipt.config_settings.is_empty() { + return Err("unsupported source-build configuration settings".into()); + } + if let Some(system) = &receipt.python { + validate_requirements(receipt, system)?; + } + if !receipt.dependencies.workers.is_empty() + || !receipt.dependencies.packages.is_empty() + || receipt + .dependencies + .libs + .iter() + .any(|lib| lib.spec.ecosystem != Ecosystem::Pip) + { + return Err("build dependency closure is not a separate Python environment".into()); + } + doctor_at(&receipt.dependencies, depth + 1)?; + // Full graph checks belong to the ordinary lock owner; recursion is + // bounded above before invoking that owner on a dependency lock. + let mut dependencies = (*receipt.dependencies).clone(); + dependencies.builds.clear(); + for lib in &mut dependencies.libs { + lib.build = None; + } + super::doctor(&dependencies)?; + } + Ok(()) +} + +fn validate_requirements(receipt: &Receipt, system: &PythonSystem) -> Result<(), String> { + let identifier = |path: &str| { + !path.is_empty() + && path.split('.').all(|part| { + let mut bytes = part.bytes(); + bytes + .next() + .is_some_and(|byte| byte.is_ascii_alphabetic() || byte == b'_') + && bytes.all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') + }) + }; + let (module, function) = system + .backend + .split_once(':') + .map(|(module, function)| (module, Some(function))) + .unwrap_or((&system.backend, None)); + if !identifier(module) + || function.is_some_and(|function| !identifier(function)) + || system.requires.len() > 1024 + || receipt.dynamic_requires.len() > 1024 + || system.backend_path.len() > 64 + { + return Err("build receipt has an invalid backend or requirement set".into()); + } + for path in &system.backend_path { + if path.is_empty() || path.starts_with('/') || path.contains(['\\', '\0']) { + return Err("invalid backend-path in receipt".into()); + } + let mut depth = 0usize; + for part in path.split('/') { + match part { + "" | "." => {} + ".." => { + depth = depth.checked_sub(1).ok_or("backend-path escapes source")?; + } + _ => depth += 1, + } + } + } + let markers = receipt + .python_markers + .as_ref() + .ok_or("Python build receipt lacks marker policy")?; + let requirements = system + .requires + .iter() + .chain(&receipt.dynamic_requires) + .cloned() + .collect::<Vec<_>>(); + let document = serde_json::json!({"info":{"requires_dist":requirements}}); + let expected = super::python_dependencies(&document, markers, &[])?; + let libraries = receipt + .dependencies + .libs + .iter() + .map(|lib| (lib.spec.name.as_str(), lib)) + .collect::<BTreeMap<_, _>>(); + let mut remaining = Vec::new(); + for (name, rule) in expected { + let lib = libraries + .get(name.as_str()) + .ok_or("Python build receipt omits a required dependency")?; + let version = lib + .spec + .version + .parse::<pep440_rs::Version>() + .map_err(|error| error.to_string())?; + if !rule.contains(&version) { + return Err("Python build dependency does not satisfy recorded requirements".into()); + } + remaining.push(lib.spec.name.as_str()); + } + let mut reached = BTreeSet::new(); + while let Some(name) = remaining.pop() { + if !reached.insert(name) { + continue; + } + let lib = libraries + .get(name) + .ok_or("build dependency closure is incomplete")?; + if lib.runtime != receipt.runtime + || lib + .artifacts + .iter() + .any(|artifact| artifact.target != receipt.tools.target) + { + return Err("build dependency runtime or target differs from its receipt".into()); + } + remaining.extend(lib.dependencies.iter().map(|edge| edge.spec.name.as_str())); + } + if reached.len() != libraries.len() { + return Err("build dependency lock contains unrelated libraries".into()); + } + Ok(()) +} + +/// Offline consumers bind recorded marker evaluation to the immutable runtime +/// manifest they have just verified, never to a lock's self-declared policy. +pub fn validate_python_policy( + lock: &LockFile, + lib: &super::LockedLib, + markers: &pep508_rs::MarkerEnvironment, +) -> Result<(), String> { + if let Some(identity) = &lib.build { + let receipt = lock + .builds + .iter() + .find(|receipt| receipt.identity().as_ref() == Ok(identity)) + .ok_or("Python build receipt missing")?; + if receipt.python_markers.as_ref() != Some(markers) { + return Err("Python build marker policy differs from signed runtime".into()); + } + } + Ok(()) +} + +fn frontend(ecosystem: &Ecosystem) -> Result<String, String> { + match ecosystem { + Ecosystem::Pip => Ok(super::sha256(include_bytes!("build/python.py"))), + Ecosystem::Npm => Ok(super::sha256( + &[ + include_bytes!("build/npm.js").as_slice(), + include_bytes!("build/npm/probe.cjs").as_slice(), + ] + .concat(), + )), + _ => Err("source-build frontend is not available for this ecosystem".into()), + } +} diff --git a/crates/dever-cli/src/libs/build/npm.js b/crates/dever-cli/src/libs/build/npm.js new file mode 100644 index 0000000..308bc09 --- /dev/null +++ b/crates/dever-cli/src/libs/build/npm.js @@ -0,0 +1,103 @@ +// Fixed registry-package lifecycle frontend. Runs only in the build sandbox. +const fs = require('node:fs'); +const path = require('node:path'); +const runScript = require('/worker/npm/frontend/node_modules/npm/node_modules/@npmcli/run-script'); +const binLinks = require('/worker/npm/frontend/node_modules/npm/node_modules/bin-links'); + +async function main() { + const request = JSON.parse(fs.readFileSync('/worker/request.json', 'utf8')); + const root = '/data/build/output/install'; + fs.mkdirSync('/data/build/output'); + fs.cpSync('/worker/install', root, { recursive: true, dereference: false, errorOnExist: true }); + fs.mkdirSync('/data/build/tmp'); + fs.mkdirSync('/data/build/home'); + Object.assign(process.env, { + PATH: '/worker/runtime/bin:/usr/bin', + CC: '/usr/bin/cc', CXX: '/usr/bin/c++', AR: '/usr/bin/ar', + PYTHON: request.python, + npm_config_python: request.python, + npm_config_nodedir: '/worker/npm', + npm_config_node_gyp: '/worker/npm/frontend/node_modules/npm/node_modules/node-gyp/bin/node-gyp.js', + npm_config_build_from_source: 'true', + npm_config_cache: '/data/build/npm-cache', + TMPDIR: '/data/build/tmp', SOURCE_DATE_EPOCH: '0', + HOME: '/data/build/home', + }); + const nodes = new Map(request.instances.map(node => [node.path, node])); + for (const node of request.instances) { + const directory = path.join(root, node.path); + const pkg = JSON.parse(fs.readFileSync(path.join(directory, 'package.json'), 'utf8')); + await binLinks({ path: directory, pkg, global: false, top: false, force: false }); + } + const visited = new Set(); + const required = new Set(request.required); + const failed = new Set(); + const nativeEntries = new Set(); + async function install(name) { + if (visited.has(name)) return; + visited.add(name); + const node = nodes.get(name); + try { + for (const edge of node.edges) { + if (edge.target && nodes.has(edge.target)) { + await install(edge.target); + if (failed.has(edge.target) && ['dependency', 'peer'].includes(edge.kind)) { + throw new Error('required lifecycle dependency failed: ' + edge.target); + } + } + } + const directory = path.join(root, name); + for (const event of ['preinstall', 'install', 'postinstall']) { + await runScript({ event, path: directory, stdio: 'inherit', scriptShell: '/bin/sh' }); + } + const native = path.join(directory, 'build', 'Release'); + if (fs.existsSync(native)) { + for (const file of fs.readdirSync(native)) { + if (file.endsWith('.node')) { + const addon = path.join(native, file); + require(addon); + nativeEntries.add(path.relative(root, addon)); + } + } + } + } catch (error) { + if (required.has(name)) throw error; + process.stderr.write('optional npm lifecycle failed: ' + name + ': ' + String(error.message || error) + '\n'); + failed.add(name); + } + } + for (const node of request.instances) await install(node.path); + let changed = true; + while (changed) { + changed = false; + for (const node of request.instances) { + if (!failed.has(node.path) && node.edges.some(edge => ['dependency', 'peer'].includes(edge.kind) && failed.has(edge.target))) { + if (required.has(node.path)) throw new Error('required lifecycle dependency failed: ' + node.path); + failed.add(node.path); + changed = true; + } + } + } + const retained = new Set(); + function retain(name) { + if (retained.has(name) || failed.has(name)) return; + retained.add(name); + for (const edge of nodes.get(name).edges) { + if (edge.target) retain(edge.target); + } + } + for (const node of request.roots) retain(node); + for (const node of request.instances) { + if (!retained.has(node.path)) fs.rmSync(path.join(root, node.path), { recursive: true, force: true }); + } + // Bin links are a build-time PATH input. The immutable Worker tree stores + // package files only; it never persists symlinks created by installer hooks. + for (const node of request.instances) { + fs.rmSync(path.join(root, node.path, 'node_modules', '.bin'), { recursive: true, force: true }); + fs.rmSync(path.join(root, node.path, 'build', 'node_gyp_bins'), { recursive: true, force: true }); + } + fs.rmSync(path.join(root, 'node_modules', '.bin'), { recursive: true, force: true }); + fs.writeFileSync('/data/build/output/result.json', JSON.stringify({ failed: [...failed].sort(), native: [...nativeEntries].filter(file => retained.has(file.split('/build/Release/')[0])).sort() })); +} + +main().catch(error => { process.stderr.write(String(error.stack || error) + '\n'); process.exitCode = 1; }); diff --git a/crates/dever-cli/src/libs/build/npm.rs b/crates/dever-cli/src/libs/build/npm.rs new file mode 100644 index 0000000..f6f053f --- /dev/null +++ b/crates/dever-cli/src/libs/build/npm.rs @@ -0,0 +1,230 @@ +//! Registry lifecycle scripts use the signed npm frontend and exact instance graph. + +use std::collections::{BTreeMap, BTreeSet, VecDeque}; +use std::io::Cursor; + +use super::{Product, Receipt, Session}; +use crate::libs::{ + Ecosystem, LockFile, LockedArtifact, LockedLib, RegistryResolver, RegistryRuntime, npm, sha256, +}; + +mod environment; +use environment::{Environment, Output}; + +impl Session<'_> { + pub(crate) fn npm( + &self, + resolver: &RegistryResolver<'_>, + libraries: &mut Vec<LockedLib>, + graph: &mut npm::Environment, + runtime: &RegistryRuntime, + ) -> Result<(), String> { + let mut lock = LockFile::new(libraries.clone())?; + lock.npm.push(graph.clone()); + let archives = libraries + .iter() + .map(|lib| { + let artifact = lib.artifacts.first().ok_or("npm source archive missing")?; + Ok(( + lib.spec.clone(), + resolver.store.verify_exact( + &artifact.sha256, + artifact.bytes, + &artifact.target, + )?, + )) + }) + .collect::<Result<BTreeMap<_, _>, String>>()?; + let files = npm::build_files(graph, &lock, |lib| { + archives + .get(&lib.spec) + .map(Vec::as_slice) + .ok_or("npm source archive missing".into()) + })?; + if !npm::requires_build(&files)? { + return Ok(()); + } + let mut sources = libraries + .iter() + .flat_map(|lib| lib.artifacts.clone()) + .collect::<Vec<_>>(); + sources.sort(); + sources.dedup(); + let key = format!( + "npm:{}:{}:{}", + sha256(&serde_json::to_vec(graph).map_err(|error| error.to_string())?), + runtime.pack.sha256, + sha256(&serde_json::to_vec(&sources).map_err(|error| error.to_string())?) + ); + if let Some(product) = self.products.borrow().get(&key) { + *graph = product + .receipt + .graph + .clone() + .ok_or("cached npm graph missing")?; + graph.build = Some(product.receipt.identity()?); + retain_archives(libraries, graph); + return Ok(()); + } + let executables = npm::build_executables(graph, &archives)?; + let environment = Environment::new(self.inputs, runtime)?; + let output = environment.run(self.inputs, graph, &files, &executables)?; + prune_failed(graph, &output.failed)?; + retain_archives(libraries, graph); + npm::validate_built_files(graph, &output.files)?; + let bytes = archive(&output)?; + let digest = resolver.store.publish(&bytes, &runtime.target)?; + if digest != sha256(&bytes) { + return Err("built npm cache identity mismatch".into()); + } + let output_artifact = LockedArtifact { + target: runtime.target.clone(), + path: format!("build/npm/{digest}/installation.tgz"), + bytes: bytes.len() as u64, + sha256: digest, + }; + let receipt = Receipt { + ecosystem: Ecosystem::Npm, + roots: graph.roots.clone(), + sources, + graph: Some(graph.clone()), + runtime: runtime.pack.clone(), + auxiliary_runtime: Some(environment.python_runtime), + native_entries: output.native, + native_rejections: output.rejections, + python_markers: None, + tools: environment.descriptor, + frontend: super::frontend(&Ecosystem::Npm)?, + python: None, + dynamic_requires: vec![], + dependencies: Box::new(LockFile::new(vec![])?), + config_settings: BTreeMap::new(), + output: output_artifact, + }; + graph.build = Some(receipt.identity()?); + self.remember( + key, + Product { + bytes, + filename: "installation.tgz".into(), + receipt, + }, + )?; + Ok(()) + } +} + +fn archive(output: &Output) -> Result<Vec<u8>, String> { + let encoder = flate2::GzBuilder::new() + .mtime(0) + .operating_system(255) + .write(Vec::new(), flate2::Compression::default()); + let mut archive = tar::Builder::new(encoder); + for (path, bytes) in &output.files { + let mut header = tar::Header::new_gnu(); + header.set_size(bytes.len() as u64); + header.set_mode(if output.executables.contains(path) { + 0o755 + } else { + 0o644 + }); + header.set_uid(0); + header.set_gid(0); + header.set_mtime(0); + header.set_cksum(); + archive + .append_data(&mut header, path, Cursor::new(bytes)) + .map_err(|error| error.to_string())?; + } + archive + .into_inner() + .and_then(|encoder| encoder.finish()) + .map_err(|error| error.to_string()) +} + +fn retain_archives(libraries: &mut Vec<LockedLib>, graph: &npm::Environment) { + let used = graph + .instances + .iter() + .map(|node| &node.source.archive) + .collect::<BTreeSet<_>>(); + libraries.retain(|lib| used.contains(&lib.spec)); +} + +fn required_nodes(graph: &npm::Environment) -> Result<BTreeSet<String>, String> { + reachable(graph, true) +} + +fn reachable(graph: &npm::Environment, mandatory_only: bool) -> Result<BTreeSet<String>, String> { + let mut remaining = graph + .roots + .iter() + .map(|root| format!("node_modules/{}", root.name)) + .collect::<VecDeque<_>>(); + let mut reached = BTreeSet::new(); + while let Some(path) = remaining.pop_front() { + if !reached.insert(path.clone()) { + continue; + } + let node = graph + .instances + .iter() + .find(|node| node.path == path) + .ok_or("npm reachable instance missing")?; + remaining.extend( + node.edges + .iter() + .filter(|edge| { + !mandatory_only || matches!(edge.kind, npm::Kind::Dependency | npm::Kind::Peer) + }) + .filter_map(|edge| edge.target.clone()), + ); + } + Ok(reached) +} + +fn prune_failed(graph: &mut npm::Environment, failed: &BTreeSet<String>) -> Result<(), String> { + if failed + .iter() + .any(|path| !graph.instances.iter().any(|node| &node.path == path)) + { + return Err("npm build reported an unknown failed instance".into()); + } + let mut failed = failed.clone(); + loop { + let before = failed.len(); + for node in &graph.instances { + if node.edges.iter().any(|edge| { + matches!(edge.kind, npm::Kind::Dependency | npm::Kind::Peer) + && edge + .target + .as_ref() + .is_some_and(|path| failed.contains(path)) + }) { + failed.insert(node.path.clone()); + } + } + if before == failed.len() { + break; + } + } + if !required_nodes(graph)?.is_disjoint(&failed) { + return Err("required npm lifecycle failed".into()); + } + graph.instances.retain(|node| !failed.contains(&node.path)); + for node in &mut graph.instances { + for edge in &mut node.edges { + if edge + .target + .as_ref() + .is_some_and(|path| failed.contains(path)) + { + edge.target = None; + edge.omission = Some(npm::Omission::BuildFailure); + } + } + } + let reached = reachable(graph, false)?; + graph.instances.retain(|node| reached.contains(&node.path)); + Ok(()) +} diff --git a/crates/dever-cli/src/libs/build/npm/environment.rs b/crates/dever-cli/src/libs/build/npm/environment.rs new file mode 100644 index 0000000..2dff7a6 --- /dev/null +++ b/crates/dever-cli/src/libs/build/npm/environment.rs @@ -0,0 +1,326 @@ +//! One immutable Node/Python/tool environment for registry lifecycle execution. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; + +use crate::libs::build::NativeRejection; +use crate::libs::build::{Inputs, pack, process::Stage}; +use crate::libs::{Ecosystem, RegistryRuntime, RuntimePack, archive_files, npm, sha256}; +use crate::workers::runtime; + +pub(super) struct Environment { + files: BTreeMap<String, (Vec<u8>, bool)>, + assets: Vec<(String, Vec<u8>)>, + node: String, + node_paths: BTreeSet<String>, + python: String, + pub python_runtime: RuntimePack, + pub descriptor: pack::Descriptor, +} + +pub(super) struct Output { + pub files: Vec<(String, Vec<u8>)>, + pub executables: BTreeSet<String>, + pub failed: BTreeSet<String>, + pub native: BTreeSet<String>, + pub rejections: BTreeMap<String, NativeRejection>, +} + +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct ResultMetadata { + failed: BTreeSet<String>, + native: BTreeSet<String>, +} + +impl Environment { + pub fn new(inputs: &dyn Inputs, runtime: &RegistryRuntime) -> Result<Self, String> { + crate::libs::build::require_execution_target(&runtime.target, "npm lifecycle build")?; + let (node, bytes) = inputs.runtime(Ecosystem::Npm)?; + if node.pack != runtime.pack || sha256(&bytes) != runtime.pack.sha256 { + return Err("npm build runtime differs from resolver".into()); + } + let node_files = archive_files("tgz", &bytes)?; + let node_manifest = runtime::validate(&node_files, &Ecosystem::Npm, &runtime.target)?; + let node_paths = node_files + .iter() + .map(|(path, _)| format!("runtime/{path}")) + .collect(); + let node = format!( + "runtime/{}", + node_manifest + .executable + .as_deref() + .ok_or("Node executable missing")? + ); + let (python, bytes) = inputs.runtime(Ecosystem::Pip)?; + if sha256(&bytes) != python.pack.sha256 { + return Err("auxiliary Python runtime hash mismatch".into()); + } + let python_files = archive_files("tgz", &bytes)?; + let python_manifest = runtime::validate(&python_files, &Ecosystem::Pip, &runtime.target)?; + let python_executable = format!( + "runtime/{}", + python_manifest + .executable + .as_deref() + .ok_or("build Python executable missing")? + ); + let (descriptor, bytes) = inputs.tools(Ecosystem::Npm)?; + descriptor.validate(runtime, Some(&python))?; + let (manifest, tools) = pack::unpack(&bytes, &descriptor)?; + let assets = inputs.assets()?; + dever_sandbox::validate_assets(&assets)?; + pack::validate_tools(&manifest, &tools, &assets)?; + let mut files = BTreeMap::new(); + for (path, bytes) in node_files { + let executable = node_manifest.executable(&path); + files.insert(format!("runtime/{path}"), (bytes, executable)); + } + for (path, bytes) in python_files { + if path == "dever-runtime.json" { + continue; + } + let executable = python_manifest.executable(&path); + let path = format!("runtime/{path}"); + if let Some((existing, _)) = files.get(&path) { + if existing != &bytes { + return Err("Node and auxiliary Python runtime paths conflict".into()); + } + } else { + files.insert(path, (bytes, executable)); + } + } + for (path, bytes) in tools { + let executable = manifest.executables.contains(&path); + if files.insert(path, (bytes, executable)).is_some() { + return Err("npm tools overlap runtimes".into()); + } + } + Ok(Self { + files, + assets, + node, + node_paths, + python: python_executable, + python_runtime: python.pack, + descriptor, + }) + } + + pub fn run( + &self, + inputs: &dyn Inputs, + graph: &npm::Environment, + files: &[(String, Vec<u8>)], + executables: &BTreeSet<String>, + ) -> Result<Output, String> { + let stage = Stage::new()?; + let input = stage.0.join("input"); + for (path, (bytes, executable)) in &self.files { + stage.write(&format!("input/{path}"), bytes, *executable)?; + } + for (path, bytes) in &self.assets { + stage.write( + &format!("sandbox/{path}"), + bytes, + dever_sandbox::asset_executable(path), + )?; + } + for (path, bytes) in files { + stage.write( + &format!("input/install/{path}"), + bytes, + executables.contains(path), + )?; + } + let request = serde_json::json!({ + "instances":graph.instances, "required":super::required_nodes(graph)?, + "roots":graph.roots.iter().map(|root|format!("node_modules/{}",root.name)).collect::<Vec<_>>(), + "python":format!("/worker/{}", self.python), + }); + stage.write( + "input/request.json", + &serde_json::to_vec(&request).map_err(|error| error.to_string())?, + false, + )?; + stage.write("input/frontend.js", include_bytes!("../npm.js"), false)?; + let work = stage.0.join("work"); + fs::create_dir(&work).map_err(|error| error.to_string())?; + let grants = [dever_sandbox::Grant { + source: work.clone(), + destination: "/data/build".into(), + writable: true, + }]; + let arguments = [input.join("frontend.js").into_os_string()]; + let executable = input.join(&self.node); + let assets = stage.0.join("sandbox"); + let launch = dever_sandbox::Launch { + assets: &assets, + worker: &input, + executable: &executable, + arguments: &arguments, + working_directory: &input, + capabilities: dever_sandbox::Capabilities { + network: false, + process: true, + }, + grants: &grants, + }; + stage.run( + launch.command_for_build(&input.join("rootfs"))?, + "isolated npm lifecycle", + &work, + )?; + inputs.diagnostic(&stage.diagnostics()?); + let mut outputs = super::super::python::output_files(&work.join("output"))?; + let response: ResultMetadata = serde_json::from_slice( + &outputs + .remove("result.json") + .ok_or("npm build result missing")?, + ) + .map_err(|error| error.to_string())?; + let mut files = Vec::new(); + let mut executables = BTreeSet::new(); + for (path, bytes) in outputs { + let path = path + .strip_prefix("install/") + .ok_or("npm lifecycle output is outside its installation")? + .to_owned(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + if fs::symlink_metadata(work.join("output/install").join(&path)) + .map_err(|error| error.to_string())? + .permissions() + .mode() + & 0o111 + != 0 + { + executables.insert(path.clone()); + } + } + files.push((path, bytes)); + } + let mut output = Output { + files, + executables, + failed: response.failed, + native: response.native, + rejections: BTreeMap::new(), + }; + // Release the large compiler tree before staging the much smaller + // runtime-only native admission sandbox. + drop(stage); + self.admit_native(inputs, &mut output)?; + Ok(output) + } + + fn admit_native(&self, inputs: &dyn Inputs, output: &mut Output) -> Result<(), String> { + let required = std::mem::take(&mut output.native); + let candidates = output + .files + .iter() + .filter(|(path, _)| path.ends_with(".node")) + .collect::<Vec<_>>(); + if candidates.is_empty() { + return if required.is_empty() { + Ok(()) + } else { + Err("npm generated native entry point is missing".into()) + }; + } + let machine = match self.descriptor.target.as_str() { + "linux-x86_64" => goblin::elf::header::EM_X86_64, + "linux-aarch64" => goblin::elf::header::EM_AARCH64, + _ => return Err("Node native admission target is unsupported".into()), + }; + let mut files = self + .files + .iter() + .filter(|(path, _)| self.node_paths.contains(*path)) + .map(|(path, (bytes, _))| (path.clone(), bytes.as_slice())) + .collect::<BTreeMap<_, _>>(); + files.extend( + self.assets + .iter() + .map(|(path, bytes)| (format!("sandbox/{path}"), bytes.as_slice())), + ); + files.extend( + output + .files + .iter() + .map(|(path, bytes)| (path.clone(), bytes.as_slice())), + ); + let stage = Stage::new()?; + for (path, bytes) in &files { + let relative = if path.starts_with("sandbox/") { + path.clone() + } else { + format!("input/{path}") + }; + let executable = path == &self.node + || path + .strip_prefix("sandbox/") + .is_some_and(dever_sandbox::asset_executable); + stage.write(&relative, bytes, executable)?; + } + stage.write("input/probe.cjs", include_bytes!("probe.cjs"), false)?; + let input = stage.0.join("input"); + let executable = input.join(&self.node); + let assets = stage.0.join("sandbox"); + for (path, bytes) in candidates { + let rejection = match goblin::elf::Elf::parse(bytes) { + Err(_) => Some(NativeRejection::Format), + Ok(elf) if !elf.is_64 || !elf.little_endian || elf.header.e_machine != machine => { + Some(NativeRejection::Target) + } + Ok(_) => { + if let Err(error) = crate::workers::python_wheel::validate_native( + &self.descriptor.target, + &[], + &self.node, + &files, + &BTreeSet::from([path.clone()]), + ) { + inputs.diagnostic( + format!("npm addon '{path}' has no packaged loader closure: {error}\n") + .as_bytes(), + ); + Some(NativeRejection::Closure) + } else { + let arguments = [ + input.join("probe.cjs").into_os_string(), + input.join(path).into_os_string(), + ]; + let launch = dever_sandbox::Launch { + assets: &assets, + worker: &input, + executable: &executable, + arguments: &arguments, + working_directory: &input, + capabilities: dever_sandbox::Capabilities::default(), + grants: &[], + }; + let admitted = stage.probe(launch.command()?)?; + inputs.diagnostic(&stage.diagnostics()?); + if admitted { + None + } else { + Some(NativeRejection::Unloadable) + } + } + } + }; + if let Some(reason) = rejection { + output.rejections.insert(path.clone(), reason); + } else { + output.native.insert(path.clone()); + } + } + if !required.is_subset(&output.native) { + return Err("source-built npm addon did not pass runtime-only native admission".into()); + } + Ok(()) + } +} diff --git a/crates/dever-cli/src/libs/build/npm/probe.cjs b/crates/dever-cli/src/libs/build/npm/probe.cjs new file mode 100644 index 0000000..d336593 --- /dev/null +++ b/crates/dever-cli/src/libs/build/npm/probe.cjs @@ -0,0 +1,9 @@ +// No package entry modules are executed. Probe exactly one immutable addon in +// an isolated runtime-only process, so addon globals cannot affect its peers. +try { + require(process.argv[2]); + process.exit(0); +} catch (error) { + process.stderr.write(String(error.stack || error) + '\n'); + process.exit(10); +} diff --git a/crates/dever-cli/src/libs/build/pack.rs b/crates/dever-cli/src/libs/build/pack.rs new file mode 100644 index 0000000..a0648b6 --- /dev/null +++ b/crates/dever-cli/src/libs/build/pack.rs @@ -0,0 +1,166 @@ +//! Signed, target-specific source-build tools. Never sourced from host PATH. + +use crate::libs::{Ecosystem, RegistryRuntime}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; + +pub const MAX_PACK_BYTES: usize = 256 * 1024 * 1024; +pub const MAX_EXPANDED_BYTES: usize = 512 * 1024 * 1024; +pub const MAX_FILES: usize = 65_536; + +pub type PackFiles = Vec<(String, Vec<u8>)>; + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Manifest { + pub format: String, + pub ecosystem: Ecosystem, + pub target: String, + pub runtime_sha256: String, + pub python_runtime_sha256: Option<String>, + pub executables: Vec<String>, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Descriptor { + pub format: String, + pub ecosystem: Ecosystem, + pub target: String, + pub runtime_sha256: String, + pub python_runtime_sha256: Option<String>, + pub sha256: String, +} + +fn hash(value: &str) -> Result<(), String> { + if value.len() != 64 + || !value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err("build pack identity must be canonical SHA-256".into()); + } + Ok(()) +} + +impl Descriptor { + pub fn validate( + &self, + runtime: &RegistryRuntime, + python: Option<&RegistryRuntime>, + ) -> Result<(), String> { + if self.format != "dever-registry-build-v1" + || self.ecosystem == Ecosystem::Go + || self.target != runtime.target + || self.runtime_sha256 != runtime.pack.sha256 + { + return Err("signed build pack does not match its runtime and target".into()); + } + hash(&self.sha256)?; + hash(&self.runtime_sha256)?; + match (&self.ecosystem, &self.python_runtime_sha256, python) { + (Ecosystem::Pip, None, _) => Ok(()), + (Ecosystem::Npm, Some(identity), Some(python)) + if identity == &python.pack.sha256 && python.target == self.target => + { + hash(identity) + } + _ => Err("build pack has no exact auxiliary Python runtime binding".into()), + } + } +} + +impl Manifest { + pub fn validate( + &self, + files: &[(String, Vec<u8>)], + descriptor: &Descriptor, + ) -> Result<(), String> { + if self.format != "dever-lib-build-v1" + || self.ecosystem != descriptor.ecosystem + || self.target != descriptor.target + || self.runtime_sha256 != descriptor.runtime_sha256 + || self.python_runtime_sha256 != descriptor.python_runtime_sha256 + { + return Err("build pack manifest differs from its signed descriptor".into()); + } + let total = files + .iter() + .try_fold(0usize, |total, (_, bytes)| total.checked_add(bytes.len())) + .ok_or("build pack expanded size overflow")?; + if files.len() > MAX_FILES || total > MAX_EXPANDED_BYTES { + return Err("build pack exceeds file or byte budget".into()); + } + let mut names = BTreeSet::new(); + for (path, _) in files { + crate::workers::valid_path(path)?; + if path != "dever-build.json" + && !path.starts_with("rootfs/usr/") + && !(self.ecosystem == Ecosystem::Pip && path.starts_with("runtime/include/")) + && !(self.ecosystem == Ecosystem::Npm + && (path.starts_with("npm/include/node/") + || path.starts_with("npm/frontend/node_modules/"))) + { + return Err(format!( + "build pack path '{path}' is outside its fixed tool layout" + )); + } + if !names.insert(path.as_str()) { + return Err("build pack repeats a path".into()); + } + } + let mut executable_names = BTreeSet::new(); + for path in &self.executables { + if !names.contains(path.as_str()) + || !executable_names.insert(path.as_str()) + || !(path.starts_with("rootfs/usr/") + || self.ecosystem == Ecosystem::Npm + && path.starts_with("npm/frontend/node_modules/")) + { + return Err( + "build executable is absent, repeated, or outside its tool root".into(), + ); + } + } + for name in ["sh", "cc", "c++", "as", "ld", "ar", "ranlib", "make"] { + if !executable_names.contains(format!("rootfs/usr/bin/{name}").as_str()) { + return Err(format!("build pack is missing required tool '{name}'")); + } + } + Ok(()) + } +} + +pub fn unpack(bytes: &[u8], descriptor: &Descriptor) -> Result<(Manifest, PackFiles), String> { + if bytes.len() > MAX_PACK_BYTES || crate::libs::sha256(bytes) != descriptor.sha256 { + return Err("build pack exceeds its budget or SHA-256 differs".into()); + } + let files = super::super::registry::tar_files(bytes, "tgz", MAX_EXPANDED_BYTES as u64)?; + let metadata = files + .iter() + .find(|(path, _)| path == "dever-build.json") + .ok_or("build pack is missing dever-build.json")?; + let manifest: Manifest = serde_json::from_slice(&metadata.1) + .map_err(|error| format!("invalid build pack manifest: {error}"))?; + manifest.validate(&files, descriptor)?; + Ok((manifest, files)) +} + +pub fn validate_tools( + manifest: &Manifest, + files: &[(String, Vec<u8>)], + assets: &[(String, Vec<u8>)], +) -> Result<(), String> { + let mut supplied = files + .iter() + .map(|(path, bytes)| (path.clone(), bytes.as_slice())) + .collect::<BTreeMap<_, _>>(); + for (path, bytes) in assets { + supplied.insert(format!("sandbox/{path}"), bytes); + } + crate::workers::python_wheel::native::validate_build_tools( + &manifest.target, + &manifest.executables, + &supplied, + ) +} diff --git a/crates/dever-cli/src/libs/build/process.rs b/crates/dever-cli/src/libs/build/process.rs new file mode 100644 index 0000000..71addac --- /dev/null +++ b/crates/dever-cli/src/libs/build/process.rs @@ -0,0 +1,186 @@ +//! One lifecycle for every managed compiler and package build hook. + +use std::fs::{self, File}; +use std::io::Read; +use std::path::Path; +use std::path::PathBuf; +use std::process::{Child, Command, Stdio}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +const DEADLINE: Duration = Duration::from_secs(300); +const MAX_OUTPUT: u64 = 16 * 1024 * 1024; +static NEXT_STAGE: AtomicU64 = AtomicU64::new(0); +mod budget; + +pub(crate) struct Stage(pub(crate) PathBuf); + +impl Stage { + pub(crate) fn new() -> Result<Self, String> { + for _ in 0..32 { + let path = std::env::temp_dir().join(format!( + "dever-lib-build-{}-{}", + std::process::id(), + NEXT_STAGE.fetch_add(1, Ordering::Relaxed) + )); + #[cfg(unix)] + let created = { + use std::os::unix::fs::DirBuilderExt; + let mut builder = fs::DirBuilder::new(); + builder.mode(0o700); + builder.create(&path) + }; + #[cfg(not(unix))] + let created = fs::create_dir(&path); + match created { + Ok(()) => return Ok(Self(path)), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => return Err(format!("cannot create private build staging: {error}")), + } + } + Err("cannot allocate private build staging".into()) + } + + pub(crate) fn write( + &self, + relative: &str, + bytes: &[u8], + executable: bool, + ) -> Result<PathBuf, String> { + crate::workers::valid_path(relative)?; + let path = self.0.join(relative); + fs::create_dir_all(path.parent().expect("validated relative path")) + .map_err(|error| format!("cannot create build input directory: {error}"))?; + // This tree is private and never a writable child mount. A second write + // must not silently replace a previously verified input. + use std::io::Write; + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&path) + .map_err(|error| format!("cannot stage build input: {error}"))?; + file.write_all(bytes) + .map_err(|error| format!("cannot write build input: {error}"))?; + #[cfg(unix)] + if executable { + use std::os::unix::fs::PermissionsExt; + file.set_permissions(fs::Permissions::from_mode(0o700)) + .map_err(|error| format!("cannot mark build tool executable: {error}"))?; + } + Ok(path) + } + + pub(crate) fn run( + &self, + command: Command, + operation: &str, + writable: &Path, + ) -> Result<Vec<u8>, String> { + let status = self.execute(command, operation, Some(writable))?; + if !status.success() { + return Err(format!( + "{operation} failed: {}", + String::from_utf8_lossy(&self.diagnostics()?) + )); + } + fs::read(self.0.join("tool-stdout")) + .map_err(|error| format!("cannot read {operation} output: {error}")) + } + + pub(crate) fn probe(&self, command: Command) -> Result<bool, String> { + match self + .execute(command, "isolated Node addon probe", None)? + .code() + { + Some(0) => Ok(true), + Some(10) => Ok(false), + _ => Err(format!( + "Node addon probe failed outside its loader check: {}", + String::from_utf8_lossy(&self.diagnostics()?) + )), + } + } + + fn execute( + &self, + mut command: Command, + operation: &str, + writable: Option<&Path>, + ) -> Result<std::process::ExitStatus, String> { + // Captures are outside every child-writable grant. Namespace teardown + // reaps descendants when the owned bootstrap is killed on any exit path. + let stdout = self.0.join("tool-stdout"); + let stderr = self.0.join("tool-stderr"); + let child = command + .stdin(Stdio::null()) + .stdout(Stdio::from( + File::create(&stdout).map_err(|error| error.to_string())?, + )) + .stderr(Stdio::from( + File::create(&stderr).map_err(|error| error.to_string())?, + )) + .spawn() + .map_err(|error| format!("cannot execute {operation}: {error}"))?; + let mut child = OwnedChild(child); + let deadline = Instant::now() + DEADLINE; + let mut next_scratch_check = Instant::now(); + let within_budget = || -> Result<(), String> { + for path in [&stdout, &stderr] { + if fs::metadata(path).map_err(|error| error.to_string())?.len() > MAX_OUTPUT { + return Err(format!("{operation} output exceeds byte limit")); + } + } + Ok(()) + }; + let status = loop { + within_budget()?; + if Instant::now() >= next_scratch_check { + if let Some(path) = writable { + budget::check(path)?; + } + next_scratch_check = Instant::now() + Duration::from_millis(250); + } + if let Some(status) = child + .0 + .try_wait() + .map_err(|error| format!("cannot wait for {operation}: {error}"))? + { + break status; + } + if Instant::now() >= deadline { + return Err(format!("{operation} exceeded its five-minute deadline")); + } + std::thread::sleep(Duration::from_millis(25)); + }; + within_budget()?; + if let Some(path) = writable { + budget::check(path)?; + } + Ok(status) + } + + pub(crate) fn diagnostics(&self) -> Result<Vec<u8>, String> { + let mut bytes = Vec::new(); + for name in ["tool-stdout", "tool-stderr"] { + File::open(self.0.join(name)) + .and_then(|file| file.take(MAX_OUTPUT).read_to_end(&mut bytes)) + .map_err(|error| format!("cannot read build diagnostic: {error}"))?; + } + Ok(bytes) + } +} + +impl Drop for Stage { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +struct OwnedChild(Child); + +impl Drop for OwnedChild { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} diff --git a/crates/dever-cli/src/libs/build/process/budget.rs b/crates/dever-cli/src/libs/build/process/budget.rs new file mode 100644 index 0000000..45953b8 --- /dev/null +++ b/crates/dever-cli/src/libs/build/process/budget.rs @@ -0,0 +1,68 @@ +//! Periodic scratch accounting is separate from immutable runtime/tool inputs. + +use std::fs; +use std::path::Path; + +const MAX_BYTES: u64 = 512 * 1024 * 1024; +const MAX_ENTRIES: usize = 65_536; +const MAX_DEPTH: usize = 128; + +#[derive(Default)] +struct Budget { + bytes: u64, + entries: usize, +} + +pub(super) fn check(root: &Path) -> Result<(), String> { + visit(root, 0, &mut Budget::default()).map_err(|error| format!("build scratch budget: {error}")) +} + +#[cfg(unix)] +fn visit(path: &Path, depth: usize, budget: &mut Budget) -> Result<(), String> { + use std::os::fd::AsRawFd; + use std::os::unix::fs::OpenOptionsExt; + if depth > MAX_DEPTH { + return Err("directory depth exceeds limit".into()); + } + // An open directory pins the inode throughout enumeration. Child-created + // symlinks and rename races cannot redirect accounting outside this tree. + let directory = match fs::OpenOptions::new() + .read(true) + .custom_flags((rustix::fs::OFlags::NOFOLLOW | rustix::fs::OFlags::DIRECTORY).bits() as i32) + .open(path) + { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(format!("cannot pin scratch directory: {error}")), + }; + let pinned = format!("/proc/self/fd/{}", directory.as_raw_fd()); + for entry in fs::read_dir(pinned).map_err(|error| error.to_string())? { + let entry = entry.map_err(|error| error.to_string())?; + let metadata = match entry.metadata() { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(error) => return Err(error.to_string()), + }; + budget.entries += 1; + if budget.entries > MAX_ENTRIES { + return Err("entry count exceeds limit".into()); + } + if metadata.is_dir() { + visit(&entry.path(), depth + 1, budget)?; + } else { + budget.bytes = budget + .bytes + .checked_add(metadata.len()) + .ok_or("byte count overflow")?; + if budget.bytes > MAX_BYTES { + return Err("bytes exceed 512 MiB limit".into()); + } + } + } + Ok(()) +} + +#[cfg(not(unix))] +fn visit(_path: &Path, _depth: usize, _budget: &mut Budget) -> Result<(), String> { + Err("managed build scratch accounting requires its Linux sandbox".into()) +} diff --git a/crates/dever-cli/src/libs/build/python.py b/crates/dever-cli/src/libs/build/python.py new file mode 100644 index 0000000..b73a7ed --- /dev/null +++ b/crates/dever-cli/src/libs/build/python.py @@ -0,0 +1,119 @@ +"""Fixed PEP 517 frontend, executed only inside the managed build sandbox.""" + +import importlib +import json +import os +from pathlib import Path +import re +import shutil +import sys +import tomllib + + +def build_system(source): + project = source / "pyproject.toml" + config = tomllib.loads(project.read_text("utf-8")) if project.exists() else {} + system = config.get("build-system") + if system is None: + system = {"requires": ["setuptools>=40.8.0"], "build-backend": "setuptools.build_meta:__legacy__"} + if not isinstance(system, dict) or not isinstance(system.get("requires"), list): + raise ValueError("build-system.requires must be an array") + requires = system["requires"] + if len(requires) > 1024 or any(not isinstance(value, str) for value in requires): + raise ValueError("invalid static build requirements") + backend = system.get("build-backend", "setuptools.build_meta:__legacy__") + identifier = r"[A-Za-z_][A-Za-z_0-9]*(?:\.[A-Za-z_][A-Za-z_0-9]*)*" + if not isinstance(backend, str) or not re.fullmatch(identifier + "(?::" + identifier + ")?", backend): + raise ValueError("invalid build-backend entry point") + paths = system.get("backend-path", []) + if not isinstance(paths, list) or len(paths) > 64: + raise ValueError("invalid backend-path") + for name in paths: + if not isinstance(name, str) or not name or Path(name).is_absolute() or "\\" in name: + raise ValueError("backend-path must be source-relative") + resolved = (source / name).resolve() + if not resolved.is_relative_to(source.resolve()) or not resolved.is_dir(): + raise ValueError("backend-path escapes the source tree") + return {"requires": requires, "backend": backend, "backend_path": paths} + + +def backend_object(source, system): + paths = [(source / path).resolve() for path in system["backend_path"]] + sys.path[:0] = [str(path) for path in paths] + module_name, _, object_name = system["backend"].partition(":") + module = importlib.import_module(module_name) + if paths: + origin = Path(module.__file__).resolve() + if not any(origin.is_relative_to(path) for path in paths): + raise ValueError("in-tree backend was loaded outside backend-path") + backend = module + for part in object_name.split(".") if object_name else []: + backend = getattr(backend, part) + return backend + + +def leaf(value, suffix): + if not isinstance(value, str) or Path(value).name != value or not value.endswith(suffix): + raise ValueError("build hook returned an invalid output name") + return value + + +def execute(request, source, output): + system = build_system(source) + if request["phase"] == "inspect": + return system + if system != request["system"]: + raise ValueError("source build configuration changed between hooks") + backend = backend_object(source, system) + settings = request["config_settings"] + if request["phase"] == "requires": + hook = getattr(backend, "get_requires_for_build_wheel", None) + requires = hook(settings) if hook else [] + if not isinstance(requires, list) or len(requires) > 1024 or any(not isinstance(value, str) for value in requires): + raise ValueError("invalid dynamic build requirements") + return {"requires": requires} + if request["phase"] == "metadata": + metadata = output / "metadata" + metadata.mkdir() + hook = getattr(backend, "prepare_metadata_for_build_wheel", None) + prepared = metadata / leaf(hook(str(metadata), settings), ".dist-info") if hook else None + if prepared is not None and (not prepared.is_dir() or prepared.is_symlink()): + raise ValueError("metadata hook did not create its declared directory") + return {"metadata": str(prepared.relative_to(output)) if prepared else None} + prepared = None + if request.get("metadata") is not None: + shutil.copytree("/worker/prepared", output / "prepared") + prepared = output / "prepared" / request["metadata"] + if not prepared.resolve().is_relative_to((output / "prepared").resolve()): + raise ValueError("prepared metadata path escapes its snapshot") + wheels = output / "wheels" + wheels.mkdir() + name = leaf(backend.build_wheel(str(wheels), settings, str(prepared) if prepared else None), ".whl") + return {"filename": name} + + +def main(): + request = json.loads(Path("/worker/request.json").read_text("utf-8")) + work = Path("/data/build") + source = work / "source" + shutil.copytree("/worker/source", source) + output = work / "output" + output.mkdir() + (work / "tmp").mkdir() + # These values are part of the fixed frontend, never project/host settings. + os.environ.update({ + "PATH": "/worker/runtime/bin:/usr/bin", + "CC": "/usr/bin/cc", "CXX": "/usr/bin/c++", + "AR": "/usr/bin/ar", "RANLIB": "/usr/bin/ranlib", + "LDSHARED": "/usr/bin/cc -shared", + "CFLAGS": "-fPIC -ffile-prefix-map=/data/build/source=.", + "CPPFLAGS": "-I/worker/runtime/include/python" + request["python_version"], + "TMPDIR": "/data/build/tmp", "SOURCE_DATE_EPOCH": "0", + }) + os.chdir(source) + result = execute(request, source, output) + (output / "result.json").write_text(json.dumps(result, sort_keys=True), "utf-8") + + +if __name__ == "__main__": + main() diff --git a/crates/dever-cli/src/libs/build/python.rs b/crates/dever-cli/src/libs/build/python.rs new file mode 100644 index 0000000..bbcbeda --- /dev/null +++ b/crates/dever-cli/src/libs/build/python.rs @@ -0,0 +1,564 @@ +//! PEP 517 source candidates enter the normal dependency solver as real wheels. + +use std::collections::{BTreeMap, BTreeSet}; +use std::ffi::OsString; +use std::fs; +use std::io::Read; +use std::path::{Path, PathBuf}; + +use super::{Product, PythonSystem, Receipt, Session, pack, process::Stage}; +use crate::libs::{ + Ecosystem, LibSpec, LockFile, LockedArtifact, RegistryResolver, RegistryRuntime, archive_files, + sha256, +}; +use crate::workers::{python_wheel::Wheel, runtime}; +use serde_json::{Value, json}; + +struct Environment { + files: BTreeMap<String, (Vec<u8>, bool)>, + assets: Vec<(String, Vec<u8>)>, + executable: String, + version: String, + extension_suffixes: Vec<String>, + descriptor: pack::Descriptor, +} + +struct Active<'a> { + session: &'a Session<'a>, + key: String, +} +impl Drop for Active<'_> { + fn drop(&mut self) { + self.session.active.borrow_mut().remove(&self.key); + } +} + +impl Session<'_> { + pub(crate) fn python( + &self, + resolver: &RegistryResolver<'_>, + spec: &LibSpec, + runtime: &RegistryRuntime, + filename: &str, + source: &[u8], + ) -> Result<Product, String> { + let key = format!( + "{}@{}:{}:{}", + spec.distribution_name(), + spec.version, + sha256(source), + runtime.pack.sha256 + ); + if let Some(product) = self.products.borrow().get(&key) { + return Ok(product.clone()); + } + if self.active.borrow().len() >= 16 || !self.active.borrow_mut().insert(key.clone()) { + return Err(format!( + "cyclic or overly deep Python build requirement for {}", + spec.key() + )); + } + let _active = Active { + session: self, + key: key.clone(), + }; + let sources = source_files(filename, source)?; + let environment = Environment::new(self, runtime)?; + let empty = LockFile::new(vec![])?; + let inspected = + environment.run(resolver, &empty, &sources, json!({"phase":"inspect"}), None)?; + let system: PythonSystem = serde_json::from_value(inspected.response) + .map_err(|error| format!("invalid Python build configuration: {error}"))?; + let mut requirements = system.requires.clone(); + let mut dependencies = resolver.build_requirements(&requirements, runtime)?; + let mut stable = None; + let mut discovered = BTreeSet::new(); + for _ in 0..8 { + let result = environment.run( + resolver, + &dependencies, + &sources, + json!({"phase":"requires","system":system}), + None, + )?; + let dynamic: Vec<String> = serde_json::from_value( + result + .response + .get("requires") + .cloned() + .ok_or("build backend omitted requirements")?, + ) + .map_err(|error| format!("invalid dynamic Python build requirements: {error}"))?; + discovered.extend(dynamic); + requirements = system.requires.iter().chain(&discovered).cloned().collect(); + let next = resolver.build_requirements(&requirements, runtime)?; + if next == dependencies { + stable = Some(discovered.iter().cloned().collect()); + break; + } + dependencies = next; + } + let dynamic = stable.ok_or("Python dynamic build requirements did not converge")?; + let metadata = environment.run( + resolver, + &dependencies, + &sources, + json!({"phase":"metadata","system":system}), + None, + )?; + let result = environment.run( + resolver, + &dependencies, + &sources, + json!({"phase":"wheel","system":system,"metadata":metadata.response.get("metadata")}), + Some(&metadata.outputs), + )?; + let filename = result + .response + .get("filename") + .and_then(Value::as_str) + .ok_or("build backend omitted wheel filename")?; + crate::workers::valid_path(filename)?; + if filename.contains('/') || !filename.ends_with(".whl") { + return Err("build backend returned invalid wheel filename".into()); + } + let bytes = result + .outputs + .get(&format!("wheels/{filename}")) + .ok_or("build backend did not produce its declared wheel")? + .clone(); + Wheel::parse(spec, Some(filename), &runtime.python_wheel_tags, &bytes)?; + if let Some(prefix) = metadata.response.get("metadata").and_then(Value::as_str) { + compare_metadata(prefix, &metadata.outputs, &bytes)?; + } + let output = artifact( + runtime, + format!( + "lib/pip/{}/{}/archive.whl", + sha256(spec.distribution_name().as_bytes()), + spec.version + ), + &bytes, + ); + let source_artifact = artifact( + runtime, + format!("build/source/{}/source", sha256(source)), + source, + ); + for (artifact, bytes) in [(&output, bytes.as_slice()), (&source_artifact, source)] { + if resolver.store.publish(bytes, &runtime.target)? != artifact.sha256 { + return Err("build artifact cache identity mismatch".into()); + } + } + let base: LibSpec = format!("pip:{}@{}", spec.distribution_name(), spec.version).parse()?; + let receipt = Receipt { + ecosystem: Ecosystem::Pip, + roots: vec![base], + sources: vec![source_artifact], + graph: None, + runtime: runtime.pack.clone(), + auxiliary_runtime: None, + native_entries: BTreeSet::new(), + native_rejections: BTreeMap::new(), + python_markers: runtime.python_markers.clone(), + tools: environment.descriptor, + frontend: super::frontend(&Ecosystem::Pip)?, + python: Some(system), + dynamic_requires: dynamic, + dependencies: Box::new(dependencies), + config_settings: BTreeMap::new(), + output, + }; + let product = Product { + bytes, + filename: filename.into(), + receipt, + }; + self.remember(key, product.clone())?; + Ok(product) + } +} + +fn artifact(runtime: &RegistryRuntime, path: String, bytes: &[u8]) -> LockedArtifact { + LockedArtifact { + target: runtime.target.clone(), + path, + bytes: bytes.len() as u64, + sha256: sha256(bytes), + } +} + +struct Source { + files: Vec<(String, Vec<u8>)>, + executables: BTreeSet<String>, +} + +fn source_files(filename: &str, bytes: &[u8]) -> Result<Source, String> { + let extension = if filename.ends_with(".tar.gz") { + "tgz" + } else if filename.ends_with(".zip") { + "zip" + } else { + return Err("unsupported Python source archive format".into()); + }; + let files = archive_files(extension, bytes)?; + let executables = crate::libs::registry::archive_executables(extension, bytes)?; + let root = files + .first() + .and_then(|(path, _)| path.split_once('/')) + .map(|(root, _)| format!("{root}/")) + .ok_or("Python source archive needs one root directory")?; + let files = files + .into_iter() + .map(|(path, bytes)| { + let path = path + .strip_prefix(&root) + .ok_or("Python source archive contains multiple roots")? + .to_owned(); + crate::workers::valid_path(&path)?; + Ok((path, bytes)) + }) + .collect::<Result<Vec<_>, String>>()?; + let executables = executables + .into_iter() + .map(|path| { + path.strip_prefix(&root) + .map(str::to_owned) + .ok_or("source executable escapes archive root".into()) + }) + .collect::<Result<_, String>>()?; + Ok(Source { files, executables }) +} + +struct ResultFiles { + response: Value, + outputs: BTreeMap<String, Vec<u8>>, +} + +impl Environment { + fn new(session: &Session<'_>, requested: &RegistryRuntime) -> Result<Self, String> { + super::require_execution_target(&requested.target, "Python source build")?; + let (runtime, bytes) = session.inputs.runtime(Ecosystem::Pip)?; + if runtime.pack != requested.pack || sha256(&bytes) != requested.pack.sha256 { + return Err("build Python runtime differs from resolver identity".into()); + } + let runtime_files = archive_files("tgz", &bytes)?; + let manifest = runtime::validate(&runtime_files, &Ecosystem::Pip, &requested.target)?; + if manifest.python_wheel_tags != requested.python_wheel_tags + || manifest.python_markers != requested.python_markers + { + return Err("build runtime policy differs from signed descriptor".into()); + } + let executable = manifest + .executable + .clone() + .ok_or("build Python executable missing")?; + let executable = format!("runtime/{executable}"); + let mut files = runtime_files + .into_iter() + .map(|(path, bytes)| { + let executable = manifest.executable(&path); + (format!("runtime/{path}"), (bytes, executable)) + }) + .collect::<BTreeMap<_, _>>(); + let (descriptor, bytes) = session.inputs.tools(Ecosystem::Pip)?; + descriptor.validate(requested, None)?; + let (tools, tool_files) = pack::unpack(&bytes, &descriptor)?; + let assets = session.inputs.assets()?; + dever_sandbox::validate_assets(&assets)?; + pack::validate_tools(&tools, &tool_files, &assets)?; + for (path, bytes) in tool_files { + let executable = tools.executables.contains(&path); + if files.insert(path, (bytes, executable)).is_some() { + return Err("build tools overlap runtime inputs".into()); + } + } + let version = requested + .python_markers + .as_ref() + .ok_or("build Python markers absent")? + .python_version() + .to_string(); + Ok(Self { + files, + assets, + executable, + version, + extension_suffixes: manifest.python_extension_suffixes, + descriptor, + }) + } + + fn run( + &self, + resolver: &RegistryResolver<'_>, + dependencies: &LockFile, + source: &Source, + mut request: Value, + prepared: Option<&BTreeMap<String, Vec<u8>>>, + ) -> Result<ResultFiles, String> { + let stage = Stage::new()?; + let input = stage.0.join("input"); + for (path, (bytes, executable)) in &self.files { + stage.write(&format!("input/{path}"), bytes, *executable)?; + } + for (path, bytes) in &self.assets { + stage.write( + &format!("sandbox/{path}"), + bytes, + dever_sandbox::asset_executable(path), + )?; + } + for (path, bytes) in &source.files { + stage.write( + &format!("input/source/{path}"), + bytes, + source.executables.contains(path), + )?; + } + if let Some(prepared) = prepared { + for (path, bytes) in prepared { + stage.write(&format!("input/prepared/{path}"), bytes, false)?; + } + } + let mut installed = BTreeMap::<String, Vec<u8>>::new(); + let runtime = resolver + .runtimes + .get(&Ecosystem::Pip) + .ok_or("build dependency runtime is missing")?; + let mut distributions = BTreeSet::new(); + for lib in &dependencies.libs { + super::validate_python_policy( + dependencies, + lib, + runtime + .python_markers + .as_ref() + .ok_or("Python build markers absent")?, + )?; + if lib.runtime != runtime.pack || lib.artifacts.len() != 1 { + return Err( + "build dependency does not use the exact Python runtime and wheel".into(), + ); + } + let artifact = &lib.artifacts[0]; + let bytes = + resolver + .store + .verify_exact(&artifact.sha256, artifact.bytes, &artifact.target)?; + let wheel = Wheel::parse(&lib.spec, None, &runtime.python_wheel_tags, &bytes)?; + wheel.validate_dependencies( + lib, + runtime + .python_markers + .as_ref() + .ok_or("Python build markers absent")?, + )?; + if !distributions.insert(( + lib.spec.distribution_name().to_owned(), + lib.spec.version.clone(), + )) { + continue; + } + for file in wheel.install(&self.executable, &self.version)? { + if self.files.contains_key(&file.path) + || installed + .insert(file.path.clone(), file.bytes.clone()) + .is_some() + { + return Err("build dependency installation has conflicting file owners".into()); + } + stage.write( + &format!("input/{}", file.path), + &file.bytes, + file.executable, + )?; + } + } + let mut supplied = self + .files + .iter() + .map(|(path, (bytes, _))| (path.clone(), bytes.as_slice())) + .collect::<BTreeMap<_, _>>(); + for (path, bytes) in &self.assets { + supplied.insert(format!("sandbox/{path}"), bytes); + } + for (path, bytes) in &installed { + supplied.insert(path.clone(), bytes); + } + crate::workers::python_wheel::validate_native( + &runtime.target, + &self.extension_suffixes, + &self.executable, + &supplied, + &installed.keys().cloned().collect(), + )?; + request["python_version"] = json!(self.version); + request["config_settings"] = json!({}); + stage.write( + "input/request.json", + &serde_json::to_vec(&request).map_err(|error| error.to_string())?, + false, + )?; + stage.write("input/frontend.py", include_bytes!("python.py"), false)?; + let work = stage.0.join("work"); + fs::create_dir(&work).map_err(|error| error.to_string())?; + let grants = [dever_sandbox::Grant { + source: work.clone(), + destination: "/data/build".into(), + writable: true, + }]; + let arguments = [ + OsString::from("-I"), + OsString::from("-B"), + input.join("frontend.py").into_os_string(), + ]; + let executable = input.join(&self.executable); + let assets = stage.0.join("sandbox"); + let launch = dever_sandbox::Launch { + assets: &assets, + worker: &input, + executable: &executable, + arguments: &arguments, + working_directory: &input, + capabilities: dever_sandbox::Capabilities { + network: false, + process: true, + }, + grants: &grants, + }; + stage.run( + launch.command_for_build(&input.join("rootfs"))?, + "isolated Python build hook", + &work, + )?; + if let Some(session) = resolver.build { + session.inputs.diagnostic(&stage.diagnostics()?); + } + let outputs = output_files(&work.join("output"))?; + let response = serde_json::from_slice( + outputs + .get("result.json") + .ok_or("build hook produced no result")?, + ) + .map_err(|error| format!("invalid build hook response: {error}"))?; + Ok(ResultFiles { response, outputs }) + } +} + +pub(super) fn output_files(root: &Path) -> Result<BTreeMap<String, Vec<u8>>, String> { + let mut files = BTreeMap::new(); + let mut pending = vec![root.to_owned()]; + let mut remaining = crate::workers::MAX_TREE_BYTES; + let mut entries = 0usize; + #[cfg(unix)] + let mut links = BTreeMap::<(u64, u64), (u64, u64)>::new(); + while let Some(directory) = pending.pop() { + if fs::symlink_metadata(&directory) + .map_err(|error| error.to_string())? + .file_type() + .is_symlink() + { + return Err("build output directory is a symbolic link".into()); + } + for entry in fs::read_dir(directory).map_err(|error| error.to_string())? { + entries += 1; + if entries > crate::workers::MAX_TREE_FILES { + return Err("build output exceeds file budget".into()); + } + let entry = entry.map_err(|error| error.to_string())?; + let kind = entry.file_type().map_err(|error| error.to_string())?; + if kind.is_dir() { + pending.push(entry.path()); + continue; + } + if !kind.is_file() { + return Err("build output contains a link or special file".into()); + } + let mut options = fs::OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(rustix::fs::OFlags::NOFOLLOW.bits() as i32); + } + let file = options + .open(entry.path()) + .map_err(|error| error.to_string())?; + let metadata = file.metadata().map_err(|error| error.to_string())?; + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let aliases = links + .entry((metadata.dev(), metadata.ino())) + .or_insert((metadata.nlink(), 0)); + if aliases.0 != metadata.nlink() { + return Err("build output link count changed during collection".into()); + } + aliases.1 += 1; + } + if !metadata.is_file() || metadata.len() > remaining as u64 { + return Err("build output exceeds byte budget".into()); + } + let mut bytes = Vec::new(); + file.take(remaining as u64 + 1) + .read_to_end(&mut bytes) + .map_err(|error| error.to_string())?; + remaining = remaining + .checked_sub(bytes.len()) + .ok_or("build output exceeds byte budget")?; + let path = entry.path(); + let name = path + .strip_prefix(root) + .map_err(|error| error.to_string())? + .to_str() + .ok_or("build output path is not UTF-8")? + .to_owned(); + crate::workers::valid_path(&name)?; + files.insert(name, bytes); + } + } + // node-gyp's COPY rule may hard-link two output names. Namespace teardown + // precedes this read: accept aliases only when every link is accounted for + // inside the output root, then publish independent immutable byte entries. + #[cfg(unix)] + if links + .values() + .any(|(expected, observed)| expected != observed) + { + return Err("build output has a hard-link alias outside its output root".into()); + } + Ok(files) +} + +fn compare_metadata( + prefix: &str, + outputs: &BTreeMap<String, Vec<u8>>, + wheel: &[u8], +) -> Result<(), String> { + crate::workers::valid_path(prefix)?; + let distribution = PathBuf::from(prefix) + .file_name() + .and_then(|name| name.to_str()) + .ok_or("invalid prepared metadata directory")? + .to_owned(); + let wheel = archive_files("whl", wheel)? + .into_iter() + .collect::<BTreeMap<_, _>>(); + let prefix = format!("{prefix}/"); + let mut count = 0; + for (path, bytes) in outputs { + if let Some(relative) = path.strip_prefix(&prefix) { + count += 1; + if wheel.get(&format!("{distribution}/{relative}")) != Some(bytes) { + return Err( + "built wheel metadata differs from prepare_metadata_for_build_wheel".into(), + ); + } + } + } + if count == 0 { + return Err("metadata hook produced an empty directory".into()); + } + Ok(()) +} diff --git a/crates/dever-cli/src/libs/mutation.rs b/crates/dever-cli/src/libs/mutation.rs new file mode 100644 index 0000000..f0eb613 --- /dev/null +++ b/crates/dever-cli/src/libs/mutation.rs @@ -0,0 +1,80 @@ +//! Serialize dependency mutations from the first project read to publication. +//! The guard file is stable: replacing dever.lock must not replace its mutex. + +use std::fs::{self, File, Metadata, OpenOptions}; +use std::path::Path; + +pub(crate) struct ProjectMutation { + _file: File, +} + +impl ProjectMutation { + pub(crate) fn for_command(command: &str, root: &Path) -> Result<Option<Self>, String> { + if !matches!(command, "add" | "update" | "remove") { + return Ok(None); + } + let directory = fs::symlink_metadata(root) + .map_err(|error| format!("cannot inspect project mutation directory: {error}"))?; + validate(&directory, true)?; + let path = root.join(".dever-project.lock"); + if let Ok(metadata) = fs::symlink_metadata(&path) { + validate(&metadata, false)?; + } + let mut options = OpenOptions::new(); + options.read(true).write(true).create(true).truncate(false); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options + .mode(0o600) + .custom_flags(rustix::fs::OFlags::NOFOLLOW.bits() as i32); + } + let file = options + .open(&path) + .map_err(|error| format!("cannot open project mutation lock: {error}"))?; + let opened = file.metadata().map_err(|error| error.to_string())?; + validate(&opened, false)?; + let current = fs::symlink_metadata(&path).map_err(|error| error.to_string())?; + validate(¤t, false)?; + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + if opened.dev() != current.dev() || opened.ino() != current.ino() { + return Err("project mutation lock changed while opening".into()); + } + } + fs2::FileExt::try_lock_exclusive(&file).map_err(|error| { + if error.kind() == std::io::ErrorKind::WouldBlock { + "project dependency mutation is busy; retry after the current command finishes" + .into() + } else { + format!("cannot acquire project mutation lock: {error}") + } + })?; + Ok(Some(Self { _file: file })) + } +} + +fn validate(metadata: &Metadata, directory: bool) -> Result<(), String> { + if metadata.file_type().is_symlink() + || (directory && !metadata.is_dir()) + || (!directory && !metadata.is_file()) + { + return Err( + "project mutation path must be a real directory or regular lock file, without symlinks" + .into(), + ); + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let forbidden = if directory { 0o022 } else { 0o077 }; + if metadata.uid() != rustix::process::geteuid().as_raw() + || metadata.mode() & forbidden != 0 + || (!directory && metadata.nlink() != 1) + { + return Err("project mutation directory must be owned and not group/other writable; lock must be owned, private and unlinked elsewhere".into()); + } + } + Ok(()) +} diff --git a/crates/dever-cli/src/libs/registry.rs b/crates/dever-cli/src/libs/registry.rs new file mode 100644 index 0000000..d9a3d8e --- /dev/null +++ b/crates/dever-cli/src/libs/registry.rs @@ -0,0 +1,1619 @@ +//! Registry protocol adapters used only by explicit `lib add` and `lib update`. + +use std::collections::{BTreeMap, BTreeSet, VecDeque}; +use std::io::{Cursor, Read}; +use std::path::{Component, Path}; +use std::str::FromStr; +use std::time::{Duration, Instant}; + +use node_semver::Version as NpmVersion; +use pep440_rs::{Version as PythonVersion, VersionSpecifiers}; +use pep508_rs::{MarkerEnvironment, Requirement, VersionOrUrl}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use sha2::{Digest, Sha512}; +use url::{Host, Url}; + +pub mod npm; +pub mod sumdb; + +use super::{ + ArtifactStore, Ecosystem, LibResolver, LibSpec, LockFile, LockedArtifact, LockedDependency, + LockedLib, ProviderManifest, RuntimePack, sha256, validate_artifact, +}; + +const MAX_METADATA_BYTES: usize = 16 * 1024 * 1024; +const MAX_ARTIFACT_BYTES: usize = 64 * 1024 * 1024; +const MAX_PACKAGES: usize = 1024; +const MAX_VERSIONS_PER_PACKAGE: usize = 10_000; +const MAX_RESOLUTION_ATTEMPTS: usize = 4096; +const REGISTRY_REQUEST_TIMEOUT: Duration = Duration::from_secs(30); +const MAX_GO_ARCHIVE_REDIRECTS: usize = 2; +const GO_ARCHIVE_BUCKET: &str = "/proxy-golang-org-prod/"; + +/// The immutable runtime identity is supplied by the verified release pack. +/// Python marker values come from that pack, not the host interpreter. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct RegistryRuntime { + pub pack: RuntimePack, + pub target: String, + pub python_markers: Option<MarkerEnvironment>, + pub python_wheel_tags: Vec<String>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub npm_libc: Option<String>, +} + +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct InstalledRegistryPack { + pub format: String, + pub runtime: RegistryRuntime, +} + +impl InstalledRegistryPack { + pub(crate) fn validate(&self, ecosystem: &Ecosystem, target: &str) -> Result<(), String> { + if self.format != "dever-registry-runtime-v1" + || self.runtime.target != target + || (ecosystem == &Ecosystem::Pip) != self.runtime.python_markers.is_some() + || (ecosystem == &Ecosystem::Pip) != !self.runtime.python_wheel_tags.is_empty() + { + return Err(format!( + "signed {} runtime manifest does not match this target", + ecosystem.as_str() + )); + } + super::validate_runtime(&self.runtime.pack)?; + super::validate_version(ecosystem, &self.runtime.pack.version)?; + if !self + .runtime + .pack + .name + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"-_.".contains(&byte)) + { + return Err("runtime pack name is invalid".into()); + } + if ecosystem == &Ecosystem::Pip { + crate::workers::python_wheel::validate_target( + target, + &self.runtime.python_wheel_tags, + self.runtime + .python_markers + .as_ref() + .expect("validated Python markers"), + )?; + } + Ok(()) + } +} + +/// A transport only fetches registry data; it cannot choose a package version. +/// Production uses fixed official origins. Tests can bind the same protocols to +/// a loopback registry without invoking host package managers. +pub trait RegistryTransport { + fn get(&self, ecosystem: Ecosystem, path_or_url: &str, limit: usize) + -> Result<Vec<u8>, String>; + + /// Only an explicit registry 404 is absence. Transport, parsing and + /// integrity failures remain errors, including for optional dependencies. + fn get_optional( + &self, + ecosystem: Ecosystem, + path: &str, + limit: usize, + ) -> Result<Option<Vec<u8>>, String> { + self.get(ecosystem, path, limit).map(Some) + } + + fn get_sumdb(&self, path: &str, limit: usize) -> Result<Vec<u8>, String> { + self.get( + Ecosystem::Go, + &format!("https://sum.golang.org{path}"), + limit, + ) + } +} + +pub struct HttpRegistry { + agent: ureq::Agent, + origins: BTreeMap<Ecosystem, String>, + artifact_origins: BTreeMap<Ecosystem, Vec<String>>, + go_archive_mirror: Option<Url>, +} + +impl HttpRegistry { + pub fn official() -> Self { + Self::new( + [ + (Ecosystem::Pip, "https://pypi.org".into()), + (Ecosystem::Npm, "https://registry.npmjs.org".into()), + (Ecosystem::Go, "https://proxy.golang.org".into()), + ], + [ + ( + Ecosystem::Pip, + vec!["https://files.pythonhosted.org".into()], + ), + (Ecosystem::Npm, vec!["https://registry.npmjs.org".into()]), + ( + Ecosystem::Go, + vec![ + "https://proxy.golang.org".into(), + "https://sum.golang.org".into(), + ], + ), + ], + ) + .with_go_archive_mirror("https://storage.googleapis.com/proxy-golang-org-prod/") + .expect("fixed official Go archive mirror") + } + + pub fn new( + origins: impl IntoIterator<Item = (Ecosystem, String)>, + artifact_origins: impl IntoIterator<Item = (Ecosystem, Vec<String>)>, + ) -> Self { + let config = ureq::Agent::config_builder() + .proxy(None) + .max_redirects(0) + .timeout_global(Some(REGISTRY_REQUEST_TIMEOUT)) + .build(); + Self { + agent: ureq::Agent::new_with_config(config), + origins: origins + .into_iter() + .map(|(ecosystem, origin)| (ecosystem, origin.trim_end_matches('/').into())) + .collect(), + artifact_origins: artifact_origins + .into_iter() + .map(|(ecosystem, origins)| { + ( + ecosystem, + origins + .into_iter() + .map(|origin| origin.trim_end_matches('/').into()) + .collect(), + ) + }) + .collect(), + go_archive_mirror: None, + } + } + + /// Explicit loopback mirrors let protocol tests exercise the same policy + /// without granting production projects a configurable registry origin. + pub fn with_go_archive_mirror(mut self, mirror: &str) -> Result<Self, String> { + let mirror_has_userinfo = has_userinfo(mirror)?; + let mirror = + Url::parse(mirror).map_err(|error| format!("invalid Go archive mirror: {error}"))?; + let go_origin = self + .origins + .get(&Ecosystem::Go) + .ok_or("Go registry origin is missing")?; + let go_origin = Url::parse(go_origin) + .map_err(|error| format!("invalid Go registry origin: {error}"))?; + let official = mirror.scheme() == "https" + && mirror.host_str() == Some("storage.googleapis.com") + && mirror.port().is_none(); + let local = mirror.scheme() == "http" + && matches!(mirror.host(), Some(Host::Ipv4(address)) if address.is_loopback()) + && go_origin.scheme() == "http" + && matches!(go_origin.host(), Some(Host::Ipv4(address)) if address.is_loopback()); + if (!official && !local) + || mirror.path() != GO_ARCHIVE_BUCKET + || mirror_has_userinfo + || mirror.query().is_some() + || mirror.fragment().is_some() + { + return Err("Go archive mirror must be the fixed official HTTPS bucket or an explicit loopback test mirror".into()); + } + self.go_archive_mirror = Some(mirror); + Ok(self) + } + + fn go_redirect(&self, location: &str) -> Result<String, String> { + let mirror = self + .go_archive_mirror + .as_ref() + .ok_or("Go archive mirror is not configured")?; + let raw_userinfo = has_userinfo(location)?; + let target = Url::parse(location) + .map_err(|error| format!("invalid Go archive redirect URL: {error}"))?; + if target.scheme() != mirror.scheme() + || target.host() != mirror.host() + || target.port_or_known_default() != mirror.port_or_known_default() + || raw_userinfo + || target.fragment().is_some() + || !target.path().starts_with(GO_ARCHIVE_BUCKET) + || target.path().len() == GO_ARCHIVE_BUCKET.len() + { + return Err("Go archive redirect is outside its trusted mirror bucket".into()); + } + Ok(target.into()) + } +} + +fn has_userinfo(raw_url: &str) -> Result<bool, String> { + let uri: ureq::http::Uri = raw_url.parse().map_err(|_| "invalid Go archive URL")?; + Ok(uri + .authority() + .is_some_and(|authority| authority.as_str().contains('@'))) +} + +impl RegistryTransport for HttpRegistry { + fn get( + &self, + ecosystem: Ecosystem, + path_or_url: &str, + limit: usize, + ) -> Result<Vec<u8>, String> { + self.fetch(ecosystem, path_or_url, limit, false)? + .ok_or("registry returned HTTP 404".into()) + } + + fn get_optional( + &self, + ecosystem: Ecosystem, + path: &str, + limit: usize, + ) -> Result<Option<Vec<u8>>, String> { + self.fetch(ecosystem, path, limit, true) + } +} + +impl HttpRegistry { + fn fetch( + &self, + ecosystem: Ecosystem, + path_or_url: &str, + limit: usize, + allow_missing: bool, + ) -> Result<Option<Vec<u8>>, String> { + let origin = self + .origins + .get(&ecosystem) + .ok_or("registry origin is missing")?; + let url = if path_or_url.starts_with('/') { + format!("{origin}{path_or_url}") + } else { + let allowed = self + .artifact_origins + .get(&ecosystem) + .ok_or("artifact origin is missing")?; + if !allowed + .iter() + .any(|origin| path_or_url.starts_with(&format!("{origin}/"))) + { + return Err("registry artifact URL has an untrusted origin".into()); + } + path_or_url.to_owned() + }; + let go_archive = ecosystem == Ecosystem::Go + && path_or_url.starts_with('/') + && path_or_url.contains("/@v/") + && path_or_url.ends_with(".zip"); + let mut current = url; + let mut visited = BTreeSet::new(); + visited.insert(current.clone()); + let mut redirects = 0; + let started = Instant::now(); + let mut response = loop { + let remaining = REGISTRY_REQUEST_TIMEOUT + .checked_sub(started.elapsed()) + .filter(|remaining| !remaining.is_zero()) + .ok_or("registry request exceeded the 30-second deadline")?; + let response = self + .agent + .get(¤t) + .config() + .timeout_global(Some(remaining)) + .build() + .call(); + if allow_missing && matches!(&response, Err(ureq::Error::StatusCode(404))) { + return Ok(None); + } + let response = response.map_err(|error| match error { + ureq::Error::StatusCode(code) => { + format!("{} registry returned HTTP {code}", ecosystem.as_str()) + } + _ => format!("{} registry request failed", ecosystem.as_str()), + })?; + if !response.status().is_redirection() { + break response; + } + if !go_archive || redirects >= MAX_GO_ARCHIVE_REDIRECTS { + return Err("registry redirect is not allowed for this endpoint".into()); + } + let mut locations = response.headers().get_all("location").iter(); + let location = locations + .next() + .ok_or("Go archive redirect lacks Location")? + .to_str() + .map_err(|_| "Go archive redirect has invalid Location")?; + if locations.next().is_some() { + return Err("Go archive redirect has multiple Locations".into()); + } + let next = self.go_redirect(location)?; + if !visited.insert(next.clone()) { + return Err("Go archive redirect loop detected".into()); + } + redirects += 1; + current = next; + }; + if !response.status().is_success() { + return Err(format!( + "{} registry returned HTTP {}", + ecosystem.as_str(), + response.status() + )); + } + let mut bytes = Vec::new(); + response + .body_mut() + .as_reader() + .take((limit + 1) as u64) + .read_to_end(&mut bytes) + .map_err(|error| format!("cannot read registry response: {error}"))?; + if bytes.len() > limit { + return Err(format!("registry response exceeds {limit} bytes")); + } + Ok(Some(bytes)) + } +} + +pub struct RegistryResolver<'a> { + pub transport: &'a dyn RegistryTransport, + pub store: &'a dyn ArtifactStore, + pub runtimes: BTreeMap<Ecosystem, RegistryRuntime>, + pub go_sumdb: sumdb::ChecksumDatabase, + pub build: Option<&'a super::build::Session<'a>>, +} + +impl LibResolver for RegistryResolver<'_> { + fn go_verifier(&self) -> sumdb::Verifier { + self.go_sumdb.verifier().clone() + } + + fn manifest(&self) -> ProviderManifest { + ProviderManifest { + ecosystem: Ecosystem::Pip, + resolver: "official-registry-protocols-v1".into(), + runtime_pack: "signed-release-runtime".into(), + sdk: "component-protocol".into(), + available: true, + } + } + + fn resolve(&self, requested: &[LibSpec]) -> Result<LockFile, String> { + let mut libs = Vec::new(); + let mut go_sumdb = Vec::new(); + let mut npm = Vec::new(); + for ecosystem in [Ecosystem::Pip, Ecosystem::Npm, Ecosystem::Go] { + let roots = requested + .iter() + .filter(|spec| spec.ecosystem == ecosystem) + .cloned() + .collect::<Vec<_>>(); + if roots.is_empty() { + continue; + } + let runtime = self + .runtimes + .get(&ecosystem) + .ok_or_else(|| format!("signed {} runtime pack is missing", ecosystem.as_str()))?; + let packages = match ecosystem { + Ecosystem::Pip => self.resolve_python(&roots, runtime)?, + Ecosystem::Npm => { + let (mut packages, mut environment) = npm::resolve(self, &roots, runtime)?; + if let Some(build) = self.build { + build.npm(self, &mut packages, &mut environment, runtime)?; + } + npm.push(environment); + packages + } + Ecosystem::Go => { + let (packages, evidence) = self.resolve_go(&roots, runtime)?; + go_sumdb.push(evidence); + packages + } + }; + libs.extend(packages); + } + let mut lock = LockFile::new(libs)?; + lock.go_sumdb = go_sumdb; + lock.npm = npm; + self.attach_builds(&mut lock)?; + Ok(lock) + } +} + +impl RegistryResolver<'_> { + fn resolve_python( + &self, + roots: &[LibSpec], + runtime: &RegistryRuntime, + ) -> Result<Vec<LockedLib>, String> { + let mut constraints = BTreeMap::<String, Vec<Constraint>>::new(); + for spec in roots { + constraints + .entry(spec.name.clone()) + .or_default() + .push(Constraint::Exact(spec.version.clone())); + } + self.python_constraints(constraints, runtime) + } + + pub(crate) fn build_requirements( + &self, + requirements: &[String], + runtime: &RegistryRuntime, + ) -> Result<LockFile, String> { + let document = serde_json::json!({"info":{"requires_dist":requirements}}); + let markers = runtime + .python_markers + .as_ref() + .ok_or("Python build runtime lacks markers")?; + let mut constraints = BTreeMap::<String, Vec<Constraint>>::new(); + for (name, rule) in python_dependencies(&document, markers, &[])? { + constraints + .entry(name) + .or_default() + .push(Constraint::Python(rule)); + } + let mut lock = LockFile::new(self.python_constraints(constraints, runtime)?)?; + self.attach_builds(&mut lock)?; + super::doctor(&lock)?; + Ok(lock) + } + + fn attach_builds(&self, lock: &mut LockFile) -> Result<(), String> { + let identities = lock + .libs + .iter() + .filter_map(|lib| lib.build.clone()) + .chain( + lock.npm + .iter() + .filter_map(|environment| environment.build.clone()), + ) + .collect::<BTreeSet<_>>(); + if !identities.is_empty() { + lock.builds = self + .build + .ok_or("build receipts lack their preparation session")? + .receipts(&identities)?; + } + Ok(()) + } + + fn python_constraints( + &self, + constraints: BTreeMap<String, Vec<Constraint>>, + runtime: &RegistryRuntime, + ) -> Result<Vec<LockedLib>, String> { + let markers = runtime + .python_markers + .as_ref() + .ok_or("Python runtime pack has no marker environment")?; + let mut remaining = constraints.keys().cloned().collect::<VecDeque<_>>(); + let mut metadata = BTreeMap::<(String, String), Value>::new(); + let mut wheels = BTreeMap::new(); + let selected = self.solve_registry(constraints, &mut metadata, &mut wheels, runtime)?; + let mut visited = BTreeSet::new(); + let mut locked = Vec::new(); + let mut distributions = BTreeMap::<String, LockedLib>::new(); + while let Some(name) = remaining.pop_front() { + if !visited.insert(name.clone()) { + continue; + } + let base = super::distribution_name(&Ecosystem::Pip, &name); + let version = selected + .get(base) + .ok_or("resolved Python version is missing")?; + let spec = LibSpec { + ecosystem: Ecosystem::Pip, + name: name.clone(), + version: version.clone(), + }; + let document = metadata + .get(&(base.to_owned(), version.clone())) + .ok_or("resolved Python metadata is missing")?; + let dependencies = + python_dependencies(document, markers, &super::python_name(&name)?.1)? + .into_iter() + .map(|(name, _)| { + let version = selected + .get(super::distribution_name(&Ecosystem::Pip, &name)) + .ok_or_else(|| format!("Python dependency {name} was not selected"))?; + selected_spec(Ecosystem::Pip, &name, version) + }) + .collect::<Result<Vec<_>, String>>()?; + let dependencies = dependencies + .into_iter() + .filter(|dependency| dependency != &spec) + .collect::<Vec<_>>(); + remaining.extend(dependencies.iter().map(|spec| spec.name.clone())); + let lib = if let Some(distribution) = distributions.get(base) { + LockedLib { + spec, + dependencies: dependencies + .into_iter() + .map(|spec| LockedDependency { spec }) + .collect(), + ..distribution.clone() + } + } else { + let candidate = wheels + .get(&(base.to_owned(), version.clone())) + .and_then(Option::as_ref) + .ok_or("resolved Python wheel is missing")?; + let mut lib = self.lock_archive( + &spec, + dependencies, + runtime, + "whl", + &candidate.bytes, + "pypi-wheel-v1", + )?; + lib.build = candidate.build.clone(); + distributions.insert(base.to_owned(), lib.clone()); + lib + }; + locked.push(lib); + } + Ok(locked) + } + + fn solve_registry( + &self, + constraints: BTreeMap<String, Vec<Constraint>>, + metadata: &mut BTreeMap<(String, String), Value>, + wheels: &mut BTreeMap<(String, String), Option<PythonCandidate>>, + runtime: &RegistryRuntime, + ) -> Result<BTreeMap<String, String>, String> { + if constraints.len() > MAX_PACKAGES { + return Err("registry dependency graph exceeds the package limit".into()); + } + let mut selected = BTreeMap::new(); + let mut state = SearchState { + metadata, + markers: runtime.python_markers.as_ref(), + runtime, + wheels, + budget: SearchBudget::default(), + }; + self.search_registry( + Ecosystem::Pip, + constraints, + BTreeSet::new(), + &mut selected, + &mut state, + ) + .map_err(SearchError::message)?; + Ok(selected) + } + + fn expand_selected( + &self, + ecosystem: &Ecosystem, + constraints: &mut BTreeMap<String, Vec<Constraint>>, + expanded: &mut BTreeSet<String>, + selected: &BTreeMap<String, String>, + state: &SearchState<'_>, + ) -> Result<(), SearchError> { + // A later edge may activate extras on an already selected distribution. + // Expand exact request variants to a fixed point inside this branch only. + loop { + if constraints.len() > MAX_PACKAGES { + return Err(SearchError::Fatal( + "registry dependency graph exceeds the package limit".into(), + )); + } + let request = constraints + .keys() + .find(|name| { + !expanded.contains(*name) + && selected.contains_key(super::distribution_name(ecosystem, name)) + }) + .cloned(); + let Some(name) = request else { break }; + let base = super::distribution_name(ecosystem, &name); + let version = &selected[base]; + let document = state + .metadata + .get(&(base.to_owned(), version.clone())) + .ok_or_else(|| { + SearchError::Fatal("selected registry metadata is missing".into()) + })?; + let dependencies = match ecosystem { + Ecosystem::Pip => { + let (_, extras) = super::python_name(&name).map_err(SearchError::Fatal)?; + validate_python_extras(document, &extras)?; + python_dependencies( + document, + state.markers.ok_or_else(|| { + SearchError::Fatal("Python marker environment is missing".into()) + })?, + &extras, + ) + .map_err(SearchError::Fatal)? + .into_iter() + .map(|(name, rule)| (name, Constraint::Python(rule))) + .collect::<Vec<_>>() + } + Ecosystem::Npm | Ecosystem::Go => unreachable!(), + }; + expanded.insert(name); + for (dependency, rule) in dependencies { + constraints.entry(dependency).or_default().push(rule); + } + } + Ok(()) + } + + fn search_registry( + &self, + ecosystem: Ecosystem, + mut constraints: BTreeMap<String, Vec<Constraint>>, + mut expanded: BTreeSet<String>, + selected: &mut BTreeMap<String, String>, + state: &mut SearchState<'_>, + ) -> Result<(), SearchError> { + self.expand_selected(&ecosystem, &mut constraints, &mut expanded, selected, state)?; + if selected.len() > MAX_PACKAGES || constraints.len() > MAX_PACKAGES { + return Err(SearchError::Fatal( + "registry dependency graph exceeds the package limit".into(), + )); + } + for (name, version) in selected.iter() { + if !constraints + .iter() + .filter(|(request, _)| super::distribution_name(&ecosystem, request) == name) + .flat_map(|(_, rules)| rules) + .all(|rule| rule.matches(version)) + { + return Err(SearchError::Conflict(format!( + "{} version conflict for {name}@{version}", + ecosystem.as_str() + ))); + } + } + let Some((name, _)) = constraints + .iter() + .find(|(name, _)| !selected.contains_key(super::distribution_name(&ecosystem, name))) + else { + return Ok(()); + }; + let name = super::distribution_name(&ecosystem, name).to_owned(); + let rules = constraints + .iter() + .filter(|(request, _)| super::distribution_name(&ecosystem, request) == name) + .flat_map(|(_, rules)| rules) + .cloned() + .collect::<Vec<_>>(); + let key = (ecosystem.clone(), name.clone()); + let versions = if let Some(versions) = state.budget.indexes.get(&key) { + versions.clone() + } else { + let versions = self + .registry_versions(ecosystem.clone(), &name) + .map_err(SearchError::Fatal)?; + if versions.len() > MAX_VERSIONS_PER_PACKAGE { + return Err(SearchError::Fatal(format!( + "registry index for {name} exceeds the version limit" + ))); + } + state.budget.indexes.insert(key, versions.clone()); + versions + }; + let mut last_conflict = None; + for version in versions + .into_iter() + .filter(|version| rules.iter().all(|rule| rule.matches(version))) + { + state.budget.attempts += 1; + if state.budget.attempts > MAX_RESOLUTION_ATTEMPTS { + return Err(SearchError::Fatal( + "registry dependency search exceeds the attempt limit".into(), + )); + } + self.registry_metadata(ecosystem.clone(), &name, &version, state.metadata) + .map_err(SearchError::Fatal)?; + let key = (name.clone(), version.clone()); + if !state.wheels.contains_key(&key) { + let spec = + selected_spec(Ecosystem::Pip, &name, &version).map_err(SearchError::Fatal)?; + let candidate = self + .python_candidate(&spec, &state.metadata[&key], state.runtime) + .map_err(SearchError::Fatal)?; + if let Some(candidate) = &candidate { + state.budget.archive_bytes = state + .budget + .archive_bytes + .checked_add(candidate.bytes.len()) + .ok_or_else(|| { + SearchError::Fatal("Python candidate cache size overflow".into()) + })?; + if state.budget.archive_bytes > crate::workers::MAX_TREE_BYTES { + return Err(SearchError::Fatal( + "Python candidate cache exceeds its 256 MiB budget".into(), + )); + } + // Release JSON is upload-time metadata and can differ per + // platform. Only this verified wheel defines solver edges. + state.metadata.get_mut(&key).unwrap()["info"] = candidate.info.clone(); + } + state.wheels.insert(key.clone(), candidate); + } + if state.wheels[&key].is_none() { + last_conflict = Some(format!( + "pip:{name}@{version} has no compatible wheel for the signed Python runtime" + )); + continue; + } + let mut branch_selected = selected.clone(); + branch_selected.insert(name.clone(), version.clone()); + match self.search_registry( + ecosystem.clone(), + constraints.clone(), + expanded.clone(), + &mut branch_selected, + state, + ) { + Ok(()) => { + *selected = branch_selected; + return Ok(()); + } + Err(SearchError::Conflict(error)) => last_conflict = Some(error), + Err(fatal) => return Err(fatal), + } + } + Err(SearchError::Conflict(last_conflict.unwrap_or_else(|| { + format!( + "no {} version satisfies constraints for {name}", + ecosystem.as_str() + ) + }))) + } + + fn registry_versions(&self, ecosystem: Ecosystem, name: &str) -> Result<Vec<String>, String> { + let path = match ecosystem { + Ecosystem::Pip => format!("/pypi/{name}/json"), + Ecosystem::Npm => format!("/{}", npm_path(name)), + Ecosystem::Go => unreachable!(), + }; + let bytes = if ecosystem == Ecosystem::Npm { + let Some(bytes) = + self.transport + .get_optional(ecosystem.clone(), &path, MAX_METADATA_BYTES)? + else { + return Ok(Vec::new()); + }; + bytes + } else { + self.transport + .get(ecosystem.clone(), &path, MAX_METADATA_BYTES)? + }; + let document = parse_json(&bytes)?; + let versions = match ecosystem { + Ecosystem::Pip => document.get("releases"), + Ecosystem::Npm => document.get("versions"), + Ecosystem::Go => unreachable!(), + } + .and_then(Value::as_object) + .ok_or("registry version index is invalid")?; + if versions.len() > MAX_VERSIONS_PER_PACKAGE { + return Err("registry version index exceeds its version budget".into()); + } + if ecosystem == Ecosystem::Npm + && versions + .keys() + .any(|version| super::validate_version(&ecosystem, version).is_err()) + { + return Err("npm registry index contains an invalid version".into()); + } + let mut versions = versions + .keys() + .filter(|version| super::validate_version(&ecosystem, version).is_ok()) + .cloned() + .collect::<Vec<_>>(); + match ecosystem { + Ecosystem::Pip => { + versions.retain(|version| PythonVersion::from_str(version).is_ok()); + versions.sort_by(|left, right| { + PythonVersion::from_str(right) + .unwrap() + .cmp(&PythonVersion::from_str(left).unwrap()) + }); + } + Ecosystem::Npm => { + versions.retain(|version| NpmVersion::from_str(version).is_ok()); + versions.sort_by(|left, right| { + NpmVersion::from_str(right) + .unwrap() + .cmp(&NpmVersion::from_str(left).unwrap()) + }); + } + Ecosystem::Go => unreachable!(), + } + Ok(versions) + } + + fn registry_metadata( + &self, + ecosystem: Ecosystem, + name: &str, + version: &str, + cache: &mut BTreeMap<(String, String), Value>, + ) -> Result<Value, String> { + let key = (name.to_owned(), version.to_owned()); + if let Some(document) = cache.get(&key) { + return Ok(document.clone()); + } + let path = match ecosystem { + Ecosystem::Pip => format!("/pypi/{name}/{version}/json"), + Ecosystem::Npm => format!("/{}/{}", npm_path(name), version), + Ecosystem::Go => unreachable!(), + }; + let document = parse_json(&self.transport.get(ecosystem, &path, MAX_METADATA_BYTES)?)?; + cache.insert(key, document.clone()); + Ok(document) + } + + fn python_candidate( + &self, + spec: &LibSpec, + document: &Value, + runtime: &RegistryRuntime, + ) -> Result<Option<PythonCandidate>, String> { + let files = document + .get("urls") + .and_then(Value::as_array) + .ok_or("PyPI release has no files")?; + let python_version = runtime + .python_markers + .as_ref() + .ok_or("Python marker environment is missing")? + .python_full_version() + .to_string() + .parse::<PythonVersion>() + .map_err(|error| error.to_string())?; + let file = runtime.python_wheel_tags.iter().find_map(|tag| { + files.iter().find(|file| { + file.get("packagetype").and_then(Value::as_str) == Some("bdist_wheel") + && file + .get("filename") + .and_then(Value::as_str) + .is_some_and(|name| crate::workers::python_wheel::matches(name, tag)) + && file + .get("requires_python") + .and_then(Value::as_str) + .is_none_or(|rule| { + rule.parse::<VersionSpecifiers>() + .is_ok_and(|rule| rule.contains(&python_version)) + }) + }) + }); + let Some(file) = file else { + return self.python_source_candidate(spec, files, runtime); + }; + let expected = file + .get("digests") + .and_then(|digests| digests.get("sha256")) + .and_then(Value::as_str) + .ok_or("PyPI file lacks SHA-256")?; + let url = field(file, "url")?; + let bytes = self + .transport + .get(Ecosystem::Pip, url, MAX_ARTIFACT_BYTES)?; + if sha256(&bytes) != expected { + return Err(format!("PyPI wheel SHA-256 mismatch for {}", spec.key())); + } + let wheel = crate::workers::python_wheel::Wheel::parse( + spec, + Some(field(file, "filename")?), + &runtime.python_wheel_tags, + &bytes, + )?; + if let Some(rule) = &wheel.requires_python + && !rule + .parse::<VersionSpecifiers>() + .map_err(|error| error.to_string())? + .contains(&python_version) + { + return Ok(None); + } + Ok(Some(PythonCandidate { + build: None, + bytes, + info: serde_json::json!({"requires_dist":wheel.requirements,"provides_extra":wheel.extras}), + })) + } + + fn python_source_candidate( + &self, + spec: &LibSpec, + files: &[Value], + runtime: &RegistryRuntime, + ) -> Result<Option<PythonCandidate>, String> { + let python = runtime + .python_markers + .as_ref() + .ok_or("Python marker environment is missing")? + .python_full_version(); + let Some(file) = files.iter().find(|file| { + file.get("packagetype").and_then(Value::as_str) == Some("sdist") + && file + .get("requires_python") + .and_then(Value::as_str) + .is_none_or(|rule| { + rule.parse::<VersionSpecifiers>() + .is_ok_and(|rule| rule.contains(python)) + }) + }) else { + return Ok(None); + }; + let session = self.build.ok_or("Python source distribution requires a signed build pack during explicit lib add/update")?; + let expected = file + .get("digests") + .and_then(|value| value.get("sha256")) + .and_then(Value::as_str) + .ok_or("PyPI source lacks SHA-256")?; + let bytes = self + .transport + .get(Ecosystem::Pip, field(file, "url")?, MAX_ARTIFACT_BYTES)?; + if sha256(&bytes) != expected { + return Err("PyPI source SHA-256 mismatch".into()); + } + let product = session.python(self, spec, runtime, field(file, "filename")?, &bytes)?; + let wheel = crate::workers::python_wheel::Wheel::parse( + spec, + Some(&product.filename), + &runtime.python_wheel_tags, + &product.bytes, + )?; + if wheel.requires_python.as_ref().is_some_and(|rule| { + rule.parse::<VersionSpecifiers>() + .is_ok_and(|rule| !rule.contains(python)) + }) { + return Ok(None); + } + Ok(Some(PythonCandidate { + bytes: product.bytes, + info: serde_json::json!({"requires_dist":wheel.requirements,"provides_extra":wheel.extras}), + build: Some(product.receipt.identity()?), + })) + } + + fn npm_archive(&self, spec: &LibSpec, document: &Value) -> Result<Vec<u8>, String> { + let dist = document + .get("dist") + .ok_or("npm version has no dist metadata")?; + let integrity = field(dist, "integrity")?; + let expected = integrity + .strip_prefix("sha512-") + .ok_or("npm dist.integrity must use SHA-512")?; + let url = field(dist, "tarball")?; + let bytes = self + .transport + .get(Ecosystem::Npm, url, MAX_ARTIFACT_BYTES)?; + let digest = Sha512::digest(&bytes); + use base64::Engine; + if base64::engine::general_purpose::STANDARD.encode(digest) != expected { + return Err(format!("npm tarball integrity mismatch for {}", spec.key())); + } + Ok(bytes) + } + + fn lock_archive( + &self, + spec: &LibSpec, + dependencies: Vec<LibSpec>, + runtime: &RegistryRuntime, + extension: &str, + bytes: &[u8], + schema: &str, + ) -> Result<LockedLib, String> { + let files = archive_files(extension, bytes)?; + if spec.ecosystem == Ecosystem::Go { + let prefix = format!("{}@v{}/", spec.name, spec.version); + if files.iter().any(|(path, _)| !path.starts_with(&prefix)) { + return Err(format!( + "Go module archive for {} has a mismatched module root", + spec.key() + )); + } + } + let path = format!( + "lib/{}/{}/{}/archive.{extension}", + spec.ecosystem.as_str(), + sha256(spec.distribution_name().as_bytes()), + spec.version + ); + let artifact = LockedArtifact { + target: runtime.target.clone(), + path, + bytes: bytes.len() as u64, + sha256: sha256(bytes), + }; + validate_artifact(&artifact)?; + let published = self.store.publish(bytes, &runtime.target)?; + if published != artifact.sha256 { + return Err(format!("artifact cache digest mismatch for {}", spec.key())); + } + let mut dependencies = dependencies + .into_iter() + .map(|spec| LockedDependency { spec }) + .collect::<Vec<_>>(); + dependencies.sort(); + dependencies.dedup(); + Ok(LockedLib { + spec: spec.clone(), + dependencies, + runtime: runtime.pack.clone(), + artifacts: vec![artifact], + schema: schema.into(), + build: None, + }) + } + + fn resolve_go( + &self, + roots: &[LibSpec], + runtime: &RegistryRuntime, + ) -> Result<(Vec<LockedLib>, sumdb::Evidence), String> { + let mut sumdb = self.go_sumdb.begin(self.transport)?; + let roots = roots + .iter() + .map(|spec| (spec.name.clone(), spec.version.clone())) + .collect::<BTreeMap<_, _>>(); + let mut selected = roots.clone(); + let mut metadata = BTreeMap::<(String, String), Vec<u8>>::new(); + for _ in 0..MAX_PACKAGES { + let mut next = roots.clone(); + let mut queue = roots.keys().cloned().collect::<VecDeque<_>>(); + let mut visited = BTreeSet::new(); + while let Some(name) = queue.pop_front() { + if !visited.insert(name.clone()) { + continue; + } + let version = selected + .get(&name) + .or_else(|| next.get(&name)) + .ok_or("Go module has no selected version")? + .clone(); + let source = self.go_mod(&name, &version, &mut metadata, &mut sumdb)?; + for (dependency, required) in + go_requirements(&source, &name, &runtime.pack.version)? + { + let old = next + .entry(dependency.clone()) + .or_insert_with(|| required.clone()); + if go_version_key(&required) > go_version_key(old) { + *old = required; + } + queue.push_back(dependency); + } + if visited.len() > MAX_PACKAGES { + return Err("Go module graph exceeds the package limit".into()); + } + } + if next == selected { + let libs = selected + .iter() + .map(|(name, version)| { + let spec = LibSpec { + ecosystem: Ecosystem::Go, + name: name.clone(), + version: version.clone(), + }; + let source = self.go_mod(name, version, &mut metadata, &mut sumdb)?; + let dependencies = go_requirements(&source, name, &runtime.pack.version)? + .into_keys() + .map(|name| { + let version = + selected.get(&name).ok_or("Go MVS dependency is missing")?; + selected_spec(Ecosystem::Go, &name, version) + }) + .collect::<Result<Vec<_>, String>>()?; + let zip = self.transport.get( + Ecosystem::Go, + &format!( + "/{}/@v/v{}.zip", + go_path(name), + version.trim_start_matches('v') + ), + MAX_ARTIFACT_BYTES, + )?; + sumdb.verify_zip(name, version, &zip)?; + self.lock_archive(&spec, dependencies, runtime, "zip", &zip, "go-sumdb-v1") + }) + .collect::<Result<Vec<_>, String>>()?; + return Ok((libs, sumdb.finish())); + } + selected = next; + } + Err("Go MVS selection did not converge".into()) + } + + fn go_mod( + &self, + name: &str, + version: &str, + cache: &mut BTreeMap<(String, String), Vec<u8>>, + sumdb: &mut sumdb::Session<'_>, + ) -> Result<Vec<u8>, String> { + let key = (name.to_owned(), version.to_owned()); + if let Some(source) = cache.get(&key) { + return Ok(source.clone()); + } + let source = self.transport.get( + Ecosystem::Go, + &format!( + "/{}/@v/v{}.mod", + go_path(name), + version.trim_start_matches('v') + ), + MAX_METADATA_BYTES, + )?; + sumdb.verify_mod(name, version, &source)?; + cache.insert(key, source.clone()); + Ok(source) + } +} + +#[derive(Clone)] +enum Constraint { + Exact(String), + Python(VersionSpecifiers), +} + +#[derive(Default)] +struct SearchBudget { + attempts: usize, + indexes: BTreeMap<(Ecosystem, String), Vec<String>>, + archive_bytes: usize, +} + +struct PythonCandidate { + bytes: Vec<u8>, + info: Value, + build: Option<String>, +} + +struct SearchState<'a> { + metadata: &'a mut BTreeMap<(String, String), Value>, + markers: Option<&'a MarkerEnvironment>, + runtime: &'a RegistryRuntime, + wheels: &'a mut BTreeMap<(String, String), Option<PythonCandidate>>, + budget: SearchBudget, +} + +enum SearchError { + Conflict(String), + Fatal(String), +} + +impl SearchError { + fn message(self) -> String { + match self { + Self::Conflict(message) | Self::Fatal(message) => message, + } + } +} + +impl Constraint { + fn matches(&self, version: &str) -> bool { + match self { + Self::Exact(exact) => exact == version, + Self::Python(rule) => { + PythonVersion::from_str(version).is_ok_and(|version| rule.contains(&version)) + } + } + } +} + +pub(crate) fn python_dependencies( + document: &Value, + markers: &MarkerEnvironment, + extras: &[pep508_rs::ExtraName], +) -> Result<Vec<(String, VersionSpecifiers)>, String> { + let dependencies = document + .get("info") + .ok_or("PyPI release lacks info metadata")? + .get("requires_dist") + .ok_or("PyPI release lacks requires_dist metadata")?; + if dependencies.is_null() { + return Ok(Vec::new()); + } + let dependencies = dependencies + .as_array() + .ok_or("PyPI requires_dist must be an array or null")?; + dependencies + .iter() + .map(|dependency| { + let requirement: Requirement = field_value(dependency)? + .parse() + .map_err(|error| format!("invalid Requires-Dist: {error}"))?; + if !requirement.evaluate_markers(markers, extras) { + return Ok(None); + } + let name = if requirement.extras.is_empty() { + requirement.name.to_string() + } else { + super::python_name(&format!( + "{}[{}]", + requirement.name, + requirement + .extras + .iter() + .map(ToString::to_string) + .collect::<Vec<_>>() + .join(",") + ))? + .0 + }; + let specifiers = match requirement.version_or_url { + Some(VersionOrUrl::VersionSpecifier(specifiers)) => specifiers, + None => VersionSpecifiers::empty(), + Some(VersionOrUrl::Url(_)) => { + return Err("PyPI dependency URL is not supported".into()); + } + }; + Ok(Some((name, specifiers))) + }) + .collect::<Result<Vec<_>, String>>() + .map(|entries| entries.into_iter().flatten().collect()) +} + +fn validate_python_extras( + document: &Value, + extras: &[pep508_rs::ExtraName], +) -> Result<(), SearchError> { + let declared = document + .get("info") + .and_then(|info| info.get("provides_extra")); + let declared = match declared { + None | Some(Value::Null) => BTreeSet::new(), + Some(Value::Array(values)) => values + .iter() + .map(|value| { + super::python_extra(field_value(value)?) + .map_err(|error| format!("invalid PyPI Provides-Extra: {error}")) + }) + .collect::<Result<BTreeSet<_>, String>>() + .map_err(SearchError::Fatal)?, + _ => { + return Err(SearchError::Fatal( + "PyPI provides_extra must be an array or null".into(), + )); + } + }; + for extra in extras { + if !declared.contains(extra) { + return Err(SearchError::Conflict(format!( + "PyPI release does not provide requested extra '{extra}'" + ))); + } + } + Ok(()) +} + +fn go_requirements( + bytes: &[u8], + module: &str, + runtime_version: &str, +) -> Result<BTreeMap<String, String>, String> { + let text = std::str::from_utf8(bytes).map_err(|_| "Go mod file is not UTF-8")?; + let mut dependencies = BTreeMap::new(); + let mut in_require = false; + let mut declared_module = false; + let runtime_version = NpmVersion::from_str(runtime_version) + .map_err(|error| format!("managed Go runtime has an invalid version: {error}"))?; + for line in text.lines() { + let line = line.split("//").next().unwrap_or("").trim(); + if line.is_empty() { + continue; + } + if line == "require (" { + in_require = true; + continue; + } + if in_require && line == ")" { + in_require = false; + continue; + } + if line.starts_with("replace ") + || line.starts_with("exclude ") + || line.starts_with("replace(") + || line.starts_with("exclude(") + { + return Err("Go replace/exclude directives require a managed module policy".into()); + } + let fields = line.split_whitespace().collect::<Vec<_>>(); + if !in_require { + match fields.as_slice() { + ["module", declared] => { + if declared_module || *declared != module { + return Err(format!("Go module declaration does not match {module}")); + } + declared_module = true; + continue; + } + ["go", version] | ["toolchain", version] => { + let minimum = go_toolchain_version(version)?; + if minimum > runtime_version { + return Err(format!( + "Go module requires toolchain {minimum}, managed runtime is {runtime_version}" + )); + } + continue; + } + ["require", ..] => {} + _ => return Err(format!("unsupported Go module directive '{line}'")), + } + } + let pair = if in_require { + fields.as_slice() + } else { + &fields[1..] + }; + if pair.len() != 2 { + return Err("invalid Go require directive".into()); + } + let spec = + format!("go:{}@{}", pair[0], pair[1].trim_start_matches('v')).parse::<LibSpec>()?; + if dependencies.insert(spec.name, spec.version).is_some() { + return Err("duplicate Go require directive".into()); + } + } + if in_require { + return Err("unclosed Go require block".into()); + } + if !declared_module { + return Err(format!("Go module declaration for {module} is missing")); + } + Ok(dependencies) +} + +fn go_toolchain_version(value: &str) -> Result<NpmVersion, String> { + let value = value.strip_prefix("go").unwrap_or(value); + let version = if value.split('.').count() == 2 { + format!("{value}.0") + } else { + value.to_owned() + }; + NpmVersion::from_str(&version) + .map_err(|error| format!("invalid Go toolchain directive: {error}")) +} + +fn go_version_key(version: &str) -> NpmVersion { + NpmVersion::from_str(version).expect("Go version was validated when parsed") +} + +fn go_path(name: &str) -> String { + name.chars() + .flat_map(|character| { + if character.is_ascii_uppercase() { + vec!['!', character.to_ascii_lowercase()] + } else { + vec![character] + } + }) + .collect() +} + +fn npm_path(name: &str) -> String { + name.replace('/', "%2f") +} + +fn selected_spec(ecosystem: Ecosystem, name: &str, version: &str) -> Result<LibSpec, String> { + format!("{}:{name}@{version}", ecosystem.as_str()).parse() +} + +fn field<'a>(document: &'a Value, name: &str) -> Result<&'a str, String> { + document + .get(name) + .and_then(Value::as_str) + .ok_or_else(|| format!("registry metadata lacks {name}")) +} + +fn field_value(value: &Value) -> Result<&str, String> { + value.as_str().ok_or("registry field must be text".into()) +} + +fn parse_json(bytes: &[u8]) -> Result<Value, String> { + serde_json::from_slice(bytes).map_err(|error| format!("invalid registry JSON: {error}")) +} + +fn safe_archive_path(path: &Path) -> Result<(), String> { + let text = path.to_str().ok_or("archive path is not UTF-8")?; + if text.is_empty() + || text.contains(['\\', ':']) + || text + .split('/') + .any(|part| part.is_empty() || part == "." || part == "..") + || path.is_absolute() + || path + .components() + .any(|part| !matches!(part, Component::Normal(_))) + { + return Err(format!("archive path '{text}' escapes its package")); + } + Ok(()) +} + +/// The same bounded extraction contract is used by registry admission and +/// managed Worker packaging. Files retain their archive-relative names. +pub(crate) fn archive_files( + extension: &str, + bytes: &[u8], +) -> Result<Vec<(String, Vec<u8>)>, String> { + if matches!(extension, "whl" | "zip") { + return Ok(zip_entries(bytes)? + .into_iter() + .filter(|(name, _)| !name.ends_with('/')) + .collect()); + } + tar_files(bytes, extension, 256 * 1024 * 1024) +} + +/// Read only permission metadata from an archive already admitted by +/// `archive_files`. Set-id bits are never propagated to an installed tree. +pub(crate) fn archive_executables( + extension: &str, + bytes: &[u8], +) -> Result<BTreeSet<String>, String> { + let mut paths = BTreeSet::new(); + if matches!(extension, "zip" | "whl") { + let mut archive = + zip::ZipArchive::new(Cursor::new(bytes)).map_err(|error| error.to_string())?; + for index in 0..archive.len() { + let entry = archive.by_index(index).map_err(|error| error.to_string())?; + if !entry.is_dir() && entry.unix_mode().unwrap_or(0) & 0o111 != 0 { + paths.insert(entry.name().into()); + } + } + } else if extension == "tgz" { + let mut archive = tar::Archive::new(flate2::read::GzDecoder::new(Cursor::new(bytes))); + for entry in archive.entries().map_err(|error| error.to_string())? { + let entry = entry.map_err(|error| error.to_string())?; + if entry.header().entry_type().is_file() + && entry.header().mode().map_err(|error| error.to_string())? & 0o111 != 0 + { + paths.insert( + entry + .path() + .map_err(|error| error.to_string())? + .to_str() + .ok_or("archive path is not UTF-8")? + .into(), + ); + } + } + } else { + return Err("unsupported archive permissions format".into()); + } + Ok(paths) +} + +/// Build-tool packs have a separate expansion budget; normal packages retain +/// the Worker limit. Both paths use the same entry and traversal validation. +pub(crate) fn tar_files( + bytes: &[u8], + extension: &str, + max_bytes: u64, +) -> Result<Vec<(String, Vec<u8>)>, String> { + let mut total = 0u64; + let mut names = BTreeSet::new(); + let mut files = Vec::new(); + match extension { + "tgz" => { + let decoder = flate2::read::GzDecoder::new(Cursor::new(bytes)); + let mut archive = tar::Archive::new(decoder); + for (index, entry) in archive + .entries() + .map_err(|error| format!("invalid tar archive: {error}"))? + .enumerate() + { + if index >= 65_536 { + return Err("archive has too many entries".into()); + } + let mut entry = entry.map_err(|error| format!("invalid tar entry: {error}"))?; + if !entry.header().entry_type().is_file() && !entry.header().entry_type().is_dir() { + return Err("archive contains a link or unsupported entry".into()); + } + let path = entry + .path() + .map_err(|error| format!("invalid tar path: {error}"))? + .into_owned(); + let name = path + .to_str() + .ok_or("archive path is not UTF-8")? + .trim_end_matches('/') + .to_owned(); + safe_archive_path(Path::new(&name))?; + if !names.insert(name.clone()) { + return Err(format!("archive repeats path '{name}'")); + } + if entry.header().entry_type().is_dir() { + continue; + } + total = total + .checked_add(entry.size()) + .ok_or("archive expanded size overflow")?; + if total > max_bytes { + return Err("archive expanded size exceeds its byte budget".into()); + } + let mut output = Vec::new(); + let expected_size = entry.size(); + let remaining = max_bytes - total + expected_size + 1; + entry + .by_ref() + .take(remaining) + .read_to_end(&mut output) + .map_err(|error| format!("corrupt tar entry: {error}"))?; + if output.len() as u64 != expected_size { + return Err("tar entry size drift".into()); + } + files.push((name, output)); + } + } + _ => return Err("unsupported registry archive format".into()), + } + if total == 0 { + return Err("registry archive is empty".into()); + } + Ok(files) +} + +/// Preserve exact ZIP names, including directory entries: Go's HashZip hashes +/// those entries too. Extraction consumers explicitly filter directories. +fn zip_entries(bytes: &[u8]) -> Result<Vec<(String, Vec<u8>)>, String> { + let mut archive = zip::ZipArchive::new(Cursor::new(bytes)) + .map_err(|error| format!("invalid ZIP archive: {error}"))?; + if archive.len() > 65_536 { + return Err("archive has too many entries".into()); + } + let mut total = 0u64; + let mut names = BTreeSet::new(); + let mut entries = Vec::new(); + for index in 0..archive.len() { + let mut entry = archive + .by_index(index) + .map_err(|error| format!("invalid ZIP entry: {error}"))?; + let name = entry.name().to_owned(); + let path = name.strip_suffix('/').unwrap_or(&name); + safe_archive_path(Path::new(path))?; + if !names.insert(path.to_owned()) { + return Err(format!("archive repeats path '{name}'")); + } + if entry + .unix_mode() + .is_some_and(|mode| !matches!(mode & 0o170000, 0 | 0o100000 | 0o040000)) + { + return Err("archive contains a link or unsupported entry".into()); + } + total = total + .checked_add(entry.size()) + .ok_or("archive expanded size overflow")?; + if total > 256 * 1024 * 1024 { + return Err("archive expanded size exceeds 256 MiB".into()); + } + let expected_size = entry.size(); + let mut output = Vec::new(); + entry + .by_ref() + .take(256 * 1024 * 1024 - total + expected_size + 1) + .read_to_end(&mut output) + .map_err(|error| format!("corrupt ZIP entry: {error}"))?; + if output.len() as u64 != expected_size { + return Err("ZIP entry size drift".into()); + } + entries.push((name, output)); + } + if total == 0 { + return Err("registry archive is empty".into()); + } + Ok(entries) +} diff --git a/crates/dever-cli/src/libs/registry/npm.rs b/crates/dever-cli/src/libs/registry/npm.rs new file mode 100644 index 0000000..6b1a68e --- /dev/null +++ b/crates/dever-cli/src/libs/registry/npm.rs @@ -0,0 +1,995 @@ +//! npm package instances follow Node's ancestor lookup, not one global version +//! per name. Archives remain content-addressed and shared between instances. + +use std::collections::{BTreeMap, BTreeSet, VecDeque}; + +use node_semver::{Range, Version}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use super::{Ecosystem, LibSpec, LockFile, LockedLib, RegistryRuntime, archive_files}; + +mod resolve; +pub(super) use resolve::resolve; + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Kind { + Dependency, + Optional, + Peer, + OptionalPeer, +} + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Omission { + Target, + Unavailable, + Conflict, + OptionalPeerAbsent, + BuildFailure, +} + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Edge { + pub name: String, + pub requirement: String, + pub kind: Kind, + pub target: Option<String>, + pub omission: Option<Omission>, +} + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Source { + pub archive: LibSpec, + pub prefix: String, +} + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Instance { + pub path: String, + pub spec: LibSpec, + pub source: Source, + /// Exact bundled layout of a root archive, including hoisted transitives. + pub bundled: BTreeMap<String, LibSpec>, + pub edges: Vec<Edge>, +} + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Environment { + pub roots: Vec<LibSpec>, + pub instances: Vec<Instance>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub build: Option<String>, +} + +impl Environment { + /// Explicit single-package graph for author fixtures with no dependencies. + /// Materialization still checks package.json against this graph. + pub fn single(spec: LibSpec) -> Self { + Self { + build: None, + roots: vec![spec.clone()], + instances: vec![Instance { + path: child("", &spec.name), + spec: spec.clone(), + source: Source { + archive: spec, + prefix: "package/".into(), + }, + bundled: BTreeMap::new(), + edges: Vec::new(), + }], + } + } +} + +pub fn roots(specs: &[LibSpec]) -> Vec<LibSpec> { + let mut roots = specs + .iter() + .filter(|spec| spec.ecosystem == Ecosystem::Npm) + .cloned() + .collect::<Vec<_>>(); + roots.sort(); + roots.dedup(); + roots +} + +pub fn environment<'a>( + lock: &'a LockFile, + requested: &[LibSpec], +) -> Result<Option<&'a Environment>, String> { + let requested = roots(requested); + if requested.is_empty() { + return Ok(None); + } + lock.npm + .iter() + .find(|env| env.roots == requested) + .map(Some) + .ok_or("npm Worker roots have no locked installation environment".into()) +} + +pub fn archive_roots(lock: &LockFile, requested: &[LibSpec]) -> Result<BTreeSet<LibSpec>, String> { + let requested = roots(requested); + if requested.is_empty() { + return Ok(BTreeSet::new()); + } + if let Some(environment) = lock + .npm + .iter() + .find(|environment| environment.roots == requested) + { + return Ok(environment + .instances + .iter() + .map(|node| node.source.archive.clone()) + .collect()); + } + // Built-in metadata fixtures are not registry packages and have no tree. + let fixtures = super::super::FixtureRegistry::builtin(); + if requested + .iter() + .all(|spec| fixtures.package(spec).is_some()) + { + return Ok(requested.into_iter().collect()); + } + Err("npm roots have no exact locked installation environment".into()) +} + +pub fn retain(lock: &LockFile, groups: &[Vec<LibSpec>]) -> Result<Vec<Environment>, String> { + let mut environments = Vec::new(); + for group in groups { + let requested = roots(group); + if requested.is_empty() { + continue; + } + let original = lock + .npm + .iter() + .find(|env| env.roots == requested) + .or_else(|| { + lock.npm + .iter() + .find(|env| requested.iter().all(|root| env.roots.contains(root))) + }); + let Some(original) = original else { + archive_roots(lock, group)?; + continue; + }; + let mut queue = requested + .iter() + .map(|spec| child("", &spec.name)) + .collect::<VecDeque<_>>(); + let mut paths = BTreeSet::new(); + while let Some(path) = queue.pop_front() { + if !paths.insert(path.clone()) { + continue; + } + let node = original + .instances + .iter() + .find(|node| node.path == path) + .ok_or("npm retained graph lacks a dependency instance")?; + queue.extend(node.edges.iter().filter_map(|edge| edge.target.clone())); + } + environments.push(Environment { + build: original.build.clone(), + roots: requested, + instances: original + .instances + .iter() + .filter(|node| paths.contains(&node.path)) + .cloned() + .collect(), + }); + } + normalize(&mut environments)?; + Ok(environments) +} + +pub(super) fn child(parent: &str, name: &str) -> String { + if parent.is_empty() { + format!("node_modules/{name}") + } else { + format!("{parent}/node_modules/{name}") + } +} + +pub(super) fn parent(path: &str) -> &str { + path.rsplit_once("/node_modules/") + .map_or("", |(parent, _)| parent) +} + +fn validate_path(path: &str) -> Result<(), String> { + if path.len() > 4096 { + return Err("npm instance path exceeds its bound".into()); + } + let path = path + .strip_prefix("node_modules/") + .ok_or("invalid npm instance path")?; + for name in path.split("/node_modules/") { + super::super::validate_name(&Ecosystem::Npm, name)?; + } + Ok(()) +} + +pub(super) fn visible<'a>( + instances: &'a BTreeMap<String, Instance>, + consumer: &str, + name: &str, + peer: bool, +) -> Option<&'a Instance> { + let mut scope = if peer { parent(consumer) } else { consumer }; + loop { + if let Some(found) = instances.get(&child(scope, name)) { + return Some(found); + } + if scope.is_empty() { + return None; + } + scope = parent(scope); + } +} + +#[derive(Clone, Debug)] +pub(super) struct Metadata { + pub spec: LibSpec, + pub edges: Vec<Edge>, + pub bundles: BTreeSet<String>, + pub document: Value, +} + +fn requirements(document: &Value, field: &str) -> Result<BTreeMap<String, String>, String> { + let Some(value) = document.get(field) else { + return Ok(BTreeMap::new()); + }; + value + .as_object() + .ok_or_else(|| format!("npm {field} must be an object"))? + .iter() + .map(|(name, value)| { + super::super::validate_name(&Ecosystem::Npm, name)?; + let rule = value.as_str().ok_or("npm dependency range must be text")?; + rule.parse::<Range>() + .map_err(|error| format!("invalid npm range for {name}: {error}"))?; + Ok((name.clone(), rule.into())) + }) + .collect() +} + +impl Metadata { + pub(super) fn parse(document: Value) -> Result<Self, String> { + let spec: LibSpec = format!( + "npm:{}@{}", + super::field(&document, "name")?, + super::field(&document, "version")? + ) + .parse()?; + let mut regular = requirements(&document, "dependencies")?; + let optional = requirements(&document, "optionalDependencies")?; + regular.retain(|name, _| !optional.contains_key(name)); + let mut peers = requirements(&document, "peerDependencies")?; + let peer_meta = document + .get("peerDependenciesMeta") + .map(|value| { + value + .as_object() + .ok_or("npm peerDependenciesMeta must be an object") + }) + .transpose()?; + if let Some(meta) = peer_meta { + for (name, value) in meta { + if !peers.contains_key(name) + || !value.is_object() + || value.get("optional").is_some_and(|flag| !flag.is_boolean()) + { + return Err("invalid npm optional peer metadata".into()); + } + } + } + // npm has one outgoing edge per name: optional > production > peer. + peers.retain(|name, _| !regular.contains_key(name) && !optional.contains_key(name)); + let mut edges = Vec::new(); + for (entries, kind) in [ + (regular.clone(), Kind::Dependency), + (optional.clone(), Kind::Optional), + (peers, Kind::Peer), + ] { + for (name, requirement) in entries { + let kind = if kind == Kind::Peer + && peer_meta + .and_then(|meta| meta.get(&name)) + .and_then(|value| value.get("optional")) + .and_then(Value::as_bool) + == Some(true) + { + Kind::OptionalPeer + } else { + kind.clone() + }; + edges.push(Edge { + name, + requirement, + kind, + target: None, + omission: None, + }); + } + } + edges.sort(); + let bundles = |value: &Value| -> Result<BTreeSet<String>, String> { + match value { + Value::Bool(true) => Ok(regular.keys().chain(optional.keys()).cloned().collect()), + Value::Bool(false) => Ok(BTreeSet::new()), + Value::Array(values) => values + .iter() + .map(|value| { + let name = value + .as_str() + .ok_or("npm bundled dependency name must be text")?; + super::super::validate_name(&Ecosystem::Npm, name)?; + if !regular.contains_key(name) && !optional.contains_key(name) { + return Err( + "npm bundled dependency is not a declared dependency".into() + ); + } + Ok(name.to_owned()) + }) + .collect(), + _ => Err("npm bundleDependencies must be an array or boolean".into()), + } + }; + let first = document + .get("bundleDependencies") + .map(bundles) + .transpose()?; + let second = document + .get("bundledDependencies") + .map(bundles) + .transpose()?; + if first.is_some() && second.is_some() && first != second { + return Err("npm bundled dependency aliases disagree".into()); + } + Ok(Self { + spec, + edges, + bundles: first.or(second).unwrap_or_default(), + document, + }) + } + + pub(super) fn eligible(&self, runtime: &RegistryRuntime) -> Result<bool, String> { + let (os, cpu) = runtime + .target + .split_once('-') + .ok_or("npm runtime target lacks platform identity")?; + let os = match os { + "windows" => "win32", + "macos" => "darwin", + other => other, + }; + let cpu = match cpu { + "x86_64" => "x64", + "aarch64" => "arm64", + other => other, + }; + for (field, selected) in [ + ("os", Some(os)), + ("cpu", Some(cpu)), + ("libc", runtime.npm_libc.as_deref()), + ] { + if field == "libc" && os != "linux" { + continue; + } + let Some(value) = self.document.get(field) else { + continue; + }; + let values = value + .as_array() + .ok_or_else(|| format!("npm {field} must be an array"))?; + let values = values + .iter() + .map(|value| value.as_str().ok_or("npm platform selector must be text")) + .collect::<Result<Vec<_>, _>>()?; + let Some(selected) = selected else { + return Ok(false); + }; + if values + .iter() + .any(|value| value.strip_prefix('!') == Some(selected)) + || (values.iter().any(|value| !value.starts_with('!')) + && !values.contains(&selected) + && !values.contains(&"any")) + { + return Ok(false); + } + } + Ok(true) + } +} + +pub(super) struct Archive { + pub files: Vec<(String, Vec<u8>)>, + pub packages: BTreeMap<String, Metadata>, +} + +impl Archive { + pub(super) fn bundled(&self) -> BTreeMap<String, LibSpec> { + self.packages + .iter() + .filter(|(prefix, _)| prefix.as_str() != "package/") + .map(|(prefix, metadata)| { + ( + prefix + .strip_prefix("package/") + .unwrap() + .trim_end_matches('/') + .into(), + metadata.spec.clone(), + ) + }) + .collect() + } + + pub(super) fn parse(spec: &LibSpec, bytes: &[u8]) -> Result<Self, String> { + let files = archive_files("tgz", bytes)?; + let mut packages = BTreeMap::<String, Metadata>::new(); + for (path, bytes) in &files { + let relative = path + .strip_prefix("package/") + .ok_or("npm archive must have a package/ root")?; + if relative == "package.json" || relative.ends_with("/package.json") { + let prefix = path.strip_suffix("package.json").unwrap(); + if prefix != "package/" { + let nested = prefix + .strip_prefix("package/") + .unwrap() + .trim_end_matches('/'); + if validate_path(nested).is_err() { + continue; + } + } + let document = super::parse_json(bytes)?; + let metadata = Metadata::parse(document)?; + if prefix == "package/" { + if metadata.spec != *spec { + return Err( + "npm package.json identity differs from its registry archive".into(), + ); + } + } else if !prefix.ends_with(&format!("node_modules/{}/", metadata.spec.name)) { + return Err("npm bundled package identity differs from its path".into()); + } + packages.insert(prefix.into(), metadata); + } + } + let root = packages + .get("package/") + .ok_or("npm archive lacks package.json")?; + for name in &root.bundles { + if !packages.contains_key(&format!("package/node_modules/{name}/")) { + return Err("npm declared bundled dependency is absent from its archive".into()); + } + } + for (path, _) in &files { + if !packages.contains_key(&source_owner(path)?) { + return Err("npm archive has unowned node_modules content".into()); + } + } + let nodes = packages + .iter() + .filter(|(prefix, _)| prefix.as_str() != "package/") + .map(|(prefix, metadata)| { + let path = prefix + .strip_prefix("package/") + .unwrap() + .trim_end_matches('/') + .to_owned(); + ( + path.clone(), + Instance { + path, + spec: metadata.spec.clone(), + source: Source { + archive: spec.clone(), + prefix: prefix.clone(), + }, + bundled: BTreeMap::new(), + edges: Vec::new(), + }, + ) + }) + .collect::<BTreeMap<_, _>>(); + let mut pending = root + .bundles + .iter() + .map(|name| child("", name)) + .collect::<VecDeque<_>>(); + let mut reached = BTreeSet::new(); + while let Some(path) = pending.pop_front() { + if !reached.insert(path.clone()) { + continue; + } + let node = nodes + .get(&path) + .ok_or("npm bundle references a missing package")?; + for edge in &packages[&node.source.prefix].edges { + let peer = matches!(edge.kind, Kind::Peer | Kind::OptionalPeer); + if let Some(found) = visible(&nodes, &path, &edge.name, peer) { + if !Version::from_str(&found.spec.version) + .map_err(|error| error.to_string())? + .satisfies( + &edge + .requirement + .parse::<Range>() + .map_err(|error| error.to_string())?, + ) + { + return Err("npm bundled dependency has an incompatible version".into()); + } + pending.push_back(found.path.clone()); + } else if edge.kind == Kind::Dependency + && edge.name != spec.name + && !root.edges.iter().any(|root| root.name == edge.name) + { + return Err("npm bundled dependency closure is incomplete".into()); + } + } + } + if reached.len() != nodes.len() { + return Err("npm archive contains undeclared or unreachable bundled packages".into()); + } + Ok(Self { files, packages }) + } +} + +fn source_owner(path: &str) -> Result<String, String> { + let Some((parent, tail)) = path.rsplit_once("/node_modules/") else { + return Ok("package/".into()); + }; + let mut parts = tail.split('/'); + let first = parts.next().ok_or("npm archive has invalid package path")?; + let name = if first.starts_with('@') { + format!( + "{first}/{}", + parts + .next() + .ok_or("npm scoped package path is incomplete")? + ) + } else { + first.into() + }; + let prefix = format!("{parent}/node_modules/{name}"); + validate_path( + prefix + .strip_prefix("package/") + .ok_or("npm bundled archive has invalid root")?, + )?; + Ok(format!("{prefix}/")) +} + +fn validate_source_owner(environment: &Environment, node: &Instance) -> Result<(), String> { + for owner in &environment.instances { + for (relative, spec) in &owner.bundled { + if node.path == format!("{}/{relative}", owner.path) + && (node.spec != *spec + || node.source.archive != owner.source.archive + || node.source.prefix != format!("package/{relative}/")) + { + return Err( + "npm bundled instance is not owned by its parent archive installation".into(), + ); + } + } + } + if node.source.prefix == "package/" { + for (relative, spec) in &node.bundled { + validate_path(relative)?; + if spec.ecosystem != Ecosystem::Npm + || !relative.ends_with(&format!("node_modules/{}", spec.name)) + { + return Err("invalid npm bundled layout identity".into()); + } + super::super::validate_name(&Ecosystem::Npm, &spec.name)?; + super::super::validate_version(&Ecosystem::Npm, &spec.version)?; + } + return if node.source.archive == node.spec { + Ok(()) + } else { + Err("npm root archive identity differs from its instance".into()) + }; + } + let relative = node + .source + .prefix + .strip_prefix("package/") + .and_then(|prefix| prefix.strip_suffix('/')) + .ok_or("invalid npm bundled source prefix")?; + validate_path(relative)?; + if !node.bundled.is_empty() + || !environment.instances.iter().any(|owner| { + owner.source.prefix == "package/" + && owner.source.archive == node.source.archive + && owner.spec == node.source.archive + && node.path == format!("{}/{relative}", owner.path) + && owner.bundled.get(relative) == Some(&node.spec) + }) + { + return Err("npm bundled instance is not owned by its parent archive installation".into()); + } + Ok(()) +} + +/// Validate immutable manifests and plan borrowed files before cloning any +/// installation output. Reused archives must not multiply past the tree budget. +pub fn files<'a>( + environment: &Environment, + lock: &LockFile, + archive: impl FnMut(&LockedLib) -> Result<&'a [u8], String>, +) -> Result<Vec<(String, Vec<u8>)>, String> { + source_files(environment, lock, archive, false) +} + +pub fn build_files<'a>( + environment: &Environment, + lock: &LockFile, + archive: impl FnMut(&LockedLib) -> Result<&'a [u8], String>, +) -> Result<Vec<(String, Vec<u8>)>, String> { + source_files(environment, lock, archive, true) +} + +pub(crate) fn build_executables( + environment: &Environment, + archives: &BTreeMap<LibSpec, Vec<u8>>, +) -> Result<BTreeSet<String>, String> { + let modes = archives + .iter() + .map(|(spec, bytes)| Ok((spec, super::archive_executables("tgz", bytes)?))) + .collect::<Result<BTreeMap<_, _>, String>>()?; + let mut installed = BTreeSet::new(); + for node in &environment.instances { + for path in modes + .get(&node.source.archive) + .ok_or("npm source modes missing")? + { + if source_owner(path)? == node.source.prefix { + let relative = path + .strip_prefix(&node.source.prefix) + .ok_or("npm source mode outside owner")?; + installed.insert(format!("{}/{relative}", node.path)); + } + } + } + Ok(installed) +} + +pub(crate) fn installed_owner(path: &str) -> Result<String, String> { + source_owner(&format!("package/{path}"))? + .strip_prefix("package/") + .map(|owner| owner.trim_end_matches('/').into()) + .ok_or("npm output is outside its installation".into()) +} + +fn source_files<'a>( + environment: &Environment, + lock: &LockFile, + mut archive: impl FnMut(&LockedLib) -> Result<&'a [u8], String>, + preparing: bool, +) -> Result<Vec<(String, Vec<u8>)>, String> { + let mut archives = BTreeMap::new(); + let mut archive_bytes = 0usize; + for node in &environment.instances { + validate_source_owner(environment, node)?; + if archives.contains_key(&node.source.archive) { + continue; + } + let lib = lock + .libs + .iter() + .find(|lib| lib.spec == node.source.archive) + .ok_or("npm graph source archive is missing")?; + let parsed = Archive::parse(&lib.spec, archive(lib)?)?; + for (_, bytes) in &parsed.files { + archive_bytes = archive_bytes + .checked_add(bytes.len()) + .ok_or("npm archive size overflow")?; + if archive_bytes > crate::workers::MAX_TREE_BYTES { + return Err("npm archive cache exceeds its tree byte budget".into()); + } + } + archives.insert(lib.spec.clone(), parsed); + } + let mut files = BTreeMap::new(); + let mut installed_bytes = 0usize; + for node in &environment.instances { + let source = &archives[&node.source.archive]; + if node.source.prefix == "package/" && node.bundled != source.bundled() { + return Err("npm bundled layout differs from its parent archive".into()); + } + let metadata = source + .packages + .get(&node.source.prefix) + .ok_or("npm instance source package is absent")?; + if metadata.spec != node.spec + || metadata.edges.len() != node.edges.len() + || metadata.edges.iter().any(|expected| { + !node.edges.iter().any(|edge| { + edge.name == expected.name + && edge.requirement == expected.requirement + && edge.kind == expected.kind + }) + }) + { + return Err("npm locked dependency graph differs from package.json".into()); + } + if !preparing + && (metadata.document.get("gypfile").and_then(Value::as_bool) == Some(true) + || metadata + .document + .get("scripts") + .and_then(Value::as_object) + .is_some_and(|scripts| { + ["preinstall", "install", "postinstall"] + .iter() + .any(|name| scripts.contains_key(*name)) + })) + { + return Err( + "managed npm Worker rejects lifecycle scripts and native build hooks".into(), + ); + } + for (path, bytes) in &source.files { + if source_owner(path)? != node.source.prefix { + continue; + } + let relative = path + .strip_prefix(&node.source.prefix) + .ok_or("npm file is outside its instance source")?; + if !preparing && (relative.ends_with(".node") || relative == "binding.gyp") { + return Err("managed npm Worker rejects native package content".into()); + } + let installed = format!("{}/{relative}", node.path); + installed_bytes = installed_bytes + .checked_add(bytes.len()) + .ok_or("npm installed size overflow")?; + if installed_bytes > crate::workers::MAX_TREE_BYTES + || files.len() >= crate::workers::MAX_TREE_FILES + { + return Err("npm installation exceeds its tree file/byte budget".into()); + } + if files.insert(installed, bytes).is_some() { + return Err("npm installed files have conflicting owners".into()); + } + } + } + Ok(files + .into_iter() + .map(|(path, bytes)| (path, bytes.clone())) + .collect()) +} + +pub fn normalize(environments: &mut Vec<Environment>) -> Result<(), String> { + for env in environments.iter_mut() { + env.roots.sort(); + env.roots.dedup(); + for node in &mut env.instances { + node.edges.sort(); + } + env.instances + .sort_by(|left, right| left.path.cmp(&right.path)); + } + environments.sort_by(|left, right| left.roots.cmp(&right.roots)); + for pair in environments.windows(2) { + if pair[0].roots == pair[1].roots && pair[0] != pair[1] { + return Err("npm roots have inconsistent installation environments".into()); + } + } + environments.dedup(); + Ok(()) +} + +pub fn requires_build(files: &[(String, Vec<u8>)]) -> Result<bool, String> { + for (path, bytes) in files { + if path.ends_with(".node") || path.ends_with("/binding.gyp") { + return Ok(true); + } + if path.ends_with("/package.json") { + let metadata: Value = serde_json::from_slice(bytes) + .map_err(|error| format!("invalid npm installation metadata: {error}"))?; + if metadata.get("gypfile").and_then(Value::as_bool) == Some(true) + || metadata + .get("scripts") + .and_then(Value::as_object) + .is_some_and(|scripts| { + ["preinstall", "install", "postinstall"] + .iter() + .any(|name| scripts.contains_key(*name)) + }) + { + return Ok(true); + } + } + } + Ok(false) +} + +/// Lifecycle output may add generated files, but cannot rewrite dependency +/// identities, create undeclared package instances, or escape the locked tree. +pub fn validate_built_files( + environment: &Environment, + files: &[(String, Vec<u8>)], +) -> Result<(), String> { + let nodes = environment + .instances + .iter() + .map(|node| (node.path.as_str(), node)) + .collect::<BTreeMap<_, _>>(); + let mut manifests = BTreeSet::new(); + let mut names = BTreeSet::new(); + let mut total = 0usize; + for (path, bytes) in files { + crate::workers::valid_path(path)?; + let owner = source_owner(&format!("package/{path}"))?; + let owner = owner + .strip_prefix("package/") + .ok_or("npm output is outside its installation")? + .trim_end_matches('/'); + let node = nodes + .get(owner) + .ok_or("npm build output introduces an unlocked instance")?; + if !names.insert(path) { + return Err("npm build output repeats a file".into()); + } + total = total + .checked_add(bytes.len()) + .ok_or("npm output byte overflow")?; + if total > crate::workers::MAX_TREE_BYTES || names.len() > crate::workers::MAX_TREE_FILES { + return Err("npm output exceeds installation budget".into()); + } + if path == &format!("{owner}/package.json") { + let metadata = + Metadata::parse(serde_json::from_slice(bytes).map_err(|error| error.to_string())?)?; + if metadata.spec != node.spec + || metadata.edges.len() != node.edges.len() + || metadata.edges.iter().any(|expected| { + !node.edges.iter().any(|edge| { + edge.name == expected.name + && edge.requirement == expected.requirement + && edge.kind == expected.kind + }) + }) + { + return Err("npm lifecycle changed locked package identity or dependencies".into()); + } + manifests.insert(owner.to_owned()); + } + } + if manifests.len() != nodes.len() { + return Err("npm lifecycle removed a locked package manifest".into()); + } + Ok(()) +} + +pub fn doctor(lock: &LockFile) -> Result<(), String> { + for env in &lock.npm { + if env.roots.is_empty() + || env.roots != roots(&env.roots) + || env.instances.len() > super::MAX_PACKAGES + { + return Err("invalid npm installation environment roots or size".into()); + } + let nodes = env + .instances + .iter() + .map(|node| (node.path.clone(), node.clone())) + .collect::<BTreeMap<_, _>>(); + if nodes.len() != env.instances.len() { + return Err("duplicate npm instance path".into()); + } + let mut pending = VecDeque::new(); + for root in &env.roots { + let path = child("", &root.name); + if nodes.get(&path).map(|node| &node.spec) != Some(root) { + return Err("npm root is not installed at its exact path".into()); + } + pending.push_back(path); + } + let mut reached = BTreeSet::new(); + while let Some(path) = pending.pop_front() { + if !reached.insert(path.clone()) { + continue; + } + let node = nodes + .get(&path) + .ok_or("npm edge references a missing instance")?; + validate_source_owner(env, node)?; + validate_path(&node.path)?; + super::super::validate_name(&Ecosystem::Npm, &node.spec.name)?; + super::super::validate_version(&Ecosystem::Npm, &node.spec.version)?; + if node.spec.ecosystem != Ecosystem::Npm + || !node + .path + .ends_with(&format!("node_modules/{}", node.spec.name)) + { + return Err("npm instance identity differs from its path".into()); + } + if !lock + .libs + .iter() + .any(|lib| lib.spec == node.source.archive && lib.spec.ecosystem == Ecosystem::Npm) + { + return Err("npm instance source archive is not locked".into()); + } + if node.source.prefix != "package/" { + validate_path( + node.source + .prefix + .strip_prefix("package/") + .and_then(|path| path.strip_suffix('/')) + .ok_or("invalid npm bundled source prefix")?, + )?; + } else if node.source.archive != node.spec { + return Err("npm root archive identity differs from its instance".into()); + } + let mut unique = BTreeSet::new(); + for edge in &node.edges { + if !unique.insert((&edge.name, &edge.kind)) { + return Err("duplicate npm dependency edge".into()); + } + let range = edge + .requirement + .parse::<Range>() + .map_err(|error| error.to_string())?; + let actual = visible( + &nodes, + &path, + &edge.name, + matches!(edge.kind, Kind::Peer | Kind::OptionalPeer), + ); + match (&edge.target, &edge.omission) { + (Some(target), None) + if actual.is_some_and(|found| { + found.path == *target + && Version::from_str(&found.spec.version) + .is_ok_and(|version| version.satisfies(&range)) + }) => + { + pending.push_back(target.clone()); + } + (None, Some(reason)) + if actual.is_none() + && (edge.kind == Kind::Optional + && *reason != Omission::OptionalPeerAbsent + || edge.kind == Kind::OptionalPeer + && matches!( + reason, + Omission::OptionalPeerAbsent | Omission::BuildFailure + )) + && (*reason != Omission::BuildFailure || env.build.is_some()) => {} + _ => { + return Err( + "npm edge differs from Node lookup or its required range".into() + ); + } + } + } + } + if reached.len() != nodes.len() { + return Err("npm environment contains unreachable package instances".into()); + } + } + for worker in &lock.workers { + if worker.ecosystem == "npm" { + environment(lock, &worker.libs)?; + } + } + Ok(()) +} + +use std::str::FromStr; diff --git a/crates/dever-cli/src/libs/registry/npm/resolve.rs b/crates/dever-cli/src/libs/registry/npm/resolve.rs new file mode 100644 index 0000000..c807a3f --- /dev/null +++ b/crates/dever-cli/src/libs/registry/npm/resolve.rs @@ -0,0 +1,436 @@ +use super::*; +use crate::libs::registry::{MAX_PACKAGES, MAX_RESOLUTION_ATTEMPTS, RegistryResolver, SearchError}; + +#[derive(Clone)] +struct Request { + consumer: Option<String>, + edge: Edge, +} + +#[derive(Clone, Default)] +struct State { + nodes: BTreeMap<String, Instance>, + pending: VecDeque<Request>, +} + +struct Catalog<'a, 'r> { + resolver: &'a RegistryResolver<'r>, + runtime: &'a RegistryRuntime, + indexes: BTreeMap<String, Vec<String>>, + metadata: BTreeMap<(String, String), Value>, + archives: BTreeMap<LibSpec, (LockedLib, Archive)>, + attempts: usize, + archive_bytes: usize, +} + +pub(crate) fn resolve( + resolver: &RegistryResolver<'_>, + requested: &[LibSpec], + runtime: &RegistryRuntime, +) -> Result<(Vec<LockedLib>, Environment), String> { + let roots = roots(requested); + let mut state = State::default(); + for spec in &roots { + state.pending.push_back(Request { + consumer: None, + edge: Edge { + name: spec.name.clone(), + requirement: format!("={}", spec.version), + kind: Kind::Dependency, + target: None, + omission: None, + }, + }); + } + let mut catalog = Catalog { + resolver, + runtime, + indexes: BTreeMap::new(), + metadata: BTreeMap::new(), + archives: BTreeMap::new(), + attempts: 0, + archive_bytes: 0, + }; + let mut state = catalog.search(state).map_err(SearchError::message)?; + let mut pending = roots + .iter() + .map(|root| child("", &root.name)) + .collect::<VecDeque<_>>(); + let mut reached = BTreeSet::new(); + while let Some(path) = pending.pop_front() { + if !reached.insert(path.clone()) { + continue; + } + pending.extend( + state.nodes[&path] + .edges + .iter() + .filter_map(|edge| edge.target.clone()), + ); + } + state.nodes.retain(|path, _| reached.contains(path)); + let used = state + .nodes + .values() + .map(|node| node.source.archive.clone()) + .collect::<BTreeSet<_>>(); + let libraries = used + .into_iter() + .map(|spec| { + catalog + .archives + .remove(&spec) + .expect("resolved npm archive") + .0 + }) + .collect(); + Ok(( + libraries, + Environment { + build: None, + roots, + instances: state.nodes.into_values().collect(), + }, + )) +} + +impl Catalog<'_, '_> { + fn archive(&mut self, spec: &LibSpec) -> Result<(), SearchError> { + if self.archives.contains_key(spec) { + return Ok(()); + } + let document = self + .resolver + .registry_metadata( + Ecosystem::Npm, + &spec.name, + &spec.version, + &mut self.metadata, + ) + .map_err(SearchError::Fatal)?; + let bytes = self + .resolver + .npm_archive(spec, &document) + .map_err(SearchError::Fatal)?; + let archive = Archive::parse(spec, &bytes).map_err(SearchError::Fatal)?; + self.archive_bytes = self + .archive_bytes + .checked_add( + archive + .files + .iter() + .map(|(_, bytes)| bytes.len()) + .sum::<usize>(), + ) + .ok_or_else(|| SearchError::Fatal("npm archive cache size overflow".into()))?; + if self.archive_bytes > 256 * 1024 * 1024 || self.archives.len() >= MAX_PACKAGES { + return Err(SearchError::Fatal( + "npm archive cache exceeds its 256 MiB/package budget".into(), + )); + } + let lib = self + .resolver + .lock_archive( + spec, + Vec::new(), + self.runtime, + "tgz", + &bytes, + "npm-instances-v1", + ) + .map_err(SearchError::Fatal)?; + self.archives.insert(spec.clone(), (lib, archive)); + Ok(()) + } + + fn metadata(&self, node: &Instance) -> &Metadata { + &self.archives[&node.source.archive].1.packages[&node.source.prefix] + } + + fn bind( + state: &mut State, + request: &Request, + target: Option<String>, + omission: Option<Omission>, + ) { + if let Some(consumer) = &request.consumer { + let edge = state + .nodes + .get_mut(consumer) + .expect("pending npm consumer") + .edges + .iter_mut() + .find(|edge| edge.name == request.edge.name && edge.kind == request.edge.kind) + .expect("pending npm edge"); + edge.target = target; + edge.omission = omission; + } + } + + fn add(&self, state: &mut State, path: &str, spec: &LibSpec) -> Result<(), SearchError> { + for (prefix, metadata) in &self.archives[spec].1.packages { + let relative = prefix + .strip_prefix("package/") + .unwrap() + .trim_end_matches('/'); + let location = if relative.is_empty() { + path.into() + } else { + format!("{path}/{relative}") + }; + let instance = Instance { + path: location.clone(), + spec: metadata.spec.clone(), + source: Source { + archive: spec.clone(), + prefix: prefix.clone(), + }, + bundled: if relative.is_empty() { + self.archives[spec].1.bundled() + } else { + BTreeMap::new() + }, + edges: metadata.edges.clone(), + }; + if state.nodes.insert(location.clone(), instance).is_some() { + return Err(SearchError::Conflict( + "npm package installation paths collide".into(), + )); + } + for edge in &metadata.edges { + if edge.kind != Kind::OptionalPeer { + state.pending.push_back(Request { + consumer: Some(location.clone()), + edge: edge.clone(), + }); + } + } + } + Ok(()) + } + + fn prune_bundled_orphans(&self, state: &mut State, root: &str) { + let mut pending = VecDeque::from([root.to_owned()]); + let mut reached = BTreeSet::new(); + while let Some(path) = pending.pop_front() { + if !reached.insert(path.clone()) { + continue; + } + let Some(node) = state.nodes.get(&path) else { + continue; + }; + for edge in &self.metadata(node).edges { + if let Some(found) = visible( + &state.nodes, + &path, + &edge.name, + matches!(edge.kind, Kind::Peer | Kind::OptionalPeer), + ) { + pending.push_back(found.path.clone()); + } + } + } + let prefix = format!("{root}/"); + state.nodes.retain(|path, node| { + node.source.prefix == "package/" || !path.starts_with(&prefix) || reached.contains(path) + }); + } + + fn search(&mut self, mut state: State) -> Result<State, SearchError> { + self.attempts += 1; + if self.attempts > MAX_RESOLUTION_ATTEMPTS + || state.nodes.len() > MAX_PACKAGES + || state.nodes.keys().any(|path| path.len() > 4096) + { + return Err(SearchError::Fatal( + "npm instance graph exceeds its resolution budget".into(), + )); + } + while let Some(request) = state.pending.pop_front() { + if request + .consumer + .as_ref() + .is_some_and(|consumer| !state.nodes.contains_key(consumer)) + { + continue; + } + let consumer = request.consumer.as_deref().unwrap_or(""); + let peer = request.edge.kind == Kind::Peer; + let scope = if peer { parent(consumer) } else { consumer }; + let location = child(scope, &request.edge.name); + let rule = request + .edge + .requirement + .parse::<Range>() + .map_err(|error| SearchError::Fatal(error.to_string()))?; + let existing = visible(&state.nodes, consumer, &request.edge.name, peer).cloned(); + if let Some(found) = &existing { + let eligible = self + .metadata(found) + .eligible(self.runtime) + .map_err(SearchError::Fatal)?; + if !eligible + && request.edge.kind == Kind::Optional + && found.source.prefix != "package/" + { + let prefix = format!("{}/", found.path); + state + .nodes + .retain(|path, _| path != &found.path && !path.starts_with(&prefix)); + let relative = found + .source + .prefix + .strip_prefix("package/") + .unwrap() + .trim_end_matches('/'); + let owner = found + .path + .strip_suffix(relative) + .unwrap() + .trim_end_matches('/'); + self.prune_bundled_orphans(&mut state, owner); + Self::bind(&mut state, &request, None, Some(Omission::Target)); + continue; + } + // Ordinary dependencies get a private nested slot. Reuse an + // ancestor only to close a real package cycle, never a sibling's + // independently resolved peer context. + let bundled = state.nodes.get(consumer).is_some_and(|node| { + node.source.prefix != "package/" && node.source.archive == found.source.archive + }); + let reusable = peer + || bundled + || found.path == location + || consumer == found.path + || consumer.starts_with(&format!("{}/", found.path)); + if reusable + && eligible + && Version::from_str(&found.spec.version) + .is_ok_and(|version| version.satisfies(&rule)) + { + Self::bind(&mut state, &request, Some(found.path.clone()), None); + continue; + } + if found.path == location { + return Err(SearchError::Conflict(format!( + "npm dependency/peer conflict for {} at {location}", + request.edge.name + ))); + } + } + if let Some(node) = request + .consumer + .as_ref() + .and_then(|consumer| state.nodes.get(consumer)) + && node.source.prefix != "package/" + && !peer + { + if request.edge.kind == Kind::Optional && existing.is_none() { + Self::bind(&mut state, &request, None, Some(Omission::Unavailable)); + continue; + } + return Err(SearchError::Fatal( + "npm bundled dependency closure is incomplete".into(), + )); + } + let versions = if let Some(versions) = self.indexes.get(&request.edge.name) { + versions.clone() + } else { + let versions = self + .resolver + .registry_versions(Ecosystem::Npm, &request.edge.name) + .map_err(SearchError::Fatal)?; + self.indexes + .insert(request.edge.name.clone(), versions.clone()); + versions + }; + let mut reason = Omission::Unavailable; + for version in versions.into_iter().filter(|version| { + Version::from_str(version).is_ok_and(|version| version.satisfies(&rule)) + }) { + self.attempts += 1; + if self.attempts > MAX_RESOLUTION_ATTEMPTS { + return Err(SearchError::Fatal( + "npm resolution exceeds its candidate budget".into(), + )); + } + let spec = LibSpec { + ecosystem: Ecosystem::Npm, + name: request.edge.name.clone(), + version, + }; + self.archive(&spec)?; + if !self.archives[&spec].1.packages["package/"] + .eligible(self.runtime) + .map_err(SearchError::Fatal)? + { + reason = Omission::Target; + continue; + } + reason = Omission::Conflict; + let mut candidate = state.clone(); + self.add(&mut candidate, &location, &spec)?; + Self::bind(&mut candidate, &request, Some(location.clone()), None); + match self.search(candidate) { + Ok(resolved) => return Ok(resolved), + Err(SearchError::Conflict(_)) => {} + Err(fatal) => return Err(fatal), + } + } + if request.edge.kind == Kind::Optional && existing.is_none() { + Self::bind(&mut state, &request, None, Some(reason)); + continue; + } + return Err(SearchError::Conflict(format!( + "no npm version satisfies {} at {location}", + request.edge.name + ))); + } + let edges = state + .nodes + .values() + .flat_map(|node| { + node.edges.iter().map(|edge| Request { + consumer: Some(node.path.clone()), + edge: edge.clone(), + }) + }) + .collect::<Vec<_>>(); + for request in edges { + let found = visible( + &state.nodes, + request.consumer.as_deref().unwrap(), + &request.edge.name, + matches!(request.edge.kind, Kind::Peer | Kind::OptionalPeer), + ); + if let Some(found) = found { + let range = request + .edge + .requirement + .parse::<Range>() + .map_err(|error| SearchError::Fatal(error.to_string()))?; + if !Version::from_str(&found.spec.version) + .is_ok_and(|version| version.satisfies(&range)) + { + return Err(SearchError::Conflict( + "npm dependency or peer conflicts with final Node lookup".into(), + )); + } + let target = found.path.clone(); + Self::bind(&mut state, &request, Some(target), None); + } else if request.edge.kind == Kind::OptionalPeer { + Self::bind( + &mut state, + &request, + None, + Some(Omission::OptionalPeerAbsent), + ); + } else if request.edge.kind != Kind::Optional || request.edge.omission.is_none() { + return Err(SearchError::Conflict( + "npm required dependency is absent from final Node lookup".into(), + )); + } + } + Ok(state) + } +} diff --git a/crates/dever-cli/src/libs/registry/sumdb.rs b/crates/dever-cli/src/libs/registry/sumdb.rs new file mode 100644 index 0000000..a5bd208 --- /dev/null +++ b/crates/dever-cli/src/libs/registry/sumdb.rs @@ -0,0 +1,524 @@ +//! Authenticated Go checksum database receipts. Trust comes from the verifier +//! supplied by the caller, never from a key stored in a project's lock file. +//! Formats follow golang.org/x/mod/sumdb/{note,tlog,dirhash} and RFC 6962. + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::{Mutex, MutexGuard}; + +use base64::{Engine, engine::general_purpose::STANDARD}; +use ring::signature::{ED25519, UnparsedPublicKey}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use super::{RegistryTransport, go_path, zip_entries}; + +mod merkle; + +pub const OFFICIAL_VERIFIER: &str = + "sum.golang.org+033de0ae+Ac4zctda0e5eza+HJyk9SxEdh+s3Ux18htTTAD8OuAn8"; +const MAX_NOTE_BYTES: usize = 16 * 1024; +const MAX_RECORDS: usize = 1024; +const MAX_EVIDENCE_BYTES: usize = 16 * 1024 * 1024; +type Hash = [u8; 32]; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct Tree { + size: u64, + hash: Hash, +} + +fn encode_hash(hash: Hash) -> String { + STANDARD.encode(hash) +} + +fn decode_hash(value: &str) -> Result<Hash, String> { + STANDARD + .decode(value) + .map_err(|_| "sumdb hash is not base64")? + .try_into() + .map_err(|_| "sumdb hash must contain 32 bytes".into()) +} + +/// Explicit verifier injection supports registry protocol tests and author +/// tooling. Released project commands always select `official()`. +#[derive(Clone)] +pub struct Verifier { + name: String, + key_hash: [u8; 4], + key: [u8; 32], +} + +impl Verifier { + pub fn official() -> Self { + Self::new(OFFICIAL_VERIFIER).expect("fixed official sumdb verifier") + } + + pub fn new(encoded: &str) -> Result<Self, String> { + // '+' also occurs inside base64; only the first two separators delimit. + let (name, rest) = encoded.split_once('+').ok_or("invalid sumdb verifier")?; + let (key_hash, encoded_key) = rest.split_once('+').ok_or("invalid sumdb verifier")?; + if name.is_empty() || name.chars().any(char::is_whitespace) || key_hash.len() != 8 { + return Err("invalid sumdb verifier identity".into()); + } + let expected = u32::from_str_radix(key_hash, 16) + .map_err(|_| "invalid sumdb key hash")? + .to_be_bytes(); + let key = STANDARD + .decode(encoded_key) + .map_err(|_| "invalid sumdb verifier encoding")?; + if key.len() != 33 || key[0] != 1 { + return Err("sumdb requires an Ed25519 verifier".into()); + } + let mut hash = Sha256::new(); + hash.update(name.as_bytes()); + hash.update(b"\n"); + hash.update(&key); + if hash.finalize()[..4] != expected { + return Err("sumdb verifier key hash mismatch".into()); + } + Ok(Self { + name: name.into(), + key_hash: expected, + key: key[1..].try_into().expect("validated Ed25519 key length"), + }) + } + + fn checkpoint(&self, note: &str) -> Result<Tree, String> { + if note.len() > MAX_NOTE_BYTES || note.bytes().any(|byte| byte < 32 && byte != b'\n') { + return Err("invalid sumdb checkpoint note".into()); + } + let (text, signatures) = note + .rsplit_once("\n\n") + .ok_or("sumdb checkpoint lacks signatures")?; + let message = format!("{text}\n"); + if !signatures.ends_with('\n') { + return Err("unterminated sumdb signature".into()); + } + let mut verified = false; + for (index, line) in signatures.lines().enumerate() { + if index >= 100 { + return Err("sumdb checkpoint has too many signatures".into()); + } + let (name, encoded) = line + .strip_prefix("— ") + .and_then(|line| line.split_once(' ')) + .ok_or("malformed sumdb signature")?; + let signature = STANDARD + .decode(encoded) + .map_err(|_| "invalid sumdb signature encoding")?; + if name.is_empty() || name.chars().any(char::is_whitespace) || signature.len() < 5 { + return Err("malformed sumdb signature identity".into()); + } + if name == self.name && signature[..4] == self.key_hash { + UnparsedPublicKey::new(&ED25519, self.key) + .verify(message.as_bytes(), &signature[4..]) + .map_err(|_| "sumdb checkpoint signature verification failed")?; + verified = true; + } + } + if !verified { + return Err("sumdb checkpoint has no trusted signature".into()); + } + let mut lines = text.lines(); + if lines.next() != Some("go.sum database tree") { + return Err("unsupported sumdb checkpoint format".into()); + } + let size_text = lines.next().ok_or("sumdb checkpoint lacks tree size")?; + let size = decimal(size_text)?; + let hash = decode_hash(lines.next().ok_or("sumdb checkpoint lacks root hash")?)?; + let empty: Hash = Sha256::digest([]).into(); + if size == 0 && hash != empty { + return Err("invalid sumdb empty root".into()); + } + Ok(Tree { size, hash }) + } +} + +fn decimal(value: &str) -> Result<u64, String> { + let number = value + .parse::<u64>() + .map_err(|_| "invalid sumdb record/tree number")?; + if number > i64::MAX as u64 || value != number.to_string() { + return Err("noncanonical sumdb record/tree number".into()); + } + Ok(number) +} + +struct Lookup<'a> { + id: u64, + text: &'a str, + checkpoint: &'a str, +} + +fn lookup(value: &str) -> Result<Lookup<'_>, String> { + if value.len() > MAX_NOTE_BYTES || value.bytes().any(|byte| byte < 32 && byte != b'\n') { + return Err("invalid sumdb lookup text".into()); + } + let (id, rest) = value + .split_once('\n') + .ok_or("sumdb lookup lacks record number")?; + let end = rest + .find("\n\n") + .ok_or("sumdb lookup lacks record terminator")?; + Ok(Lookup { + id: decimal(id)?, + text: &rest[..end + 1], + checkpoint: &rest[end + 2..], + }) +} + +fn hashes(record: &str, module: &str, version: &str) -> Result<(String, String), String> { + let version = format!("v{version}"); + let mod_version = format!("{version}/go.mod"); + let mut values = BTreeMap::new(); + for line in record.lines() { + let parts = line.split(' ').collect::<Vec<_>>(); + if parts.len() != 3 + || parts[0] != module + || (parts[1] != version && parts[1] != mod_version) + { + return Err("sumdb lookup record has a different module/version identity".into()); + } + decode_hash( + parts[2] + .strip_prefix("h1:") + .ok_or("sumdb record requires h1 hashes")?, + )?; + if values.insert(parts[1], parts[2]).is_some() { + return Err("sumdb lookup repeats a module checksum".into()); + } + } + Ok(( + values + .get(mod_version.as_str()) + .ok_or("sumdb lookup lacks go.mod hash")? + .to_string(), + values + .get(version.as_str()) + .ok_or("sumdb lookup lacks module ZIP hash")? + .to_string(), + )) +} + +fn dirhash(files: &[(String, Vec<u8>)]) -> Result<String, String> { + let mut files = files.iter().collect::<Vec<_>>(); + files.sort_by(|left, right| left.0.cmp(&right.0)); + let mut seen = BTreeSet::new(); + let mut hash = Sha256::new(); + for (name, bytes) in files { + if name.contains('\n') || !seen.insert(name) { + return Err("invalid or duplicate Go dirhash filename".into()); + } + hash.update(format!("{} {name}\n", crate::libs::sha256(bytes))); + } + Ok(format!("h1:{}", STANDARD.encode(hash.finalize()))) +} + +pub fn mod_hash(bytes: &[u8]) -> String { + // The Go protocol hashes a synthetic directory containing only go.mod. + dirhash(&[("go.mod".into(), bytes.to_vec())]).expect("fixed dirhash filename") +} + +pub fn zip_hash(module: &str, version: &str, bytes: &[u8]) -> Result<String, String> { + let files = zip_entries(bytes)?; + let prefix = format!("{module}@v{version}/"); + if files.iter().any(|(path, _)| !path.starts_with(&prefix)) { + return Err("Go module archive has a mismatched module root".into()); + } + dirhash(&files) +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Record { + pub module: String, + pub version: String, + pub go_mod: String, + pub lookup: String, + pub inclusion: Vec<String>, + pub consistency: Vec<String>, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Evidence { + pub previous: Option<String>, + pub checkpoint: String, + pub consistency: Vec<String>, + pub records: Vec<Record>, +} + +#[derive(Debug)] +pub struct Verified { + pub checkpoint: String, + checksums: BTreeMap<(String, String), String>, +} + +impl Verified { + pub fn verify_zip(&self, module: &str, version: &str, bytes: &[u8]) -> Result<(), String> { + let expected = self + .checksums + .get(&(module.into(), version.into())) + .ok_or("Go module lacks authenticated sumdb evidence")?; + if zip_hash(module, version, bytes)? != *expected { + return Err("Go module ZIP does not match authenticated sumdb h1".into()); + } + Ok(()) + } + + pub fn contains(&self, module: &str, version: &str) -> bool { + self.checksums + .contains_key(&(module.into(), version.into())) + } +} + +impl Evidence { + pub fn verify(&self, verifier: &Verifier) -> Result<Verified, String> { + if self.records.len() > MAX_RECORDS + || serde_json::to_vec(self).map_err(|e| e.to_string())?.len() > MAX_EVIDENCE_BYTES + { + return Err("sumdb evidence exceeds its size budget".into()); + } + let mut current = verifier.checkpoint(&self.checkpoint)?; + if let Some(previous) = &self.previous { + let older = verifier.checkpoint(previous)?; + if older.size > current.size { + return Err("sumdb checkpoint rollback rejected".into()); + } + merkle::check_consistency(&self.consistency, older, current)?; + } else if !self.consistency.is_empty() { + return Err("sumdb first checkpoint has an unexpected consistency proof".into()); + } + let mut latest = self.checkpoint.clone(); + let mut checksums = BTreeMap::new(); + for record in &self.records { + let entry = lookup(&record.lookup)?; + let tree = verifier.checkpoint(entry.checkpoint)?; + merkle::check_record(&record.inclusion, tree, entry.id, entry.text.as_bytes())?; + let (older, newer) = if tree.size < current.size { + (tree, current) + } else { + (current, tree) + }; + merkle::check_consistency(&record.consistency, older, newer)?; + let (mod_h1, zip_h1) = hashes(entry.text, &record.module, &record.version)?; + if mod_hash(record.go_mod.as_bytes()) != mod_h1 { + return Err("Go go.mod does not match authenticated sumdb h1".into()); + } + if checksums + .insert((record.module.clone(), record.version.clone()), zip_h1) + .is_some() + { + return Err("sumdb evidence repeats a module/version".into()); + } + if tree.size > current.size { + current = tree; + latest = entry.checkpoint.into(); + } + } + Ok(Verified { + checkpoint: latest, + checksums, + }) + } +} + +/// One explicit resolution sequence owns the checkpoint. The project lock is +/// its persistent anchor; deleting that lock intentionally starts first trust. +pub struct ChecksumDatabase { + verifier: Verifier, + latest: Mutex<Option<String>>, +} + +impl ChecksumDatabase { + pub fn verifier(&self) -> &Verifier { + &self.verifier + } + + pub fn new(verifier: Verifier, previous: Option<String>) -> Result<Self, String> { + if let Some(note) = &previous { + verifier.checkpoint(note)?; + } + Ok(Self { + verifier, + latest: Mutex::new(previous), + }) + } + + pub fn official(previous: Option<String>) -> Result<Self, String> { + Self::new(Verifier::official(), previous) + } + + pub(super) fn begin<'a>( + &'a self, + transport: &'a dyn RegistryTransport, + ) -> Result<Session<'a>, String> { + let guard = self + .latest + .lock() + .map_err(|_| "sumdb checkpoint state poisoned")?; + let checkpoint = String::from_utf8(transport.get_sumdb("/latest", MAX_NOTE_BYTES)?) + .map_err(|_| "sumdb checkpoint is not UTF-8")?; + let current = self.verifier.checkpoint(&checkpoint)?; + let mut tiles = merkle::Tiles::new(transport); + let consistency = if let Some(previous) = &*guard { + let older = self.verifier.checkpoint(previous)?; + if older.size > current.size { + return Err("sumdb checkpoint rollback rejected".into()); + } + tiles.consistency(older, current)? + } else { + Vec::new() + }; + let evidence = Evidence { + previous: guard.clone(), + checkpoint: checkpoint.clone(), + consistency, + records: Vec::new(), + }; + Ok(Session { + verifier: &self.verifier, + transport, + guard, + tiles, + current, + latest: checkpoint, + evidence, + }) + } +} + +pub(super) struct Session<'a> { + verifier: &'a Verifier, + transport: &'a dyn RegistryTransport, + guard: MutexGuard<'a, Option<String>>, + tiles: merkle::Tiles<'a>, + current: Tree, + latest: String, + evidence: Evidence, +} + +impl Session<'_> { + pub(super) fn verify_mod( + &mut self, + module: &str, + version: &str, + source: &[u8], + ) -> Result<(), String> { + if let Some(record) = self + .evidence + .records + .iter() + .find(|record| record.module == module && record.version == version) + { + if record.go_mod.as_bytes() != source { + return Err("Go go.mod changed during resolution".into()); + } + return Ok(()); + } + if self.evidence.records.len() >= MAX_RECORDS { + return Err("sumdb resolution exceeds record limit".into()); + } + let path = format!("/lookup/{}@v{}", go_path(module), go_path(version)); + let response = String::from_utf8(self.transport.get_sumdb(&path, MAX_NOTE_BYTES)?) + .map_err(|_| "sumdb lookup is not UTF-8")?; + let entry = lookup(&response)?; + let tree = self.verifier.checkpoint(entry.checkpoint)?; + let inclusion = self.tiles.inclusion(tree.size, entry.id)?; + merkle::check_record(&inclusion, tree, entry.id, entry.text.as_bytes())?; + let (older, newer) = if tree.size < self.current.size { + (tree, self.current) + } else { + (self.current, tree) + }; + let consistency = self.tiles.consistency(older, newer)?; + let (expected, _) = hashes(entry.text, module, version)?; + if mod_hash(source) != expected { + return Err("Go go.mod does not match authenticated sumdb h1".into()); + } + if tree.size > self.current.size { + self.current = tree; + self.latest = entry.checkpoint.into(); + } + let record = Record { + module: module.into(), + version: version.into(), + go_mod: String::from_utf8(source.to_vec()).map_err(|_| "Go go.mod is not UTF-8")?, + lookup: response, + inclusion, + consistency, + }; + self.evidence.records.push(record); + if serde_json::to_vec(&self.evidence) + .map_err(|e| e.to_string())? + .len() + > MAX_EVIDENCE_BYTES + { + return Err("sumdb evidence exceeds its size budget".into()); + } + Ok(()) + } + + pub(super) fn verify_zip( + &self, + module: &str, + version: &str, + bytes: &[u8], + ) -> Result<(), String> { + let record = self + .evidence + .records + .iter() + .find(|record| record.module == module && record.version == version) + .ok_or("Go module ZIP has no authenticated metadata")?; + let (_, expected) = hashes(lookup(&record.lookup)?.text, module, version)?; + if zip_hash(module, version, bytes)? != expected { + return Err("Go module ZIP does not match authenticated sumdb h1".into()); + } + Ok(()) + } + + pub(super) fn finish(mut self) -> Evidence { + *self.guard = Some(self.latest); + self.evidence + } +} + +pub fn verify_chain(evidence: &[Evidence], verifier: &Verifier) -> Result<Verified, String> { + let mut latest = None; + let mut checksums = BTreeMap::new(); + if evidence.len() > MAX_RECORDS { + return Err("sumdb resolution history exceeds its limit".into()); + } + let mut bytes = 0usize; + for receipt in evidence { + bytes = bytes + .checked_add( + serde_json::to_vec(receipt) + .map_err(|error| error.to_string())? + .len(), + ) + .ok_or("sumdb resolution history size overflow")?; + if bytes > MAX_EVIDENCE_BYTES { + return Err("sumdb resolution history exceeds its size budget".into()); + } + if let Some(previous) = &latest + && receipt.previous.as_ref() != Some(previous) + { + return Err("sumdb resolution checkpoints are not connected".into()); + } + let verified = receipt.verify(verifier)?; + for (identity, hash) in verified.checksums { + if let Some(old) = checksums.insert(identity, hash.clone()) + && old != hash + { + return Err("sumdb resolutions disagree about module content".into()); + } + } + latest = Some(verified.checkpoint); + } + Ok(Verified { + checkpoint: latest.unwrap_or_default(), + checksums, + }) +} diff --git a/crates/dever-cli/src/libs/registry/sumdb/merkle.rs b/crates/dever-cli/src/libs/registry/sumdb/merkle.rs new file mode 100644 index 0000000..94a3dcf --- /dev/null +++ b/crates/dever-cli/src/libs/registry/sumdb/merkle.rs @@ -0,0 +1,260 @@ +//! RFC 6962 tree proofs and Go's height-eight tile addressing. +//! Protocol reference: golang.org/x/mod/sumdb/tlog (BSD-3-Clause). + +use std::collections::BTreeMap; + +use sha2::{Digest, Sha256}; + +use super::{Hash, Tree, decode_hash, encode_hash}; +use crate::libs::RegistryTransport; + +pub(super) fn leaf(bytes: &[u8]) -> Hash { + let mut hash = Sha256::new(); + hash.update([0]); + hash.update(bytes); + hash.finalize().into() +} + +fn node(left: Hash, right: Hash) -> Hash { + let mut hash = Sha256::new(); + hash.update([1]); + hash.update(left); + hash.update(right); + hash.finalize().into() +} + +fn split(size: u64) -> u64 { + 1 << (63 - (size - 1).leading_zeros()) +} + +pub(super) fn check_record( + proof: &[String], + tree: Tree, + id: u64, + bytes: &[u8], +) -> Result<(), String> { + if id >= tree.size || proof.len() > 63 { + return Err("sumdb record proof has invalid bounds".into()); + } + fn root(proof: &[String], size: u64, id: u64, value: Hash) -> Result<Hash, String> { + if size == 1 { + return if proof.is_empty() { + Ok(value) + } else { + Err("sumdb inclusion proof has surplus hashes".into()) + }; + } + let (last, rest) = proof + .split_last() + .ok_or("sumdb inclusion proof is incomplete")?; + let sibling = decode_hash(last)?; + let left = split(size); + if id < left { + Ok(node(root(rest, left, id, value)?, sibling)) + } else { + Ok(node(sibling, root(rest, size - left, id - left, value)?)) + } + } + if root(proof, tree.size, id, leaf(bytes))? != tree.hash { + return Err("sumdb inclusion proof does not match signed checkpoint".into()); + } + Ok(()) +} + +pub(super) fn check_consistency(proof: &[String], older: Tree, newer: Tree) -> Result<(), String> { + if older.size > newer.size || proof.len() > 63 { + return Err("sumdb consistency proof has invalid bounds".into()); + } + if older.size == 0 { + let empty: Hash = Sha256::digest([]).into(); + return if proof.is_empty() && older.hash == empty { + Ok(()) + } else { + Err("sumdb empty checkpoint is invalid".into()) + }; + } + fn roots( + proof: &[String], + lo: u64, + size: u64, + old_size: u64, + old_hash: Hash, + ) -> Result<(Hash, Hash), String> { + if old_size == size { + return if lo == 0 && proof.is_empty() { + Ok((old_hash, old_hash)) + } else if lo != 0 && proof.len() == 1 { + let hash = decode_hash(&proof[0])?; + Ok((hash, hash)) + } else { + Err("sumdb consistency proof has surplus or missing hashes".into()) + }; + } + let (last, rest) = proof + .split_last() + .ok_or("sumdb consistency proof is incomplete")?; + let sibling = decode_hash(last)?; + let left = split(size); + if old_size <= left { + let (old, new) = roots(rest, lo, left, old_size, old_hash)?; + Ok((old, node(new, sibling))) + } else { + let (old, new) = roots(rest, lo + left, size - left, old_size - left, old_hash)?; + Ok((node(sibling, old), node(sibling, new))) + } + } + let (old, new) = roots(proof, 0, newer.size, older.size, older.hash)?; + if old != older.hash || new != newer.hash { + return Err("sumdb consistency proof detects a conflicting history".into()); + } + Ok(()) +} + +/// Fetched tiles are only inputs to proofs; no byte is trusted merely because +/// it is cached. Persisted Evidence contains only completely verified proofs. +pub(super) struct Tiles<'a> { + transport: &'a dyn RegistryTransport, + cache: BTreeMap<String, Vec<u8>>, +} + +impl<'a> Tiles<'a> { + pub(super) fn new(transport: &'a dyn RegistryTransport) -> Self { + Self { + transport, + cache: BTreeMap::new(), + } + } + + fn subtree(&mut self, lo: u64, size: u64, tree_size: u64) -> Result<Hash, String> { + if size.is_power_of_two() { + let level = size.trailing_zeros(); + let tile_level = level / 8; + let tile_scale = tile_level * 8; + let count = 1u64 << (level % 8); + let first = lo >> tile_scale; + let index = first / 256; + let offset = first % 256; + let width = (tree_size >> tile_scale) + .checked_sub(index * 256) + .ok_or("sumdb tile lies outside checkpoint")? + .min(256); + if offset + count > width { + return Err("sumdb tile does not cover requested subtree".into()); + } + let mut chunks = vec![format!("{:03}", index % 1000)]; + let mut prefix = index / 1000; + while prefix > 0 { + chunks.push(format!("x{:03}", prefix % 1000)); + prefix /= 1000; + } + chunks.reverse(); + let full = format!("/tile/8/{tile_level}/{}", chunks.join("/")); + let path = if width == 256 { + full.clone() + } else { + format!("{full}.p/{width}") + }; + if !self.cache.contains_key(&path) { + if self.cache.len() >= 4096 { + return Err("sumdb tile cache exceeds its 32 MiB budget".into()); + } + let bytes = match self.transport.get_sumdb(&path, 8192) { + Ok(bytes) => bytes, + Err(_) if width < 256 => { + let mut bytes = self.transport.get_sumdb(&full, 8192)?; + if bytes.len() != 8192 { + return Err("sumdb full tile has wrong length".into()); + } + bytes.truncate(width as usize * 32); + bytes + } + Err(error) => return Err(error), + }; + if bytes.len() != width as usize * 32 { + return Err("sumdb tile has wrong length".into()); + } + self.cache.insert(path.clone(), bytes); + } + fn fold(bytes: &[u8]) -> Hash { + if bytes.len() == 32 { + return bytes.try_into().expect("one tile hash"); + } + let (left, right) = bytes.split_at(bytes.len() / 2); + node(fold(left), fold(right)) + } + return Ok(fold( + &self.cache[&path][offset as usize * 32..(offset + count) as usize * 32], + )); + } + let left = split(size); + Ok(node( + self.subtree(lo, left, tree_size)?, + self.subtree(lo + left, size - left, tree_size)?, + )) + } + + pub(super) fn inclusion(&mut self, size: u64, id: u64) -> Result<Vec<String>, String> { + fn walk( + tiles: &mut Tiles<'_>, + lo: u64, + size: u64, + id: u64, + total: u64, + proof: &mut Vec<String>, + ) -> Result<(), String> { + if size == 1 { + return Ok(()); + } + let left = split(size); + let sibling = if id < lo + left { + walk(tiles, lo, left, id, total, proof)?; + tiles.subtree(lo + left, size - left, total)? + } else { + walk(tiles, lo + left, size - left, id, total, proof)?; + tiles.subtree(lo, left, total)? + }; + proof.push(encode_hash(sibling)); + Ok(()) + } + if id >= size { + return Err("sumdb record lies outside signed checkpoint".into()); + } + let mut proof = Vec::new(); + walk(self, 0, size, id, size, &mut proof)?; + Ok(proof) + } + + pub(super) fn consistency(&mut self, older: Tree, newer: Tree) -> Result<Vec<String>, String> { + fn walk( + tiles: &mut Tiles<'_>, + lo: u64, + size: u64, + old_size: u64, + total: u64, + proof: &mut Vec<String>, + ) -> Result<(), String> { + if old_size == size { + if lo != 0 { + proof.push(encode_hash(tiles.subtree(lo, size, total)?)); + } + return Ok(()); + } + let left = split(size); + let sibling = if old_size <= left { + walk(tiles, lo, left, old_size, total, proof)?; + tiles.subtree(lo + left, size - left, total)? + } else { + walk(tiles, lo + left, size - left, old_size - left, total, proof)?; + tiles.subtree(lo, left, total)? + }; + proof.push(encode_hash(sibling)); + Ok(()) + } + let mut proof = Vec::new(); + if older.size != 0 { + walk(self, 0, newer.size, older.size, newer.size, &mut proof)?; + } + check_consistency(&proof, older, newer)?; + Ok(proof) + } +} diff --git a/crates/dever-cli/src/main.rs b/crates/dever-cli/src/main.rs new file mode 100644 index 0000000..7877992 --- /dev/null +++ b/crates/dever-cli/src/main.rs @@ -0,0 +1,754 @@ +use std::env; +use std::ffi::OsString; +use std::io::{self, Write}; +use std::path::PathBuf; +use std::process::{Command, ExitCode}; + +use dever_cli::libs; +use dever_cli::toolchain::BuildTarget; +use dever_core::source::SourceMap; + +mod api; +mod compile; +mod format; +mod test_runner; +mod worker; + +#[cfg(test)] +#[path = "../../../test/dever-cli-tests/build_arguments.rs"] +mod build_arguments; + +const USAGE: &str = "Usage:\n dever check <project-root>\n dever api <project-root> [--output <new-file>]\n dever fmt <project-root> [--check]\n dever test <project-root>\n dever run <project-root> [-- <component>.<domain>.<cmd> '<json-object>']\n dever run <project-root> --tenant <positive-tenant-id> -- <component>.<domain>.<cmd> '<json-object>'\n dever build <project-root> --output <new-file> [--target linux-x86_64|linux-aarch64]\n dever clean <project-root>\n dever tenant migrate <project-root> <positive-tenant-id>\n dever tenant owner <project-root> <positive-tenant-id> <site> <positive-user-id>\n dever tenant component enable <project-root> <positive-tenant-id> <component>\n dever tenant component disable <project-root> <positive-tenant-id> <component>\n dever lib add|list|update|remove|doctor <project-root> [--target linux-x86_64|linux-aarch64] [spec ...]\n dever package add|list|update|remove|doctor <project-root> [spec ...]\n\nApplications use API, Job, and CMD declarations under module/ and config/setting.json. Native builds and tests are offline."; + +enum Action { + Check, + Api { + output: Option<PathBuf>, + }, + Format { + check: bool, + }, + Test, + Run { + tenant_id: Option<i64>, + arguments: Vec<OsString>, + }, + Build { + output: PathBuf, + target: BuildTarget, + }, + Clean, + TenantMigrate { + tenant_id: i64, + }, + TenantOwner { + tenant_id: i64, + site: String, + user_id: i64, + }, + TenantComponent { + tenant_id: i64, + component: String, + enabled: bool, + }, + Lib { + command: String, + specs: Vec<String>, + target: BuildTarget, + }, + Package { + command: String, + specs: Vec<String>, + }, +} + +impl Action { + fn loads_settings(&self) -> bool { + matches!(self, Self::Check | Self::Api { .. }) || self.compiles_application() + } + + fn compiles_application(&self) -> bool { + matches!( + self, + Self::Run { .. } + | Self::Build { .. } + | Self::TenantMigrate { .. } + | Self::TenantOwner { .. } + | Self::TenantComponent { .. } + ) + } +} + +fn main() -> ExitCode { + match execute() { + Ok(code) => code, + Err(error) => { + // Preserve failure status even if stderr is unavailable. + let _ = writeln!(io::stderr().lock(), "{error}"); + ExitCode::FAILURE + } + } +} + +fn execute() -> Result<ExitCode, String> { + let arguments: Vec<_> = env::args_os().skip(1).collect(); + if arguments.first().is_some_and(|argument| argument == "new") { + let (destination, markdown) = match &arguments[1..] { + [destination] if !destination.to_string_lossy().starts_with('-') => { + (destination, false) + } + [destination, option] + if !destination.to_string_lossy().starts_with('-') && option == "--markdown" => + { + (destination, true) + } + _ => return Err("Usage: dever new <new-directory> [--markdown]".into()), + }; + dever_cli::project::create(std::path::Path::new(destination), markdown)?; + writeln!( + io::stdout().lock(), + "created Dever project {}", + destination.to_string_lossy() + ) + .map_err(|error| error.to_string())?; + return Ok(ExitCode::SUCCESS); + } + if arguments.as_slice() == [OsString::from("--dever-compile-worker")] { + worker::execute()?; + return Ok(ExitCode::SUCCESS); + } + if arguments.as_slice() == [OsString::from("--dever-health")] { + return Ok(ExitCode::SUCCESS); + } + if arguments.as_slice() == [OsString::from("--version")] { + writeln!(io::stdout().lock(), "{}", env!("CARGO_PKG_VERSION")) + .map_err(|error| error.to_string())?; + return Ok(ExitCode::SUCCESS); + } + if arguments.as_slice() == [OsString::from("--help")] { + writeln!( + io::stdout().lock(), + "Create a project: dever new <new-directory> [--markdown]" + ) + .map_err(|error| error.to_string())?; + writeln!(io::stdout().lock(), "{USAGE}").map_err(|error| error.to_string())?; + return Ok(ExitCode::SUCCESS); + } + let (root, action) = arguments_for(&arguments)?; + if let Action::Lib { + command, + specs, + target, + } = &action + { + let message = dever_cli::libs::execute_for_target(command, &root, specs, *target)?; + if !message.is_empty() { + writeln!(io::stdout().lock(), "{message}").map_err(|error| error.to_string())?; + } + return Ok(ExitCode::SUCCESS); + } + if let Action::Package { command, specs } = &action { + let message = dever_cli::packages::execute(command, &root, specs)?; + if !message.is_empty() { + writeln!(io::stdout().lock(), "{message}").map_err(|error| error.to_string())?; + } + return Ok(ExitCode::SUCCESS); + } + if matches!(action, Action::Clean) { + let summary = dever_core::native::clean_project_artifacts(&root) + .map_err(|error| format!("cannot clean Dever artifacts: {error}"))?; + writeln!( + io::stdout().lock(), + "cleaned {} directories, {} files, {} bytes", + summary.directories, + summary.files, + summary.bytes, + ) + .map_err(|error| error.to_string())?; + return Ok(ExitCode::SUCCESS); + } + let source_root = root.join("module"); + let packages = if matches!(action, Action::Format { .. }) { + Vec::new() + } else { + dever_cli::packages::sources(&root)? + }; + let sources = SourceMap::load_with_packages( + &source_root, + matches!(action, Action::Test | Action::Format { .. }) + .then(|| root.join("test")) + .as_deref(), + &packages, + ) + .map_err(|errors| { + errors + .iter() + .map(ToString::to_string) + .collect::<Vec<_>>() + .join("\n") + })?; + if !sources.files().iter().any(|source| !source.is_test()) { + return Err(format!( + "{}: no .dever or .dever.md source files", + source_root.display() + )); + } + if action.compiles_application() + && sources.files().iter().any(|source| { + source + .path() + .file_name() + .is_some_and(|name| name == "main.dever" || name == "main.dever.md") + }) + { + return Err("application entry is generated from API, CMD, and Job declarations; remove module/main.dever".into()); + } + if let Action::Format { check } = action { + return format::execute(&root, &sources, check); + } + let settings = action + .loads_settings() + .then(|| dever_runtime::config::Settings::load_project(&root)) + .transpose()?; + let checked = match settings.as_ref() { + Some(settings) => dever_core::check_with_settings(&sources, settings), + None => dever_core::check(&sources), + }; + let program = checked.map_err(|errors| { + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + })?; + for warning in program.warnings() { + if SourceMap::is_standard(warning.primary.source) { + continue; + } + write!(io::stderr().lock(), "{}", warning.render(&sources)) + .map_err(|error| error.to_string())?; + } + if let Action::Api { output } = action { + return api::write(&program, output.as_deref()); + } + api::check_baseline(&program, &root)?; + if matches!(action, Action::Check) { + return Ok(ExitCode::SUCCESS); + } + if matches!(action, Action::Test) { + return test_runner::execute(&program, &sources); + } + let target = match &action { + Action::Build { target, .. } => *target, + _ => BuildTarget::host()?, + }; + let external_report = if action.compiles_application() { + libs::prepare_program(&root, &program, target)? + } else { + None + }; + if action.compiles_application() { + let errors = program.application_errors()?; + if !errors.is_empty() { + return Err(errors.iter().map(|error| error.render(&sources)).collect()); + } + } + match &action { + Action::Check => unreachable!("check returned before target selection"), + Action::Lib { .. } => unreachable!("lib returned before source checking"), + Action::Package { .. } => unreachable!("package returned before source checking"), + Action::Api { .. } => unreachable!("api returned before baseline checking"), + Action::Format { .. } => unreachable!("fmt returned before checking"), + Action::Test => unreachable!("test returned before application compilation"), + Action::Run { .. } + | Action::Build { .. } + | Action::TenantMigrate { .. } + | Action::TenantOwner { .. } + | Action::TenantComponent { .. } => {} + Action::Clean => unreachable!("clean returned before source loading"), + } + let settings = settings + .as_ref() + .expect("application action loaded settings before checking"); + program.validate_database_settings(settings)?; + let resources = libs::prepare_resources(&root, &program, target)?; + let native = compile::application( + &program, + &sources, + &settings.compilation_bindings(), + &resources, + target, + )?; + match action { + Action::Package { .. } => unreachable!("package returned before source checking"), + Action::Build { output, .. } => { + native.save(&output).map_err(|error| { + format!("cannot save executable '{}': {error}", output.display()) + })?; + if let Some(report) = external_report { + eprintln!("{}", report.summary()); + libs::write_report(&output, &report)?; + } + Ok(ExitCode::SUCCESS) + } + Action::Run { + tenant_id, + arguments, + } => { + let arguments = match tenant_id { + Some(tenant_id) => [ + vec![ + OsString::from("--dever-tenant-cmd"), + OsString::from(tenant_id.to_string()), + ], + arguments, + ] + .concat(), + None => arguments, + }; + run_native(&native, &root, &arguments) + } + Action::TenantMigrate { tenant_id } => run_native( + &native, + &root, + &[ + OsString::from("--dever-tenant-migrate"), + OsString::from(tenant_id.to_string()), + ], + ), + Action::TenantOwner { + tenant_id, + site, + user_id, + } => run_native( + &native, + &root, + &[ + OsString::from("--dever-tenant-owner"), + OsString::from(tenant_id.to_string()), + OsString::from(site), + OsString::from(user_id.to_string()), + ], + ), + Action::TenantComponent { + tenant_id, + component, + enabled, + } => run_native( + &native, + &root, + &[ + OsString::from("--dever-tenant-component"), + OsString::from(tenant_id.to_string()), + OsString::from(if enabled { "enable" } else { "disable" }), + OsString::from(component), + ], + ), + Action::Check + | Action::Api { .. } + | Action::Format { .. } + | Action::Test + | Action::Lib { .. } + | Action::Clean => { + unreachable!("check and fmt returned before native compilation") + } + } +} + +fn run_native( + native: &dever_core::native::NativeProgram, + project_root: &PathBuf, + arguments: &[OsString], +) -> Result<ExitCode, String> { + stage_runtime_config(native.executable(), project_root)?; + let status = Command::new(native.executable()) + .current_dir(project_root) + .args(arguments) + .status() + .map_err(|error| format!("cannot run native program: {error}"))?; + Ok(if status.success() { + ExitCode::SUCCESS + } else { + ExitCode::FAILURE + }) +} + +fn stage_runtime_config( + executable: &std::path::Path, + project_root: &std::path::Path, +) -> Result<(), String> { + let source = project_root.join("config"); + let destination = executable + .parent() + .ok_or("native executable has no parent directory")? + .join("config"); + copy_config_directory(&source, &destination) +} + +fn copy_config_directory( + source: &std::path::Path, + destination: &std::path::Path, +) -> Result<(), String> { + let metadata = std::fs::symlink_metadata(source) + .map_err(|error| format!("cannot inspect '{}': {error}", source.display()))?; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Err(format!( + "project config '{}' must be a real directory", + source.display() + )); + } + std::fs::create_dir(destination) + .map_err(|error| format!("cannot stage config '{}': {error}", destination.display()))?; + for entry in std::fs::read_dir(source) + .map_err(|error| format!("cannot read '{}': {error}", source.display()))? + { + let entry = + entry.map_err(|error| format!("cannot read '{}': {error}", source.display()))?; + let source_path = entry.path(); + let destination_path = destination.join(entry.file_name()); + let metadata = std::fs::symlink_metadata(&source_path) + .map_err(|error| format!("cannot inspect '{}': {error}", source_path.display()))?; + if metadata.file_type().is_symlink() { + return Err(format!( + "project config '{}' must not contain symbolic links", + source_path.display() + )); + } + if metadata.is_dir() { + copy_config_directory(&source_path, &destination_path)?; + } else if metadata.is_file() { + std::fs::copy(&source_path, &destination_path).map_err(|error| { + format!("cannot stage config '{}': {error}", source_path.display()) + })?; + } else { + return Err(format!( + "project config '{}' must contain only files and directories", + source_path.display() + )); + } + } + Ok(()) +} + +fn arguments_for(arguments: &[OsString]) -> Result<(PathBuf, Action), String> { + match arguments { + [command, operation, root, rest @ ..] + if command == "lib" + && matches!( + operation.to_str(), + Some("add" | "list" | "update" | "remove" | "doctor") + ) => + { + let command = operation.to_str().unwrap().to_owned(); + let (specs, target) = lib_arguments(rest)?; + Ok(( + root.into(), + Action::Lib { + command, + specs, + target, + }, + )) + } + [command, operation, root, rest @ ..] + if command == "package" + && matches!( + operation.to_str(), + Some("add" | "list" | "update" | "remove" | "doctor") + ) => + { + let command = operation.to_str().unwrap().to_owned(); + let specs = rest + .iter() + .map(|value| { + value + .to_str() + .map(str::to_owned) + .ok_or("Package spec must be UTF-8".to_owned()) + }) + .collect::<Result<Vec<_>, _>>()?; + Ok((root.into(), Action::Package { command, specs })) + } + [command, root] if command == "check" => Ok((root.into(), Action::Check)), + [command, root] if command == "api" => Ok((root.into(), Action::Api { output: None })), + [command, root, flag, output] if command == "api" && flag == "--output" => Ok(( + root.into(), + Action::Api { + output: Some(output.into()), + }, + )), + [command, root] if command == "fmt" => Ok((root.into(), Action::Format { check: false })), + [command, root, flag] if command == "fmt" && flag == "--check" => { + Ok((root.into(), Action::Format { check: true })) + } + [command, root] if command == "run" => Ok(( + root.into(), + Action::Run { + tenant_id: None, + arguments: Vec::new(), + }, + )), + [command, root, separator, name, input] if command == "run" && separator == "--" => Ok(( + root.into(), + Action::Run { + tenant_id: None, + arguments: vec![name.clone(), input.clone()], + }, + )), + [command, root, tenant, tenant_id, separator, name, input] + if command == "run" && tenant == "--tenant" && separator == "--" => + { + let tenant_id = positive_tenant_id(tenant_id)?; + Ok(( + root.into(), + Action::Run { + tenant_id: Some(tenant_id), + arguments: vec![name.clone(), input.clone()], + }, + )) + } + [command, root] if command == "test" => Ok((root.into(), Action::Test)), + [command, root] if command == "clean" => Ok((root.into(), Action::Clean)), + [command, root, rest @ ..] if command == "build" => { + let mut output = None; + let mut target = None; + let mut options = rest.iter(); + while let Some(option) = options.next() { + match option.to_str() { + Some("--output") if output.is_none() => { + output = Some(PathBuf::from(option_value(&mut options, "--output")?)); + } + Some("--target") if target.is_none() => { + target = Some(parse_target(option_value(&mut options, "--target")?)?); + } + _ => return Err("build accepts --output and --target exactly once; unknown or duplicate option".into()), + } + } + Ok(( + root.into(), + Action::Build { + output: output.ok_or("build requires --output <new-file>")?, + target: target.map_or_else(BuildTarget::host, Ok)?, + }, + )) + } + [tenant, migrate, root, tenant_id] if tenant == "tenant" && migrate == "migrate" => { + let tenant_id = positive_tenant_id(tenant_id)?; + Ok((root.into(), Action::TenantMigrate { tenant_id })) + } + [tenant, owner, root, tenant_id, site, user_id] + if tenant == "tenant" && owner == "owner" => + { + let tenant_id = positive_id(tenant_id, "tenant id")?; + let site = site + .to_str() + .ok_or_else(|| "site must be UTF-8".to_owned())? + .to_owned(); + let user_id = positive_id(user_id, "user id")?; + Ok(( + root.into(), + Action::TenantOwner { + tenant_id, + site, + user_id, + }, + )) + } + [ + tenant, + component_command, + operation, + root, + tenant_id, + component, + ] if tenant == "tenant" + && component_command == "component" + && (operation == "enable" || operation == "disable") => + { + let tenant_id = positive_tenant_id(tenant_id)?; + let component = component + .to_str() + .ok_or_else(|| "component must be UTF-8".to_owned())? + .to_owned(); + Ok(( + root.into(), + Action::TenantComponent { + tenant_id, + component, + enabled: operation == "enable", + }, + )) + } + _ => Err(USAGE.into()), + } +} + +fn option_value<'a>( + options: &mut impl Iterator<Item = &'a OsString>, + option: &str, +) -> Result<&'a OsString, String> { + options + .next() + .filter(|value| !value.is_empty() && !value.to_string_lossy().starts_with("--")) + .ok_or_else(|| format!("{option} requires a value")) +} + +fn parse_target(value: &OsString) -> Result<BuildTarget, String> { + value.to_str().ok_or("build target must be UTF-8")?.parse() +} + +fn lib_arguments(arguments: &[OsString]) -> Result<(Vec<String>, BuildTarget), String> { + let mut specs = Vec::new(); + let mut target = None; + let mut arguments = arguments.iter(); + while let Some(argument) = arguments.next() { + if argument == "--target" && target.is_none() { + target = Some(parse_target(option_value(&mut arguments, "--target")?)?); + } else { + let spec = argument.to_str().ok_or("lib spec must be UTF-8")?; + if spec.starts_with('-') { + return Err("unknown or duplicate lib option".into()); + } + specs.push(spec.to_owned()); + } + } + Ok((specs, target.map_or_else(BuildTarget::host, Ok)?)) +} + +fn positive_tenant_id(value: &OsString) -> Result<i64, String> { + positive_id(value, "tenant id") +} + +fn positive_id(value: &OsString, label: &str) -> Result<i64, String> { + let id = value + .to_str() + .ok_or_else(|| format!("{label} must be a positive integer"))? + .parse::<i64>() + .map_err(|_| format!("{label} must be a positive integer"))?; + if id <= 0 { + return Err(format!("{label} must be a positive integer")); + } + Ok(id) +} + +#[cfg(test)] +mod tests { + use super::{Action, arguments_for}; + use std::ffi::OsString; + + #[test] + fn run_accepts_only_one_named_json_command_after_separator() { + let args = [ + "run", + "project", + "--", + "user.account.greeting", + "{\"name\":\"Ada\"}", + ] + .map(OsString::from); + let (root, action) = arguments_for(&args).unwrap(); + assert_eq!(root.to_string_lossy(), "project"); + let Action::Run { + tenant_id, + arguments, + } = action + else { + panic!("expected run action") + }; + assert_eq!(tenant_id, None); + assert_eq!(arguments, args[3..]); + assert!(arguments_for(&args[..4]).is_err()); + assert!(arguments_for(&[args.as_slice(), &[OsString::from("extra")]].concat()).is_err()); + } + + #[test] + fn tenant_command_uses_a_separate_positive_control_plane_id() { + let args = [ + "run", + "project", + "--tenant", + "42", + "--", + "content.article.rebuild", + "{\"force\":true}", + ] + .map(OsString::from); + let (root, action) = arguments_for(&args).unwrap(); + assert_eq!(root.to_string_lossy(), "project"); + let Action::Run { + tenant_id, + arguments, + } = action + else { + panic!("expected run action") + }; + assert_eq!(tenant_id, Some(42)); + assert_eq!(arguments, args[5..]); + + for invalid in ["0", "-1", "tenant-1", "9223372036854775808"] { + let args = [ + "run", + "project", + "--tenant", + invalid, + "--", + "content.article.rebuild", + "{}", + ] + .map(OsString::from); + let error = arguments_for(&args) + .err() + .expect("invalid tenant id must fail"); + assert_eq!(error, "tenant id must be a positive integer"); + } + } + + #[test] + fn tenant_migrate_requires_a_positive_integer_id() { + let args = ["tenant", "migrate", "project", "42"].map(OsString::from); + let (root, action) = arguments_for(&args).unwrap(); + assert_eq!(root.to_string_lossy(), "project"); + assert!(matches!(action, Action::TenantMigrate { tenant_id: 42 })); + + for invalid in ["0", "-1", "tenant-1", "9223372036854775808"] { + let args = ["tenant", "migrate", "project", invalid].map(OsString::from); + let error = arguments_for(&args) + .err() + .expect("invalid tenant id must fail"); + assert_eq!(error, "tenant id must be a positive integer"); + } + } + + #[test] + fn tenant_component_command_selects_one_manifest_component() { + for (operation, expected) in [("enable", true), ("disable", false)] { + let args = + ["tenant", "component", operation, "project", "42", "news"].map(OsString::from); + let (root, action) = arguments_for(&args).unwrap(); + assert_eq!(root.to_string_lossy(), "project"); + let Action::TenantComponent { + tenant_id, + component, + enabled, + } = action + else { + panic!("expected tenant component action"); + }; + assert_eq!(tenant_id, 42); + assert_eq!(component, "news"); + assert_eq!(enabled, expected); + } + + let invalid = + ["tenant", "component", "toggle", "project", "42", "news"].map(OsString::from); + assert!(arguments_for(&invalid).is_err()); + } + + #[test] + fn clean_accepts_exactly_one_project_root() { + let args = ["clean", "project"].map(OsString::from); + let (root, action) = arguments_for(&args).unwrap(); + assert_eq!(root.to_string_lossy(), "project"); + assert!(matches!(action, Action::Clean)); + assert!(arguments_for(&[args.as_slice(), &[OsString::from("extra")]].concat()).is_err()); + } +} diff --git a/crates/dever-cli/src/packages.rs b/crates/dever-cli/src/packages.rs new file mode 100644 index 0000000..951436a --- /dev/null +++ b/crates/dever-cli/src/packages.rs @@ -0,0 +1,1220 @@ +//! Explicit Package resolution and offline, lock-bound source loading. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::str::FromStr; +use std::time::Duration; + +use dever_core::source::PackageSource; +use node_semver::{Range, Version}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use url::Url; + +use crate::libs::{self, ArtifactStore, LibSpec}; + +const INDEX_FORMAT: &str = "dever-package-index-v1"; +const MANIFEST_FORMAT: &str = "dever-package-v1"; +const MAX_INDEX_BYTES: usize = 1024 * 1024; +const MAX_ARCHIVE_BYTES: u64 = 64 * 1024 * 1024; +const MAX_PACKAGES: usize = 256; +const MAX_SEARCH_STEPS: usize = 4096; + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PackageRequest { + pub name: String, + pub requirement: String, +} + +impl FromStr for PackageRequest { + type Err = String; + + fn from_str(value: &str) -> Result<Self, Self::Err> { + let (name, requirement) = value + .rsplit_once('@') + .ok_or("Package must be <name>@<version-range>")?; + valid_name(name)?; + requirement + .parse::<Range>() + .map_err(|error| format!("invalid Package version range: {error}"))?; + Ok(Self { + name: name.into(), + requirement: requirement.into(), + }) + } +} + +impl PackageRequest { + pub fn key(&self) -> String { + format!("{}@{}", self.name, self.requirement) + } + + fn matches(&self, version: &str) -> Result<bool, String> { + let range = self + .requirement + .parse::<Range>() + .map_err(|error| error.to_string())?; + let version = version + .parse::<Version>() + .map_err(|error| error.to_string())?; + Ok(version.satisfies(&range)) + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PackageDependency { + pub name: String, + pub version: String, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LockedPackage { + pub name: String, + pub version: String, + pub sha256: String, + pub bytes: u64, + pub dependencies: Vec<PackageDependency>, + pub manifest_sha256: String, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct PackageManifest { + format: String, + name: String, + version: String, + #[serde(default)] + dependencies: BTreeMap<String, String>, + #[serde(default)] + lib: Vec<String>, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct PackageIndex { + format: String, + name: String, + versions: Vec<IndexVersion>, +} + +#[derive(Clone, Deserialize)] +#[serde(deny_unknown_fields)] +struct IndexVersion { + version: String, + sha256: String, + bytes: u64, +} + +pub trait PackageTransport { + fn get(&self, path: &str, limit: usize) -> Result<Vec<u8>, String>; +} + +pub struct HttpPackageTransport { + origin: Url, + agent: ureq::Agent, +} + +impl HttpPackageTransport { + pub fn new(origin: &str) -> Result<Self, String> { + let origin = Url::parse(origin).map_err(|_| "invalid Package registry URL")?; + if origin.scheme() != "https" + || origin.host_str().is_none() + || origin.username() != "" + || origin.password().is_some() + || origin.query().is_some() + || origin.fragment().is_some() + || origin.path() != "/" + { + return Err("Package registry must be an HTTPS origin".into()); + } + Ok(Self::at(origin)) + } + + fn at(origin: Url) -> Self { + let config = ureq::Agent::config_builder() + .proxy(None) + .max_redirects(0) + .timeout_global(Some(Duration::from_secs(30))) + .build(); + Self { + origin, + agent: ureq::Agent::new_with_config(config), + } + } + + /// Only a caller-owned loopback peer may replace the HTTPS origin in tests. + pub fn loopback(origin: &str) -> Result<Self, String> { + let origin = Url::parse(origin).map_err(|_| "invalid loopback Package registry")?; + if origin.scheme() != "http" + || !matches!(origin.host(), Some(url::Host::Ipv4(ip)) if ip.is_loopback()) + || origin.username() != "" + || origin.password().is_some() + || origin.query().is_some() + || origin.fragment().is_some() + || origin.path() != "/" + { + return Err("Package test registry must be an owned IPv4 loopback origin".into()); + } + Ok(Self::at(origin)) + } +} + +impl PackageTransport for HttpPackageTransport { + fn get(&self, path: &str, limit: usize) -> Result<Vec<u8>, String> { + if !path.starts_with('/') + || path.contains(['?', '#', '\\']) + || path + .split('/') + .skip(1) + .any(|part| part.is_empty() || part == "." || part == "..") + { + return Err("invalid Package registry path".into()); + } + let url = self + .origin + .join(&path[1..]) + .map_err(|_| "invalid Package registry path")?; + let mut response = self + .agent + .get(url.as_str()) + .call() + .map_err(|_| "Package registry request failed")?; + if !response.status().is_success() { + return Err("Package registry returned a non-success response".into()); + } + response + .body_mut() + .with_config() + .limit(limit as u64 + 1) + .read_to_vec() + .map_err(|_| "cannot read Package registry response".into()) + .and_then(|bytes| { + (bytes.len() <= limit) + .then_some(bytes) + .ok_or("Package registry response exceeds limit".into()) + }) + } +} + +fn valid_name(name: &str) -> Result<(), String> { + if name.is_empty() + || name.len() > 64 + || !name + .bytes() + .next() + .is_some_and(|byte| byte.is_ascii_lowercase()) + || !name + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_') + { + return Err("Package name must use lower_snake_case".into()); + } + Ok(()) +} + +fn digest(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) +} + +fn validate_digest(value: &str) -> Result<(), String> { + if value.len() != 64 + || !value + .bytes() + .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) + { + return Err("Package SHA-256 must be lowercase hexadecimal".into()); + } + Ok(()) +} + +struct ManifestArchive { + manifest: PackageManifest, + files: Vec<(String, Vec<u8>)>, + manifest_sha256: String, +} + +fn manifest( + archive: &[u8], + expected_name: &str, + expected_version: &str, +) -> Result<ManifestArchive, String> { + let files = libs::archive_files("zip", archive)?; + let manifest_bytes = files + .iter() + .find(|(path, _)| path == "dever-package.json") + .map(|(_, bytes)| bytes) + .ok_or("Package archive lacks dever-package.json")?; + let text = std::str::from_utf8(manifest_bytes).map_err(|_| "Package manifest is not UTF-8")?; + dever_runtime::wire::parse(text) + .map_err(|error| format!("invalid Package manifest: {error}"))?; + let manifest: PackageManifest = serde_json::from_slice(manifest_bytes) + .map_err(|error| format!("invalid Package manifest: {error}"))?; + if manifest.format != MANIFEST_FORMAT + || manifest.name != expected_name + || manifest.version != expected_version + { + return Err("Package archive manifest identity differs from registry metadata".into()); + } + valid_name(&manifest.name)?; + manifest + .version + .parse::<Version>() + .map_err(|_| "invalid Package version")?; + for (name, requirement) in &manifest.dependencies { + valid_name(name)?; + requirement + .parse::<Range>() + .map_err(|_| "invalid Package dependency range")?; + } + for request in &manifest.lib { + request.parse::<LibSpec>()?; + } + let mut source_count = 0; + for (path, _) in &files { + if path == "dever-package.json" { + continue; + } + let valid_owner = path.starts_with(&format!("module/{expected_name}/")) + || path.starts_with(&format!("worker/{expected_name}/")); + if !valid_owner { + return Err(format!("Package archive contains unowned path '{path}'")); + } + if path.starts_with("module/") && (path.ends_with(".dever") || path.ends_with(".dever.md")) + { + source_count += 1; + } + } + if source_count == 0 { + return Err("Package archive has no Dever source".into()); + } + let manifest_sha256 = digest(manifest_bytes); + Ok(ManifestArchive { + manifest, + files, + manifest_sha256, + }) +} + +pub fn normalize_lock(packages: &mut Vec<LockedPackage>) -> Result<(), String> { + packages.sort_by(|left, right| left.name.cmp(&right.name)); + let mut names = BTreeSet::new(); + for package in packages { + valid_name(&package.name)?; + package + .version + .parse::<Version>() + .map_err(|_| "invalid locked Package version")?; + validate_digest(&package.sha256)?; + validate_digest(&package.manifest_sha256)?; + if package.bytes == 0 + || package.bytes > MAX_ARCHIVE_BYTES + || !names.insert(package.name.clone()) + { + return Err("invalid or duplicate locked Package".into()); + } + package + .dependencies + .sort_by(|left, right| left.name.cmp(&right.name)); + for dependency in &package.dependencies { + valid_name(&dependency.name)?; + dependency + .version + .parse::<Version>() + .map_err(|_| "invalid locked Package dependency")?; + } + } + Ok(()) +} + +pub fn doctor_lock(packages: &[LockedPackage]) -> Result<(), String> { + let mut canonical = packages.to_vec(); + normalize_lock(&mut canonical)?; + if canonical != packages { + return Err("Package lock is not canonical".into()); + } + let by_name = packages + .iter() + .map(|package| (package.name.as_str(), package.version.as_str())) + .collect::<BTreeMap<_, _>>(); + for package in packages { + let mut dependencies = BTreeSet::new(); + for dependency in &package.dependencies { + if !dependencies.insert(&dependency.name) || dependency.name == package.name { + return Err(format!( + "locked Package {} has duplicate or self dependency", + package.name + )); + } + if by_name.get(dependency.name.as_str()) != Some(&dependency.version.as_str()) { + return Err(format!( + "locked Package {} has a missing or mismatched dependency {}", + package.name, dependency.name + )); + } + } + } + let mut complete = BTreeSet::new(); + while complete.len() < packages.len() { + let before = complete.len(); + for package in packages { + if package + .dependencies + .iter() + .all(|dependency| complete.contains(&dependency.name)) + { + complete.insert(package.name.clone()); + } + } + if complete.len() == before { + return Err("cyclic locked Package dependencies".into()); + } + } + Ok(()) +} + +fn settings(project_root: &Path) -> Result<(String, Vec<PackageRequest>), String> { + if !project_root.join("config/setting.json").exists() { + return Ok((String::new(), Vec::new())); + } + let settings = dever_runtime::config::Settings::load_project(project_root)?; + match settings.packages() { + Some(package) => { + HttpPackageTransport::new(package.registry())?; + let mut roots = package + .roots() + .iter() + .map(|root| root.parse::<PackageRequest>()) + .collect::<Result<Vec<_>, _>>()?; + roots.sort(); + roots.dedup(); + if roots + .iter() + .map(|root| &root.name) + .collect::<BTreeSet<_>>() + .len() + != roots.len() + { + return Err("setting.json package.use repeats a Package name".into()); + } + Ok((package.registry().into(), roots)) + } + None => Ok((String::new(), Vec::new())), + } +} + +#[derive(Clone)] +struct SelectedVersion { + index: IndexVersion, + manifest: PackageManifest, + manifest_sha256: String, +} + +struct PackageResolver<'a> { + transport: &'a dyn PackageTransport, + store: &'a dyn ArtifactStore, + indexes: BTreeMap<String, Vec<IndexVersion>>, + steps: usize, +} + +impl PackageResolver<'_> { + fn index(&mut self, name: &str) -> Result<Vec<IndexVersion>, String> { + if let Some(index) = self.indexes.get(name) { + return Ok(index.clone()); + } + let bytes = self + .transport + .get(&format!("/v1/packages/{name}/index.json"), MAX_INDEX_BYTES)?; + let text = std::str::from_utf8(&bytes).map_err(|_| "Package index is not UTF-8")?; + dever_runtime::wire::parse(text) + .map_err(|error| format!("invalid Package index: {error}"))?; + let index: PackageIndex = serde_json::from_slice(&bytes) + .map_err(|error| format!("invalid Package index: {error}"))?; + if index.format != INDEX_FORMAT || index.name != name { + return Err(format!("Package index identity differs for {name}")); + } + let mut seen = BTreeSet::new(); + for version in &index.versions { + version + .version + .parse::<Version>() + .map_err(|_| "invalid indexed Package version")?; + validate_digest(&version.sha256)?; + if version.bytes == 0 + || version.bytes > MAX_ARCHIVE_BYTES + || !seen.insert(version.version.clone()) + { + return Err(format!( + "invalid or duplicate Package index version for {name}" + )); + } + } + let mut versions = index.versions; + versions.sort_by(|left, right| { + right + .version + .parse::<Version>() + .expect("validated version") + .cmp(&left.version.parse::<Version>().expect("validated version")) + }); + self.indexes.insert(name.into(), versions.clone()); + Ok(versions) + } + + fn archive(&self, name: &str, index: &IndexVersion) -> Result<Vec<u8>, String> { + let bytes = self.transport.get( + &format!("/v1/packages/{name}/{}.zip", index.version), + index.bytes as usize, + )?; + if bytes.len() as u64 != index.bytes || digest(&bytes) != index.sha256 { + return Err(format!( + "Package {name}@{} differs from registry index", + index.version + )); + } + let published = self.store.publish(&bytes, "package")?; + if published != index.sha256 { + return Err("Package cache receipt differs from registry index".into()); + } + Ok(bytes) + } + + fn select( + &mut self, + constraints: BTreeMap<String, Vec<PackageRequest>>, + selected: BTreeMap<String, SelectedVersion>, + ) -> Result<BTreeMap<String, SelectedVersion>, String> { + self.steps += 1; + if self.steps > MAX_SEARCH_STEPS || constraints.len() > MAX_PACKAGES { + return Err("Package resolution budget exceeded".into()); + } + for (name, requests) in &constraints { + if let Some(chosen) = selected.get(name) + && !requests + .iter() + .all(|request| request.matches(&chosen.index.version).unwrap_or(false)) + { + return Err(format!( + "Package {name} has incompatible version constraints" + )); + } + } + let Some((name, requests)) = constraints + .iter() + .find(|(name, _)| !selected.contains_key(*name)) + else { + return Ok(selected); + }; + let name = name.clone(); + let requests = requests.clone(); + let mut conflict = None; + for index in self.index(&name)? { + if !requests + .iter() + .all(|request| request.matches(&index.version).unwrap_or(false)) + { + continue; + } + let archive = self.archive(&name, &index)?; + let ManifestArchive { + manifest, + manifest_sha256, + .. + } = manifest(&archive, &name, &index.version)?; + let mut next_constraints = constraints.clone(); + for (dependency, requirement) in &manifest.dependencies { + next_constraints + .entry(dependency.clone()) + .or_default() + .push(PackageRequest { + name: dependency.clone(), + requirement: requirement.clone(), + }); + } + let mut next_selected = selected.clone(); + next_selected.insert( + name.clone(), + SelectedVersion { + index, + manifest, + manifest_sha256, + }, + ); + match self.select(next_constraints, next_selected) { + Ok(result) => return Ok(result), + Err(error) + if error.contains("incompatible version constraints") + || error.contains("no version satisfies") => + { + conflict = Some(error) + } + Err(error) => return Err(error), + } + } + Err(conflict + .unwrap_or_else(|| format!("no Package version satisfies constraints for {name}"))) + } +} + +struct ResolvedClosure { + packages: Vec<LockedPackage>, + libs: Vec<LibSpec>, + sources: Vec<PackageSource>, +} + +fn resolve_with( + roots: &[PackageRequest], + transport: &dyn PackageTransport, + store: &dyn ArtifactStore, +) -> Result<ResolvedClosure, String> { + let mut constraints = BTreeMap::<String, Vec<PackageRequest>>::new(); + for request in roots { + constraints + .entry(request.name.clone()) + .or_default() + .push(request.clone()); + } + let selected = PackageResolver { + transport, + store, + indexes: BTreeMap::new(), + steps: 0, + } + .select(constraints, BTreeMap::new())?; + let mut packages = Vec::new(); + let mut libs = BTreeSet::new(); + let mut sources = Vec::new(); + for (name, chosen) in &selected { + let archive = store.verify_exact(&chosen.index.sha256, chosen.index.bytes, "package")?; + let ManifestArchive { + files, + manifest_sha256, + .. + } = manifest(&archive, name, &chosen.index.version)?; + if manifest_sha256 != chosen.manifest_sha256 { + return Err(format!("Package {name} manifest changed")); + } + for request in &chosen.manifest.lib { + libs.insert(request.parse::<LibSpec>()?); + } + sources.extend(package_sources(files)?); + let dependencies = chosen + .manifest + .dependencies + .keys() + .map(|dependency| PackageDependency { + name: dependency.clone(), + version: selected[dependency].index.version.clone(), + }) + .collect(); + packages.push(LockedPackage { + name: name.clone(), + version: chosen.index.version.clone(), + sha256: chosen.index.sha256.clone(), + bytes: chosen.index.bytes, + dependencies, + manifest_sha256: chosen.manifest_sha256.clone(), + }); + } + normalize_lock(&mut packages)?; + Ok(ResolvedClosure { + packages, + libs: libs.into_iter().collect(), + sources, + }) +} + +fn retain_locked_closure( + project_root: &Path, + roots: &[PackageRequest], + lock: &libs::LockFile, + store: &dyn ArtifactStore, +) -> Result<ResolvedClosure, String> { + let files = owned_files_with(project_root, store)?; + let by_name = lock + .packages + .iter() + .map(|package| (package.name.as_str(), package)) + .collect::<BTreeMap<_, _>>(); + let mut names = BTreeSet::new(); + let mut pending = roots + .iter() + .map(|root| root.name.as_str()) + .collect::<Vec<_>>(); + while let Some(name) = pending.pop() { + if !names.insert(name) { + continue; + } + let package = by_name + .get(name) + .ok_or_else(|| format!("Package {name} is missing from dever.lock"))?; + pending.extend( + package + .dependencies + .iter() + .map(|dependency| dependency.name.as_str()), + ); + } + let packages = lock + .packages + .iter() + .filter(|package| names.contains(package.name.as_str())) + .cloned() + .collect::<Vec<_>>(); + let mut libs = BTreeSet::new(); + for package in &packages { + let archive = store.verify_exact(&package.sha256, package.bytes, "package")?; + let ManifestArchive { manifest, .. } = manifest(&archive, &package.name, &package.version)?; + for spec in manifest.lib { + libs.insert(spec.parse::<LibSpec>()?); + } + } + let sources = package_sources( + files + .into_iter() + .filter(|(path, _)| { + path.strip_prefix("module/") + .and_then(|path| path.split('/').next()) + .is_some_and(|name| names.contains(name)) + }) + .collect(), + )?; + Ok(ResolvedClosure { + packages, + libs: libs.into_iter().collect(), + sources, + }) +} + +fn package_sources(files: Vec<(String, Vec<u8>)>) -> Result<Vec<PackageSource>, String> { + files + .into_iter() + .filter(|(path, _)| { + path.starts_with("module/") && (path.ends_with(".dever") || path.ends_with(".dever.md")) + }) + .map(|(path, bytes)| { + let text = String::from_utf8(bytes) + .map_err(|_| format!("Package source '{path}' is not UTF-8"))?; + Ok(PackageSource { + path: PathBuf::from(path.strip_prefix("module/").expect("filtered module path")), + display: PathBuf::from(format!("package/{path}")), + text, + }) + }) + .collect() +} + +pub fn owned_files(project_root: &Path) -> Result<BTreeMap<String, Vec<u8>>, String> { + let (_, roots) = settings(project_root)?; + if roots.is_empty() { + if project_root.join("dever.lock").exists() + && !libs::read_lock(&project_root.join("dever.lock"))? + .packages + .is_empty() + { + return Err("Package lock has no setting.json roots".into()); + } + return Ok(BTreeMap::new()); + } + let store = libs::managed_artifact_store()?; + owned_files_with(project_root, &store) +} + +pub fn owned_files_with( + project_root: &Path, + store: &dyn ArtifactStore, +) -> Result<BTreeMap<String, Vec<u8>>, String> { + let (_, roots) = settings(project_root)?; + let lock_path = project_root.join("dever.lock"); + if roots.is_empty() && !lock_path.exists() { + return Ok(BTreeMap::new()); + } + if !lock_path.exists() { + return Err("Package declarations require dever.lock; run 'dever package update'".into()); + } + let lock = libs::read_lock(&lock_path)?; + let packages = &lock.packages; + validate_locked_roots(&roots, packages)?; + let mut files = BTreeMap::new(); + for package in packages { + if fs::symlink_metadata(project_root.join("module").join(&package.name)).is_ok() { + return Err(format!( + "Package '{}' collides with a local component", + package.name + )); + } + let archive = store.verify_exact(&package.sha256, package.bytes, "package")?; + let ManifestArchive { + manifest, + files: entries, + manifest_sha256, + } = manifest(&archive, &package.name, &package.version)?; + if manifest_sha256 != package.manifest_sha256 { + return Err(format!("Package {} manifest changed", package.name)); + } + let dependencies = manifest + .dependencies + .iter() + .map(|(name, requirement)| { + let dependency = packages + .iter() + .find(|package| &package.name == name) + .ok_or_else(|| { + format!("Package {} dependency {name} is missing", package.name) + })?; + if !(PackageRequest { + name: name.clone(), + requirement: requirement.clone(), + }) + .matches(&dependency.version)? + { + return Err(format!( + "Package {} dependency {name} lock is stale", + package.name + )); + } + Ok(PackageDependency { + name: name.clone(), + version: dependency.version.clone(), + }) + }) + .collect::<Result<Vec<_>, String>>()?; + if dependencies != package.dependencies { + return Err(format!("Package {} dependencies changed", package.name)); + } + for (path, bytes) in entries { + if path == "dever-package.json" { + continue; + } + let owned_path = project_root.join(&path); + match fs::symlink_metadata(&owned_path) { + Ok(_) => { + return Err(format!( + "Package '{}' collides with project path '{}'", + package.name, path + )); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(format!("cannot inspect Package path '{path}': {error}")), + } + if files.insert(path.clone(), bytes).is_some() { + return Err(format!("Package path '{path}' is duplicated")); + } + } + } + Ok(files) +} + +fn validate_locked_roots( + roots: &[PackageRequest], + packages: &[LockedPackage], +) -> Result<(), String> { + doctor_lock(packages)?; + for root in roots { + let package = packages + .iter() + .find(|package| package.name == root.name) + .ok_or_else(|| format!("Package {} is missing from dever.lock", root.name))?; + if !root.matches(&package.version)? { + return Err(format!("Package {} lock is stale", root.name)); + } + } + if roots.is_empty() && !packages.is_empty() { + return Err("Package lock has no setting.json roots".into()); + } + let by_name = packages + .iter() + .map(|package| (package.name.as_str(), package)) + .collect::<BTreeMap<_, _>>(); + let mut reachable = BTreeSet::new(); + let mut pending = roots + .iter() + .map(|root| root.name.as_str()) + .collect::<Vec<_>>(); + while let Some(name) = pending.pop() { + if !reachable.insert(name) { + continue; + } + pending.extend( + by_name[name] + .dependencies + .iter() + .map(|dependency| dependency.name.as_str()), + ); + } + if reachable.len() != packages.len() { + return Err("Package lock contains unreferenced Package entries".into()); + } + Ok(()) +} + +pub fn sources(project_root: &Path) -> Result<Vec<PackageSource>, String> { + if settings(project_root)?.1.is_empty() { + return package_sources(owned_files(project_root)?.into_iter().collect()); + } + let store = libs::managed_artifact_store()?; + sources_with(project_root, &store) +} + +pub fn sources_with( + project_root: &Path, + store: &dyn ArtifactStore, +) -> Result<Vec<PackageSource>, String> { + package_sources(owned_files_with(project_root, store)?.into_iter().collect()) +} + +pub fn owned_worker_bytes(project_root: &Path, entry: &str) -> Result<Option<Vec<u8>>, String> { + if !owns_worker(project_root, entry)? { + return Ok(None); + } + let store = libs::managed_artifact_store()?; + locked_worker_bytes(project_root, entry, &store) +} + +pub fn owned_worker_bytes_with( + project_root: &Path, + entry: &str, + store: &dyn ArtifactStore, +) -> Result<Option<Vec<u8>>, String> { + if !owns_worker(project_root, entry)? { + return Ok(None); + } + locked_worker_bytes(project_root, entry, store) +} + +fn locked_worker_bytes( + project_root: &Path, + entry: &str, + store: &dyn ArtifactStore, +) -> Result<Option<Vec<u8>>, String> { + owned_files_with(project_root, store)? + .remove(entry) + .map(Some) + .ok_or_else(|| format!("Package Worker entry '{entry}' is missing from its locked archive")) +} + +pub fn owns_worker(project_root: &Path, entry: &str) -> Result<bool, String> { + let Some((name, relative_entry)) = entry + .strip_prefix("worker/") + .or_else(|| entry.strip_prefix("module/")) + .and_then(|path| path.split_once('/')) + else { + return Ok(false); + }; + if name.is_empty() || relative_entry.is_empty() { + return Ok(false); + } + let (_, roots) = settings(project_root)?; + let lock_path = project_root.join("dever.lock"); + if roots.is_empty() && !lock_path.exists() { + return Ok(false); + } + if !lock_path.exists() { + return Err("Package declarations require dever.lock; run 'dever package update'".into()); + } + let lock = libs::read_lock(&lock_path)?; + validate_locked_roots(&roots, &lock.packages)?; + Ok(lock.packages.iter().any(|package| package.name == name)) +} + +pub fn declared_libs(project_root: &Path) -> Result<Vec<LibSpec>, String> { + if settings(project_root)?.1.is_empty() { + return Ok(Vec::new()); + } + let store = libs::managed_artifact_store()?; + declared_libs_with(project_root, &store) +} + +pub fn declared_libs_with( + project_root: &Path, + store: &dyn ArtifactStore, +) -> Result<Vec<LibSpec>, String> { + let (_, roots) = settings(project_root)?; + if roots.is_empty() { + return Ok(Vec::new()); + } + owned_files_with(project_root, store)?; + let lock = libs::read_lock(&project_root.join("dever.lock"))?; + let mut libs = BTreeSet::new(); + for package in lock.packages { + let archive = store.verify_exact(&package.sha256, package.bytes, "package")?; + let ManifestArchive { + manifest, + manifest_sha256, + .. + } = manifest(&archive, &package.name, &package.version)?; + if manifest_sha256 != package.manifest_sha256 { + return Err(format!("Package {} manifest changed", package.name)); + } + for request in manifest.lib { + libs.insert(request.parse::<LibSpec>()?); + } + } + Ok(libs.into_iter().collect()) +} + +fn write_roots( + project_root: &Path, + registry: &str, + roots: &[PackageRequest], +) -> Result<(), String> { + libs::reject_declaration_symlinks(project_root)?; + let path = project_root.join("config/setting.json"); + let text = fs::read_to_string(&path) + .map_err(|error| format!("cannot read '{}': {error}", path.display()))?; + dever_runtime::wire::parse(&text).map_err(|error| format!("invalid setting.json: {error}"))?; + let mut document: serde_json::Map<String, serde_json::Value> = + serde_json::from_str(&text).map_err(|error| format!("invalid setting.json: {error}"))?; + document.insert( + "package".into(), + serde_json::json!({ + "registry": registry, + "use": roots.iter().map(PackageRequest::key).collect::<Vec<_>>(), + }), + ); + let mut bytes = serde_json::to_vec_pretty(&document).map_err(|error| error.to_string())?; + bytes.push(b'\n'); + replace_settings(&path, &bytes) +} + +fn replace_settings(path: &Path, bytes: &[u8]) -> Result<(), String> { + let temporary = + path.with_file_name(format!(".setting.json.package.{}.tmp", std::process::id())); + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&temporary) + .map_err(|error| format!("cannot stage setting.json: {error}"))?; + use std::io::Write; + let result = file.write_all(bytes).and_then(|_| file.sync_all()); + drop(file); + if let Err(error) = result { + let _ = fs::remove_file(&temporary); + return Err(format!("cannot stage setting.json: {error}")); + } + fs::rename(&temporary, path).map_err(|error| { + let _ = fs::remove_file(&temporary); + format!("cannot replace setting.json: {error}") + }) +} + +fn selected_roots( + current: &[PackageRequest], + command: &str, + specs: &[String], +) -> Result<Vec<PackageRequest>, String> { + let mut roots = current.to_vec(); + match command { + "add" | "update" => { + for spec in specs { + let request = spec.parse::<PackageRequest>()?; + roots.retain(|root| root.name != request.name); + roots.push(request); + } + } + "remove" => { + if specs.is_empty() { + return Err("dever package remove requires a Package name".into()); + } + for name in specs { + valid_name(name)?; + if !roots.iter().any(|root| &root.name == name) { + return Err(format!( + "Package {name} is not a direct setting.json dependency" + )); + } + roots.retain(|root| &root.name != name); + } + } + _ => unreachable!("command checked by caller"), + } + roots.sort(); + if roots + .iter() + .map(|root| &root.name) + .collect::<BTreeSet<_>>() + .len() + != roots.len() + { + return Err("duplicate Package root".into()); + } + Ok(roots) +} + +fn check_project_collisions( + project_root: &Path, + packages: &[LockedPackage], + store: &dyn ArtifactStore, +) -> Result<(), String> { + for package in packages { + if fs::symlink_metadata(project_root.join("module").join(&package.name)).is_ok() { + return Err(format!( + "Package '{}' collides with a local component", + package.name + )); + } + let archive = store.verify_exact(&package.sha256, package.bytes, "package")?; + let ManifestArchive { files: entries, .. } = + manifest(&archive, &package.name, &package.version)?; + for (path, _) in entries { + if path == "dever-package.json" { + continue; + } + match fs::symlink_metadata(project_root.join(&path)) { + Ok(_) => { + return Err(format!( + "Package '{}' collides with project path '{path}'", + package.name + )); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(format!("cannot inspect Package path '{path}': {error}")), + } + } + } + Ok(()) +} + +pub fn execute(command: &str, project_root: &Path, specs: &[String]) -> Result<String, String> { + let _mutation = libs::ProjectMutation::for_command(command, project_root)?; + let (registry, _) = settings(project_root)?; + if command == "list" { + let lock = libs::read_lock(&project_root.join("dever.lock"))?; + return Ok(lock + .packages + .iter() + .map(|package| format!("{}@{}", package.name, package.version)) + .collect::<Vec<_>>() + .join("\n")); + } + let transport = HttpPackageTransport::new(®istry)?; + let store = libs::managed_artifact_store()?; + execute_locked(command, project_root, specs, &transport, &store) +} + +pub fn execute_with( + command: &str, + project_root: &Path, + specs: &[String], + transport: &dyn PackageTransport, + store: &dyn ArtifactStore, +) -> Result<String, String> { + let _mutation = libs::ProjectMutation::for_command(command, project_root)?; + execute_locked(command, project_root, specs, transport, store) +} + +fn execute_locked( + command: &str, + project_root: &Path, + specs: &[String], + transport: &dyn PackageTransport, + store: &dyn ArtifactStore, +) -> Result<String, String> { + let (registry, current) = settings(project_root)?; + let lock_path = project_root.join("dever.lock"); + match command { + "list" => { + let lock = libs::read_lock(&lock_path)?; + Ok(lock + .packages + .iter() + .map(|package| format!("{}@{}", package.name, package.version)) + .collect::<Vec<_>>() + .join("\n")) + } + "doctor" => { + if !specs.is_empty() { + return Err("dever package doctor takes no specs".into()); + } + let sources = + package_sources(owned_files_with(project_root, store)?.into_iter().collect())?; + let lock = libs::read_lock(&lock_path)?; + libs::doctor(&lock)?; + let workers = libs::source_workers_with_packages( + project_root, + &sources, + crate::toolchain::BuildTarget::host()?, + )?; + if workers != lock.workers { + return Err("Package Worker binding differs from dever.lock".into()); + } + Ok(format!("{} is valid", lock_path.display())) + } + "add" | "update" | "remove" => { + let roots = selected_roots(¤t, command, specs)?; + if command == "add" && specs.is_empty() { + return Err("dever package add requires <name>@<version-range>".into()); + } + if command == "update" && roots.is_empty() { + return Err("dever package update requires Package roots".into()); + } + let previous = if lock_path.exists() { + Some(libs::read_lock(&lock_path)?) + } else { + None + }; + let closure = if command == "remove" { + retain_locked_closure( + project_root, + &roots, + previous + .as_ref() + .ok_or("dever.lock is required for offline Package removal")?, + store, + )? + } else { + resolve_with(&roots, transport, store)? + }; + check_project_collisions(project_root, &closure.packages, store)?; + let workers = libs::source_workers_with_packages( + project_root, + &closure.sources, + crate::toolchain::BuildTarget::host()?, + )?; + let mut requested = libs::read_declarations(project_root)?; + requested.extend(closure.libs); + requested.sort(); + requested.dedup(); + let mut lock = if command == "remove" { + libs::retain_locked_libs( + previous.as_ref().expect("remove requires lock"), + &requested, + workers, + )? + } else { + libs::resolve_project( + project_root, + &requested, + workers, + crate::toolchain::BuildTarget::host()?, + )? + }; + lock.packages = closure.packages; + libs::doctor(&lock)?; + lock.encode()?; + let previous_settings = fs::read(project_root.join("config/setting.json")) + .map_err(|error| format!("cannot save setting.json for rollback: {error}"))?; + write_roots(project_root, ®istry, &roots)?; + if let Err(error) = lock.write_atomic(project_root) { + let settings_path = project_root.join("config/setting.json"); + return match replace_settings(&settings_path, &previous_settings) { + Ok(()) => Err(error), + Err(rollback) => { + Err(format!("{error}; setting.json rollback failed: {rollback}")) + } + }; + } + Ok(format!( + "locked {} Package(s) in {}", + lock.packages.len(), + lock_path.display() + )) + } + _ => Err( + "Usage: dever package add|update|remove|list|doctor <project-root> [spec ...]".into(), + ), + } +} diff --git a/crates/dever-cli/src/project.rs b/crates/dever-cli/src/project.rs new file mode 100644 index 0000000..8f817b6 --- /dev/null +++ b/crates/dever-cli/src/project.rs @@ -0,0 +1,128 @@ +//! Create complete starter projects without replacing caller-owned paths. + +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +static NEXT_STAGING: AtomicU64 = AtomicU64::new(0); + +macro_rules! template { + ($path:literal) => { + include_str!(concat!("../../../skills/assets/template/", $path)) + }; +} + +const FILES: &[(&str, &str)] = &[ + ("config/setting.json", template!("config/setting.json")), + ("AGENTS.md", template!("AGENTS.md")), + ("README.md", template!("README.md")), + (".gitignore", template!(".gitignore")), +]; + +const SOURCES: &[(&str, &str, &str)] = &[ + ( + "module/hello/greeting/app", + template!("module/hello/greeting/app.dever"), + template!("module/hello/greeting/app.md"), + ), + ( + "module/hello/greeting/api", + template!("module/hello/greeting/api.dever"), + template!("module/hello/greeting/api.md"), + ), + ( + "test/hello/greeting/greet", + template!("test/hello/greeting/greet.dever"), + template!("test/hello/greeting/greet.md"), + ), +]; + +/// Publish a new project beside its private staging directory. Existing targets, +/// including empty directories and dangling symlinks, are never replaced. +pub fn create(destination: &Path, markdown: bool) -> Result<(), String> { + let parent = destination + .parent() + .filter(|path| !path.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + let name = destination + .file_name() + .ok_or_else(|| "new project requires a new directory name".to_owned())?; + let parent = fs::canonicalize(parent) + .map_err(|error| format!("cannot resolve new project parent: {error}"))?; + let destination = parent.join(name); + match fs::symlink_metadata(&destination) { + Ok(_) => { + return Err(format!( + "project target already exists: {}", + destination.display() + )); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(format!("cannot inspect project target: {error}")), + } + let staging = Staging::create(&parent)?; + for (path, content) in FILES { + write(&staging.0, path, content)?; + } + for (path, source, document) in SOURCES { + let (suffix, content) = if markdown { + ("dever.md", document.replace("{{source}}", source)) + } else { + ("dever", (*source).to_owned()) + }; + write(&staging.0, &format!("{path}.{suffix}"), &content)?; + } + publish(&staging.0, &destination) +} + +fn write(root: &Path, path: &str, content: &str) -> Result<(), String> { + let path = root.join(path); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent) + .map_err(|error| format!("cannot create template directory: {error}"))?; + } + fs::write(&path, content) + .map_err(|error| format!("cannot write template {}: {error}", path.display())) +} + +struct Staging(PathBuf); + +impl Staging { + fn create(parent: &Path) -> Result<Self, String> { + loop { + let sequence = NEXT_STAGING.fetch_add(1, Ordering::Relaxed); + let path = parent.join(format!(".dever-new-{}-{sequence}", std::process::id())); + let mut builder = fs::DirBuilder::new(); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + match builder.create(&path) { + Ok(()) => return Ok(Self(path)), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => return Err(format!("cannot stage new project: {error}")), + } + } + } +} + +impl Drop for Staging { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +fn publish(staging: &Path, destination: &Path) -> Result<(), String> { + #[cfg(target_os = "linux")] + { + use rustix::fs::{CWD, RenameFlags, renameat_with}; + renameat_with(CWD, staging, CWD, destination, RenameFlags::NOREPLACE) + .map_err(|error| format!("cannot publish new project: {error}")) + } + #[cfg(not(target_os = "linux"))] + { + let _ = (staging, destination); + Err("new project publication is not available for this platform".into()) + } +} diff --git a/crates/dever-cli/src/test_runner.rs b/crates/dever-cli/src/test_runner.rs new file mode 100644 index 0000000..7227132 --- /dev/null +++ b/crates/dever-cli/src/test_runner.rs @@ -0,0 +1,203 @@ +use std::fs; +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Command, ExitCode}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use dever_core::hir::{Program, TestCase}; +use dever_core::source::SourceMap; + +pub(super) fn execute(program: &Program, sources: &SourceMap) -> Result<ExitCode, String> { + let mut stdout = io::stdout().lock(); + writeln!(stdout, "running {} tests", program.tests().len()).map_err(io_error)?; + + let mut passed = 0; + let mut failures = Vec::new(); + if !program.tests().is_empty() { + match crate::compile::test_suite(program, sources) { + Ok(native) => { + for (index, test) in program.tests().iter().enumerate() { + match run_case(program, test, &native, index) { + Ok(()) => { + passed += 1; + writeln!(stdout, "test {} ... ok", test.name()).map_err(io_error)?; + } + Err(failure) => { + writeln!(stdout, "test {} ... FAILED", test.name()) + .map_err(io_error)?; + failures.push((test.name(), failure)); + } + } + } + } + Err(message) => { + for test in program.tests() { + writeln!(stdout, "test {} ... FAILED", test.name()).map_err(io_error)?; + failures.push((test.name(), CaseFailure::message(message.clone()))); + } + } + } + } + + for (name, failure) in &failures { + writeln!(stdout, "\n---- {name} failure ----").map_err(io_error)?; + writeln!(stdout, "{}", failure.message).map_err(io_error)?; + write_output(&mut stdout, "stdout", &failure.stdout)?; + write_output(&mut stdout, "stderr", &failure.stderr)?; + } + + let failed = failures.len(); + let status = if failed == 0 { "ok" } else { "FAILED" }; + writeln!( + stdout, + "\ntest result: {status}. {passed} passed; {failed} failed; 0 not run" + ) + .map_err(io_error)?; + Ok(if failed == 0 { + ExitCode::SUCCESS + } else { + ExitCode::FAILURE + }) +} + +fn run_case( + program: &Program, + test: &TestCase, + native: &dever_core::native::NativeProgram, + index: usize, +) -> Result<(), CaseFailure> { + let project = TemporaryProject::new().map_err(CaseFailure::message)?; + if test.uses_database() { + write_settings(project.path(), test).map_err(CaseFailure::message)?; + let settings = dever_runtime::config::Settings::load_project(project.path()) + .map_err(CaseFailure::message)?; + let profile = program + .validate_database_settings(&settings) + .map_err(CaseFailure::message)?; + if !profile.sqlite || profile.postgres { + return Err(CaseFailure::message( + "generated test settings must select SQLite only".into(), + )); + } + } + + let executable = project + .path() + .join(format!("test-program{}", std::env::consts::EXE_SUFFIX)); + native.save(&executable).map_err(|error| { + CaseFailure::message(format!( + "cannot stage test executable '{}': {error}", + executable.display() + )) + })?; + let output = Command::new(&executable) + .arg(index.to_string()) + .current_dir(project.path()) + .output() + .map_err(|error| CaseFailure::message(format!("cannot run test process: {error}")))?; + if output.status.success() { + Ok(()) + } else { + Err(CaseFailure { + message: format!("test process exited with {}", output.status), + stdout: output.stdout, + stderr: output.stderr, + }) + } +} + +fn write_settings(root: &Path, test: &TestCase) -> Result<(), String> { + let config = root.join("config"); + fs::create_dir(&config) + .map_err(|error| format!("cannot create '{}': {error}", config.display()))?; + let mut databases = serde_json::Map::new(); + for name in test.database_connections() { + databases.insert( + name.clone(), + serde_json::json!({ + "type": "sqlite", + "path": format!("data/db/{name}.db"), + "max_connections": 1, + "max_page_size": 100 + }), + ); + } + let document = serde_json::json!({ "database": databases }); + let path = config.join("setting.json"); + let bytes = serde_json::to_vec_pretty(&document) + .map_err(|error| format!("cannot encode test settings: {error}"))?; + fs::write(&path, bytes).map_err(|error| format!("cannot write '{}': {error}", path.display())) +} + +fn write_output(output: &mut impl Write, label: &str, bytes: &[u8]) -> Result<(), String> { + if bytes.is_empty() { + return Ok(()); + } + writeln!(output, "---- {label} ----").map_err(io_error)?; + output.write_all(bytes).map_err(io_error)?; + if !bytes.ends_with(b"\n") { + writeln!(output).map_err(io_error)?; + } + Ok(()) +} + +fn io_error(error: io::Error) -> String { + error.to_string() +} + +struct CaseFailure { + message: String, + stdout: Vec<u8>, + stderr: Vec<u8>, +} + +impl CaseFailure { + fn message(message: String) -> Self { + Self { + message, + stdout: Vec::new(), + stderr: Vec::new(), + } + } +} + +struct TemporaryProject(PathBuf); + +impl TemporaryProject { + fn new() -> Result<Self, String> { + static NEXT: AtomicU64 = AtomicU64::new(0); + loop { + let path = std::env::temp_dir().join(format!( + "dever-application-test-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let mut builder = fs::DirBuilder::new(); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + match builder.create(&path) { + Ok(()) => return Ok(Self(path)), + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue, + Err(error) => { + return Err(format!( + "cannot create temporary test project '{}': {error}", + path.display() + )); + } + } + } + } + + fn path(&self) -> &Path { + &self.0 + } +} + +impl Drop for TemporaryProject { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} diff --git a/crates/dever-cli/src/toolchain/bootstrap.rs b/crates/dever-cli/src/toolchain/bootstrap.rs new file mode 100644 index 0000000..1d67cce --- /dev/null +++ b/crates/dever-cli/src/toolchain/bootstrap.rs @@ -0,0 +1,434 @@ +//! Explicit Linux first installation. The caller must already trust this installer. +//! Release keys come from an independent administrator input, never the release. +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Write}; +use std::os::unix::fs::{MetadataExt, OpenOptionsExt, PermissionsExt}; +use std::path::{Component, Path, PathBuf}; + +use serde::Deserialize; +use sha2::{Digest, Sha256}; + +use super::release::{self, InstallLock}; +use super::{Artifact, Layout, MachineManager, ReleaseManifest, Version}; + +mod sandbox; +mod service; +mod transaction; +pub use service::{RecoveryAction, recovery_actions}; + +pub(crate) const SERVICE: &str = "[Unit]\nDescription=Dever shared compiler service\nAfter=local-fs.target\n\n[Service]\nType=exec\nUser=root\nGroup=root\nExecStart=/opt/dever/bin/deverd --root /opt/dever\nRuntimeDirectory=dever\nRuntimeDirectoryMode=0755\nUMask=0077\nRestart=on-failure\nRestartSec=1\nKillMode=control-group\nTimeoutStopSec=30\nNoNewPrivileges=yes\nPrivateTmp=yes\nProtectHome=yes\nProtectSystem=strict\nReadWritePaths=/opt/dever/state /opt/dever/cache /run/dever\n\n[Install]\nWantedBy=multi-user.target\n"; + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Settings { + release: PathBuf, + system_root: PathBuf, + trusted_key: PathBuf, + #[serde(default)] + activate_service: bool, +} + +/// Private distribution entry; ordinary version install/update remains unchanged. +pub fn install(installer_root: &Path) -> Result<Version, String> { + require_root()?; + absolute_root(installer_root)?; + trusted_directory(installer_root)?; + let config = installer_root.join("config/setting.json"); + trusted_file(&config)?; + let settings: Settings = + serde_json::from_slice(&read_bounded(&config, 1024 * 1024)?).map_err(|error| { + format!( + "invalid installer config at {}:{}", + error.line(), + error.column() + ) + })?; + absolute_root(&settings.system_root)?; + trusted_directory(&settings.system_root)?; + if settings.system_root == Path::new("/") { + // Publishing an AppArmor file can trigger a later system-wide reload, + // even when this installer does not load profiles or activate services. + dever_sandbox::validate_host_policy()?; + } + if settings.activate_service && settings.system_root != Path::new("/") { + return Err( + "service activation requires the actual system root; image installs are offline".into(), + ); + } + let package = absolute_input(installer_root, &settings.release)?; + let key_path = absolute_input(installer_root, &settings.trusted_key)?; + trusted_file(&key_path)?; + let package = fs::canonicalize(package).map_err(|error| error.to_string())?; + if key_path.starts_with(&package) { + return Err("trusted key must be supplied independently outside the release".into()); + } + let key = release::decode_hex_file(&key_path, 32)?; + create_directory(&settings.system_root, "opt")?; + let layout = Layout::new(settings.system_root.join("opt/dever")); + if layout.root().exists() { + trusted_directory(layout.root())?; + } + for directory in [ + layout.bin(), + layout.state(), + layout.versions(), + layout.downloads(), + layout.staging(), + layout.native_cache(), + ] { + let relative = directory + .strip_prefix(&settings.system_root) + .map_err(|error| error.to_string())? + .to_str() + .ok_or("installer directory is not UTF-8")?; + create_directory(&settings.system_root, relative)?; + } + layout.validate_machine_permissions()?; + let _lock = InstallLock::acquire(&layout)?; + transaction::recover(&settings.system_root, &layout)?; + let pinned = layout.state().join("trusted-release-key"); + if pinned.exists() { + trusted_file(&pinned)?; + if release::decode_hex_file(&pinned, 32)? != key { + return Err("installer key differs from the machine's pinned release key".into()); + } + } + // The private copy is checked again by the version owner before any health + // command executes. A writable download cannot race signed bytes into it. + let manifest = MachineManager::verify_package(&package, None, &key)?; + validate_payload(&package, &manifest.artifacts, true)?; + pin_key(&layout, &key)?; + let manager = MachineManager::new(layout.clone()); + if manager + .active_version()? + .is_some_and(|active| manifest.version < active) + { + return Err("bootstrap installation cannot downgrade the active compiler".into()); + } + let installed = layout.versions().join(manifest.version.as_str()); + if installed.exists() + && super::sha256_file(&installed.join("manifest.json"))? + != super::sha256_file(&package.join("manifest.json"))? + { + return Err("installed version already has a different signed release identity".into()); + } + manager.install_package_locked(&package, &manifest.version)?; + transaction::install( + &settings.system_root, + &layout, + &installed, + &manifest, + &key, + settings.activate_service, + )?; + Ok(manifest.version) +} + +/// Recover only this installer's fixed, recorded targets. Normal version +/// operations must finish a pending bootstrap transaction first. +pub fn recover(system_root: &Path) -> Result<(), String> { + require_root()?; + absolute_root(system_root)?; + trusted_directory(system_root)?; + let layout = Layout::new(system_root.join("opt/dever")); + layout.validate_machine_permissions()?; + let _lock = InstallLock::acquire(&layout)?; + transaction::recover(system_root, &layout) +} + +pub(crate) fn validate_payload( + root: &Path, + artifacts: &[Artifact], + current_template: bool, +) -> Result<(), String> { + let scoped = artifacts + .iter() + .filter_map(|artifact| { + artifact + .path + .strip_prefix("bootstrap/") + .map(|path| Artifact { + path: path.into(), + bytes: artifact.bytes, + sha256: artifact.sha256.clone(), + }) + }) + .collect::<Vec<_>>(); + validate_files(&root.join("bootstrap"), &scoped, current_template) +} + +fn validate_files(root: &Path, artifacts: &[Artifact], new_payload: bool) -> Result<(), String> { + let bytes = artifacts + .iter() + .try_fold(0u64, |total, artifact| total.checked_add(artifact.bytes)) + .ok_or("bootstrap size overflow")?; + if artifacts.len() > 64 || bytes > 64 * 1024 * 1024 { + return Err("bootstrap closure exceeds its 64-file/64-MiB budget".into()); + } + for artifact in artifacts { + if !matches!( + artifact.path.as_str(), + "dever" | "deverd" | "deverd.service" | "dever-sandbox" + ) && !artifact.path.starts_with("lib/") + { + return Err("bootstrap has an unsupported artifact".into()); + } + let path = super::runtime_pack::checked_path(root, &artifact.path, true)?; + let bytes = read_bounded(&path, artifact.bytes)?; + if bytes.len() as u64 != artifact.bytes || digest(&bytes) != artifact.sha256 { + return Err(format!( + "bootstrap artifact '{}' differs from its signature", + artifact.path + )); + } + } + if !artifacts + .iter() + .any(|artifact| artifact.path == "deverd.service") + || new_payload && read_bounded(&root.join("deverd.service"), 8192)? != SERVICE.as_bytes() + { + return Err( + "bootstrap service definition is absent or differs from the supported contract".into(), + ); + } + if new_payload + && (!artifacts + .iter() + .any(|artifact| artifact.path == "dever-sandbox") + || read_bounded(&root.join("dever-sandbox"), 8192)? + != dever_sandbox::APPARMOR_PROFILE.as_bytes()) + { + return Err( + "bootstrap AppArmor profile is absent or differs from the supported contract".into(), + ); + } + super::core_libraries::validate_entries(root, artifacts, &["dever", "deverd"]) +} + +pub(super) fn health_check(path: &Path, flag: &str) -> Result<(), String> { + if !service::run( + std::process::Command::new(path).arg(flag), + "health-check installed executable", + )? { + return Err(format!( + "installed executable '{}' failed its health check", + path.display() + )); + } + Ok(()) +} + +pub(super) fn require_no_pending(layout: &Layout) -> Result<(), String> { + if fs::symlink_metadata(layout.state().join("bootstrap-install.json")).is_ok() { + return Err( + "bootstrap installation is incomplete; rerun the explicit installer to recover it" + .into(), + ); + } + Ok(()) +} + +fn validate_bin(root: &Path, key: &[u8]) -> Result<Vec<u8>, String> { + trusted_directory(root)?; + let manifest = MachineManager::signed_manifest(root, key)?; + let artifacts = manifest + .artifacts + .iter() + .filter_map(|artifact| { + artifact + .path + .strip_prefix("bootstrap/") + .map(|path| Artifact { + path: path.into(), + bytes: artifact.bytes, + sha256: artifact.sha256.clone(), + }) + }) + .collect::<Vec<_>>(); + validate_files(root, &artifacts, false)?; + let expected = artifacts + .iter() + .map(|artifact| artifact.path.as_str()) + .chain(["manifest.json", "manifest.sig"]) + .collect::<std::collections::BTreeSet<_>>(); + let found = transaction::files(root)?; + if found + .iter() + .map(String::as_str) + .collect::<std::collections::BTreeSet<_>>() + != expected + { + return Err("installed bootstrap contains unsigned files".into()); + } + read_bounded(&root.join("deverd.service"), 8192) +} + +fn pin_key(layout: &Layout, key: &[u8]) -> Result<(), String> { + let path = layout.state().join("trusted-release-key"); + let next = layout.state().join(".trusted-release-key.next"); + if fs::symlink_metadata(&next).is_ok() { + trusted_file(&next)?; + if fs::metadata(&next) + .map_err(|error| error.to_string())? + .len() + > 65 + { + return Err("temporary release key exceeds its fixed size".into()); + } + fs::remove_file(&next).map_err(|error| error.to_string())?; + release::sync_directory(&layout.state()).map_err(|error| error.to_string())?; + } + if fs::symlink_metadata(&path).is_ok() { + trusted_file(&path)?; + if release::decode_hex_file(&path, 32)? != key { + return Err("installer key differs from the machine's pinned release key".into()); + } + return Ok(()); + } + write_new(&next, format!("{}\n", hex(key)).as_bytes(), 0o644)?; + rustix::fs::renameat_with( + rustix::fs::CWD, + &next, + rustix::fs::CWD, + &path, + rustix::fs::RenameFlags::NOREPLACE, + ) + .map_err(|error| format!("cannot publish initial release key: {error}"))?; + release::sync_directory(&layout.state()).map_err(|error| error.to_string()) +} + +fn absolute_root(path: &Path) -> Result<(), String> { + if !path.is_absolute() + || path + .components() + .any(|part| matches!(part, Component::ParentDir)) + { + return Err("installer roots must be absolute paths without '..'".into()); + } + Ok(()) +} + +fn require_root() -> Result<(), String> { + if rustix::process::geteuid().as_raw() != 0 { + return Err("machine bootstrap installation requires root".into()); + } + Ok(()) +} + +fn absolute_input(root: &Path, path: &Path) -> Result<PathBuf, String> { + let path = if path.is_absolute() { + path.to_path_buf() + } else { + root.join(path) + }; + if path + .components() + .any(|part| matches!(part, Component::ParentDir)) + { + return Err("installer paths cannot contain '..'".into()); + } + Ok(path) +} + +fn trusted_directory(path: &Path) -> Result<(), String> { + trusted_path(path, true) +} + +fn trusted_file(path: &Path) -> Result<(), String> { + trusted_path(path, false) +} + +fn trusted_path(path: &Path, directory: bool) -> Result<(), String> { + let metadata = fs::symlink_metadata(path).map_err(|error| { + format!( + "cannot inspect installer path '{}': {error}", + path.display() + ) + })?; + if metadata.file_type().is_symlink() + || metadata.uid() != 0 + || metadata.mode() & 0o022 != 0 + || directory && !metadata.is_dir() + || !directory && (!metadata.is_file() || metadata.nlink() != 1) + { + return Err(format!( + "installer path '{}' must be a real root-owned {} without shared write permission", + path.display(), + if directory { "directory" } else { "file" } + )); + } + for ancestor in path.ancestors().skip(1) { + if ancestor.as_os_str().is_empty() { + continue; + } + let parent = fs::symlink_metadata(ancestor).map_err(|error| error.to_string())?; + // A root-owned sticky temporary ancestor is safe for an owned fixture; + // the actual installation root itself may never be shared-writable. + if !parent.is_dir() + || parent.file_type().is_symlink() + || parent.uid() != 0 + || parent.mode() & 0o022 != 0 && parent.mode() & 0o1000 == 0 + { + return Err("installer path crosses an untrusted ancestor".into()); + } + } + Ok(()) +} + +fn create_directory(root: &Path, relative: &str) -> Result<PathBuf, String> { + let mut path = root.to_owned(); + for part in Path::new(relative).components() { + let Component::Normal(part) = part else { + return Err("invalid installer directory".into()); + }; + path.push(part); + match fs::create_dir(&path) { + Ok(()) => { + fs::set_permissions(&path, fs::Permissions::from_mode(0o755)) + .map_err(|error| error.to_string())?; + release::sync_directory(&path).map_err(|error| error.to_string())?; + release::sync_directory(path.parent().ok_or("installer directory has no parent")?) + .map_err(|error| error.to_string())?; + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error.to_string()), + } + trusted_directory(&path)?; + } + Ok(path) +} + +fn read_bounded(path: &Path, maximum: u64) -> Result<Vec<u8>, String> { + let mut bytes = Vec::new(); + File::open(path) + .and_then(|file| file.take(maximum.saturating_add(1)).read_to_end(&mut bytes)) + .map_err(|error| error.to_string())?; + if bytes.len() as u64 > maximum { + return Err("installer input exceeds its size bound".into()); + } + Ok(bytes) +} + +fn write_new(path: &Path, bytes: &[u8], mode: u32) -> Result<(), String> { + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(path) + .map_err(|error| error.to_string())?; + file.write_all(bytes) + .and_then(|()| file.sync_all()) + .map_err(|error| error.to_string())?; + fs::set_permissions(path, fs::Permissions::from_mode(mode)) + .map_err(|error| error.to_string())?; + file.sync_all().map_err(|error| error.to_string())?; + release::sync_directory(path.parent().ok_or("installer file has no parent")?) + .map_err(|error| error.to_string()) +} + +fn digest(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) +} +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} diff --git a/crates/dever-cli/src/toolchain/bootstrap/sandbox.rs b/crates/dever-cli/src/toolchain/bootstrap/sandbox.rs new file mode 100644 index 0000000..a786427 --- /dev/null +++ b/crates/dever-cli/src/toolchain/bootstrap/sandbox.rs @@ -0,0 +1,138 @@ +//! Preserve signed content-addressed helpers across compiler upgrades/removal. +use std::fs; +use std::os::unix::fs::PermissionsExt; +use std::path::Path; + +use super::{ + MachineManager, ReleaseManifest, digest, read_bounded, release, trusted_directory, trusted_file, +}; + +const MAX_HELPERS: usize = 24; +const MAX_HELPER_BYTES: u64 = 32 * 1024 * 1024; + +fn artifact_path() -> String { + format!( + "sandbox/{}/bin/bwrap", + crate::toolchain::platform_identity() + ) +} + +pub(super) fn stage( + installed: &Path, + next: &Path, + source: &Path, + manifest: &ReleaseManifest, + key: &[u8], +) -> Result<(), String> { + fs::create_dir(next).map_err(|error| error.to_string())?; + fs::set_permissions(next, fs::Permissions::from_mode(0o755)) + .map_err(|error| error.to_string())?; + let mut hashes = std::collections::BTreeSet::new(); + if installed.exists() { + trusted_directory(installed)?; + for entry in fs::read_dir(installed).map_err(|error| error.to_string())? { + let entry = entry.map_err(|error| error.to_string())?; + let hash = entry + .file_name() + .into_string() + .map_err(|_| "sandbox helper identity is not UTF-8")?; + if hashes.len() >= MAX_HELPERS || !valid_hash(&hash) { + return Err("installed sandbox helper store has unsupported entries or exceeds its version bound".into()); + } + validate_installed(&entry.path(), &hash, key)?; + let destination = next.join(&hash); + copy_helper(&entry.path().join("bwrap"), &entry.path(), &destination)?; + hashes.insert(hash); + } + } + if let Some(artifact) = manifest + .artifacts + .iter() + .find(|artifact| artifact.path == artifact_path()) + && !hashes.contains(&artifact.sha256) + { + if hashes.len() >= MAX_HELPERS { + return Err("sandbox helper version store is full; retained applications must be retired before removing old helpers".into()); + } + let bytes = read_bounded(&source.join(&artifact.path), MAX_HELPER_BYTES)?; + if bytes.len() as u64 != artifact.bytes + || dever_sandbox::validate_bwrap(&bytes)? != artifact.sha256 + { + return Err("sandbox helper differs from its signed release".into()); + } + let destination = next.join(&artifact.sha256); + copy_helper(&source.join(&artifact.path), source, &destination)?; + validate_installed(&destination, &artifact.sha256, key)?; + } + release::sync_tree(next) +} + +fn copy_helper(helper: &Path, signed_release: &Path, destination: &Path) -> Result<(), String> { + fs::create_dir(destination).map_err(|error| error.to_string())?; + fs::set_permissions(destination, fs::Permissions::from_mode(0o755)) + .map_err(|error| error.to_string())?; + release::copy_file(helper, &destination.join("bwrap"), true)?; + for name in ["manifest.json", "manifest.sig"] { + release::copy_file(&signed_release.join(name), &destination.join(name), false)?; + } + Ok(()) +} + +fn valid_hash(hash: &str) -> bool { + hash.len() == 64 + && hash + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +fn validate_installed(root: &Path, hash: &str, key: &[u8]) -> Result<(), String> { + use std::os::unix::fs::MetadataExt; + trusted_directory(root)?; + let names = fs::read_dir(root) + .map_err(|error| error.to_string())? + .map(|entry| entry.map(|entry| entry.file_name())) + .collect::<Result<std::collections::BTreeSet<_>, _>>() + .map_err(|error| error.to_string())?; + let expected = ["bwrap", "manifest.json", "manifest.sig"] + .map(std::ffi::OsString::from) + .into_iter() + .collect(); + if names != expected { + return Err("installed sandbox helper contains unsigned entries".into()); + } + for name in ["bwrap", "manifest.json", "manifest.sig"] { + trusted_file(&root.join(name))?; + } + let helper = root.join("bwrap"); + if fs::metadata(&helper) + .map_err(|error| error.to_string())? + .mode() + & 0o6111 + != 0o111 + { + return Err("installed sandbox helper has unsupported executable permissions".into()); + } + match rustix::fs::lgetxattr(&helper, "security.capability", &mut [0u8; 64]) { + Err(rustix::io::Errno::NODATA | rustix::io::Errno::OPNOTSUPP) => {} + Ok(_) => return Err("installed sandbox helper must not have file capabilities".into()), + Err(error) => { + return Err(format!( + "cannot inspect installed sandbox helper capabilities: {error}" + )); + } + } + let manifest = MachineManager::signed_manifest(root, key)?; + let artifact = manifest + .artifacts + .iter() + .find(|artifact| artifact.path == artifact_path()) + .ok_or("installed sandbox helper has no signed artifact")?; + let bytes = read_bounded(&helper, MAX_HELPER_BYTES)?; + if artifact.sha256 != hash || artifact.bytes != bytes.len() as u64 || digest(&bytes) != hash { + return Err( + "installed sandbox helper differs from its signature or directory identity".into(), + ); + } + dever_sandbox::validate_bwrap(&bytes)?; + Ok(()) +} diff --git a/crates/dever-cli/src/toolchain/bootstrap/service.rs b/crates/dever-cli/src/toolchain/bootstrap/service.rs new file mode 100644 index 0000000..a1d1a14 --- /dev/null +++ b/crates/dever-cli/src/toolchain/bootstrap/service.rs @@ -0,0 +1,174 @@ +//! Bounded Linux platform actions. Image installation never calls systemctl. +use std::io::Read; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant}; + +struct OwnedChild(Child); +impl Drop for OwnedChild { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +pub(super) fn run(command: &mut Command, operation: &str) -> Result<bool, String> { + let mut child = OwnedChild( + command + .env_clear() + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .map_err(|error| format!("cannot {operation}: {error}"))?, + ); + let deadline = Instant::now() + Duration::from_secs(30); + loop { + if let Some(status) = child + .0 + .try_wait() + .map_err(|error| format!("cannot wait for {operation}: {error}"))? + { + return Ok(status.success()); + } + if Instant::now() >= deadline { + return Err(format!("{operation} exceeded 30 seconds")); + } + std::thread::sleep(Duration::from_millis(20)); + } +} + +pub(super) fn active() -> Result<bool, String> { + if !loaded()? { + return Ok(false); + } + match property("ActiveState")?.as_str() { + "inactive" | "failed" => Ok(false), + "active" | "activating" | "deactivating" | "reloading" => Ok(true), + _ => Err("Dever service has an unsupported active state".into()), + } +} + +pub(super) fn loaded() -> Result<bool, String> { + match property("LoadState")?.as_str() { + "loaded" => Ok(true), + "not-found" => Ok(false), + _ => Err("Dever service has an unsupported load state".into()), + } +} + +fn property(name: &str) -> Result<String, String> { + let mut command = Command::new("/usr/bin/systemctl"); + command + .env_clear() + .args(["show", "--value", "--property", name, "deverd.service"]) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()); + let mut child = OwnedChild( + command + .spawn() + .map_err(|error| format!("cannot inspect Dever service: {error}"))?, + ); + let deadline = Instant::now() + Duration::from_secs(30); + let status = loop { + if let Some(status) = child.0.try_wait().map_err(|error| error.to_string())? { + break status; + } + if Instant::now() >= deadline { + return Err("service state inspection exceeded 30 seconds".into()); + } + std::thread::sleep(Duration::from_millis(20)); + }; + let mut bytes = Vec::new(); + child + .0 + .stdout + .take() + .ok_or("service state output missing")? + .take(4097) + .read_to_end(&mut bytes) + .map_err(|error| error.to_string())?; + if bytes.len() > 4096 { + return Err("service state output exceeded its bound".into()); + } + let value = String::from_utf8(bytes) + .map_err(|_| "service state output is not UTF-8")? + .trim() + .to_owned(); + if !status.success() + && !(status.code() == Some(1) && name == "LoadState" && value == "not-found") + { + return Err(format!("cannot inspect Dever service {name}: {status}")); + } + Ok(value) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RecoveryAction { + Stop, + Reload, + Restart, +} + +/// Files are restored between these two action lists. A service that never +/// existed is not stopped after its new unit has already been removed. +pub fn recovery_actions( + start_requested: bool, + was_active: bool, + currently_loaded: bool, +) -> (Vec<RecoveryAction>, Vec<RecoveryAction>) { + if !start_requested { + return (vec![], vec![]); + } + let before = if currently_loaded { + vec![RecoveryAction::Stop] + } else { + vec![] + }; + let mut after = vec![RecoveryAction::Reload]; + if was_active { + after.push(RecoveryAction::Restart); + } + (before, after) +} + +pub(super) fn execute(actions: &[RecoveryAction]) -> Result<(), String> { + for action in actions { + let arguments = match action { + RecoveryAction::Reload => vec!["daemon-reload"], + RecoveryAction::Stop => vec!["stop", "deverd.service"], + RecoveryAction::Restart => vec!["restart", "deverd.service"], + }; + if !run( + Command::new("/usr/bin/systemctl").args(&arguments), + "restore Dever service", + )? { + return Err(format!("systemctl {} failed", arguments.join(" "))); + } + } + Ok(()) +} + +pub(super) fn apply(restart: bool) -> Result<(), String> { + execute(&[ + RecoveryAction::Reload, + if restart { + RecoveryAction::Restart + } else { + RecoveryAction::Stop + }, + ]) +} + +pub(super) fn ready(layout: &crate::toolchain::Layout) -> Result<(), String> { + let deadline = Instant::now() + Duration::from_secs(15); + loop { + match crate::toolchain::cache_status(layout) { + Ok(_) => return Ok(()), + Err(error) if Instant::now() >= deadline => { + return Err(format!("Dever service did not become ready: {error}")); + } + Err(_) => std::thread::sleep(Duration::from_millis(50)), + } + } +} diff --git a/crates/dever-cli/src/toolchain/bootstrap/transaction.rs b/crates/dever-cli/src/toolchain/bootstrap/transaction.rs new file mode 100644 index 0000000..38ee744 --- /dev/null +++ b/crates/dever-cli/src/toolchain/bootstrap/transaction.rs @@ -0,0 +1,522 @@ +//! One bounded journal for the fixed bootstrap destinations. It contains +//! identities, never caller-selected recovery paths or arbitrary commands. +use std::collections::BTreeSet; +use std::fs; +use std::os::unix::fs::{MetadataExt, symlink}; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use super::{ + Layout, ReleaseManifest, create_directory, digest, read_bounded, release, service, + trusted_directory, trusted_file, validate_bin, write_new, +}; + +const JOURNAL: &str = "bootstrap-install.json"; +const JOURNAL_NEXT: &str = ".bootstrap-install.next"; +const MAX_TREE_BYTES: u64 = 80 * 1024 * 1024; + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +enum Target { + Bin, + Sandbox, + Apparmor, + Service, + Entry, + Enable, + Active, +} + +const TARGETS: [Target; 7] = [ + Target::Sandbox, + Target::Apparmor, + Target::Bin, + Target::Service, + Target::Entry, + Target::Enable, + Target::Active, +]; + +impl Target { + fn relative(self) -> &'static str { + match self { + Self::Bin => "opt/dever/bin", + Self::Sandbox => "opt/dever/sandbox", + Self::Apparmor => "etc/apparmor.d/dever-sandbox", + Self::Service => "etc/systemd/system/deverd.service", + Self::Entry => "usr/local/bin/dever", + Self::Enable => "etc/systemd/system/multi-user.target.wants/deverd.service", + Self::Active => "opt/dever/state/active-version", + } + } + fn path(self, root: &Path) -> PathBuf { + root.join(self.relative()) + } + fn next(self, root: &Path) -> PathBuf { + let path = self.path(root); + path.with_file_name(format!( + ".dever-bootstrap-{}.next", + path.file_name().unwrap().to_string_lossy() + )) + } + fn link(self) -> Option<&'static str> { + match self { + Self::Entry => Some("/opt/dever/bin/dever"), + Self::Enable => Some("../deverd.service"), + _ => None, + } + } +} + +#[derive(Clone, Copy, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +enum Phase { + Preparing, + Prepared, + Committed, +} + +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct Change { + target: Target, + before: Option<String>, + after: Option<String>, +} + +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct Journal { + format: String, + phase: Phase, + service_was_active: Option<bool>, + service_start_requested: bool, + changes: Vec<Change>, +} + +pub(super) fn install( + root: &Path, + layout: &Layout, + source: &Path, + manifest: &ReleaseManifest, + key: &[u8], + activate_service: bool, +) -> Result<(), String> { + let previous_service = if fs::read_dir(layout.bin()) + .map_err(|error| error.to_string())? + .next() + .is_some() + { + Some(validate_bin(&layout.bin(), key)?) + } else { + None + }; + for target in TARGETS { + let parent = Path::new(target.relative()) + .parent() + .unwrap() + .to_str() + .unwrap(); + create_directory(root, parent)?; + if fs::symlink_metadata(target.next(root)).is_ok() { + return Err("unrecorded bootstrap staging destination already exists".into()); + } + if let Some(link) = target.link() { + if let Ok(metadata) = fs::symlink_metadata(target.path(root)) + && (previous_service.is_none() + || metadata.uid() != 0 + || !metadata.file_type().is_symlink() + || fs::read_link(target.path(root)).map_err(|error| error.to_string())? + != Path::new(link)) + { + return Err( + "bootstrap cannot replace an unrelated public entry or service link".into(), + ); + } + } else if target == Target::Apparmor && target.path(root).exists() { + trusted_file(&target.path(root))?; + let installed_profile = layout.bin().join("dever-sandbox"); + if previous_service.is_none() + || !installed_profile.exists() + || read_bounded(&target.path(root), 8192)? + != read_bounded(&installed_profile, 8192)? + { + return Err("bootstrap cannot replace an unrelated AppArmor profile".into()); + } + } else if target == Target::Service && target.path(root).exists() { + trusted_file(&target.path(root))?; + if previous_service.as_deref() + != Some(read_bounded(&target.path(root), 8192)?.as_slice()) + { + return Err( + "bootstrap cannot replace an unrelated system service definition".into(), + ); + } + } + } + let changes = TARGETS + .into_iter() + .map(|target| { + Ok(Change { + target, + before: identity(&target.path(root), target.link())?, + after: None, + }) + }) + .collect::<Result<Vec<_>, String>>()?; + let mut journal = Journal { + format: "dever-bootstrap-transaction-v1".into(), + phase: Phase::Preparing, + service_was_active: if activate_service { + Some(service::active()?) + } else { + None + }, + service_start_requested: false, + changes, + }; + save(layout, &journal)?; + let result = (|| { + stage(root, source, manifest, key)?; + for change in &mut journal.changes { + change.after = identity(&change.target.next(root), change.target.link())?; + if change.after.is_none() { + return Err("bootstrap staging is incomplete".into()); + } + } + journal.phase = Phase::Prepared; + save(layout, &journal)?; + for index in 0..journal.changes.len() { + if journal.changes[index].target == Target::Active && activate_service { + journal.service_start_requested = true; + save(layout, &journal)?; + service::apply(true)?; + service::ready(layout)?; + } + publish(root, &journal.changes[index])?; + } + journal.phase = Phase::Committed; + save(layout, &journal) + })(); + if let Err(primary) = result { + // The journal is still Prepared unless its commit was persisted. Read + // the durable phase rather than trusting the in-memory assignment. + return match recover(root, layout) { + Ok(()) => Err(primary), + Err(rollback) => Err(format!("{primary}; bootstrap recovery failed: {rollback}")), + }; + } + if let Err(error) = recover(root, layout) { + // Publication is already committed. Never roll it back because an old + // staging copy could not be removed; preserve the journal for retry. + return Err(format!( + "bootstrap publication committed, but cleanup requires recovery: {error}" + )); + } + Ok(()) +} + +fn stage(root: &Path, source: &Path, manifest: &ReleaseManifest, key: &[u8]) -> Result<(), String> { + super::sandbox::stage( + &Target::Sandbox.path(root), + &Target::Sandbox.next(root), + source, + manifest, + key, + )?; + write_new( + &Target::Apparmor.next(root), + dever_sandbox::APPARMOR_PROFILE.as_bytes(), + 0o644, + )?; + let bin = Target::Bin.next(root); + fs::create_dir(&bin).map_err(|error| error.to_string())?; + for artifact in &manifest.artifacts { + let Some(relative) = artifact.path.strip_prefix("bootstrap/") else { + continue; + }; + release::copy_file( + &source.join(&artifact.path), + &bin.join(relative), + matches!(relative, "dever" | "deverd"), + )?; + } + for file in ["manifest.json", "manifest.sig"] { + release::copy_file(&source.join(file), &bin.join(file), false)?; + } + validate_bin(&bin, key)?; + for name in ["dever", "deverd"] { + if !service::run( + Command::new(bin.join(name)).arg("--dever-bootstrap-health"), + "health-check bootstrap", + )? { + return Err(format!("bootstrap {name} failed its health check")); + } + } + write_new( + &Target::Service.next(root), + super::SERVICE.as_bytes(), + 0o644, + )?; + for target in [Target::Entry, Target::Enable] { + symlink(target.link().unwrap(), target.next(root)).map_err(|error| error.to_string())?; + sync_parent(&target.next(root))?; + } + let active = Target::Active.path(root); + let next = Target::Active.next(root); + if active.exists() { + release::copy_file(&active, &next, false)?; + } + release::append_active_version(&next, &manifest.version).map_err(|error| error.to_string())?; + release::sync_tree(&bin)?; + sync_parent(&bin) +} + +fn publish(root: &Path, change: &Change) -> Result<(), String> { + if change.target == Target::Apparmor && root == Path::new("/") { + dever_sandbox::validate_host_policy()?; + } + let path = change.target.path(root); + let next = change.target.next(root); + if identity(&path, change.target.link())? != change.before + || identity(&next, change.target.link())? != change.after + { + return Err("bootstrap destination changed during installation".into()); + } + if change.before == change.after { + return Ok(()); + } + if change.before.is_some() { + exchange(&path, &next)?; + } else { + rustix::fs::renameat_with( + rustix::fs::CWD, + &next, + rustix::fs::CWD, + &path, + rustix::fs::RenameFlags::NOREPLACE, + ) + .map_err(|error| error.to_string())?; + } + sync_parent(&path) +} + +pub(super) fn recover(root: &Path, layout: &Layout) -> Result<(), String> { + let temporary = layout.state().join(JOURNAL_NEXT); + if fs::symlink_metadata(&temporary).is_ok() { + trusted_file(&temporary)?; + fs::remove_file(&temporary).map_err(|error| error.to_string())?; + } + let path = layout.state().join(JOURNAL); + let bytes = match fs::symlink_metadata(&path) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error.to_string()), + Ok(_) => { + trusted_file(&path)?; + read_bounded(&path, 64 * 1024)? + } + }; + let journal: Journal = serde_json::from_slice(&bytes) + .map_err(|error| format!("invalid bootstrap recovery journal: {error}"))?; + if journal.format != "dever-bootstrap-transaction-v1" + || journal + .changes + .iter() + .map(|change| change.target) + .collect::<Vec<_>>() + != TARGETS + || journal.service_was_active.is_some() && root != Path::new("/") + || journal.service_start_requested + && (journal.service_was_active.is_none() || journal.phase == Phase::Preparing) + || journal.changes.iter().any(|change| { + change.before.iter().chain(&change.after).any(|hash| { + hash.len() != 64 + || !hash + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + }) + }) + || journal.phase != Phase::Preparing + && journal.changes.iter().any(|change| change.after.is_none()) + { + return Err("bootstrap recovery journal has unsupported targets or identities".into()); + } + // Validate all affected parents before touching any target. The journal can + // never turn an image-root path into a write through a substituted symlink. + for change in &journal.changes { + trusted_directory(change.target.path(root).parent().unwrap())?; + } + if journal.phase == Phase::Committed { + for change in &journal.changes { + if identity(&change.target.path(root), change.target.link())? != change.after { + return Err("committed bootstrap destination changed before cleanup".into()); + } + } + } else { + let (before_files, after_files) = if let Some(was_active) = journal.service_was_active { + service::recovery_actions( + journal.service_start_requested, + was_active, + journal.service_start_requested && service::loaded()?, + ) + } else { + (vec![], vec![]) + }; + service::execute(&before_files)?; + for change in journal.changes.iter().rev() { + rollback(root, change, journal.phase)?; + } + service::execute(&after_files)?; + } + for change in &journal.changes { + remove_owned(&change.target.next(root), change.target.link())?; + } + fs::remove_file(&path).map_err(|error| error.to_string())?; + sync_parent(&path) +} + +fn rollback(root: &Path, change: &Change, phase: Phase) -> Result<(), String> { + let path = change.target.path(root); + let next = change.target.next(root); + let current = identity(&path, change.target.link())?; + if current == change.before { + return Ok(()); + } + if phase != Phase::Prepared || current != change.after { + return Err("bootstrap recovery refuses a destination outside this transaction".into()); + } + if let Some(before) = &change.before { + if identity(&next, change.target.link())?.as_ref() != Some(before) { + return Err("bootstrap rollback copy is missing or changed".into()); + } + exchange(&path, &next)?; + } else { + remove_owned(&path, change.target.link())?; + } + sync_parent(&path) +} + +fn save(layout: &Layout, journal: &Journal) -> Result<(), String> { + let next = layout.state().join(JOURNAL_NEXT); + write_new( + &next, + &serde_json::to_vec(journal).map_err(|error| error.to_string())?, + 0o600, + )?; + fs::rename(&next, layout.state().join(JOURNAL)).map_err(|error| error.to_string())?; + release::sync_directory(&layout.state()).map_err(|error| error.to_string()) +} + +fn exchange(path: &Path, next: &Path) -> Result<(), String> { + rustix::fs::renameat_with( + rustix::fs::CWD, + path, + rustix::fs::CWD, + next, + rustix::fs::RenameFlags::EXCHANGE, + ) + .map_err(|error| format!("atomic bootstrap exchange failed: {error}")) +} + +fn sync_parent(path: &Path) -> Result<(), String> { + release::sync_directory(path.parent().ok_or("bootstrap destination has no parent")?) + .map_err(|error| error.to_string()) +} + +fn remove_owned(path: &Path, link: Option<&str>) -> Result<(), String> { + let Some(_) = identity(path, link)? else { + return Ok(()); + }; + if fs::symlink_metadata(path) + .map_err(|error| error.to_string())? + .is_dir() + { + fs::remove_dir_all(path).map_err(|error| error.to_string())?; + } else { + fs::remove_file(path).map_err(|error| error.to_string())?; + } + sync_parent(path) +} + +fn identity(path: &Path, link: Option<&str>) -> Result<Option<String>, String> { + let metadata = match fs::symlink_metadata(path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error.to_string()), + }; + if let Some(link) = link { + if !metadata.file_type().is_symlink() + || metadata.uid() != 0 + || fs::read_link(path).map_err(|error| error.to_string())? != Path::new(link) + { + return Err("bootstrap link has an unsupported owner or target".into()); + } + return Ok(Some(digest(format!("link\0{link}").as_bytes()))); + } + let mut tree = Tree::default(); + tree.visit(path, path, 0)?; + Ok(Some(format!("{:x}", tree.hash.finalize()))) +} + +pub(super) fn files(root: &Path) -> Result<Vec<String>, String> { + let mut tree = Tree::default(); + tree.visit(root, root, 0)?; + Ok(tree.files.into_iter().collect()) +} + +#[derive(Default)] +struct Tree { + hash: Sha256, + bytes: u64, + entries: usize, + files: BTreeSet<String>, +} + +impl Tree { + fn visit(&mut self, root: &Path, path: &Path, depth: usize) -> Result<(), String> { + self.entries += 1; + if self.entries > 128 || depth > 4 { + return Err("bootstrap tree exceeds its entry/depth bound".into()); + } + let metadata = fs::symlink_metadata(path).map_err(|error| error.to_string())?; + let name = path + .strip_prefix(root) + .unwrap() + .to_str() + .ok_or("bootstrap path is not UTF-8")?; + self.hash.update((name.len() as u64).to_le_bytes()); + self.hash.update(name.as_bytes()); + self.hash.update((metadata.mode() & 0o777).to_le_bytes()); + if metadata.is_dir() && !metadata.file_type().is_symlink() { + trusted_directory(path)?; + self.hash.update(b"directory"); + let mut entries = fs::read_dir(path) + .map_err(|error| error.to_string())? + .map(|entry| entry.map(|entry| entry.path())) + .collect::<Result<Vec<_>, _>>() + .map_err(|error| error.to_string())?; + entries.sort(); + for entry in entries { + self.visit(root, &entry, depth + 1)?; + } + } else { + trusted_file(path)?; + self.bytes = self + .bytes + .checked_add(metadata.len()) + .ok_or("bootstrap tree size overflow")?; + if self.bytes > MAX_TREE_BYTES { + return Err("bootstrap tree exceeds its byte bound".into()); + } + let bytes = read_bounded(path, metadata.len())?; + self.hash.update(b"file"); + self.hash.update((bytes.len() as u64).to_le_bytes()); + self.hash.update(bytes); + self.files.insert(name.into()); + } + Ok(()) + } +} diff --git a/crates/dever-cli/src/toolchain/cache.rs b/crates/dever-cli/src/toolchain/cache.rs new file mode 100644 index 0000000..00c1be9 --- /dev/null +++ b/crates/dever-cli/src/toolchain/cache.rs @@ -0,0 +1,778 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::fs::{self, File, OpenOptions}; +use std::io::{self, Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use super::release::{Layout, Version, hex}; + +static NEXT_CACHE_OPERATION: AtomicU64 = AtomicU64::new(0); + +#[derive(Clone, Debug)] +pub struct BuildIdentity<'a> { + pub version: &'a Version, + pub compiler: &'a [u8], + pub runtime: &'a [u8], + pub target: &'a str, + pub generated: &'a [u8], + pub options: &'a [u8], + pub environment: &'a [u8], +} + +impl BuildIdentity<'_> { + pub fn key(&self) -> String { + let mut digest = Sha256::new(); + digest.update(b"dever-machine-cache-v1\0"); + for value in [ + self.version.as_str().as_bytes(), + self.compiler, + self.runtime, + self.target.as_bytes(), + self.generated, + self.options, + self.environment, + ] { + digest.update((value.len() as u64).to_le_bytes()); + digest.update(value); + } + hex(&digest.finalize()) + } +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct CacheEntry { + pub format: String, + pub key: String, + pub version: Version, + pub target: String, + pub bytes: u64, + pub sha256: String, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)] +pub struct CacheStatus { + #[serde(default)] + pub verified: bool, + pub entries: u64, + pub bytes: u64, + pub versions: BTreeMap<String, u64>, + pub corrupt_entries: u64, + pub staging_entries: u64, + #[serde(default)] + pub artifact_entries: u64, + #[serde(default)] + pub artifact_bytes: u64, + #[serde(default)] + pub corrupt_artifacts: u64, +} + +/// The service accepts content identities, never caller-selected cache paths. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ArtifactReceipt { + pub sha256: String, + pub bytes: u64, +} + +pub const MAX_ARTIFACT_BYTES: u64 = 64 * 1024 * 1024; +pub(super) const MAX_COMPILED_BYTES: u64 = 320 * 1024 * 1024; +const MAX_ARTIFACT_CACHE_BYTES: u64 = 2 * 1024 * 1024 * 1024; +const MAX_ARTIFACT_ENTRIES: usize = 4096; + +impl ArtifactReceipt { + pub fn validate(&self) -> Result<(), String> { + validate_key(&self.sha256)?; + if self.bytes > MAX_ARTIFACT_BYTES { + return Err("shared artifact exceeds the 64 MiB transfer limit".into()); + } + Ok(()) + } + + pub(super) fn validate_compiled(&self) -> Result<(), String> { + validate_key(&self.sha256)?; + if self.bytes == 0 || self.bytes > MAX_COMPILED_BYTES { + return Err("compiled output must be between 1 byte and 320 MiB".into()); + } + Ok(()) + } +} + +pub struct StoredArtifact { + pub receipt: ArtifactReceipt, + pub file: File, + _operation: CacheOperation, +} + +/// Filesystem engine owned by the privileged cache process. It deliberately has +/// no transport or caller-authentication semantics; project processes must use +/// the authenticated `deverd` IPC client rather than constructing this store directly. +pub struct CacheStore { + root: PathBuf, +} + +impl CacheStore { + pub fn new(layout: &Layout) -> Self { + Self { + root: layout.native_cache(), + } + } + + /// Hold across input staging, compilation, publication and response transfer. + pub(crate) fn operation(&self) -> Result<CacheOperation, String> { + CacheOperation::acquire(&self.root) + } + + pub(crate) fn compilation_directory(&self) -> Result<CompilationDirectory, String> { + let root = self.root.join("staging"); + create_directory(&root)?; + let path = root.join(format!( + "compile-{}-{}", + std::process::id(), + NEXT_CACHE_OPERATION.fetch_add(1, Ordering::Relaxed) + )); + let mut builder = fs::DirBuilder::new(); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + builder + .create(&path) + .map_err(|error| format!("cannot stage compilation: {error}"))?; + Ok(CompilationDirectory(path)) + } + + /// Hash while receiving into an owned staging file. Only verified complete + /// bytes are published, and concurrent identical uploads share one entry. + pub fn publish_artifact( + &self, + expected: &ArtifactReceipt, + input: &mut impl Read, + ) -> Result<ArtifactReceipt, String> { + expected.validate()?; + let _operation = CacheOperation::acquire(&self.root)?; + let staging_root = self.root.join("staging"); + create_directory(&staging_root)?; + let staging = staging_root.join(format!( + "artifact-{}-{}", + std::process::id(), + NEXT_CACHE_OPERATION.fetch_add(1, Ordering::Relaxed) + )); + let result = (|| { + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options + .open(&staging) + .map_err(|error| format!("cannot stage shared artifact: {error}"))?; + let mut digest = Sha256::new(); + let mut remaining = expected.bytes; + let mut buffer = [0_u8; 64 * 1024]; + while remaining > 0 { + let limit = remaining.min(buffer.len() as u64) as usize; + input + .read_exact(&mut buffer[..limit]) + .map_err(|error| format!("incomplete shared artifact upload: {error}"))?; + file.write_all(&buffer[..limit]) + .map_err(|error| format!("cannot stage shared artifact: {error}"))?; + digest.update(&buffer[..limit]); + remaining -= limit as u64; + } + if hex(&digest.finalize()) != expected.sha256 { + return Err("shared artifact upload failed SHA-256 verification".into()); + } + file.sync_all() + .map_err(|error| format!("cannot sync shared artifact: {error}"))?; + drop(file); + let artifacts = self.root.join("artifacts"); + create_directory(&artifacts)?; + let _publication = ArtifactPublication::acquire(&self.root)?; + let destination = artifacts.join(&expected.sha256); + match fs::symlink_metadata(&destination) { + Ok(_) => { + self.read_artifact_file(expected)?; + } + Err(error) if error.kind() == io::ErrorKind::NotFound => { + let (used, count) = directory_entries(&artifacts)?.try_fold( + (0_u64, 0_usize), + |(sum, count), entry| { + let metadata = fs::symlink_metadata(entry.path()) + .map_err(|error| error.to_string())?; + if !metadata.is_file() || metadata.file_type().is_symlink() { + return Err( + "shared artifact cache contains a non-regular file".into() + ); + } + Ok::<_, String>((sum.saturating_add(metadata.len()), count + 1)) + }, + )?; + if count >= MAX_ARTIFACT_ENTRIES + || used.saturating_add(expected.bytes) > MAX_ARTIFACT_CACHE_BYTES + { + return Err("shared artifact cache is full; administrator must run 'dever cache clean'".into()); + } + fs::rename(&staging, &destination) + .map_err(|error| format!("cannot publish shared artifact: {error}"))?; + sync_directory(&artifacts) + .map_err(|error| format!("cannot sync shared artifacts: {error}"))?; + } + Err(error) => return Err(format!("cannot inspect shared artifact: {error}")), + } + Ok(expected.clone()) + })(); + if staging.exists() { + let _ = fs::remove_file(&staging); + } + result + } + + pub fn artifact(&self, expected: &ArtifactReceipt) -> Result<StoredArtifact, String> { + expected.validate()?; + let operation = CacheOperation::acquire(&self.root)?; + let file = self.read_artifact_file(expected)?; + Ok(StoredArtifact { + receipt: expected.clone(), + file, + _operation: operation, + }) + } + + fn read_artifact_file(&self, expected: &ArtifactReceipt) -> Result<File, String> { + let path = self.artifact_path(expected)?; + if sha256_file(&path)? != expected.sha256 { + return Err("shared artifact failed SHA-256 verification".into()); + } + File::open(&path).map_err(|error| format!("cannot read shared artifact: {error}")) + } + + fn artifact_path(&self, expected: &ArtifactReceipt) -> Result<PathBuf, String> { + expected.validate()?; + let directory = self.root.join("artifacts"); + ensure_real_directory(&directory) + .map_err(|error| format!("shared artifact unavailable: {error}"))?; + let path = directory.join(&expected.sha256); + let metadata = fs::symlink_metadata(&path) + .map_err(|error| format!("shared artifact unavailable: {error}"))?; + if !metadata.is_file() + || metadata.file_type().is_symlink() + || metadata.len() != expected.bytes + { + return Err("shared artifact is incomplete or modified".into()); + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let owner = fs::symlink_metadata(&self.root) + .map_err(|error| error.to_string())? + .uid(); + if metadata.uid() != owner || metadata.mode() & 0o077 != 0 { + return Err("shared artifact must be private to the service owner".into()); + } + } + Ok(path) + } + + pub fn publish( + &self, + identity: &BuildIdentity<'_>, + executable: &Path, + ) -> Result<String, String> { + let _operation = CacheOperation::acquire(&self.root)?; + let _publication = ArtifactPublication::acquire(&self.root)?; + let key = identity.key(); + let destination = self.root.join("entries").join(&key); + if destination.exists() { + self.read_entry(&key)?; + return Ok(key); + } + let staging_root = self.root.join("staging"); + create_directory(&staging_root)?; + create_directory(&self.root.join("entries"))?; + let bytes = fs::symlink_metadata(executable) + .map_err(|error| format!("cannot inspect compiled output: {error}"))? + .len(); + if bytes == 0 || bytes > MAX_COMPILED_BYTES { + return Err("compiled output must be between 1 byte and 320 MiB".into()); + } + let (used, count) = directory_entries(&self.root.join("entries"))?.try_fold( + (0_u64, 0_usize), + |(used, count), entry| { + let key = entry.file_name().to_string_lossy().into_owned(); + let manifest = self.inspect_entry(&key, false)?; + Ok::<_, String>((used.saturating_add(manifest.bytes), count + 1)) + }, + )?; + if count >= MAX_ARTIFACT_ENTRIES || used.saturating_add(bytes) > MAX_ARTIFACT_CACHE_BYTES { + return Err("shared compilation cache quota exceeded (2 GiB / 4096 entries)".into()); + } + let staging = staging_root.join(format!( + "publish-{}-{}", + std::process::id(), + NEXT_CACHE_OPERATION.fetch_add(1, Ordering::Relaxed), + )); + create_directory(&staging)?; + let result = (|| { + let program = staging.join("program"); + copy_regular_file(executable, &program)?; + let metadata = fs::metadata(&program).map_err(|error| error.to_string())?; + let entry = CacheEntry { + format: "dever-machine-cache-entry-v1".into(), + key: key.clone(), + version: identity.version.clone(), + target: identity.target.to_owned(), + bytes: metadata.len(), + sha256: sha256_file(&program)?, + }; + let manifest = serde_json::to_vec(&entry).map_err(|error| error.to_string())?; + write_new(&staging.join("entry.json"), &manifest)?; + sync_directory(&staging).map_err(|error| error.to_string())?; + match fs::rename(&staging, &destination) { + Ok(()) => { + sync_directory(&self.root.join("entries")).map_err(|error| error.to_string()) + } + Err(_error) if destination.exists() => { + self.read_entry(&key)?; + Ok(()) + } + Err(error) => Err(format!("cannot publish shared cache entry: {error}")), + } + })(); + if staging.exists() { + let _ = fs::remove_dir_all(&staging); + } + result.map(|()| key) + } + + pub fn restore(&self, identity: &BuildIdentity<'_>) -> Result<Option<CachedArtifact>, String> { + let key = identity.key(); + let entry = match self.read_entry(&key) { + Ok(entry) => entry, + Err(error) if error.starts_with("shared cache miss:") => return Ok(None), + Err(error) => return Err(error), + }; + if entry.version != *identity.version || entry.target != identity.target { + return Err("shared cache identity collision".into()); + } + let lease = CacheLease::acquire(&self.root, &key)?; + let path = self.root.join("entries").join(&key).join("program"); + let bytes = read_regular_file(&path)?; + if bytes.len() as u64 != entry.bytes || sha256(&bytes) != entry.sha256 { + return Err(format!( + "shared cache entry {key} is incomplete or modified" + )); + } + Ok(Some(CachedArtifact { + bytes, + _lease: lease, + })) + } + + pub fn status(&self) -> Result<CacheStatus, String> { + self.inspect_status(true) + } + + /// Unprivileged status is metadata-only: it cannot force hashing all + /// libraries/programs. It never establishes an artifact integrity claim. + pub fn summary(&self) -> Result<CacheStatus, String> { + self.inspect_status(false) + } + + fn inspect_status(&self, verify_bytes: bool) -> Result<CacheStatus, String> { + let mut status = CacheStatus { + verified: verify_bytes, + ..CacheStatus::default() + }; + for entry in directory_entries(&self.root.join("staging"))? { + let _ = entry; + status.staging_entries += 1; + } + for entry in directory_entries(&self.root.join("entries"))? { + let key = entry + .file_name() + .into_string() + .map_err(|_| "shared cache key is not UTF-8".to_owned())?; + match self.inspect_entry(&key, verify_bytes) { + Ok(manifest) => { + status.entries += 1; + status.bytes = status.bytes.saturating_add(manifest.bytes); + *status + .versions + .entry(manifest.version.to_string()) + .or_default() += 1; + } + Err(_) => status.corrupt_entries += 1, + } + } + for entry in directory_entries(&self.root.join("artifacts"))? { + let metadata = fs::symlink_metadata(entry.path()).map_err(|error| error.to_string())?; + let expected = ArtifactReceipt { + sha256: entry.file_name().to_string_lossy().into_owned(), + bytes: metadata.len(), + }; + let inspected = if verify_bytes { + self.read_artifact_file(&expected).map(|_| ()) + } else { + self.artifact_path(&expected).map(|_| ()) + }; + match inspected { + Ok(_) => { + status.artifact_entries += 1; + status.artifact_bytes = status.artifact_bytes.saturating_add(expected.bytes); + } + Err(_) => status.corrupt_artifacts += 1, + } + } + Ok(status) + } + + pub fn clean(&self, installed: &BTreeSet<Version>) -> Result<CacheStatus, String> { + let _lock = CacheCleanLock::acquire(&self.root)?; + if directory_entries(&self.root.join("operations"))? + .next() + .is_some() + { + return Err("shared cache is busy; clean did not remove active build state".into()); + } + remove_children(&self.root.join("staging"))?; + // Artifacts are a recoverable download cache. Explicit administrator + // cleanup removes them only while no upload or reader is active. + remove_children(&self.root.join("artifacts"))?; + for entry in directory_entries(&self.root.join("entries"))? { + let key = entry + .file_name() + .into_string() + .map_err(|_| "shared cache key is not UTF-8".to_owned())?; + if self.has_lease(&key)? { + continue; + } + let remove = self + .read_entry(&key) + .map(|entry| !installed.contains(&entry.version)) + .unwrap_or(true); + if remove { + remove_entry(entry.path())?; + } + } + self.status() + } + + fn read_entry(&self, key: &str) -> Result<CacheEntry, String> { + self.inspect_entry(key, true) + } + + fn inspect_entry(&self, key: &str, verify_bytes: bool) -> Result<CacheEntry, String> { + validate_key(key)?; + let root = self.root.join("entries").join(key); + if !root.exists() { + return Err(format!("shared cache miss: {key}")); + } + ensure_real_directory(&root)?; + let manifest: CacheEntry = + serde_json::from_slice(&read_regular_file(&root.join("entry.json"))?) + .map_err(|error| format!("invalid shared cache manifest {key}: {error}"))?; + if manifest.format != "dever-machine-cache-entry-v1" + || manifest.key != key + || manifest.bytes == 0 + || manifest.bytes > MAX_COMPILED_BYTES + { + return Err(format!("invalid shared cache manifest {key}")); + } + let program = root.join("program"); + let metadata = fs::symlink_metadata(&program) + .map_err(|error| format!("cannot inspect shared cache entry {key}: {error}"))?; + if !metadata.file_type().is_file() + || metadata.len() != manifest.bytes + || (verify_bytes && sha256_file(&program)? != manifest.sha256) + { + return Err(format!( + "shared cache entry {key} is incomplete or modified" + )); + } + Ok(manifest) + } + + fn has_lease(&self, key: &str) -> Result<bool, String> { + let directory = self.root.join("leases").join(key); + Ok(directory_entries(&directory)?.next().is_some()) + } +} + +pub struct CachedArtifact { + bytes: Vec<u8>, + _lease: CacheLease, +} + +impl CachedArtifact { + pub fn as_bytes(&self) -> &[u8] { + &self.bytes + } +} + +struct CacheLease { + path: PathBuf, +} + +impl CacheLease { + fn acquire(root: &Path, key: &str) -> Result<Self, String> { + if root.join("clean.lock").exists() { + return Err("shared cache maintenance is active".into()); + } + let directory = root.join("leases").join(key); + create_directory(&directory)?; + let path = directory.join(format!( + "{}-{}", + std::process::id(), + NEXT_CACHE_OPERATION.fetch_add(1, Ordering::Relaxed), + )); + write_new(&path, b"dever-cache-lease-v1\n")?; + if root.join("clean.lock").exists() { + let _ = fs::remove_file(&path); + return Err("shared cache maintenance is active".into()); + } + Ok(Self { path }) + } +} + +impl Drop for CacheLease { + fn drop(&mut self) { + let parent = self.path.parent().map(Path::to_path_buf); + let _ = fs::remove_file(&self.path); + if let Some(parent) = parent { + let _ = fs::remove_dir(parent); + } + } +} + +struct CacheCleanLock(PathBuf); + +impl CacheCleanLock { + fn acquire(root: &Path) -> Result<Self, String> { + create_directory(root)?; + let path = root.join("clean.lock"); + write_new(&path, format!("{}\n", std::process::id()).as_bytes())?; + Ok(Self(path)) + } +} + +pub(crate) struct CacheOperation(PathBuf); + +pub(crate) struct CompilationDirectory(pub(crate) PathBuf); + +impl Drop for CompilationDirectory { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +impl CacheOperation { + fn acquire(root: &Path) -> Result<Self, String> { + create_directory(root)?; + if root.join("clean.lock").exists() { + return Err("shared cache maintenance is active".into()); + } + let directory = root.join("operations"); + create_directory(&directory)?; + let path = directory.join(format!( + "{}-{}", + std::process::id(), + NEXT_CACHE_OPERATION.fetch_add(1, Ordering::Relaxed), + )); + write_new(&path, b"dever-cache-operation-v1\n")?; + if root.join("clean.lock").exists() { + let _ = fs::remove_file(&path); + return Err("shared cache maintenance is active".into()); + } + Ok(Self(path)) + } +} + +impl Drop for CacheOperation { + fn drop(&mut self) { + let _ = fs::remove_file(&self.0); + } +} + +impl Drop for CacheCleanLock { + fn drop(&mut self) { + let _ = fs::remove_file(&self.0); + } +} + +/// Serializes quota accounting and publication, not network reception. +struct ArtifactPublication(File); + +impl ArtifactPublication { + fn acquire(root: &Path) -> Result<Self, String> { + let path = root.join("artifact-publish.lock"); + let mut options = OpenOptions::new(); + options.read(true).write(true); + match fs::symlink_metadata(&path) { + Ok(metadata) if metadata.is_file() && !metadata.file_type().is_symlink() => {} + Ok(_) => return Err("artifact publication lock must be a regular file".into()), + Err(error) if error.kind() == io::ErrorKind::NotFound => { + options.create_new(true); + } + Err(error) => return Err(format!("cannot inspect artifact publication lock: {error}")), + } + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let file = match options.open(&path) { + Ok(file) => file, + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => OpenOptions::new() + .read(true) + .write(true) + .open(&path) + .map_err(|error| format!("cannot open artifact publication lock: {error}"))?, + Err(error) => return Err(format!("cannot open artifact publication lock: {error}")), + }; + fs2::FileExt::lock_exclusive(&file) + .map_err(|error| format!("cannot lock artifact publication: {error}"))?; + Ok(Self(file)) + } +} + +impl Drop for ArtifactPublication { + fn drop(&mut self) { + let _ = fs2::FileExt::unlock(&self.0); + } +} + +fn validate_key(key: &str) -> Result<(), String> { + if key.len() != 64 + || !key + .bytes() + .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) + { + return Err("invalid shared cache key".into()); + } + Ok(()) +} + +fn directory_entries(path: &Path) -> Result<impl Iterator<Item = fs::DirEntry>, String> { + let entries = match fs::read_dir(path) { + Ok(entries) => entries, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(Vec::new().into_iter()), + Err(error) => return Err(format!("cannot inspect '{}': {error}", path.display())), + }; + let entries = entries + .collect::<Result<Vec<_>, _>>() + .map_err(|error| format!("cannot inspect '{}': {error}", path.display()))?; + Ok(entries.into_iter()) +} + +fn remove_children(path: &Path) -> Result<(), String> { + for entry in directory_entries(path)? { + remove_entry(entry.path())?; + } + Ok(()) +} + +fn remove_entry(path: PathBuf) -> Result<(), String> { + let metadata = fs::symlink_metadata(&path) + .map_err(|error| format!("cannot inspect '{}': {error}", path.display()))?; + if metadata.is_dir() && !metadata.file_type().is_symlink() { + fs::remove_dir_all(&path) + } else { + fs::remove_file(&path) + } + .map_err(|error| format!("cannot remove '{}': {error}", path.display())) +} + +fn create_directory(path: &Path) -> Result<(), String> { + let mut builder = fs::DirBuilder::new(); + builder.recursive(true); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + builder + .create(path) + .map_err(|error| format!("cannot create '{}': {error}", path.display()))?; + ensure_real_directory(path) +} + +fn ensure_real_directory(path: &Path) -> Result<(), String> { + let metadata = fs::symlink_metadata(path) + .map_err(|error| format!("cannot inspect '{}': {error}", path.display()))?; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Err(format!("'{}' must be a real directory", path.display())); + } + Ok(()) +} + +fn copy_regular_file(source: &Path, destination: &Path) -> Result<(), String> { + let metadata = fs::symlink_metadata(source) + .map_err(|error| format!("cannot inspect '{}': {error}", source.display()))?; + if !metadata.file_type().is_file() { + return Err(format!("'{}' must be a regular file", source.display())); + } + fs::copy(source, destination) + .map_err(|error| format!("cannot copy '{}': {error}", source.display()))?; + fs::set_permissions(destination, metadata.permissions()) + .map_err(|error| format!("cannot preserve '{}': {error}", destination.display())) +} + +fn read_regular_file(path: &Path) -> Result<Vec<u8>, String> { + let metadata = fs::symlink_metadata(path) + .map_err(|error| format!("cannot inspect '{}': {error}", path.display()))?; + if !metadata.file_type().is_file() { + return Err(format!("'{}' must be a regular file", path.display())); + } + fs::read(path).map_err(|error| format!("cannot read '{}': {error}", path.display())) +} + +fn write_new(path: &Path, value: &[u8]) -> Result<(), String> { + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .map_err(|error| format!("cannot create '{}': {error}", path.display()))?; + file.write_all(value) + .and_then(|()| file.sync_all()) + .map_err(|error| format!("cannot write '{}': {error}", path.display())) +} + +fn sha256_file(path: &Path) -> Result<String, String> { + let mut file = + File::open(path).map_err(|error| format!("cannot read '{}': {error}", path.display()))?; + let mut digest = Sha256::new(); + let mut buffer = [0_u8; 64 * 1024]; + loop { + let length = file + .read(&mut buffer) + .map_err(|error| format!("cannot hash '{}': {error}", path.display()))?; + if length == 0 { + break; + } + digest.update(&buffer[..length]); + } + Ok(hex(&digest.finalize())) +} + +fn sha256(value: &[u8]) -> String { + hex(&Sha256::digest(value)) +} + +fn sync_directory(path: &Path) -> io::Result<()> { + #[cfg(unix)] + { + File::open(path)?.sync_all() + } + #[cfg(windows)] + { + let _ = path; + Ok(()) + } +} diff --git a/crates/dever-cli/src/toolchain/compilation.rs b/crates/dever-cli/src/toolchain/compilation.rs new file mode 100644 index 0000000..92d0416 --- /dev/null +++ b/crates/dever-cli/src/toolchain/compilation.rs @@ -0,0 +1,326 @@ +//! Bounded source-only protocol shared by callers, the service and its worker. + +use super::BuildTarget; +use base64::Engine; +use base64::engine::general_purpose::STANDARD; +use dever_core::native::EmbeddedResource; +use dever_core::source::SourceMap; +use dever_runtime::config::CompilationBindings; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::BTreeSet; + +// Base64 plus source/metadata must fit while embedding a complete interpreter. +pub const MAX_COMPILE_BYTES: usize = 384 * 1024 * 1024; +const MAX_RESOURCE_BYTES: usize = 128 * 1024 * 1024; +const MAX_RESOURCE_TOTAL: usize = 256 * 1024 * 1024; +pub const WORKER_REQUEST_FILE: &str = "request.json"; +pub const WORKER_OUTPUT_FILE: &str = "program"; + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, Eq, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum CompileKind { + Application, + TestSuite, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct CompileRequest { + pub version: String, + pub kind: CompileKind, + pub target: BuildTarget, + pub sources: Vec<CompilationSource>, + pub bindings: Option<CompilationBindings>, + pub resources: Vec<CompilationResource>, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct CompilationSource { + pub logical_path: String, + pub is_test: bool, + pub text: String, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct CompilationResource { + pub path: String, + #[serde(with = "resource_bytes")] + pub bytes: Vec<u8>, + pub sha256: String, + pub executable: bool, +} + +impl CompileRequest { + pub fn new( + version: String, + kind: CompileKind, + sources: &SourceMap, + bindings: Option<CompilationBindings>, + resources: &[EmbeddedResource], + target: BuildTarget, + ) -> Result<Self, String> { + let request = Self { + version, + kind, + target, + sources: sources + .compilation_sources()? + .into_iter() + .map(|source| CompilationSource { + logical_path: source.logical_path, + is_test: source.is_test, + text: source.text, + }) + .collect(), + bindings, + resources: resources + .iter() + .map(|resource| CompilationResource { + path: resource.path.clone(), + bytes: resource.bytes.clone(), + sha256: resource.sha256.clone(), + executable: resource.executable, + }) + .collect(), + }; + request.validate()?; + Ok(request) + } + + pub fn validate(&self) -> Result<(), String> { + super::Version::parse(&self.version)?; + if self.kind == CompileKind::TestSuite && self.target != BuildTarget::host()? { + return Err("test compilation requires the compiler host target".into()); + } + if self.sources.is_empty() + || self.sources.len() > 4096 + || self + .sources + .iter() + .map(|source| source.text.len()) + .sum::<usize>() + > 16 * 1024 * 1024 + || self + .sources + .iter() + .any(|source| source.logical_path.len() > 4096) + || !self.sources.iter().any(|source| !source.is_test) + { + return Err("compilation requires application sources, at most 4096 files and 16 MiB of source text".into()); + } + self.source_map()?; + match self.kind { + CompileKind::Application => { + if self.sources.iter().any(|source| source.is_test) { + return Err("application compilation cannot contain test sources".into()); + } + self.bindings + .as_ref() + .ok_or("application compilation requires deployment bindings")? + .validate()?; + } + CompileKind::TestSuite if self.bindings.is_some() || !self.resources.is_empty() => { + return Err("test compilation derives case-local bindings and cannot contain deployment bindings or external resources".into()); + } + CompileKind::TestSuite => {} + } + if self.resources.len() > 4096 + || self + .resources + .iter() + .map(|resource| resource.bytes.len()) + .sum::<usize>() + > MAX_RESOURCE_TOTAL + { + return Err("compilation resources exceed 4096 files or 256 MiB".into()); + } + let mut paths = BTreeSet::new(); + for resource in &self.resources { + relative_path(&resource.path)?; + if !paths.insert(&resource.path) { + return Err("duplicate compilation resource path".into()); + } + if resource.bytes.len() > MAX_RESOURCE_BYTES + || format!("{:x}", Sha256::digest(&resource.bytes)) != resource.sha256 + { + return Err(format!( + "compilation resource '{}' exceeds 128 MiB or has an invalid digest", + resource.path + )); + } + } + Ok(()) + } + + pub fn source_map(&self) -> Result<SourceMap, String> { + SourceMap::from_compilation_sources( + &self + .sources + .iter() + .map(|source| dever_core::source::CompilationSource { + logical_path: source.logical_path.clone(), + is_test: source.is_test, + text: source.text.clone(), + }) + .collect::<Vec<_>>(), + ) + } + + pub fn embedded_resources(&self) -> Vec<EmbeddedResource> { + self.resources + .iter() + .map(|resource| EmbeddedResource { + path: resource.path.clone(), + bytes: resource.bytes.clone(), + sha256: resource.sha256.clone(), + executable: resource.executable, + }) + .collect() + } + + pub fn encode(&self) -> Result<Vec<u8>, String> { + self.validate()?; + #[derive(Serialize)] + struct Canonical<'a> { + version: &'a str, + kind: CompileKind, + target: BuildTarget, + sources: Vec<&'a CompilationSource>, + bindings: &'a Option<CompilationBindings>, + resources: Vec<&'a CompilationResource>, + } + // Sort references: interpreter payloads must not be cloned to order JSON. + let mut sources = self.sources.iter().collect::<Vec<_>>(); + sources.sort_by(|left, right| { + (left.is_test, &left.logical_path).cmp(&(right.is_test, &right.logical_path)) + }); + let mut resources = self.resources.iter().collect::<Vec<_>>(); + resources.sort_by(|left, right| left.path.cmp(&right.path)); + let canonical = Canonical { + version: &self.version, + kind: self.kind, + target: self.target, + sources, + bindings: &self.bindings, + resources, + }; + let bytes = serde_json::to_vec(&canonical) + .map_err(|error| format!("cannot encode compile request: {error}"))?; + if bytes.len() > MAX_COMPILE_BYTES { + return Err("compile request exceeds 384 MiB".into()); + } + Ok(bytes) + } + + pub fn decode(bytes: &[u8]) -> Result<Self, String> { + if bytes.len() > MAX_COMPILE_BYTES { + return Err("compile request exceeds 384 MiB".into()); + } + let request: Self = serde_json::from_slice(bytes) + .map_err(|error| format!("invalid compile request: {error}"))?; + // Requiring one representation rejects duplicate map keys as well as + // ambiguous ordering/escapes before requests acquire a cache identity. + if request.encode()? != bytes { + return Err("compile request must use canonical JSON without duplicate keys".into()); + } + Ok(request) + } +} + +fn relative_path(path: &str) -> Result<(), String> { + if path.is_empty() + || path.len() > 4096 + || path.contains(['\\', ':', '\0']) + || path + .split('/') + .any(|part| part.is_empty() || part == "." || part == "..") + { + return Err("compilation resources require normalized relative paths".into()); + } + Ok(()) +} + +/// Workers accept inputs only inside the service owner's private compile stage. +/// This prevents the hidden entry from becoming a client-side managed fallback. +pub fn worker_directory(layout: &super::Layout) -> Result<std::path::PathBuf, String> { + #[cfg(unix)] + { + use std::fs; + use std::os::unix::fs::MetadataExt; + + let directory = std::env::current_dir() + .map_err(|error| format!("cannot locate compiler worker directory: {error}"))?; + let staging = layout.native_cache().join("staging"); + if directory.parent() != Some(staging.as_path()) + || !directory + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| { + name.strip_prefix("compile-") + .is_some_and(|suffix| !suffix.is_empty()) + }) + { + return Err( + "compile worker requires a service-owned native cache staging directory".into(), + ); + } + let owner = rustix::process::geteuid().as_raw(); + for (path, private) in [ + (layout.state(), false), + (layout.cache(), false), + (layout.native_cache(), false), + (staging, true), + (directory.clone(), true), + ] { + let metadata = fs::symlink_metadata(&path) + .map_err(|error| format!("cannot inspect compiler worker directory: {error}"))?; + if metadata.file_type().is_symlink() + || !metadata.is_dir() + || metadata.uid() != owner + || metadata.mode() & 0o022 != 0 + || private && metadata.mode() & 0o777 != 0o700 + { + return Err("compiler worker directories must be real and service-owned without group/other writes; staging must be private (0700)".into()); + } + } + let request = fs::symlink_metadata(directory.join(WORKER_REQUEST_FILE)) + .map_err(|error| format!("cannot inspect compiler worker request: {error}"))?; + if request.file_type().is_symlink() + || !request.is_file() + || request.uid() != owner + || request.mode() & 0o777 != 0o600 + { + return Err( + "compiler worker request must be a service-owned regular file with mode 0600" + .into(), + ); + } + Ok(directory) + } + #[cfg(not(unix))] + { + let _ = layout; + Err("trusted compiler worker directories are unavailable on this platform".into()) + } +} + +mod resource_bytes { + use super::*; + pub fn serialize<S: serde::Serializer>(bytes: &[u8], serializer: S) -> Result<S::Ok, S::Error> { + serializer.serialize_str(&STANDARD.encode(bytes)) + } + pub fn deserialize<'de, D: serde::Deserializer<'de>>( + deserializer: D, + ) -> Result<Vec<u8>, D::Error> { + let encoded = String::deserialize(deserializer)?; + if encoded.len() > MAX_RESOURCE_BYTES * 4 / 3 + 4 { + return Err(serde::de::Error::custom( + "compilation resource exceeds 128 MiB", + )); + } + STANDARD.decode(encoded).map_err(serde::de::Error::custom) + } +} diff --git a/crates/dever-cli/src/toolchain/core_libraries.rs b/crates/dever-cli/src/toolchain/core_libraries.rs new file mode 100644 index 0000000..f27a4c2 --- /dev/null +++ b/crates/dever-cli/src/toolchain/core_libraries.rs @@ -0,0 +1,212 @@ +//! Native compiler loader closure, shared by authoring and signed installation. +use std::collections::{BTreeMap, BTreeSet}; +use std::fs::File; +use std::io::Read; +use std::path::Path; + +use goblin::elf::{Elf, dynamic, header}; + +use super::{Artifact, runtime_pack}; + +const MAX_FILES: usize = 64; +const MAX_BYTES: u64 = 512 * 1024 * 1024; + +pub(super) fn library_name(name: &str) -> Result<(), String> { + if name.is_empty() || matches!(name, "." | "..") || name.contains(['/', '\\', '\0', ':']) { + return Err("compiler dependency must use a plain library name".into()); + } + Ok(()) +} + +fn os_library(name: &str) -> bool { + // These are the declared GNU OS ABI, not host-discovered dependencies. + matches!( + name, + "libc.so.6" + | "libm.so.6" + | "libdl.so.2" + | "libpthread.so.0" + | "librt.so.1" + | "libresolv.so.2" + | "libutil.so.1" + ) || name == loader_name() +} + +fn loader_name() -> &'static str { + if cfg!(target_arch = "aarch64") { + "ld-linux-aarch64.so.1" + } else { + "ld-linux-x86-64.so.2" + } +} + +pub(super) fn validate(root: &Path, artifacts: &[Artifact]) -> Result<(), String> { + validate_entries(root, artifacts, &["dever-core"]) +} + +/// Both release components keep their complete non-OS loader closure in lib/. +/// Their entry sets differ; the loader and file admission rules do not. +pub(super) fn validate_entries( + root: &Path, + artifacts: &[Artifact], + entries: &[&str], +) -> Result<(), String> { + runtime_pack::host_target()?; + dever_sandbox::require_no_loader_preload()?; + let mut files = BTreeMap::new(); + let mut total = 0u64; + for artifact in artifacts { + if !entries.contains(&artifact.path.as_str()) && !artifact.path.starts_with("lib/") { + continue; + } + if let Some(name) = artifact.path.strip_prefix("lib/") { + library_name(name)?; + if os_library(name) { + return Err("compiler pack must not override the GNU OS ABI libraries".into()); + } + } + total = total + .checked_add(artifact.bytes) + .ok_or("compiler closure size overflow")?; + if total > MAX_BYTES || files.len() >= MAX_FILES { + return Err("compiler loader closure exceeds its file or byte budget".into()); + } + if files.insert(artifact.path.as_str(), artifact).is_some() { + return Err("compiler loader closure has duplicate files".into()); + } + } + validate_library_directory(root, &files)?; + let mut remaining = entries + .iter() + .map(|entry| (*entry).to_owned()) + .collect::<Vec<_>>(); + let mut reached = BTreeSet::new(); + while let Some(path) = remaining.pop() { + if !reached.insert(path.clone()) { + continue; + } + let artifact = files.get(path.as_str()).ok_or_else(|| { + format!("compiler dependency '{path}' is not supplied by the signed release") + })?; + let input = runtime_pack::checked_path(root, &path, true)?; + let mut bytes = Vec::new(); + File::open(input) + .and_then(|file| file.take(artifact.bytes + 1).read_to_end(&mut bytes)) + .map_err(|error| format!("cannot read compiler loader input: {error}"))?; + if bytes.len() as u64 != artifact.bytes { + return Err("compiler loader input size differs from its signed declaration".into()); + } + remaining.extend(dependencies( + &path, + &bytes, + entries.contains(&path.as_str()), + )?); + } + if reached.len() != files.len() { + return Err("compiler closure contains libraries unused by the core".into()); + } + Ok(()) +} + +fn dependencies(path: &str, bytes: &[u8], executable: bool) -> Result<Vec<String>, String> { + let elf = Elf::parse(bytes).map_err(|error| format!("invalid compiler ELF input: {error}"))?; + let machine = if cfg!(target_arch = "aarch64") { + header::EM_AARCH64 + } else { + header::EM_X86_64 + }; + if !elf.is_64 + || !elf.little_endian + || elf.header.e_machine != machine + || !matches!(elf.header.e_type, header::ET_DYN | header::ET_EXEC) + { + return Err("compiler ELF input has an incompatible target or object type".into()); + } + // GNU auditing/config and ELF filter tags can load libraries outside + // DT_NEEDED; they are not part of this release's loader contract. + if elf.dynamic.as_ref().is_some_and(|table| { + table.dyns.iter().any(|entry| { + matches!( + entry.d_tag, + dynamic::DT_CONFIG + | dynamic::DT_AUDIT + | dynamic::DT_DEPAUDIT + | 0x7fff_fffd + | 0x7fff_ffff + ) + }) + }) { + return Err("compiler ELF input contains unsupported dynamic loader hooks".into()); + } + if executable { + let interpreter = if cfg!(target_arch = "aarch64") { + "/lib/ld-linux-aarch64.so.1" + } else { + "/lib64/ld-linux-x86-64.so.2" + }; + if elf.interpreter != Some(interpreter) + || elf.rpaths != ["$ORIGIN/lib"] + || !elf.runpaths.is_empty() + { + return Err("compiler requires the GNU loader and inherited RPATH=$ORIGIN/lib".into()); + } + } else { + if elf.header.e_type != header::ET_DYN || elf.soname != path.strip_prefix("lib/") { + return Err("compiler library SONAME must match its signed filename".into()); + } + // Upstream LLVM uses $ORIGIN/../lib, which resolves to this same + // private lib directory. No search entry may reach the host tree. + if elf + .rpaths + .iter() + .chain(&elf.runpaths) + .any(|path| !matches!(*path, "$ORIGIN" | "$ORIGIN/../lib")) + { + return Err("compiler library search path leaves its private library directory".into()); + } + } + let mut dependencies = Vec::new(); + for name in elf.libraries { + library_name(name)?; + if !os_library(name) { + dependencies.push(format!("lib/{name}")); + } + } + Ok(dependencies) +} + +fn validate_library_directory( + root: &Path, + files: &BTreeMap<&str, &Artifact>, +) -> Result<(), String> { + let mut expected = files + .keys() + .filter_map(|path| path.strip_prefix("lib/")) + .collect::<BTreeSet<_>>(); + match std::fs::symlink_metadata(root.join("lib")) { + Err(error) if expected.is_empty() && error.kind() == std::io::ErrorKind::NotFound => { + return Ok(()); + } + Err(error) => return Err(format!("cannot inspect compiler lib directory: {error}")), + Ok(_) => {} + } + let directory = runtime_pack::checked_path(root, "lib", false)?; + for entry in std::fs::read_dir(directory).map_err(|error| error.to_string())? { + let entry = entry.map_err(|error| error.to_string())?; + let name = entry.file_name(); + if !entry + .file_type() + .map_err(|error| error.to_string())? + .is_file() + || !name.to_str().is_some_and(|name| expected.remove(name)) + { + return Err( + "compiler lib directory contains an unsigned file, directory or link".into(), + ); + } + } + if !expected.is_empty() { + return Err("compiler lib directory is missing a signed library".into()); + } + Ok(()) +} diff --git a/crates/dever-cli/src/toolchain/launcher.rs b/crates/dever-cli/src/toolchain/launcher.rs new file mode 100644 index 0000000..4053d38 --- /dev/null +++ b/crates/dever-cli/src/toolchain/launcher.rs @@ -0,0 +1,205 @@ +use std::ffi::{OsStr, OsString}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use super::release::{Layout, MachineManager, Version, platform_contract}; + +const USAGE: &str = "Usage:\n dever install <version|latest>\n dever update\n dever use <version>\n dever uninstall <version>\n dever version [project-root]\n dever cache status\n dever cache clean\n dever lib add|list|update|remove|doctor <project-root> [--target linux-x86_64|linux-aarch64] [spec ...]\n dever package add|list|update|remove|doctor <project-root> [spec ...]\n dever check <project-root>\n dever api <project-root> [--output <new-file>]\n dever fmt <project-root> [--check]\n dever test <project-root>\n dever run <project-root> [-- <component>.<domain>.<cmd> '<json-object>']\n dever build <project-root> --output <new-file> [--target linux-x86_64|linux-aarch64]\n dever clean <project-root>\n dever tenant ..."; + +#[derive(Debug)] +pub enum CommandResult { + Exited(i32), + Message(String), +} + +pub fn execute(executable: &Path, arguments: &[OsString]) -> Result<CommandResult, String> { + if platform_contract(std::env::consts::OS).is_none() { + return Err(format!( + "Dever does not support platform '{}'", + std::env::consts::OS + )); + } + let layout = Layout::from_launcher(executable)?; + execute_in(&layout, arguments) +} + +pub fn execute_in(layout: &Layout, arguments: &[OsString]) -> Result<CommandResult, String> { + layout.validate_machine_permissions()?; + #[cfg(target_os = "linux")] + super::bootstrap::require_no_pending(layout)?; + let manager = MachineManager::new(layout.clone()); + match arguments { + [command, requested] if command == "install" => { + let requested = utf8(requested, "version")?; + super::release_source::prepare(layout, requested)?; + let version = manager.install_requested(requested)?; + Ok(CommandResult::Message(format!("installed Dever {version}"))) + } + [command] if command == "update" => { + super::release_source::prepare(layout, "latest")?; + let version = manager.update()?; + Ok(CommandResult::Message(format!( + "updated Dever and dever-language skill to {version}" + ))) + } + [command, version] if command == "use" => { + let version = Version::parse(utf8(version, "version")?)?; + manager.activate(&version)?; + Ok(CommandResult::Message(format!("active Dever {version}"))) + } + [command, version] if command == "uninstall" => { + let version = Version::parse(utf8(version, "version")?)?; + manager.uninstall(&version)?; + Ok(CommandResult::Message(format!( + "uninstalled Dever {version}" + ))) + } + [command] if command == "version" => { + let version = manager + .active_version()? + .ok_or("no active Dever version; run 'dever install latest'")?; + manager.resolve_core(&version)?; + Ok(CommandResult::Message(version.to_string())) + } + [command, project] if command == "version" => { + let root = PathBuf::from(project); + let version = select_version(&manager, &root)?; + Ok(CommandResult::Message(version.to_string())) + } + [cache, status] if cache == "cache" && status == "status" => { + let status = crate::toolchain::cache_status(layout)?; + Ok(CommandResult::Message( + serde_json::to_string(&status).map_err(|error| error.to_string())?, + )) + } + [cache, clean] if cache == "cache" && clean == "clean" => { + let status = crate::toolchain::cache_clean( + layout, + &manager.installed_versions()?.into_iter().collect(), + )?; + Ok(CommandResult::Message( + serde_json::to_string(&status).map_err(|error| error.to_string())?, + )) + } + [skill, path] if skill == "skill" && path == "path" => Ok(CommandResult::Message( + super::skill::path(&manager)?.display().to_string(), + )), + [skill, path, project] if skill == "skill" && path == "path" => { + let version = select_version(&manager, Path::new(project))?; + Ok(CommandResult::Message( + manager.resolve_skill(&version)?.display().to_string(), + )) + } + [skill, install, directory] if skill == "skill" && install == "install" => { + super::skill::install(&manager, Path::new(directory))?; + Ok(CommandResult::Message(format!( + "installed dever-language skill entry in '{}'", + Path::new(directory).display() + ))) + } + [command, ..] if command == "new" => { + let version = manager + .active_version()? + .ok_or("no active Dever version; run 'dever install latest'")?; + dispatch_core(&manager, &version, arguments) + } + [help] if help == "--help" => Ok(CommandResult::Message(format!( + "{USAGE}\n dever new <new-directory> [--markdown]\n dever skill path [project-root]\n dever skill install <new-directory>" + ))), + [] => Err(USAGE.into()), + _ => dispatch_project_command(&manager, arguments), + } +} + +pub fn project_version(project_root: &Path) -> Result<Option<Version>, String> { + let path = project_root.join("config/setting.json"); + let text = fs::read_to_string(&path) + .map_err(|error| format!("cannot read '{}': {error}", path.display()))?; + let root = dever_runtime::wire::parse(&text) + .map_err(|error| format!("invalid '{}': {error}", path.display()))?; + let fields = root + .object() + .map_err(|_| format!("invalid '{}': expected a JSON object", path.display()))?; + let Some(dever) = fields.get("dever") else { + return Ok(None); + }; + let fields = dever + .fields(&["version"]) + .map_err(|error| format!("invalid '{}': dever: {error}", path.display()))?; + let version = fields + .get("version") + .ok_or_else(|| format!("invalid '{}': dever.version is required", path.display()))? + .text() + .map_err(|error| format!("invalid '{}': dever.version: {error}", path.display()))?; + Version::parse(version) + .map(Some) + .map_err(|error| format!("invalid '{}': {error}", path.display())) +} + +fn dispatch_project_command( + manager: &MachineManager, + arguments: &[OsString], +) -> Result<CommandResult, String> { + let root = project_root(arguments).ok_or_else(|| USAGE.to_owned())?; + let version = select_version(manager, &root)?; + dispatch_core(manager, &version, arguments) +} + +fn dispatch_core( + manager: &MachineManager, + version: &Version, + arguments: &[OsString], +) -> Result<CommandResult, String> { + let core = manager.resolve_core(version)?; + let status = Command::new(&core) + .args(arguments) + .env_clear() + .status() + .map_err(|error| format!("cannot run Dever {version} '{}': {error}", core.display()))?; + Ok(CommandResult::Exited(status.code().unwrap_or(1))) +} + +fn select_version(manager: &MachineManager, project_root: &Path) -> Result<Version, String> { + let version = match project_version(project_root)? { + Some(version) => version, + None => manager + .active_version()? + .ok_or("no active Dever version; run 'dever install latest'")?, + }; + manager.resolve_core(&version)?; + Ok(version) +} + +fn project_root(arguments: &[OsString]) -> Option<PathBuf> { + match arguments { + [command, _operation, root, ..] if command == "lib" || command == "package" => { + Some(root.into()) + } + [command, root, ..] + if matches!( + command.to_str(), + Some("check" | "api" | "fmt" | "test" | "run" | "build" | "clean") + ) => + { + Some(root.into()) + } + [tenant, operation, root, ..] + if tenant == "tenant" && matches!(operation.to_str(), Some("migrate" | "owner")) => + { + Some(root.into()) + } + [tenant, component, _operation, root, ..] + if tenant == "tenant" && component == "component" => + { + Some(root.into()) + } + _ => None, + } +} + +fn utf8<'a>(value: &'a OsStr, label: &str) -> Result<&'a str, String> { + value + .to_str() + .ok_or_else(|| format!("{label} must be UTF-8")) +} diff --git a/crates/dever-cli/src/toolchain/mod.rs b/crates/dever-cli/src/toolchain/mod.rs new file mode 100644 index 0000000..7d0932b --- /dev/null +++ b/crates/dever-cli/src/toolchain/mod.rs @@ -0,0 +1,25 @@ +#[cfg(target_os = "linux")] +pub mod bootstrap; +mod cache; +pub mod compilation; +mod core_libraries; +mod launcher; +pub mod packaging; +mod release; +mod release_source; +pub mod runtime_pack; +mod service; +mod skill; +mod target; + +pub use target::BuildTarget; + +pub use cache::{ + ArtifactReceipt, BuildIdentity, CacheEntry, CacheStatus, CacheStore, CachedArtifact, +}; +pub use launcher::{CommandResult, execute, execute_in, project_version}; +pub use release::{ + Artifact, DownloadedReleases, Layout, MachineManager, PlatformContract, ReleaseManifest, + Version, platform_contract, platform_identity, sha256_file, +}; +pub use service::{artifact_get, artifact_put, cache_clean, cache_status, compile, serve}; diff --git a/crates/dever-cli/src/toolchain/packaging.rs b/crates/dever-cli/src/toolchain/packaging.rs new file mode 100644 index 0000000..8925376 --- /dev/null +++ b/crates/dever-cli/src/toolchain/packaging.rs @@ -0,0 +1,520 @@ +//! Private authoring of signed native releases from explicitly prepared inputs. +//! Compilation and tool discovery do not belong to this entry point. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs::{self, File}; +use std::io::Read; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use ring::signature::Ed25519KeyPair; +use serde::Deserialize; +use sha2::{Digest, Sha256}; + +use super::runtime_pack::{self, Manifest, Profile, Profiles, RuntimeArtifact}; +use super::{Artifact, BuildTarget, ReleaseManifest, Version, sha256_file}; + +static NEXT_STAGING: AtomicU64 = AtomicU64::new(0); + +mod bootstrap; +mod builds; +mod ecosystems; +mod sandbox; + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Settings { + format: String, + version: String, + target: String, + signing_key: String, + core: Input, + core_libraries: Vec<LinkInput>, + skill: Vec<LinkInput>, + #[serde(default)] + bootstrap: Option<bootstrap::Inputs>, + profiles: ArchiveInputs, + start: Vec<LinkInput>, + libraries: Vec<LinkInput>, + end: Vec<LinkInput>, + #[serde(default)] + native_targets: BTreeMap<BuildTarget, NativeInputs>, + #[serde(default)] + runtimes: ecosystems::RuntimeInputs, + #[serde(default)] + sandbox: Option<Vec<LinkInput>>, + #[serde(default)] + builds: builds::Inputs, + #[serde(default)] + ecosystem_targets: BTreeMap<BuildTarget, ecosystems::RuntimeInputs>, + #[serde(default)] + build_targets: BTreeMap<BuildTarget, builds::Inputs>, + #[serde(default)] + sandbox_targets: BTreeMap<BuildTarget, Vec<LinkInput>>, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct NativeInputs { + profiles: ArchiveInputs, + start: Vec<LinkInput>, + libraries: Vec<LinkInput>, + end: Vec<LinkInput>, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Input { + source: String, + sha256: String, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct LinkInput { + source: String, + path: String, + sha256: String, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ArchiveInputs { + base: Input, + sqlite: Input, + postgres: Input, + both: Input, +} + +struct SigningKey { + pair: Ed25519KeyPair, + sha256: String, +} + +struct PayloadWriter<'a> { + root: &'a Path, + staging: &'a Path, + key: &'a SigningKey, + destinations: BTreeSet<String>, +} + +/// Create a new release directory. Inputs come only from author-root/config/setting.json. +pub fn create(author_root: &Path, output: &Path) -> Result<ReleaseManifest, String> { + let target = runtime_pack::host_target()?; + let metadata = fs::symlink_metadata(author_root) + .map_err(|error| format!("cannot inspect author root: {error}"))?; + if !metadata.is_dir() || metadata.file_type().is_symlink() { + return Err("author root must be a real directory".into()); + } + let root = fs::canonicalize(author_root) + .map_err(|error| format!("cannot resolve author root: {error}"))?; + let config = runtime_pack::checked_path(&root, "config/setting.json", true)?; + let bytes = read_bounded(&config, 16 * 1024 * 1024, "author configuration")?; + // Do not echo configuration values: they include the private signing-key location. + let settings: Settings = serde_json::from_slice(&bytes).map_err(|error| { + format!( + "invalid author configuration at line {}, column {}", + error.line(), + error.column() + ) + })?; + if settings.format != "dever-native-release-input-v1" || settings.target != target { + return Err("unsupported native release input format or target".into()); + } + let version = Version::parse(&settings.version)?; + let key = read_signing_key(&root, &settings.signing_key)?; + let output = output_path(output)?; + let staging = create_staging(output.parent().expect("resolved output parent"))?; + let result = assemble(&root, &staging, settings, version, &key) + .and_then(|manifest| publish(&staging, &output).map(|()| manifest)); + if result.is_err() + && let Err(error) = fs::remove_dir_all(&staging) + { + return Err(format!( + "{}; cannot remove owned release staging: {error}", + result.expect_err("failed release creation") + )); + } + result +} + +fn assemble( + root: &Path, + staging: &Path, + settings: Settings, + version: Version, + key: &SigningKey, +) -> Result<ReleaseManifest, String> { + let host = BuildTarget::host()?; + let mut writer = PayloadWriter { + root, + staging, + key, + destinations: BTreeSet::from(["manifest.json".to_owned(), "manifest.sig".to_owned()]), + }; + let mut artifacts = Vec::new(); + artifacts.push(writer.copy( + &settings.core.source, + "dever-core", + &settings.core.sha256, + true, + )?); + for input in &settings.core_libraries { + super::core_libraries::library_name(&input.path)?; + artifacts.push(writer.copy( + &input.source, + &format!("lib/{}", input.path), + &input.sha256, + false, + )?); + } + super::core_libraries::validate(staging, &artifacts)?; + if settings.skill.len() > 256 || !settings.skill.iter().any(|input| input.path == "SKILL.md") { + return Err("release skill requires SKILL.md and at most 256 files".into()); + } + let mut skill_bytes = 0_u64; + for input in settings.skill { + let source = writer.source(&input.source, &input.path, &input.sha256)?; + let bytes = fs::metadata(source) + .map_err(|error| error.to_string())? + .len(); + skill_bytes = skill_bytes + .checked_add(bytes) + .ok_or("release skill size overflow")?; + if bytes > 1024 * 1024 || skill_bytes > 8 * 1024 * 1024 { + return Err("release skill exceeds its 1 MiB file or 8 MiB total limit".into()); + } + artifacts.push(writer.copy( + &input.source, + &format!("skills/dever-language/{}", input.path), + &input.sha256, + false, + )?); + } + if let Some(input) = settings.bootstrap { + artifacts.extend(bootstrap::assemble(&mut writer, input)?); + } + artifacts.extend(assemble_native( + &mut writer, + &version, + host, + NativeInputs { + profiles: settings.profiles, + start: settings.start, + libraries: settings.libraries, + end: settings.end, + }, + )?); + for (target, inputs) in settings.native_targets { + artifacts.extend(assemble_native(&mut writer, &version, target, inputs)?); + } + for (ecosystem, input) in settings.runtimes.entries() { + artifacts.extend(ecosystems::assemble(&mut writer, ecosystem, input, host)?); + } + for (target, inputs) in settings.ecosystem_targets { + for (ecosystem, input) in inputs.entries() { + artifacts.extend(ecosystems::assemble(&mut writer, ecosystem, input, target)?); + } + } + if let Some(inputs) = settings.sandbox { + artifacts.extend(sandbox::assemble(&mut writer, inputs, host)?); + } + for (target, inputs) in settings.sandbox_targets { + artifacts.extend(sandbox::assemble(&mut writer, inputs, target)?); + } + for (ecosystem, input) in settings.builds.entries() { + artifacts.extend(builds::assemble(&mut writer, ecosystem, input, host)?); + } + for (target, inputs) in settings.build_targets { + for (ecosystem, input) in inputs.entries() { + artifacts.extend(builds::assemble(&mut writer, ecosystem, input, target)?); + } + } + artifacts.sort_by(|left, right| left.path.cmp(&right.path)); + let manifest = ReleaseManifest::new(version, artifacts); + let bytes = write_metadata(&staging.join("manifest.json"), &manifest)?; + let signature = key.pair.sign(&bytes); + let hex: String = signature + .as_ref() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(); + fs::write(staging.join("manifest.sig"), format!("{hex}\n")) + .map_err(|error| format!("cannot write release signature: {error}"))?; + set_mode(&staging.join("manifest.sig"), false)?; + Ok(manifest) +} + +fn assemble_native( + writer: &mut PayloadWriter<'_>, + version: &Version, + target: BuildTarget, + inputs: NativeInputs, +) -> Result<Vec<Artifact>, String> { + let runtime_prefix = format!("runtime/{}", target.platform()); + let manifest_path = writer.reserve(&format!("{runtime_prefix}/manifest.json"))?; + let archive_inputs = inputs.profiles; + let mut files = Vec::new(); + for (name, input) in [ + ("base", archive_inputs.base), + ("sqlite", archive_inputs.sqlite), + ("postgres", archive_inputs.postgres), + ("both", archive_inputs.both), + ] { + let path = format!("archives/{name}.a"); + let artifact = writer.copy( + &input.source, + &format!("{runtime_prefix}/{path}"), + &input.sha256, + false, + )?; + files.push(RuntimeArtifact { + path, + bytes: artifact.bytes, + sha256: artifact.sha256, + }); + } + for input in inputs + .start + .iter() + .chain(&inputs.libraries) + .chain(&inputs.end) + { + runtime_pack::relative_path(&input.path)?; + let artifact = writer.copy( + &input.source, + &format!("{runtime_prefix}/{}", input.path), + &input.sha256, + false, + )?; + files.push(RuntimeArtifact { + path: input.path.clone(), + bytes: artifact.bytes, + sha256: artifact.sha256, + }); + } + files.sort_by(|left, right| left.path.cmp(&right.path)); + let link_paths = |inputs: &[LinkInput]| inputs.iter().map(|input| input.path.clone()).collect(); + let profile = |name| { + Some(Profile { + start: link_paths(&inputs.start), + runtime: format!("archives/{name}.a"), + libraries: link_paths(&inputs.libraries), + end: link_paths(&inputs.end), + }) + }; + let runtime = Manifest { + format: "dever-native-runtime-v1".into(), + compiler_version: version.to_string(), + abi: 1, + platform: target.platform().into(), + target: target.triple().into(), + profiles: Profiles { + base: profile("base"), + sqlite: profile("sqlite"), + postgres: profile("postgres"), + both: profile("both"), + }, + files, + }; + write_metadata(&manifest_path, &runtime)?; + Ok( + runtime_pack::validate(writer.staging, version.as_str(), target)? + .into_iter() + .map(|artifact| Artifact { + path: artifact.path, + bytes: artifact.bytes, + sha256: artifact.sha256, + }) + .collect(), + ) +} + +impl PayloadWriter<'_> { + fn reserve(&mut self, destination: &str) -> Result<PathBuf, String> { + runtime_pack::relative_path(destination)?; + if self.destinations.iter().any(|path| { + path == destination + || path.starts_with(&format!("{destination}/")) + || destination.starts_with(&format!("{path}/")) + }) { + return Err("duplicate release destination or file/directory conflict".into()); + } + self.destinations.insert(destination.to_owned()); + let output = self.staging.join(destination); + create_directories(self.staging, output.parent().expect("payload parent"))?; + Ok(output) + } + + fn source(&self, source: &str, destination: &str, digest: &str) -> Result<PathBuf, String> { + if digest.len() != 64 + || !digest + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err("payload requires a lowercase SHA-256 digest".into()); + } + if digest == self.key.sha256 { + return Err("signing key cannot be included as a release payload".into()); + } + let source = runtime_pack::checked_path(self.root, source, true) + .map_err(|_| format!("invalid source for release payload '{destination}'"))?; + if sha256_file(&source).map_err(|_| "cannot hash source payload")? != digest { + return Err(format!( + "release payload '{destination}' failed SHA-256 verification" + )); + } + Ok(source) + } + + fn copy( + &mut self, + source: &str, + destination: &str, + digest: &str, + executable: bool, + ) -> Result<Artifact, String> { + let output = self.reserve(destination)?; + let source = self.source(source, destination, digest)?; + fs::copy(&source, &output) + .map_err(|error| format!("cannot copy release payload '{destination}': {error}"))?; + if sha256_file(&output)? != digest { + return Err(format!( + "copied release payload '{destination}' failed SHA-256 verification" + )); + } + set_mode(&output, executable)?; + Ok(Artifact { + path: destination.to_owned(), + bytes: fs::metadata(output) + .map_err(|error| error.to_string())? + .len(), + sha256: digest.to_owned(), + }) + } +} + +fn read_signing_key(root: &Path, relative: &str) -> Result<SigningKey, String> { + let path = runtime_pack::checked_path(root, relative, true) + .map_err(|_| "signing key must be a real file within the author root")?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let metadata = fs::metadata(&path).map_err(|_| "cannot inspect signing key")?; + if metadata.permissions().mode() & 0o077 != 0 { + return Err("signing key permissions must be private to its owner".into()); + } + } + let bytes = read_bounded(&path, 4096, "signing key")?; + let pair = Ed25519KeyPair::from_pkcs8(&bytes) + .map_err(|_| "signing key is not a valid Ed25519 PKCS#8 key")?; + Ok(SigningKey { + pair, + sha256: format!("{:x}", Sha256::digest(bytes)), + }) +} + +fn read_bounded(path: &Path, limit: u64, label: &str) -> Result<Vec<u8>, String> { + let mut bytes = Vec::new(); + File::open(path) + .and_then(|file| file.take(limit + 1).read_to_end(&mut bytes)) + .map_err(|error| format!("cannot read {label}: {error}"))?; + if bytes.len() as u64 > limit { + return Err(format!("{label} exceeds its size limit")); + } + Ok(bytes) +} + +fn write_metadata(path: &Path, value: &impl serde::Serialize) -> Result<Vec<u8>, String> { + let mut bytes = serde_json::to_vec_pretty(value).map_err(|error| error.to_string())?; + bytes.push(b'\n'); + fs::write(path, &bytes).map_err(|error| format!("cannot write release metadata: {error}"))?; + set_mode(path, false)?; + Ok(bytes) +} + +fn output_path(output: &Path) -> Result<PathBuf, String> { + let name = output + .file_name() + .ok_or("release output requires a new directory name")?; + let parent = output + .parent() + .filter(|path| !path.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + let parent = fs::canonicalize(parent) + .map_err(|error| format!("cannot resolve release output parent: {error}"))?; + let output = parent.join(name); + match fs::symlink_metadata(&output) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(output), + Ok(_) => Err("release output already exists".into()), + Err(error) => Err(format!("cannot inspect release output: {error}")), + } +} + +fn create_staging(parent: &Path) -> Result<PathBuf, String> { + let mut builder = fs::DirBuilder::new(); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + loop { + let path = parent.join(format!( + ".native-release-{}-{}", + std::process::id(), + NEXT_STAGING.fetch_add(1, Ordering::Relaxed) + )); + match builder.create(&path) { + Ok(()) => return Ok(path), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => return Err(format!("cannot create release staging: {error}")), + } + } +} + +fn create_directories(root: &Path, directory: &Path) -> Result<(), String> { + let mut current = root.to_owned(); + for component in directory + .strip_prefix(root) + .expect("owned staging path") + .components() + { + current.push(component); + if !current.exists() { + fs::create_dir(¤t) + .map_err(|error| format!("cannot create payload directory: {error}"))?; + set_mode(¤t, true)?; + } + } + Ok(()) +} + +fn set_mode(path: &Path, executable: bool) -> Result<(), String> { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions( + path, + fs::Permissions::from_mode(if executable { 0o755 } else { 0o644 }), + ) + .map_err(|error| format!("cannot set release permissions: {error}"))?; + } + #[cfg(not(unix))] + let _ = (path, executable); + Ok(()) +} + +fn publish(staging: &Path, output: &Path) -> Result<(), String> { + set_mode(staging, true)?; + #[cfg(target_os = "linux")] + { + use rustix::fs::{CWD, RenameFlags, renameat_with}; + renameat_with(CWD, staging, CWD, output, RenameFlags::NOREPLACE) + .map_err(|error| format!("cannot publish new release directory: {error}")) + } + #[cfg(not(target_os = "linux"))] + { + let _ = output; + Err("native release publication is not available for this platform".into()) + } +} diff --git a/crates/dever-cli/src/toolchain/packaging/bootstrap.rs b/crates/dever-cli/src/toolchain/packaging/bootstrap.rs new file mode 100644 index 0000000..43761c4 --- /dev/null +++ b/crates/dever-cli/src/toolchain/packaging/bootstrap.rs @@ -0,0 +1,64 @@ +//! The stable Linux entry points share a small, separately checked loader closure. +use serde::Deserialize; + +use super::{Artifact, Input, LinkInput, PayloadWriter}; + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Inputs { + launcher: Input, + daemon: Input, + libraries: Vec<LinkInput>, +} + +pub(super) fn assemble( + writer: &mut PayloadWriter<'_>, + input: Inputs, +) -> Result<Vec<Artifact>, String> { + if !cfg!(target_os = "linux") { + return Err("bootstrap authoring currently requires Linux".into()); + } + let mut artifacts = Vec::new(); + for (name, input) in [("dever", input.launcher), ("deverd", input.daemon)] { + artifacts.push(writer.copy( + &input.source, + &format!("bootstrap/{name}"), + &input.sha256, + true, + )?); + } + for library in input.libraries { + crate::toolchain::core_libraries::library_name(&library.path)?; + artifacts.push(writer.copy( + &library.source, + &format!("bootstrap/lib/{}", library.path), + &library.sha256, + false, + )?); + } + #[cfg(target_os = "linux")] + { + use sha2::{Digest, Sha256}; + for (path, bytes) in [ + ( + "bootstrap/deverd.service", + crate::toolchain::bootstrap::SERVICE.as_bytes(), + ), + ( + "bootstrap/dever-sandbox", + dever_sandbox::APPARMOR_PROFILE.as_bytes(), + ), + ] { + let destination = writer.reserve(path)?; + std::fs::write(&destination, bytes).map_err(|error| error.to_string())?; + super::set_mode(&destination, false)?; + artifacts.push(Artifact { + path: path.into(), + bytes: bytes.len() as u64, + sha256: format!("{:x}", Sha256::digest(bytes)), + }); + } + crate::toolchain::bootstrap::validate_payload(writer.staging, &artifacts, true)?; + } + Ok(artifacts) +} diff --git a/crates/dever-cli/src/toolchain/packaging/builds.rs b/crates/dever-cli/src/toolchain/packaging/builds.rs new file mode 100644 index 0000000..f0deef0 --- /dev/null +++ b/crates/dever-cli/src/toolchain/packaging/builds.rs @@ -0,0 +1,180 @@ +//! Assemble source-build tools against already prepared runtime identities. + +use flate2::{Compression, GzBuilder}; +use serde::Deserialize; +use sha2::{Digest, Sha256}; +use std::fs::{self, File}; + +use super::{Artifact, LinkInput, PayloadWriter, read_bounded, set_mode, write_metadata}; +use crate::libs::build::pack::{self, Descriptor, Manifest}; +use crate::libs::{Ecosystem, InstalledRegistryPack}; + +#[derive(Default, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Inputs { + pip: Option<Input>, + npm: Option<Input>, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Input { + executables: Vec<String>, + files: Vec<LinkInput>, +} + +impl Inputs { + pub(super) fn entries(self) -> impl Iterator<Item = (Ecosystem, Input)> { + [(Ecosystem::Pip, self.pip), (Ecosystem::Npm, self.npm)] + .into_iter() + .filter_map(|(ecosystem, input)| input.map(|input| (ecosystem, input))) + } +} + +pub(super) fn assemble( + writer: &mut PayloadWriter<'_>, + ecosystem: Ecosystem, + input: Input, + target: crate::toolchain::BuildTarget, +) -> Result<Vec<Artifact>, String> { + let target = target.platform().to_owned(); + let runtime = |ecosystem: &Ecosystem| -> Result<InstalledRegistryPack, String> { + let path = writer.staging.join(format!( + "runtime/{}/{target}/manifest.json", + ecosystem.as_str() + )); + let manifest: InstalledRegistryPack = serde_json::from_slice(&read_bounded( + &path, + 1024 * 1024, + "build runtime descriptor", + )?) + .map_err(|error| format!("invalid build runtime descriptor: {error}"))?; + manifest.validate(ecosystem, &target)?; + Ok(manifest) + }; + let runtime = runtime(&ecosystem)?; + let python = if ecosystem == Ecosystem::Npm { + let path = writer + .staging + .join(format!("runtime/pip/{target}/manifest.json")); + let manifest: InstalledRegistryPack = serde_json::from_slice(&read_bounded( + &path, + 1024 * 1024, + "auxiliary Python runtime descriptor", + )?) + .map_err(|error| format!("invalid auxiliary Python runtime: {error}"))?; + manifest.validate(&Ecosystem::Pip, &target)?; + Some(manifest) + } else { + None + }; + let mut descriptor = Descriptor { + format: "dever-registry-build-v1".into(), + ecosystem: ecosystem.clone(), + target: target.clone(), + runtime_sha256: runtime.runtime.pack.sha256.clone(), + python_runtime_sha256: python + .as_ref() + .map(|runtime| runtime.runtime.pack.sha256.clone()), + sha256: "0".repeat(64), + }; + descriptor.validate( + &runtime.runtime, + python.as_ref().map(|manifest| &manifest.runtime), + )?; + let manifest = Manifest { + format: "dever-lib-build-v1".into(), + ecosystem: ecosystem.clone(), + target: target.clone(), + runtime_sha256: descriptor.runtime_sha256.clone(), + python_runtime_sha256: descriptor.python_runtime_sha256.clone(), + executables: input.executables, + }; + if input.files.len() >= pack::MAX_FILES { + return Err("build pack has too many leaves".into()); + } + let metadata = serde_json::to_vec(&manifest).map_err(|error| error.to_string())?; + let mut remaining = pack::MAX_EXPANDED_BYTES + .checked_sub(metadata.len()) + .ok_or("build manifest is too large")? as u64; + let mut files = vec![("dever-build.json".into(), metadata)]; + for input in input.files { + let path = writer.source(&input.source, &input.path, &input.sha256)?; + let bytes = read_bounded(&path, remaining, "build tool leaf")?; + if format!("{:x}", Sha256::digest(&bytes)) != input.sha256 { + return Err("build tool changed during packaging".into()); + } + remaining -= bytes.len() as u64; + files.push((input.path, bytes)); + } + files.sort_by(|left, right| left.0.cmp(&right.0)); + manifest.validate(&files, &descriptor)?; + let mut assets = Vec::new(); + for directory in ["bin", "lib"] { + for entry in fs::read_dir(writer.staging.join(format!("sandbox/{target}/{directory}"))) + .map_err(|error| format!("build pack requires prepared sandbox assets: {error}"))? + { + let entry = entry.map_err(|error| error.to_string())?; + let name = entry + .file_name() + .into_string() + .map_err(|_| "sandbox asset name is not UTF-8")?; + assets.push(( + format!("{directory}/{name}"), + read_bounded(&entry.path(), 32 * 1024 * 1024, "sandbox build asset")?, + )); + } + } + dever_sandbox::validate_assets_for_target(&assets, &target)?; + pack::validate_tools(&manifest, &files, &assets)?; + let prefix = format!("build/{}/{target}", ecosystem.as_str()); + let archive_name = format!("{prefix}/build.pack"); + let path = writer.reserve(&archive_name)?; + let encoder = GzBuilder::new().mtime(0).operating_system(255).write( + File::create(&path).map_err(|error| error.to_string())?, + Compression::default(), + ); + let mut archive = tar::Builder::new(encoder); + for (path, bytes) in files { + let mut header = tar::Header::new_gnu(); + header.set_size(bytes.len() as u64); + header.set_mode(if manifest.executables.contains(&path) { + 0o755 + } else { + 0o644 + }); + header.set_uid(0); + header.set_gid(0); + header.set_mtime(0); + header.set_cksum(); + archive + .append_data(&mut header, path, bytes.as_slice()) + .map_err(|error| error.to_string())?; + } + archive + .into_inner() + .and_then(|encoder| encoder.finish()) + .map_err(|error| error.to_string())?; + set_mode(&path, false)?; + let bytes = fs::metadata(&path) + .map_err(|error| error.to_string())? + .len(); + if bytes > pack::MAX_PACK_BYTES as u64 { + return Err("compressed build pack exceeds 256 MiB".into()); + } + descriptor.sha256 = super::sha256_file(&path)?; + let descriptor_name = format!("{prefix}/manifest.json"); + let metadata = write_metadata(&writer.reserve(&descriptor_name)?, &descriptor)?; + Ok(vec![ + Artifact { + path: archive_name, + bytes, + sha256: descriptor.sha256, + }, + Artifact { + path: descriptor_name, + bytes: metadata.len() as u64, + sha256: format!("{:x}", Sha256::digest(&metadata)), + }, + ]) +} diff --git a/crates/dever-cli/src/toolchain/packaging/ecosystems.rs b/crates/dever-cli/src/toolchain/packaging/ecosystems.rs new file mode 100644 index 0000000..2d9d9b5 --- /dev/null +++ b/crates/dever-cli/src/toolchain/packaging/ecosystems.rs @@ -0,0 +1,152 @@ +//! Deterministic ecosystem packs inside the same signed native release. + +use std::fs::{self, File}; + +use flate2::{Compression, GzBuilder}; +use pep508_rs::MarkerEnvironment; +use serde::Deserialize; +use sha2::{Digest, Sha256}; + +use super::{ + Artifact, LinkInput, PayloadWriter, read_bounded, set_mode, sha256_file, write_metadata, +}; +use crate::libs::{Ecosystem, InstalledRegistryPack, RegistryRuntime, RuntimePack}; +use crate::workers::{MAX_TREE_BYTES, MAX_TREE_FILES, runtime}; + +#[derive(Default, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct RuntimeInputs { + pip: Option<RuntimeInput>, + npm: Option<RuntimeInput>, + go: Option<RuntimeInput>, +} + +impl RuntimeInputs { + pub(super) fn entries(self) -> impl Iterator<Item = (Ecosystem, RuntimeInput)> { + [ + (Ecosystem::Pip, self.pip), + (Ecosystem::Npm, self.npm), + (Ecosystem::Go, self.go), + ] + .into_iter() + .filter_map(|(ecosystem, input)| input.map(|input| (ecosystem, input))) + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct RuntimeInput { + name: String, + version: String, + python_markers: Option<MarkerEnvironment>, + #[serde(default)] + python_wheel_tags: Vec<String>, + npm_libc: Option<String>, + files: Vec<LinkInput>, +} + +pub(super) fn assemble( + writer: &mut PayloadWriter<'_>, + ecosystem: Ecosystem, + input: RuntimeInput, + target: crate::toolchain::BuildTarget, +) -> Result<Vec<Artifact>, String> { + let target = target.platform().to_owned(); + let mut metadata = InstalledRegistryPack { + format: "dever-registry-runtime-v1".into(), + runtime: RegistryRuntime { + pack: RuntimePack { + name: input.name, + version: input.version, + sha256: "0".repeat(64), + }, + target: target.clone(), + python_markers: input.python_markers, + python_wheel_tags: input.python_wheel_tags, + npm_libc: input.npm_libc, + }, + }; + metadata.validate(&ecosystem, &target)?; + if input.files.len() > MAX_TREE_FILES { + return Err("managed runtime pack has too many files".into()); + } + let mut files = Vec::with_capacity(input.files.len()); + let mut remaining = MAX_TREE_BYTES as u64; + for input in input.files { + super::runtime_pack::relative_path(&input.path)?; + // A runtime archive is built from its leaf files, never another completed pack. + if input.path == "runtime.pack" { + return Err("runtime pack cannot include another runtime.pack".into()); + } + let source = writer.source(&input.source, &input.path, &input.sha256)?; + let bytes = read_bounded(&source, remaining, "runtime payload")?; + if format!("{:x}", Sha256::digest(&bytes)) != input.sha256 { + return Err("runtime payload changed during packaging".into()); + } + remaining -= bytes.len() as u64; + files.push((input.path, bytes)); + } + files.sort_by(|left, right| left.0.cmp(&right.0)); + let manifest = runtime::validate(&files, &ecosystem, &target)?; + if manifest.python_wheel_tags != metadata.runtime.python_wheel_tags + || manifest.python_markers != metadata.runtime.python_markers + { + return Err( + "runtime pack Python metadata differs from its signed registry descriptor".into(), + ); + } + let prefix = format!("runtime/{}/{target}", ecosystem.as_str()); + let archive_name = format!("{prefix}/runtime.pack"); + let archive_path = writer.reserve(&archive_name)?; + let output = File::create(&archive_path) + .map_err(|error| format!("cannot create runtime pack: {error}"))?; + let encoder = GzBuilder::new() + .mtime(0) + .operating_system(255) + .write(output, Compression::default()); + let mut archive = tar::Builder::new(encoder); + for (path, bytes) in &files { + let mut header = tar::Header::new_gnu(); + header.set_size(bytes.len() as u64); + header.set_mode(if manifest.executable(path) { + 0o755 + } else { + 0o644 + }); + header.set_uid(0); + header.set_gid(0); + header.set_mtime(0); + header.set_cksum(); + archive + .append_data(&mut header, path, bytes.as_slice()) + .map_err(|error| format!("cannot write runtime pack entry: {error}"))?; + } + archive + .into_inner() + .and_then(|encoder| encoder.finish()) + .map_err(|error| format!("cannot finish runtime pack: {error}"))?; + set_mode(&archive_path, false)?; + let size = fs::metadata(&archive_path) + .map_err(|error| error.to_string())? + .len(); + if size > MAX_TREE_BYTES as u64 { + return Err("compressed runtime pack exceeds 256 MiB".into()); + } + metadata.runtime.pack.sha256 = sha256_file(&archive_path)?; + let pack = Artifact { + path: archive_name, + bytes: size, + sha256: metadata.runtime.pack.sha256.clone(), + }; + let metadata_name = format!("{prefix}/manifest.json"); + let metadata_path = writer.reserve(&metadata_name)?; + let bytes = write_metadata(&metadata_path, &metadata)?; + Ok(vec![ + pack, + Artifact { + path: metadata_name, + bytes: bytes.len() as u64, + sha256: format!("{:x}", Sha256::digest(bytes)), + }, + ]) +} diff --git a/crates/dever-cli/src/toolchain/packaging/sandbox.rs b/crates/dever-cli/src/toolchain/packaging/sandbox.rs new file mode 100644 index 0000000..ece49e8 --- /dev/null +++ b/crates/dever-cli/src/toolchain/packaging/sandbox.rs @@ -0,0 +1,43 @@ +//! Sandbox tools and OS libraries are authenticated by the existing release. + +use super::{Artifact, LinkInput, PayloadWriter, read_bounded}; +use sha2::{Digest, Sha256}; + +pub(super) fn assemble( + writer: &mut PayloadWriter<'_>, + inputs: Vec<LinkInput>, + target: crate::toolchain::BuildTarget, +) -> Result<Vec<Artifact>, String> { + if inputs.len() > 128 { + return Err("sandbox pack contains too many files".into()); + } + let mut files = Vec::new(); + let mut remaining = 32 * 1024 * 1024; + for input in inputs { + super::runtime_pack::relative_path(&input.path)?; + let source = writer.source(&input.source, &input.path, &input.sha256)?; + let bytes = read_bounded(&source, remaining, "sandbox asset")?; + if format!("{:x}", Sha256::digest(&bytes)) != input.sha256 { + return Err("sandbox asset changed during packaging".into()); + } + remaining -= bytes.len() as u64; + files.push((input.path, bytes)); + } + files.sort_by(|left, right| left.0.cmp(&right.0)); + dever_sandbox::validate_assets_for_target(&files, target.platform())?; + // Leaf artifacts avoid a second manifest/signature or extraction protocol. + let mut artifacts = Vec::new(); + for (path, bytes) in files { + let destination = format!("sandbox/{}/{path}", target.platform()); + let output = writer.reserve(&destination)?; + std::fs::write(&output, &bytes) + .map_err(|error| format!("cannot write sandbox asset: {error}"))?; + super::set_mode(&output, dever_sandbox::asset_executable(&path))?; + artifacts.push(Artifact { + path: destination, + bytes: bytes.len() as u64, + sha256: format!("{:x}", Sha256::digest(bytes)), + }); + } + Ok(artifacts) +} diff --git a/crates/dever-cli/src/toolchain/release.rs b/crates/dever-cli/src/toolchain/release.rs new file mode 100644 index 0000000..83776d9 --- /dev/null +++ b/crates/dever-cli/src/toolchain/release.rs @@ -0,0 +1,1194 @@ +use std::collections::BTreeSet; +use std::fmt; +use std::fs::{self, File, OpenOptions}; +use std::io::{self, Read, Write}; +use std::path::{Component, Path, PathBuf}; +#[cfg(not(target_os = "linux"))] +use std::process::Command; +use std::str::FromStr; +use std::sync::atomic::{AtomicU64, Ordering}; + +use ring::signature::{ED25519, UnparsedPublicKey}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +static NEXT_TRANSACTION: AtomicU64 = AtomicU64::new(0); + +const MANIFEST_NAME: &str = "manifest.json"; +const SIGNATURE_NAME: &str = "manifest.sig"; +const TRUSTED_KEY_NAME: &str = "trusted-release-key"; + +#[derive(Clone, Debug, Eq, Hash, PartialEq, Serialize, Deserialize)] +#[serde(transparent)] +pub struct Version(String); + +impl Version { + pub fn parse(value: &str) -> Result<Self, String> { + dever_runtime::config::validate_dever_version(value)?; + Ok(Self(value.to_owned())) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Display for Version { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.0) + } +} + +impl Ord for Version { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + version_components(self.as_str()).cmp(&version_components(other.as_str())) + } +} + +impl PartialOrd for Version { + fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> { + Some(self.cmp(other)) + } +} + +impl FromStr for Version { + type Err = String; + + fn from_str(value: &str) -> Result<Self, Self::Err> { + Self::parse(value) + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Artifact { + pub path: String, + pub bytes: u64, + pub sha256: String, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ReleaseManifest { + pub format: String, + pub version: Version, + pub platform: String, + pub artifacts: Vec<Artifact>, +} + +impl ReleaseManifest { + pub fn new(version: Version, artifacts: Vec<Artifact>) -> Self { + Self { + format: "dever-release-v1".into(), + version, + platform: platform_identity(), + artifacts, + } + } +} + +#[derive(Clone, Debug)] +pub struct Layout { + root: PathBuf, +} + +impl Layout { + pub fn new(root: impl Into<PathBuf>) -> Self { + Self { root: root.into() } + } + + pub fn from_launcher(executable: &Path) -> Result<Self, String> { + let executable = fs::canonicalize(executable).map_err(|error| { + format!( + "cannot resolve launcher '{}': {error}", + executable.display() + ) + })?; + let bin = executable + .parent() + .ok_or("dever launcher has no bin directory")?; + if bin.file_name().and_then(|name| name.to_str()) != Some("bin") { + return Err("dever launcher must be installed under <toolchain>/bin".into()); + } + let root = bin.parent().ok_or("dever launcher has no toolchain root")?; + Ok(Self::new(root)) + } + + pub fn root(&self) -> &Path { + &self.root + } + pub fn bin(&self) -> PathBuf { + self.root.join("bin") + } + pub fn state(&self) -> PathBuf { + self.root.join("state") + } + pub fn versions(&self) -> PathBuf { + self.root.join("versions") + } + pub fn cache(&self) -> PathBuf { + self.root.join("cache") + } + pub fn downloads(&self) -> PathBuf { + self.cache().join("downloads") + } + pub fn staging(&self) -> PathBuf { + self.cache().join("staging") + } + pub fn native_cache(&self) -> PathBuf { + self.cache().join("native") + } + /// The local control socket is kept inside the machine state directory for + /// development fixtures. Platform installers may bind the documented + /// system endpoint to this service, but callers never fall back to the + /// cache directory itself. + pub fn service_socket(&self) -> PathBuf { + if self.root + == Path::new( + platform_contract(std::env::consts::OS) + .map(|contract| contract.machine_root) + .unwrap_or(""), + ) + { + return PathBuf::from( + platform_contract(std::env::consts::OS) + .expect("current platform contract") + .ipc_endpoint, + ); + } + self.state().join("deverd.sock") + } + pub fn service_token(&self) -> PathBuf { + self.state().join("deverd.token") + } + + pub fn initialize(&self) -> Result<(), String> { + reject_symlink_ancestors(&self.root)?; + for directory in [ + self.bin(), + self.state(), + self.versions(), + self.downloads(), + self.staging(), + self.native_cache(), + ] { + create_private_directory(&directory)?; + } + Ok(()) + } + + pub fn validate_machine_permissions(&self) -> Result<(), String> { + for path in [self.root.clone(), self.bin(), self.state(), self.versions()] { + let metadata = fs::symlink_metadata(&path).map_err(|error| { + format!("cannot inspect machine path '{}': {error}", path.display()) + })?; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Err(format!( + "machine path '{}' must be a real directory", + path.display() + )); + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + if metadata.uid() != 0 || metadata.mode() & 0o022 != 0 { + return Err(format!( + "machine path '{}' must be owned by root and not writable by group or other users", + path.display() + )); + } + } + } + let cache = fs::symlink_metadata(self.cache()) + .map_err(|error| format!("cannot inspect machine cache: {error}"))?; + if cache.file_type().is_symlink() || !cache.is_dir() { + return Err("machine cache must be a real directory".into()); + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + if cache.mode() & 0o002 != 0 { + return Err("machine cache must not be writable by other users".into()); + } + } + Ok(()) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct PlatformContract { + pub machine_root: &'static str, + pub public_entry: &'static str, + pub service_definition: &'static str, + pub ipc_endpoint: &'static str, +} + +pub fn platform_contract(os: &str) -> Option<PlatformContract> { + match os { + "linux" => Some(PlatformContract { + machine_root: "/opt/dever", + public_entry: "/usr/local/bin/dever", + service_definition: "/etc/systemd/system/deverd.service", + ipc_endpoint: "/run/dever/deverd.sock", + }), + "macos" => Some(PlatformContract { + machine_root: "/Library/Application Support/Dever", + public_entry: "/usr/local/bin/dever", + service_definition: "/Library/LaunchDaemons/dev.dever.deverd.plist", + ipc_endpoint: "/var/run/dever/deverd.sock", + }), + "windows" => Some(PlatformContract { + machine_root: r"C:\Program Files\Dever", + public_entry: r"C:\Program Files\Dever\bin\dever.exe", + service_definition: "DeverBuildService", + ipc_endpoint: r"\\.\pipe\deverd", + }), + _ => None, + } +} + +pub fn platform_identity() -> String { + format!("{}-{}", std::env::consts::OS, std::env::consts::ARCH) +} + +#[derive(Clone, Debug)] +pub struct DownloadedReleases { + root: PathBuf, +} + +impl DownloadedReleases { + pub fn new(root: impl Into<PathBuf>) -> Self { + Self { root: root.into() } + } + + pub fn package(&self, version: &Version) -> PathBuf { + self.root.join(version.as_str()) + } + + pub fn latest(&self) -> Result<Version, String> { + let path = self.root.join("latest"); + let value = String::from_utf8(read_regular_file(&path)?) + .map_err(|_| format!("release catalog '{}' must be UTF-8", path.display()))?; + Version::parse(value.trim()) + .map_err(|error| format!("invalid latest release in '{}': {error}", path.display())) + } +} + +pub struct MachineManager { + layout: Layout, + releases: DownloadedReleases, +} + +impl MachineManager { + pub fn new(layout: Layout) -> Self { + let releases = DownloadedReleases::new(layout.downloads()); + Self { layout, releases } + } + + pub fn layout(&self) -> &Layout { + &self.layout + } + + pub fn install_requested(&self, requested: &str) -> Result<Version, String> { + let version = if requested == "latest" { + self.releases.latest()? + } else { + Version::parse(requested)? + }; + let _lock = InstallLock::acquire(&self.layout)?; + #[cfg(target_os = "linux")] + super::bootstrap::require_no_pending(&self.layout)?; + self.cleanup_staging()?; + self.install_locked(&version)?; + if self.active_version()?.is_none() { + self.activate_locked(&version)?; + } + Ok(version) + } + + pub fn update(&self) -> Result<Version, String> { + let version = self.releases.latest()?; + let _lock = InstallLock::acquire(&self.layout)?; + #[cfg(target_os = "linux")] + super::bootstrap::require_no_pending(&self.layout)?; + if self + .active_version()? + .is_some_and(|active| version < active) + { + return Err(format!( + "refusing to downgrade active Dever with 'dever update' to {version}; use 'dever use {version}' for an explicit installed-version switch" + )); + } + self.cleanup_staging()?; + self.install_locked(&version)?; + self.activate_locked(&version)?; + Ok(version) + } + + pub fn activate(&self, version: &Version) -> Result<(), String> { + let _lock = InstallLock::acquire(&self.layout)?; + #[cfg(target_os = "linux")] + super::bootstrap::require_no_pending(&self.layout)?; + self.activate_locked(version) + } + + pub fn uninstall(&self, version: &Version) -> Result<(), String> { + let _lock = InstallLock::acquire(&self.layout)?; + #[cfg(target_os = "linux")] + super::bootstrap::require_no_pending(&self.layout)?; + if self.active_version()?.as_ref() == Some(version) { + return Err(format!( + "cannot uninstall active Dever {version}; run 'dever use <other-version>' first" + )); + } + let installed = self.layout.versions().join(version.as_str()); + ensure_real_directory(&installed, "installed version")?; + let staging = self.transaction_path("uninstall"); + fs::rename(&installed, &staging) + .map_err(|error| format!("cannot stage Dever {version} for removal: {error}"))?; + if let Err(error) = sync_directory(&self.layout.versions()) { + let rollback = fs::rename(&staging, &installed) + .and_then(|()| sync_directory(&self.layout.versions())); + return match rollback { + Ok(()) => Err(format!( + "cannot persist Dever {version} uninstallation: {error}" + )), + Err(rollback_error) => Err(format!( + "cannot persist Dever {version} uninstallation ({error}) or restore it ({rollback_error})" + )), + }; + } + // The durable rename is the logical uninstall commit. Physical removal is + // recoverable staging cleanup and must not report failure after that commit. + let _ = fs::remove_dir_all(&staging); + Ok(()) + } + + pub fn active_version(&self) -> Result<Option<Version>, String> { + let path = self.layout.state().join("active-version"); + match fs::symlink_metadata(&path) { + Ok(_) => { + validate_shared_file(&path, &self.layout.state(), "active Dever state", false)? + } + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(format!("cannot inspect '{}': {error}", path.display())), + } + let value = match fs::read_to_string(&path) { + Ok(value) => value, + Err(error) => return Err(format!("cannot read '{}': {error}", path.display())), + }; + let mut lines = value.lines(); + if lines.next() != Some("dever-active-version-v1") { + return Err(format!( + "invalid active Dever state in '{}'", + path.display() + )); + } + let mut selected = None; + for line in lines { + let Some((version, checksum)) = line.split_once(' ') else { + continue; + }; + if checksum == active_checksum(version) { + selected = Some(Version::parse(version).map_err(|error| { + format!( + "invalid active Dever version in '{}': {error}", + path.display() + ) + })?); + } + } + Ok(selected) + } + + pub fn installed_versions(&self) -> Result<Vec<Version>, String> { + let mut versions = Vec::new(); + for entry in fs::read_dir(self.layout.versions()) + .map_err(|error| format!("cannot list installed Dever versions: {error}"))? + { + let entry = + entry.map_err(|error| format!("cannot list installed Dever versions: {error}"))?; + if entry + .file_type() + .map_err(|error| error.to_string())? + .is_dir() + { + let name = entry + .file_name() + .into_string() + .map_err(|_| "installed Dever version name is not UTF-8".to_owned())?; + versions.push( + Version::parse(&name).map_err(|_| { + format!("invalid directory in machine version store: {name}") + })?, + ); + } + } + versions.sort(); + Ok(versions) + } + + pub fn resolve_core(&self, version: &Version) -> Result<PathBuf, String> { + let root = self.layout.versions().join(version.as_str()); + if !root.exists() { + return Err(format!( + "Dever {version} is not installed; run 'dever install {version}'" + )); + } + let manifest = self.verify_release(&root, Some(version))?; + validate_installed_release(&self.layout, &root, &manifest)?; + Ok(root.join(core_path(&manifest)?)) + } + + pub(super) fn resolve_skill(&self, version: &Version) -> Result<PathBuf, String> { + let root = self.layout.versions().join(version.as_str()); + let manifest = self.verify_release(&root, Some(version))?; + validate_installed_release(&self.layout, &root, &manifest)?; + if !manifest + .artifacts + .iter() + .any(|artifact| artifact.path == "skills/dever-language/SKILL.md") + { + return Err(format!( + "Dever {version} has no signed dever-language skill; install a complete release" + )); + } + Ok(root.join("skills/dever-language")) + } + + /// The lease prevents installation/removal while a trusted worker uses its + /// compiler and link inputs. Callers cannot select a compiler filesystem path. + pub(crate) fn compilation(&self, version: &Version) -> Result<InstalledCompiler, String> { + let lease = InstallLock::shared(&self.layout)?; + let root = self.layout.versions().join(version.as_str()); + let manifest = self.verify_release(&root, Some(version))?; + validate_installed_release(&self.layout, &root, &manifest)?; + let core = fs::canonicalize(root.join(core_path(&manifest)?)) + .map_err(|error| format!("cannot resolve installed compiler: {error}"))?; + Ok(InstalledCompiler { + core, + manifest, + identity: sha256_file(&root.join(MANIFEST_NAME))?, + _lease: lease, + }) + } + + fn install_locked(&self, version: &Version) -> Result<(), String> { + self.install_package_locked(&self.releases.package(version), version) + } + + pub(super) fn install_package_locked( + &self, + package: &Path, + version: &Version, + ) -> Result<(), String> { + let destination = self.layout.versions().join(version.as_str()); + if destination.exists() { + let manifest = self.verify_release(&destination, Some(version))?; + validate_installed_release(&self.layout, &destination, &manifest)?; + return Ok(()); + } + let manifest = self.verify_release(package, Some(version))?; + let core_relative = core_path(&manifest)?.to_path_buf(); + let staging = self.transaction_path("install"); + create_private_directory(&staging)?; + let result = (|| { + copy_file( + &package.join(MANIFEST_NAME), + &staging.join(MANIFEST_NAME), + false, + )?; + copy_file( + &package.join(SIGNATURE_NAME), + &staging.join(SIGNATURE_NAME), + false, + )?; + for artifact in &manifest.artifacts { + let relative = safe_relative(&artifact.path)?; + copy_file( + &artifact_path(package, relative)?, + &staging.join(relative), + relative == core_relative || bootstrap_executable(relative), + )?; + } + self.verify_release(&staging, Some(version))?; + validate_installed_release(&self.layout, &staging, &manifest)?; + let core = staging.join(core_path(&manifest)?); + #[cfg(target_os = "linux")] + super::bootstrap::health_check(&core, "--dever-health")?; + #[cfg(not(target_os = "linux"))] + { + let status = Command::new(&core) + .arg("--dever-health") + .env_clear() + .status() + .map_err(|error| format!("cannot health-check Dever {version}: {error}"))?; + if !status.success() { + return Err(format!( + "Dever {version} failed its installation health check" + )); + } + } + sync_tree(&staging)?; + fs::rename(&staging, &destination) + .map_err(|error| format!("cannot publish Dever {version}: {error}"))?; + sync_directory(&self.layout.versions()) + .map_err(|error| format!("cannot persist Dever {version} installation: {error}"))?; + Ok(()) + })(); + if result.is_err() { + let _ = fs::remove_dir_all(&staging); + } + result + } + + fn activate_locked(&self, version: &Version) -> Result<(), String> { + self.resolve_core(version)?; + append_active_version(&self.layout.state().join("active-version"), version) + .map_err(|error| format!("cannot activate Dever {version}: {error}")) + } + + fn verify_release( + &self, + root: &Path, + expected: Option<&Version>, + ) -> Result<ReleaseManifest, String> { + let trusted_key_path = self.layout.state().join(TRUSTED_KEY_NAME); + validate_shared_file( + &trusted_key_path, + &self.layout.state(), + "trusted release key", + false, + )?; + let trusted_key = decode_hex_file(&trusted_key_path, 32)?; + Self::verify_package(root, expected, &trusted_key) + } + + pub(super) fn verify_package( + root: &Path, + expected: Option<&Version>, + trusted_key: &[u8], + ) -> Result<ReleaseManifest, String> { + let manifest = Self::signed_manifest(root, trusted_key)?; + if expected.is_some_and(|expected| expected != &manifest.version) { + return Err(format!( + "release version does not match requested {expected:?}" + )); + } + Self::verify_artifacts(root, manifest) + } + + pub(super) fn signed_manifest( + root: &Path, + trusted_key: &[u8], + ) -> Result<ReleaseManifest, String> { + ensure_real_directory(root, "release")?; + let manifest_path = root.join(MANIFEST_NAME); + let manifest_bytes = read_regular_file(&manifest_path)?; + let signature = decode_hex_file(&root.join(SIGNATURE_NAME), 64)?; + UnparsedPublicKey::new(&ED25519, trusted_key) + .verify(&manifest_bytes, &signature) + .map_err(|_| { + format!( + "release signature verification failed for '{}'", + root.display() + ) + })?; + let manifest: ReleaseManifest = + serde_json::from_slice(&manifest_bytes).map_err(|error| { + format!( + "invalid release manifest '{}': {error}", + manifest_path.display() + ) + })?; + if manifest.format != "dever-release-v1" { + return Err("unsupported Dever release manifest format".into()); + } + if manifest.platform != platform_identity() { + return Err(format!( + "release platform '{}' does not match '{}'", + manifest.platform, + platform_identity() + )); + } + Ok(manifest) + } + + fn verify_artifacts(root: &Path, manifest: ReleaseManifest) -> Result<ReleaseManifest, String> { + let mut paths = BTreeSet::new(); + for artifact in &manifest.artifacts { + let relative = safe_relative(&artifact.path)?; + if !paths.insert(relative.to_path_buf()) { + return Err(format!("duplicate release artifact '{}'", artifact.path)); + } + if artifact.sha256.len() != 64 + || !artifact + .sha256 + .bytes() + .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) + { + return Err(format!( + "invalid SHA-256 for release artifact '{}'", + artifact.path + )); + } + let path = artifact_path(root, relative)?; + let metadata = fs::symlink_metadata(&path).map_err(|error| { + format!( + "cannot inspect release artifact '{}': {error}", + path.display() + ) + })?; + if !metadata.file_type().is_file() { + return Err(format!( + "release artifact '{}' is not a regular file", + path.display() + )); + } + if metadata.len() != artifact.bytes || sha256_file(&path)? != artifact.sha256 { + return Err(format!( + "release artifact '{}' failed integrity verification", + artifact.path + )); + } + } + core_path(&manifest)?; + let native_manifest = format!("runtime/{}/manifest.json", platform_identity()); + if manifest + .artifacts + .iter() + .any(|artifact| artifact.path == native_manifest) + { + super::core_libraries::validate(root, &manifest.artifacts)?; + } + #[cfg(target_os = "linux")] + if manifest + .artifacts + .iter() + .any(|artifact| artifact.path.starts_with("bootstrap/")) + { + // Older installed versions retain their own signed service template. + // Only the explicit bootstrap installer applies the current template. + super::bootstrap::validate_payload(root, &manifest.artifacts, false)?; + } + Ok(manifest) + } + + fn cleanup_staging(&self) -> Result<(), String> { + for entry in fs::read_dir(self.layout.staging()) + .map_err(|error| format!("cannot inspect toolchain staging: {error}"))? + { + let entry = + entry.map_err(|error| format!("cannot inspect toolchain staging: {error}"))?; + let metadata = fs::symlink_metadata(entry.path()) + .map_err(|error| format!("cannot inspect toolchain staging entry: {error}"))?; + if metadata.file_type().is_symlink() { + fs::remove_file(entry.path()).map_err(|error| error.to_string())?; + } else if metadata.is_dir() { + fs::remove_dir_all(entry.path()).map_err(|error| error.to_string())?; + } else { + fs::remove_file(entry.path()).map_err(|error| error.to_string())?; + } + } + Ok(()) + } + + fn transaction_path(&self, operation: &str) -> PathBuf { + self.layout.staging().join(format!( + "{operation}-{}-{}", + std::process::id(), + NEXT_TRANSACTION.fetch_add(1, Ordering::Relaxed), + )) + } +} + +pub(crate) struct InstalledCompiler { + pub(crate) core: PathBuf, + pub(crate) manifest: ReleaseManifest, + pub(crate) identity: String, + _lease: InstallLock, +} + +pub(super) struct InstallLock(File); + +impl InstallLock { + pub(super) fn acquire(layout: &Layout) -> Result<Self, String> { + Self::open(layout, false) + } + + fn shared(layout: &Layout) -> Result<Self, String> { + Self::open(layout, true) + } + + fn open(layout: &Layout, shared: bool) -> Result<Self, String> { + let path = layout.state().join("install.lock"); + let mut options = OpenOptions::new(); + options.read(true).write(true).create(true).truncate(false); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + match fs::symlink_metadata(&path) { + Ok(_) => { + validate_install_lock(layout, &path)?; + } + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(format!("cannot inspect installation lock: {error}")), + } + let file = options + .open(&path) + .map_err(|error| format!("cannot open Dever installation lock: {error}"))?; + let current = validate_install_lock(layout, &path)?; + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let opened = file.metadata().map_err(|error| error.to_string())?; + if current.dev() != opened.dev() || current.ino() != opened.ino() { + return Err("installation lock changed while opening".into()); + } + } + #[cfg(not(unix))] + let _ = current; + let locked = if shared { + fs2::FileExt::try_lock_shared(&file) + } else { + fs2::FileExt::try_lock_exclusive(&file) + }; + locked.map_err(|error| { + format!("another Dever installation operation or compilation is active: {error}") + })?; + Ok(Self(file)) + } +} + +fn validate_install_lock(layout: &Layout, path: &Path) -> Result<fs::Metadata, String> { + let metadata = fs::symlink_metadata(path) + .map_err(|error| format!("cannot inspect installation lock: {error}"))?; + if !metadata.is_file() || metadata.file_type().is_symlink() { + return Err("installation lock must be a regular file".into()); + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let state = fs::symlink_metadata(layout.state()).map_err(|error| error.to_string())?; + if metadata.uid() != state.uid() || metadata.mode() & 0o077 != 0 { + return Err("installation lock must be private to the service owner".into()); + } + } + #[cfg(not(unix))] + let _ = layout; + Ok(metadata) +} + +impl Drop for InstallLock { + fn drop(&mut self) { + // Keep the inode: unlinking a locked file permits a second independent lock. + let _ = fs2::FileExt::unlock(&self.0); + } +} + +fn core_path(manifest: &ReleaseManifest) -> Result<&Path, String> { + let expected = format!("dever-core{}", std::env::consts::EXE_SUFFIX); + let mut matching = manifest + .artifacts + .iter() + .filter(|artifact| artifact.path == expected); + let core = matching + .next() + .ok_or("release manifest is missing dever-core")?; + if matching.next().is_some() { + return Err("release manifest contains multiple dever-core artifacts".into()); + } + Ok(Path::new(&core.path)) +} + +fn bootstrap_executable(path: &Path) -> bool { + matches!(path.to_str(), Some("bootstrap/dever" | "bootstrap/deverd")) +} + +fn version_components(version: &str) -> [u64; 3] { + let mut parts = version.split('.').map(|part| { + part.parse::<u64>() + .expect("validated Dever version component") + }); + [ + parts.next().expect("validated Dever major version"), + parts.next().expect("validated Dever minor version"), + parts.next().expect("validated Dever patch version"), + ] +} + +fn safe_relative(path: &str) -> Result<&Path, String> { + let path = Path::new(path); + if path.as_os_str().is_empty() + || path.is_absolute() + || path + .components() + .any(|component| !matches!(component, Component::Normal(_))) + { + return Err(format!( + "release artifact path '{path:?}' must be a normalized relative path" + )); + } + Ok(path) +} + +fn artifact_path(root: &Path, relative: &Path) -> Result<PathBuf, String> { + let mut path = root.to_path_buf(); + let count = relative.components().count(); + for (index, component) in relative.components().enumerate() { + let Component::Normal(component) = component else { + return Err("release artifact path must be normalized".into()); + }; + path.push(component); + let metadata = fs::symlink_metadata(&path).map_err(|error| { + format!( + "cannot inspect release artifact '{}': {error}", + path.display() + ) + })?; + if metadata.file_type().is_symlink() + || (index + 1 < count && !metadata.is_dir()) + || (index + 1 == count && !metadata.file_type().is_file()) + { + return Err(format!( + "release artifact '{}' crosses a symlink or invalid path", + path.display() + )); + } + } + Ok(path) +} + +pub(super) fn reject_symlink_ancestors(path: &Path) -> Result<(), String> { + let mut current = PathBuf::new(); + for component in path.components() { + current.push(component); + match fs::symlink_metadata(¤t) { + Ok(metadata) if metadata.file_type().is_symlink() => { + return Err(format!( + "machine path '{}' must not be a symlink", + current.display() + )); + } + Ok(_) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => break, + Err(error) => return Err(format!("cannot inspect '{}': {error}", current.display())), + } + } + Ok(()) +} + +pub(super) fn ensure_real_directory(path: &Path, label: &str) -> Result<(), String> { + let metadata = fs::symlink_metadata(path) + .map_err(|error| format!("cannot inspect {label} '{}': {error}", path.display()))?; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Err(format!( + "{label} '{}' must be a real directory", + path.display() + )); + } + Ok(()) +} + +fn create_private_directory(path: &Path) -> Result<(), String> { + let mut builder = fs::DirBuilder::new(); + builder.recursive(true); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o755); + } + builder + .create(path) + .map_err(|error| format!("cannot create '{}': {error}", path.display()))?; + ensure_real_directory(path, "machine directory") +} + +fn read_regular_file(path: &Path) -> Result<Vec<u8>, String> { + let metadata = fs::symlink_metadata(path) + .map_err(|error| format!("cannot inspect '{}': {error}", path.display()))?; + if !metadata.file_type().is_file() { + return Err(format!("'{}' must be a regular file", path.display())); + } + fs::read(path).map_err(|error| format!("cannot read '{}': {error}", path.display())) +} + +pub(super) fn copy_file(source: &Path, destination: &Path, executable: bool) -> Result<(), String> { + if let Some(parent) = destination.parent() { + create_private_directory(parent)?; + } + let mut source_file = File::open(source) + .map_err(|error| format!("cannot read '{}': {error}", source.display()))?; + if !source_file + .metadata() + .map_err(|error| error.to_string())? + .is_file() + { + return Err(format!("'{}' must be a regular file", source.display())); + } + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut destination_file = options + .open(destination) + .map_err(|error| format!("cannot create '{}': {error}", destination.display()))?; + io::copy(&mut source_file, &mut destination_file) + .and_then(|_| destination_file.sync_all()) + .map_err(|error| format!("cannot copy '{}': {error}", source.display()))?; + set_shared_file_permissions(destination, executable)?; + destination_file.sync_all().map_err(|error| { + format!( + "cannot persist final file mode '{}': {error}", + destination.display() + ) + }) +} + +/// Persist every child directory after its leaves, before publishing a tree. +pub(super) fn sync_tree(root: &Path) -> Result<(), String> { + for entry in fs::read_dir(root).map_err(|error| error.to_string())? { + let entry = entry.map_err(|error| error.to_string())?; + let kind = entry.file_type().map_err(|error| error.to_string())?; + if kind.is_dir() { + sync_tree(&entry.path())?; + } else if !kind.is_file() { + return Err("installation tree contains a non-regular entry".into()); + } + } + sync_directory(root).map_err(|error| error.to_string()) +} + +fn validate_installed_release( + layout: &Layout, + root: &Path, + manifest: &ReleaseManifest, +) -> Result<(), String> { + validate_shared_directory(root, &layout.versions(), "installed release")?; + validate_shared_file(&root.join(MANIFEST_NAME), root, "release manifest", false)?; + validate_shared_file(&root.join(SIGNATURE_NAME), root, "release signature", false)?; + let core = core_path(manifest)?; + for artifact in &manifest.artifacts { + let relative = safe_relative(&artifact.path)?; + let mut parent = root.to_path_buf(); + let components: Vec<_> = relative.components().collect(); + for component in &components[..components.len().saturating_sub(1)] { + let Component::Normal(component) = component else { + return Err("release artifact path must be normalized".into()); + }; + let directory = parent.join(component); + validate_shared_directory(&directory, &parent, "release artifact directory")?; + parent = directory; + } + validate_shared_file( + &root.join(relative), + &parent, + "installed release artifact", + relative == core || bootstrap_executable(relative), + )?; + } + Ok(()) +} + +fn validate_shared_directory(path: &Path, parent: &Path, label: &str) -> Result<(), String> { + let metadata = fs::symlink_metadata(path) + .map_err(|error| format!("cannot inspect {label} '{}': {error}", path.display()))?; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Err(format!( + "{label} '{}' must be a real directory", + path.display() + )); + } + validate_shared_permissions(path, parent, &metadata, label, true, false) +} + +pub(super) fn validate_shared_file( + path: &Path, + parent: &Path, + label: &str, + executable: bool, +) -> Result<(), String> { + let metadata = fs::symlink_metadata(path) + .map_err(|error| format!("cannot inspect {label} '{}': {error}", path.display()))?; + if !metadata.file_type().is_file() { + return Err(format!( + "{label} '{}' must be a regular file", + path.display() + )); + } + validate_shared_permissions(path, parent, &metadata, label, false, executable) +} + +fn validate_shared_permissions( + path: &Path, + parent: &Path, + metadata: &fs::Metadata, + label: &str, + directory: bool, + executable: bool, +) -> Result<(), String> { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let parent_metadata = fs::symlink_metadata(parent).map_err(|error| { + format!( + "cannot inspect {label} parent '{}': {error}", + parent.display() + ) + })?; + let required = if directory || executable { + 0o005 + } else { + 0o004 + }; + if metadata.uid() != parent_metadata.uid() + || metadata.mode() & 0o022 != 0 + || metadata.mode() & required != required + { + return Err(format!( + "{label} '{}' must be owned by the machine store owner, shared read-only, and not writable by group or other users", + path.display() + )); + } + } + #[cfg(windows)] + { + let _ = (path, parent, metadata, label, directory, executable); + } + Ok(()) +} + +pub(super) fn set_shared_file_permissions(path: &Path, executable: bool) -> Result<(), String> { + #[cfg(unix)] + { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + let metadata = fs::symlink_metadata(path) + .map_err(|error| format!("cannot inspect '{}': {error}", path.display()))?; + let mode = if executable { 0o755 } else { 0o644 }; + fs::set_permissions(path, fs::Permissions::from_mode(mode)) + .map_err(|error| format!("cannot protect '{}': {error}", path.display()))?; + let protected = fs::symlink_metadata(path) + .map_err(|error| format!("cannot inspect '{}': {error}", path.display()))?; + if protected.uid() != metadata.uid() || protected.mode() & 0o777 != mode { + return Err(format!( + "cannot protect '{}': permissions did not persist", + path.display() + )); + } + } + #[cfg(windows)] + { + let mut permissions = fs::metadata(path) + .map_err(|error| format!("cannot inspect '{}': {error}", path.display()))? + .permissions(); + let _ = executable; + permissions.set_readonly(true); + fs::set_permissions(path, permissions) + .map_err(|error| format!("cannot protect '{}': {error}", path.display()))?; + } + Ok(()) +} + +pub(super) fn decode_hex_file(path: &Path, expected_bytes: usize) -> Result<Vec<u8>, String> { + let text = String::from_utf8(read_regular_file(path)?) + .map_err(|_| format!("'{}' must contain UTF-8 hexadecimal data", path.display()))?; + let text = text.trim(); + if text.len() != expected_bytes * 2 || !text.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err(format!( + "'{}' must contain {} hexadecimal bytes", + path.display(), + expected_bytes + )); + } + text.as_bytes() + .as_chunks::<2>() + .0 + .iter() + .map(|pair| { + let digits = std::str::from_utf8(pair).expect("hexadecimal bytes are UTF-8"); + u8::from_str_radix(digits, 16) + .map_err(|_| format!("invalid hexadecimal data in '{}'", path.display())) + }) + .collect() +} + +pub fn sha256_file(path: &Path) -> Result<String, String> { + let mut file = + File::open(path).map_err(|error| format!("cannot read '{}': {error}", path.display()))?; + // Runtime packs are checked on every compilation, including cache hits. + // Reuse the existing cryptographic backend's optimized streaming SHA-256. + let mut digest = ring::digest::Context::new(&ring::digest::SHA256); + let mut buffer = [0_u8; 64 * 1024]; + loop { + let length = file + .read(&mut buffer) + .map_err(|error| format!("cannot hash '{}': {error}", path.display()))?; + if length == 0 { + break; + } + digest.update(&buffer[..length]); + } + Ok(hex(digest.finish().as_ref())) +} + +pub fn hex(bytes: &[u8]) -> String { + const DIGITS: &[u8; 16] = b"0123456789abcdef"; + let mut output = String::with_capacity(bytes.len() * 2); + for byte in bytes { + output.push(DIGITS[usize::from(byte >> 4)] as char); + output.push(DIGITS[usize::from(byte & 0x0f)] as char); + } + output +} + +pub(super) fn append_active_version(path: &Path, version: &Version) -> io::Result<()> { + let parent = path + .parent() + .ok_or_else(|| io::Error::other("active state has no parent"))?; + match fs::symlink_metadata(path) { + Ok(_) => validate_shared_file(path, parent, "active Dever state", false) + .map_err(io::Error::other)?, + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + let mut options = OpenOptions::new(); + options.append(true).create(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options.open(path)?; + if file.metadata()?.len() == 0 { + file.write_all(b"dever-active-version-v1\n")?; + } + // A leading newline keeps the next record recoverable even if a previous + // append stopped halfway. Readers use only complete checksum-valid records. + writeln!(file, "\n{} {}", version, active_checksum(version.as_str()))?; + file.sync_all()?; + set_shared_file_permissions(path, false).map_err(io::Error::other)?; + sync_directory(parent) +} + +fn active_checksum(version: &str) -> String { + hex(&Sha256::digest( + [b"dever-active-version-v1\0".as_slice(), version.as_bytes()].concat(), + )) +} + +pub(super) fn sync_directory(path: &Path) -> io::Result<()> { + #[cfg(unix)] + { + File::open(path)?.sync_all() + } + #[cfg(windows)] + { + let _ = path; + Ok(()) + } +} diff --git a/crates/dever-cli/src/toolchain/release_source.rs b/crates/dever-cli/src/toolchain/release_source.rs new file mode 100644 index 0000000..cdf6ef7 --- /dev/null +++ b/crates/dever-cli/src/toolchain/release_source.rs @@ -0,0 +1,387 @@ +//! Explicit installation/update downloads. Project commands never enter here. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs::{self, OpenOptions}; +use std::io::{self, Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +use super::release::{ + InstallLock, Layout, MachineManager, ReleaseManifest, Version, decode_hex_file, + ensure_real_directory, platform_identity, reject_symlink_ancestors, + set_shared_file_permissions, sync_directory, sync_tree, validate_shared_file, +}; + +const OFFICIAL_RELEASES: &str = "https://github.com/shemic/dever-main/releases"; +const DOWNLOAD_TIMEOUT: Duration = Duration::from_secs(300); +const METADATA_LIMIT: u64 = 2 * 1024 * 1024; +const RELEASE_LIMIT: u64 = 2 * 1024 * 1024 * 1024; +static NEXT_DOWNLOAD: AtomicU64 = AtomicU64::new(0); + +struct ReleaseSource { + base: String, + agent: ureq::Agent, +} + +impl ReleaseSource { + fn official() -> Self { + Self::new(OFFICIAL_RELEASES.to_owned()) + } + + fn new(base: String) -> Self { + Self { + base, + agent: ureq::Agent::new_with_config( + ureq::Agent::config_builder() + .proxy(None) + .max_redirects(0) + .timeout_global(Some(DOWNLOAD_TIMEOUT)) + .build(), + ), + } + } + + fn asset(&self, requested: &str, suffix: &str) -> String { + let selection = if requested == "latest" { + "latest/download".to_owned() + } else { + format!("download/v{requested}") + }; + format!( + "{}/{selection}/dever-{}.{suffix}", + self.base, + platform_identity() + ) + } + + fn response( + &self, + mut address: String, + started: Instant, + ) -> Result<ureq::http::Response<ureq::Body>, String> { + for _ in 0..=5 { + let remaining = DOWNLOAD_TIMEOUT + .checked_sub(started.elapsed()) + .filter(|duration| !duration.is_zero()) + .ok_or("Dever release download exceeded its five-minute deadline")?; + let response = self + .agent + .get(&address) + .config() + .timeout_global(Some(remaining)) + .build() + .call() + .map_err(|error| match error { + ureq::Error::StatusCode(404) => { + "the official Dever release is not published for this version/platform; no installed version was changed".to_owned() + } + ureq::Error::StatusCode(code) => { + format!("official Dever release download returned HTTP {code}") + } + _ => "cannot download the official Dever release".to_owned(), + })?; + if response.status().is_success() { + return Ok(response); + } + if !response.status().is_redirection() { + return Err("unexpected official release HTTP status".into()); + } + let mut locations = response.headers().get_all("location").iter(); + let location = locations + .next() + .and_then(|value| value.to_str().ok()) + .ok_or("release redirect is missing a valid Location")?; + if locations.next().is_some() { + return Err("release redirect has multiple Locations".into()); + } + address = redirect(&address, location)?; + } + Err("official release download exceeded its redirect limit".into()) + } + + fn metadata( + &self, + requested: &str, + suffix: &str, + path: &Path, + started: Instant, + ) -> Result<(), String> { + let mut response = self.response(self.asset(requested, suffix), started)?; + let mut bytes = Vec::new(); + response + .body_mut() + .as_reader() + .take(METADATA_LIMIT + 1) + .read_to_end(&mut bytes) + .map_err(|_| "cannot read official release metadata")?; + if bytes.len() as u64 > METADATA_LIMIT { + return Err("official release metadata exceeds its size limit".into()); + } + write_new(path, &bytes) + } +} + +fn redirect(current: &str, location: &str) -> Result<String, String> { + let next = url::Url::parse(current) + .and_then(|base| base.join(location)) + .map_err(|_| "invalid release redirect")?; + if next.scheme() != "https" + || !next.username().is_empty() + || next.password().is_some() + || next.port_or_known_default() != Some(443) + || next.fragment().is_some() + || !matches!( + next.host_str(), + Some( + "github.com" + | "release-assets.githubusercontent.com" + | "objects.githubusercontent.com" + ) + ) + { + return Err("release redirect must remain on the official HTTPS asset hosts".into()); + } + Ok(next.into()) +} + +pub(super) fn prepare(layout: &Layout, requested: &str) -> Result<Version, String> { + prepare_from(layout, requested, &ReleaseSource::official()) +} + +fn prepare_from( + layout: &Layout, + requested: &str, + source: &ReleaseSource, +) -> Result<Version, String> { + if requested != "latest" { + Version::parse(requested)?; + } + layout.validate_machine_permissions()?; + for directory in [layout.downloads(), layout.staging()] { + reject_symlink_ancestors(&directory)?; + ensure_real_directory(&directory, "release download directory")?; + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let metadata = fs::metadata(&directory).map_err(|error| error.to_string())?; + if metadata.uid() != 0 || metadata.mode() & 0o022 != 0 { + return Err( + "release download directories must be root-owned and not publicly writable" + .into(), + ); + } + } + } + let _lock = InstallLock::acquire(layout)?; + #[cfg(target_os = "linux")] + super::bootstrap::require_no_pending(layout)?; + let key_path = layout.state().join("trusted-release-key"); + validate_shared_file(&key_path, &layout.state(), "trusted release key", false)?; + let key = decode_hex_file(&key_path, 32)?; + let staging = Download::new(&layout.staging())?; + let started = Instant::now(); + source.metadata( + requested, + "manifest.json", + &staging.0.join("manifest.json"), + started, + )?; + source.metadata( + requested, + "manifest.sig", + &staging.0.join("manifest.sig"), + started, + )?; + let manifest = MachineManager::signed_manifest(&staging.0, &key)?; + if !manifest + .artifacts + .iter() + .any(|artifact| artifact.path == "skills/dever-language/SKILL.md") + { + return Err("official releases must include the matching dever-language skill".into()); + } + if requested != "latest" && manifest.version.as_str() != requested { + return Err("downloaded release version does not match the requested version".into()); + } + validate_catalog(&manifest)?; + let version = manifest.version.clone(); + let destination = layout.downloads().join(version.as_str()); + match fs::symlink_metadata(&destination) { + Ok(_) => { + MachineManager::verify_package(&destination, Some(&version), &key)?; + if fs::read(destination.join("manifest.json")).map_err(|error| error.to_string())? + != fs::read(staging.0.join("manifest.json")).map_err(|error| error.to_string())? + { + return Err("published Dever version changed its signed manifest".into()); + } + } + Err(error) if error.kind() == io::ErrorKind::NotFound => { + let mut response = + source.response(source.asset(version.as_str(), "tar.gz"), started)?; + let compressed = response.body_mut().as_reader().take(RELEASE_LIMIT + 1); + unpack( + flate2::read::GzDecoder::new(compressed), + &staging.0, + &manifest, + )?; + MachineManager::verify_package(&staging.0, Some(&version), &key)?; + sync_tree(&staging.0)?; + publish(&staging.0, &destination)?; + sync_directory(&layout.downloads()).map_err(|error| error.to_string())?; + } + Err(error) => return Err(format!("cannot inspect downloaded release: {error}")), + } + // Latest is only a catalog hint; the signed version and active journal own trust. + if requested == "latest" { + let latest = layout.downloads().join("latest"); + match fs::symlink_metadata(&latest) { + Ok(_) => validate_shared_file(&latest, &layout.downloads(), "release catalog", false)?, + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(format!("cannot inspect release catalog: {error}")), + } + let temporary = Download::new(&layout.staging())?; + let catalog = temporary.0.join("latest"); + write_new(&catalog, format!("{version}\n").as_bytes())?; + fs::rename(catalog, latest) + .map_err(|error| format!("cannot publish release catalog: {error}"))?; + sync_directory(&layout.downloads()).map_err(|error| error.to_string())?; + } + Ok(version) +} + +fn validate_catalog(manifest: &ReleaseManifest) -> Result<(), String> { + if manifest.artifacts.is_empty() || manifest.artifacts.len() > 4096 { + return Err("release artifact count is outside 1..=4096".into()); + } + let mut paths = BTreeSet::new(); + let mut size = 0_u64; + for artifact in &manifest.artifacts { + if artifact + .path + .split('/') + .any(|part| part.is_empty() || part == "." || part == "..") + || artifact.path.contains(['\\', ':', '\0']) + || matches!(artifact.path.as_str(), "manifest.json" | "manifest.sig") + || !paths.insert(&artifact.path) + { + return Err("release has an unsafe or duplicate artifact path".into()); + } + size = size + .checked_add(artifact.bytes) + .ok_or("release size overflow")?; + if size > RELEASE_LIMIT { + return Err("release exceeds its two-GiB unpacked size limit".into()); + } + } + Ok(()) +} + +fn unpack(reader: impl Read, root: &Path, manifest: &ReleaseManifest) -> Result<(), String> { + validate_catalog(manifest)?; + let expected: BTreeMap<_, _> = manifest + .artifacts + .iter() + .map(|artifact| (artifact.path.as_str(), artifact)) + .collect(); + let mut seen = BTreeSet::new(); + // Raw entries reject PAX/GNU extensions before tar can buffer arbitrary metadata. + let mut archive = tar::Archive::new(reader.take(RELEASE_LIMIT + 4096 * 1024)); + for entry in archive + .entries() + .map_err(|error| format!("invalid release archive: {error}"))? + .raw(true) + { + let mut entry = entry.map_err(|error| format!("invalid release archive entry: {error}"))?; + if !entry.header().entry_type().is_file() { + return Err("release archives may contain only regular files".into()); + } + let bytes = entry.path_bytes(); + let name = std::str::from_utf8(&bytes) + .map_err(|_| "release archive path is not UTF-8")? + .to_owned(); + let artifact = expected + .get(name.as_str()) + .ok_or("release archive contains an unsigned file")?; + if !seen.insert(name.clone()) || entry.size() != artifact.bytes { + return Err("release archive contains a duplicate or wrong-sized file".into()); + } + let path = root.join(&name); + fs::create_dir_all(path.parent().expect("artifact has a parent")) + .map_err(|error| format!("cannot create release artifact directory: {error}"))?; + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(&path) + .map_err(|error| format!("cannot create release artifact: {error}"))?; + let bytes = io::copy(&mut entry, &mut file) + .map_err(|error| format!("cannot extract release artifact: {error}"))?; + if bytes != artifact.bytes { + return Err("release archive contains a truncated file".into()); + } + set_shared_file_permissions(&path, false)?; + file.sync_all().map_err(|error| error.to_string())?; + } + if seen.len() != expected.len() { + return Err("release archive is missing signed artifacts".into()); + } + Ok(()) +} + +fn write_new(path: &Path, bytes: &[u8]) -> Result<(), String> { + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .map_err(|error| format!("cannot create release metadata: {error}"))?; + file.write_all(bytes) + .and_then(|()| file.sync_all()) + .map_err(|error| error.to_string())?; + set_shared_file_permissions(path, false) +} + +fn publish(source: &Path, destination: &Path) -> Result<(), String> { + #[cfg(target_os = "linux")] + { + use rustix::fs::{CWD, RenameFlags, renameat_with}; + renameat_with(CWD, source, CWD, destination, RenameFlags::NOREPLACE) + .map_err(|error| format!("cannot publish downloaded release: {error}")) + } + #[cfg(not(target_os = "linux"))] + { + let _ = (source, destination); + Err("official release downloads currently require Linux".into()) + } +} + +struct Download(PathBuf); + +impl Download { + fn new(parent: &Path) -> Result<Self, String> { + let path = parent.join(format!( + "download-{}-{}", + std::process::id(), + NEXT_DOWNLOAD.fetch_add(1, Ordering::Relaxed) + )); + let mut builder = fs::DirBuilder::new(); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + builder + .create(&path) + .map_err(|error| format!("cannot create release download staging: {error}"))?; + Ok(Self(path)) + } +} + +impl Drop for Download { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +#[cfg(test)] +#[path = "../../../../test/dever-cli-tests/release_source.rs"] +mod tests; diff --git a/crates/dever-cli/src/toolchain/runtime_pack.rs b/crates/dever-cli/src/toolchain/runtime_pack.rs new file mode 100644 index 0000000..596f1d2 --- /dev/null +++ b/crates/dever-cli/src/toolchain/runtime_pack.rs @@ -0,0 +1,464 @@ +//! Compiler-relative, explicitly supplied native link inputs. This is not a +//! package resolver or a substitute for the managed compiler service. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::io::{Read, Seek, SeekFrom}; +use std::path::{Path, PathBuf}; + +use dever_runtime::config::RuntimeProfile; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use super::{BuildTarget, sha256_file}; + +const MANIFEST_LIMIT: u64 = 1024 * 1024; + +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Manifest { + pub(super) format: String, + pub(super) compiler_version: String, + pub(super) abi: u32, + pub(super) platform: String, + pub(super) target: String, + pub(super) profiles: Profiles, + pub(super) files: Vec<RuntimeArtifact>, +} + +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Profiles { + pub(super) base: Option<Profile>, + pub(super) sqlite: Option<Profile>, + pub(super) postgres: Option<Profile>, + pub(super) both: Option<Profile>, +} + +impl Profiles { + fn all(&self) -> [(&'static str, Option<&Profile>); 4] { + [ + ("base", self.base.as_ref()), + ("sqlite", self.sqlite.as_ref()), + ("postgres", self.postgres.as_ref()), + ("both", self.both.as_ref()), + ] + } +} + +#[derive(Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Profile { + pub(super) start: Vec<String>, + pub(super) runtime: String, + pub(super) libraries: Vec<String>, + pub(super) end: Vec<String>, +} + +impl Profile { + fn inputs(&self) -> impl Iterator<Item = &str> { + self.start + .iter() + .chain(std::iter::once(&self.runtime)) + .chain(&self.libraries) + .chain(&self.end) + .map(String::as_str) + } +} + +#[derive(Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct RuntimeArtifact { + pub path: String, + pub bytes: u64, + pub sha256: String, +} + +pub struct RuntimePack { + root: PathBuf, + pub target: BuildTarget, + pub identity: String, + profile: Profile, + files: BTreeMap<String, RuntimeArtifact>, +} + +pub struct LinkInputs { + pub start: Vec<PathBuf>, + pub libraries: Vec<PathBuf>, + pub end: Vec<PathBuf>, +} + +struct LoadedManifest { + root: PathBuf, + target: BuildTarget, + bytes: Vec<u8>, + profiles: Profiles, + files: BTreeMap<String, RuntimeArtifact>, +} + +fn load_manifest( + directory: &Path, + compiler_version: &str, + selected: BuildTarget, +) -> Result<LoadedManifest, String> { + let target = selected.triple(); + let platform = selected.platform(); + let root = checked_path(directory, &format!("runtime/{platform}"), false)?; + let path = checked_path(&root, "manifest.json", true)?; + let mut bytes = Vec::new(); + fs::File::open(&path) + .and_then(|file| file.take(MANIFEST_LIMIT + 1).read_to_end(&mut bytes)) + .map_err(|error| format!("cannot read runtime pack manifest: {error}"))?; + if bytes.len() as u64 > MANIFEST_LIMIT { + return Err("runtime pack manifest exceeds 1 MiB".into()); + } + let manifest: Manifest = serde_json::from_slice(&bytes) + .map_err(|error| format!("invalid runtime pack manifest: {error}"))?; + if manifest.format != "dever-native-runtime-v1" + || manifest.compiler_version != compiler_version + // A v1 manifest describes v1 runtime symbols. Even bridge metadata + // references would link the native SDK into the launcher and daemon. + || manifest.abi != 1 + || manifest.platform != platform + || manifest.target != target + { + return Err( + "runtime pack format, compiler version, ABI or target does not match this compiler" + .into(), + ); + } + let files = validate_files(manifest.files)?; + validate_profiles(&manifest.profiles, &files)?; + Ok(LoadedManifest { + root, + target: selected, + bytes, + profiles: manifest.profiles, + files, + }) +} + +/// Return the entire runtime closure that must be covered by the signed release. +pub fn validate( + directory: &Path, + compiler_version: &str, + target: BuildTarget, +) -> Result<Vec<RuntimeArtifact>, String> { + let manifest = load_manifest(directory, compiler_version, target)?; + let prefix = format!("runtime/{}", target.platform()); + let mut artifacts = vec![RuntimeArtifact { + path: format!("{prefix}/manifest.json"), + bytes: manifest.bytes.len() as u64, + sha256: format!("{:x}", Sha256::digest(&manifest.bytes)), + }]; + for (path, artifact) in manifest.files { + verify_file( + &checked_path(&manifest.root, &path, true)?, + &artifact, + target, + )?; + artifacts.push(RuntimeArtifact { + path: format!("{prefix}/{path}"), + ..artifact + }); + } + Ok(artifacts) +} + +pub(super) fn host_target() -> Result<&'static str, String> { + Ok(BuildTarget::host()?.triple()) +} + +impl RuntimePack { + pub fn load( + directory: &Path, + profile: RuntimeProfile, + selected: BuildTarget, + ) -> Result<Self, String> { + let LoadedManifest { + root, + target, + bytes, + profiles, + files, + } = load_manifest(directory, env!("CARGO_PKG_VERSION"), selected)?; + let name = match (profile.sqlite, profile.postgres) { + (false, false) => "base", + (true, false) => "sqlite", + (false, true) => "postgres", + (true, true) => "both", + }; + let profile = profiles + .all() + .into_iter() + .find_map(|(key, value)| (key == name).then_some(value).flatten()) + .ok_or_else(|| format!("runtime pack is missing the required '{name}' profile"))? + .clone(); + // Validate even on a cache hit: a corrupt installed pack is never + // silently replaced with an old cached executable. + for input in profile.inputs() { + verify_file(&checked_path(&root, input, true)?, &files[input], selected)?; + } + Ok(Self { + root, + target, + identity: format!("{name}:{:x}", Sha256::digest(bytes)), + profile, + files, + }) + } + + pub fn snapshot(&self, directory: &Path) -> Result<LinkInputs, String> { + let destination = directory.join("runtime"); + fs::create_dir(&destination) + .map_err(|error| format!("cannot create runtime input snapshot: {error}"))?; + let mut inputs = Vec::new(); + for (index, input) in self.profile.inputs().enumerate() { + let source = checked_path(&self.root, input, true)?; + let extension = Path::new(input) + .extension() + .expect("validated input extension"); + let output = destination + .join(index.to_string()) + .with_extension(extension); + fs::copy(&source, &output) + .map_err(|error| format!("cannot snapshot runtime input '{input}': {error}"))?; + verify_file(&output, &self.files[input], self.target)?; + inputs.push(output); + } + let start = inputs.drain(..self.profile.start.len()).collect(); + let end = inputs.split_off(1 + self.profile.libraries.len()); + Ok(LinkInputs { + start, + libraries: inputs, + end, + }) + } +} + +fn validate_files( + artifacts: Vec<RuntimeArtifact>, +) -> Result<BTreeMap<String, RuntimeArtifact>, String> { + if artifacts.is_empty() || artifacts.len() > 4096 { + return Err("runtime pack must declare 1..4096 files".into()); + } + let mut files = BTreeMap::new(); + for artifact in artifacts { + relative_path(&artifact.path)?; + if artifact.bytes == 0 + || artifact.bytes > 1024 * 1024 * 1024 + || artifact.sha256.len() != 64 + || !artifact + .sha256 + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err(format!( + "invalid runtime file digest or size for '{}'", + artifact.path + )); + } + if files.insert(artifact.path.clone(), artifact).is_some() { + return Err("duplicate runtime file path".into()); + } + } + Ok(files) +} + +fn validate_profiles( + profiles: &Profiles, + files: &BTreeMap<String, RuntimeArtifact>, +) -> Result<(), String> { + let mut referenced = BTreeSet::new(); + for (_, profile) in profiles.all() { + let Some(profile) = profile else { continue }; + if profile.start.is_empty() || profile.end.is_empty() || profile.libraries.is_empty() { + return Err("runtime profile requires CRT start/end and system libraries".into()); + } + let mut unique = BTreeSet::new(); + for input in profile.inputs() { + if !files.contains_key(input) || !unique.insert(input) { + return Err(format!( + "runtime profile has an undeclared or duplicate input '{input}'" + )); + } + referenced.insert(input); + } + for (paths, extension) in [ + (profile.start.as_slice(), "o"), + (profile.end.as_slice(), "o"), + (profile.libraries.as_slice(), "a"), + (std::slice::from_ref(&profile.runtime), "a"), + ] { + if paths.iter().any(|path| { + Path::new(path).extension().and_then(|value| value.to_str()) != Some(extension) + }) { + return Err( + "runtime profiles accept only explicit object files and static archives".into(), + ); + } + } + } + if referenced.len() != files.len() { + return Err("runtime pack contains unreferenced files".into()); + } + Ok(()) +} + +pub(super) fn relative_path(path: &str) -> Result<(), String> { + if path.is_empty() + || path.len() > 4096 + || path.contains(['\\', ':', '\0']) + || path + .split('/') + .any(|part| part.is_empty() || part == "." || part == "..") + { + return Err("runtime pack paths must be normalized relative paths".into()); + } + Ok(()) +} + +pub(super) fn checked_path(root: &Path, relative: &str, file: bool) -> Result<PathBuf, String> { + relative_path(relative)?; + let mut path = root.to_owned(); + let mut segments = relative.split('/').peekable(); + while let Some(segment) = segments.next() { + path.push(segment); + let metadata = fs::symlink_metadata(&path).map_err(|error| { + format!("cannot inspect runtime pack '{}': {error}", path.display()) + })?; + let expect_file = file && segments.peek().is_none(); + if metadata.file_type().is_symlink() + || if expect_file { + !metadata.is_file() + } else { + !metadata.is_dir() + } + { + return Err(format!( + "runtime pack '{}' must be a real {}", + path.display(), + if expect_file { "file" } else { "directory" } + )); + } + } + Ok(path) +} + +fn verify_file(path: &Path, artifact: &RuntimeArtifact, target: BuildTarget) -> Result<(), String> { + let length = fs::metadata(path) + .map_err(|error| format!("cannot inspect runtime file '{}': {error}", path.display()))? + .len(); + if length != artifact.bytes || sha256_file(path)? != artifact.sha256 { + return Err(format!( + "runtime file '{}' failed size or SHA-256 verification", + artifact.path + )); + } + let mut prefix = [0_u8; 8]; + fs::File::open(path) + .and_then(|mut file| file.read_exact(&mut prefix)) + .map_err(|error| format!("cannot read runtime input header: {error}"))?; + let valid = match Path::new(&artifact.path) + .extension() + .and_then(|value| value.to_str()) + { + Some("a") => &prefix == b"!<arch>\n", + Some("o") => &prefix[..4] == b"\x7fELF", + _ => false, + }; + if !valid { + return Err(format!( + "runtime input '{}' is not an ELF object or self-contained archive; linker scripts and thin archives are forbidden", + artifact.path + )); + } + validate_target(path, length, target).map_err(|error| { + format!( + "runtime input '{}' failed target verification: {error}", + artifact.path + ) + })?; + Ok(()) +} + +fn validate_target(path: &Path, length: u64, target: BuildTarget) -> Result<(), String> { + let mut file = fs::File::open(path).map_err(|error| error.to_string())?; + if path.extension().is_some_and(|extension| extension == "o") { + return validate_object(&mut file, length, target); + } + // Walk archive members without loading large runtime archives into memory. + let mut offset = 8_u64; + while offset < length { + file.seek(SeekFrom::Start(offset)) + .map_err(|error| error.to_string())?; + let mut header = [0; 60]; + file.read_exact(&mut header) + .map_err(|_| "truncated runtime archive member")?; + if &header[58..] != b"`\n" { + return Err("invalid runtime archive member header".into()); + } + let size = std::str::from_utf8(&header[48..58]) + .ok() + .and_then(|size| size.trim().parse::<u64>().ok()) + .ok_or("invalid runtime archive member size")?; + let start = offset + .checked_add(60) + .ok_or("runtime archive size overflow")?; + let end = start + .checked_add(size) + .filter(|end| *end <= length) + .ok_or("truncated runtime archive member")?; + let name = std::str::from_utf8(&header[..16]) + .map_err(|_| "invalid runtime archive member name")? + .trim(); + if !matches!( + name, + "/" | "//" | "/SYM64/" | "__.SYMDEF" | "__.SYMDEF SORTED" + ) { + let name_bytes = match name.strip_prefix("#1/") { + Some(bytes) => bytes + .parse::<u64>() + .map_err(|_| "invalid archive extended name")?, + None => 0, + }; + let object_size = size + .checked_sub(name_bytes) + .ok_or("invalid archive extended name size")?; + file.seek(SeekFrom::Start(start + name_bytes)) + .map_err(|error| error.to_string())?; + validate_object(&mut file, object_size, target)?; + } + offset = end + .checked_add(size % 2) + .filter(|end| *end <= length) + .ok_or("truncated runtime archive padding")?; + } + Ok(()) +} + +fn validate_object(file: &mut fs::File, length: u64, target: BuildTarget) -> Result<(), String> { + let mut header = [0; 20]; + if length < header.len() as u64 { + return Err("runtime input requires a complete target ELF object header".into()); + } + file.read_exact(&mut header) + .map_err(|error| format!("cannot read runtime object: {error}"))?; + let machine = match target { + BuildTarget::LinuxX86_64 => 62, + BuildTarget::LinuxAarch64 => 183, + }; + if &header[..4] != b"\x7fELF" + || header[4] != 2 + || header[5] != 1 + || header[6] != 1 + || u16::from_le_bytes([header[16], header[17]]) != 1 + || u16::from_le_bytes([header[18], header[19]]) != machine + { + return Err(format!( + "runtime input is not a relocatable ELF object for {}", + target.platform() + )); + } + Ok(()) +} diff --git a/crates/dever-cli/src/toolchain/service.rs b/crates/dever-cli/src/toolchain/service.rs new file mode 100644 index 0000000..60f333e --- /dev/null +++ b/crates/dever-cli/src/toolchain/service.rs @@ -0,0 +1,732 @@ +//! The small authenticated boundary around the machine cache. +//! +//! `CacheStore` remains an on-disk engine owned by this process. Clients use +//! this module's length-delimited local IPC instead of constructing the store +//! in the launcher. Platform installers can replace the fixture socket with +//! the documented system endpoint without changing the request contract. + +use std::collections::BTreeSet; +#[cfg(unix)] +use std::fs::File; +use std::fs::{self, OpenOptions}; +use std::io::{Read, Write}; +use std::path::Path; +#[cfg(unix)] +use std::sync::{Arc, Mutex, mpsc}; + +#[cfg(unix)] +use fs2::FileExt; +use ring::rand::{SecureRandom, SystemRandom}; +use serde::{Deserialize, Serialize}; + +use super::cache::{ArtifactReceipt, CacheStatus, CacheStore}; +use super::release::{Layout, Version}; +use sha2::{Digest, Sha256}; + +#[cfg(unix)] +mod compile; + +pub use self::compile_request::compile; + +mod compile_request; + +const TOKEN_BYTES: usize = 32; +const MAX_MESSAGE: usize = 1024 * 1024; +#[cfg(unix)] +const CLIENT_WORKERS: usize = 8; + +#[cfg(unix)] +struct Client { + stream: std::os::unix::net::UnixStream, + caller: Option<u32>, + #[cfg(target_os = "linux")] + _admission: Option<ForeignAdmission>, +} + +#[cfg(target_os = "linux")] +struct ForeignAdmission { + uid: u32, + users: Arc<Mutex<std::collections::BTreeMap<u32, usize>>>, +} + +#[cfg(target_os = "linux")] +impl ForeignAdmission { + fn acquire( + users: &Arc<Mutex<std::collections::BTreeMap<u32, usize>>>, + uid: u32, + ) -> Option<Self> { + let mut counts = users.lock().unwrap(); + // One stalled caller cannot fill the worker pool. Two service slots + // stay available to the owner even when foreign users are saturated. + if counts.values().sum::<usize>() >= CLIENT_WORKERS - 2 + || counts.get(&uid).copied().unwrap_or(0) >= 2 + { + return None; + } + *counts.entry(uid).or_default() += 1; + Some(Self { + uid, + users: Arc::clone(users), + }) + } +} + +#[cfg(target_os = "linux")] +impl Drop for ForeignAdmission { + fn drop(&mut self) { + let mut counts = self.users.lock().unwrap(); + if let Some(count) = counts.get_mut(&self.uid) { + *count -= 1; + if *count == 0 { + counts.remove(&self.uid); + } + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Request { + #[serde(default)] + token: String, + operation: Operation, + #[serde(default)] + installed: Vec<Version>, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(rename_all = "snake_case", deny_unknown_fields)] +enum Operation { + Status, + Clean, + ArtifactPut(ArtifactReceipt), + ArtifactGet(ArtifactReceipt), + Compile { bytes: u64 }, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Response { + ok: bool, + status: Option<CacheStatus>, + error: Option<String>, + #[serde(default)] + artifact: Option<ArtifactReceipt>, +} + +/// Ensure a per-machine authentication token exists. The token is not an +/// environment variable and is never accepted from a command line argument. +pub fn ensure_token(layout: &Layout) -> Result<(), String> { + if layout.service_token().exists() { + validate_token_file(layout)?; + return Ok(()); + } + let mut token = [0_u8; TOKEN_BYTES]; + SystemRandom::new() + .fill(&mut token) + .map_err(|_| "cannot generate deverd authentication token".to_owned())?; + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options + .open(layout.service_token()) + .map_err(|error| format!("cannot create deverd authentication token: {error}"))?; + file.write_all(hex(&token).as_bytes()) + .and_then(|_| file.write_all(b"\n")) + .and_then(|_| file.sync_all()) + .map_err(|error| format!("cannot write deverd authentication token: {error}"))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(layout.service_token(), fs::Permissions::from_mode(0o600)) + .map_err(|error| format!("cannot protect deverd authentication token: {error}"))?; + } + Ok(()) +} + +/// Serve requests until the process is stopped. This is intentionally a +/// local service primitive; OS service registration and privilege adapters +/// belong to the distribution layer. +pub fn serve(layout: &Layout) -> Result<(), String> { + #[cfg(unix)] + let _service_lock = acquire_service_lock(layout)?; + ensure_token(layout)?; + #[cfg(unix)] + { + use std::os::unix::fs::FileTypeExt; + use std::os::unix::net::UnixListener; + if let Some(parent) = layout.service_socket().parent() { + fs::create_dir_all(parent) + .map_err(|error| format!("cannot create deverd state directory: {error}"))?; + } + match fs::symlink_metadata(layout.service_socket()) { + Ok(metadata) + if metadata.file_type().is_symlink() || !metadata.file_type().is_socket() => + { + return Err("deverd IPC endpoint must be a real Unix socket".into()); + } + Ok(_) => fs::remove_file(layout.service_socket()) + .map_err(|error| format!("cannot replace deverd IPC endpoint: {error}"))?, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(format!("cannot inspect deverd IPC endpoint: {error}")), + } + let listener = UnixListener::bind(layout.service_socket()) + .map_err(|error| format!("cannot bind deverd IPC endpoint: {error}"))?; + use std::os::unix::fs::PermissionsExt; + // Linux obtains the calling uid from the kernel. The cache directories + // remain private; only the socket is usable by other machine users. + let mode = if cfg!(target_os = "linux") { + 0o666 + } else { + 0o600 + }; + fs::set_permissions(layout.service_socket(), fs::Permissions::from_mode(mode)) + .map_err(|error| format!("cannot protect deverd IPC endpoint: {error}"))?; + let (sender, receiver) = mpsc::sync_channel(CLIENT_WORKERS); + let receiver = Arc::new(Mutex::new(receiver)); + #[cfg(target_os = "linux")] + let (owner, foreign_users) = { + use std::os::unix::fs::MetadataExt; + let owner = fs::symlink_metadata(layout.state()) + .map_err(|error| format!("cannot inspect deverd state: {error}"))? + .uid(); + if owner != rustix::process::geteuid().as_raw() { + return Err("deverd must run as the state directory owner".into()); + } + ( + owner, + Arc::new(Mutex::new(std::collections::BTreeMap::new())), + ) + }; + std::thread::scope(|scope| { + for _ in 0..CLIENT_WORKERS { + let receiver = Arc::clone(&receiver); + scope.spawn(move || { + loop { + let next = { receiver.lock().unwrap().recv() }; + match next { + Ok(client) => { + let Client { + stream, + caller, + #[cfg(target_os = "linux")] + _admission, + } = client; + serve_client(layout, stream, caller); + } + Err(_) => break, + } + } + }); + } + let result = loop { + match listener.accept() { + Ok((stream, _)) => { + #[cfg(target_os = "linux")] + let caller = match rustix::net::sockopt::socket_peercred(&stream) { + Ok(credentials) => Some(credentials.uid.as_raw()), + Err(_) => continue, + }; + #[cfg(not(target_os = "linux"))] + let caller = None; + #[cfg(target_os = "linux")] + let admission = if caller != Some(owner) { + match ForeignAdmission::acquire( + &foreign_users, + caller.expect("checked kernel uid"), + ) { + Some(admission) => Some(admission), + None => continue, + } + } else { + None + }; + let client = Client { + stream, + caller, + #[cfg(target_os = "linux")] + _admission: admission, + }; + let _ = sender.try_send(client); + } + Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue, + Err(error) => break Err(format!("deverd IPC accept failed: {error}")), + } + }; + drop(sender); + result + }) + } + #[cfg(not(unix))] + { + let _ = layout; + Err("deverd authenticated IPC is not implemented for this platform yet".into()) + } +} + +pub fn cache_status(layout: &Layout) -> Result<CacheStatus, String> { + request(layout, Operation::Status, &[]).and_then(|response| { + response.status.ok_or_else(|| { + response + .error + .unwrap_or_else(|| "deverd returned no cache status".into()) + }) + }) +} + +pub fn cache_clean(layout: &Layout, installed: &BTreeSet<Version>) -> Result<CacheStatus, String> { + request( + layout, + Operation::Clean, + &installed.iter().cloned().collect::<Vec<_>>(), + ) + .and_then(|response| { + response.status.ok_or_else(|| { + response + .error + .unwrap_or_else(|| "deverd returned no cache status".into()) + }) + }) +} + +pub fn artifact_put(layout: &Layout, bytes: &[u8]) -> Result<ArtifactReceipt, String> { + let expected = ArtifactReceipt { + sha256: hex(&Sha256::digest(bytes)), + bytes: bytes.len() as u64, + }; + expected.validate()?; + #[cfg(unix)] + { + let (mut stream, token) = connect(layout)?; + write_frame( + &mut stream, + &Request { + token, + operation: Operation::ArtifactPut(expected.clone()), + installed: Vec::new(), + }, + )?; + require_artifact(read_frame(&mut stream)?, &expected)?; + stream + .write_all(bytes) + .map_err(|error| format!("deverd artifact upload failed: {error}"))?; + require_artifact(read_frame(&mut stream)?, &expected)?; + Ok(expected) + } + #[cfg(not(unix))] + { + let _ = layout; + Err("deverd artifact IPC is not implemented for this platform".into()) + } +} + +pub fn artifact_get(layout: &Layout, sha256: &str, bytes: u64) -> Result<Vec<u8>, String> { + let expected = ArtifactReceipt { + sha256: sha256.to_owned(), + bytes, + }; + expected.validate()?; + #[cfg(unix)] + { + let (mut stream, token) = connect(layout)?; + write_frame( + &mut stream, + &Request { + token, + operation: Operation::ArtifactGet(expected.clone()), + installed: Vec::new(), + }, + )?; + require_artifact(read_frame(&mut stream)?, &expected)?; + let mut bytes = vec![0; expected.bytes as usize]; + stream + .read_exact(&mut bytes) + .map_err(|error| format!("deverd artifact download is incomplete: {error}"))?; + if hex(&Sha256::digest(&bytes)) != expected.sha256 { + return Err("deverd artifact download failed SHA-256 verification".into()); + } + Ok(bytes) + } + #[cfg(not(unix))] + { + let _ = layout; + Err("deverd artifact IPC is not implemented for this platform".into()) + } +} + +fn require_artifact(response: Response, expected: &ArtifactReceipt) -> Result<(), String> { + if !response.ok { + return Err(response + .error + .unwrap_or_else(|| "deverd artifact request failed".into())); + } + if response.artifact.as_ref() != Some(expected) { + return Err("deverd returned a different artifact identity".into()); + } + Ok(()) +} + +#[cfg(unix)] +fn connect(layout: &Layout) -> Result<(std::os::unix::net::UnixStream, String), String> { + use std::os::unix::fs::{FileTypeExt, MetadataExt}; + use std::os::unix::net::UnixStream; + + let state = fs::symlink_metadata(layout.state()) + .map_err(|error| format!("cannot inspect deverd state directory: {error}"))?; + if !state.is_dir() || state.file_type().is_symlink() || state.mode() & 0o022 != 0 { + return Err("deverd state directory must not be writable by other users".into()); + } + let endpoint = fs::symlink_metadata(layout.service_socket()) + .map_err(|error| format!("authenticated deverd cache service unavailable: {error}"))?; + if !endpoint.file_type().is_socket() || endpoint.uid() != state.uid() { + return Err("deverd IPC endpoint does not belong to the service owner".into()); + } + let stream = UnixStream::connect(layout.service_socket()) + .map_err(|error| format!("authenticated deverd cache service unavailable: {error}"))?; + #[cfg(target_os = "linux")] + if rustix::net::sockopt::socket_peercred(&stream) + .map_err(|error| format!("cannot authenticate deverd OS peer: {error}"))? + .uid + .as_raw() + != state.uid() + { + return Err("deverd OS peer does not match the service owner".into()); + } + stream + .set_read_timeout(Some(std::time::Duration::from_secs(5))) + .and_then(|()| stream.set_write_timeout(Some(std::time::Duration::from_secs(5)))) + .map_err(|error| format!("cannot configure deverd IPC timeout: {error}"))?; + #[cfg(target_os = "linux")] + if rustix::process::geteuid().as_raw() != state.uid() { + return Ok((stream, String::new())); + } + validate_token_file(layout)?; + let token = String::from_utf8(read_file(&layout.service_token())?) + .map_err(|_| "invalid deverd authentication token".to_owned())?; + Ok((stream, token.trim().to_owned())) +} + +fn request( + layout: &Layout, + operation: Operation, + installed: &[Version], +) -> Result<Response, String> { + #[cfg(unix)] + { + let (mut stream, token) = connect(layout)?; + let request = Request { + token, + operation, + installed: installed.to_owned(), + }; + write_frame(&mut stream, &request)?; + read_frame(&mut stream) + } + #[cfg(not(unix))] + { + let _ = (layout, operation, installed); + Err( + "authenticated deverd cache service unavailable: platform IPC is not implemented" + .into(), + ) + } +} + +#[cfg(unix)] +fn handle( + layout: &Layout, + stream: &mut std::os::unix::net::UnixStream, + caller: Option<u32>, +) -> Response { + let request: Result<Request, _> = read_frame(stream); + let request = match request { + Ok(request) => request, + Err(error) => return failure(error), + }; + if let Err(error) = validate_token_file(layout) { + return failure(error); + } + let bytes = match read_file(&layout.service_token()) { + Ok(bytes) => bytes, + Err(error) => return failure(error), + }; + let expected = match String::from_utf8(bytes) { + Ok(token) => token.trim().as_bytes().to_vec(), + Err(_) => return failure("invalid deverd authentication token".into()), + }; + #[cfg(target_os = "linux")] + let foreign = match caller { + Some(uid) => { + use std::os::unix::fs::MetadataExt; + match fs::symlink_metadata(layout.state()) { + Ok(state) => uid != state.uid(), + Err(error) => return failure(format!("cannot inspect deverd state: {error}")), + } + } + None => return failure("deverd OS caller credentials are unavailable".into()), + }; + #[cfg(not(target_os = "linux"))] + let foreign = { + let _ = caller; + false + }; + if foreign { + if matches!(request.operation, Operation::Clean) { + return failure("deverd cache clean requires the service owner".into()); + } + } else if !constant_time_equal(request.token.as_bytes(), &expected) { + return failure("deverd authentication failed".into()); + } + let store = CacheStore::new(layout); + let result = match request.operation { + Operation::Status => { + if foreign { + store.summary() + } else { + store.status() + } + } + Operation::Clean => { + if request.installed.len() > 256 { + return failure("too many installed Dever versions".into()); + } + store.clean(&request.installed.into_iter().collect()) + } + Operation::ArtifactPut(expected) => { + if let Err(error) = expected.validate() { + return failure(error); + } + if let Err(error) = write_frame(stream, &artifact_response(expected.clone())) { + return failure(error); + } + let mut input = TransferReader { + input: stream, + started: std::time::Instant::now(), + }; + return match store.publish_artifact(&expected, &mut input) { + Ok(receipt) => artifact_response(receipt), + Err(error) => failure(error), + }; + } + Operation::ArtifactGet(expected) => { + let mut artifact = match store.artifact(&expected) { + Ok(artifact) => artifact, + Err(error) => return failure(error), + }; + if let Err(error) = write_frame(stream, &artifact_response(artifact.receipt)) { + return failure(error); + } + if let Err(error) = std::io::copy(&mut artifact.file, stream) { + return failure(format!("deverd artifact transfer failed: {error}")); + } + return Response { + ok: true, + status: None, + artifact: None, + error: None, + }; + } + Operation::Compile { bytes } => { + return match compile::serve(layout, stream, bytes) { + Ok(()) => Response { + ok: true, + status: None, + artifact: None, + error: None, + }, + Err(error) => failure(error), + }; + } + }; + match result { + Ok(status) => Response { + ok: true, + status: Some(status), + error: None, + artifact: None, + }, + Err(error) => failure(error), + } +} + +fn failure(error: String) -> Response { + Response { + ok: false, + status: None, + error: Some(error), + artifact: None, + } +} + +fn artifact_response(artifact: ArtifactReceipt) -> Response { + Response { + ok: true, + status: None, + error: None, + artifact: Some(artifact), + } +} + +struct TransferReader<'a, R> { + input: &'a mut R, + started: std::time::Instant, +} + +impl<R: Read> Read for TransferReader<'_, R> { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result<usize> { + if self.started.elapsed() >= std::time::Duration::from_secs(60) { + return Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "deverd artifact transfer deadline exceeded", + )); + } + self.input.read(buffer) + } +} + +#[cfg(unix)] +fn serve_client(layout: &Layout, mut stream: std::os::unix::net::UnixStream, caller: Option<u32>) { + if stream + .set_read_timeout(Some(std::time::Duration::from_secs(5))) + .and_then(|()| stream.set_write_timeout(Some(std::time::Duration::from_secs(5)))) + .is_err() + { + return; + } + let response = handle(layout, &mut stream, caller); + let _ = write_frame(&mut stream, &response); +} + +#[cfg(unix)] +fn acquire_service_lock(layout: &Layout) -> Result<File, String> { + use std::os::unix::fs::{MetadataExt, OpenOptionsExt}; + + let path = layout.state().join("deverd.lock"); + let mut options = OpenOptions::new(); + options.read(true).write(true); + match fs::symlink_metadata(&path) { + Ok(_) => { + validate_owned_file(layout, &path, "deverd lock")?; + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + options.create_new(true).mode(0o600); + } + Err(error) => return Err(format!("cannot inspect deverd lock: {error}")), + } + let file = options + .open(&path) + .map_err(|error| format!("cannot open deverd lock: {error}"))?; + let opened = file + .metadata() + .map_err(|error| format!("cannot inspect deverd lock: {error}"))?; + let current = validate_owned_file(layout, &path, "deverd lock")?; + if opened.dev() != current.dev() || opened.ino() != current.ino() { + return Err("deverd lock changed while opening".into()); + } + FileExt::try_lock_exclusive(&file).map_err(|error| { + if error.kind() == std::io::ErrorKind::WouldBlock { + "deverd is already running for this machine root".into() + } else { + format!("cannot lock deverd state: {error}") + } + })?; + Ok(file) +} + +fn validate_token_file(layout: &Layout) -> Result<(), String> { + let path = layout.service_token(); + validate_owned_file(layout, &path, "deverd authentication token")?; + let token = read_file(&path)?; + let token = token.strip_suffix(b"\n").unwrap_or(&token); + if token.len() != TOKEN_BYTES * 2 + || !token + .iter() + .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) + { + return Err("invalid deverd authentication token".into()); + } + Ok(()) +} + +fn validate_owned_file(layout: &Layout, path: &Path, name: &str) -> Result<fs::Metadata, String> { + let metadata = + fs::symlink_metadata(path).map_err(|error| format!("cannot inspect {name}: {error}"))?; + if metadata.file_type().is_symlink() || !metadata.is_file() { + return Err(format!("{name} must be a regular file")); + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let state = fs::symlink_metadata(layout.state()) + .map_err(|error| format!("cannot inspect deverd state directory: {error}"))?; + if !state.is_dir() || state.uid() != metadata.uid() { + return Err(format!("{name} must have the state directory owner")); + } + if metadata.mode() & 0o077 != 0 { + return Err(format!("{name} must be accessible only to its owner")); + } + } + Ok(metadata) +} + +fn write_frame<T: Serialize>(stream: &mut impl Write, value: &T) -> Result<(), String> { + let payload = + serde_json::to_vec(value).map_err(|error| format!("deverd IPC encode failed: {error}"))?; + if payload.len() > MAX_MESSAGE { + return Err("deverd IPC message is too large".into()); + } + stream + .write_all(&(payload.len() as u32).to_be_bytes()) + .and_then(|_| stream.write_all(&payload)) + .and_then(|_| stream.flush()) + .map_err(|error| format!("deverd IPC write failed: {error}")) +} + +fn read_frame<T: for<'de> Deserialize<'de>>(stream: &mut impl Read) -> Result<T, String> { + let mut length = [0_u8; 4]; + stream + .read_exact(&mut length) + .map_err(|error| format!("deverd IPC read failed: {error}"))?; + let length = u32::from_be_bytes(length) as usize; + if length > MAX_MESSAGE { + return Err("deverd IPC message is too large".into()); + } + let mut payload = vec![0; length]; + stream + .read_exact(&mut payload) + .map_err(|error| format!("deverd IPC read failed: {error}"))?; + let text = + std::str::from_utf8(&payload).map_err(|_| "deverd IPC payload must be UTF-8 JSON")?; + dever_runtime::wire::parse(text) + .map_err(|error| format!("deverd IPC decode failed: {error}"))?; + serde_json::from_slice(&payload).map_err(|error| format!("deverd IPC decode failed: {error}")) +} + +fn read_file(path: &Path) -> Result<Vec<u8>, String> { + fs::read(path).map_err(|error| format!("cannot read '{}': {error}", path.display())) +} + +fn constant_time_equal(left: &[u8], right: &[u8]) -> bool { + if left.len() != right.len() { + return false; + } + let mut difference = 0_u8; + for (left, right) in left.iter().zip(right) { + difference |= left ^ right; + } + difference == 0 +} + +fn hex(bytes: &[u8]) -> String { + const DIGITS: &[u8; 16] = b"0123456789abcdef"; + let mut output = String::with_capacity(bytes.len() * 2); + for byte in bytes { + output.push(DIGITS[usize::from(byte >> 4)] as char); + output.push(DIGITS[usize::from(byte & 0x0f)] as char); + } + output +} diff --git a/crates/dever-cli/src/toolchain/service/compile.rs b/crates/dever-cli/src/toolchain/service/compile.rs new file mode 100644 index 0000000..2455c36 --- /dev/null +++ b/crates/dever-cli/src/toolchain/service/compile.rs @@ -0,0 +1,273 @@ +//! Only this service path may publish compiled entries. Artifact uploads remain +//! opaque bytes and can never satisfy a compilation lookup. +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Write}; +use std::os::unix::net::UnixStream; +use std::path::Path; +use std::process::{Child, Command, Stdio}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{Duration, Instant}; + +use super::super::cache::{ArtifactReceipt, BuildIdentity, CacheStore, MAX_COMPILED_BYTES}; +use super::super::compilation::{ + CompileRequest, MAX_COMPILE_BYTES, WORKER_OUTPUT_FILE, WORKER_REQUEST_FILE, +}; +use super::super::release::{InstalledCompiler, Layout, MachineManager, Version}; +use super::{Response, artifact_response, hex, write_frame}; + +const COMPILE_TIMEOUT: Duration = Duration::from_secs(180); +const MAX_DIAGNOSTIC_BYTES: u64 = 64 * 1024; +static ACTIVE_COMPILATIONS: AtomicUsize = AtomicUsize::new(0); + +struct Admission; + +impl Admission { + fn acquire() -> Result<Self, String> { + ACTIVE_COMPILATIONS + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| { + (count < 2).then_some(count + 1) + }) + .map_err(|_| "deverd compilation capacity is busy; retry later".to_owned())?; + Ok(Self) + } +} + +impl Drop for Admission { + fn drop(&mut self) { + ACTIVE_COMPILATIONS.fetch_sub(1, Ordering::Release); + } +} + +pub(super) fn serve(layout: &Layout, stream: &mut UnixStream, bytes: u64) -> Result<(), String> { + if bytes == 0 || bytes > MAX_COMPILE_BYTES as u64 { + return Err("deverd compilation request exceeds its transfer limit".into()); + } + let _admission = Admission::acquire()?; + let started = Instant::now(); + let store = CacheStore::new(layout); + let _operation = store.operation()?; + write_frame( + stream, + &Response { + ok: true, + status: None, + artifact: None, + error: None, + }, + )?; + let mut payload = vec![0; bytes as usize]; + super::TransferReader { + input: stream, + started, + } + .read_exact(&mut payload) + .map_err(|error| format!("incomplete compilation request: {error}"))?; + let request = CompileRequest::decode(&payload)?; + let version = Version::parse(&request.version)?; + let target = request.target; + // Canonical bytes own the cache identity; decoded interpreter copies are no + // longer needed by the service while the worker rereads its private request. + drop(request); + let compiler = MachineManager::new(layout.clone()).compilation(&version)?; + verify_pack(&compiler, target)?; + let identity = BuildIdentity { + version: &version, + compiler: compiler.identity.as_bytes(), + runtime: compiler.identity.as_bytes(), + target: target.platform(), + generated: &payload, + options: b"dever-trusted-compilation-v1", + environment: b"", + }; + stream + .set_read_timeout(Some(Duration::from_millis(1))) + .map_err(|error| format!("cannot monitor compilation caller: {error}"))?; + require_connected(stream, started)?; + if let Some(artifact) = store.restore(&identity)? { + return send_output(stream, artifact.as_bytes(), started); + } + let directory = store.compilation_directory()?; + write_private(&directory.0.join(WORKER_REQUEST_FILE), &payload)?; + run_worker(&compiler.core, &directory.0, stream, started)?; + require_connected(stream, started)?; + let output = directory.0.join(WORKER_OUTPUT_FILE); + let metadata = fs::symlink_metadata(&output) + .map_err(|error| format!("compiler produced no executable: {error}"))?; + if !metadata.is_file() + || metadata.file_type().is_symlink() + || metadata.len() == 0 + || metadata.len() > MAX_COMPILED_BYTES + { + return Err("compiler output must be a regular executable of at most 320 MiB".into()); + } + store.publish(&identity, &output)?; + let artifact = store + .restore(&identity)? + .ok_or("compiled output was not published")?; + send_output(stream, artifact.as_bytes(), started) +} + +fn verify_pack( + compiler: &InstalledCompiler, + target: super::super::BuildTarget, +) -> Result<(), String> { + let root = compiler + .core + .parent() + .ok_or("installed compiler has no directory")?; + let artifacts = + super::super::runtime_pack::validate(root, compiler.manifest.version.as_str(), target)?; + for input in artifacts { + if !compiler.manifest.artifacts.iter().any(|signed| { + signed.path == input.path + && signed.bytes == input.bytes + && signed.sha256 == input.sha256 + }) { + return Err(format!( + "native runtime input '{}' is not covered by the signed release", + input.path + )); + } + } + // This is the embedded compiler's fixed Linux LLVM dependency, not a PATH lookup. + if cfg!(target_os = "linux") + && !compiler + .manifest + .artifacts + .iter() + .any(|artifact| artifact.path == "lib/libLLVM.so.18.1") + { + return Err("signed release is missing lib/libLLVM.so.18.1".into()); + } + Ok(()) +} + +fn send_output(stream: &mut UnixStream, bytes: &[u8], started: Instant) -> Result<(), String> { + let receipt = ArtifactReceipt { + sha256: hex(ring::digest::digest(&ring::digest::SHA256, bytes).as_ref()), + bytes: bytes.len() as u64, + }; + receipt.validate_compiled()?; + let timeout = || { + COMPILE_TIMEOUT + .checked_sub(started.elapsed()) + .filter(|remaining| !remaining.is_zero()) + .map(|remaining| remaining.min(Duration::from_secs(5))) + .ok_or_else(|| "deverd compilation transfer deadline exceeded".to_owned()) + }; + stream + .set_write_timeout(Some(timeout()?)) + .map_err(|error| error.to_string())?; + write_frame(stream, &artifact_response(receipt))?; + let mut remaining = bytes; + while !remaining.is_empty() { + stream + .set_write_timeout(Some(timeout()?)) + .map_err(|error| error.to_string())?; + match stream.write(&remaining[..remaining.len().min(64 * 1024)]) { + Ok(0) => return Err("compiled output transfer closed".into()), + Ok(written) => remaining = &remaining[written..], + Err(error) if error.kind() == std::io::ErrorKind::Interrupted => {} + Err(error) => return Err(format!("compiled output transfer failed: {error}")), + } + } + Ok(()) +} + +fn require_connected(stream: &mut UnixStream, started: Instant) -> Result<(), String> { + if started.elapsed() >= COMPILE_TIMEOUT { + return Err("deverd compilation deadline exceeded".into()); + } + let mut signal = [0_u8; 1]; + match stream.read(&mut signal) { + // Any byte after the exact request body explicitly cancels this request. + Ok(_) => Err("deverd compilation cancelled by caller".into()), + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::WouldBlock + | std::io::ErrorKind::TimedOut + | std::io::ErrorKind::Interrupted + ) => + { + Ok(()) + } + Err(error) => Err(format!("deverd compilation caller disconnected: {error}")), + } +} + +struct Worker(Child); + +impl Drop for Worker { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +fn run_worker( + core: &Path, + directory: &Path, + stream: &mut UnixStream, + started: Instant, +) -> Result<(), String> { + let diagnostic = directory.join("diagnostic"); + let error_output = private_file(&diagnostic)?; + let mut worker = Worker( + Command::new(core) + .arg("--dever-compile-worker") + .env_clear() + .current_dir(directory) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(error_output) + .spawn() + .map_err(|error| format!("cannot start trusted compile worker: {error}"))?, + ); + loop { + require_connected(stream, started)?; + if fs::metadata(&diagnostic) + .map_err(|error| error.to_string())? + .len() + > MAX_DIAGNOSTIC_BYTES + { + return Err("compile worker diagnostic exceeded 64 KiB".into()); + } + if let Some(status) = worker + .0 + .try_wait() + .map_err(|error| format!("cannot wait for compile worker: {error}"))? + { + if status.success() { + return Ok(()); + } + let mut bytes = Vec::new(); + File::open(&diagnostic) + .map_err(|error| error.to_string())? + .take(MAX_DIAGNOSTIC_BYTES) + .read_to_end(&mut bytes) + .map_err(|error| error.to_string())?; + return Err(format!( + "trusted compile worker failed ({status}): {}", + String::from_utf8_lossy(&bytes) + )); + } + std::thread::sleep(Duration::from_millis(10)); + } +} + +fn private_file(path: &Path) -> Result<File, String> { + use std::os::unix::fs::OpenOptionsExt; + OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(path) + .map_err(|error| format!("cannot create compilation input: {error}")) +} + +fn write_private(path: &Path, bytes: &[u8]) -> Result<(), String> { + private_file(path)? + .write_all(bytes) + .map_err(|error| format!("cannot write compilation input: {error}")) +} diff --git a/crates/dever-cli/src/toolchain/service/compile_request.rs b/crates/dever-cli/src/toolchain/service/compile_request.rs new file mode 100644 index 0000000..7419b47 --- /dev/null +++ b/crates/dever-cli/src/toolchain/service/compile_request.rs @@ -0,0 +1,87 @@ +use super::super::compilation::CompileRequest; +use super::super::release::Layout; + +/// Submit source and compile-only bindings, never a client-chosen cache key or IR. +pub fn compile(layout: &Layout, request: &CompileRequest) -> Result<Vec<u8>, String> { + #[cfg(unix)] + { + use super::{Operation, Request, Response, connect, read_frame, write_frame}; + use std::io::{Read, Write}; + use std::time::{Duration, Instant}; + let payload = request.encode()?; + let (mut stream, token) = connect(layout)?; + write_frame( + &mut stream, + &Request { + token, + operation: Operation::Compile { + bytes: payload.len() as u64, + }, + installed: Vec::new(), + }, + )?; + let ready: Response = read_frame(&mut stream)?; + if !ready.ok { + return Err(ready + .error + .unwrap_or_else(|| "deverd rejected compilation".into())); + } + stream + .write_all(&payload) + .map_err(|error| format!("deverd compilation upload failed: {error}"))?; + drop(payload); + let mut download = CompilationDownload { + stream: &mut stream, + deadline: Instant::now() + Duration::from_secs(190), + }; + let response: Response = read_frame(&mut download)?; + if !response.ok { + return Err(response + .error + .unwrap_or_else(|| "deverd compilation failed".into())); + } + let receipt = response + .artifact + .ok_or("deverd returned no compiled output identity")?; + receipt.validate_compiled()?; + let mut output = vec![0; receipt.bytes as usize]; + download + .read_exact(&mut output) + .map_err(|error| format!("deverd compilation download is incomplete: {error}"))?; + if super::hex(ring::digest::digest(&ring::digest::SHA256, &output).as_ref()) + != receipt.sha256 + { + return Err("deverd compiled output failed SHA-256 verification".into()); + } + Ok(output) + } + #[cfg(not(unix))] + { + let _ = (layout, request); + Err("trusted deverd compilation is not implemented for this platform".into()) + } +} + +#[cfg(unix)] +struct CompilationDownload<'a> { + stream: &'a mut std::os::unix::net::UnixStream, + deadline: std::time::Instant, +} + +#[cfg(unix)] +impl std::io::Read for CompilationDownload<'_> { + fn read(&mut self, bytes: &mut [u8]) -> std::io::Result<usize> { + let remaining = self + .deadline + .checked_duration_since(std::time::Instant::now()) + .filter(|duration| !duration.is_zero()) + .ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::TimedOut, + "compiled output download deadline exceeded", + ) + })?; + self.stream.set_read_timeout(Some(remaining))?; + self.stream.read(bytes) + } +} diff --git a/crates/dever-cli/src/toolchain/skill.rs b/crates/dever-cli/src/toolchain/skill.rs new file mode 100644 index 0000000..174b04e --- /dev/null +++ b/crates/dever-cli/src/toolchain/skill.rs @@ -0,0 +1,91 @@ +//! A stable AI entry resolves the same active-version journal as the launcher. + +use std::fs::{self, OpenOptions}; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use super::release::MachineManager; + +static NEXT_STAGING: AtomicU64 = AtomicU64::new(0); + +pub(super) fn path(manager: &MachineManager) -> Result<PathBuf, String> { + let version = manager + .active_version()? + .ok_or("no active Dever version; install Dever first")?; + // Like launcher dispatch, this is read-only and available to every machine user. + manager.resolve_skill(&version) +} + +pub(super) fn install(manager: &MachineManager, destination: &Path) -> Result<(), String> { + path(manager)?; + let launcher = manager.layout().bin().join("dever"); + let launcher = launcher + .to_str() + .ok_or("launcher path must be UTF-8 for an AI skill")?; + if launcher.chars().any(char::is_control) || launcher.contains('`') { + return Err("launcher path cannot contain control characters or backticks".into()); + } + let quoted = format!("'{}'", launcher.replace('\'', "'\\''")); + let content = format!( + "---\nname: dever-language\ndescription: Develop, review and debug Dever language applications (.dever and .dever.md), manage Dever Packages and Libs, and install or update Dever. Not the Go Dever framework.\n---\n\n# Dever language\n\nBefore every Dever task, run this read-only command:\n\n```sh\n{quoted} skill path\n```\n\nRead `SKILL.md` in the returned directory completely, then follow its instructions. Resolve its reference and asset paths against that directory. This entry follows the machine's active Dever version, so `dever update` and `dever use` update the compiler and development guidance together. If resolution fails, report the error; do not use stale guidance or infer language syntax. This entry grants no permission to install, update or change a project.\n" + ); + let name = destination + .file_name() + .ok_or("skill destination requires a new directory name")?; + let parent = destination + .parent() + .filter(|path| !path.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + let parent = fs::canonicalize(parent) + .map_err(|error| format!("cannot resolve skill destination parent: {error}"))?; + let destination = parent.join(name); + let staging = stage(&parent)?; + let entry = staging.join("SKILL.md"); + let result = (|| { + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(&entry) + .map_err(|error| format!("cannot create skill entry: {error}"))?; + file.write_all(content.as_bytes()) + .and_then(|()| file.sync_all()) + .map_err(|error| format!("cannot persist skill entry: {error}"))?; + #[cfg(target_os = "linux")] + { + use rustix::fs::{CWD, RenameFlags, renameat_with}; + renameat_with(CWD, &staging, CWD, &destination, RenameFlags::NOREPLACE) + .map_err(|error| format!("cannot publish new skill directory: {error}")) + } + #[cfg(not(target_os = "linux"))] + { + Err("skill installation is not available for this platform".into()) + } + })(); + if result.is_err() { + let _ = fs::remove_file(&entry); + let _ = fs::remove_dir(&staging); + } + result +} + +fn stage(parent: &Path) -> Result<PathBuf, String> { + loop { + let path = parent.join(format!( + ".dever-skill-{}-{}", + std::process::id(), + NEXT_STAGING.fetch_add(1, Ordering::Relaxed) + )); + let mut builder = fs::DirBuilder::new(); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + match builder.create(&path) { + Ok(()) => return Ok(path), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => return Err(format!("cannot stage skill entry: {error}")), + } + } +} diff --git a/crates/dever-cli/src/toolchain/target.rs b/crates/dever-cli/src/toolchain/target.rs new file mode 100644 index 0000000..66f7a94 --- /dev/null +++ b/crates/dever-cli/src/toolchain/target.rs @@ -0,0 +1,50 @@ +//! Application deployment targets, independent of the compiler host identity. + +use serde::{Deserialize, Serialize}; +use std::str::FromStr; + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Deserialize, Serialize)] +pub enum BuildTarget { + #[serde(rename = "linux-x86_64")] + LinuxX86_64, + #[serde(rename = "linux-aarch64")] + LinuxAarch64, +} + +impl BuildTarget { + pub fn host() -> Result<Self, String> { + match (std::env::consts::OS, std::env::consts::ARCH) { + ("linux", "x86_64") => Ok(Self::LinuxX86_64), + ("linux", "aarch64") => Ok(Self::LinuxAarch64), + _ => Err("native executable runtime packs are not available for this platform".into()), + } + } + + pub fn platform(self) -> &'static str { + match self { + Self::LinuxX86_64 => "linux-x86_64", + Self::LinuxAarch64 => "linux-aarch64", + } + } + + pub fn triple(self) -> &'static str { + match self { + Self::LinuxX86_64 => "x86_64-unknown-linux-gnu", + Self::LinuxAarch64 => "aarch64-unknown-linux-gnu", + } + } +} + +impl FromStr for BuildTarget { + type Err = String; + + fn from_str(value: &str) -> Result<Self, Self::Err> { + match value { + "linux-x86_64" => Ok(Self::LinuxX86_64), + "linux-aarch64" => Ok(Self::LinuxAarch64), + _ => Err(format!( + "unsupported build target '{value}'; expected linux-x86_64 or linux-aarch64" + )), + } + } +} diff --git a/crates/dever-cli/src/worker.rs b/crates/dever-cli/src/worker.rs new file mode 100644 index 0000000..48d3895 --- /dev/null +++ b/crates/dever-cli/src/worker.rs @@ -0,0 +1,97 @@ +//! Hidden source compiler. It never prepares dependencies or executes a project. + +use dever_cli::toolchain::compilation::{ + CompileKind, CompileRequest, MAX_COMPILE_BYTES, WORKER_OUTPUT_FILE, WORKER_REQUEST_FILE, + worker_directory, +}; +use std::fs; +use std::io::Read; + +pub(super) fn execute() -> Result<(), String> { + resource_limits()?; + let (layout, version) = crate::compile::managed_compiler()? + .ok_or("compile workers require an installed managed compiler")?; + let directory = worker_directory(&layout)?; + let path = directory.join(WORKER_REQUEST_FILE); + let metadata = fs::symlink_metadata(&path) + .map_err(|error| format!("cannot inspect compile request: {error}"))?; + if !metadata.is_file() + || metadata.file_type().is_symlink() + || metadata.len() > MAX_COMPILE_BYTES as u64 + { + return Err("compile request must be a bounded regular file".into()); + } + let mut bytes = Vec::new(); + fs::File::open(&path) + .and_then(|file| { + file.take(MAX_COMPILE_BYTES as u64 + 1) + .read_to_end(&mut bytes) + }) + .map_err(|error| format!("cannot read compile request: {error}"))?; + let request = CompileRequest::decode(&bytes)?; + drop(bytes); + if request.version != version { + return Err("compile request version does not match worker".into()); + } + let sources = request.source_map()?; + let checked = match &request.bindings { + Some(bindings) => dever_core::check_with_bindings(&sources, bindings), + None => dever_core::check(&sources), + }; + let render = |errors: Vec<dever_core::diagnostic::Diagnostic>| { + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + }; + let program = checked.map_err(render)?; + if request.kind == CompileKind::Application { + if request.sources.iter().any(|source| { + source + .logical_path + .rsplit('/') + .next() + .is_some_and(|name| name == "main.dever" || name == "main.dever.md") + }) { + return Err("application entry is generated from API, CMD, and Job declarations; remove module/main.dever".into()); + } + let errors = program.application_errors()?; + if !errors.is_empty() { + return Err(render(errors)); + } + } + crate::compile::worker_compile( + &program, + &sources, + &request, + &directory, + &directory.join(WORKER_OUTPUT_FILE), + ) +} + +#[cfg(unix)] +fn resource_limits() -> Result<(), String> { + use rustix::process::{Resource, Rlimit, setrlimit}; + for (resource, limit) in [ + (Resource::Cpu, 120), + (Resource::As, 2 * 1024 * 1024 * 1024), + (Resource::Fsize, 1024 * 1024 * 1024), + (Resource::Nofile, 128), + (Resource::Core, 0), + ] { + setrlimit( + resource, + Rlimit { + current: Some(limit), + maximum: Some(limit), + }, + ) + .map_err(|error| format!("cannot restrict compiler worker resources: {error}"))?; + } + Ok(()) +} + +#[cfg(not(unix))] +fn resource_limits() -> Result<(), String> { + Err("managed compiler worker resource limits are unavailable on this platform".into()) +} diff --git a/crates/dever-cli/src/workers.rs b/crates/dever-cli/src/workers.rs new file mode 100644 index 0000000..7ed3099 --- /dev/null +++ b/crates/dever-cli/src/workers.rs @@ -0,0 +1,833 @@ +//! Offline, per-Adapter execution trees for managed external Workers. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::path::{Component, Path}; + +use dever_core::hir::{ExternalWorkerContract, Program}; +use dever_core::native::EmbeddedResource; +use serde_json::json; +use sha2::{Digest, Sha256}; + +use crate::libs::{self, Ecosystem, LibSpec, LockFile, LockedLib}; + +mod command; +mod elf; +mod go_build; +pub mod python_wheel; +pub(crate) mod runtime; + +pub(crate) const MAX_TREE_BYTES: usize = 256 * 1024 * 1024; +pub(crate) const MAX_TREE_FILES: usize = 65_536; +const IGNORED_SOURCE_DIRECTORIES: &[&str] = &["node_modules", "__pycache__", ".git", ".dever"]; +const PYTHON_SDK: &[u8] = include_bytes!("../../../sdk/python/dever_component.py"); +const JAVASCRIPT_SDK: &[u8] = include_bytes!("../../../sdk/javascript/dever_component.mjs"); + +const PYTHON_RUNNER: &str = r#"import importlib.util +import json +from pathlib import Path +import sys + +base = Path(__file__).resolve().parent +sys.stdout = sys.stderr +sys.path.insert(0, str(base)) +from dever_component import Worker, main + +sys.path.insert(0, str(base / "runtime" / "lib" / "python__PYTHON_VERSION__" / "site-packages")) +sys.path.insert(0, str(base / "source")) +spec = importlib.util.spec_from_file_location("dever_worker_entry", base / "source" / "__ENTRY__") +if spec is None or spec.loader is None: + raise RuntimeError("managed Worker entry cannot be loaded") +entry = importlib.util.module_from_spec(spec) +spec.loader.exec_module(entry) +manifest = json.loads((base / "contract.json").read_text(encoding="utf-8")) +handlers = {operation: getattr(entry, operation) for operation in manifest["operations"]} +main(Worker.from_manifest(manifest, handlers)) +"#; + +const JAVASCRIPT_RUNNER: &str = r#"import { Console } from 'node:console'; +import { readFileSync } from 'node:fs'; +import { Worker, main } from './dever_component.mjs'; + +globalThis.console = new Console({ stdout: process.stderr, stderr: process.stderr }); +const manifest = JSON.parse(readFileSync(new URL('./contract.json', import.meta.url), 'utf8')); +const entry = await import(new URL('./source/__ENTRY__', import.meta.url)); +const handlers = Object.fromEntries(manifest.operations.map(operation => [operation, entry[operation] ?? entry.default?.[operation]])); +await main(Worker.fromManifest(manifest, handlers)); +"#; + +pub fn prepare( + project_root: &Path, + program: &Program, + resources: &[EmbeddedResource], +) -> Result<Vec<EmbeddedResource>, String> { + prepare_for_target( + project_root, + program, + resources, + crate::toolchain::BuildTarget::host()?, + resources, + ) +} + +pub fn prepare_for_target( + project_root: &Path, + program: &Program, + resources: &[EmbeddedResource], + target: crate::toolchain::BuildTarget, + build_assets: &[EmbeddedResource], +) -> Result<Vec<EmbeddedResource>, String> { + let workers = program.external_worker_contracts(); + let commands = program.external_command_contracts(); + if workers.is_empty() && commands.is_empty() { + return Ok(Vec::new()); + } + let lock = if workers.iter().any(|worker| worker.ecosystem != "exec") { + let lock_path = project_root.join("dever.lock"); + if fs::symlink_metadata(&lock_path) + .map_err(|error| format!("cannot inspect dever.lock: {error}"))? + .file_type() + .is_symlink() + { + return Err("managed Worker lock file must not be a symbolic link".into()); + } + let lock = LockFile::decode( + &fs::read(&lock_path).map_err(|error| format!("cannot read dever.lock: {error}"))?, + )?; + libs::doctor(&lock)?; + Some(lock) + } else { + None + }; + let mut supplied = BTreeMap::new(); + for resource in resources { + if supplied.insert(resource.path.as_str(), resource).is_some() { + return Err(format!( + "duplicate locked Worker resource '{}'", + resource.path + )); + } + if digest(&resource.bytes) != resource.sha256 { + return Err(format!( + "locked Worker resource '{}' has a digest mismatch", + resource.path + )); + } + } + let mut output = Vec::new(); + let package_files = crate::packages::owned_files(project_root)?; + for contract in commands { + output.extend(command::prepare( + project_root, + &contract, + &package_files, + &supplied, + target, + )?); + } + for contract in workers { + if contract.ecosystem == "exec" { + output.extend(prepare_exec(&contract, &supplied, target)?); + continue; + } + let lock = lock.as_ref().expect("managed Worker has a lock"); + let locked = lock + .workers + .iter() + .find(|worker| worker.port == contract.port && worker.adapter == contract.adapter) + .ok_or_else(|| { + format!( + "managed Worker '{}.{}' is not locked", + contract.port, contract.adapter + ) + })?; + if locked.entry != contract.entry + || locked.schema != contract.schema + || locked.ecosystem != contract.ecosystem + || locked.operations != contract.operations + || locked.capabilities != contract.capabilities + { + return Err(format!( + "managed Worker '{}.{}' differs from its checked Port", + contract.port, contract.adapter + )); + } + let requested = contract + .libs + .iter() + .map(|value| value.parse::<LibSpec>()) + .collect::<Result<BTreeSet<_>, _>>()?; + if requested != locked.libs.iter().cloned().collect() { + return Err("managed Worker Lib roots differ from its checked Adapter".into()); + } + output.extend(prepare_worker( + project_root, + &contract, + lock, + &supplied, + &package_files, + target, + build_assets, + )?); + } + Ok(output) +} + +fn prepare_exec( + contract: &ExternalWorkerContract, + supplied: &BTreeMap<&str, &EmbeddedResource>, + target: crate::toolchain::BuildTarget, +) -> Result<Vec<EmbeddedResource>, String> { + let entry = supplied + .get(contract.entry.as_str()) + .ok_or("exec Worker is missing from verified resources")?; + if !entry.executable || entry.bytes.is_empty() { + return Err("exec Worker entry must be a nonempty executable resource".into()); + } + runtime::validate_binary(&contract.entry, &entry.bytes, target.platform())?; + let identity = + serde_json::to_vec(&contract.sdk_manifest()).map_err(|error| error.to_string())?; + let tree = format!("workers/{}", digest(&identity)); + let executable = format!("{tree}/worker"); + let manifest = json!({ + "format": "dever-worker-launch-v1", "ecosystem": "exec", "entry": contract.entry, + "executable": executable, "arguments": [], "working_directory": tree, + }); + Ok(vec![ + resource(executable, entry.bytes.clone(), true), + resource( + format!("{}.dever-worker.json", contract.entry), + serde_json::to_vec(&manifest).map_err(|error| error.to_string())?, + false, + ), + ]) +} + +fn prepare_worker( + project_root: &Path, + contract: &ExternalWorkerContract, + lock: &LockFile, + supplied: &BTreeMap<&str, &EmbeddedResource>, + package_files: &BTreeMap<String, Vec<u8>>, + target: crate::toolchain::BuildTarget, + build_assets: &[EmbeddedResource], +) -> Result<Vec<EmbeddedResource>, String> { + let ecosystem: Ecosystem = contract.ecosystem.parse()?; + let identity = + serde_json::to_vec(&contract.sdk_manifest()).map_err(|error| error.to_string())?; + let tree = format!("workers/{}", digest(&identity)); + let target = target.platform(); + let runtime_path = format!("lib/runtime/{}/{target}/runtime.pack", ecosystem.as_str()); + let runtime = supplied + .get(runtime_path.as_str()) + .ok_or_else(|| format!("managed {} runtime pack is missing", ecosystem.as_str()))?; + let locked_worker = lock + .workers + .iter() + .find(|worker| worker.port == contract.port && worker.adapter == contract.adapter) + .expect("validated Worker lock identity"); + let locked_runtime = locked_worker + .runtime + .as_ref() + .ok_or("managed Worker lacks a locked runtime pack")?; + if runtime.sha256 != locked_runtime.sha256 { + return Err("managed Worker runtime pack differs from its lock".into()); + } + let files = libs::archive_files("tgz", &runtime.bytes)?; + let manifest = runtime::validate(&files, &ecosystem, target)?; + let mut tree_files = Tree::new(tree.clone()); + let runtime_executable = if ecosystem == Ecosystem::Go { + None + } else { + let executable = manifest + .executable + .as_deref() + .expect("validated interpreter executable"); + for (path, bytes) in &files { + tree_files.add( + &format!("runtime/{path}"), + bytes.clone(), + path == executable, + )?; + } + let path = format!("{tree}/runtime/{executable}"); + Some(path) + }; + add_adapter_source(project_root, contract, package_files, &mut tree_files)?; + let runner = match ecosystem { + Ecosystem::Pip => { + add_python_dependencies(contract, lock, supplied, &manifest, &mut tree_files)?; + tree_files.add("dever_component.py", PYTHON_SDK.to_vec(), false)?; + tree_files.add( + "runner.py", + PYTHON_RUNNER + .replace("__ENTRY__", entry_name(&contract.entry)?) + .replace( + "__PYTHON_VERSION__", + &manifest + .python_markers + .as_ref() + .expect("validated Python markers") + .python_version() + .to_string(), + ) + .into_bytes(), + false, + )?; + "runner.py" + } + Ecosystem::Npm => { + ensure_javascript_format(&contract.entry, &mut tree_files)?; + add_javascript_dependencies(contract, lock, supplied, &manifest, &mut tree_files)?; + tree_files.add("dever_component.mjs", JAVASCRIPT_SDK.to_vec(), false)?; + tree_files.add( + "runner.mjs", + JAVASCRIPT_RUNNER + .replace("__ENTRY__", entry_name(&contract.entry)?) + .into_bytes(), + false, + )?; + "runner.mjs" + } + Ecosystem::Go => { + tree_files.add("contract.json", identity.clone(), false)?; + let binary = go_build::compile_worker( + contract, + lock, + supplied, + go_build::Inputs { + files: &files, + manifest: manifest + .build + .as_ref() + .expect("validated Go build descriptor"), + sandbox: build_assets, + }, + &mut tree_files, + target, + )?; + tree_files.remove_prefix("source/"); + let launch = json!({ + "format": "dever-worker-launch-v1", "ecosystem": contract.ecosystem, + "entry": contract.entry, "executable": format!("{tree}/worker{}", std::env::consts::EXE_SUFFIX), + "arguments": [], "working_directory": tree, + }); + let mut output = tree_files.finish(); + output.push(resource( + format!("{}.dever-worker.json", contract.entry), + serde_json::to_vec(&launch).map_err(|error| error.to_string())?, + false, + )); + debug_assert_eq!(binary, format!("worker{}", std::env::consts::EXE_SUFFIX)); + return Ok(output); + } + }; + tree_files.add("contract.json", identity, false)?; + let runner_path = format!("{tree}/{runner}"); + let arguments = manifest + .arguments + .iter() + .map(|value| json!({"literal": value})) + .chain(std::iter::once(json!({"resource": runner_path}))) + .collect::<Vec<_>>(); + let launch = json!({ + "format": "dever-worker-launch-v1", "ecosystem": contract.ecosystem, + "entry": contract.entry, "executable": runtime_executable.expect("interpreter branch"), + "arguments": arguments, "working_directory": tree, + }); + let mut output = tree_files.finish(); + output.push(resource( + format!("{}.dever-worker.json", contract.entry), + serde_json::to_vec(&launch).map_err(|error| error.to_string())?, + false, + )); + Ok(output) +} + +fn digest(bytes: &[u8]) -> String { + Sha256::digest(bytes) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +fn resource(path: String, bytes: Vec<u8>, executable: bool) -> EmbeddedResource { + EmbeddedResource { + sha256: digest(&bytes), + path, + bytes, + executable, + } +} + +pub(crate) fn valid_path(path: &str) -> Result<(), String> { + let relative = Path::new(path); + if path.is_empty() + || path.contains(['\\', ':', '\0']) + || relative.is_absolute() + || relative + .components() + .any(|part| !matches!(part, Component::Normal(_))) + || path.split('/').any(str::is_empty) + { + return Err(format!( + "managed Worker path '{path}' is not portable and relative" + )); + } + Ok(()) +} + +fn entry_name(entry: &str) -> Result<&str, String> { + valid_path(entry)?; + let name = Path::new(entry) + .file_name() + .and_then(|name| name.to_str()) + .ok_or("invalid managed Worker entry name")?; + if !name + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-')) + { + return Err("managed Worker entry name contains an unsafe character".into()); + } + Ok(name) +} + +fn ensure_javascript_format(entry: &str, tree: &mut Tree) -> Result<(), String> { + if !entry_name(entry)?.ends_with(".js") { + return Ok(()); + } + let path = format!("{}/source/package.json", tree.prefix); + if let Some(package) = tree.files.get(&path) { + let metadata: serde_json::Value = serde_json::from_slice(&package.bytes) + .map_err(|error| format!("invalid Adapter package.json: {error}"))?; + if !matches!( + metadata.get("type").and_then(|value| value.as_str()), + Some("module" | "commonjs") + ) { + return Err( + "managed .js Worker entry requires package.json type module or commonjs".into(), + ); + } + } else { + tree.add( + "source/package.json", + b"{\"type\":\"module\"}\n".to_vec(), + false, + )?; + } + Ok(()) +} + +fn add_adapter_source( + project_root: &Path, + contract: &ExternalWorkerContract, + package_files: &BTreeMap<String, Vec<u8>>, + tree: &mut Tree, +) -> Result<(), String> { + let name = entry_name(&contract.entry)?; + let allowed: &[&str] = match contract.ecosystem.as_str() { + "pip" if name.ends_with(".py") => &["py", "pyi", "json"], + "npm" if name.ends_with(".js") || name.ends_with(".mjs") || name.ends_with(".cjs") => { + &["js", "mjs", "cjs", "json"] + } + "go" if name.ends_with(".go") => &[], + _ => return Err("managed Worker entry has an unsupported language extension".into()), + }; + let package_owned = crate::packages::owns_worker(project_root, &contract.entry)?; + if add_package_adapter_source(&contract.entry, allowed, package_files, tree, package_owned)? { + return Ok(()); + } + let entry = libs::owned_worker(project_root, &contract.entry)?; + let parent = entry + .parent() + .ok_or("managed Worker entry has no source directory")?; + let mut pending = vec![parent.to_path_buf()]; + while let Some(directory) = pending.pop() { + let mut children = fs::read_dir(&directory) + .map_err(|error| format!("cannot read Adapter source: {error}"))? + .map(|result| { + result + .map(|entry| entry.path()) + .map_err(|error| error.to_string()) + }) + .collect::<Result<Vec<_>, _>>()?; + children.sort(); + for path in children { + let metadata = fs::symlink_metadata(&path) + .map_err(|error| format!("cannot inspect Adapter source: {error}"))?; + if metadata.file_type().is_symlink() { + return Err("Adapter source contains a symbolic link".into()); + } + let relative = path + .strip_prefix(parent) + .map_err(|error| error.to_string())? + .to_str() + .ok_or("Adapter source path is not UTF-8")? + .replace(std::path::MAIN_SEPARATOR, "/"); + valid_path(&relative)?; + if metadata.is_dir() { + if !path + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| IGNORED_SOURCE_DIRECTORIES.contains(&name)) + { + pending.push(path); + } + } else if metadata.is_file() { + if allowed.is_empty() + || path + .extension() + .and_then(|value| value.to_str()) + .is_some_and(|extension| allowed.contains(&extension)) + { + tree.add( + &format!("source/{relative}"), + fs::read(&path) + .map_err(|error| format!("cannot read Adapter source: {error}"))?, + false, + )?; + } + } else { + return Err("Adapter source contains a non-regular file".into()); + } + } + } + Ok(()) +} + +fn add_package_adapter_source( + entry: &str, + allowed: &[&str], + package_files: &BTreeMap<String, Vec<u8>>, + tree: &mut Tree, + package_owned: bool, +) -> Result<bool, String> { + // Ownership comes from the checked Package lock, not from file presence: + // a missing entire source directory must never enable a local fallback. + if !package_owned { + return Ok(false); + } + if !package_files.contains_key(entry) { + return Err(format!("locked Package Worker entry '{entry}' is missing")); + } + let parent = Path::new(entry) + .parent() + .and_then(|path| path.to_str()) + .ok_or("Package Worker entry has no source directory")?; + let prefix = format!("{parent}/"); + for (path, bytes) in package_files { + let Some(relative) = path.strip_prefix(&prefix) else { + continue; + }; + valid_path(relative)?; + if relative + .split('/') + .any(|part| IGNORED_SOURCE_DIRECTORIES.contains(&part)) + { + continue; + } + if allowed.is_empty() + || Path::new(relative) + .extension() + .and_then(|extension| extension.to_str()) + .is_some_and(|extension| allowed.contains(&extension)) + { + tree.add(&format!("source/{relative}"), bytes.clone(), false)?; + } + } + Ok(true) +} + +fn dependency_closure<'a>( + contract: &ExternalWorkerContract, + lock: &'a LockFile, +) -> Result<Vec<&'a LockedLib>, String> { + let mut remaining = contract + .libs + .iter() + .map(|value| value.parse::<LibSpec>()) + .collect::<Result<Vec<_>, _>>()?; + let mut visited = BTreeSet::new(); + let mut closure = Vec::new(); + while let Some(spec) = remaining.pop() { + if !visited.insert(spec.clone()) { + continue; + } + if spec.ecosystem.as_str() != contract.ecosystem { + return Err("Worker Lib ecosystem differs from Adapter".into()); + } + let lib = lock + .libs + .iter() + .find(|lib| lib.spec == spec) + .ok_or_else(|| format!("locked Worker Lib {} is missing", spec.key()))?; + remaining.extend( + lib.dependencies + .iter() + .map(|dependency| dependency.spec.clone()), + ); + closure.push(lib); + } + closure.sort_by(|left, right| left.spec.cmp(&right.spec)); + Ok(closure) +} + +fn locked_archive<'a>( + lib: &LockedLib, + extension: &str, + supplied: &'a BTreeMap<&str, &EmbeddedResource>, + target: &str, +) -> Result<&'a [u8], String> { + let artifact = lib + .artifacts + .iter() + .find(|artifact| { + artifact.target == target && artifact.path.ends_with(&format!(".{extension}")) + }) + .ok_or_else(|| format!("locked {} archive for {target} is missing", lib.spec.key()))?; + let path = format!( + "lib/{}/{}/{}/archive.{extension}", + lib.spec.ecosystem.as_str(), + digest(lib.spec.distribution_name().as_bytes()), + lib.spec.version + ); + if artifact.path != path { + return Err(format!( + "locked Worker archive path differs from the {} installation layout", + lib.spec.key() + )); + } + let resource = supplied + .get(path.as_str()) + .ok_or_else(|| format!("locked Worker archive '{path}' is missing"))?; + if resource.sha256 != artifact.sha256 || resource.bytes.len() as u64 != artifact.bytes { + return Err(format!( + "locked Worker archive '{path}' differs from its lock" + )); + } + Ok(&resource.bytes) +} + +fn add_python_dependencies( + contract: &ExternalWorkerContract, + lock: &LockFile, + supplied: &BTreeMap<&str, &EmbeddedResource>, + runtime: &runtime::Manifest, + tree: &mut Tree, +) -> Result<(), String> { + let mut installed_distributions = BTreeMap::<&str, &LockedLib>::new(); + let mut wheel_files = BTreeSet::new(); + for lib in dependency_closure(contract, lock)? { + let wheel = python_wheel::Wheel::parse( + &lib.spec, + None, + &runtime.python_wheel_tags, + locked_archive(lib, "whl", supplied, &runtime.target)?, + )?; + wheel.validate_dependencies( + lib, + runtime + .python_markers + .as_ref() + .expect("validated Python markers"), + )?; + libs::build::validate_python_policy( + lock, + lib, + runtime + .python_markers + .as_ref() + .expect("validated Python markers"), + )?; + if let Some(previous) = installed_distributions.insert(lib.spec.distribution_name(), lib) { + if previous.spec.version != lib.spec.version || previous.artifacts != lib.artifacts { + return Err("Python extras refer to conflicting distribution artifacts".into()); + } + continue; + } + for file in wheel.install( + &format!( + "runtime/{}", + runtime + .executable + .as_deref() + .expect("validated Python interpreter") + ), + &runtime + .python_markers + .as_ref() + .expect("validated Python markers") + .python_version() + .to_string(), + )? { + wheel_files.insert(file.path.clone()); + tree.add(&file.path, file.bytes, file.executable)?; + } + } + validate_worker_native( + runtime, + tree, + supplied, + &wheel_files, + &runtime.python_extension_suffixes, + ) +} + +fn validate_worker_native( + runtime: &runtime::Manifest, + tree: &Tree, + supplied: &BTreeMap<&str, &EmbeddedResource>, + entries: &BTreeSet<String>, + suffixes: &[String], +) -> Result<(), String> { + let prefix = format!("{}/", tree.prefix); + let mut files = tree + .files + .iter() + .map(|(path, resource)| { + ( + path.strip_prefix(&prefix) + .expect("owned Worker tree") + .to_owned(), + resource.bytes.as_slice(), + ) + }) + .collect::<BTreeMap<_, _>>(); + files.extend( + supplied + .iter() + .filter(|(path, _)| path.starts_with("sandbox/lib/")) + .map(|(path, resource)| ((*path).to_owned(), resource.bytes.as_slice())), + ); + python_wheel::validate_native( + &runtime.target, + suffixes, + &format!( + "runtime/{}", + runtime + .executable + .as_deref() + .expect("validated interpreter") + ), + &files, + entries, + )?; + Ok(()) +} + +fn add_javascript_dependencies( + contract: &ExternalWorkerContract, + lock: &LockFile, + supplied: &BTreeMap<&str, &EmbeddedResource>, + runtime: &runtime::Manifest, + tree: &mut Tree, +) -> Result<(), String> { + let requested = contract + .libs + .iter() + .map(|value| value.parse()) + .collect::<Result<Vec<LibSpec>, _>>()?; + let Some(environment) = libs::npm::environment(lock, &requested)? else { + return Ok(()); + }; + if let Some(receipt) = libs::build::npm_receipt(lock, environment)? { + if receipt.output.target != runtime.target || receipt.tools.target != runtime.target { + return Err("locked npm build receipt differs from the selected Worker target".into()); + } + let resource = supplied + .get(receipt.output.path.as_str()) + .ok_or("locked npm installation resource is missing")?; + for (path, bytes, executable) in + libs::build::npm_installation(lock, environment, &resource.bytes)? + { + tree.add(&path, bytes, executable)?; + } + let entries = receipt + .native_entries + .iter() + .filter(|path| { + libs::npm::installed_owner(path) + .is_ok_and(|owner| environment.instances.iter().any(|node| node.path == owner)) + }) + .cloned() + .collect(); + return validate_worker_native(runtime, tree, supplied, &entries, &[]); + } + for (path, bytes) in libs::npm::files(environment, lock, |lib| { + locked_archive(lib, "tgz", supplied, &runtime.target) + })? { + tree.add(&path, bytes, false)?; + } + Ok(()) +} + +struct Tree { + prefix: String, + files: BTreeMap<String, EmbeddedResource>, + bytes: usize, +} + +impl Tree { + fn new(prefix: String) -> Self { + Self { + prefix, + files: BTreeMap::new(), + bytes: 0, + } + } + + fn add(&mut self, relative: &str, bytes: Vec<u8>, executable: bool) -> Result<(), String> { + valid_path(relative)?; + let total = self + .bytes + .checked_add(bytes.len()) + .ok_or("managed Worker tree size overflow")?; + if total > MAX_TREE_BYTES || self.files.len() >= MAX_TREE_FILES { + return Err("managed Worker tree exceeds file or byte budget".into()); + } + let path = format!("{}/{relative}", self.prefix); + if self.files.contains_key(&path) { + return Err(format!("managed Worker tree repeats '{path}'")); + } + let mut ancestor = path.as_str(); + while let Some((parent, _)) = ancestor.rsplit_once('/') { + if self.files.contains_key(parent) { + return Err(format!( + "managed Worker tree has a file/directory conflict at '{parent}'" + )); + } + ancestor = parent; + } + let descendant_prefix = format!("{path}/"); + if self + .files + .range(descendant_prefix.clone()..) + .next() + .is_some_and(|(existing, _)| existing.starts_with(&descendant_prefix)) + { + return Err(format!( + "managed Worker tree has a file/directory conflict at '{path}'" + )); + } + self.bytes = total; + self.files + .insert(path.clone(), resource(path, bytes, executable)); + Ok(()) + } + + fn finish(self) -> Vec<EmbeddedResource> { + self.files.into_values().collect() + } + + fn remove_prefix(&mut self, relative: &str) { + let prefix = format!("{}/{relative}", self.prefix); + self.files.retain(|path, resource| { + if path.starts_with(&prefix) { + self.bytes -= resource.bytes.len(); + false + } else { + true + } + }); + } +} diff --git a/crates/dever-cli/src/workers/command.rs b/crates/dever-cli/src/workers/command.rs new file mode 100644 index 0000000..0187590 --- /dev/null +++ b/crates/dever-cli/src/workers/command.rs @@ -0,0 +1,239 @@ +//! Offline native command trees, including their explicitly supplied libraries. + +use std::collections::BTreeMap; +use std::fs::{self, File}; +use std::io::Read; +use std::path::Path; + +use dever_core::hir::ExternalWorkerContract; +use dever_core::native::EmbeddedResource; +use goblin::elf::{dynamic, header}; +use serde_json::json; + +use super::{MAX_TREE_BYTES, MAX_TREE_FILES, Tree, digest, elf, resource, runtime, valid_path}; +use crate::toolchain::BuildTarget; + +pub(super) fn prepare( + root: &Path, + contract: &ExternalWorkerContract, + package_files: &BTreeMap<String, Vec<u8>>, + supplied: &BTreeMap<&str, &EmbeddedResource>, + target: BuildTarget, +) -> Result<Vec<EmbeddedResource>, String> { + let prefix = format!( + "commands/{}", + digest(&serde_json::to_vec(&contract.sdk_manifest()).map_err(|error| error.to_string())?) + ); + let mut tree = Tree::new(prefix.clone()); + let source = command_files(root, &contract.entry, package_files)?; + let binary = &source["program/tool"]; + runtime::validate_binary(&contract.entry, binary, target.platform())?; + let machine = match target { + BuildTarget::LinuxX86_64 => header::EM_X86_64, + BuildTarget::LinuxAarch64 => header::EM_AARCH64, + }; + let executable = elf::object("program/tool", binary, machine)?; + let interpreter = executable.interpreter.map(str::to_owned); + if interpreter.is_none() && !executable.libraries.is_empty() { + return Err("command executable has dynamic dependencies without an interpreter".into()); + } + let mut files = source + .iter() + .map(|(path, bytes)| (path.clone(), bytes.as_slice())) + .collect::<BTreeMap<_, _>>(); + files.extend( + supplied + .iter() + .filter(|(path, _)| path.starts_with("sandbox/lib/")) + .map(|(path, entry)| ((*path).to_owned(), entry.bytes.as_slice())), + ); + let closure = elf::Closure { + files: &files, + machine, + directories: vec!["program/lib".into(), "sandbox/lib".into()], + }; + for (path, bytes) in &source { + let object = elf::object(path, bytes, machine)?; + reject_loader_hooks(&object)?; + // The logical program root becomes /worker in the sandbox. A path + // escaping it could otherwise resolve a library from a file grant. + for fields in [&object.rpaths, &object.runpaths] { + for directory in elf::origin_directories(path, fields)? { + if directory != "program" && !directory.starts_with("program/") { + return Err("command library search path escapes its packaged tree".into()); + } + } + } + if let Some(name) = path.strip_prefix("program/lib/") { + if supplied.contains_key(format!("sandbox/lib/{name}").as_str()) { + return Err(format!( + "command library '{name}' overrides a signed sandbox library" + )); + } + if object.header.e_type != header::ET_DYN + || object.interpreter.is_some() + || object.soname.is_some_and(|soname| soname != name) + { + return Err(format!( + "command library '{name}' is not a matching SONAME shared object" + )); + } + } + closure.follow(path, Vec::new(), &BTreeMap::new())?; + } + for (path, bytes) in source { + let name = path.strip_prefix("program/").expect("owned command tree"); + tree.add(name, bytes, name == "tool")?; + } + let (executable, arguments) = if let Some(interpreter) = interpreter { + let name = interpreter + .rsplit('/') + .next() + .expect("validated GNU loader"); + let loader = supplied + .get(format!("sandbox/lib/{name}").as_str()) + .ok_or("command requires the signed target GNU loader")?; + runtime::validate_binary(name, &loader.bytes, target.platform())?; + tree.add("loader", loader.bytes.clone(), true)?; + ( + format!("{prefix}/loader"), + vec![ + json!({"literal": "--inhibit-cache"}), + json!({"literal": "--library-path"}), + json!({"literal": "lib:/lib"}), + json!({"resource": format!("{prefix}/tool")}), + ], + ) + } else { + (format!("{prefix}/tool"), Vec::new()) + }; + let manifest = json!({ + "format": "dever-command-launch-v1", "ecosystem": "command", + "entry": contract.entry, "executable": executable, + "arguments": arguments, "working_directory": prefix, + }); + let mut output = tree.finish(); + output.push(resource( + format!("{}.dever-command.json", contract.entry), + serde_json::to_vec(&manifest).map_err(|error| error.to_string())?, + false, + )); + Ok(output) +} + +fn reject_loader_hooks(elf: &goblin::elf::Elf<'_>) -> Result<(), String> { + if elf.dynamic.as_ref().is_some_and(|table| { + table.dyns.iter().any(|entry| { + matches!( + entry.d_tag, + dynamic::DT_CONFIG + | dynamic::DT_AUDIT + | dynamic::DT_DEPAUDIT + | 0x7fff_fffd + | 0x7fff_ffff + ) + }) + }) { + return Err("command ELF contains unsupported dynamic loader hooks".into()); + } + Ok(()) +} + +fn command_files( + root: &Path, + entry: &str, + package_files: &BTreeMap<String, Vec<u8>>, +) -> Result<BTreeMap<String, Vec<u8>>, String> { + valid_path(entry)?; + let parent = entry + .rsplit_once('/') + .map(|(parent, _)| parent) + .ok_or("command entry must belong to a domain")?; + let library_prefix = format!("{parent}/lib/"); + let packaged = crate::packages::owns_worker(root, entry)?; + let binary = if packaged { + package_files + .get(entry) + .cloned() + .ok_or("Package command executable is missing")? + } else { + read_file(root, entry)? + }; + let mut files = BTreeMap::from([("program/tool".to_owned(), binary)]); + let mut bytes = files["program/tool"].len(); + if bytes == 0 || bytes > MAX_TREE_BYTES { + return Err("command executable is empty or exceeds the byte budget".into()); + } + let names = if packaged { + package_files + .keys() + .filter_map(|path| path.strip_prefix(&library_prefix)) + .map(str::to_owned) + .collect::<Vec<_>>() + } else { + library_names(&root.join(&library_prefix))? + }; + for name in names { + valid_path(&name)?; + if name.contains('/') || files.len() >= MAX_TREE_FILES { + return Err("command lib directory must contain bounded regular library files".into()); + } + let path = format!("{library_prefix}{name}"); + let library = if packaged { + package_files[&path].clone() + } else { + read_file(root, &path)? + }; + bytes = bytes + .checked_add(library.len()) + .ok_or("command library size overflow")?; + if bytes > MAX_TREE_BYTES { + return Err("command libraries exceed the byte budget".into()); + } + files.insert(format!("program/lib/{name}"), library); + } + Ok(files) +} + +fn library_names(directory: &Path) -> Result<Vec<String>, String> { + let metadata = match fs::symlink_metadata(directory) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(error) => return Err(format!("cannot inspect command lib directory: {error}")), + }; + if !metadata.is_dir() || metadata.file_type().is_symlink() { + return Err("command lib directory must be a real directory".into()); + } + let mut names = Vec::new(); + for entry in fs::read_dir(directory).map_err(|error| error.to_string())? { + let entry = entry.map_err(|error| error.to_string())?; + if names.len() >= MAX_TREE_FILES + || !entry + .file_type() + .map_err(|error| error.to_string())? + .is_file() + { + return Err("command lib directory must contain bounded regular library files".into()); + } + names.push( + entry + .file_name() + .into_string() + .map_err(|_| "command library name is not UTF-8")?, + ); + } + names.sort(); + Ok(names) +} + +fn read_file(root: &Path, entry: &str) -> Result<Vec<u8>, String> { + let path = crate::libs::owned_worker(root, entry)?; + let mut bytes = Vec::new(); + File::open(path) + .and_then(|file| file.take(MAX_TREE_BYTES as u64 + 1).read_to_end(&mut bytes)) + .map_err(|error| format!("cannot read command resource '{entry}': {error}"))?; + if bytes.len() > MAX_TREE_BYTES { + return Err("command resource exceeds the byte budget".into()); + } + Ok(bytes) +} diff --git a/crates/dever-cli/src/workers/elf.rs b/crates/dever-cli/src/workers/elf.rs new file mode 100644 index 0000000..cb74ae9 --- /dev/null +++ b/crates/dever-cli/src/workers/elf.rs @@ -0,0 +1,176 @@ +//! Shared ELF architecture, loader search and versioned dependency closure. + +use std::collections::{BTreeMap, BTreeSet, VecDeque}; + +use goblin::elf::{Elf, header}; + +pub(crate) fn object<'a>(path: &str, bytes: &'a [u8], machine: u16) -> Result<Elf<'a>, String> { + let elf = Elf::parse(bytes).map_err(|error| format!("invalid native ELF '{path}': {error}"))?; + if !elf.is_64 + || !elf.little_endian + || elf.header.e_machine != machine + || !matches!(elf.header.e_type, header::ET_DYN | header::ET_EXEC) + { + return Err(format!( + "native ELF '{path}' has an incompatible target or object type" + )); + } + Ok(elf) +} + +pub(crate) fn origin_directories(path: &str, fields: &[&str]) -> Result<Vec<String>, String> { + let parent = path.rsplit_once('/').map_or("", |(parent, _)| parent); + let mut result = Vec::new(); + for field in fields { + for directory in field.split(':') { + let relative = directory + .strip_prefix("$ORIGIN") + .or_else(|| directory.strip_prefix("${ORIGIN}")) + .ok_or("native library search path must be relative to $ORIGIN")?; + if !relative.is_empty() && !relative.starts_with('/') + || relative.contains(['$', '\\', '\0']) + { + return Err("native library search path has an unsupported token".into()); + } + let combined = format!("{parent}{relative}"); + let mut normalized = Vec::new(); + for part in combined.split('/') { + match part { + "" | "." => {} + ".." => { + normalized + .pop() + .ok_or("native library path escapes the installation tree")?; + } + part => normalized.push(part), + } + } + let normalized = normalized.join("/"); + if normalized.is_empty() || normalized.starts_with("sandbox/") { + return Err("native library path escapes its Worker installation".into()); + } + if !result.contains(&normalized) { + result.push(normalized); + } + } + } + Ok(result) +} + +fn plain_library(name: &str) -> Result<(), String> { + if name.is_empty() || matches!(name, "." | "..") || name.contains(['/', '\\', '\0']) { + return Err("native DT_NEEDED must be a plain library name".into()); + } + Ok(()) +} + +fn validate_versions(consumer: &Elf<'_>, library: &str, provider: &Elf<'_>) -> Result<(), String> { + let mut provided = BTreeSet::new(); + if let Some(definitions) = &provider.verdef { + for definition in definitions.iter() { + for name in definition.iter() { + provided.insert( + provider + .dynstrtab + .get_at(name.vda_name) + .ok_or("invalid ELF version definition")? + .to_owned(), + ); + } + } + } + if let Some(requirements) = &consumer.verneed { + for requirement in requirements.iter() { + if consumer.dynstrtab.get_at(requirement.vn_file) != Some(library) { + continue; + } + for version in requirement.iter() { + let name = consumer + .dynstrtab + .get_at(version.vna_name) + .ok_or("invalid ELF version requirement")?; + if version.vna_flags & 2 == 0 && !provided.contains(name) { + return Err(format!( + "packaged library '{library}' does not provide native ABI version '{name}'" + )); + } + } + } + } + Ok(()) +} + +pub(crate) struct Closure<'a> { + pub(crate) files: &'a BTreeMap<String, &'a [u8]>, + pub(crate) machine: u16, + pub(crate) directories: Vec<String>, +} + +impl Closure<'_> { + pub(crate) fn follow( + &self, + start: &str, + inherited: Vec<String>, + globals: &BTreeMap<String, String>, + ) -> Result<(BTreeMap<String, String>, BTreeSet<String>), String> { + let mut loaded = globals.clone(); + let mut pending = VecDeque::from([(start.to_owned(), inherited)]); + let mut seen = BTreeSet::new(); + while let Some((path, inherited)) = pending.pop_front() { + if !seen.insert(path.clone()) { + continue; + } + let bytes = self + .files + .get(&path) + .ok_or_else(|| format!("native dependency '{path}' is absent"))?; + let elf = object(&path, bytes, self.machine)?; + let rpaths = if elf.runpaths.is_empty() { + origin_directories(&path, &elf.rpaths)? + } else { + Vec::new() + }; + let runpaths = origin_directories(&path, &elf.runpaths)?; + let mut inherited_next = rpaths.clone(); + inherited_next.extend(inherited.clone()); + let inherited_search = if elf.runpaths.is_empty() { + inherited.as_slice() + } else { + &[] + }; + for name in &elf.libraries { + plain_library(name)?; + let supplied = loaded + .get(*name) + .cloned() + .or_else(|| { + rpaths + .iter() + .chain(inherited_search.iter()) + .chain(runpaths.iter()) + .map(|directory| format!("{directory}/{name}")) + .chain( + self.directories + .iter() + .map(|directory| format!("{directory}/{name}")), + ) + .find(|candidate| self.files.contains_key(candidate)) + }) + .ok_or_else(|| { + format!("native library '{name}' is not in the packaged dependency closure") + })?; + let provider = object(&supplied, self.files[&supplied], self.machine)?; + validate_versions(&elf, name, &provider)?; + loaded.insert((*name).to_owned(), supplied.clone()); + if let Some(soname) = provider.soname { + plain_library(soname)?; + if soname != *name { + return Err("native library SONAME differs from DT_NEEDED".into()); + } + } + pending.push_back((supplied, inherited_next.clone())); + } + } + Ok((loaded, seen)) + } +} diff --git a/crates/dever-cli/src/workers/go_build.rs b/crates/dever-cli/src/workers/go_build.rs new file mode 100644 index 0000000..38d57c9 --- /dev/null +++ b/crates/dever-cli/src/workers/go_build.rs @@ -0,0 +1,574 @@ +//! Compile a checked Go Adapter with the exact tools and archives in its locked pack. + +use std::collections::{BTreeMap, BTreeSet}; +use std::ffi::OsString; +use std::fs; +use std::io::Read; +use std::path::{Path, PathBuf}; + +use dever_core::hir::ExternalWorkerContract; +use dever_core::native::EmbeddedResource; +use serde::Deserialize; +use serde_json::json; + +use super::{Tree, dependency_closure, digest, entry_name, locked_archive, valid_path}; +use crate::libs::build::process::Stage; +use crate::libs::{self, LockFile}; + +const SDK: &[u8] = include_bytes!("../../../../sdk/go/component.go"); + +#[derive(Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct GoBuildManifest { + host: crate::toolchain::BuildTarget, + compiler: String, + linker: String, + analyzer: String, + stdlib_importcfg: String, + goos: String, + goarch: String, + go_version: String, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct SelectedPackage { + name: String, + files: Vec<String>, + imports: Vec<String>, + embed: BTreeMap<String, Vec<String>>, +} + +struct Builder { + stage: Stage, + input: PathBuf, + assets: PathBuf, + grants: Vec<dever_sandbox::Grant>, + tools: GoBuildManifest, + entry: String, + stdlib: BTreeMap<String, PathBuf>, + modules: BTreeMap<String, PathBuf>, + archives: BTreeMap<String, PathBuf>, + visiting: BTreeSet<String>, +} + +pub(super) struct Inputs<'a> { + pub files: &'a [(String, Vec<u8>)], + pub manifest: &'a GoBuildManifest, + pub sandbox: &'a [EmbeddedResource], +} + +pub(super) fn compile_worker( + contract: &ExternalWorkerContract, + lock: &LockFile, + supplied: &BTreeMap<&str, &EmbeddedResource>, + inputs: Inputs<'_>, + tree: &mut Tree, + target: &str, +) -> Result<String, String> { + let Inputs { + files: pack_files, + manifest: build, + sandbox: build_assets, + } = inputs; + let references = build.validate_files(pack_files, target)?; + if build.host != crate::toolchain::BuildTarget::host()? { + return Err("Go build tools require a different build host".into()); + } + let stage = Stage::new()?; + let input = stage.0.join("input"); + let assets = stage.0.join("sandbox"); + for resource in build_assets + .iter() + .filter(|resource| resource.path.starts_with("sandbox/")) + { + if digest(&resource.bytes) != resource.sha256 { + return Err("Go build sandbox asset has a digest mismatch".into()); + } + stage.write(&resource.path, &resource.bytes, resource.executable)?; + } + let build_directory = stage.0.join("build"); + fs::create_dir(&build_directory) + .map_err(|error| format!("cannot create Go build output: {error}"))?; + let grants = vec![dever_sandbox::Grant { + source: build_directory, + destination: "/data/build".into(), + writable: true, + }]; + for (path, bytes) in pack_files { + stage.write( + &format!("input/{path}"), + bytes, + path == &build.compiler || path == &build.linker || path == &build.analyzer, + )?; + } + let stdlib = references + .into_iter() + .map(|(name, path)| (name, input.join(path))) + .collect(); + let source_prefix = format!("{}/source/", tree.prefix); + let mut entry_present = false; + for (path, resource) in &tree.files { + if let Some(relative) = path.strip_prefix(&source_prefix) { + if relative == entry_name(&contract.entry)? { + entry_present = true; + } + stage.write(&format!("input/source/{relative}"), &resource.bytes, false)?; + } + } + if !entry_present { + return Err("managed Go Worker entry source is missing".into()); + } + if tree + .files + .contains_key(&format!("{}/source/dever_worker_runner.go", tree.prefix)) + { + return Err("Adapter source conflicts with the generated Go Worker runner".into()); + } + stage.write("input/sdk/component.go", SDK, false)?; + let runner = runner_source(contract)?; + stage.write( + "input/source/dever_worker_runner.go", + runner.as_bytes(), + false, + )?; + let mut modules = BTreeMap::new(); + let authenticated = + libs::sumdb::verify_chain(&lock.go_sumdb, &libs::sumdb::Verifier::official())?; + for lib in dependency_closure(contract, lock)? { + let root = format!("{}@v{}/", lib.spec.name, lib.spec.version); + let directory = format!("modules/{}", digest(lib.spec.key().as_bytes())); + let archive = locked_archive(lib, "zip", supplied, target)?; + authenticated.verify_zip(&lib.spec.name, &lib.spec.version, archive)?; + let files = libs::archive_files("zip", archive)?; + if files.is_empty() { + return Err(format!("locked Go module {} is empty", lib.spec.key())); + } + for (path, bytes) in files { + let relative = path.strip_prefix(&root).ok_or_else(|| { + format!( + "locked Go module {} has a mismatched archive root", + lib.spec.key() + ) + })?; + stage.write(&format!("input/{directory}/{relative}"), &bytes, false)?; + } + if modules + .insert(lib.spec.name.clone(), input.join(directory)) + .is_some() + { + return Err("locked Go module graph selects multiple versions of one module".into()); + } + } + let mut builder = Builder { + stage, + input, + assets, + grants, + tools: build.clone(), + entry: entry_name(&contract.entry)?.into(), + stdlib, + modules, + archives: BTreeMap::new(), + visiting: BTreeSet::new(), + }; + builder.verify_tools()?; + builder.compile("dever-component")?; + builder.compile("main")?; + let binary_name = format!("worker{}", std::env::consts::EXE_SUFFIX); + builder.link(&binary_name)?; + let path = builder.stage.0.join("build").join(&binary_name); + let mut options = fs::OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(rustix::fs::OFlags::NOFOLLOW.bits() as i32); + } + let file = options + .open(path) + .map_err(|error| format!("cannot open linked Go Worker: {error}"))?; + let metadata = file.metadata().map_err(|error| error.to_string())?; + if !metadata.is_file() || metadata.len() > 128 * 1024 * 1024 { + return Err("linked Go Worker is not a bounded regular file".into()); + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + if metadata.nlink() != 1 { + return Err("linked Go Worker must not have hard links".into()); + } + } + let mut binary = Vec::new(); + file.take(128 * 1024 * 1024 + 1) + .read_to_end(&mut binary) + .map_err(|error| format!("cannot read linked Go Worker: {error}"))?; + if binary.len() > 128 * 1024 * 1024 { + return Err("linked Go Worker exceeds the byte limit".into()); + } + if binary.is_empty() { + return Err("linked Go Worker is empty".into()); + } + super::runtime::validate_binary("Go Worker", &binary, target)?; + tree.add(&binary_name, binary, true)?; + Ok(binary_name) +} + +impl GoBuildManifest { + pub(super) fn executable(&self, path: &str) -> bool { + [&self.compiler, &self.linker, &self.analyzer] + .iter() + .any(|tool| tool.as_str() == path) + } + + pub(super) fn validate_files( + &self, + files: &[(String, Vec<u8>)], + target: &str, + ) -> Result<BTreeMap<String, String>, String> { + self.validate(target)?; + let mut pack = BTreeMap::new(); + for (path, bytes) in files { + if pack.insert(path.as_str(), bytes.as_slice()).is_some() { + return Err("Go build pack repeats a file".into()); + } + } + for tool in [&self.compiler, &self.linker, &self.analyzer] { + if !tool.starts_with("bin/") + || !pack + .get(tool.as_str()) + .is_some_and(|bytes| !bytes.is_empty()) + { + return Err(format!("locked Go build tool '{tool}' is missing")); + } + super::runtime::validate_binary(tool, pack[tool.as_str()], self.host.platform())?; + } + if self.stdlib_importcfg != "stdlib/importcfg" { + return Err("locked Go standard library importcfg path is invalid".into()); + } + let importcfg = pack + .get(self.stdlib_importcfg.as_str()) + .ok_or("locked Go standard library importcfg is missing")?; + let importcfg = std::str::from_utf8(importcfg) + .map_err(|_| "locked Go standard library importcfg is not UTF-8")?; + let mut stdlib = BTreeMap::new(); + for line in importcfg.lines() { + let (name, path) = line + .strip_prefix("packagefile ") + .and_then(|line| line.split_once('=')) + .ok_or("invalid locked Go standard library importcfg")?; + valid_import(name)?; + if path != format!("stdlib/{name}.a") + || !pack.get(path).is_some_and(|bytes| !bytes.is_empty()) + || stdlib.insert(name.into(), path.into()).is_some() + { + return Err( + "locked Go standard library importcfg has a missing or repeated archive".into(), + ); + } + let archive = pack[path]; + let version = self + .go_version + .split_whitespace() + .nth(2) + .ok_or("invalid Go version")?; + let object_identity = format!("go object {} {} {version} ", self.goos, self.goarch); + if !archive.starts_with(b"!<arch>\n__.PKGDEF") + || !archive + .get(68..) + .is_some_and(|bytes| bytes.starts_with(object_identity.as_bytes())) + { + return Err(format!( + "locked Go standard library archive '{name}' differs from its target or compiler version" + )); + } + } + if !["runtime", "context", "encoding/json"] + .iter() + .all(|name| stdlib.contains_key(*name)) + { + return Err("locked Go standard library importcfg is incomplete".into()); + } + Ok(stdlib) + } + + fn validate(&self, target: &str) -> Result<(), String> { + let expected = match target { + "linux-x86_64" => ("linux", "amd64"), + "linux-aarch64" => ("linux", "arm64"), + _ => return Err("managed Go build target is unsupported".into()), + }; + if (self.goos.as_str(), self.goarch.as_str()) != expected + || !self.go_version.starts_with("go version go") + || !self.go_version.ends_with(match self.host { + crate::toolchain::BuildTarget::LinuxX86_64 => " linux/amd64", + crate::toolchain::BuildTarget::LinuxAarch64 => " linux/arm64", + }) + { + return Err("managed Go build tools do not match the selected target".into()); + } + for path in [ + &self.compiler, + &self.linker, + &self.analyzer, + &self.stdlib_importcfg, + ] { + valid_path(path)?; + } + Ok(()) + } +} + +impl Builder { + fn verify_tools(&self) -> Result<(), String> { + let version = self + .tools + .go_version + .split_whitespace() + .nth(2) + .ok_or("locked Go build version is invalid")?; + for (kind, path) in [ + ("compile", &self.tools.compiler), + ("link", &self.tools.linker), + ] { + let output = self.run(&self.input.join(path), &["-V"])?; + if output != format!("{kind} version {version}\n").as_bytes() { + return Err(format!( + "locked Go {kind} tool version differs from its pack" + )); + } + } + Ok(()) + } + + fn package_dir(&self, name: &str) -> Result<PathBuf, String> { + if name == "main" { + return Ok(self.input.join("source")); + } + if name == "dever-component" { + return Ok(self.input.join("sdk")); + } + if let Some(relative) = name.strip_prefix("dever-worker/") { + valid_path(relative)?; + return Ok(self.input.join("source").join(relative)); + } + let (module, root) = self + .modules + .iter() + .filter(|(module, _)| { + name == module.as_str() || name.starts_with(&format!("{module}/")) + }) + .max_by_key(|(module, _)| module.len()) + .ok_or_else(|| format!("Go import '{name}' is not in the locked module graph"))?; + let suffix = name + .strip_prefix(module) + .expect("selected module prefix") + .trim_start_matches('/'); + if !suffix.is_empty() { + valid_path(suffix)?; + } + Ok(root.join(suffix)) + } + + fn compile(&mut self, name: &str) -> Result<(), String> { + if self.stdlib.contains_key(name) || name == "unsafe" || self.archives.contains_key(name) { + return Ok(()); + } + valid_import(name)?; + if !self.visiting.insert(name.to_owned()) { + return Err(format!("Go import cycle includes '{name}'")); + } + let directory = self.package_dir(name)?; + let analyzer = self.input.join(&self.tools.analyzer); + let output = self.run( + &analyzer, + &[ + self.tools.goos.as_str(), + self.tools.goarch.as_str(), + directory.to_str().ok_or("Go package path is not UTF-8")?, + ], + )?; + let selected: SelectedPackage = serde_json::from_slice(&output) + .map_err(|error| format!("invalid locked Go analyzer output: {error}"))?; + if selected.files.is_empty() || (name == "main") != (selected.name == "main") { + return Err(format!( + "Go package '{name}' has an invalid package declaration" + )); + } + if name == "main" && !selected.files.contains(&self.entry) { + // The generated runner may be selected while the checked entry is excluded by tags. + return Err("checked Go Worker entry is excluded by target build constraints".into()); + } + for import in &selected.imports { + self.compile(import)?; + } + let cfg = self.importcfg()?; + let id = digest(name.as_bytes()); + let cfg_path = self.stage.write( + &format!("input/config/{id}.importcfg"), + cfg.as_bytes(), + false, + )?; + let archive = self.stage.0.join(format!("build/{id}.a")); + let mut args = vec![ + "-pack".to_owned(), + "-complete".into(), + "-nolocalimports".into(), + // Staging is invocation-owned; recorded source paths must not make + // identical locked Workers differ across run/build or projects. + "-trimpath".into(), + self.input.to_string_lossy().into_owned(), + "-p".into(), + name.into(), + "-importcfg".into(), + cfg_path.to_string_lossy().into_owned(), + "-o".into(), + archive.to_string_lossy().into_owned(), + ]; + if !selected.embed.is_empty() { + let mut files = BTreeMap::new(); + for paths in selected.embed.values() { + for path in paths { + valid_path(path)?; + let file = directory.join(path); + if !file.is_file() { + return Err("Go embed file is absent from locked source".into()); + } + files.insert(path.clone(), self.visible_path(&file)?); + } + } + let embed = json!({"Patterns":selected.embed,"Files":files}); + let path = self.stage.write( + &format!("input/config/{id}.embedcfg"), + &serde_json::to_vec(&embed).map_err(|error| error.to_string())?, + false, + )?; + args.extend(["-embedcfg".into(), path.to_string_lossy().into_owned()]); + } + for file in selected.files { + valid_path(&file)?; + args.push(directory.join(file).to_string_lossy().into_owned()); + } + let compiler = self.input.join(&self.tools.compiler); + let arguments = args.iter().map(String::as_str).collect::<Vec<_>>(); + self.run(&compiler, &arguments)?; + self.archives.insert(name.into(), archive); + self.visiting.remove(name); + Ok(()) + } + + fn importcfg(&self) -> Result<String, String> { + let mut cfg = String::new(); + for (name, path) in self.stdlib.iter().chain(&self.archives) { + cfg.push_str(&format!( + "packagefile {name}={}\n", + self.visible_path(path)? + )); + } + Ok(cfg) + } + + fn link(&self, output: &str) -> Result<(), String> { + let cfg = self.stage.write( + "input/config/link.importcfg", + self.importcfg()?.as_bytes(), + false, + )?; + let main = self + .archives + .get("main") + .ok_or("Go Worker main archive is missing")?; + let linker = self.input.join(&self.tools.linker); + let output_path = self.stage.0.join("build").join(output); + let temporary_directory = self.stage.0.join("build"); + self.run( + &linker, + &[ + "-linkmode=internal", + "-buildmode=exe", + "-tmpdir", + temporary_directory + .to_str() + .ok_or("Go build path is not UTF-8")?, + "-importcfg", + cfg.to_str().ok_or("Go importcfg path is not UTF-8")?, + "-o", + output_path.to_str().ok_or("Go output path is not UTF-8")?, + main.to_str().ok_or("Go archive path is not UTF-8")?, + ], + )?; + Ok(()) + } + + fn run(&self, executable: &Path, args: &[&str]) -> Result<Vec<u8>, String> { + let arguments = args.iter().map(OsString::from).collect::<Vec<_>>(); + self.stage.run( + self.sandbox(executable, &arguments).command()?, + "locked Go build tool", + &self.stage.0.join("build"), + ) + } + + fn sandbox<'a>( + &'a self, + executable: &'a Path, + arguments: &'a [OsString], + ) -> dever_sandbox::Launch<'a> { + dever_sandbox::Launch { + assets: &self.assets, + worker: &self.input, + executable, + arguments, + working_directory: &self.input, + capabilities: dever_sandbox::Capabilities::default(), + grants: &self.grants, + } + } + + fn visible_path(&self, path: &Path) -> Result<String, String> { + self.sandbox(&self.input, &[]) + .argument_path(path)? + .into_os_string() + .into_string() + .map_err(|_| "Go sandbox path is not UTF-8".into()) + } +} + +fn valid_import(name: &str) -> Result<(), String> { + if name.is_empty() + || name.split('/').any(str::is_empty) + || !name.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'/' | b'.' | b'_' | b'-' | b'~') + }) + { + return Err(format!("invalid locked Go import '{name}'")); + } + Ok(()) +} + +fn runner_source(contract: &ExternalWorkerContract) -> Result<String, String> { + let mut handlers = String::new(); + let mut names = BTreeSet::new(); + for operation in &contract.operations { + let mut function = String::new(); + for part in operation.split(['.', '_', '-']) { + if part.is_empty() || !part.bytes().all(|byte| byte.is_ascii_alphanumeric()) { + return Err(format!( + "Go operation '{operation}' cannot bind to an exported function" + )); + } + let mut letters = part.chars(); + function.push(letters.next().expect("nonempty part").to_ascii_uppercase()); + function.extend(letters); + } + if !names.insert(function.clone()) + || !function.starts_with(|character: char| character.is_ascii_uppercase()) + { + return Err("Go operation names map to duplicate or invalid exported functions".into()); + } + handlers.push_str(&format!("\t\t{operation:?}: {function},\n")); + } + Ok(format!( + "package main\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"os\"\n\tcomponent \"dever-component\"\n)\n\nfunc main() {{\n\tmanifest, err := os.ReadFile(\"contract.json\")\n\tif err == nil {{\n\t\terr = component.ServeManifest(manifest, map[string]func(context.Context, any, any) (any, error){{\n{handlers}\t\t}}, os.Stdin, os.Stdout)\n\t}}\n\tif err != nil {{\n\t\tfmt.Fprintln(os.Stderr, err)\n\t\tos.Exit(1)\n\t}}\n}}\n" + )) +} diff --git a/crates/dever-cli/src/workers/python_wheel.rs b/crates/dever-cli/src/workers/python_wheel.rs new file mode 100644 index 0000000..6e8ca34 --- /dev/null +++ b/crates/dever-cli/src/workers/python_wheel.rs @@ -0,0 +1,588 @@ +//! One wheel admission and installation contract for registry downloads, +//! offline Workers and isolated build dependencies. + +use std::collections::{BTreeMap, BTreeSet}; + +use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; +use pep440_rs::Version; +use pep508_rs::Requirement; +use sha2::{Digest, Sha256, Sha384, Sha512}; + +use crate::libs::{LibSpec, archive_files}; + +pub(crate) mod native; +mod scripts; +pub use native::validate_native; + +pub struct Wheel { + files: Vec<(String, Vec<u8>)>, + stem: String, + pub requirements: Vec<String>, + pub extras: Vec<String>, + pub requires_python: Option<String>, +} + +pub struct InstalledFile { + pub path: String, + pub bytes: Vec<u8>, + pub executable: bool, +} + +/// Filename tags can be compressed; runtime and WHEEL tags must be expanded. +pub fn validate_tags(tags: &[String]) -> Result<(), String> { + let mut unique = BTreeSet::new(); + if tags.is_empty() || tags.len() > 4096 { + return Err("Python wheel compatibility tags are missing or exceed their bound".into()); + } + for tag in tags { + if tag.split('-').count() != 3 + || tag.split('-').any(|part| { + part.is_empty() + || !part + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') + }) + || !unique.insert(tag) + { + return Err("Python wheel compatibility tags are invalid or repeated".into()); + } + } + Ok(()) +} + +pub(crate) fn validate_target( + target: &str, + tags: &[String], + markers: &pep508_rs::MarkerEnvironment, +) -> Result<(), String> { + validate_tags(tags)?; + let architecture = match target.parse::<crate::toolchain::BuildTarget>()? { + crate::toolchain::BuildTarget::LinuxX86_64 => "x86_64", + crate::toolchain::BuildTarget::LinuxAarch64 => "aarch64", + }; + if markers.sys_platform() != "linux" || markers.platform_machine() != architecture { + return Err("Python marker environment differs from the selected target".into()); + } + for tag in tags { + let platform = tag.rsplit('-').next().expect("validated wheel tag"); + if platform != "any" + && (!(platform.starts_with("linux_") + || platform.starts_with("manylinux") + || platform.starts_with("musllinux")) + || !platform.ends_with(&format!("_{architecture}"))) + { + return Err("Python wheel platform tag differs from the selected target".into()); + } + } + Ok(()) +} + +fn expanded_tags(python: &str, abi: &str, platform: &str) -> Result<BTreeSet<String>, String> { + let mut tags = BTreeSet::new(); + for python in python.split('.') { + for abi in abi.split('.') { + for platform in platform.split('.') { + if tags.len() >= 4096 || !tags.insert(format!("{python}-{abi}-{platform}")) { + return Err("wheel filename tags are repeated or exceed their bound".into()); + } + } + } + } + validate_tags(&tags.iter().cloned().collect::<Vec<_>>())?; + Ok(tags) +} + +fn filename( + filename: &str, + spec: Option<&LibSpec>, +) -> Result<(BTreeSet<String>, Option<String>), String> { + super::valid_path(filename)?; + if filename.contains('/') { + return Err("wheel filename must be a leaf name".into()); + } + let fields = filename + .strip_suffix(".whl") + .ok_or("wheel filename lacks .whl")? + .split('-') + .collect::<Vec<_>>(); + if !matches!(fields.len(), 5 | 6) { + return Err("invalid wheel filename fields".into()); + } + if let Some(spec) = spec { + validate_identity(fields[0], fields[1], spec)?; + } + let build = if fields.len() == 6 { + validate_build(fields[2])?; + Some(fields[2].to_owned()) + } else { + None + }; + let tail = &fields[fields.len() - 3..]; + Ok((expanded_tags(tail[0], tail[1], tail[2])?, build)) +} + +pub fn matches(filename_value: &str, selected_tag: &str) -> bool { + filename(filename_value, None).is_ok_and(|(tags, _)| tags.contains(selected_tag)) +} + +fn validate_build(value: &str) -> Result<(), String> { + if !value.as_bytes().first().is_some_and(u8::is_ascii_digit) + || !value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') + { + return Err("invalid wheel build tag".into()); + } + Ok(()) +} + +fn validate_identity(name: &str, version: &str, spec: &LibSpec) -> Result<(), String> { + name.parse::<pep508_rs::PackageName>() + .map_err(|error| format!("invalid wheel distribution name: {error}"))?; + let identity: LibSpec = format!("pip:{name}@{version}").parse()?; + if identity.distribution_name() != spec.distribution_name() + || version + .parse::<Version>() + .map_err(|error| error.to_string())? + != spec + .version + .parse::<Version>() + .map_err(|error| error.to_string())? + { + return Err("wheel distribution identity differs from its lock".into()); + } + Ok(()) +} + +fn headers(bytes: &[u8]) -> Result<BTreeMap<String, Vec<String>>, String> { + if bytes.len() > 16 * 1024 * 1024 { + return Err("wheel metadata exceeds 16 MiB".into()); + } + let text = std::str::from_utf8(bytes).map_err(|_| "wheel metadata is not UTF-8")?; + let mut result = BTreeMap::<String, Vec<String>>::new(); + let mut previous: Option<String> = None; + for line in text.lines() { + if line.is_empty() { + break; + } + if line.starts_with([' ', '\t']) { + let key = previous + .as_ref() + .ok_or("wheel metadata starts with a folded line")?; + let value = result.get_mut(key).unwrap().last_mut().unwrap(); + value.push(' '); + value.push_str(line.trim()); + continue; + } + let (key, value) = line + .split_once(':') + .ok_or("invalid wheel metadata header")?; + if key.is_empty() + || !key + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-') + || value.contains('\0') + { + return Err("invalid wheel metadata header".into()); + } + let key = key.to_ascii_lowercase(); + result + .entry(key.clone()) + .or_default() + .push(value.trim().into()); + previous = Some(key); + } + Ok(result) +} + +fn optional_header<'a>( + headers: &'a BTreeMap<String, Vec<String>>, + key: &str, +) -> Result<Option<&'a str>, String> { + match headers.get(key) { + None => Ok(None), + Some(values) if values.len() == 1 => Ok(Some(values[0].as_str())), + _ => Err(format!("wheel metadata repeats {key}")), + } +} + +fn required_header<'a>( + headers: &'a BTreeMap<String, Vec<String>>, + key: &str, +) -> Result<&'a str, String> { + optional_header(headers, key)?.ok_or_else(|| format!("wheel metadata lacks {key}")) +} + +impl Wheel { + pub(crate) fn validate_dependencies( + &self, + lib: &crate::libs::LockedLib, + markers: &pep508_rs::MarkerEnvironment, + ) -> Result<(), String> { + let extras = lib + .spec + .name + .split_once('[') + .map(|(_, extras)| extras.trim_end_matches(']')) + .unwrap_or("") + .split(',') + .filter(|extra| !extra.is_empty()) + .map(str::parse::<pep508_rs::ExtraName>) + .collect::<Result<Vec<_>, _>>() + .map_err(|error| error.to_string())?; + let supplied_extras = self + .extras + .iter() + .map(|extra| extra.parse::<pep508_rs::ExtraName>()) + .collect::<Result<BTreeSet<_>, _>>() + .map_err(|error| error.to_string())?; + if extras.iter().any(|extra| !supplied_extras.contains(extra)) { + return Err("wheel does not provide a requested extra".into()); + } + let document = serde_json::json!({"info":{"requires_dist":self.requirements}}); + let expected = crate::libs::python_dependencies(&document, markers, &extras)?; + let mut names = BTreeSet::new(); + for (name, rule) in expected { + if name == lib.spec.name + && rule.contains( + &lib.spec + .version + .parse::<Version>() + .map_err(|error| error.to_string())?, + ) + { + continue; + } + let dependency = lib + .dependencies + .iter() + .find(|dependency| dependency.spec.name == name) + .ok_or("wheel dependency is missing from the exact lock")?; + if !rule.contains( + &dependency + .spec + .version + .parse::<Version>() + .map_err(|error| error.to_string())?, + ) { + return Err("locked wheel dependency does not satisfy METADATA".into()); + } + names.insert(name); + } + if lib + .dependencies + .iter() + .any(|dependency| !names.contains(&dependency.spec.name)) + { + return Err("wheel lock has an undeclared dependency".into()); + } + if let Some(rule) = &self.requires_python + && !rule + .parse::<pep440_rs::VersionSpecifiers>() + .map_err(|error| error.to_string())? + .contains(markers.python_full_version()) + { + return Err("wheel Requires-Python excludes the signed runtime".into()); + } + Ok(()) + } + pub fn parse( + spec: &LibSpec, + filename_value: Option<&str>, + runtime_tags: &[String], + bytes: &[u8], + ) -> Result<Self, String> { + validate_tags(runtime_tags)?; + let files = archive_files("whl", bytes)?; + let mut info = BTreeSet::new(); + let mut data = BTreeSet::new(); + for (path, _) in &files { + super::valid_path(path)?; + if let Some((root, _)) = path.split_once('/') { + if root.ends_with(".dist-info") { + info.insert(root); + } + if root.ends_with(".data") { + data.insert(root); + } + } + } + if info.len() != 1 { + return Err("wheel must contain exactly one dist-info directory".into()); + } + let info = info.into_iter().next().unwrap(); + let stem = info.strip_suffix(".dist-info").unwrap(); + let (name, version) = stem + .rsplit_once('-') + .ok_or("invalid wheel dist-info directory")?; + validate_identity(name, version, spec)?; + if data.iter().any(|root| *root != format!("{stem}.data")) { + return Err("wheel data directory differs from its distribution identity".into()); + } + let read = |name: &str| { + files + .iter() + .find(|(path, _)| path == &format!("{info}/{name}")) + .map(|(_, bytes)| bytes.as_slice()) + .ok_or_else(|| format!("wheel lacks {name}")) + }; + let metadata = headers(read("METADATA")?)?; + let metadata_version = required_header(&metadata, "metadata-version")?; + if metadata_version + .split_once('.') + .is_none_or(|(major, minor)| { + !matches!(major, "1" | "2") || minor.parse::<u32>().is_err() + }) + { + return Err("unsupported wheel core metadata version".into()); + } + validate_identity( + required_header(&metadata, "name")?, + required_header(&metadata, "version")?, + spec, + )?; + let wheel = headers(read("WHEEL")?)?; + let wheel_version = required_header(&wheel, "wheel-version")?; + if wheel_version + .split_once('.') + .is_none_or(|(major, minor)| major != "1" || minor.parse::<u32>().is_err()) + { + return Err("unsupported Wheel-Version".into()); + } + if !matches!( + required_header(&wheel, "root-is-purelib")?, + "true" | "false" + ) { + return Err("invalid wheel Root-Is-Purelib".into()); + } + let tags = wheel.get("tag").ok_or("wheel has no compatibility Tag")?; + validate_tags(tags)?; + if !tags.iter().any(|tag| runtime_tags.contains(tag)) { + return Err("wheel tags do not match the signed Python runtime".into()); + } + if tags.iter().all(|tag| tag.ends_with("-any")) + && files.iter().any(|(path, bytes)| { + bytes.starts_with(b"\x7fELF") + || path.ends_with(".pyd") + || path.ends_with(".dll") + || path.ends_with(".dylib") + }) + { + return Err("platform-independent wheel contains a native binary".into()); + } + let build = optional_header(&wheel, "build")?; + if let Some(build) = build { + validate_build(build)?; + } + if let Some(value) = filename_value { + let (file_tags, file_build) = filename(value, Some(spec))?; + if file_tags != tags.iter().cloned().collect() || file_build.as_deref() != build { + return Err("wheel filename tags/build differ from WHEEL metadata".into()); + } + } + validate_record(&files, info, read("RECORD")?)?; + let requirements = metadata.get("requires-dist").cloned().unwrap_or_default(); + for requirement in &requirements { + requirement + .parse::<Requirement>() + .map_err(|error| format!("invalid wheel Requires-Dist: {error}"))?; + } + let extras = metadata.get("provides-extra").cloned().unwrap_or_default(); + for extra in &extras { + extra + .parse::<pep508_rs::ExtraName>() + .map_err(|error| format!("invalid wheel Provides-Extra: {error}"))?; + } + let requires_python = optional_header(&metadata, "requires-python")?.map(str::to_owned); + if let Some(rule) = &requires_python { + rule.parse::<pep440_rs::VersionSpecifiers>() + .map_err(|error| error.to_string())?; + } + let stem = stem.to_owned(); + Ok(Self { + files, + stem, + requirements, + extras, + requires_python, + }) + } + + /// Install into the interpreter's real prefix so sys.prefix, sysconfig, + /// standalone scripts and the isolated runner agree on resource locations. + pub fn install( + self, + interpreter: &str, + python_version: &str, + ) -> Result<Vec<InstalledFile>, String> { + super::valid_path(interpreter)?; + let prefix = interpreter + .rsplit_once("/bin/") + .map(|(prefix, _)| prefix) + .ok_or("Python installation executable must live in its prefix/bin")?; + super::valid_path(prefix)?; + if python_version.split('.').count() != 2 + || python_version + .split('.') + .any(|part| part.parse::<u32>().is_err()) + { + return Err("Python installation requires a major.minor version".into()); + } + let library = format!("{prefix}/lib/python{python_version}/site-packages"); + let name = self.stem.rsplit_once('-').unwrap().0; + let data_prefix = format!("{}.data/", self.stem); + let mut installed = BTreeSet::new(); + let mut output = Vec::with_capacity(self.files.len()); + let entry_points = self + .files + .iter() + .find(|(path, _)| path == &format!("{}.dist-info/entry_points.txt", self.stem)) + .map(|(_, bytes)| scripts::entry_points(bytes, prefix, interpreter)) + .transpose()? + .unwrap_or_default(); + for (path, mut bytes) in self.files { + let (destination, executable) = if let Some(relative) = path.strip_prefix(&data_prefix) + { + let (scheme, relative) = relative + .split_once('/') + .ok_or("wheel data scheme has no file")?; + let destination = match scheme { + "purelib" | "platlib" => format!("{library}/{relative}"), + "headers" => { + format!("{prefix}/include/python{python_version}/{name}/{relative}") + } + "scripts" if !relative.contains('/') => format!("{prefix}/bin/{relative}"), + "data" => format!("{prefix}/{relative}"), + _ => return Err("invalid wheel data installation scheme".into()), + }; + if scheme == "scripts" + && (bytes.starts_with(b"#!python\n") + || bytes.starts_with(b"#!pythonw\n") + || bytes.starts_with(b"#!python\r\n") + || bytes.starts_with(b"#!pythonw\r\n")) + { + let newline = bytes.iter().position(|byte| *byte == b'\n').unwrap(); + bytes = [ + format!("#!/worker/{interpreter} -I\n").as_bytes(), + &bytes[newline + 1..], + ] + .concat(); + } + (destination, scheme == "scripts") + } else { + (format!("{library}/{path}"), false) + }; + super::valid_path(&destination)?; + if !installed.insert(destination.clone()) { + return Err("wheel installation schemes produce conflicting files".into()); + } + output.push(InstalledFile { + path: destination, + bytes, + executable, + }); + } + for file in entry_points { + if !installed.insert(file.path.clone()) { + return Err("wheel entry point conflicts with another installed script".into()); + } + output.push(file); + } + let record_path = format!("{library}/{}.dist-info/RECORD", self.stem); + let mut record = csv::WriterBuilder::new() + .has_headers(false) + .from_writer(Vec::new()); + for file in &output { + let base = library.split('/').collect::<Vec<_>>(); + let target = file.path.split('/').collect::<Vec<_>>(); + let common = base + .iter() + .zip(&target) + .take_while(|(left, right)| left == right) + .count(); + let relative = format!( + "{}{}", + "../".repeat(base.len() - common), + target[common..].join("/") + ); + let (hash, size) = if file.path == record_path { + (String::new(), String::new()) + } else { + ( + format!( + "sha256={}", + URL_SAFE_NO_PAD.encode(Sha256::digest(&file.bytes)) + ), + file.bytes.len().to_string(), + ) + }; + record + .write_record([relative, hash, size]) + .map_err(|error| error.to_string())?; + } + output + .iter_mut() + .find(|file| file.path == record_path) + .expect("validated RECORD") + .bytes = record.into_inner().map_err(|error| error.to_string())?; + Ok(output) + } +} + +fn validate_record(files: &[(String, Vec<u8>)], info: &str, bytes: &[u8]) -> Result<(), String> { + let entries = files + .iter() + .map(|(path, bytes)| (path.as_str(), bytes)) + .collect::<BTreeMap<_, _>>(); + let record_path = format!("{info}/RECORD"); + let mut seen = BTreeSet::new(); + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .flexible(false) + .from_reader(bytes); + for row in reader.records() { + let row = row.map_err(|error| format!("invalid wheel RECORD CSV: {error}"))?; + if row.len() != 3 { + return Err("wheel RECORD requires three fields".into()); + } + let path = &row[0]; + super::valid_path(path)?; + if !seen.insert(path.to_owned()) { + return Err("wheel RECORD repeats a file".into()); + } + let content = entries + .get(path) + .ok_or("wheel RECORD references an absent file")?; + if path == record_path { + if !row[1].is_empty() || !row[2].is_empty() { + return Err("wheel RECORD cannot hash itself".into()); + } + continue; + } + if path == format!("{info}/RECORD.jws") || path == format!("{info}/RECORD.p7s") { + return Err("wheel signatures must not be listed in RECORD".into()); + } + let (algorithm, expected) = row[1] + .split_once('=') + .ok_or("wheel RECORD lacks a secure hash")?; + let actual = match algorithm { + "sha256" => URL_SAFE_NO_PAD.encode(Sha256::digest(content)), + "sha384" => URL_SAFE_NO_PAD.encode(Sha384::digest(content)), + "sha512" => URL_SAFE_NO_PAD.encode(Sha512::digest(content)), + _ => return Err("wheel RECORD hash must use SHA-256 or stronger SHA-2".into()), + }; + if actual != expected || row[2].parse::<usize>().ok() != Some(content.len()) { + return Err("wheel RECORD hash or size does not match its file".into()); + } + } + if !seen.contains(&record_path) + || entries.keys().any(|path| { + !seen.contains(*path) + && **path != format!("{info}/RECORD.jws") + && **path != format!("{info}/RECORD.p7s") + }) + { + return Err("wheel RECORD does not cover every file".into()); + } + Ok(()) +} diff --git a/crates/dever-cli/src/workers/python_wheel/native.rs b/crates/dever-cli/src/workers/python_wheel/native.rs new file mode 100644 index 0000000..da47e09 --- /dev/null +++ b/crates/dever-cli/src/workers/python_wheel/native.rs @@ -0,0 +1,154 @@ +//! Validate extension ABI and the dynamic loader's closed, packaged search tree. + +use super::super::elf::{Closure, object, origin_directories}; +use goblin::elf::header; +use std::collections::{BTreeMap, BTreeSet}; + +/// `files` contains Worker-relative installation files and separately verified +/// OS ABI files at sandbox/lib/*. No host directory is searched or executed. +pub fn validate_native( + target: &str, + suffixes: &[String], + interpreter: &str, + files: &BTreeMap<String, &[u8]>, + wheel_files: &BTreeSet<String>, +) -> Result<(), String> { + let native = files + .iter() + .filter(|(path, _)| wheel_files.contains(*path)) + .filter(|(path, bytes)| { + bytes.starts_with(b"\x7fELF") + || path.ends_with(".so") + || path.contains(".so.") + || path.ends_with(".pyd") + || path.ends_with(".dll") + || path.ends_with(".dylib") + }) + .collect::<Vec<_>>(); + if native.is_empty() { + return Ok(()); + } + let machine = match target { + "linux-x86_64" => header::EM_X86_64, + "linux-aarch64" => header::EM_AARCH64, + _ => return Err("native Python wheel target is unsupported".into()), + }; + let closure = Closure { + files, + machine, + directories: vec!["sandbox/lib".into()], + }; + let interpreter_elf = object( + interpreter, + files + .get(interpreter) + .ok_or("native wheel requires its packaged Python executable")?, + machine, + )?; + let inherited = if interpreter_elf.runpaths.is_empty() { + origin_directories(interpreter, &interpreter_elf.rpaths)? + } else { + vec![] + }; + let (globals, _) = closure.follow(interpreter, vec![], &BTreeMap::new())?; + let mut entries = Vec::new(); + for (path, bytes) in native { + if path.ends_with(".pyd") || path.ends_with(".dll") || path.ends_with(".dylib") { + return Err("native Python wheel contains a binary for a different OS".into()); + } + let elf = object(path, bytes, machine)?; + if elf.header.e_type != header::ET_DYN || elf.interpreter.is_some() { + return Err("native wheel content must be a shared library, not an executable".into()); + } + let module = elf.dynsyms.iter().any(|symbol| { + symbol.st_shndx != 0 + && elf + .dynstrtab + .get_at(symbol.st_name) + .is_some_and(|name| name.starts_with("PyInit_")) + }); + let module = module || path.contains(".cpython-") || path.ends_with(".abi3.so"); + if module { + let compatible = suffixes.iter().any(|suffix| { + path.ends_with(suffix) + && (suffix != ".so" + || !path.contains(".cpython-") && !path.ends_with(".abi3.so")) + }); + if !compatible { + return Err( + "native Python extension suffix differs from the signed interpreter ABI".into(), + ); + } + } + entries.push((path, module)); + } + entries.sort_by_key(|(_, module)| !*module); + let mut reached = BTreeSet::new(); + for (path, module) in entries { + // Vendored helpers inherit the importing extension's loader context; + // reaching one does not imply an independent direct dlopen entry. + if !module && reached.contains(path) { + continue; + } + let (_, closure_paths) = closure.follow(path, inherited.clone(), &globals)?; + reached.extend(closure_paths); + } + Ok(()) +} + +/// Compiler processes use the signed sysroot's standard library locations. +/// Reuse the ELF version/SONAME closure checks used by installed extensions. +pub(crate) fn validate_build_tools( + target: &str, + executables: &[String], + files: &BTreeMap<String, &[u8]>, +) -> Result<(), String> { + let (machine, multiarch, interpreter) = match target { + "linux-x86_64" => ( + header::EM_X86_64, + "x86_64-linux-gnu", + "/lib64/ld-linux-x86-64.so.2", + ), + "linux-aarch64" => ( + header::EM_AARCH64, + "aarch64-linux-gnu", + "/lib/ld-linux-aarch64.so.1", + ), + _ => return Err("build tools target is unsupported".into()), + }; + let closure = Closure { + files, + machine, + directories: vec![ + format!("rootfs/usr/lib/{multiarch}"), + "sandbox/lib".into(), + "rootfs/usr/lib".into(), + ], + }; + for path in executables { + let bytes = files.get(path).ok_or("build executable is missing")?; + if bytes.starts_with(b"#!") { + let first = bytes + .split(|byte| *byte == b'\n') + .next() + .unwrap_or_default(); + if !matches!( + first, + b"#!/bin/sh" + | b"#!/usr/bin/sh" + | b"#!/usr/bin/env sh" + | b"#!/usr/bin/env node" + | b"#!/usr/bin/env python3" + ) { + return Err("build tool script must name the signed shell directly".into()); + } + continue; + } + let elf = object(path, bytes, machine)?; + if elf.interpreter.is_some_and(|value| value != interpreter) { + return Err("build tool interpreter differs from signed sandbox loader".into()); + } + closure.follow(path, vec![], &BTreeMap::new())?; + } + Ok(()) +} diff --git a/crates/dever-cli/src/workers/python_wheel/scripts.rs b/crates/dever-cli/src/workers/python_wheel/scripts.rs new file mode 100644 index 0000000..1508855 --- /dev/null +++ b/crates/dever-cli/src/workers/python_wheel/scripts.rs @@ -0,0 +1,88 @@ +//! Wheel console/gui entry points share the installed interpreter prefix. + +use super::InstalledFile; +use std::collections::{BTreeMap, BTreeSet}; + +fn identifier(path: &str) -> bool { + !path.is_empty() + && path.split('.').all(|part| { + let mut bytes = part.bytes(); + bytes + .next() + .is_some_and(|byte| byte.is_ascii_alphabetic() || byte == b'_') + && bytes.all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') + }) +} + +pub(super) fn entry_points( + bytes: &[u8], + prefix: &str, + interpreter: &str, +) -> Result<Vec<InstalledFile>, String> { + let text = std::str::from_utf8(bytes).map_err(|_| "wheel entry points must be UTF-8")?; + let mut active = false; + let mut sections = BTreeSet::new(); + let mut entries = BTreeMap::<String, String>::new(); + let mut previous: Option<String> = None; + for line in text.lines() { + let trimmed = line.trim(); + if trimmed.is_empty() || trimmed.starts_with(['#', ';']) { + continue; + } + if let Some(section) = trimmed + .strip_prefix('[') + .and_then(|value| value.strip_suffix(']')) + { + active = matches!(section, "console_scripts" | "gui_scripts"); + if active && !sections.insert(section.to_owned()) { + return Err("wheel repeats an entry-point group".into()); + } + previous = None; + continue; + } + if !active { + continue; + } + if line.starts_with(char::is_whitespace) && !trimmed.contains('=') { + let name = previous + .as_ref() + .ok_or("wheel entry point has an orphan continuation")?; + let value = entries.get_mut(name).expect("previous entry"); + value.push(' '); + value.push_str(trimmed); + continue; + } + let (name, value) = trimmed + .split_once('=') + .ok_or("invalid wheel script entry point")?; + let name = name.trim(); + if name.is_empty() + || !name + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"-_.".contains(&byte)) + || matches!(name, "." | "..") + { + return Err("wheel script name must be a portable file name".into()); + } + if entries + .insert(name.to_owned(), value.trim().to_owned()) + .is_some() + { + return Err("wheel repeats a script name".into()); + } + previous = Some(name.to_owned()); + } + entries.into_iter().map(|(name,value)| { + let value = if let Some((target,extras))=value.split_once('[') { + let extras=extras.strip_suffix(']').ok_or("invalid entry-point extras")?; + for extra in extras.split(',') { + extra.trim().parse::<pep508_rs::ExtraName>().map_err(|error|format!("invalid entry-point extra: {error}"))?; + } + target.trim() + }else{value.trim()}; + let (module,function)=value.split_once(':').ok_or("wheel console script must name a callable")?; + if !identifier(module) || !identifier(function) { return Err("wheel script has an invalid Python entry point".into()); } + let source=format!("#!/worker/{interpreter} -I\nimport importlib\nimport re\nimport sys\nentry = importlib.import_module({module:?})\nfor part in {function:?}.split('.'):\n entry = getattr(entry, part)\nsys.argv[0] = re.sub(r'(-script\\.pyw|\\.exe)?$', '', sys.argv[0])\nsys.exit(entry())\n"); + Ok(InstalledFile {path:format!("{prefix}/bin/{name}"),bytes:source.into_bytes(),executable:true}) + }).collect() +} diff --git a/crates/dever-cli/src/workers/runtime.rs b/crates/dever-cli/src/workers/runtime.rs new file mode 100644 index 0000000..a69bb94 --- /dev/null +++ b/crates/dever-cli/src/workers/runtime.rs @@ -0,0 +1,183 @@ +//! One runtime descriptor and file-closure contract for authoring and Worker preparation. + +use std::collections::BTreeSet; +use std::path::Path; + +use serde::Deserialize; + +use super::{MAX_TREE_BYTES, MAX_TREE_FILES, go_build, valid_path}; +use crate::libs::Ecosystem; + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct Manifest { + format: String, + ecosystem: String, + pub(crate) target: String, + pub(crate) executable: Option<String>, + #[serde(default)] + pub(super) arguments: Vec<String>, + pub(super) build: Option<go_build::GoBuildManifest>, + #[serde(default)] + pub(crate) python_wheel_tags: Vec<String>, + #[serde(default)] + pub(crate) python_extension_suffixes: Vec<String>, + pub(crate) python_markers: Option<pep508_rs::MarkerEnvironment>, +} + +impl Manifest { + pub(crate) fn executable(&self, path: &str) -> bool { + self.executable.as_deref() == Some(path) + || self + .build + .as_ref() + .is_some_and(|build| build.executable(path)) + } +} + +pub(crate) fn validate( + files: &[(String, Vec<u8>)], + ecosystem: &Ecosystem, + target: &str, +) -> Result<Manifest, String> { + if files.len() > MAX_TREE_FILES { + return Err("managed runtime pack has too many files".into()); + } + let mut paths = BTreeSet::new(); + let mut total = 0usize; + for (path, bytes) in files { + valid_path(path)?; + if !paths.insert(path.as_str()) { + return Err("managed runtime pack repeats a file".into()); + } + total = total + .checked_add(bytes.len()) + .ok_or("managed runtime pack size overflow")?; + if total > MAX_TREE_BYTES { + return Err("managed runtime pack exceeds 256 MiB".into()); + } + } + for path in &paths { + for parent in Path::new(path).ancestors().skip(1) { + if parent.to_str().is_some_and(|name| paths.contains(name)) { + return Err("managed runtime pack has a file/directory conflict".into()); + } + } + } + let metadata = files + .iter() + .find(|(path, _)| path == "dever-runtime.json") + .ok_or("managed runtime pack has no dever-runtime.json")?; + let manifest: Manifest = serde_json::from_slice(&metadata.1) + .map_err(|error| format!("invalid managed runtime manifest: {error}"))?; + if manifest.format != "dever-worker-runtime-v1" + || manifest.ecosystem != ecosystem.as_str() + || manifest.target != target + { + return Err("managed runtime pack identity differs from its lock and target".into()); + } + if ecosystem != &Ecosystem::Pip + && (!manifest.python_wheel_tags.is_empty() + || !manifest.python_extension_suffixes.is_empty() + || manifest.python_markers.is_some()) + { + return Err("non-Python runtime must not declare Python ABI metadata".into()); + } + if ecosystem == &Ecosystem::Go { + if manifest.executable.is_some() || !manifest.arguments.is_empty() { + return Err("managed Go build pack must not declare a runtime interpreter".into()); + } + manifest + .build + .as_ref() + .ok_or("managed Go Worker requires a locked compile/link build pack")? + .validate_files(files, target)?; + } else { + if manifest.build.is_some() { + return Err("managed interpreter pack must not declare Go build tools".into()); + } + let executable = manifest + .executable + .as_deref() + .ok_or("managed runtime executable is missing")?; + valid_path(executable)?; + if !executable.starts_with("bin/") + || manifest.arguments.len() > 64 + || manifest + .arguments + .iter() + .any(|argument| !argument.starts_with('-') || argument.contains('\0')) + { + return Err("managed runtime executable or fixed flags are invalid".into()); + } + if ecosystem == &Ecosystem::Pip && manifest.arguments != ["-I", "-S", "-B"] { + return Err( + "managed Python runtime requires the fixed -I -S -B isolation flags".into(), + ); + } + if ecosystem == &Ecosystem::Pip { + if manifest.python_markers.is_none() { + return Err("managed Python runtime lacks its marker environment".into()); + } + super::python_wheel::validate_target( + target, + &manifest.python_wheel_tags, + manifest + .python_markers + .as_ref() + .expect("validated Python markers"), + )?; + let mut suffixes = BTreeSet::new(); + if manifest.python_extension_suffixes.is_empty() + || manifest.python_extension_suffixes.len() > 16 + || manifest.python_extension_suffixes.iter().any(|suffix| { + !suffix.starts_with('.') + || !suffix.ends_with(".so") + || !suffix + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"._-".contains(&byte)) + || !suffixes.insert(suffix) + }) + { + return Err( + "managed Python extension suffixes are missing, invalid or repeated".into(), + ); + } + } + if !files + .iter() + .any(|(path, bytes)| path == executable && !bytes.is_empty()) + { + return Err("managed runtime executable is missing from its pack".into()); + } + for (path, bytes) in files { + if path == executable || bytes.starts_with(b"\x7fELF") { + validate_binary(path, bytes, target)?; + } + } + } + Ok(manifest) +} + +pub(crate) fn validate_binary(path: &str, bytes: &[u8], target: &str) -> Result<(), String> { + use goblin::elf::{Elf, header}; + let (machine, interpreter) = match target.parse::<crate::toolchain::BuildTarget>()? { + crate::toolchain::BuildTarget::LinuxX86_64 => { + (header::EM_X86_64, "/lib64/ld-linux-x86-64.so.2") + } + crate::toolchain::BuildTarget::LinuxAarch64 => { + (header::EM_AARCH64, "/lib/ld-linux-aarch64.so.1") + } + }; + let elf = + Elf::parse(bytes).map_err(|error| format!("invalid runtime ELF '{path}': {error}"))?; + if !elf.is_64 + || !elf.little_endian + || elf.header.e_machine != machine + || !matches!(elf.header.e_type, header::ET_DYN | header::ET_EXEC) + || elf.interpreter.is_some_and(|actual| actual != interpreter) + { + return Err(format!("runtime ELF '{path}' differs from target {target}")); + } + Ok(()) +} diff --git a/crates/dever-core/Cargo.toml b/crates/dever-core/Cargo.toml new file mode 100644 index 0000000..51cd126 --- /dev/null +++ b/crates/dever-core/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "dever-core" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +publish.workspace = true + +[lints] +workspace = true + +[dependencies] +dever-runtime = { workspace = true, features = ["database"] } +pulldown-cmark.workspace = true +sha2.workspace = true +serde_json.workspace = true + +[features] +reference = ["dever-runtime/crypto"] diff --git a/crates/dever-core/src/api.rs b/crates/dever-core/src/api.rs new file mode 100644 index 0000000..9c62a0d --- /dev/null +++ b/crates/dever-core/src/api.rs @@ -0,0 +1,304 @@ +use std::collections::BTreeSet; + +use crate::diagnostic::Diagnostic; +use crate::hir::{Atom, CallArgument, CallTarget, Domain, ExpressionKind, HandlerTarget, Program}; +use crate::specialize::{self, Specialization}; +use crate::types::{Field, Parameter, Shape, Type}; + +impl Program { + /// Application entries are known only for run/build; library checks retain unused exports. + pub fn application_errors(&self) -> Result<Vec<Diagnostic>, String> { + let roots = self + .api_routes + .iter() + .map(|route| route.function) + .chain(self.api_commands.iter().map(|command| command.function)) + .chain(self.jobs.iter().map(|job| job.function)) + .chain(self.auth.iter().map(|hook| hook.function)) + .chain(self.rest_binding_roots()) + .map(|function| Specialization { + function, + handlers: Vec::new(), + }); + let reachable = specialize::execution_reachable(&self.functions, roots) + .map_err(|_| "application contains a specialization cycle".to_owned())? + .iter() + .map(|instance| instance.function) + .collect::<BTreeSet<_>>(); + Ok(crate::contracts::application_errors(self, &reachable)) + } + + /// Application reachability cannot be inferred for a library with unknown consumers. + pub fn application_warnings(&self, entry: &str) -> Result<Vec<Diagnostic>, String> { + let entry_id = self.entry(entry)?; + let instances = specialize::reachable( + &self.functions, + [Specialization { + function: entry_id, + handlers: Vec::new(), + }], + ) + .map_err(|_| "application contains a specialization cycle".to_owned())?; + let reachable = instances + .iter() + .map(|instance| instance.function) + .collect::<BTreeSet<_>>(); + let mut required_exports = BTreeSet::from([entry_id]); + for id in reachable { + let function = &self.functions[id]; + crate::check::visit(function, |expression| { + let mut reference = |target: HandlerTarget| { + if let HandlerTarget::Function(target) = target { + let caller = &self.packages[function.owner]; + let callee = &self.packages[self.functions[target].owner]; + let internal_app_call = caller.role == Some(crate::source::SourceRole::App) + && callee.role == Some(crate::source::SourceRole::App) + && caller.domain.is_some() + && caller.domain == callee.domain; + if self.functions[target].owner != function.owner && !internal_app_call { + required_exports.insert(target); + } + } + }; + if let Some((target, arguments)) = expression.kind.static_call() { + match target { + CallTarget::Function(id) => reference(HandlerTarget::Function(id)), + CallTarget::Handler(handler) => reference(handler), + } + for argument in arguments { + if let CallArgument::Handler(handler) = argument { + reference(*handler); + } + } + } else if let ExpressionKind::Collection { + handler: Some(handler), + .. + } = &expression.kind + { + reference(*handler); + } + }); + } + let mut warnings = Vec::new(); + for (id, function) in self.functions.iter().enumerate() { + if !function.public + || self.packages[function.owner].bundled + || required_exports.contains(&id) + { + continue; + } + warnings.push(Diagnostic::error( + "W007", + format!("exported function '{}' is not used outside its source owner by application '{entry}'; its exposure may be unnecessary", function.name), + function.span, + )); + } + warnings.sort_by_key(|warning| (warning.primary.source, warning.primary.start)); + Ok(warnings) + } + + /// A stable, reviewable public contract. Source paths and internal IDs are excluded. + pub fn api_snapshot(&self) -> String { + let mut lines = BTreeSet::from(["dever-api 5".to_owned()]); + let mut exposed_types = BTreeSet::new(); + for (id, definition) in self.types.iter().enumerate() { + if definition.public && !self.packages[definition.owner].bundled { + exposed_types.insert(id); + } + } + for (id, function) in self.functions.iter().enumerate() { + if !function.public || self.packages[function.owner].bundled { + continue; + } + let parameters = function + .parameters + .iter() + .map(|parameter| self.api_parameter(parameter, &mut exposed_types)) + .collect::<Vec<_>>() + .join(", "); + for field in &function.outputs { + referenced_types(&field.ty, &mut exposed_types); + } + lines.insert(format!( + "{}function {}({parameters}) ({}) pure={} recover={:?} effects=[{}] errors=[{}] recoveries={:?}", + if function.kind == crate::syntax::FunctionKind::Transaction { + "transaction " + } else { + "" + }, + function.name, + self.api_fields(&function.outputs), + function.pure, + function.recovery, + self.effects[id] + .iter() + .copied() + .collect::<Vec<_>>() + .join(", "), + crate::contracts::failure_names(self, &self.failures[id]), + self.recoveries[id], + )); + } + let mut visited = BTreeSet::new(); + while let Some(id) = exposed_types.pop_first() { + if !visited.insert(id) { + continue; + } + let definition = &self.types[id]; + let shape = self.api_shape(&definition.shape, &mut exposed_types); + lines.insert(format!("type {} {shape}", definition.name)); + } + lines.extend(self.model_snapshot().lines().skip(1).map(str::to_owned)); + lines.extend(self.permissions.iter().map(|permission| { + format!( + "permission {} method={} component={} domain={} site={} action={}", + permission.key, + permission.method, + permission.component, + permission.domain, + permission.site, + permission.action, + ) + })); + lines.into_iter().collect::<Vec<_>>().join("\n") + "\n" + } + + /// Baselines are generated explicitly; checking never updates the approved contract. + pub fn check_api(&self, baseline: &str) -> Result<(), String> { + let normalize = |text: &str| { + text.lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .map(str::to_owned) + .collect::<BTreeSet<_>>() + }; + let expected = normalize(baseline); + if !expected.contains("dever-api 5") { + return Err("invalid API baseline: expected dever-api 5".into()); + } + let actual = normalize(&self.api_snapshot()); + if actual == expected { + return Ok(()); + } + let mut changes = vec!["public API differs from the approved baseline:".to_owned()]; + changes.extend(expected.difference(&actual).map(|line| format!("- {line}"))); + changes.extend(actual.difference(&expected).map(|line| format!("+ {line}"))); + Err(changes.join("\n")) + } + + fn api_parameter(&self, parameter: &Parameter, exposed_types: &mut BTreeSet<usize>) -> String { + match parameter { + Parameter::Value(ty) => { + referenced_types(ty, exposed_types); + ty.label(&self.types) + } + Parameter::Handler(signature) => { + for ty in &signature.parameters { + referenced_types(ty, exposed_types); + } + for field in &signature.outputs { + referenced_types(&field.ty, exposed_types); + } + let inputs = signature + .parameters + .iter() + .zip(&signature.bounds) + .map(|(ty, bounds)| format!("{} [{}]", ty.label(&self.types), domain(bounds))) + .collect::<Vec<_>>() + .join(", "); + format!("handler({inputs})({})", self.api_fields(&signature.outputs)) + } + } + } + + fn api_shape(&self, shape: &Shape, exposed_types: &mut BTreeSet<usize>) -> String { + match shape { + Shape::Record(fields) => { + let visible = fields + .iter() + .filter(|field| !field.private) + .cloned() + .collect::<Vec<_>>(); + for field in &visible { + referenced_types(&field.ty, exposed_types); + } + let construction = if fields.iter().any(|field| field.private) { + "private" + } else { + "public" + }; + format!( + "record construct={construction} {{{}}}", + self.api_fields(&visible) + ) + } + Shape::Choice(variants) => variants + .iter() + .map(|variant| { + for field in &variant.fields { + referenced_types(&field.ty, exposed_types); + } + format!( + "{}{}({})", + if variant.error { "error " } else { "" }, + variant.name, + self.api_fields(&variant.fields) + ) + }) + .collect::<Vec<_>>() + .join(" | "), + } + } + + fn api_fields(&self, fields: &[Field]) -> String { + fields + .iter() + .map(|field| { + format!( + "{}: {} [{}]", + field.name, + field.ty.label(&self.types), + domain(&field.bounds) + ) + }) + .collect::<Vec<_>>() + .join(", ") + } +} + +fn referenced_types(ty: &Type, references: &mut BTreeSet<usize>) { + match ty { + Type::Named(id) => { + references.insert(*id); + } + Type::Nullable(base) + | Type::List(base) + | Type::Stream(base) + | Type::AsyncStream(base) + | Type::RowStream(base) + | Type::Related(base) + | Type::Channel(base) => referenced_types(base, references), + Type::Map(key, value) | Type::MapEntry(key, value) => { + referenced_types(key, references); + referenced_types(value, references); + } + Type::Outputs(fields) | Type::Task(fields) => { + for field in fields { + referenced_types(&field.ty, references); + } + } + _ => {} + } +} + +fn domain(bounds: &Domain) -> String { + bounds + .iter() + .map(|atom| match atom { + Atom::Int(lower, upper) => format!("{lower}..{upper}"), + Atom::Decimal(lower, upper) => format!("{lower}..{upper}"), + _ => unreachable!("checked field bounds are numeric intervals"), + }) + .collect::<Vec<_>>() + .join(" or ") +} diff --git a/crates/dever-core/src/capture.rs b/crates/dever-core/src/capture.rs new file mode 100644 index 0000000..b626740 --- /dev/null +++ b/crates/dever-core/src/capture.rs @@ -0,0 +1,95 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use crate::hir::Failure; +use crate::types::{Definition, Field, Shape, Type}; + +/// A capture may return its own error variant or wrap an error-only choice. +/// Wrapping preserves the source error's nominal identity and payload. +#[derive(Clone, Copy)] +pub(crate) struct Target { + pub variant: usize, + pub wrapped: bool, +} + +pub(crate) fn targets( + types: &[Definition], + choice: usize, + actual: &BTreeSet<Failure>, +) -> Result<BTreeMap<Failure, Target>, &'static str> { + let Shape::Choice(variants) = &types[choice].shape else { + unreachable!("checked capture choice") + }; + let mut targets = BTreeMap::new(); + for (variant, definition) in variants.iter().enumerate().filter(|(_, value)| value.error) { + let nested = error_only_payload(types, &definition.fields); + let direct = Failure { + ty: choice, + variant, + }; + if nested.is_none() || actual.contains(&direct) { + targets.insert( + direct, + Target { + variant, + wrapped: false, + }, + ); + } + let Some((ty, count)) = nested else { + continue; + }; + // An explicitly raised outer variant remains nominal; it is not + // implicitly unwrapped merely because it carries another error. + if actual.contains(&direct) && !actual.iter().any(|failure| failure.ty == ty) { + continue; + } + for nested_variant in 0..count { + let failure = Failure { + ty, + variant: nested_variant, + }; + if targets + .insert( + failure, + Target { + variant, + wrapped: true, + }, + ) + .is_some() + { + return Err("result capture contains overlapping error wrappers"); + } + } + } + Ok(targets) +} + +fn error_only_payload(types: &[Definition], fields: &[Field]) -> Option<(usize, usize)> { + let [field] = fields else { + return None; + }; + let Type::Named(ty) = field.ty else { + return None; + }; + let Shape::Choice(variants) = &types[ty].shape else { + return None; + }; + variants + .iter() + .all(|variant| variant.error) + .then_some((ty, variants.len())) +} + +pub(crate) fn database_errors(types: &[Definition]) -> Vec<Failure> { + let ty = types + .iter() + .position(|definition| definition.name == "dever.database.Error") + .expect("bundled database error choice"); + let Shape::Choice(variants) = &types[ty].shape else { + unreachable!("bundled database error choice") + }; + (0..variants.len()) + .map(|variant| Failure { ty, variant }) + .collect() +} diff --git a/crates/dever-core/src/check.rs b/crates/dever-core/src/check.rs new file mode 100644 index 0000000..917ec9a --- /dev/null +++ b/crates/dever-core/src/check.rs @@ -0,0 +1,850 @@ +use std::collections::BTreeMap; + +use crate::diagnostic::Diagnostic; +use crate::hir::{Function, Program, TestCase}; +use crate::source::{SourceLayout, SourceMap, SourceRole}; +use crate::syntax::{Declaration, FunctionClause, Package}; +use crate::types::{Definition, DefinitionKind, Shape}; + +mod api; +mod auth; +mod body; +mod clauses; +mod collections; +mod components; +mod concurrency; +mod constants; +mod cycles; +mod expressions; +mod intrinsics; +mod jobs; +mod layout; +mod model; +mod names; +mod operators; +mod orm; +mod ports; +mod sql_read; +mod upload; +pub(crate) use ports::test_program; +mod symbols; +mod values; +pub(crate) use body::check_suspensions; +pub(crate) use cycles::{visit, visit_expression}; + +type Checked<T> = Result<T, Diagnostic>; + +struct Symbols { + functions: BTreeMap<(String, usize), usize>, + types: BTreeMap<String, usize>, + packages: Vec<PackageScope>, + strict_layout: bool, +} + +struct PackageScope { + name: String, + layout: SourceLayout, + bundled: bool, + implemented_ports: std::collections::BTreeSet<String>, + contract_types: std::collections::BTreeSet<String>, +} + +struct Context<'a> { + symbols: &'a Symbols, + types: &'a [Definition], + functions: &'a [Function], + models: &'a [crate::model::ModelSchema], + owner: usize, +} + +/// Checks every declaration, including bundled standard source and unused functions. +pub fn check(sources: &SourceMap) -> Result<Program, Vec<Diagnostic>> { + cache::check(sources, check_uncached) +} + +mod cache; + +fn check_uncached(sources: &SourceMap) -> Result<Program, Vec<Diagnostic>> { + check_configured(sources, None) +} + +/// Deployment bindings participate in checking before ambient identity types resolve. +/// They intentionally bypass the source-only cache; secrets are never retained in HIR. +pub fn check_with_settings( + sources: &SourceMap, + settings: &dever_runtime::config::Settings, +) -> Result<Program, Vec<Diagnostic>> { + check_with_bindings(sources, &settings.compilation_bindings()) +} + +pub fn check_with_bindings( + sources: &SourceMap, + bindings: &dever_runtime::config::CompilationBindings, +) -> Result<Program, Vec<Diagnostic>> { + check_configured(sources, Some(bindings)) +} + +fn check_configured( + sources: &SourceMap, + settings: Option<&dever_runtime::config::CompilationBindings>, +) -> Result<Program, Vec<Diagnostic>> { + let mut errors = layout::validate(sources); + if !errors.is_empty() { + return Err(sorted(errors)); + } + let packages: Vec<_> = sources + .all_files() + .filter_map(|source| match crate::parse(source) { + Ok(package) => Some(package), + Err(diagnostics) => { + errors.extend(diagnostics); + None + } + }) + .collect(); + if !errors.is_empty() { + return Err(sorted(errors)); + } + + let mut symbols = Symbols { + functions: BTreeMap::new(), + types: BTreeMap::new(), + packages: Vec::new(), + strict_layout: sources.strict_layout(), + }; + let mut types = Vec::new(); + let mut type_sources = Vec::new(); + let mut functions: Vec<Function> = Vec::new(); + let mut groups: Vec<Vec<&FunctionClause>> = Vec::new(); + for (owner, package) in packages.iter().enumerate() { + if let Err(error) = names::check(package) { + errors.push(error); + } + check_package(package, sources.strict_layout(), &mut errors); + let package_name = path_name(&package.name); + if symbols + .packages + .iter() + .any(|scope| scope.name == package_name) + { + errors.push(Diagnostic::error("C002", "duplicate package", package.span)); + } + let bundled = SourceMap::is_standard(package.span.source); + symbols.packages.push(PackageScope { + name: package_name.clone(), + layout: package.layout.clone(), + bundled, + implemented_ports: package + .declarations + .iter() + .filter_map(|declaration| { + let Declaration::Function(function) = declaration else { + return None; + }; + let (port, _) = function.name.text.rsplit_once('.')?; + Some(format!("{}.{}", package.layout.app_prefix()?, port)) + }) + .collect(), + contract_types: std::collections::BTreeSet::new(), + }); + let mut last_function = None; + for declaration in &package.declarations { + match declaration { + Declaration::Type(ty) => { + let qualified = declaration_name(package, &package_name, &ty.name.text); + let public = declaration_public(package, bundled, ty.public); + last_function = None; + if symbols + .types + .insert(qualified.clone(), types.len()) + .is_some() + { + errors.push(Diagnostic::error("C002", "duplicate type", ty.span)); + } + types.push(Definition { + name: qualified, + owner, + public, + shape: Shape::Record(Vec::new()), + span: ty.span, + properties: Default::default(), + kind: DefinitionKind::Regular, + }); + type_sources.push(Some(ty)); + } + Declaration::Function(clause) => { + let qualified = declaration_name(package, &package_name, &clause.name.text); + let public = declaration_public(package, bundled, clause.public); + let key = (qualified.clone(), clause.inputs.len()); + if let Some(id) = symbols.functions.get(&key).copied() { + if functions[id].owner != owner { + errors.push(Diagnostic::error( + "C002", + format!("duplicate App function '{qualified}'"), + clause.span, + )); + last_function = None; + continue; + } + if functions[id].public != public { + errors.push(Diagnostic::error( + "C006", + "all clauses of a function must use the same public visibility", + clause.span, + )); + } + if last_function != Some(id) { + errors.push(Diagnostic::error( + "C002", + "function clauses must be contiguous", + clause.span, + )); + } + groups[id].push(clause); + last_function = Some(id); + } else { + let id = functions.len(); + symbols.functions.insert(key, id); + functions.push(Function { + name: qualified, + owner, + public, + parameters: Vec::new(), + outputs: Vec::new(), + clauses: Vec::new(), + span: clause.span, + suspends: false, + pure: clause.pure, + recovery: clause.recovery.clone(), + kind: clause.kind, + port: None, + implementation: clause.name.text.contains('.'), + setting: None, + external: None, + }); + groups.push(vec![clause]); + last_function = Some(id); + } + } + Declaration::Database(_) + | Declaration::Schedule(_) + | Declaration::ModelIndex(_) + | Declaration::Relation(_) + | Declaration::Seed(_) + | Declaration::Migration(_) + | Declaration::ModelSql(_) + | Declaration::Api(_) + | Declaration::Rest(_) + | Declaration::External(_) => last_function = None, + } + } + } + let model_registrations = model::register( + &packages, + sources, + &mut symbols, + &mut types, + &mut type_sources, + &mut errors, + ); + if !errors.is_empty() { + return Err(sorted(errors)); + } + ports::expand_contract_types(&packages, &mut symbols, &types, &type_sources); + for (id, source) in type_sources.iter().enumerate() { + let Some(source) = source else { + continue; + }; + let context = Context { + symbols: &symbols, + types: &types, + functions: &functions, + models: &[], + owner: types[id].owner, + }; + match context.shape(&source.shape) { + Ok(shape) => types[id].shape = shape, + Err(error) => errors.push(error), + } + } + model::synthesize_fields(&model_registrations, &mut types, &mut errors); + if !errors.is_empty() { + return Err(sorted(errors)); + } + let models = model::schemas( + &model_registrations, + &packages, + &symbols, + &types, + &mut errors, + ); + let model_rest_sources = model::rest_sources(&models, &packages, &types); + if !errors.is_empty() { + return Err(sorted(errors)); + } + model::synthesize_relation_fields(&models, &mut types, &mut errors); + symbols::validate_types(&types, &mut errors); + if !errors.is_empty() { + return Err(sorted(errors)); + } + for (id, group) in groups.iter().enumerate() { + let context = Context { + symbols: &symbols, + types: &types, + functions: &functions, + models: &models, + owner: functions[id].owner, + }; + match clauses::signature(group, &context) { + Ok((parameters, outputs)) => { + functions[id].parameters = parameters; + functions[id].outputs = outputs; + } + Err(error) => errors.push(error), + } + } + if !errors.is_empty() { + return Err(sorted(errors)); + } + let mut adapters = ports::register( + &packages, + &groups, + &symbols, + &types, + &mut functions, + &mut errors, + ); + ports::settings( + &packages, + &types, + &mut functions, + &mut adapters, + &mut errors, + ); + let tests = test_cases(&packages, &functions, &mut errors); + let api::Entries { + routes: mut api_routes, + commands: api_commands, + rest: mut api_rest, + } = api::register( + &packages, + &symbols, + &functions, + &types, + &models, + &model_rest_sources, + &mut errors, + ); + upload::validate_signatures(&functions, &api_routes, &mut errors); + let jobs = jobs::register(&packages, &functions, &types, &mut errors); + let auth::Registration { + hooks: auth, + permissions, + } = auth::register( + settings, + &packages, + &functions, + &types, + &mut api_routes, + &mut api_rest, + &mut errors, + ); + validate_public_boundaries(&functions, &types, &mut errors); + validate_sensitive_boundaries(&packages, &functions, &types, &mut errors); + // Markdown documents the unified function interface, after clause patterns resolve to types. + for (owner, package) in packages.iter().enumerate() { + let source = sources.get(package.span.source); + if source.is_markdown() { + errors.extend(crate::markdown::validate_contract( + source, package, owner, &types, &functions, + )); + } + } + if !errors.is_empty() { + return Err(sorted(errors)); + } + let mut warnings = Vec::new(); + for (id, group) in groups.iter().enumerate() { + if functions[id].port.is_some() { + continue; + } + let context = Context { + symbols: &symbols, + types: &types, + functions: &functions, + models: &models, + owner: functions[id].owner, + }; + let result = (|| { + let patterns = clauses::check(group, &functions[id].parameters, &context)?; + group + .iter() + .zip(patterns) + .map(|(source, domains)| { + body::check(source, domains, &functions[id], &context, &mut warnings) + }) + .collect::<Checked<Vec<_>>>() + })(); + match result { + Ok(clauses) => functions[id].clauses = clauses, + Err(error) => errors.push(error), + } + } + if errors.is_empty() { + cycles::check(&functions, &types, &symbols, &mut errors); + } + if errors.is_empty() { + let mut program = Program { + types, + functions, + warnings: sorted(warnings), + effects: Vec::new(), + recoveries: Vec::new(), + suspension_handlers: Vec::new(), + failures: Vec::new(), + failure_handlers: Vec::new(), + database_effects: Vec::new(), + models, + jobs, + api_routes, + api_commands, + api_rest, + permissions, + tenant_components: Vec::new(), + tests, + adapters, + auth, + packages: packages + .iter() + .map(|package| crate::hir::PackageContract { + bundled: SourceMap::is_standard(package.span.source), + domain: package + .layout + .domain() + .map(|(component, domain)| (component.to_owned(), domain.to_owned())), + role: package.layout.role(), + }) + .collect(), + }; + // Test bodies are checked under their own fake bindings, never production effects. + let test_owners: std::collections::BTreeSet<_> = program + .tests + .iter() + .map(|test| program.functions[test.function].owner) + .collect(); + let test_bodies: Vec<_> = program + .functions + .iter_mut() + .enumerate() + .filter(|(_, function)| test_owners.contains(&function.owner)) + .map(|(id, function)| (id, std::mem::take(&mut function.clauses))) + .collect(); + crate::contracts::check(&mut program, &mut errors).map_err(|error| vec![error])?; + components::register(&mut program, &mut errors); + api::validate_rest_bindings(&program, &mut errors); + api::validate_get_effects(&program, &mut errors); + api::validate_write_transactions(&program, &mut errors); + api::validate_context_roots(&program, &mut errors); + api::validate_context_detachment(&program, &mut errors); + auth::validate(&program, settings.is_some(), &mut errors); + ports::validate_failures(&program, &mut errors); + validate_exported_failures(&program, &mut errors); + for (id, clauses) in test_bodies { + program.functions[id].clauses = clauses; + } + finalize_tests(&mut program, &mut errors); + program.warnings = sorted(program.warnings); + if errors.is_empty() { + Ok(program) + } else { + Err(sorted(errors)) + } + } else { + Err(sorted(errors)) + } +} + +fn declaration_name(package: &Package, physical: &str, name: &str) -> String { + match package.layout.role() { + Some(SourceRole::App) => format!("{}.{}", package.layout.app_prefix().unwrap(), name), + _ => format!("{physical}.{name}"), + } +} + +fn declaration_public(package: &Package, bundled: bool, explicit: bool) -> bool { + if bundled || matches!(&package.layout, SourceLayout::Main | SourceLayout::Loose) { + explicit + } else { + package.layout.role() == Some(SourceRole::App) + } +} + +fn test_cases( + packages: &[Package], + functions: &[Function], + errors: &mut Vec<Diagnostic>, +) -> Vec<TestCase> { + let mut tests = Vec::new(); + for (owner, package) in packages.iter().enumerate() { + let SourceLayout::Test { + component, + domain, + topic, + } = &package.layout + else { + continue; + }; + let entry_name = format!("{}.{}", path_name(&package.name), topic); + let matching = functions + .iter() + .enumerate() + .filter(|(_, function)| function.owner == owner && function.name == entry_name) + .collect::<Vec<_>>(); + let [(function, entry)] = matching.as_slice() else { + errors.push(Diagnostic::error( + "C005", + if matching.is_empty() { + format!("test file must declare the ordinary '{topic}() ()' entry") + } else { + format!("test entry '{topic}' cannot be overloaded") + }, + package.span, + )); + continue; + }; + if entry.kind != crate::syntax::FunctionKind::Ordinary + || !entry.parameters.is_empty() + || !entry.outputs.is_empty() + { + errors.push(Diagnostic::error( + "C005", + format!( + "test entry '{topic}' must be an ordinary zero-input, zero-output function" + ), + entry.span, + )); + continue; + } + tests.push(TestCase { + name: format!("{component}/{domain}/{topic}"), + function: *function, + span: entry.span, + database_connections: Vec::new(), + port_bindings: BTreeMap::new(), + }); + } + tests.sort_by(|left, right| left.name.cmp(&right.name)); + tests +} + +fn finalize_tests(program: &mut Program, errors: &mut Vec<Diagnostic>) { + ports::bind_tests(program); + if !errors.is_empty() { + return; + } + for index in 0..program.tests.len() { + let case = match ports::test_program(program, index) { + Ok(case) => case, + Err(case_errors) => { + errors.extend(case_errors); + continue; + } + }; + let function = program.tests[index].function; + let span = program.tests[index].span; + let root = crate::specialize::Specialization { + function, + handlers: Vec::new(), + }; + let instances = match crate::specialize::reachable(&case.functions, [root.clone()]) { + Ok(instances) => instances, + Err(cycle) => { + errors.push(Diagnostic::error( + "C007", + "recursive specialized test calls are not allowed", + cycle, + )); + continue; + } + }; + if instances.iter().any(|instance| { + matches!( + program.functions[instance.function].name.as_str(), + "dever.api.serve" | "dever.job.serve" + ) + }) { + errors.push(Diagnostic::error( + "C006", + "tests cannot start the application HTTP API or Job worker", + span, + )); + } + if crate::specialize::has_effect(&case, &root, "identity") { + errors.push(Diagnostic::error( + "C006", + "HTTP identity and site capabilities are not available from tests", + span, + )); + } + let models = crate::specialize::database_effects(&case, &root); + if models.is_empty() { + continue; + } + let mut connections = std::collections::BTreeSet::from(["default".to_owned()]); + for model in &program.models { + if let crate::model::ConnectionSelector::Explicit(name) = &model.connection { + connections.insert(name.clone()); + } + } + for job in &program.jobs { + if let crate::model::ConnectionSelector::Explicit(name) = &job.connection { + connections.insert(name.clone()); + } + } + program.tests[index].database_connections = connections.into_iter().collect(); + } +} + +fn check_package(package: &Package, strict_layout: bool, errors: &mut Vec<Diagnostic>) { + let name = path_name(&package.name); + if !SourceMap::is_standard(package.span.source) + && (package.name[0].text == "dever" + || matches!(name.as_str(), "text" | "math" | "decimal" | "float" | "int")) + { + errors.push(Diagnostic::error( + "C003", + "reserved standard package namespace", + package.name[0].span, + )); + } + match package.layout.role() { + None if strict_layout && matches!(&package.layout, SourceLayout::Main) => { + for declaration in &package.declarations { + if !matches!(declaration, Declaration::Function(function) + if function.name.text == "main" + && function.kind == crate::syntax::FunctionKind::Ordinary) + { + errors.push(Diagnostic::error( + "C005", + "main.dever may contain only the ordinary main() boot function", + crate::markdown::declaration_span(declaration), + )); + } + } + } + Some(SourceRole::Model) => { + for declaration in &package.declarations { + if let Declaration::Function(function) = declaration { + errors.push(Diagnostic::error( + "C005", + "model source declares storage only; move business behavior to app or domain", + function.span, + )); + } + } + } + Some(SourceRole::Port) => { + for declaration in &package.declarations { + if !matches!(declaration, Declaration::Type(_) | Declaration::Function(_)) { + errors.push(Diagnostic::error( + "C005", + "Port declares only types and bodyless contracts", + crate::markdown::declaration_span(declaration), + )); + } + } + } + Some(SourceRole::Adapter) => { + let externals = package + .declarations + .iter() + .filter(|declaration| matches!(declaration, Declaration::External(_))) + .count(); + if externals > 1 { + errors.push(Diagnostic::error( + "C002", + "Adapter source may declare only one external implementation", + package.span, + )); + } + if externals == 1 { + for declaration in &package.declarations { + let allowed = matches!(declaration, Declaration::External(_)) + || matches!(declaration, Declaration::Type(ty) if ty.setting); + if !allowed { + errors.push(Diagnostic::error( + "C005", + "external Adapter may contain only its external declaration and optional setting", + crate::markdown::declaration_span(declaration), + )); + } + } + } + } + None if matches!(package.layout, SourceLayout::Test { .. }) => { + for declaration in &package.declarations { + if let Declaration::Function(function) = declaration + && matches!( + function.name.text.as_str(), + "assert" | "assert_eq" | "secret" + ) + { + errors.push(Diagnostic::error( + "C002", + "assert, assert_eq and secret are reserved test primitives", + function.name.span, + )); + } + } + } + _ => {} + } +} + +fn validate_public_boundaries( + functions: &[Function], + types: &[Definition], + errors: &mut Vec<Diagnostic>, +) { + let mut check_type = |ty: &crate::types::Type, name: &str, span| { + symbols::referenced_types(ty, &mut |id| { + if !types[id].public { + errors.push(Diagnostic::error( + "C006", + format!( + "exported declaration '{name}' cannot expose private type '{}'", + types[id].name, + ), + span, + )); + } + }); + }; + for definition in types.iter().filter(|definition| definition.public) { + for field in symbols::shape_fields(&definition.shape) + .into_iter() + .filter(|field| !field.private) + { + check_type(&field.ty, &definition.name, definition.span); + } + } + for function in functions.iter().filter(|function| function.public) { + let mut check_type = |ty| check_type(ty, &function.name, function.span); + for parameter in &function.parameters { + match parameter { + crate::types::Parameter::Value(ty) => check_type(ty), + crate::types::Parameter::Handler(signature) => { + for ty in &signature.parameters { + check_type(ty); + } + for field in &signature.outputs { + check_type(&field.ty); + } + } + } + } + for output in &function.outputs { + check_type(&output.ty); + } + } +} + +fn validate_exported_failures(program: &Program, errors: &mut Vec<Diagnostic>) { + for (id, function) in program.functions.iter().enumerate() { + if program.packages[function.owner].role != Some(SourceRole::App) { + continue; + } + let private_types = program.failures[id] + .iter() + .map(|failure| failure.ty) + .filter(|ty| !program.types[*ty].public) + .collect::<std::collections::BTreeSet<_>>(); + for ty in private_types { + errors.push(Diagnostic::error( + "C006", + format!( + "exported function '{}' cannot expose private error type '{}'", + function.name, program.types[ty].name, + ), + function.span, + )); + } + } +} + +fn validate_sensitive_boundaries( + packages: &[crate::syntax::Package], + functions: &[Function], + types: &[Definition], + errors: &mut Vec<Diagnostic>, +) { + for definition in types { + if let crate::types::Shape::Choice(variants) = &definition.shape { + for variant in variants.iter().filter(|variant| variant.error) { + if variant + .fields + .iter() + .any(|field| !field.ty.observable(types)) + { + errors.push(Diagnostic::error( + "C006", + "error payload cannot contain Secret or private Model data", + definition.span, + )); + } + } + } + } + for function in functions { + let app = packages[function.owner].layout.role() == Some(SourceRole::App); + let entry = function.name == "main.main" + || (function.public && !crate::source::SourceMap::is_standard(function.span.source)); + if !app && !entry { + continue; + } + let sensitive_input = function.parameters.iter().any(|parameter| match parameter { + crate::types::Parameter::Value(ty) => ty.contains_private_model(types), + crate::types::Parameter::Handler(signature) => { + signature + .parameters + .iter() + .any(|ty| ty.contains_private_model(types)) + || signature + .outputs + .iter() + .any(|field| !field.ty.observable(types)) + } + }); + if sensitive_input + || function + .outputs + .iter() + .any(|field| !field.ty.observable(types)) + { + errors.push(Diagnostic::error( + "C006", + "App and entry boundaries cannot expose private Model data or output Secret; return an explicit View", + function.span, + )); + } + } +} + +fn path_name(path: &[crate::syntax::Name]) -> String { + path.iter() + .map(|name| name.text.as_str()) + .collect::<Vec<_>>() + .join(".") +} + +fn sorted(mut errors: Vec<Diagnostic>) -> Vec<Diagnostic> { + errors.sort_by_key(|error| { + ( + error.primary.source, + error.primary.start, + error.primary.end, + error.code, + ) + }); + errors +} diff --git a/crates/dever-core/src/check/api.rs b/crates/dever-core/src/check/api.rs new file mode 100644 index 0000000..0b7dbf2 --- /dev/null +++ b/crates/dever-core/src/check/api.rs @@ -0,0 +1,1005 @@ +use std::collections::BTreeMap; + +use crate::diagnostic::Diagnostic; +use crate::hir::{ + ApiCommand, ApiRest, ApiRoute, CallTarget, CollectionOp, ExpressionKind, Function, + HandlerTarget, Program, +}; +use crate::model::{ModelFieldType, RestSearch, RestValueSource, RestWriteField}; +use crate::source::{SourceLayout, SourceRole, Span}; +use crate::syntax::{ + ApiBinding, ApiKind, Declaration, FunctionClause, InputKind, Package, Pattern, +}; +use crate::types::{Definition, Parameter, Type}; +use crate::wire::{Policy, Schema}; + +pub(super) struct Entries { + pub routes: Vec<ApiRoute>, + pub commands: Vec<ApiCommand>, + pub rest: Vec<ApiRest>, +} + +pub(super) fn validate_get_effects(program: &Program, errors: &mut Vec<Diagnostic>) { + let writes: BTreeMap<_, _> = program + .functions + .iter() + .enumerate() + .filter_map(|(id, function)| { + let mut write = None; + super::visit(function, |expression| { + let writes_database = match &expression.kind { + ExpressionKind::ModelOperation { operation, .. } => operation.writes_database(), + ExpressionKind::JobEnqueue { .. } => true, + ExpressionKind::Intrinsic { operation, .. } => authorization_write(*operation), + _ => false, + }; + if writes_database { + write.get_or_insert(expression.span); + } + }); + write.map(|span| (id, span)) + }) + .collect(); + for route in program + .api_routes + .iter() + .filter(|route| route.method == "GET") + { + let root = crate::specialize::Specialization { + function: route.function, + handlers: Vec::new(), + }; + let Ok(reachable) = crate::specialize::reachable(&program.functions, [root]) else { + continue; + }; + if let Some(write) = reachable + .iter() + .find_map(|instance| writes.get(&instance.function)) + { + let mut error = Diagnostic::error( + "C013", + "GET App call chains cannot write database or authorization state or enqueue a Job", + program.functions[route.function].span, + ); + error.related.push(crate::diagnostic::Label { + span: *write, + message: "persistent write reachable from this GET binding".into(), + }); + errors.push(error); + } + } +} + +pub(super) fn validate_rest_bindings(program: &Program, errors: &mut Vec<Diagnostic>) { + for route in &program.api_rest { + let model_domain = program.models[route.model] + .package + .split('.') + .take(2) + .collect::<Vec<_>>() + .join("."); + for expression in route + .contract + .create + .iter() + .chain(&route.contract.replace) + .filter_map(|write| match &write.source { + RestValueSource::Binding { expression, .. } => Some(expression), + RestValueSource::Input => None, + }) + .chain( + route + .contract + .search + .iter() + .map(|search| &search.expression), + ) + { + super::visit_expression(expression, |value| match &value.kind { + ExpressionKind::Call { + target: CallTarget::Function(target), + .. + } => { + if !rest_function_allowed(program, *target, &model_domain) { + errors.push(Diagnostic::error("C013", "REST field bindings call only pure same-domain Domain functions or pure standard library functions", value.span)); + } + } + ExpressionKind::Call { .. } + | ExpressionKind::ModelOperation { .. } + | ExpressionKind::JobEnqueue { .. } + | ExpressionKind::RunCall { .. } + | ExpressionKind::AwaitTask(_) + | ExpressionKind::StopTask(_) + | ExpressionKind::Group(_) + | ExpressionKind::AwaitGroup(_) + | ExpressionKind::StopGroup(_) + | ExpressionKind::ParallelCall { .. } + | ExpressionKind::BlockingCall { .. } + | ExpressionKind::Channel { .. } + | ExpressionKind::ChannelSend { .. } + | ExpressionKind::ChannelReceive(_) + | ExpressionKind::ChannelClose(_) + | ExpressionKind::CaptureResult { .. } => { + errors.push(Diagnostic::error("C013", "REST field bindings cannot perform database, Job, handler, task, concurrency, or captured-failure operations", value.span)); + } + ExpressionKind::Intrinsic { operation, .. } + if !rest_intrinsic_allowed(*operation) => + { + errors.push(Diagnostic::error( + "C013", + "REST field binding uses a forbidden system capability", + value.span, + )); + } + ExpressionKind::Collection { + operation: CollectionOp::ParallelEach, + .. + } => { + errors.push(Diagnostic::error( + "C013", + "REST field bindings cannot start concurrent collection work", + value.span, + )); + } + ExpressionKind::Collection { + handler: Some(HandlerTarget::Function(target)), + .. + } if !rest_function_allowed(program, *target, &model_domain) => { + errors.push(Diagnostic::error("C013", "REST field binding collection handlers must be pure same-domain Domain functions or pure standard library functions", value.span)); + } + ExpressionKind::Collection { + handler: Some(HandlerTarget::Parameter(_)), + .. + } => { + errors.push(Diagnostic::error( + "C013", + "REST field bindings cannot use unconstrained handlers", + value.span, + )); + } + _ => {} + }); + } + } +} + +fn rest_function_allowed(program: &Program, target: usize, model_domain: &str) -> bool { + let function = &program.functions[target]; + let package = &program.packages[function.owner]; + let same_domain = package + .domain + .as_ref() + .is_some_and(|(component, domain)| format!("{component}.{domain}") == model_domain); + let context = package.bundled + && matches!( + function.name.as_str(), + "dever.api.request_id" + | "dever.api.method" + | "dever.api.path" + | "dever.api.client_address" + | "dever.api.header" + | "dever.api.cookie" + | "dever.api.secret_cookie" + | "dever.auth.id" + | "dever.auth.session" + | "dever.auth.user_id" + | "dever.auth.tenant_id" + | "dever.site.key" + ); + context + || (package.bundled && program.effects[target].is_empty()) + || (!package.bundled + && package.role == Some(SourceRole::Domain) + && same_domain + && function.pure + && program.effects[target].is_empty()) +} + +fn rest_intrinsic_allowed(operation: crate::intrinsic::Intrinsic) -> bool { + use crate::intrinsic::Intrinsic::*; + matches!( + operation, + CryptoPasswordVerify + | CryptoSha256 + | CryptoHmacSha256 + | CryptoConstantTimeEq + | TextTrim + | TextLower + | TextUpper + | TextCodepoint + | TextFromCodepoint + | TextAt + | TextSlice + | TextIndexOf + | TextContains + | TextStartsWith + | TextEndsWith + | TextSplit + | TextReplace + | IntParse + | IntToText + | DecimalParse + | DecimalToText + | DecimalFromInt + | DecimalRound + | FloatParse + | FloatToText + | FloatFromInt + | FloatSqrt + | FloatSin + | FloatCos + | FloatLog + | FloatPow + | FloatIsNan + | FloatIsFinite + | FloatIsInfinite + | IdFromText + | IdToText + | UuidParse + | UuidToText + | BytesFromText + | BytesToText + | BytesFromInts + | BytesLength + | BytesAt + | BytesSlice + | BytesConcat + | TimeParseDateTime + | TimeFormatDateTime + | TimeParseDate + | TimeFormatDate + | TimeParseTime + | TimeFormatTime + | TimeDuration + | TimeAdd + | TimeSubtract + | TimeDifference + | ApiRequestId + | ApiMethod + | ApiPath + | ApiClientAddress + | ApiHeader + | ApiCookie + | ApiSecretCookie + | AuthId + | AuthSession + | AuthUserId + | AuthTenantId + | SiteKey + ) +} + +pub(super) fn validate_write_transactions(program: &Program, errors: &mut Vec<Diagnostic>) { + let write_routes = program + .api_routes + .iter() + .filter(|route| route.method != "GET") + .map(|route| route.function) + .collect::<std::collections::BTreeSet<_>>(); + for function in &write_routes { + let root = crate::specialize::Specialization { + function: *function, + handlers: Vec::new(), + }; + let password_crypto = crate::specialize::uses_intrinsic( + program, + &root, + crate::intrinsic::Intrinsic::CryptoPasswordVerify, + ) || crate::specialize::uses_intrinsic( + program, + &root, + crate::intrinsic::Intrinsic::CryptoPasswordHash, + ); + if password_crypto + && let Some(span) = crate::specialize::write_outside_transaction(program, &root) + { + let mut error = Diagnostic::error( + "C014", + "password crypto API writes must be enclosed in an explicit transaction function", + program.functions[*function].span, + ); + error.related.push(crate::diagnostic::Label { + span, + message: "database write occurs outside the short transaction boundary".into(), + }); + errors.push(error); + } + } + let roots = write_routes + .into_iter() + .chain(program.api_commands.iter().map(|command| command.function)) + .chain(program.jobs.iter().map(|job| job.function)) + .collect::<std::collections::BTreeSet<_>>(); + for function in roots { + let root = crate::specialize::Specialization { + function, + handlers: Vec::new(), + }; + if crate::specialize::writes_database(program, &root) + && crate::specialize::starts_transaction_unsafe_work(program, &root) + { + errors.push(Diagnostic::error( + "C014", + "an API, CMD, or Job database transaction cannot start concurrent work", + program.functions[function].span, + )); + } + } +} + +pub(super) fn validate_context_roots(program: &Program, errors: &mut Vec<Diagnostic>) { + for route in program.api_routes.iter().filter(|route| route.anonymous) { + let root = crate::specialize::Specialization { + function: route.function, + handlers: Vec::new(), + }; + let enqueues = crate::specialize::reachable(&program.functions, [root.clone()]).is_ok_and( + |reachable| { + reachable.iter().any(|instance| { + let mut found = false; + super::visit(&program.functions[instance.function], |expression| { + found |= matches!(expression.kind, ExpressionKind::JobEnqueue { .. }); + }); + found + }) + }, + ); + if enqueues { + errors.push(Diagnostic::error( + "C006", + "public API call chains cannot enqueue a Job without a trusted user identity", + route.span, + )); + } + let administers_authorization = crate::specialize::reachable(&program.functions, [root]) + .is_ok_and(|reachable| { + reachable.iter().any(|instance| { + let mut found = false; + super::visit(&program.functions[instance.function], |expression| { + found |= matches!( + expression.kind, + ExpressionKind::Intrinsic { operation, .. } + if authorization_administration(operation) + ); + }); + found + }) + }); + if administers_authorization { + errors.push(Diagnostic::error( + "C006", + "public API call chains cannot administer authorization", + route.span, + )); + } + } + let roots = program + .api_commands + .iter() + .map(|command| command.function) + .chain(program.jobs.iter().map(|job| job.function)) + .collect::<std::collections::BTreeSet<_>>(); + for function in roots { + let root = crate::specialize::Specialization { + function, + handlers: Vec::new(), + }; + let request = crate::specialize::has_effect(program, &root, "request"); + let identity = crate::specialize::has_effect(program, &root, "identity"); + if request || identity { + errors.push(Diagnostic::error( + "C006", + if request { + "HTTP request capabilities are not available from CMD or Job entries" + } else { + "HTTP identity and site capabilities are not available from CMD or Job entries" + }, + program.functions[function].span, + )); + } + } +} + +fn authorization_write(operation: crate::intrinsic::Intrinsic) -> bool { + use crate::intrinsic::Intrinsic::*; + matches!( + operation, + AuthSaveRole | AuthGrantRole | AuthRevokeRole | AuthDisableRole + ) +} + +fn authorization_administration(operation: crate::intrinsic::Intrinsic) -> bool { + matches!(operation, crate::intrinsic::Intrinsic::AuthPermissions) + || authorization_write(operation) +} + +pub(super) fn validate_context_detachment(program: &Program, errors: &mut Vec<Diagnostic>) { + let roots = program + .api_routes + .iter() + .map(|route| route.function) + .chain(program.api_commands.iter().map(|command| command.function)) + .chain(program.jobs.iter().map(|job| job.function)) + .collect::<std::collections::BTreeSet<_>>(); + let Ok(instances) = crate::specialize::reachable( + &program.functions, + roots + .into_iter() + .map(|function| crate::specialize::Specialization { + function, + handlers: Vec::new(), + }), + ) else { + return; + }; + let mut detached = Vec::new(); + for instance in instances { + let bindings = + crate::specialize::handler_bindings(&program.functions[instance.function], &instance); + super::visit(&program.functions[instance.function], |expression| { + let target = match &expression.kind { + ExpressionKind::RunCall { target, .. } + | ExpressionKind::ParallelCall { target, .. } + | ExpressionKind::BlockingCall { target, .. } => Some(match target { + CallTarget::Function(function) => *function, + CallTarget::Handler(handler) => { + crate::specialize::resolve_handler(*handler, &bindings) + } + }), + ExpressionKind::Collection { + operation: CollectionOp::ParallelEach, + handler: Some(handler), + .. + } => Some(match handler { + HandlerTarget::Function(function) => *function, + HandlerTarget::Parameter(parameter) => bindings[parameter], + }), + _ => None, + }; + if target.is_some_and(|function| { + program.effects[function].contains("request") + || program.effects[function].contains("identity") + }) && !detached.contains(&expression.span) + { + detached.push(expression.span); + } + }); + } + for span in detached { + errors.push(Diagnostic::error( + "C006", + "HTTP request, identity, and site capabilities cannot be used inside detached or parallel work", + span, + )); + } +} + +pub(super) fn register( + packages: &[Package], + symbols: &super::Symbols, + functions: &[Function], + types: &[Definition], + models: &[crate::model::ModelSchema], + rest_sources: &[Vec<super::model::RestFieldSource>], + errors: &mut Vec<Diagnostic>, +) -> Entries { + let mut entries = Entries { + routes: legacy_routes(packages, functions, types, errors), + commands: Vec::new(), + rest: Vec::new(), + }; + let mut routes: BTreeMap<_, _> = entries + .routes + .iter() + .map(|route| { + ( + (route.method, route.path.clone()), + functions[route.function].span, + ) + }) + .collect(); + let mut commands = BTreeMap::new(); + for package in packages { + let SourceLayout::Role { + component, + domain, + role: SourceRole::Api, + topics, + } = &package.layout + else { + continue; + }; + let domain_name = format!("{component}.{domain}"); + let directory = topics[..topics.len().saturating_sub(1)].to_vec(); + let prefix = std::iter::once(component.as_str()) + .chain(std::iter::once(domain.as_str())) + .chain(topics.iter().map(String::as_str)) + .collect::<Vec<_>>() + .join("/"); + for declaration in &package.declarations { + match declaration { + Declaration::Api(binding) => { + let Some((id, inputs)) = binding_target( + binding, &domain_name, packages, functions, types, errors, + ) else { continue; }; + if let Some(method) = binding.kind.method() { + let path = format!("/{prefix}/{}", binding.action.text); + if occupy(&mut routes, (method, path.clone()), binding.span, "HTTP route", errors) { + entries.routes.push(ApiRoute { method, path, function: id, inputs, + directory: directory.clone(), anonymous: binding.anonymous, + component: component.clone(), domain: domain.clone(), + action: binding.action.text.clone(), auth: None, permission: None, + components: Vec::new(), + span: binding.span, + multipart: functions[id].parameters.iter().any(|parameter| parameter.value_type() == Some(&Type::Upload)) }); + } + } else { + let name = format!("{domain_name}.{}", binding.action.text); + if occupy(&mut commands, name.clone(), binding.span, "command", errors) { + entries.commands.push(ApiCommand { name, function: id, inputs, components: Vec::new() }); + } + } + } + Declaration::Rest(rest) => { + let selected = rest.model.as_ref() + .map(|selector| format!("{domain_name}.{}", super::path_name(selector))); + let candidates: Vec<_> = models.iter().enumerate() + .filter(|(_, model)| { + let model_package = &packages[types[model.record].owner]; + model_package.layout.domain() == package.layout.domain() + && selected.as_ref().is_none_or(|name| model.package == *name) + }) + .collect(); + let resolved = candidates.first().filter(|(_, schema)| { + candidates.len() == 1 && (selected.is_some() + || schema.package == format!("{domain_name}.model")) + }); + let Some((model, schema)) = resolved else { + errors.push(Diagnostic::error( + "C005", + "rest requires one same-domain Model; use 'rest model' or 'rest model.topic' to select it explicitly", + rest.span, + )); + continue; + }; + let mut contract = match schema.rest_contract(types) { + Ok(contract) => contract, + Err(reason) => { errors.push(Diagnostic::error("C014", reason, rest.span)); continue; } + }; + let checker_owner = packages.iter().enumerate().find_map(|(owner, candidate)| { + (candidate.layout.role() == Some(SourceRole::Domain) + && candidate.layout.domain() == package.layout.domain()).then_some(owner) + }).unwrap_or(types[schema.record].owner); + let context = super::Context { symbols, types, functions, models, owner: checker_owner }; + let mut warnings = Vec::new(); + for source in &rest_sources[*model] { + let field = &schema.fields[source.field]; + if (field.generated || field.owner.is_some()) + && (source.create.is_some() || source.replace.is_some() || source.search.is_some()) + { + errors.push(Diagnostic::error("C014", "generated and owner fields cannot declare REST bindings", source.span)); + continue; + } + let expected = &match &types[schema.record].shape { + crate::types::Shape::Record(fields) => &fields[source.field].ty, + _ => unreachable!(), + }; + for (expression, writes, label) in [ + (source.create.as_ref(), &mut contract.create, "create"), + (source.replace.as_ref(), &mut contract.replace, "replace"), + ] { + let Some(expression) = expression else { continue }; + match super::body::expression(expression, expected, expected, &context, &mut warnings) { + Ok(expression) => { + let uses_input = expression_uses_input(&expression); + if source.private && uses_input { + errors.push(Diagnostic::error("C014", format!("private REST {label} binding cannot use request input"), source.span)); + continue; + } + writes.retain(|write| write.field != source.field); + writes.push(RestWriteField { field: source.field, source: RestValueSource::Binding { expression, uses_input } }); + } + Err(error) => errors.push(error), + } + } + if let Some(expression) = &source.search { + let valid_type = matches!(field.ty, + ModelFieldType::Bool | ModelFieldType::Int | ModelFieldType::Float + | ModelFieldType::Decimal { .. } | ModelFieldType::Text { .. } + | ModelFieldType::Uuid | ModelFieldType::DateTime | ModelFieldType::Date + | ModelFieldType::Time | ModelFieldType::Duration + | ModelFieldType::ModelId(_)); + if matches!(field.name.as_str(), "page" | "size") { + errors.push(Diagnostic::error("C014", "REST search field cannot use the reserved page or size query name", source.span)); + } else if source.private || field.nullable || !valid_type { + errors.push(Diagnostic::error("C014", "REST search requires one public, non-null, indexable scalar field", source.span)); + } else if contract.search.is_some() { + errors.push(Diagnostic::error("C014", "REST supports at most one search field", source.span)); + } else { + match super::body::expression(expression, expected, expected, &context, &mut warnings) { + Ok(expression) if expression_uses_input(&expression) => { + contract.search = Some(RestSearch { field: source.field, expression }); + } + Ok(_) => errors.push(Diagnostic::error("C014", "REST search binding must use input", source.span)), + Err(error) => errors.push(error), + } + } + } + } + for (index, field) in schema.fields.iter().enumerate() { + if source_private(schema, types, index) + && !field.nullable && !field.generated && field.default.is_none() && field.owner.is_none() + && !contract.create.iter().any(|write| write.field == index) + { + errors.push(Diagnostic::error("C014", format!("REST Model '{}.{}' is private and required, but has no generated, default, owner, or create binding value", schema.package, field.name), rest.span)); + } + } + let path = format!("/{prefix}"); + let mut valid = true; + for (method, suffix) in [ + ("GET", ""), ("GET", "/{id}"), ("POST", ""), + ("PUT", "/{id}"), ("DELETE", "/{id}"), + ] { + valid &= occupy(&mut routes, (method, format!("{path}{suffix}")), rest.span, "HTTP route", errors); + } + if valid { + entries.rest.push(ApiRest { + path, + model: *model, + span: rest.span, + directory: directory.clone(), + component: component.clone(), + domain: domain.clone(), + auth: None, + permissions: Default::default(), + components: Vec::new(), + contract, + }); + } + } + _ => errors.push(Diagnostic::error("C005", "API role files accept only rest and get/post/put/delete/cmd App bindings; handler function bodies and legacy prefixes are not allowed", crate::markdown::declaration_span(declaration))), + } + } + } + entries + .routes + .sort_by(|left, right| (&left.path, left.method).cmp(&(&right.path, right.method))); + entries + .commands + .sort_by(|left, right| left.name.cmp(&right.name)); + entries + .rest + .sort_by(|left, right| left.path.cmp(&right.path)); + entries +} + +fn source_private(schema: &crate::model::ModelSchema, types: &[Definition], index: usize) -> bool { + let crate::types::Shape::Record(fields) = &types[schema.record].shape else { + unreachable!() + }; + fields[index].private +} + +fn expression_uses_input(expression: &crate::hir::Expression) -> bool { + let mut uses = false; + super::visit_expression(expression, |value| { + uses |= matches!(value.kind, ExpressionKind::Local(0)) + }); + uses +} + +fn occupy<K: Ord>( + occupied: &mut BTreeMap<K, Span>, + key: K, + span: Span, + kind: &str, + errors: &mut Vec<Diagnostic>, +) -> bool { + if let Some(previous) = occupied.insert(key, span) { + let mut error = Diagnostic::error("C002", format!("duplicate {kind}"), span); + error.related.push(crate::diagnostic::Label { + span: previous, + message: format!("first {kind} declaration"), + }); + errors.push(error); + return false; + } + true +} + +fn binding_target( + binding: &ApiBinding, + domain: &str, + packages: &[Package], + functions: &[Function], + types: &[Definition], + errors: &mut Vec<Diagnostic>, +) -> Option<(usize, Vec<String>)> { + let target = match binding.target.as_slice() { + [app, target] if app.text == "app" => target, + _ => { + errors.push(Diagnostic::error( + "C006", + "API bindings must target app.<function> in the same domain", + binding.span, + )); + return None; + } + }; + let name = format!("{domain}.{}", target.text); + let matches: Vec<_> = functions + .iter() + .enumerate() + .filter(|(_, function)| { + function.name == name && packages[function.owner].layout.role() == Some(SourceRole::App) + }) + .collect(); + let [(id, function)] = matches.as_slice() else { + errors.push(Diagnostic::error( + "C004", + "API binding requires one unambiguous same-domain App function", + binding.span, + )); + return None; + }; + if function.outputs.len() != 1 { + errors.push(Diagnostic::error( + "C005", + "API/CMD bindings require exactly one JSON-serializable App output", + binding.span, + )); + return None; + } + if let Err(reason) = Schema::from_field(&function.outputs[0], types, Policy::Output) { + errors.push(Diagnostic::error( + "C005", + format!("API/CMD output is not JSON-serializable: {reason}"), + binding.span, + )); + } + let clauses: Vec<&FunctionClause> = packages[function.owner] + .declarations + .iter() + .filter_map(|declaration| match declaration { + Declaration::Function(clause) if clause.name.text == target.text => Some(clause), + _ => None, + }) + .collect(); + let inputs: Vec<_> = clauses[0] + .inputs + .iter() + .map(|input| input.name.text.clone()) + .collect(); + for clause in &clauses { + if clause + .inputs + .iter() + .map(|input| &input.name.text) + .ne(inputs.iter()) + { + errors.push(Diagnostic::error( + "C005", + "API-bound App clauses must use identical input names", + clause.span, + )); + } + } + for parameter in &function.parameters { + let Parameter::Value(ty) = parameter else { + errors.push(Diagnostic::error( + "C005", + "API/CMD inputs cannot be handler parameters", + binding.span, + )); + continue; + }; + if ty == &Type::Upload { + if binding.kind != ApiKind::Post { + errors.push(Diagnostic::error( + "C005", + "Upload is allowed only as a direct POST input", + binding.span, + )); + } + continue; + } + if function + .parameters + .iter() + .any(|parameter| parameter.value_type() == Some(&Type::Upload)) + && !matches!( + ty.base(), + Type::Text + | Type::Int + | Type::Bool + | Type::Float + | Type::Decimal + | Type::Uuid + | Type::DateTime + | Type::Date + | Type::Time + | Type::Duration + ) + { + errors.push(Diagnostic::error( + "C005", + "multipart inputs must be Upload or wire-safe scalar values", + binding.span, + )); + } + match Schema::build(ty, types, Policy::ApiInput) { + Err(reason) => errors.push(Diagnostic::error( + "C005", + format!("API/CMD input is not JSON-decodable: {reason}"), + binding.span, + )), + Ok(schema) if matches!(binding.kind, ApiKind::Get | ApiKind::Delete) => { + let mut node = &schema.nodes[schema.root]; + if let crate::wire::Node::Nullable(inner) = node { + node = &schema.nodes[*inner]; + } + if matches!( + node, + crate::wire::Node::Record { .. } + | crate::wire::Node::List(_) + | crate::wire::Node::Json + | crate::wire::Node::Nullable(_) + ) { + errors.push(Diagnostic::error( + "C005", + "GET/DELETE inputs must be wire-safe scalars or nullable scalars", + binding.span, + )); + } + } + Ok(_) => {} + } + } + Some((*id, inputs)) +} + +fn legacy_routes( + packages: &[Package], + functions: &[Function], + types: &[Definition], + errors: &mut Vec<Diagnostic>, +) -> Vec<ApiRoute> { + let mut routes = Vec::new(); + let mut occupied = BTreeMap::new(); + for (owner, package) in packages.iter().enumerate() { + if crate::source::SourceMap::is_standard(package.span.source) { + continue; + } + let segments: Vec<_> = package.name.iter().map(|part| part.text.as_str()).collect(); + let mut prefix: Vec<_> = match &package.layout { + SourceLayout::Loose => { + let Some(index) = segments.iter().position(|part| *part == "api") else { + continue; + }; + if index == 0 || index + 1 == segments.len() { + continue; + } + segments[..index] + .iter() + .chain(segments[index + 1..].iter()) + .copied() + .collect() + } + _ => continue, + }; + for declaration in &package.declarations { + let Declaration::Function(clause) = declaration else { + errors.push(Diagnostic::error( + "C005", + "api/ files may contain only HTTP handler functions", + crate::markdown::declaration_span(declaration), + )); + continue; + }; + let (method, action) = if let Some(action) = clause.name.text.strip_prefix("get_") { + ("GET", action) + } else if let Some(action) = clause.name.text.strip_prefix("post_") { + ("POST", action) + } else if let Some(action) = clause.name.text.strip_prefix("delete_") { + ("DELETE", action) + } else { + errors.push(Diagnostic::error( + "C005", + "API handler names must start with get_, post_ or delete_", + clause.name.span, + )); + continue; + }; + if action.is_empty() + || clause.public + || clause.kind != crate::syntax::FunctionKind::Ordinary + { + errors.push(Diagnostic::error( + "C005", + "API handlers must be private ordinary functions with an action after the HTTP prefix", + clause.span, + )); + continue; + } + let qualified = format!("{}.{}", segments.join("."), clause.name.text); + let Some((id, function)) = functions.iter().enumerate().find(|(_, function)| { + function.owner == owner + && function.name == qualified + && function.parameters.len() == clause.inputs.len() + }) else { + continue; + }; + prefix.push(action); + let path = format!("/{}", prefix.join("/")); + prefix.pop(); + if let Some(previous) = occupied.insert((method, path.clone()), clause.span) { + let mut error = Diagnostic::error("C002", "duplicate HTTP route", clause.span); + error.related.push(crate::diagnostic::Label { + span: previous, + message: "first handler for this method and path".into(), + }); + errors.push(error); + continue; + } + if function.outputs.len() != 1 || function.outputs[0].name != "response" { + errors.push(Diagnostic::error( + "C005", + "API handlers must have exactly one output named response", + clause.span, + )); + continue; + } + let mut inputs = Vec::new(); + for (index, input) in clause.inputs.iter().enumerate() { + let supported = matches!(&input.kind, + InputKind::Value(Pattern::Typed { bounds, .. }) if bounds.is_empty()) + && function.parameters.get(index).is_some_and( + |parameter| matches!(parameter, Parameter::Value(ty) if input_type(ty)), + ); + if !supported { + errors.push(Diagnostic::error( + "C005", + "API inputs must be plain Text, Int, Bool or nullable variants", + input.span, + )); + } + inputs.push(input.name.text.clone()); + } + if let Err(reason) = Schema::from_field(&function.outputs[0], types, Policy::Output) { + errors.push(Diagnostic::error( + "C005", + format!( + "API response must be JSON-serializable without private fields: {reason}" + ), + clause.span, + )); + } + routes.push(ApiRoute { + method, + path, + function: id, + inputs, + directory: Vec::new(), + anonymous: false, + span: clause.span, + component: function.name.split('.').next().unwrap_or("app").to_owned(), + domain: function.name.split('.').nth(1).unwrap_or("main").to_owned(), + action: action.to_owned(), + auth: None, + permission: None, + components: Vec::new(), + multipart: false, + }); + } + } + routes.sort_by(|left, right| (&left.path, left.method).cmp(&(&right.path, right.method))); + routes +} + +fn input_type(ty: &Type) -> bool { + match ty { + Type::Nullable(inner) => matches!(inner.as_ref(), Type::Text | Type::Int | Type::Bool), + Type::Text | Type::Int | Type::Bool => true, + _ => false, + } +} diff --git a/crates/dever-core/src/check/auth.rs b/crates/dever-core/src/check/auth.rs new file mode 100644 index 0000000..9b135c1 --- /dev/null +++ b/crates/dever-core/src/check/auth.rs @@ -0,0 +1,554 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use crate::diagnostic::Diagnostic; +use crate::hir::{ + ApiRest, ApiRoute, AuthHook, Expression, ExpressionKind, Function, Permission, Program, + RestPermissions, +}; +use crate::source::{SourceRole, Span}; +use crate::syntax::{FunctionKind, Package}; +use crate::types::{Definition, DefinitionKind, Parameter, Shape, Type}; + +pub(super) struct Registration { + pub hooks: Vec<AuthHook>, + pub permissions: Vec<Permission>, +} + +struct SiteBinding { + key: String, + provider: String, +} + +pub(super) fn register( + settings: Option<&dever_runtime::config::CompilationBindings>, + packages: &[Package], + functions: &[Function], + types: &[Definition], + routes: &mut [ApiRoute], + rest: &mut [ApiRest], + errors: &mut Vec<Diagnostic>, +) -> Registration { + let Some(settings) = settings else { + return Registration { + hooks: Vec::new(), + permissions: Vec::new(), + }; + }; + if routes.is_empty() && rest.is_empty() { + return Registration { + hooks: Vec::new(), + permissions: Vec::new(), + }; + } + + let claims = types + .iter() + .position(|definition| definition.name == "dever.auth.Claims") + .expect("official dever.auth.Claims type"); + let mut route_providers = Vec::with_capacity(routes.len()); + let mut rest_providers = Vec::with_capacity(rest.len()); + let mut providers = BTreeSet::new(); + + for route in routes.iter() { + let binding = site_binding(settings, &route.directory, route.span, errors); + if let Some(binding) = &binding { + providers.insert(binding.provider.clone()); + } + route_providers.push(binding); + } + for declaration in rest.iter() { + let binding = site_binding(settings, &declaration.directory, declaration.span, errors); + if let Some(binding) = &binding { + providers.insert(binding.provider.clone()); + } + rest_providers.push(binding); + } + + let mut hooks = Vec::new(); + let mut hook_by_provider = BTreeMap::new(); + for provider_key in providers { + let Some(verify) = settings.providers.get(&provider_key) else { + errors.push(Diagnostic::error( + "C005", + format!("site authentication provider '{provider_key}' is not configured"), + first_provider_span( + &provider_key, + &route_providers, + routes, + &rest_providers, + rest, + ), + )); + continue; + }; + let span = first_provider_span( + &provider_key, + &route_providers, + routes, + &rest_providers, + rest, + ); + let Some((function, identity)) = + verify_contract(verify, claims, packages, functions, types, span, errors) + else { + continue; + }; + let hook = hooks.len(); + hooks.push(AuthHook { + provider: provider_key.clone(), + function, + claims, + identity, + }); + hook_by_provider.insert(provider_key, hook); + } + + let mut permissions = Vec::new(); + let mut occupied = BTreeMap::new(); + for (route, binding) in routes.iter_mut().zip(&route_providers) { + route.auth = binding + .as_ref() + .and_then(|binding| hook_by_provider.get(&binding.provider).copied()); + if !route.anonymous { + route.permission = binding.as_ref().and_then(|binding| { + register_permission( + &mut permissions, + &mut occupied, + Permission { + key: permission_key( + &route.component, + &route.domain, + &binding.key, + &route.action, + ), + component: route.component.clone(), + domain: route.domain.clone(), + site: binding.key.clone(), + action: route.action.clone(), + method: route.method, + }, + route.span, + errors, + ) + }); + } + } + for (declaration, binding) in rest.iter_mut().zip(&rest_providers) { + declaration.auth = binding + .as_ref() + .and_then(|binding| hook_by_provider.get(&binding.provider).copied()); + let Some(binding) = binding else { continue }; + let mut permission = |action, method| { + register_permission( + &mut permissions, + &mut occupied, + Permission { + key: permission_key( + &declaration.component, + &declaration.domain, + &binding.key, + action, + ), + component: declaration.component.clone(), + domain: declaration.domain.clone(), + site: binding.key.clone(), + action: action.to_owned(), + method, + }, + declaration.span, + errors, + ) + }; + declaration.permissions = RestPermissions { + read: permission("read", "GET"), + create: permission("create", "POST"), + replace: permission("replace", "PUT"), + delete: permission("delete", "DELETE"), + }; + } + Registration { hooks, permissions } +} + +fn site_binding( + settings: &dever_runtime::config::CompilationBindings, + directory: &[String], + span: Span, + errors: &mut Vec<Diagnostic>, +) -> Option<SiteBinding> { + let directory = directory.iter().map(String::as_str).collect::<Vec<_>>(); + match settings.site_for_directory(&directory) { + Ok(Some((key, site))) => Some(SiteBinding { + key: key.to_owned(), + provider: site.auth.clone(), + }), + Ok(None) => { + errors.push(Diagnostic::error( + "C005", + "HTTP API source directory is not assigned to a configured site", + span, + )); + None + } + Err(message) => { + errors.push(Diagnostic::error("C005", message, span)); + None + } + } +} + +fn permission_key(component: &str, domain: &str, site: &str, action: &str) -> String { + format!("{component}.{domain}.{site}.{action}") +} + +fn register_permission( + permissions: &mut Vec<Permission>, + occupied: &mut BTreeMap<String, Span>, + permission: Permission, + span: Span, + errors: &mut Vec<Diagnostic>, +) -> Option<usize> { + if let Some(previous) = occupied.insert(permission.key.clone(), span) { + let mut error = Diagnostic::error( + "C002", + format!("duplicate API permission '{}'", permission.key), + span, + ); + error.related.push(crate::diagnostic::Label { + span: previous, + message: "first declaration of this permission".into(), + }); + errors.push(error); + return None; + } + let id = permissions.len(); + permissions.push(permission); + Some(id) +} + +fn first_provider_span( + provider: &str, + route_providers: &[Option<SiteBinding>], + routes: &[ApiRoute], + rest_providers: &[Option<SiteBinding>], + rest: &[ApiRest], +) -> Span { + route_providers + .iter() + .zip(routes) + .find_map(|(candidate, route)| { + (candidate.as_ref().map(|binding| binding.provider.as_str()) == Some(provider)) + .then_some(route.span) + }) + .or_else(|| { + rest_providers + .iter() + .zip(rest) + .find_map(|(candidate, declaration)| { + (candidate.as_ref().map(|binding| binding.provider.as_str()) == Some(provider)) + .then_some(declaration.span) + }) + }) + .expect("provider originated from an API declaration") +} + +fn verify_contract( + name: &str, + claims: usize, + packages: &[Package], + functions: &[Function], + types: &[Definition], + span: Span, + errors: &mut Vec<Diagnostic>, +) -> Option<(usize, usize)> { + let candidates = functions + .iter() + .enumerate() + .filter(|(_, function)| function.name == name) + .collect::<Vec<_>>(); + let [(function_id, function)] = candidates.as_slice() else { + errors.push(Diagnostic::error( + "C005", + if candidates.is_empty() { + format!("authentication verify App '{name}' does not exist") + } else { + format!("authentication verify App '{name}' is overloaded") + }, + span, + )); + return None; + }; + if packages[function.owner].layout.role() != Some(SourceRole::App) + || !function.public + || function.kind != FunctionKind::Ordinary + || function.parameters.as_slice() != [Parameter::Value(Type::Named(claims))] + || function.outputs.len() != 1 + { + errors.push(Diagnostic::error( + "C005", + "authentication verify must be one ordinary App function with one dever.auth.Claims input and one identity record output", + function.span, + )); + return None; + } + let Type::Named(identity) = function.outputs[0].ty else { + errors.push(Diagnostic::error( + "C005", + "authentication verify output must be an App-owned identity record", + function.span, + )); + return None; + }; + if types[identity].owner != function.owner + || !types[identity].public + || types[identity].kind != DefinitionKind::Regular + || !identity_fields(&types[identity], types) + { + errors.push(Diagnostic::error( + "C005", + "authentication identity must be an App-owned public record with id: Text, user_id: ModelId?, and tenant_id: ModelId?", + types[identity].span, + )); + return None; + } + Some((*function_id, identity)) +} + +fn identity_fields(identity: &Definition, types: &[Definition]) -> bool { + let Shape::Record(fields) = &identity.shape else { + return false; + }; + let [id, user_id, tenant_id] = fields.as_slice() else { + return false; + }; + fields + .iter() + .all(|field| !field.private && field.bounds.is_empty()) + && id.name == "id" + && id.ty == Type::Text + && user_id.name == "user_id" + && optional_model_id(&user_id.ty, types) + && tenant_id.name == "tenant_id" + && optional_model_id(&tenant_id.ty, types) +} + +fn optional_model_id(ty: &Type, types: &[Definition]) -> bool { + matches!(ty, Type::Nullable(inner) if matches!(inner.as_ref(), Type::Named(id) if types[*id].kind == DefinitionKind::ModelId)) +} + +pub(super) fn validate(program: &Program, configured: bool, errors: &mut Vec<Diagnostic>) { + validate_identity_types(program, configured, errors); + for hook in &program.auth { + let root = crate::specialize::Specialization { + function: hook.function, + handlers: Vec::new(), + }; + let Ok(reachable) = crate::specialize::reachable(&program.functions, [root]) else { + continue; + }; + let mut forbidden = None; + let mut tenant_model = None; + for instance in reachable { + crate::check::visit(&program.functions[instance.function], |expression| { + let writes = match &expression.kind { + ExpressionKind::ModelOperation { operation, .. } => operation.writes_database(), + ExpressionKind::JobEnqueue { .. } => true, + _ => false, + }; + if writes && forbidden.is_none() { + forbidden = Some(( + expression.span, + "authentication verify cannot write to a database or enqueue a Job", + )); + } + if let ExpressionKind::ModelOperation { model, .. } = &expression.kind + && program.models[*model].scope == crate::model::ModelScope::Tenant + && tenant_model.is_none() + { + tenant_model = Some((expression.span, program.models[*model].package.clone())); + } + }); + } + if let Some((span, message)) = forbidden { + errors.push(Diagnostic::error("C013", message, span)); + } + if let Some((span, model)) = tenant_model { + errors.push(Diagnostic::error( + "C014", + format!( + "authentication verify may access only global Models; '{model}' is tenant-scoped" + ), + span, + )); + } + if program.effects[hook.function].contains("request") { + errors.push(Diagnostic::error( + "C006", + "authentication verify receives checked Claims and cannot read HTTP request capabilities", + program.functions[hook.function].span, + )); + } + if program.effects[hook.function].contains("identity") { + errors.push(Diagnostic::error( + "C006", + "authentication verify cannot read or require an identity before it establishes one", + program.functions[hook.function].span, + )); + } + } + + for (function_id, function) in program.functions.iter().enumerate() { + let package = &program.packages[function.owner]; + if package.bundled || !program.effects[function_id].contains("identity") { + continue; + } + if package.role != Some(SourceRole::App) { + errors.push(Diagnostic::error( + "C006", + "authentication and site capabilities are available only from App functions", + function.span, + )); + continue; + } + } +} + +fn validate_identity_types(program: &Program, configured: bool, errors: &mut Vec<Diagnostic>) { + for route in &program.api_routes { + let context = IdentityContext { + program, + hook: route.auth.map(|index| &program.auth[index]), + protected: !route.anonymous, + configured, + binding_span: route.span, + }; + let root = crate::specialize::Specialization { + function: route.function, + handlers: Vec::new(), + }; + let Ok(reachable) = crate::specialize::reachable(&program.functions, [root]) else { + continue; + }; + for instance in reachable { + super::visit(&program.functions[instance.function], |expression| { + context.validate_expression(expression, errors); + }); + } + } + for route in &program.api_rest { + let context = IdentityContext { + program, + hook: route.auth.map(|index| &program.auth[index]), + protected: true, + configured, + binding_span: route.span, + }; + if let Some((field, source)) = route.contract.owner { + let Shape::Record(fields) = &program.types[program.models[route.model].record].shape + else { + unreachable!("checked Model record") + }; + let (name, identity_field) = match source { + crate::model::RestOwnerSource::User => ("user_id", 1), + crate::model::RestOwnerSource::Tenant => ("tenant_id", 2), + }; + context.validate_type( + name, + identity_field, + fields[field].ty.clone().nullable(), + program.types[program.models[route.model].record].span, + errors, + ); + } + for expression in route + .contract + .create + .iter() + .chain(&route.contract.replace) + .filter_map(|write| match &write.source { + crate::model::RestValueSource::Binding { expression, .. } => Some(expression), + crate::model::RestValueSource::Input => None, + }) + .chain( + route + .contract + .search + .iter() + .map(|search| &search.expression), + ) + { + super::visit_expression(expression, |value| { + context.validate_expression(value, errors); + }); + } + } +} + +struct IdentityContext<'a> { + program: &'a Program, + hook: Option<&'a AuthHook>, + protected: bool, + configured: bool, + binding_span: Span, +} + +impl IdentityContext<'_> { + fn validate_expression(&self, expression: &Expression, errors: &mut Vec<Diagnostic>) { + use crate::intrinsic::Intrinsic; + + let ExpressionKind::Intrinsic { + operation, + arguments, + .. + } = &expression.kind + else { + return; + }; + let (name, field, actual) = match operation { + Intrinsic::AuthUserId => ("user_id", 1, expression.ty.clone()), + Intrinsic::AuthTenantId => ("tenant_id", 2, expression.ty.clone()), + Intrinsic::AuthOwnsUser => ("owns_user", 1, arguments[0].ty.clone().nullable()), + _ => return, + }; + self.validate_type(name, field, actual, expression.span, errors); + } + + fn validate_type( + &self, + name: &str, + field: usize, + actual: Type, + span: Span, + errors: &mut Vec<Diagnostic>, + ) { + if !optional_model_id(&actual, &self.program.types) || (!self.configured && self.protected) + { + return; + } + let expected = self.hook.and_then(|hook| { + let Shape::Record(fields) = &self.program.types[hook.identity].shape else { + return None; + }; + Some(&fields[field].ty) + }); + if self.protected && expected == Some(&actual) { + return; + } + let message = match (self.protected, self.hook, expected) { + (true, Some(hook), Some(expected)) => format!( + "dever.auth.{name} nominal ID must match provider '{}' identity field {} rather than {}", + hook.provider, + expected.label(&self.program.types), + actual.label(&self.program.types), + ), + _ => format!( + "typed dever.auth.{name} requires a protected API with a configured identity provider" + ), + }; + let mut error = Diagnostic::error("C006", message, span); + error.related.push(crate::diagnostic::Label { + span: self.binding_span, + message: "authentication context belongs to this API binding".into(), + }); + errors.push(error); + } +} diff --git a/crates/dever-core/src/check/body.rs b/crates/dever-core/src/check/body.rs new file mode 100644 index 0000000..2649736 --- /dev/null +++ b/crates/dever-core/src/check/body.rs @@ -0,0 +1,648 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use crate::diagnostic::Diagnostic; +use crate::hir::{self, Atom, Binding, Constant, Domain, ExpressionKind, Function, Projection}; +use crate::source::Span; +use crate::syntax::{FunctionClause, InputKind, Pattern, StatementKind}; +use crate::types::{HandlerSignature, Parameter, Shape, Type}; + +use super::{Checked, Context, path_name}; + +pub(super) struct Local { + pub slot: usize, + pub ty: Type, + pub initialized: bool, + pub last_write: Option<Span>, + pub read: bool, + pub known: Option<ExpressionKind>, + pub affine_consumed: bool, +} + +pub(super) struct Body<'a, 'b> { + pub context: &'a Context<'b>, + pub locals: BTreeMap<String, Local>, + pub handlers: BTreeMap<String, HandlerLocal>, + pub slots: Vec<Type>, + pub warnings: &'a mut Vec<Diagnostic>, +} + +pub(super) struct HandlerLocal { + pub parameter: usize, + pub signature: HandlerSignature, +} + +pub(super) fn expression( + source: &crate::syntax::Expression, + input: &Type, + expected: &Type, + context: &Context<'_>, + warnings: &mut Vec<Diagnostic>, +) -> Checked<hir::Expression> { + let mut checker = Body { + context, + locals: BTreeMap::new(), + handlers: BTreeMap::new(), + slots: Vec::new(), + warnings, + }; + checker.declare("input", input.clone(), true, source.span)?; + checker.expression(source, Some(expected)) +} + +pub(super) fn check( + source: &FunctionClause, + patterns: Vec<Domain>, + function: &Function, + context: &Context<'_>, + warnings: &mut Vec<Diagnostic>, +) -> Checked<hir::Clause> { + let mut checker = Body { + context, + locals: BTreeMap::new(), + handlers: BTreeMap::new(), + slots: Vec::new(), + warnings, + }; + let mut bindings = Vec::new(); + let mut domains = patterns.iter(); + for (input_index, (input, parameter)) in + source.inputs.iter().zip(&function.parameters).enumerate() + { + let Parameter::Value(parameter) = parameter else { + let Parameter::Handler(signature) = parameter else { + unreachable!() + }; + if checker.locals.contains_key(&input.name.text) + || checker.handlers.contains_key(&input.name.text) + { + return Err(Diagnostic::error( + "C002", + format!("duplicate local binding '{}'", input.name.text), + input.name.span, + )); + } + checker.handlers.insert( + input.name.text.clone(), + HandlerLocal { + parameter: input_index, + signature: signature.clone(), + }, + ); + continue; + }; + let InputKind::Value(input_pattern) = &input.kind else { + unreachable!("checked parameter kind") + }; + let domain = domains.next().expect("checked value parameter domain"); + let non_null = matches!(parameter, Type::Nullable(_)) + && !domain.iter().any(|atom| matches!(atom, Atom::Null)); + let ty = if non_null { + parameter.base().clone() + } else { + parameter.clone() + }; + let slot = checker.declare(&input.name.text, ty, true, input.name.span)?; + checker + .locals + .get_mut(&input.name.text) + .expect("declared input") + .known = match domain.as_slice() { + [Atom::Null] => Some(ExpressionKind::Constant(Constant::Null)), + [Atom::Bool(value)] => Some(ExpressionKind::Constant(Constant::Bool(*value))), + [Atom::Int(lo, hi)] if lo == hi => Some(ExpressionKind::Constant(Constant::Int(*lo))), + [Atom::Decimal(lo, hi)] if lo == hi => { + Some(ExpressionKind::Constant(Constant::Decimal(*lo))) + } + [Atom::Literal(Constant::Text(value))] => { + Some(ExpressionKind::Constant(Constant::Text(value.clone()))) + } + _ => None, + }; + bindings.push(Binding { + slot, + input: input_index, + projection: if non_null { + Projection::NonNull + } else { + Projection::Whole + }, + }); + if let Pattern::Variant { + name, + arguments, + bindings: names, + span, + } = input_pattern + { + let qualified = path_name(name); + let (variant, fields) = if let Some((_, variant, fields)) = + context.related_variant(&qualified, arguments, *span)? + { + (variant, fields) + } else { + let (id, variant) = context + .variant(&qualified, *span)? + .expect("checked choice pattern"); + let Shape::Choice(variants) = &context.types[id].shape else { + unreachable!() + }; + (variant, variants[variant].fields.clone()) + }; + for (field, (name, payload)) in names.iter().zip(&fields).enumerate() { + if name.text == "_" { + continue; + } + let slot = checker.declare(&name.text, payload.ty.clone(), true, name.span)?; + bindings.push(Binding { + slot, + input: input_index, + projection: Projection::Payload { variant, field }, + }); + } + } + } + debug_assert!(domains.next().is_none()); + let mut outputs = Vec::new(); + for (field, source) in function.outputs.iter().zip(&source.outputs) { + outputs.push(checker.declare(&field.name, field.ty.clone(), false, source.span)?); + } + let mut body = Vec::new(); + let mut terminates = false; + for (index, statement) in source.body.iter().enumerate() { + match &statement.kind { + StatementKind::Assign { target, value } => { + let name = &target[0].text; + if name == "setting" + && checker.context.symbols.packages[checker.context.owner] + .layout + .role() + == Some(crate::source::SourceRole::Adapter) + { + return Err(Diagnostic::error( + "C006", + "Adapter setting is read-only and cannot be shadowed", + target[0].span, + )); + } + if target.len() == 1 { + let expected = checker.locals.get(name).map(|local| local.ty.clone()); + let value = checker.expression(value, expected.as_ref())?; + if value.ty == Type::Unit { + return Err(Diagnostic::error( + "C005", + "a zero-output action cannot be assigned", + value.span, + )); + } + let slot = match checker.locals.get_mut(name) { + Some(local) => { + if is_affine(&local.ty) && !local.affine_consumed { + return Err(Diagnostic::error( + "C005", + format!( + "{} must be consumed before it can be overwritten", + affine_label(&local.ty) + ), + statement.span, + )); + } + warn_overwrite(local, checker.warnings); + local.initialized = true; + local.affine_consumed = false; + local.read = false; + local.last_write = Some(statement.span); + local.known = known_value(&value.kind); + local.slot + } + None => { + let slot = + checker.declare(name, value.ty.clone(), true, target[0].span)?; + let local = checker.locals.get_mut(name).expect("declared local"); + local.last_write = Some(statement.span); + local.known = known_value(&value.kind); + slot + } + }; + body.push(hir::Statement::Assign { + slot, + fields: Vec::new(), + value, + }); + } else { + let local = checker.read_local(name, target[0].span)?; + let slot = local.slot; + let mut ty = local.ty.clone(); + let mut fields = Vec::new(); + for name in &target[1..] { + if matches!(ty, Type::Outputs(_)) { + return Err(Diagnostic::error( + "C005", + "named call outputs are read-only projections", + name.span, + )); + } + let (index, field_type) = checker.field_type(&ty, name)?; + fields.push(index); + ty = field_type; + } + let value = checker.expression(value, Some(&ty))?; + let local = checker.locals.get_mut(name).expect("checked local"); + local.last_write = Some(statement.span); + local.read = false; + local.known = None; + body.push(hir::Statement::Assign { + slot, + fields, + value, + }); + } + } + StatementKind::Call(expression) => { + let value = checker.expression(expression, None)?; + if value.ty != Type::Unit { + return Err(Diagnostic::error( + "C005", + "only zero-output calls may be action statements", + expression.span, + )); + } + if matches!(value.kind, ExpressionKind::Fail(_)) { + if index + 1 != source.body.len() { + return Err(Diagnostic::error( + "C005", + "fail must be the final statement in a function clause", + expression.span, + )); + } + terminates = true; + } + body.push(hir::Statement::Call(value)); + } + } + } + if !terminates { + for field in &function.outputs { + checker.read_local(&field.name, source.span)?; + } + } + if !terminates + && let Some((_, local)) = checker + .locals + .iter() + .find(|(_, local)| is_affine(&local.ty) && !local.affine_consumed) + { + return Err(Diagnostic::error( + "C005", + format!( + "{} must be consumed before the function exits", + affine_label(&local.ty) + ), + local.last_write.unwrap_or(source.span), + )); + } + for local in checker.locals.values() { + if !local.read + && let Some(span) = local.last_write + { + checker.warnings.push(Diagnostic::error( + "W001", + "value is written but never read", + span, + )); + } + } + Ok(hir::Clause { + patterns, + bindings, + locals: checker.slots, + outputs, + body, + terminates, + }) +} + +fn warn_overwrite(local: &Local, warnings: &mut Vec<Diagnostic>) { + if !local.read + && let Some(span) = local.last_write + { + warnings.push(Diagnostic::error( + "W001", + "value is overwritten before it is read", + span, + )); + } +} + +impl Body<'_, '_> { + fn declare(&mut self, name: &str, ty: Type, initialized: bool, span: Span) -> Checked<usize> { + if self.locals.contains_key(name) || self.handlers.contains_key(name) { + return Err(Diagnostic::error( + "C002", + format!("duplicate local binding '{name}'"), + span, + )); + } + let slot = self.slots.len(); + self.slots.push(ty.clone()); + self.locals.insert( + name.into(), + Local { + slot, + ty, + initialized, + last_write: None, + read: false, + known: None, + affine_consumed: false, + }, + ); + Ok(slot) + } + + pub(super) fn read_local(&mut self, name: &str, span: Span) -> Checked<&Local> { + let local = self + .locals + .get_mut(name) + .filter(|local| local.initialized) + .ok_or_else(|| { + Diagnostic::error( + "C004", + format!("unknown or uninitialized local '{name}'"), + span, + ) + })?; + if is_affine(&local.ty) { + return Err(Diagnostic::error( + "C005", + format!( + "{} is affine and may only be used by its task operation", + affine_label(&local.ty) + ), + span, + )); + } + local.read = true; + Ok(local) + } + + pub(super) fn affine_local( + &mut self, + name: &str, + span: Span, + consume: bool, + ) -> Checked<(usize, Type)> { + let local = self + .locals + .get_mut(name) + .ok_or_else(|| Diagnostic::error("C004", format!("unknown local '{name}'"), span))?; + if !is_affine(&local.ty) { + return Err(Diagnostic::error( + "C005", + "this operation requires a Task or Group local", + span, + )); + } + if local.affine_consumed { + return Err(Diagnostic::error( + "C005", + format!("{} has already been consumed", affine_label(&local.ty)), + span, + )); + } + local.read = true; + if consume { + local.affine_consumed = true; + } + Ok((local.slot, local.ty.clone())) + } +} + +fn is_affine(ty: &Type) -> bool { + matches!(ty, Type::Task(_) | Type::Group | Type::Upload) +} + +fn affine_label(ty: &Type) -> &'static str { + match ty { + Type::Task(_) => "Task", + Type::Group => "Group", + Type::Upload => "Upload", + _ => unreachable!("affine label requires Task or Group"), + } +} + +pub(crate) fn check_suspensions(program: &crate::hir::Program, errors: &mut Vec<Diagnostic>) { + for function in &program.functions { + for clause in &function.clauses { + if let Err(error) = check_suspension_safety(clause, program) { + errors.push(error); + } + } + } +} + +fn check_suspension_safety(clause: &hir::Clause, program: &crate::hir::Program) -> Checked<()> { + let body = &clause.body; + let locals = &clause.locals; + let outputs = &clause.outputs; + let mut live: BTreeSet<_> = outputs.iter().copied().collect(); + for statement in body.iter().rev() { + match statement { + hir::Statement::Assign { + slot, + fields, + value, + } => { + if fields.is_empty() { + live.remove(slot); + } else { + live.insert(*slot); + } + expression_liveness(value, locals, program, &mut live)?; + } + hir::Statement::Call(value) => { + expression_liveness(value, locals, program, &mut live)?; + } + } + } + Ok(()) +} + +fn expression_liveness( + expression: &hir::Expression, + locals: &[Type], + program: &crate::hir::Program, + live: &mut BTreeSet<usize>, +) -> Checked<()> { + if suspension_point(&expression.kind, program) + && let Some(slot) = live.iter().copied().find(|slot| { + let ty = &locals[*slot]; + !matches!(ty, Type::Task(_) | Type::Group | Type::Upload) + && !ty.transferable(&program.types) + }) + { + return Err(Diagnostic::error( + "C005", + format!( + "{} local cannot remain live across a suspension point", + locals[slot].label(&program.types) + ), + expression.span, + )); + } + + match &expression.kind { + ExpressionKind::Local(slot) => { + live.insert(*slot); + } + ExpressionKind::Field { value, .. } + | ExpressionKind::Fail(value) + | ExpressionKind::Unary { value, .. } + | ExpressionKind::Promote(value) + | ExpressionKind::Some(value) + | ExpressionKind::AwaitTask(value) + | ExpressionKind::StopTask(value) + | ExpressionKind::Group(value) + | ExpressionKind::AwaitGroup(value) + | ExpressionKind::StopGroup(value) + | ExpressionKind::ChannelReceive(value) + | ExpressionKind::ChannelClose(value) => { + expression_liveness(value, locals, program, live)?; + } + ExpressionKind::Call { arguments, .. } + | ExpressionKind::CaptureResult { arguments, .. } + | ExpressionKind::ParallelCall { arguments, .. } + | ExpressionKind::BlockingCall { arguments, .. } => { + call_arguments_liveness(arguments, locals, program, live)?; + } + ExpressionKind::ModelOperation { operation, .. } => { + for value in operation.expressions().into_iter().rev() { + expression_liveness(value, locals, program, live)?; + } + } + ExpressionKind::RunCall { + group, arguments, .. + } => { + call_arguments_liveness(arguments, locals, program, live)?; + if let Some(group) = group { + expression_liveness(group, locals, program, live)?; + } + } + ExpressionKind::Channel { capacity, .. } => { + expression_liveness(capacity, locals, program, live)?; + } + ExpressionKind::ChannelSend { channel, value } => { + expression_liveness(value, locals, program, live)?; + expression_liveness(channel, locals, program, live)?; + } + ExpressionKind::Intrinsic { arguments, .. } + | ExpressionKind::JobEnqueue { arguments, .. } + | ExpressionKind::Collection { arguments, .. } + | ExpressionKind::Variant { arguments, .. } + | ExpressionKind::List(arguments) => { + for argument in arguments.iter().rev() { + expression_liveness(argument, locals, program, live)?; + } + } + ExpressionKind::Record { fields } => { + for (_, value) in fields.iter().rev() { + expression_liveness(value, locals, program, live)?; + } + } + ExpressionKind::Map(entries) => { + for (key, value) in entries.iter().rev() { + expression_liveness(value, locals, program, live)?; + expression_liveness(key, locals, program, live)?; + } + } + ExpressionKind::Binary { left, right, .. } => { + expression_liveness(right, locals, program, live)?; + expression_liveness(left, locals, program, live)?; + } + ExpressionKind::Constant(_) | ExpressionKind::ChoiceOptions { .. } => {} + } + Ok(()) +} + +fn call_arguments_liveness( + arguments: &[hir::CallArgument], + locals: &[Type], + program: &crate::hir::Program, + live: &mut BTreeSet<usize>, +) -> Checked<()> { + for argument in arguments.iter().rev() { + if let hir::CallArgument::Value(value) = argument { + expression_liveness(value, locals, program, live)?; + } + } + Ok(()) +} + +fn suspension_point(kind: &ExpressionKind, program: &crate::hir::Program) -> bool { + if let ExpressionKind::Intrinsic { operation, .. } = kind { + return operation.is_async(); + } + if let ExpressionKind::Call { target, .. } | ExpressionKind::CaptureResult { target, .. } = kind + { + return call_target_may_suspend(*target, program); + } + if matches!( + kind, + ExpressionKind::ModelOperation { .. } | ExpressionKind::JobEnqueue { .. } + ) { + return true; + } + if let ExpressionKind::Collection { + operation, + handler, + sequence, + .. + } = kind + { + return *operation == hir::CollectionOp::ParallelEach + || (matches!( + sequence, + Some(hir::SequenceKind::AsyncStream | hir::SequenceKind::RowStream) + ) && *operation != hir::CollectionOp::Close) + || handler.is_some_and(|target| handler_may_suspend(target, program)); + } + matches!( + kind, + ExpressionKind::RunCall { .. } + | ExpressionKind::AwaitTask(_) + | ExpressionKind::StopTask(_) + | ExpressionKind::AwaitGroup(_) + | ExpressionKind::StopGroup(_) + | ExpressionKind::ParallelCall { .. } + | ExpressionKind::BlockingCall { .. } + | ExpressionKind::ChannelSend { .. } + | ExpressionKind::ChannelReceive(_) + | ExpressionKind::ChannelClose(_) + ) +} + +fn call_target_may_suspend(target: hir::CallTarget, program: &crate::hir::Program) -> bool { + match target { + hir::CallTarget::Function(id) => function_may_suspend(id, program), + hir::CallTarget::Handler(target) => handler_may_suspend(target, program), + } +} + +fn handler_may_suspend(target: hir::HandlerTarget, program: &crate::hir::Program) -> bool { + match target { + hir::HandlerTarget::Function(id) => function_may_suspend(id, program), + hir::HandlerTarget::Parameter(_) => true, + } +} + +fn function_may_suspend(id: usize, program: &crate::hir::Program) -> bool { + program.functions[id].suspends || !program.suspension_handlers[id].is_empty() +} + +fn known_value(value: &ExpressionKind) -> Option<ExpressionKind> { + match value { + ExpressionKind::Constant(_) => Some(value.clone()), + ExpressionKind::Variant { arguments, .. } if arguments.is_empty() => Some(value.clone()), + ExpressionKind::Some(inner) if known_value(&inner.kind).is_some() => Some(value.clone()), + _ => None, + } +} diff --git a/crates/dever-core/src/check/cache.rs b/crates/dever-core/src/check/cache.rs new file mode 100644 index 0000000..ca2a495 --- /dev/null +++ b/crates/dever-core/src/check/cache.rs @@ -0,0 +1,63 @@ +use std::cell::RefCell; +use std::path::PathBuf; + +use crate::diagnostic::Diagnostic; +use crate::hir::Program; +use crate::source::{SourceMap, SourceOrigin}; + +type CheckedProgram = Result<Program, Vec<Diagnostic>>; + +struct CachedProgram { + sources: Vec<(PathBuf, PathBuf, SourceOrigin, String)>, + strict_layout: bool, + result: CheckedProgram, +} + +thread_local! { + // Keep one compilation only: editors and embedding callers can reuse an unchanged + // program without retaining every project ever checked by a long-lived process. + static PREVIOUS: RefCell<Option<CachedProgram>> = const { RefCell::new(None) }; +} + +pub(super) fn check( + sources: &SourceMap, + compute: impl FnOnce(&SourceMap) -> CheckedProgram, +) -> CheckedProgram { + let cached = PREVIOUS.with_borrow(|previous| { + let previous = previous.as_ref()?; + let matches = previous.strict_layout == sources.strict_layout() + && previous.sources.len() == sources.files().len() + && previous.sources.iter().zip(sources.files()).all( + |((path, logical_path, origin, text), source)| { + path == source.path() + && logical_path == source.logical_path() + && *origin == source.origin() + && text == source.text() + }, + ); + matches.then(|| previous.result.clone()) + }); + if let Some(result) = cached { + return result; + } + let result = compute(sources); + PREVIOUS.with_borrow_mut(|previous| { + *previous = Some(CachedProgram { + strict_layout: sources.strict_layout(), + sources: sources + .files() + .iter() + .map(|source| { + ( + source.path().to_owned(), + source.logical_path().to_owned(), + source.origin(), + source.text().to_owned(), + ) + }) + .collect(), + result: result.clone(), + }); + }); + result +} diff --git a/crates/dever-core/src/check/clauses.rs b/crates/dever-core/src/check/clauses.rs new file mode 100644 index 0000000..a5f229d --- /dev/null +++ b/crates/dever-core/src/check/clauses.rs @@ -0,0 +1,628 @@ +use std::collections::BTreeSet; + +use dever_runtime::number::DecimalValue; + +use crate::diagnostic::{Diagnostic, Label}; +use crate::hir::{Atom, Constant, Domain, ExpressionKind}; +use crate::syntax::{BinaryOperator as Op, FunctionClause, InputKind, Literal, Pattern}; +use crate::types::{Field, HandlerSignature, Parameter, Shape, Type}; + +use super::{Checked, Context, constants, path_name, symbols}; + +#[derive(Clone)] +enum PatternSet { + Any(bool), + Null, + Bool(bool), + Variant(usize), + Literal(Constant), + Int(Option<(i64, i64)>), + Decimal(Option<(DecimalValue, DecimalValue)>), + Other, +} + +pub(super) fn signature( + group: &[&FunctionClause], + context: &Context<'_>, +) -> Checked<(Vec<Parameter>, Vec<Field>)> { + if group + .iter() + .any(|clause| clause.pure != group[0].pure || clause.recovery != group[0].recovery) + { + return Err(Diagnostic::error( + "C013", + "all clauses must have identical pure and recovery contracts", + group[0].span, + )); + } + let mut parameters = Vec::new(); + for axis in 0..group[0].inputs.len() { + if let InputKind::Handler(source) = &group[0].inputs[axis].kind { + let signature = handler_signature(source, context)?; + for clause in &group[1..] { + let InputKind::Handler(candidate) = &clause.inputs[axis].kind else { + return Err(Diagnostic::error( + "C005", + "all clauses must agree whether an input is a handler", + clause.inputs[axis].span, + )); + }; + if handler_signature(candidate, context)? != signature { + return Err(Diagnostic::error( + "C005", + "all clauses must use the same handler signature", + clause.inputs[axis].span, + )); + } + } + parameters.push(Parameter::Handler(signature)); + continue; + } + let mut inferred = None; + let mut nullable = false; + for clause in group { + let input = &clause.inputs[axis]; + let InputKind::Value(pattern) = &input.kind else { + return Err(Diagnostic::error( + "C005", + "all clauses must agree whether an input is a handler", + input.span, + )); + }; + let ty = match pattern { + Pattern::Typed { ty, .. } => { + match context.variant(&path_name(&ty.name), ty.span)? { + Some((id, _)) => Some(Type::Named(id)), + None => Some(context.resolve_type(ty)?), + } + } + Pattern::Variant { + name, + arguments, + span, + .. + } => { + let name = path_name(name); + if let Some((ty, _, _)) = context.related_variant(&name, arguments, *span)? { + Some(ty) + } else { + if !arguments.is_empty() { + return Err(Diagnostic::error( + "C005", + "choice variants do not accept type arguments", + *span, + )); + } + Some(Type::Named( + context + .variant(&name, *span)? + .ok_or_else(|| { + Diagnostic::error( + "C005", + "expected a choice variant pattern", + *span, + ) + })? + .0, + )) + } + } + _ => None, + }; + if let Some(ty) = ty { + merge(&mut inferred, &mut nullable, ty, input.span)?; + } + } + let explicitly_typed = inferred.is_some(); + let mut literal_types = Vec::new(); + for clause in group { + let input = &clause.inputs[axis]; + let InputKind::Value(pattern) = &input.kind else { + unreachable!("input kind checked above") + }; + if let Pattern::Literal { value, span } = pattern { + if matches!(value, Literal::Null) { + nullable = true; + continue; + } + let value = constants::literal(value, inferred.as_ref(), *span)?; + literal_types.push((value.ty, input.span)); + } + } + if !explicitly_typed && literal_types.iter().any(|(ty, _)| *ty == Type::Decimal) { + inferred = Some(Type::Decimal); + } + for (ty, span) in literal_types { + let ty = if ty == Type::Int && inferred == Some(Type::Decimal) { + Type::Decimal + } else { + ty + }; + merge(&mut inferred, &mut nullable, ty, span)?; + } + let ty = inferred.ok_or_else(|| { + Diagnostic::error( + "C005", + "cannot infer this parameter's base type from null/other alone", + group[0].inputs[axis].span, + ) + })?; + let ty = if nullable { ty.nullable() } else { ty }; + symbols::validate_type_constraints(&ty, context.types, group[0].inputs[axis].span)?; + parameters.push(Parameter::Value(ty)); + } + let outputs = context.fields(&group[0].outputs)?; + for field in &outputs { + symbols::validate_type_constraints(&field.ty, context.types, group[0].span)?; + } + for clause in &group[1..] { + if context.fields(&clause.outputs)? != outputs { + return Err(Diagnostic::error( + "C005", + "all clauses must have identical ordered output names and types", + clause.span, + )); + } + } + Ok((parameters, outputs)) +} + +fn handler_signature( + source: &crate::syntax::HandlerSignature, + context: &Context<'_>, +) -> Checked<HandlerSignature> { + let inputs = context.fields(&source.inputs)?; + for (field, source) in inputs.iter().zip(&source.inputs) { + symbols::validate_type_constraints(&field.ty, context.types, source.span)?; + } + let outputs = context.fields(&source.outputs)?; + for (field, source) in outputs.iter().zip(&source.outputs) { + symbols::validate_type_constraints(&field.ty, context.types, source.span)?; + } + Ok(HandlerSignature { + bounds: inputs.iter().map(|field| field.bounds.clone()).collect(), + parameters: inputs.into_iter().map(|field| field.ty).collect(), + outputs, + }) +} + +pub(super) fn field_bounds( + field: &crate::syntax::Field, + ty: &Type, + context: &Context<'_>, +) -> Checked<Domain> { + if field.bounds.is_empty() { + return Ok(Vec::new()); + } + let pattern = Pattern::Typed { + ty: field.ty.clone(), + bounds: field.bounds.clone(), + }; + let atom = match normalize(&pattern, ty, context, field.span)? { + PatternSet::Int(Some((lo, hi))) => Atom::Int(lo, hi), + PatternSet::Decimal(Some((lo, hi))) => Atom::Decimal(lo, hi), + _ => { + return Err(Diagnostic::error( + "C011", + "field constraint has an empty domain", + field.span, + )); + } + }; + Ok(vec![atom]) +} + +fn merge( + inferred: &mut Option<Type>, + nullable: &mut bool, + ty: Type, + span: crate::source::Span, +) -> Checked<()> { + *nullable |= matches!(ty, Type::Nullable(_)); + let base = ty.base().clone(); + match inferred { + Some(previous) if *previous != base => Err(Diagnostic::error( + "C005", + "clauses disagree on this parameter's base type", + span, + )), + _ => { + *inferred = Some(base); + Ok(()) + } + } +} + +pub(super) fn check( + group: &[&FunctionClause], + parameters: &[Parameter], + context: &Context<'_>, +) -> Checked<Vec<Vec<Domain>>> { + let value_axes = parameters + .iter() + .enumerate() + .filter_map(|(axis, parameter)| parameter.value_type().map(|ty| (axis, ty))) + .collect::<Vec<_>>(); + let patterns: Vec<Vec<_>> = group + .iter() + .map(|clause| { + value_axes + .iter() + .map(|(axis, ty)| { + let input = &clause.inputs[*axis]; + let InputKind::Value(pattern) = &input.kind else { + return Err(Diagnostic::error( + "C005", + "all clauses must agree whether an input is a handler", + input.span, + )); + }; + normalize(pattern, ty, context, input.span) + }) + .collect::<Checked<_>>() + }) + .collect::<Checked<_>>()?; + let atoms: Vec<_> = value_axes + .iter() + .enumerate() + .map(|(axis, (_, ty))| partition(ty, patterns.iter().map(|row| &row[axis]), context)) + .collect(); + let mut products: Vec<Vec<BTreeSet<usize>>> = Vec::new(); + for (row, clause) in patterns.iter().zip(group) { + let mut product = Vec::new(); + for (axis, pattern) in row.iter().enumerate() { + let selected: BTreeSet<_> = atoms[axis] + .iter() + .enumerate() + .filter_map(|(id, atom)| { + let included = if matches!(pattern, PatternSet::Other) { + !patterns.iter().any(|row| { + !matches!(row[axis], PatternSet::Other) && contains(&row[axis], atom) + }) + } else { + contains(pattern, atom) + }; + included.then_some(id) + }) + .collect(); + if selected.is_empty() { + return Err(Diagnostic::error( + "C009", + "unreachable clause: this pattern selects no values", + clause.inputs[value_axes[axis].0].span, + )); + } + product.push(selected); + } + for (previous, other) in products.iter().enumerate() { + if product + .iter() + .zip(other) + .all(|(left, right)| !left.is_disjoint(right)) + { + let mut error = Diagnostic::error("C009", "function clauses overlap", clause.span); + error.related.push(Label { + span: group[previous].span, + message: "overlapping clause".into(), + }); + return Err(error); + } + } + products.push(product); + } + // Subtract disjoint rectangles without enumerating the input Cartesian product. + let mut remaining = vec![ + atoms + .iter() + .map(|axis| (0..axis.len()).collect()) + .collect::<Vec<BTreeSet<_>>>(), + ]; + for covered in &products { + remaining = remaining + .into_iter() + .flat_map(|region| subtract(region, covered)) + .collect(); + } + if !remaining.is_empty() { + return Err(Diagnostic::error( + "C009", + "function clauses do not cover every input combination", + group[0].span, + )); + } + Ok(products + .into_iter() + .map(|row| { + row.into_iter() + .enumerate() + .map(|(axis, selected)| { + selected + .into_iter() + .map(|id| atoms[axis][id].clone()) + .collect() + }) + .collect() + }) + .collect()) +} + +fn subtract( + region: Vec<BTreeSet<usize>>, + covered: &[BTreeSet<usize>], +) -> Vec<Vec<BTreeSet<usize>>> { + let intersections: Vec<_> = region + .iter() + .zip(covered) + .map(|(left, right)| left.intersection(right).copied().collect::<BTreeSet<_>>()) + .collect(); + if intersections.iter().any(BTreeSet::is_empty) { + return vec![region]; + } + let mut remainder = Vec::new(); + for axis in 0..region.len() { + let difference: BTreeSet<_> = region[axis].difference(&covered[axis]).copied().collect(); + if difference.is_empty() { + continue; + } + let mut piece = intersections[..axis].to_vec(); + piece.push(difference); + piece.extend_from_slice(®ion[axis + 1..]); + remainder.push(piece); + } + remainder +} + +fn normalize( + pattern: &Pattern, + parameter: &Type, + context: &Context<'_>, + span: crate::source::Span, +) -> Checked<PatternSet> { + match pattern { + Pattern::Other(_) => Ok(PatternSet::Other), + Pattern::Variant { + name, + arguments, + bindings, + .. + } => { + let name = path_name(name); + let (ty, variant, fields) = + if let Some(found) = context.related_variant(&name, arguments, span)? { + found + } else { + if !arguments.is_empty() { + return Err(Diagnostic::error( + "C005", + "choice variants do not accept type arguments", + span, + )); + } + let (id, variant) = context.variant(&name, span)?.ok_or_else(|| { + Diagnostic::error("C005", "expected choice variant", span) + })?; + let Shape::Choice(variants) = &context.types[id].shape else { + unreachable!() + }; + (Type::Named(id), variant, variants[variant].fields.clone()) + }; + if ty != *parameter.base() { + return Err(Diagnostic::error( + "C005", + "variant pattern has the wrong parameter type", + span, + )); + } + if fields.len() != bindings.len() { + return Err(Diagnostic::error( + "C005", + "choice pattern payload arity mismatch", + span, + )); + } + Ok(PatternSet::Variant(variant)) + } + Pattern::Literal { + value: Literal::Null, + .. + } => Ok(PatternSet::Null), + Pattern::Literal { value, .. } => { + let mut value = constants::literal(value, Some(parameter), span)?; + if value.ty == Type::Int && parameter.base() == &Type::Decimal { + value = constants::promote(value); + } + let ExpressionKind::Constant(value) = value.kind else { + unreachable!() + }; + Ok(match value { + Constant::Bool(value) => PatternSet::Bool(value), + Constant::Int(value) => PatternSet::Int(Some((value, value))), + Constant::Decimal(value) => PatternSet::Decimal(Some((value, value))), + _ => PatternSet::Literal(value), + }) + } + Pattern::Typed { ty, bounds } => { + if let Some((id, variant)) = context.variant(&path_name(&ty.name), span)? { + let Shape::Choice(variants) = &context.types[id].shape else { + unreachable!() + }; + if ty.nullable || !bounds.is_empty() || !variants[variant].fields.is_empty() { + return Err(Diagnostic::error( + "C005", + "payload variants require an explicit binding list", + span, + )); + } + return Ok(PatternSet::Variant(variant)); + } + if bounds.is_empty() { + return Ok(PatternSet::Any(ty.nullable)); + } + if ty.nullable || !matches!(parameter.base(), Type::Int | Type::Decimal) { + return Err(Diagnostic::error( + "C005", + "range bounds require non-nullable Int or Decimal patterns", + span, + )); + } + let mut lower = false; + let mut upper = false; + let mut integers = Some((i64::MIN, i64::MAX)); + let mut decimals = Some((DecimalValue::minimum(), DecimalValue::maximum())); + for bound in bounds { + let seen = match bound.comparison { + Op::Greater | Op::GreaterEqual => &mut lower, + Op::Less | Op::LessEqual => &mut upper, + _ => unreachable!(), + }; + if *seen { + return Err(Diagnostic::error( + "C009", + "a range may have at most one lower and one upper bound", + bound.span, + )); + } + *seen = true; + let mut value = constants::literal( + &Literal::Number(bound.number.clone()), + Some(parameter), + bound.span, + )?; + if value.ty == Type::Int && parameter.base() == &Type::Decimal { + value = constants::promote(value); + } + match value.kind { + ExpressionKind::Constant(Constant::Int(value)) + if parameter.base() == &Type::Int => + { + integers = integers.and_then(|(lo, hi)| { + Some(match bound.comparison { + Op::Greater => (value.checked_add(1)?, hi), + Op::GreaterEqual => (value, hi), + Op::Less => (lo, value.checked_sub(1)?), + Op::LessEqual => (lo, value), + _ => unreachable!(), + }) + }); + } + ExpressionKind::Constant(Constant::Decimal(value)) + if parameter.base() == &Type::Decimal => + { + decimals = decimals.and_then(|(lo, hi)| { + Some(match bound.comparison { + Op::Greater => (value.next_up()?, hi), + Op::GreaterEqual => (value, hi), + Op::Less => (lo, value.next_down()?), + Op::LessEqual => (lo, value), + _ => unreachable!(), + }) + }); + } + _ => { + return Err(Diagnostic::error( + "C005", + "range bound has the wrong numeric type", + bound.span, + )); + } + } + } + Ok(if parameter.base() == &Type::Int { + PatternSet::Int(integers.filter(|(lo, hi)| lo <= hi)) + } else { + PatternSet::Decimal(decimals.filter(|(lo, hi)| lo <= hi)) + }) + } + } +} + +fn partition<'a>( + ty: &Type, + patterns: impl Iterator<Item = &'a PatternSet>, + context: &Context<'_>, +) -> Domain { + let patterns: Vec<_> = patterns.collect(); + let mut atoms = match ty.base() { + Type::Bool => vec![Atom::Bool(false), Atom::Bool(true)], + Type::Named(id) => match &context.types[*id].shape { + Shape::Choice(variants) => (0..variants.len()).map(Atom::Variant).collect(), + _ => vec![Atom::Any], + }, + Type::Related(_) => vec![Atom::Variant(0), Atom::Variant(1)], + Type::Int => { + let mut cuts = BTreeSet::from([i64::MIN]); + for pattern in &patterns { + if let PatternSet::Int(Some((lo, hi))) = pattern { + cuts.insert(*lo); + if let Some(next) = hi.checked_add(1) { + cuts.insert(next); + } + } + } + let cuts: Vec<_> = cuts.into_iter().collect(); + cuts.iter() + .enumerate() + .map(|(index, lo)| { + Atom::Int(*lo, cuts.get(index + 1).map_or(i64::MAX, |next| next - 1)) + }) + .collect() + } + Type::Decimal => { + let mut cuts = BTreeSet::from([DecimalValue::minimum()]); + for pattern in &patterns { + if let PatternSet::Decimal(Some((lo, hi))) = pattern { + cuts.insert(*lo); + if let Some(next) = hi.next_up() { + cuts.insert(next); + } + } + } + let cuts: Vec<_> = cuts.into_iter().collect(); + cuts.iter() + .enumerate() + .map(|(index, lo)| { + Atom::Decimal( + *lo, + cuts.get(index + 1) + .and_then(|next| next.next_down()) + .unwrap_or_else(DecimalValue::maximum), + ) + }) + .collect() + } + Type::Text | Type::Float => { + let mut constants = Vec::new(); + for pattern in &patterns { + if let PatternSet::Literal(value) = pattern + && !constants.contains(value) + { + constants.push(value.clone()); + } + } + let mut atoms: Vec<_> = constants.iter().cloned().map(Atom::Literal).collect(); + atoms.push(Atom::Remainder(constants)); + atoms + } + _ => vec![Atom::Any], + }; + if matches!(ty, Type::Nullable(_)) { + atoms.push(Atom::Null); + } + atoms +} + +fn contains(pattern: &PatternSet, atom: &Atom) -> bool { + match (pattern, atom) { + (PatternSet::Any(nullable), Atom::Null) => *nullable, + (PatternSet::Any(_), _) => true, + (PatternSet::Null, Atom::Null) => true, + (PatternSet::Bool(left), Atom::Bool(right)) => left == right, + (PatternSet::Variant(left), Atom::Variant(right)) => left == right, + (PatternSet::Literal(left), Atom::Literal(right)) => left == right, + (PatternSet::Int(Some((lo, hi))), Atom::Int(start, _)) => lo <= start && start <= hi, + (PatternSet::Decimal(Some((lo, hi))), Atom::Decimal(start, _)) => { + lo <= start && start <= hi + } + _ => false, + } +} diff --git a/crates/dever-core/src/check/collections.rs b/crates/dever-core/src/check/collections.rs new file mode 100644 index 0000000..1bdc3e2 --- /dev/null +++ b/crates/dever-core/src/check/collections.rs @@ -0,0 +1,403 @@ +use crate::diagnostic::Diagnostic; +use crate::hir::{CollectionOp as Op, Expression as Value, ExpressionKind, SequenceKind}; +use crate::source::Span; +use crate::syntax::{Expression, ExpressionKind as Ast}; +use crate::types::{Type, output_type}; + +use super::{Checked, body::Body, expressions::ungroup}; + +pub(super) fn operation(name: &str) -> Option<Op> { + Some(match name { + "each" => Op::Each, + "parallel_each" => Op::ParallelEach, + "reduce" => Op::Reduce, + "reduce_until" => Op::ReduceUntil, + "filter" => Op::Filter, + "find" => Op::Find, + "sum" => Op::Sum, + "append" => Op::Append, + "first" => Op::First, + "get" => Op::Get, + "put" => Op::Put, + "remove" => Op::Remove, + "entries" => Op::Entries, + "length" => Op::Length, + "close" => Op::Close, + _ => return None, + }) +} + +struct Sequence { + value: Value, + kind: SequenceKind, + element: Type, +} + +impl Body<'_, '_> { + pub(super) fn collection( + &mut self, + operation: Op, + sources: &[Expression], + expected: Option<&Type>, + span: Span, + ) -> Checked<Value> { + if matches!( + operation, + Op::Each | Op::ParallelEach | Op::Filter | Op::Find | Op::Reduce | Op::ReduceUntil + ) || (operation == Op::Sum && sources.len() != 1) + { + return self.handler_sequence(operation, sources, span); + } + let arity = match operation { + Op::Put => 3, + Op::Get | Op::Remove | Op::Append => 2, + _ => 1, + }; + if sources.len() != arity { + return Err(Diagnostic::error( + "C005", + format!("collection operation expects {arity} argument(s)"), + span, + )); + } + let mut arguments = Vec::new(); + let mut sequence = None; + let ty = match operation { + Op::Append => { + let list = if let Some(Type::List(_)) = expected { + self.expression(&sources[0], expected)? + } else if matches!(&ungroup(&sources[0]).kind, Ast::List(elements) if elements.is_empty()) + { + let value = self.expression(&sources[1], None)?; + let hint = Type::List(Box::new(value.ty.clone())); + self.expression(&sources[0], Some(&hint))? + } else { + self.expression(&sources[0], None)? + }; + let Type::List(element) = &list.ty else { + return Err(expected_collection(span)); + }; + let value = self.expression(&sources[1], Some(element))?; + let ty = list.ty.clone(); + arguments.extend([list, value]); + ty + } + Op::Sum | Op::First => { + let hint = expected.map(|ty| Type::List(Box::new(ty.clone()))); + let list = self.expression( + &sources[0], + hint.as_ref() + .filter(|_| super::expressions::needs_context(&sources[0])), + )?; + let Type::List(element) = &list.ty else { + return Err(expected_collection(span)); + }; + let ty = if operation == Op::First { + element.as_ref().clone().nullable() + } else { + if !element.numeric() { + return Err(Diagnostic::error( + "C005", + "sum requires Int, Decimal or Float elements", + span, + )); + } + element.as_ref().clone() + }; + arguments.push(list); + if operation == Op::Sum { + sequence = Some(SequenceKind::List); + } + ty + } + Op::Get | Op::Put | Op::Remove => { + let hint = if matches!(operation, Op::Put | Op::Remove) { + expected.filter(|ty| matches!(ty, Type::Map(_, _))) + } else { + None + }; + let map = if hint.is_none() + && operation == Op::Put + && matches!(&ungroup(&sources[0]).kind, Ast::Map(entries) if entries.is_empty()) + { + let key = self.expression(&sources[1], None)?; + let value = self.expression(&sources[2], None)?; + let ty = Type::Map(Box::new(key.ty), Box::new(value.ty)); + self.expression(&sources[0], Some(&ty))? + } else { + self.expression(&sources[0], hint)? + }; + let Type::Map(key, value) = &map.ty else { + return Err(expected_collection(span)); + }; + let key = self.expression(&sources[1], Some(key))?; + let ty = if operation == Op::Get { + value.as_ref().clone().nullable() + } else { + map.ty.clone() + }; + let replacement = if operation == Op::Put { + Some(self.expression(&sources[2], Some(value))?) + } else { + None + }; + arguments.extend([map, key]); + arguments.extend(replacement); + ty + } + Op::Entries => { + let map = self.expression(&sources[0], None)?; + let Type::Map(key, value) = &map.ty else { + return Err(expected_collection(span)); + }; + let ty = Type::List(Box::new(Type::MapEntry(key.clone(), value.clone()))); + arguments.push(map); + ty + } + Op::Length => { + let value = self.expression(&sources[0], None)?; + if !matches!( + value.ty, + Type::List(_) | Type::Map(_, _) | Type::Text | Type::Bytes + ) { + return Err(expected_collection(span)); + } + arguments.push(value); + Type::Int + } + Op::Close => { + let value = self.expression(&sources[0], None)?; + if !matches!( + value.ty, + Type::Stream(_) | Type::AsyncStream(_) | Type::RowStream(_) + ) { + return Err(Diagnostic::error( + "C005", + "close requires a Stream or AsyncStream value", + sources[0].span, + )); + } + sequence = Some(match value.ty { + Type::AsyncStream(_) => SequenceKind::AsyncStream, + Type::RowStream(_) => SequenceKind::RowStream, + _ => SequenceKind::Stream, + }); + arguments.push(value); + Type::Unit + } + _ => unreachable!("handler operations dispatched above"), + }; + Ok(Value { + kind: ExpressionKind::Collection { + operation, + handler: None, + sequence, + arguments, + }, + ty, + span, + }) + } + + fn handler_sequence( + &mut self, + operation: Op, + sources: &[Expression], + span: Span, + ) -> Checked<Value> { + let (handler_arity, valid_arity) = match operation { + Op::ParallelEach => ( + sources.len().saturating_sub(2), + (3..=4).contains(&sources.len()), + ), + Op::Reduce | Op::ReduceUntil => (2, sources.len() == 3), + _ => ( + sources.len().saturating_sub(1), + (2..=3).contains(&sources.len()), + ), + }; + if !valid_arity || !(1..=2).contains(&handler_arity) { + return Err(Diagnostic::error( + "C005", + if matches!(operation, Op::Reduce | Op::ReduceUntil) { + "expected handler, sequence and initial state" + } else if operation == Op::ParallelEach { + "expected handler, sequence, worker count and optional context" + } else { + "expected handler, sequence and optional context" + }, + span, + )); + } + let (handler, signature) = self.resolve_handler(&sources[0], handler_arity)?; + let sequence_value = self.sequence(&sources[1], Some(&signature.parameters[0]))?; + if operation == Op::ParallelEach && sequence_value.kind == SequenceKind::RowStream { + return Err(Diagnostic::error( + "C005", + "database row streams use sequential each to retain one bounded connection", + sources[1].span, + )); + } + if sequence_value.element != signature.parameters[0] { + return Err(Diagnostic::error( + "C005", + "handler element type does not match the sequence", + sources[0].span, + )); + } + if matches!(operation, Op::Filter | Op::Find | Op::Sum) + && sequence_value.kind != SequenceKind::List + { + return Err(Diagnostic::error( + "C005", + "this operation only accepts List sequences", + sources[1].span, + )); + } + let output = output_type(&signature.outputs); + let mut arguments = vec![sequence_value.value]; + let ty = match operation { + Op::ParallelEach => { + if output != Type::Unit { + return Err(Diagnostic::error( + "C005", + "parallel_each requires a zero-output handler", + sources[0].span, + )); + } + if !signature + .parameters + .iter() + .all(|ty| ty.transferable(self.context.types)) + { + return Err(Diagnostic::error( + "C005", + "parallel_each handler inputs cannot contain Stream values", + sources[0].span, + )); + } + arguments.push(self.expression(&sources[2], Some(&Type::Int))?); + if signature.parameters.len() == 2 { + arguments.push(self.expression(&sources[3], Some(&signature.parameters[1]))?); + } + Type::Unit + } + Op::Each => { + if signature.parameters.len() == 2 { + arguments.push(self.expression(&sources[2], Some(&signature.parameters[1]))?); + } + if matches!( + sequence_value.kind, + SequenceKind::Stream | SequenceKind::AsyncStream | SequenceKind::RowStream + ) && output != Type::Unit + { + return Err(Diagnostic::error( + "C005", + "each on a stream requires a zero-output handler", + sources[0].span, + )); + } + if output == Type::Unit { + Type::Unit + } else if output.value() { + Type::List(Box::new(output)) + } else { + return Err(invalid_handler(sources[0].span)); + } + } + Op::Filter if output == Type::Bool => { + if signature.parameters.len() == 2 { + arguments.push(self.expression(&sources[2], Some(&signature.parameters[1]))?); + } + Type::List(Box::new(sequence_value.element.clone())) + } + Op::Find if output == Type::Bool => { + if signature.parameters.len() == 2 { + arguments.push(self.expression(&sources[2], Some(&signature.parameters[1]))?); + } + sequence_value.element.clone().nullable() + } + Op::Sum if output.numeric() => { + if signature.parameters.len() == 2 { + arguments.push(self.expression(&sources[2], Some(&signature.parameters[1]))?); + } + output + } + Op::Reduce => { + let state = self.expression(&sources[2], Some(&signature.parameters[1]))?; + if signature.outputs.len() != 1 || output != state.ty { + return Err(invalid_handler(sources[0].span)); + } + arguments.push(state); + output + } + Op::ReduceUntil => { + let state = self.expression(&sources[2], Some(&signature.parameters[1]))?; + if signature.outputs.len() != 2 + || signature.outputs[0].ty != state.ty + || signature.outputs[1].name != "stop" + || signature.outputs[1].ty != Type::Bool + { + return Err(Diagnostic::error( + "C005", + "reduce_until handler must output state followed by 'stop: Bool'", + sources[0].span, + )); + } + arguments.push(state); + signature.parameters[1].clone() + } + _ => return Err(invalid_handler(sources[0].span)), + }; + Ok(Value { + kind: ExpressionKind::Collection { + operation, + handler: Some(handler), + sequence: Some(sequence_value.kind), + arguments, + }, + ty, + span, + }) + } + + fn sequence(&mut self, source: &Expression, expected: Option<&Type>) -> Checked<Sequence> { + let hint = expected + .map(|element| Type::List(Box::new(element.clone()))) + .filter(|_| super::expressions::needs_context(source)); + let value = self.expression(source, hint.as_ref())?; + let (kind, element) = match &value.ty { + Type::List(element) => (SequenceKind::List, element.as_ref().clone()), + Type::Bytes => (SequenceKind::Bytes, Type::Int), + Type::Stream(element) => (SequenceKind::Stream, element.as_ref().clone()), + Type::AsyncStream(element) => (SequenceKind::AsyncStream, element.as_ref().clone()), + Type::RowStream(element) => (SequenceKind::RowStream, element.as_ref().clone()), + _ => return Err(expected_sequence(source.span)), + }; + Ok(Sequence { + value, + kind, + element, + }) + } +} + +fn expected_collection(span: Span) -> Diagnostic { + Diagnostic::error("C005", "incorrect collection operand type", span) +} + +fn expected_sequence(span: Span) -> Diagnostic { + Diagnostic::error( + "C005", + "expected List, Bytes, Stream or AsyncStream sequence", + span, + ) +} + +fn invalid_handler(span: Span) -> Diagnostic { + Diagnostic::error( + "C005", + "handler has invalid outputs for this collection operation", + span, + ) +} diff --git a/crates/dever-core/src/check/components.rs b/crates/dever-core/src/check/components.rs new file mode 100644 index 0000000..05f4df8 --- /dev/null +++ b/crates/dever-core/src/check/components.rs @@ -0,0 +1,123 @@ +use std::collections::BTreeSet; + +use crate::diagnostic::Diagnostic; +use crate::hir::Program; +use crate::model::{DatabaseOwner, ModelScope}; +use crate::specialize::{self, Specialization}; + +pub(super) fn register(program: &mut Program, errors: &mut Vec<Diagnostic>) { + let mut tenant_components = program + .models + .iter() + .filter(|model| model.scope == ModelScope::Tenant) + .filter_map(|model| model.package.split('.').next().map(str::to_owned)) + .collect::<BTreeSet<_>>(); + for job in &program.jobs { + if job_uses_tenant_storage(program, job.function) + && let Some(component) = function_component(program, job.function) + { + tenant_components.insert(component.to_owned()); + } + } + program.tenant_components = tenant_components.iter().cloned().collect(); + + let route_components = program + .api_routes + .iter() + .map(|route| reachable_components(program, route.function, &tenant_components)) + .collect::<Vec<_>>(); + for (route, components) in program.api_routes.iter_mut().zip(route_components) { + if route.anonymous && !components.is_empty() { + errors.push(Diagnostic::error( + "C014", + format!( + "public API cannot reach tenant components [{}]; tenant access requires an authenticated API", + components.join(", ") + ), + route.span, + )); + } + route.components = components; + } + + let command_components = program + .api_commands + .iter() + .map(|command| reachable_components(program, command.function, &tenant_components)) + .collect::<Vec<_>>(); + for (command, components) in program.api_commands.iter_mut().zip(command_components) { + command.components = components; + } + + for rest in &mut program.api_rest { + let component = program.models[rest.model] + .package + .split('.') + .next() + .filter(|component| tenant_components.contains(*component)) + .map(str::to_owned); + rest.components = component.into_iter().collect(); + } + + let job_components = program + .jobs + .iter() + .map(|job| reachable_components(program, job.function, &tenant_components)) + .collect::<Vec<_>>(); + for (job, components) in program.jobs.iter_mut().zip(job_components) { + job.components = components; + } +} + +fn reachable_components( + program: &Program, + function: usize, + tenant_components: &BTreeSet<String>, +) -> Vec<String> { + let root = Specialization { + function, + handlers: Vec::new(), + }; + let Ok(reachable) = specialize::execution_reachable(&program.functions, [root]) else { + return Vec::new(); + }; + let mut components = BTreeSet::new(); + for instance in reachable { + if let Some(component) = function_component(program, instance.function) + && tenant_components.contains(component) + { + components.insert(component.to_owned()); + } + for owner in &program.database_effects[instance.function] { + match owner { + DatabaseOwner::Model(model) + if program.models[*model].scope == ModelScope::Tenant => + { + if let Some(component) = program.models[*model].package.split('.').next() { + components.insert(component.to_owned()); + } + } + DatabaseOwner::Job(job) if job_uses_tenant_storage(program, *job) => { + if let Some(component) = function_component(program, *job) { + components.insert(component.to_owned()); + } + } + _ => {} + } + } + } + components.into_iter().collect() +} + +fn job_uses_tenant_storage(program: &Program, function: usize) -> bool { + program.database_effects[function].iter().any(|owner| { + matches!(owner, DatabaseOwner::Model(model) if program.models[*model].scope == ModelScope::Tenant) + }) +} + +fn function_component(program: &Program, function: usize) -> Option<&str> { + program.packages[program.functions[function].owner] + .domain + .as_ref() + .map(|(component, _)| component.as_str()) +} diff --git a/crates/dever-core/src/check/concurrency.rs b/crates/dever-core/src/check/concurrency.rs new file mode 100644 index 0000000..000258c --- /dev/null +++ b/crates/dever-core/src/check/concurrency.rs @@ -0,0 +1,469 @@ +use crate::diagnostic::Diagnostic; +use crate::hir::{CallArgument, CollectionOp, Expression, ExpressionKind, SequenceKind}; +use crate::source::Span; +use crate::syntax::{ContextualExpression, Expression as SourceExpression, ExpressionKind as Ast}; +use crate::types::{Field, Type, output_type}; + +use super::expressions::ungroup; +use super::{Checked, body::Body}; + +impl Body<'_, '_> { + pub(super) fn contextual( + &mut self, + operation: &ContextualExpression, + span: Span, + ) -> Checked<Expression> { + match operation { + ContextualExpression::Channel { element, capacity } => { + let element = self.context.resolve_type(element)?; + super::symbols::validate_type_constraints(&element, self.context.types, span)?; + if !element.transferable(self.context.types) { + return Err(Diagnostic::error( + "C005", + "a Channel item type must be transferable between threads", + span, + )); + } + let capacity = self.capacity( + capacity, + dever_runtime::channel::MAX_CHANNEL_CAPACITY, + "channel capacity", + )?; + Ok(Expression { + kind: ExpressionKind::Channel { + element: element.clone(), + capacity: Box::new(capacity), + }, + ty: Type::Channel(Box::new(element)), + span, + }) + } + } + } + + pub(super) fn named_concurrency( + &mut self, + name: &str, + arguments: &[SourceExpression], + span: Span, + ) -> Option<Checked<Expression>> { + let result = match name { + "run" => self.run(arguments, span), + "wait" => self.wait(arguments, span), + "stop" => self.stop(arguments, span), + "group" => self.group(arguments, span), + "parallel" => self.parallel(arguments, span), + "blocking" => self.blocking(arguments, span), + "send" => self.send(arguments, span), + "receive" => self.receive(arguments, span), + "close" => self.close(arguments, span), + "timeout" => self.timeout(arguments, span), + "race" => self.race(arguments, span), + "stream" => self.stream(arguments, span), + _ => return None, + }; + Some(result) + } + + fn timeout(&mut self, arguments: &[SourceExpression], span: Span) -> Checked<Expression> { + exact_arity("timeout", arguments, 3, span)?; + let (task, Type::Task(outputs)) = self.affine_operand(&arguments[0], true)? else { + return Err(Diagnostic::error("C005", "timeout requires a Task", span)); + }; + let duration = self.expression(&arguments[1], Some(&Type::Int))?; + let handler = self.handler_reference( + &arguments[2], + &crate::types::HandlerSignature { + parameters: vec![], + bounds: vec![], + outputs: outputs.clone(), + }, + )?; + Ok(Expression { + kind: ExpressionKind::Intrinsic { + operation: crate::intrinsic::Intrinsic::TaskTimeout, + handler: Some(handler), + arguments: vec![task, duration], + }, + ty: output_type(&outputs), + span, + }) + } + + fn race(&mut self, sources: &[SourceExpression], span: Span) -> Checked<Expression> { + if sources.len() < 2 { + return Err(Diagnostic::error( + "C005", + "race requires at least two Tasks", + span, + )); + } + let mut arguments = Vec::new(); + let mut signature = None; + for source in sources { + let (task, Type::Task(outputs)) = self.affine_operand(source, true)? else { + return Err(Diagnostic::error( + "C005", + "race requires Tasks", + source.span, + )); + }; + if signature + .as_ref() + .is_some_and(|expected| expected != &outputs) + { + return Err(Diagnostic::error( + "C005", + "race Tasks must have identical output signatures", + source.span, + )); + } + signature = Some(outputs); + arguments.push(task); + } + Ok(Expression { + kind: ExpressionKind::Intrinsic { + operation: crate::intrinsic::Intrinsic::TaskRace, + handler: None, + arguments, + }, + ty: output_type(&signature.expect("at least two tasks")), + span, + }) + } + + fn stream(&mut self, sources: &[SourceExpression], span: Span) -> Checked<Expression> { + exact_arity("stream", sources, 1, span)?; + let value = self.expression(&sources[0], None)?; + let element = match &value.ty { + Type::Channel(element) | Type::List(element) => element.as_ref().clone(), + Type::Bytes => Type::Int, + _ => { + return Err(Diagnostic::error( + "C005", + "stream requires Channel, List or Bytes", + span, + )); + } + }; + if !element.transferable(self.context.types) { + return Err(Diagnostic::error( + "C005", + "async stream elements must be transferable", + span, + )); + } + Ok(Expression { + kind: ExpressionKind::Intrinsic { + operation: crate::intrinsic::Intrinsic::StreamOf, + handler: None, + arguments: vec![value], + }, + ty: Type::AsyncStream(Box::new(element)), + span, + }) + } + + fn wait(&mut self, arguments: &[SourceExpression], span: Span) -> Checked<Expression> { + exact_arity("wait", arguments, 1, span)?; + let (task, ty) = self.affine_operand(&arguments[0], true)?; + match ty { + Type::Task(outputs) => Ok(Expression { + kind: ExpressionKind::AwaitTask(Box::new(task)), + ty: output_type(&outputs), + span, + }), + Type::Group => Ok(Expression { + kind: ExpressionKind::AwaitGroup(Box::new(task)), + ty: Type::Unit, + span, + }), + _ => unreachable!("affine operands are Task or Group"), + } + } + + fn run(&mut self, arguments: &[SourceExpression], span: Span) -> Checked<Expression> { + match arguments { + [call] => { + let call = self.checked_static_call(call)?; + self.require_transferable_call(&call.arguments, &call.outputs, span)?; + let outputs = call.outputs.clone(); + Ok(Expression { + kind: ExpressionKind::RunCall { + group: None, + target: call.target, + arguments: call.arguments, + }, + ty: Type::Task(outputs), + span, + }) + } + [group, call] => { + let (group, ty) = self.affine_operand(group, false)?; + if ty != Type::Group { + return Err(Diagnostic::error( + "C005", + "the first argument to run(group, call) must be a Group local", + arguments[0].span, + )); + } + let call = self.checked_static_call(call)?; + if !call.outputs.is_empty() { + return Err(Diagnostic::error( + "C005", + "a Group accepts only zero-output calls", + arguments[1].span, + )); + } + self.require_transferable_call(&call.arguments, &call.outputs, span)?; + Ok(Expression { + kind: ExpressionKind::RunCall { + group: Some(Box::new(group)), + target: call.target, + arguments: call.arguments, + }, + ty: Type::Unit, + span, + }) + } + _ => Err(Diagnostic::error( + "C005", + "'run' expects a call, optionally preceded by a Group", + span, + )), + } + } + + fn stop(&mut self, arguments: &[SourceExpression], span: Span) -> Checked<Expression> { + exact_arity("stop", arguments, 1, span)?; + let (value, ty) = self.affine_operand(&arguments[0], true)?; + let kind = match ty { + Type::Task(outputs) if outputs.is_empty() => ExpressionKind::StopTask(Box::new(value)), + Type::Task(_) => { + return Err(Diagnostic::error( + "C005", + "stop accepts only a zero-output Task; wait for result-bearing tasks", + arguments[0].span, + )); + } + Type::Group => ExpressionKind::StopGroup(Box::new(value)), + _ => unreachable!("affine operands are Task or Group"), + }; + Ok(Expression { + kind, + ty: Type::Unit, + span, + }) + } + + fn group(&mut self, arguments: &[SourceExpression], span: Span) -> Checked<Expression> { + exact_arity("group", arguments, 1, span)?; + let limit = self.capacity( + &arguments[0], + dever_runtime::task::MAX_GROUP_TASKS, + "group limit", + )?; + Ok(Expression { + kind: ExpressionKind::Group(Box::new(limit)), + ty: Type::Group, + span, + }) + } + + fn parallel(&mut self, arguments: &[SourceExpression], span: Span) -> Checked<Expression> { + exact_arity("parallel", arguments, 1, span)?; + let call = self.checked_static_call(&arguments[0])?; + if !call.pure { + return Err(Diagnostic::error( + "C005", + "parallel requires a pure function call", + arguments[0].span, + )); + } + self.require_transferable_call(&call.arguments, &call.outputs, span)?; + Ok(Expression { + kind: ExpressionKind::ParallelCall { + target: call.target, + arguments: call.arguments, + }, + ty: output_type(&call.outputs), + span, + }) + } + + fn blocking(&mut self, arguments: &[SourceExpression], span: Span) -> Checked<Expression> { + exact_arity("blocking", arguments, 1, span)?; + let call = self.checked_static_call(&arguments[0])?; + self.require_transferable_call(&call.arguments, &call.outputs, span)?; + Ok(Expression { + kind: ExpressionKind::BlockingCall { + target: call.target, + arguments: call.arguments, + }, + ty: output_type(&call.outputs), + span, + }) + } + + fn send(&mut self, arguments: &[SourceExpression], span: Span) -> Checked<Expression> { + exact_arity("send", arguments, 2, span)?; + let channel = self.expression(&arguments[0], None)?; + let Type::Channel(element) = &channel.ty else { + return Err(Diagnostic::error( + "C005", + "send requires a Channel as its first argument", + arguments[0].span, + )); + }; + let value = self.expression(&arguments[1], Some(element))?; + Ok(Expression { + kind: ExpressionKind::ChannelSend { + channel: Box::new(channel), + value: Box::new(value), + }, + ty: Type::Unit, + span, + }) + } + + fn receive(&mut self, arguments: &[SourceExpression], span: Span) -> Checked<Expression> { + exact_arity("receive", arguments, 1, span)?; + let channel = self.expression(&arguments[0], None)?; + let Type::Channel(element) = &channel.ty else { + return Err(Diagnostic::error( + "C005", + "receive requires a Channel value", + arguments[0].span, + )); + }; + let ty = element.as_ref().clone().nullable(); + Ok(Expression { + kind: ExpressionKind::ChannelReceive(Box::new(channel)), + ty, + span, + }) + } + + fn close(&mut self, arguments: &[SourceExpression], span: Span) -> Checked<Expression> { + exact_arity("close", arguments, 1, span)?; + let value = self.expression(&arguments[0], None)?; + let kind = match &value.ty { + Type::Channel(_) => ExpressionKind::ChannelClose(Box::new(value)), + Type::Stream(_) | Type::AsyncStream(_) | Type::RowStream(_) => { + ExpressionKind::Collection { + operation: CollectionOp::Close, + handler: None, + sequence: Some(match value.ty { + Type::AsyncStream(_) => SequenceKind::AsyncStream, + Type::RowStream(_) => SequenceKind::RowStream, + _ => SequenceKind::Stream, + }), + arguments: vec![value], + } + } + _ => { + return Err(Diagnostic::error( + "C005", + "close requires a Stream, AsyncStream or Channel value", + arguments[0].span, + )); + } + }; + Ok(Expression { + kind, + ty: Type::Unit, + span, + }) + } + + fn affine_operand( + &mut self, + source: &SourceExpression, + consume: bool, + ) -> Checked<(Expression, Type)> { + let Ast::Name(path) = &ungroup(source).kind else { + return Err(Diagnostic::error( + "C005", + "Task and Group operations require a direct local name", + source.span, + )); + }; + if path.len() != 1 { + return Err(Diagnostic::error( + "C005", + "Task and Group cannot be stored or projected", + source.span, + )); + } + let (slot, ty) = self.affine_local(&path[0].text, source.span, consume)?; + Ok(( + Expression { + kind: ExpressionKind::Local(slot), + ty: ty.clone(), + span: source.span, + }, + ty, + )) + } + + fn capacity( + &mut self, + source: &SourceExpression, + maximum: usize, + label: &str, + ) -> Checked<Expression> { + let value = self.expression(source, Some(&Type::Int))?; + if let ExpressionKind::Constant(crate::hir::Constant::Int(constant)) = &value.kind + && (*constant <= 0 + || usize::try_from(*constant).map_or(true, |constant| constant > maximum)) + { + return Err(Diagnostic::error( + "C005", + format!("{label} must be between 1 and {maximum}"), + source.span, + )); + } + Ok(value) + } + + fn require_transferable_call( + &self, + arguments: &[CallArgument], + outputs: &[Field], + span: Span, + ) -> Checked<()> { + let arguments_transfer = arguments.iter().all(|argument| match argument { + CallArgument::Value(value) => value.ty.transferable(self.context.types), + CallArgument::Handler(_) => true, + }); + let outputs_transfer = outputs + .iter() + .all(|field| field.ty.transferable(self.context.types)); + if arguments_transfer && outputs_transfer { + Ok(()) + } else { + Err(Diagnostic::error( + "C005", + "task inputs and outputs must be transferable between threads", + span, + )) + } + } +} + +fn exact_arity( + name: &str, + arguments: &[SourceExpression], + expected: usize, + span: Span, +) -> Checked<()> { + if arguments.len() == expected { + Ok(()) + } else { + Err(Diagnostic::error( + "C005", + format!("'{name}' expects {expected} argument(s)"), + span, + )) + } +} diff --git a/crates/dever-core/src/check/constants.rs b/crates/dever-core/src/check/constants.rs new file mode 100644 index 0000000..6c7d33c --- /dev/null +++ b/crates/dever-core/src/check/constants.rs @@ -0,0 +1,148 @@ +use dever_runtime::number::{self, DecimalValue}; + +use crate::diagnostic::Diagnostic; +use crate::hir::{Constant, Expression, ExpressionKind}; +use crate::source::Span; +use crate::syntax::{BinaryOperator as Op, Literal, UnaryOperator}; +use crate::types::Type; + +use super::Checked; + +pub(super) fn literal(value: &Literal, expected: Option<&Type>, span: Span) -> Checked<Expression> { + let (constant, ty) = match value { + Literal::Bool(value) => (Constant::Bool(*value), Type::Bool), + Literal::Text(value) if expected.map(Type::base) == Some(&Type::Uuid) => { + dever_runtime::orm::Uuid::parse(value) + .map_err(|error| Diagnostic::error("C005", error.to_string(), span))?; + (Constant::Text(value.clone()), Type::Uuid) + } + Literal::Text(value) => (Constant::Text(value.clone()), Type::Text), + Literal::Null => { + let Some(ty @ Type::Nullable(_)) = expected else { + return Err(Diagnostic::error( + "C005", + "null needs a nullable type context", + span, + )); + }; + (Constant::Null, ty.clone()) + } + Literal::Number(text) => { + if text.contains(['e', 'E']) + || (text.contains('.') && expected.map(Type::base) == Some(&Type::Float)) + { + let value: f64 = text + .parse() + .map_err(|_| Diagnostic::error("C005", "invalid Float literal", span))?; + if !value.is_finite() { + return Err(Diagnostic::error( + "C005", + "Float literal must be finite", + span, + )); + } + (Constant::Float(value), Type::Float) + } else if text.contains('.') { + ( + Constant::Decimal( + DecimalValue::parse(text) + .map_err(|message| Diagnostic::error("C005", message, span))?, + ), + Type::Decimal, + ) + } else { + ( + Constant::Int(text.parse().map_err(|_| { + Diagnostic::error("C005", "Int literal is outside the 64-bit range", span) + })?), + Type::Int, + ) + } + } + }; + Ok(Expression { + kind: ExpressionKind::Constant(constant), + ty, + span, + }) +} + +pub(super) fn promote(value: Expression) -> Expression { + let span = value.span; + let kind = match value.kind { + ExpressionKind::Constant(Constant::Int(integer)) => { + ExpressionKind::Constant(Constant::Decimal(DecimalValue::from_int(integer))) + } + _ => ExpressionKind::Promote(Box::new(value)), + }; + Expression { + kind, + ty: Type::Decimal, + span, + } +} + +pub(super) fn unary(operator: UnaryOperator, value: &Constant, span: Span) -> Checked<Constant> { + let result = match (operator, value) { + (UnaryOperator::Not, Constant::Bool(value)) => Ok(Constant::Bool(!value)), + (UnaryOperator::Negate, Constant::Int(value)) => number::int_neg(*value).map(Constant::Int), + (UnaryOperator::Negate, Constant::Decimal(value)) => { + value.checked_neg().map(Constant::Decimal) + } + (UnaryOperator::Negate, Constant::Float(value)) => Ok(Constant::Float(-value)), + _ => unreachable!("checked unary operand"), + }; + result.map_err(|message| Diagnostic::error("C008", message, span)) +} + +pub(super) fn binary( + left: &Constant, + operator: Op, + right: &Constant, + span: Span, +) -> Checked<Constant> { + use Constant as C; + let result = match (left, operator, right) { + (left, Op::Equal, right) => Ok(C::Bool(left == right)), + (left, Op::NotEqual, right) => Ok(C::Bool(left != right)), + (C::Bool(left), Op::And, C::Bool(right)) => Ok(C::Bool(*left && *right)), + (C::Bool(left), Op::Or, C::Bool(right)) => Ok(C::Bool(*left || *right)), + (C::Text(left), Op::Add, C::Text(right)) => Ok(C::Text(format!("{left}{right}"))), + (C::Int(left), op, C::Int(right)) => match op { + Op::Add => number::int_add(*left, *right).map(C::Int), + Op::Subtract => number::int_sub(*left, *right).map(C::Int), + Op::Multiply => number::int_mul(*left, *right).map(C::Int), + Op::IntegerDivide => number::int_div(*left, *right).map(C::Int), + Op::Remainder => number::int_rem(*left, *right).map(C::Int), + _ => Ok(C::Bool(compare(left.partial_cmp(right), op))), + }, + (C::Decimal(left), op, C::Decimal(right)) => match op { + Op::Add => left.checked_add(*right).map(C::Decimal), + Op::Subtract => left.checked_sub(*right).map(C::Decimal), + Op::Multiply => left.checked_mul(*right).map(C::Decimal), + Op::Divide => left.checked_div(*right).map(C::Decimal), + _ => Ok(C::Bool(compare(left.partial_cmp(right), op))), + }, + (C::Float(left), op, C::Float(right)) => Ok(match op { + Op::Add => C::Float(left + right), + Op::Subtract => C::Float(left - right), + Op::Multiply => C::Float(left * right), + Op::Divide => C::Float(left / right), + _ => C::Bool(compare(left.partial_cmp(right), op)), + }), + (C::Text(left), op, C::Text(right)) => Ok(C::Bool(compare(left.partial_cmp(right), op))), + _ => unreachable!("checked binary operands"), + }; + result.map_err(|message| Diagnostic::error("C008", message, span)) +} + +fn compare(order: Option<std::cmp::Ordering>, operator: Op) -> bool { + use std::cmp::Ordering::{Equal, Greater, Less}; + match operator { + Op::Less => order == Some(Less), + Op::LessEqual => matches!(order, Some(Less | Equal)), + Op::Greater => order == Some(Greater), + Op::GreaterEqual => matches!(order, Some(Greater | Equal)), + _ => unreachable!("comparison operator"), + } +} diff --git a/crates/dever-core/src/check/cycles.rs b/crates/dever-core/src/check/cycles.rs new file mode 100644 index 0000000..587139d --- /dev/null +++ b/crates/dever-core/src/check/cycles.rs @@ -0,0 +1,253 @@ +use crate::diagnostic::Diagnostic; +use crate::hir::{ + CallArgument, CallTarget, Expression, ExpressionKind, Function, HandlerTarget, Statement, +}; +use crate::source::Span; +use crate::types::{Definition, Parameter}; + +use super::{ + Symbols, + symbols::{referenced_types, shape_fields}, +}; + +pub(super) fn check( + functions: &[Function], + types: &[Definition], + symbols: &Symbols, + errors: &mut Vec<Diagnostic>, +) { + let mut calls = vec![Vec::new(); functions.len()]; + let mut dependencies = vec![Vec::new(); symbols.packages.len()]; + for definition in types { + for field in shape_fields(&definition.shape) { + referenced_types(&field.ty, &mut |id| { + dependency( + &mut dependencies, + definition.owner, + types[id].owner, + definition.span, + symbols, + ) + }); + } + } + for (caller, function) in functions.iter().enumerate() { + for ty in function + .parameters + .iter() + .flat_map(|parameter| match parameter { + Parameter::Value(ty) => vec![ty], + Parameter::Handler(signature) => signature + .parameters + .iter() + .chain(signature.outputs.iter().map(|field| &field.ty)) + .collect(), + }) + .chain(function.outputs.iter().map(|field| &field.ty)) + { + referenced_types(ty, &mut |id| { + dependency( + &mut dependencies, + function.owner, + types[id].owner, + function.span, + symbols, + ) + }); + } + visit(function, |expression| { + referenced_types(&expression.ty, &mut |id| { + dependency( + &mut dependencies, + function.owner, + types[id].owner, + expression.span, + symbols, + ) + }); + let callee = match expression.kind.static_call().map(|(target, _)| target) { + Some(CallTarget::Function(function)) + | Some(CallTarget::Handler(HandlerTarget::Function(function))) => Some(function), + _ => match expression.kind.callback() { + Some(HandlerTarget::Function(function)) => Some(function), + _ => None, + }, + }; + if let Some(callee) = callee { + calls[caller].push((callee, expression.span)); + dependency( + &mut dependencies, + function.owner, + functions[callee].owner, + expression.span, + symbols, + ); + } + if let Some((_, arguments)) = expression.kind.static_call() { + for argument in arguments { + if let CallArgument::Handler(HandlerTarget::Function(target)) = argument { + dependency( + &mut dependencies, + function.owner, + functions[*target].owner, + expression.span, + symbols, + ); + } + } + } + }); + } + for (graph, message) in [ + (&calls, "recursive function calls are not allowed"), + (&dependencies, "cyclic package dependencies are not allowed"), + ] { + if let Some(span) = cycle(graph) { + errors.push(Diagnostic::error("C007", message, span)); + } + } + let roots = functions + .iter() + .enumerate() + .filter(|(_, function)| { + function + .parameters + .iter() + .all(|parameter| matches!(parameter, Parameter::Value(_))) + }) + .map(|(function, _)| crate::specialize::Specialization { + function, + handlers: Vec::new(), + }); + if let Err(span) = crate::specialize::reachable(functions, roots) { + errors.push(Diagnostic::error( + "C007", + "recursive specialized handler calls are not allowed", + span, + )); + } +} + +// Explicit DFS frames keep long acyclic call chains off the compiler's call stack. +pub(super) fn cycle(graph: &[Vec<(usize, Span)>]) -> Option<Span> { + let mut state = vec![0_u8; graph.len()]; + for root in 0..graph.len() { + if state[root] != 0 { + continue; + } + state[root] = 1; + let mut pending = vec![(root, 0)]; + while let Some((node, edge)) = pending.last_mut() { + if *edge == graph[*node].len() { + state[*node] = 2; + pending.pop(); + continue; + } + let (next, span) = graph[*node][*edge]; + *edge += 1; + match state[next] { + 1 => return Some(span), + 0 => { + state[next] = 1; + pending.push((next, 0)); + } + _ => {} + } + } + } + None +} + +fn dependency( + graph: &mut [Vec<(usize, Span)>], + owner: usize, + target: usize, + span: Span, + symbols: &Symbols, +) { + let same_domain = symbols.packages[owner].layout.domain().is_some() + && symbols.packages[owner].layout.domain() == symbols.packages[target].layout.domain(); + if owner != target && !same_domain { + graph[owner].push((target, span)); + } +} + +pub(crate) fn visit(function: &Function, mut apply: impl FnMut(&Expression)) { + let mut pending = Vec::new(); + for clause in &function.clauses { + for statement in &clause.body { + pending.push(match statement { + Statement::Assign { value, .. } | Statement::Call(value) => value, + }); + } + } + visit_pending(pending, &mut apply); +} + +pub(crate) fn visit_expression(expression: &Expression, mut apply: impl FnMut(&Expression)) { + visit_pending(vec![expression], &mut apply); +} + +fn visit_pending<'a>(mut pending: Vec<&'a Expression>, apply: &mut impl FnMut(&'a Expression)) { + while let Some(expression) = pending.pop() { + apply(expression); + match &expression.kind { + ExpressionKind::Field { value, .. } + | ExpressionKind::Fail(value) + | ExpressionKind::Unary { value, .. } + | ExpressionKind::Promote(value) + | ExpressionKind::Some(value) + | ExpressionKind::AwaitTask(value) + | ExpressionKind::StopTask(value) + | ExpressionKind::Group(value) + | ExpressionKind::AwaitGroup(value) + | ExpressionKind::StopGroup(value) + | ExpressionKind::ChannelReceive(value) + | ExpressionKind::ChannelClose(value) => pending.push(value), + ExpressionKind::Call { arguments, .. } + | ExpressionKind::CaptureResult { arguments, .. } + | ExpressionKind::ParallelCall { arguments, .. } + | ExpressionKind::BlockingCall { arguments, .. } => { + pending.extend(arguments.iter().filter_map(|argument| match argument { + CallArgument::Value(value) => Some(value), + CallArgument::Handler(_) => None, + })); + } + ExpressionKind::RunCall { + group, arguments, .. + } => { + pending.extend(group.iter().map(Box::as_ref)); + pending.extend(arguments.iter().filter_map(|argument| match argument { + CallArgument::Value(value) => Some(value), + CallArgument::Handler(_) => None, + })); + } + ExpressionKind::ModelOperation { operation, .. } => { + pending.extend(operation.expressions()); + } + ExpressionKind::Channel { capacity, .. } => pending.push(capacity), + ExpressionKind::ChannelSend { channel, value } => { + pending.extend([channel.as_ref(), value.as_ref()]); + } + ExpressionKind::Intrinsic { arguments, .. } + | ExpressionKind::JobEnqueue { arguments, .. } + | ExpressionKind::Collection { arguments, .. } + | ExpressionKind::Variant { arguments, .. } + | ExpressionKind::List(arguments) => pending.extend(arguments), + ExpressionKind::Record { fields } => { + pending.extend(fields.iter().map(|(_, value)| value)); + } + ExpressionKind::Map(entries) => { + for (key, value) in entries { + pending.extend([key, value]); + } + } + ExpressionKind::Binary { left, right, .. } => { + pending.extend([left.as_ref(), right.as_ref()]); + } + ExpressionKind::Constant(_) + | ExpressionKind::Local(_) + | ExpressionKind::ChoiceOptions { .. } => {} + } + } +} diff --git a/crates/dever-core/src/check/expressions.rs b/crates/dever-core/src/check/expressions.rs new file mode 100644 index 0000000..9ede6f9 --- /dev/null +++ b/crates/dever-core/src/check/expressions.rs @@ -0,0 +1,890 @@ +use crate::diagnostic::Diagnostic; +use crate::hir::{ + CallArgument, CallTarget, Expression as Value, ExpressionKind as Expr, HandlerTarget, +}; +use crate::intrinsic::Intrinsic; +use crate::source::Span; +use crate::syntax::{self, Expression, ExpressionKind as Ast, Literal}; +use crate::types::{Field, HandlerSignature, Parameter, Shape, Type, output_type}; + +use super::{Checked, body::Body, constants, path_name}; + +pub(super) struct CheckedCall { + pub target: CallTarget, + pub arguments: Vec<CallArgument>, + pub outputs: Vec<Field>, + pub pure: bool, +} + +impl Body<'_, '_> { + pub(super) fn expression( + &mut self, + source: &Expression, + expected: Option<&Type>, + ) -> Checked<Value> { + let mut value = match &source.kind { + Ast::Literal(literal) => constants::literal(literal, expected, source.span)?, + Ast::Group(inner) => self.expression(inner, expected)?, + Ast::Name(path) => self.name(path, source.span)?, + Ast::Field { value, name } => { + if name.text == "options" { + if let Ast::Name(path) = &value.kind { + if let Some(choice) = + self.context.model_choice(&path_name(path), value.span)? + { + Value { + kind: Expr::ChoiceOptions { choice }, + ty: Type::Map(Box::new(Type::Named(choice)), Box::new(Type::Text)), + span: source.span, + } + } else { + let value = self.expression(value, None)?; + self.field(value, name)? + } + } else { + let value = self.expression(value, None)?; + self.field(value, name)? + } + } else { + let value = self.expression(value, None)?; + self.field(value, name)? + } + } + Ast::Unary { operator, value } => { + self.unary(*operator, value, expected, source.span)? + } + Ast::Binary { + left, + operator, + right, + } => self.binary(left, *operator, right, expected, source.span)?, + Ast::Record { name, fields } => self.record(name, fields, source.span)?, + Ast::List(elements) => self.list(elements, expected, source.span)?, + Ast::Map(entries) => self.map(entries, expected, source.span)?, + Ast::Call { + function, + arguments, + } => self.call(function, arguments, expected, source.span)?, + Ast::Fail(error) => self.fail(error, source.span)?, + Ast::CaptureResult(call) => self.capture_result(call, expected, source.span)?, + Ast::Contextual(operation) => self.contextual(operation, source.span)?, + }; + value.span = source.span; + if nested_affine(&value.ty) { + return Err(Diagnostic::error( + "C005", + "Task and Group values cannot be stored in another value", + source.span, + )); + } + match expected { + Some(expected) => self.coerce(value, expected), + None => Ok(value), + } + } + + pub(super) fn coerce(&self, value: Value, expected: &Type) -> Checked<Value> { + if &value.ty == expected { + return Ok(value); + } + if let Type::Nullable(base) = expected + && !matches!(value.ty, Type::Nullable(_)) + { + let inner = self.coerce(value, base)?; + return Ok(Value { + span: inner.span, + ty: expected.clone(), + kind: Expr::Some(Box::new(inner)), + }); + } + if value.ty == Type::Int && expected == &Type::Decimal { + return Ok(constants::promote(value)); + } + Err(Diagnostic::error( + "C005", + format!( + "expected {}, found {}", + expected.label(self.context.types), + value.ty.label(self.context.types) + ), + value.span, + )) + } + + fn name(&mut self, path: &syntax::Path, span: Span) -> Checked<Value> { + if path[0].text == "setting" + && self.context.symbols.packages[self.context.owner] + .layout + .role() + == Some(crate::source::SourceRole::Adapter) + { + let (id, function) = self + .context + .functions + .iter() + .enumerate() + .find(|(_, function)| { + function.owner == self.context.owner && function.setting.is_some() + }) + .ok_or_else(|| { + Diagnostic::error( + "C006", + "setting is available only in its declaring Adapter", + span, + ) + })?; + let mut value = Value { + kind: Expr::Call { + target: CallTarget::Function(id), + arguments: Vec::new(), + }, + ty: Type::Named(function.setting.unwrap()), + span, + }; + for name in &path[1..] { + value = self.field(value, name)?; + } + return Ok(value); + } + if path.len() > 1 && path.last().is_some_and(|name| name.text == "options") { + let choice_name = path_name(&path[..path.len() - 1]); + if let Some(choice) = self.context.model_choice(&choice_name, span)? { + return Ok(self.choice_options(choice, span)); + } + } + if path.len() == 1 && self.handlers.contains_key(&path[0].text) { + return Err(Diagnostic::error( + "C005", + "handler parameters are not runtime values", + span, + )); + } + if path.len() > 1 + && let Some((id, variant)) = self.context.variant(&path_name(path), span)? + { + return self.construct_variant(id, variant, &[], span); + } + if self.locals.contains_key(&path[0].text) { + let local = self.read_local(&path[0].text, span)?; + let mut value = Value { + kind: local.known.clone().unwrap_or(Expr::Local(local.slot)), + ty: local.ty.clone(), + span, + }; + for name in &path[1..] { + value = self.field(value, name)?; + } + return Ok(value); + } + if let Some((id, variant)) = self.context.variant(&path_name(path), span)? { + return self.construct_variant(id, variant, &[], span); + } + Err(Diagnostic::error( + "C004", + format!("unknown or uninitialized local '{}'", path_name(path)), + span, + )) + } + + fn choice_options(&self, choice: usize, span: Span) -> Value { + Value { + kind: Expr::ChoiceOptions { choice }, + ty: Type::Map(Box::new(Type::Named(choice)), Box::new(Type::Text)), + span, + } + } + + pub(super) fn field_type(&self, ty: &Type, name: &syntax::Name) -> Checked<(usize, Type)> { + if let Type::Named(id) = ty + && !self.context.owns_private_fields(*id) + && let Shape::Record(fields) = &self.context.types[*id].shape + && fields + .iter() + .any(|field| field.name == name.text && field.private) + { + return Err(Diagnostic::error( + "C006", + "private field is accessible only to its owner (Model: owning domain App)", + name.span, + )); + } + let fields = self.context.value_fields(ty).ok_or_else(|| { + Diagnostic::error( + "C005", + "field access requires an initialized record, MapEntry or named outputs", + name.span, + ) + })?; + fields + .into_iter() + .enumerate() + .find(|(_, field)| field.name == name.text) + .map(|(index, field)| (index, field.ty)) + .ok_or_else(|| { + Diagnostic::error("C004", format!("unknown field '{}'", name.text), name.span) + }) + } + + fn field(&self, value: Value, name: &syntax::Name) -> Checked<Value> { + let (index, ty) = self.field_type(&value.ty, name)?; + Ok(Value { + kind: Expr::Field { + value: Box::new(value), + index, + }, + ty, + span: name.span, + }) + } + + fn call( + &mut self, + function: &Expression, + arguments: &[Expression], + expected: Option<&Type>, + span: Span, + ) -> Checked<Value> { + let Ast::Name(path) = &ungroup(function).kind else { + return Err(Diagnostic::error( + "C005", + "only named functions are callable", + function.span, + )); + }; + let name = path_name(path); + if let Some(value) = self.upload_call(&name, arguments, span) { + return value; + } + if matches!(name.as_str(), "dever.job.enqueue" | "dever.job.enqueue_at") { + return self.enqueue_job(arguments, name.ends_with("_at"), span); + } + if self.context.is_test() + && path.len() == 1 + && let Some(assertion) = self.test_assertion(&name, arguments, span)? + { + return Ok(assertion); + } + if path.len() == 1 + && let Some(handler) = self.handlers.get(&path[0].text) + { + let parameter = handler.parameter; + let signature = handler.signature.clone(); + let checked = self.handler_call(parameter, &signature, arguments, span)?; + return Ok(Value { + kind: Expr::Call { + target: checked.target, + arguments: checked.arguments, + }, + ty: output_type(&checked.outputs), + span, + }); + } + if let Some((id, variant)) = self.context.variant(&name, function.span)? { + return self.construct_variant(id, variant, arguments, span); + } + if let Some(value) = self.model_call(&name, arguments, span)? { + return Ok(value); + } + if path.len() == 1 && self.locals.contains_key(&path[0].text) { + return Err(Diagnostic::error( + "C005", + "local values are not callable", + function.span, + )); + } + let local_function = self.context.has_function(&name, arguments.len()); + if !local_function { + if let Some(result) = self.named_concurrency(&name, arguments, span) { + if self.context.symbols.packages[self.context.owner] + .layout + .role() + == Some(crate::source::SourceRole::Job) + { + return Err(Diagnostic::error( + "C006", + "Job entries call only their same-domain App", + span, + )); + } + return result; + } + if let Some(operation) = super::collections::operation(&name) { + if self.context.symbols.packages[self.context.owner] + .layout + .role() + == Some(crate::source::SourceRole::Job) + { + return Err(Diagnostic::error( + "C006", + "Job entries call only their same-domain App", + span, + )); + } + return self.collection(operation, arguments, expected, span); + } + } + if let Some(operation) = Intrinsic::from_name(&name) { + if matches!(operation, Intrinsic::AuthOwnsUser) { + let [argument] = arguments else { + return Err(Diagnostic::error( + "C005", + "dever.auth.owns_user expects one Model id", + span, + )); + }; + let argument = self.expression(argument, None)?; + let model_id = matches!(argument.ty, Type::Named(id) + if self.context.types[id].kind == crate::types::DefinitionKind::ModelId); + if !model_id { + return Err(Diagnostic::error( + "C005", + "dever.auth.owns_user expects one Model id", + argument.span, + )); + } + return Ok(Value { + kind: Expr::Intrinsic { + operation, + handler: None, + arguments: vec![argument], + }, + ty: Type::Bool, + span, + }); + } + if self.context.symbols.packages[self.context.owner] + .layout + .role() + == Some(crate::source::SourceRole::Job) + { + return Err(Diagnostic::error( + "C006", + "Job entries call only their same-domain App", + span, + )); + } + let mut signature = super::intrinsics::signature(operation, self.context, span)?; + let shared_context = if matches!( + operation, + Intrinsic::HttpServe + | Intrinsic::HttpServeLive + | Intrinsic::HttpServeTls + | Intrinsic::HttpServeLiveTls + ) && arguments.len() == signature.parameters.len() + 2 + { + let context = self.expression(arguments.last().unwrap(), None)?; + if !context.ty.transferable(self.context.types) { + return Err(Diagnostic::error( + "C005", + "HTTP handler context must be transferable", + context.span, + )); + } + let handler = signature.handler.as_mut().expect("HTTP handler"); + handler.parameters.push(context.ty.clone()); + handler.bounds.push(Default::default()); + Some(context) + } else { + None + }; + let count = signature.parameters.len() + usize::from(signature.handler.is_some()); + if count + usize::from(shared_context.is_some()) != arguments.len() { + return Err(Diagnostic::error( + "C005", + format!("'{name}' expects {count} argument(s)"), + span, + )); + } + let (handler, arguments) = match &signature.handler { + Some(expected) => ( + Some(self.handler_reference(&arguments[0], expected)?), + &arguments[1..], + ), + None => (None, arguments), + }; + let mut arguments: Vec<_> = arguments + .iter() + .zip(signature.parameters) + .map(|(source, ty)| self.expression(source, Some(&ty))) + .collect::<Checked<_>>()?; + arguments.extend(shared_context); + return Ok(Value { + kind: Expr::Intrinsic { + operation, + handler, + arguments, + }, + ty: signature.output, + span, + }); + } + let id = self + .context + .function(&name, arguments.len(), function.span)?; + // The standard getter retains absence; only an explicit nominal context + // selects the trusted ID representation. Route/provider validation follows. + if let Some(Type::Nullable(inner)) = expected + && let Type::Named(model_id) = inner.as_ref() + && self.context.types[*model_id].kind == crate::types::DefinitionKind::ModelId + && let Some(operation) = match name.as_str() { + "dever.auth.user_id" => Some(Intrinsic::AuthUserId), + "dever.auth.tenant_id" => Some(Intrinsic::AuthTenantId), + _ => None, + } + { + return Ok(Value { + kind: Expr::Intrinsic { + operation, + handler: None, + arguments: Vec::new(), + }, + ty: expected.expect("matched explicit nullable type").clone(), + span, + }); + } + let checked = self.function_call(id, arguments)?; + Ok(Value { + kind: Expr::Call { + target: checked.target, + arguments: checked.arguments, + }, + ty: output_type(&checked.outputs), + span, + }) + } + + fn test_assertion( + &mut self, + name: &str, + arguments: &[Expression], + span: Span, + ) -> Checked<Option<Value>> { + let mut output = Type::Unit; + let (operation, checked) = match name { + "secret" => { + let [input] = arguments else { + return Err(Diagnostic::error( + "C005", + "secret expects one Text argument", + span, + )); + }; + output = Type::Secret; + ( + Intrinsic::TestSecret, + vec![self.expression(input, Some(&Type::Text))?], + ) + } + "assert" => { + let [condition] = arguments else { + return Err(Diagnostic::error( + "C005", + "'assert' expects one Bool argument", + span, + )); + }; + ( + Intrinsic::TestAssert, + vec![self.expression(condition, Some(&Type::Bool))?], + ) + } + "assert_eq" => { + let [actual, expected] = arguments else { + return Err(Diagnostic::error( + "C005", + "'assert_eq' expects two arguments", + span, + )); + }; + let actual = self.expression(actual, None)?; + let expected = + self.expression(expected, needs_context(expected).then_some(&actual.ty))?; + if actual.ty != expected.ty { + return Err(Diagnostic::error( + "C005", + format!( + "assert_eq requires the same type on both sides; found {} and {}", + actual.ty.label(self.context.types), + expected.ty.label(self.context.types) + ), + span, + )); + } + if !actual.ty.comparable(self.context.types) + || !actual.ty.observable(self.context.types) + { + return Err(Diagnostic::error( + "C005", + format!( + "assert_eq does not support {}", + actual.ty.label(self.context.types) + ), + span, + )); + } + (Intrinsic::TestAssertEq, vec![actual, expected]) + } + _ => return Ok(None), + }; + Ok(Some(Value { + kind: Expr::Intrinsic { + operation, + handler: None, + arguments: checked, + }, + ty: output, + span, + })) + } + + fn fail(&mut self, source: &Expression, span: Span) -> Checked<Value> { + let source = ungroup(source); + let (path, arguments) = match &source.kind { + Ast::Name(path) => (path, &[][..]), + Ast::Call { + function, + arguments, + } => { + let Ast::Name(path) = &ungroup(function).kind else { + return Err(Diagnostic::error( + "C005", + "fail requires a direct error choice variant", + source.span, + )); + }; + (path, arguments.as_slice()) + } + _ => { + return Err(Diagnostic::error( + "C005", + "fail requires a direct error choice variant", + source.span, + )); + } + }; + let Some((id, variant)) = self.context.variant(&path_name(path), source.span)? else { + return Err(Diagnostic::error( + "C005", + "fail requires a qualified error choice variant", + source.span, + )); + }; + let error = self.construct_failure_variant(id, variant, arguments, source.span)?; + Ok(Value { + kind: Expr::Fail(Box::new(error)), + ty: Type::Unit, + span, + }) + } + + fn capture_result( + &mut self, + source: &Expression, + expected: Option<&Type>, + span: Span, + ) -> Checked<Value> { + let Some(Type::Named(choice)) = expected.map(Type::base) else { + return Err(Diagnostic::error( + "C005", + "result(call) requires an expected choice type", + span, + )); + }; + let Shape::Choice(variants) = &self.context.types[*choice].shape else { + return Err(Diagnostic::error( + "C005", + "result(call) requires an expected choice type", + span, + )); + }; + let success_variants = variants + .iter() + .enumerate() + .filter(|(_, variant)| !variant.error) + .collect::<Vec<_>>(); + let [(success, success_variant)] = success_variants.as_slice() else { + return Err(Diagnostic::error( + "C005", + "a result capture choice must declare exactly one success variant", + self.context.types[*choice].span, + )); + }; + let checked = self.checked_static_call(source)?; + let outputs_match = checked.outputs.len() == success_variant.fields.len() + && checked + .outputs + .iter() + .zip(&success_variant.fields) + .all(|(output, field)| output.ty == field.ty); + if !outputs_match { + return Err(Diagnostic::error( + "C005", + "result capture success payload must match the called function outputs", + span, + )); + } + Ok(Value { + kind: Expr::CaptureResult { + target: checked.target, + arguments: checked.arguments, + choice: *choice, + success: *success, + }, + ty: Type::Named(*choice), + span, + }) + } + + pub(super) fn checked_static_call(&mut self, source: &Expression) -> Checked<CheckedCall> { + let Ast::Call { + function, + arguments, + } = &ungroup(source).kind + else { + return Err(Diagnostic::error( + "C005", + "this operation requires a direct named function call", + source.span, + )); + }; + let Ast::Name(path) = &ungroup(function).kind else { + return Err(Diagnostic::error( + "C005", + "only named functions are callable", + function.span, + )); + }; + if path.len() == 1 { + if let Some(handler) = self.handlers.get(&path[0].text) { + return self.handler_call( + handler.parameter, + &handler.signature.clone(), + arguments, + source.span, + ); + } + if path.len() == 1 && self.locals.contains_key(&path[0].text) { + return Err(Diagnostic::error( + "C005", + "local values are not callable", + function.span, + )); + } + } + let id = self + .context + .function(&path_name(path), arguments.len(), function.span)?; + self.function_call(id, arguments) + } + + fn handler_call( + &mut self, + parameter: usize, + signature: &HandlerSignature, + arguments: &[Expression], + span: Span, + ) -> Checked<CheckedCall> { + if signature.parameters.len() != arguments.len() { + return Err(Diagnostic::error( + "C005", + format!("handler expects {} argument(s)", signature.parameters.len()), + span, + )); + } + let arguments = arguments + .iter() + .zip(&signature.parameters) + .map(|(argument, ty)| self.expression(argument, Some(ty)).map(CallArgument::Value)) + .collect::<Checked<_>>()?; + Ok(CheckedCall { + target: CallTarget::Handler(HandlerTarget::Parameter(parameter)), + arguments, + outputs: signature.outputs.clone(), + pure: false, + }) + } + + fn function_call(&mut self, id: usize, arguments: &[Expression]) -> Checked<CheckedCall> { + let function = &self.context.functions[id]; + let parameters = function.parameters.clone(); + let storage_sink = function.name == "dever.storage.put"; + if storage_sink + && self.context.symbols.packages[self.context.owner] + .layout + .role() + != Some(crate::source::SourceRole::Adapter) + { + return Err(Diagnostic::error( + "C006", + "storage.put must be called by a storage Adapter through its Port", + function.span, + )); + } + let upload_transfer = function.port.is_some() || storage_sink; + let checked_arguments = arguments + .iter() + .zip(¶meters) + .map(|(argument, parameter)| match parameter { + Parameter::Value(ty) => { + if ty == &Type::Upload { + if !upload_transfer { return Err(Diagnostic::error("C006", "Upload ownership may transfer only through a Port to its storage Adapter", argument.span)); } + let Ast::Name(path) = &argument.kind else { return Err(Diagnostic::error("C005", "Upload transfer requires a single local", argument.span)); }; + if path.len() != 1 { return Err(Diagnostic::error("C005", "Upload transfer requires a single local", argument.span)); } + let (slot, actual) = self.affine_local(&path[0].text, argument.span, true)?; + if actual != Type::Upload { return Err(Diagnostic::error("C005", "expected Upload local", argument.span)); } + return Ok(CallArgument::Value(Value { kind: Expr::Local(slot), ty: actual, span: argument.span })); + } + self.expression(argument, Some(ty)).map(CallArgument::Value) + } + Parameter::Handler(expected) => self + .handler_reference(argument, expected) + .map(CallArgument::Handler), + }) + .collect::<Checked<_>>()?; + Ok(CheckedCall { + target: CallTarget::Function(id), + arguments: checked_arguments, + outputs: function.outputs.clone(), + pure: function.pure, + }) + } + + pub(super) fn handler_reference( + &self, + source: &Expression, + expected: &HandlerSignature, + ) -> Checked<HandlerTarget> { + let (target, actual) = self.resolve_handler(source, expected.parameters.len())?; + let inputs_match = actual.parameters == expected.parameters + && expected + .bounds + .iter() + .zip(&actual.bounds) + .all(|(required, accepted)| crate::contracts::bounds_subset(required, accepted)); + let outputs_match = actual.outputs.len() == expected.outputs.len() + && actual + .outputs + .iter() + .zip(&expected.outputs) + .all(|(actual, expected)| { + actual.name == expected.name + && actual.ty == expected.ty + && (matches!(target, HandlerTarget::Function(_)) + || crate::contracts::bounds_subset(&actual.bounds, &expected.bounds)) + }); + // Concrete output guarantees are inferred from checked bodies by the + // contract pass. Forwarded handlers have only their declared guarantees. + if !inputs_match || !outputs_match { + return Err(Diagnostic::error( + "C005", + "handler signature does not match the required signature", + source.span, + )); + } + Ok(target) + } + + pub(super) fn resolve_handler( + &self, + source: &Expression, + arity: usize, + ) -> Checked<(HandlerTarget, HandlerSignature)> { + let Ast::Name(path) = &ungroup(source).kind else { + return Err(Diagnostic::error( + "C005", + "handler must be a named function or handler parameter", + source.span, + )); + }; + if path.len() == 1 { + if let Some(handler) = self.handlers.get(&path[0].text) { + if handler.signature.parameters.len() != arity { + return Err(Diagnostic::error( + "C005", + format!( + "handler expects {} argument(s)", + handler.signature.parameters.len() + ), + source.span, + )); + } + return Ok(( + HandlerTarget::Parameter(handler.parameter), + handler.signature.clone(), + )); + } + if path.len() == 1 && self.locals.contains_key(&path[0].text) { + return Err(Diagnostic::error( + "C005", + "local values cannot be handlers", + source.span, + )); + } + } + let id = self + .context + .function(&path_name(path), arity, source.span)?; + let function = &self.context.functions[id]; + let parameters = function + .parameters + .iter() + .map(|parameter| match parameter { + Parameter::Value(ty) => Ok(ty.clone()), + Parameter::Handler(_) => Err(Diagnostic::error( + "C005", + "a handler target cannot itself require handler inputs", + source.span, + )), + }) + .collect::<Checked<Vec<_>>>()?; + let signature = HandlerSignature { + bounds: vec![Vec::new(); parameters.len()], + parameters, + outputs: function.outputs.clone(), + }; + Ok((HandlerTarget::Function(id), signature)) + } +} + +fn nested_affine(ty: &Type) -> bool { + match ty { + Type::Task(_) | Type::Group | Type::Upload => false, + Type::Nullable(inner) + | Type::List(inner) + | Type::Stream(inner) + | Type::AsyncStream(inner) + | Type::RowStream(inner) + | Type::Channel(inner) => contains_affine(inner), + Type::Map(key, value) | Type::MapEntry(key, value) => { + contains_affine(key) || contains_affine(value) + } + Type::Outputs(fields) => fields.iter().any(|field| contains_affine(&field.ty)), + _ => false, + } +} + +fn contains_affine(ty: &Type) -> bool { + matches!(ty, Type::Task(_) | Type::Group | Type::Upload) || nested_affine(ty) +} + +pub(super) fn needs_context(expression: &Expression) -> bool { + match &expression.kind { + Ast::Literal(Literal::Null) => true, + Ast::List(elements) => elements.iter().all(needs_context), + Ast::Map(entries) => entries.is_empty(), + Ast::Group(inner) => needs_context(inner), + Ast::CaptureResult(_) => true, + _ => false, + } +} + +pub(super) fn ungroup(mut expression: &Expression) -> &Expression { + while let Ast::Group(inner) = &expression.kind { + expression = inner; + } + expression +} + +pub(super) fn is_null(expression: &Expression) -> bool { + matches!(ungroup(expression).kind, Ast::Literal(Literal::Null)) +} diff --git a/crates/dever-core/src/check/intrinsics.rs b/crates/dever-core/src/check/intrinsics.rs new file mode 100644 index 0000000..e62f39e --- /dev/null +++ b/crates/dever-core/src/check/intrinsics.rs @@ -0,0 +1,640 @@ +use crate::diagnostic::Diagnostic; +use crate::intrinsic::Intrinsic; +use crate::source::Span; +use crate::types::{Field, HandlerSignature, Shape, Type}; + +use super::{Checked, Context}; + +pub(super) struct Signature { + pub handler: Option<HandlerSignature>, + pub parameters: Vec<Type>, + pub output: Type, +} + +pub(super) fn signature( + operation: Intrinsic, + context: &Context<'_>, + span: Span, +) -> Checked<Signature> { + use Intrinsic::*; + use Type::*; + let mut handler = None; + let result = |name: &str, success: &str, payload: &[Type], eof: bool| -> Checked<Type> { + let id = context.named_type(name, span)?; + let Shape::Choice(variants) = &context.types[id].shape else { + return Err(Diagnostic::error( + "C005", + "system result must be a choice type", + span, + )); + }; + let mut expected = vec![(success, payload.to_vec()), ("Failed", vec![Text])]; + if eof { + expected.push(("End", vec![])); + } + if variants.len() != expected.len() + || !expected.iter().all(|(name, payload)| { + variants.iter().any(|variant| { + variant.name == *name + && variant + .fields + .iter() + .map(|field| &field.ty) + .eq(payload.iter()) + }) + }) + { + return Err(Diagnostic::error( + "C005", + "official result type does not match its system primitive contract", + span, + )); + } + Ok(Named(id)) + }; + let (parameters, output) = match operation { + TestAssert | TestAssertEq | TestSecret | AuthOwnsUser => { + unreachable!("polymorphic primitives are checked contextually") + } + CryptoToken => ( + vec![Int], + result("dever.crypto.TokenResult", "Ready", &[Secret], false)?, + ), + CryptoPasswordHash => ( + vec![Secret], + result("dever.crypto.HashResult", "Ready", &[Text], false)?, + ), + CryptoPasswordVerify => ( + vec![Secret, Text], + result("dever.crypto.VerifyResult", "Ready", &[Bool], false)?, + ), + CryptoSha256 => (vec![Bytes], Bytes), + CryptoHmacSha256 => (vec![Secret, Bytes], Bytes), + CryptoConstantTimeEq => (vec![Bytes, Bytes], Bool), + ApiServe => (vec![], Unit), + UploadFilename | UploadContentType => (vec![Upload], Text), + UploadSize => (vec![Upload], Int), + UploadClose => (vec![Upload], Unit), + UploadStore => ( + vec![Upload], + result("dever.storage.PutResult", "Stored", &[Uuid], false)?, + ), + ApiRequestId | ApiMethod | ApiPath => (vec![], Text), + ApiClientAddress => (vec![], Text.nullable()), + ApiHeader | ApiCookie => (vec![Text], Text.nullable()), + ApiSecretCookie => (vec![Text], Secret.nullable()), + ApiResponseHeader => (vec![Text, Text], Unit), + ApiResponseCookie | ApiResponseSecretCookie => { + let options = context.named_type("dever.api.CookieOptions", span)?; + let value = if matches!(operation, ApiResponseCookie) { + Text + } else { + Secret + }; + (vec![Text, value, Named(options)], Unit) + } + AuthIssue => (vec![Text, Text, Text.nullable()], Secret), + AuthIssueCookie => (vec![Text, Text, Text.nullable()], Unit), + AuthClearCookie => (vec![], Unit), + AuthId | AuthSession | SiteKey => (vec![], Text), + AuthUserId | AuthTenantId => (vec![], Int.nullable()), + AuthPermissions => { + let permission = context.named_type("dever.auth.Permission", span)?; + (vec![], List(Box::new(Named(permission)))) + } + AuthSaveRole => (vec![Text, Text, Bool, List(Box::new(Text))], Unit), + AuthGrantRole | AuthRevokeRole => (vec![Int, Text], Unit), + AuthDisableRole => (vec![Text], Unit), + JobServe => { + if !context.symbols.packages[context.owner].bundled + && !matches!( + context.symbols.packages[context.owner].layout, + crate::source::SourceLayout::Main + ) + { + return Err(Diagnostic::error( + "C006", + "only main may start the Job worker", + span, + )); + } + (vec![], Unit) + } + TestJobDrain | TestClockAdvance => { + if !context.is_test() { + return Err(Diagnostic::error( + "C006", + "Job test controls are available only in test sources", + span, + )); + } + ( + vec![Int], + if matches!(operation, TestJobDrain) { + Int + } else { + Unit + }, + ) + } + TaskTimeout | TaskRace | StreamOf => unreachable!("polymorphic core operation"), + TaskTicks => (vec![Int], AsyncStream(Box::new(Int))), + TextTrim | TextLower | TextUpper => (vec![Text], Text), + TextCodepoint => (vec![Text], Int.nullable()), + TextFromCodepoint => (vec![Int], Text.nullable()), + TextAt => (vec![Text, Int], Text.nullable()), + TextSlice => (vec![Text, Int, Int], Text.nullable()), + TextIndexOf => (vec![Text, Text], Int.nullable()), + TextContains | TextStartsWith | TextEndsWith => (vec![Text, Text], Bool), + TextSplit => (vec![Text, Text], List(Box::new(Text))), + TextReplace => (vec![Text, Text, Text], Text), + IntParse => (vec![Text], Int.nullable()), + IntToText => (vec![Int], Text), + DecimalParse => (vec![Text], Decimal.nullable()), + DecimalToText => (vec![Decimal], Text), + DecimalFromInt => (vec![Int], Decimal), + FloatParse => (vec![Text], Float.nullable()), + FloatToText => (vec![Float], Text), + ProcessArguments => ( + vec![], + result( + "dever.process.ArgumentsResult", + "Read", + &[List(Box::new(Text))], + false, + )?, + ), + TimeUnixMillis | TimeMonotonicNanos => ( + vec![], + result("dever.time.ClockResult", "Read", &[Int], false)?, + ), + TimeNow => ( + vec![], + result("dever.time.DateTimeResult", "Read", &[DateTime], false)?, + ), + TimeParseDateTime => ( + vec![Text], + result("dever.time.DateTimeResult", "Read", &[DateTime], false)?, + ), + TimeParseDate => ( + vec![Text], + result("dever.time.DateResult", "Read", &[Date], false)?, + ), + TimeParseTime => ( + vec![Text], + result("dever.time.TimeResult", "Read", &[Time], false)?, + ), + TimeFormatDateTime => ( + vec![DateTime], + result("dever.time.TextResult", "Read", &[Text], false)?, + ), + TimeFormatDate => ( + vec![Date], + result("dever.time.TextResult", "Read", &[Text], false)?, + ), + TimeFormatTime => ( + vec![Time], + result("dever.time.TextResult", "Read", &[Text], false)?, + ), + TimeDuration => (vec![Int], Duration), + TimeAdd | TimeSubtract => ( + vec![DateTime, Duration], + result("dever.time.DateTimeResult", "Read", &[DateTime], false)?, + ), + TimeDifference => ( + vec![DateTime, DateTime], + result("dever.time.DurationResult", "Read", &[Duration], false)?, + ), + TimeSleep | TaskSleep => ( + vec![Int], + result("dever.time.SleepResult", "Done", &[], false)?, + ), + DecimalRound => (vec![Decimal, Int], Decimal), + FloatFromInt => (vec![Int], Float), + FloatSqrt | FloatSin | FloatCos | FloatLog => (vec![Float], Float), + FloatPow => (vec![Float, Float], Float), + FloatIsNan | FloatIsFinite | FloatIsInfinite => (vec![Float], Bool), + StdoutWrite => (vec![Text], Unit), + LogDebug | LogInfo | LogWarn | LogError => { + (vec![Text, Map(Box::new(Text), Box::new(Text))], Unit) + } + IdFromText => (vec![Text], Id), + IdToText => (vec![Id], Text), + UuidParse => (vec![Text], Uuid.nullable()), + UuidToText => (vec![Uuid], Text), + BytesFromText => (vec![Text], Bytes), + BytesToText => ( + vec![Bytes], + result("dever.bytes.DecodeResult", "Decoded", &[Text], false)?, + ), + BytesFromInts => ( + vec![List(Box::new(Int))], + result("dever.bytes.BufferResult", "Ready", &[Bytes], false)?, + ), + BytesLength => (vec![Bytes], Int), + BytesAt => (vec![Bytes, Int], Int.nullable()), + BytesSlice => ( + vec![Bytes, Int, Int], + result("dever.bytes.BufferResult", "Ready", &[Bytes], false)?, + ), + BytesConcat => (vec![Bytes, Bytes], Bytes), + FileOpen | FileCreate => ( + vec![Text], + result("dever.io.OpenResult", "Opened", &[File], false)?, + ), + FileRead => ( + vec![File, Int], + result("dever.io.SystemReadResult", "Read", &[Bytes], true)?, + ), + FileWrite => ( + vec![File, Bytes], + result("dever.io.WriteResult", "Done", &[], false)?, + ), + FileClose => ( + vec![File], + result("dever.io.CloseResult", "Closed", &[], false)?, + ), + FileChunks | TcpChunks => { + let event = result("dever.io.ReadEvent", "Chunk", &[Bytes], false)?; + let (source, stream, result_name) = if matches!(operation, FileChunks) { + (File, Stream(Box::new(event)), "dever.io.ReadStreamResult") + } else { + ( + Socket, + AsyncStream(Box::new(event)), + "dever.net.ReadStreamResult", + ) + }; + ( + vec![source, Int], + result(result_name, "Streaming", &[stream], false)?, + ) + } + TcpConnect => ( + vec![Text, Int], + result("dever.net.ConnectResult", "Connected", &[Socket], false)?, + ), + TcpConnectTimeout => ( + vec![Text, Int, Int], + result("dever.net.ConnectResult", "Connected", &[Socket], false)?, + ), + TcpListen => ( + vec![Text, Int], + result("dever.net.ListenResult", "Listening", &[Listener], false)?, + ), + TcpAccept => ( + vec![Listener], + result("dever.net.ConnectResult", "Connected", &[Socket], false)?, + ), + TcpPort => ( + vec![Listener], + result("dever.net.PortResult", "Bound", &[Int], false)?, + ), + TcpRead => ( + vec![Socket, Int], + result("dever.net.SystemReadResult", "Read", &[Bytes], true)?, + ), + TcpWrite => ( + vec![Socket, Bytes], + result("dever.io.WriteResult", "Done", &[], false)?, + ), + TcpTimeout => ( + vec![Socket, Int], + result("dever.io.WriteResult", "Done", &[], false)?, + ), + TcpClose => ( + vec![Socket], + result("dever.io.CloseResult", "Closed", &[], false)?, + ), + TcpCloseListener => ( + vec![Listener], + result("dever.io.CloseResult", "Closed", &[], false)?, + ), + TcpConnections => { + let connection = result("dever.net.ConnectResult", "Connected", &[Socket], false)?; + (vec![Listener], AsyncStream(Box::new(connection))) + } + TlsSystem => (vec![], ClientTls), + TlsClient => ( + vec![Bytes], + result("dever.tls.ClientResult", "Ready", &[ClientTls], false)?, + ), + TlsServer => ( + vec![Bytes, Bytes], + result("dever.tls.ServerResult", "Ready", &[ServerTls], false)?, + ), + HttpCloseClient => ( + vec![HttpClient], + result("dever.io.CloseResult", "Closed", &[], false)?, + ), + HttpServe | HttpServeTls | HttpSend | HttpServeLive | HttpServeLiveTls | HttpRespond + | HttpStart | SseStart | HttpClientNew | HttpRequest | HttpOpen | HttpOpenStream => { + let header = record( + context, + "dever.http.Header", + &[("name", Text), ("value", Bytes)], + span, + )?; + let headers = List(Box::new(header)); + let request = record( + context, + "dever.http.Request", + &[ + ("method", Text), + ("target", Text), + ("headers", headers.clone()), + ("body", Bytes), + ], + span, + )?; + let response = record( + context, + "dever.http.Response", + &[ + ("status", Int), + ("headers", headers.clone()), + ("body", Bytes), + ], + span, + )?; + let http2 = record( + context, + "dever.http.Http2Limits", + &[ + ("streams", Int), + ("stream_window_bytes", Int), + ("connection_window_bytes", Int), + ], + span, + )?; + let limits = record( + context, + "dever.http.Limits", + &[ + ("header_bytes", Int), + ("body_bytes", Int), + ("timeout_ms", Int), + ("connections", Int), + ("http2", http2.nullable()), + ], + span, + )?; + match operation { + HttpServeLive | HttpServeLiveTls => { + let live = record( + context, + "dever.http.LiveLimits", + &[ + ("chunk_bytes", Int), + ("idle_ms", Int), + ("heartbeat_ms", Int), + ], + span, + )?; + handler = Some(HandlerSignature { + parameters: vec![request, HttpReply], + bounds: vec![Default::default(); 2], + outputs: vec![], + }); + let mut parameters = vec![Listener, limits, live]; + if matches!(operation, HttpServeLiveTls) { + parameters.push(ServerTls); + } + (parameters, Unit) + } + HttpServe | HttpServeTls => { + handler = Some(HandlerSignature { + parameters: vec![request], + bounds: vec![Default::default()], + outputs: vec![Field { + name: "response".into(), + ty: response, + private: false, + bounds: Default::default(), + }], + }); + let mut parameters = vec![Listener, limits]; + if matches!(operation, HttpServeTls) { + parameters.push(ServerTls); + } + (parameters, Unit) + } + HttpRespond | HttpStart | SseStart => { + let parameters = match operation { + HttpRespond => vec![HttpReply, response], + HttpStart => vec![HttpReply, Int, headers], + SseStart => vec![HttpReply, headers], + _ => unreachable!(), + }; + ( + parameters, + result("dever.io.WriteResult", "Done", &[], false)?, + ) + } + HttpClientNew => { + let pool = record( + context, + "dever.http.PoolLimits", + &[("idle_ms", Int), ("chunk_bytes", Int), ("read_ms", Int)], + span, + )?; + ( + vec![Text, ClientTls, limits, pool], + result("dever.http.ClientResult", "Ready", &[HttpClient], false)?, + ) + } + HttpSend => ( + vec![Text, Int, request, limits], + result("dever.http.ResponseResult", "Read", &[response], false)?, + ), + HttpRequest => ( + vec![HttpClient, request], + result("dever.http.ResponseResult", "Read", &[response], false)?, + ), + HttpOpen | HttpOpenStream => { + let event = result("dever.io.ReadEvent", "Chunk", &[Bytes], false)?; + let body = AsyncStream(Box::new(event)); + let response = record( + context, + "dever.http.StreamResponse", + &[ + ("status", Int), + ("headers", headers.clone()), + ("body", body.clone()), + ], + span, + )?; + let request = if matches!(operation, HttpOpenStream) { + record( + context, + "dever.http.StreamRequest", + &[ + ("method", Text), + ("target", Text), + ("headers", headers), + ("body", body), + ], + span, + )? + } else { + request + }; + ( + vec![HttpClient, request], + result( + "dever.http.StreamResponseResult", + "Read", + &[response], + false, + )?, + ) + } + _ => unreachable!("HTTP record operation"), + } + } + HttpWrite => ( + vec![HttpReply, Bytes], + result("dever.io.WriteResult", "Done", &[], false)?, + ), + HttpFinish => ( + vec![HttpReply], + result("dever.io.CloseResult", "Closed", &[], false)?, + ), + SseSend => { + let event = record( + context, + "dever.sse.Event", + &[ + ("event", Text), + ("data", Text), + ("id", Text.nullable()), + ("retry_ms", Int.nullable()), + ], + span, + )?; + ( + vec![HttpReply, event], + result("dever.io.WriteResult", "Done", &[], false)?, + ) + } + WsAccept | WsConnect | WsOpen => { + let limits = record( + context, + "dever.websocket.Limits", + &[("message_bytes", Int), ("idle_ms", Int), ("write_ms", Int)], + span, + )?; + let parameters = match operation { + WsAccept => vec![HttpReply, limits], + WsOpen => vec![Text, ClientTls, limits], + _ => vec![Text, Int, Text, limits], + }; + ( + parameters, + result( + "dever.websocket.ConnectResult", + "Connected", + &[WebSocket], + false, + )?, + ) + } + WsSend | WsReceive | WsMessages => { + let message = websocket_message(context, span)?; + match operation { + WsSend => ( + vec![WebSocket, message], + result("dever.io.WriteResult", "Done", &[], false)?, + ), + WsReceive => ( + vec![WebSocket], + result( + "dever.websocket.SystemReceiveResult", + "Read", + &[message], + true, + )?, + ), + WsMessages => { + let event = result("dever.websocket.MessageEvent", "Read", &[message], false)?; + (vec![WebSocket], AsyncStream(Box::new(event))) + } + _ => unreachable!(), + } + } + WsClose => ( + vec![WebSocket, Int, Text], + result("dever.io.CloseResult", "Closed", &[], false)?, + ), + }; + Ok(Signature { + handler, + parameters, + output, + }) +} + +fn record( + context: &Context<'_>, + name: &str, + expected: &[(&str, Type)], + span: Span, +) -> Checked<Type> { + let id = context.named_type(name, span)?; + let valid = match &context.types[id].shape { + Shape::Record(fields) => { + fields.len() == expected.len() + && expected.iter().all(|(name, ty)| { + fields.iter().any(|field| { + field.name == *name + && field.ty == *ty + && !field.private + && field.bounds.is_empty() + }) + }) + } + Shape::Choice(_) => false, + }; + if !valid { + return Err(Diagnostic::error( + "C005", + format!("'{name}' does not match its system record contract"), + span, + )); + } + Ok(Type::Named(id)) +} + +fn websocket_message(context: &Context<'_>, span: Span) -> Checked<Type> { + let id = context.named_type("dever.websocket.Message", span)?; + let expected = [ + ("Text", Type::Text), + ("Binary", Type::Bytes), + ("Ping", Type::Bytes), + ("Pong", Type::Bytes), + ]; + let valid = match &context.types[id].shape { + Shape::Choice(variants) => { + variants.len() == expected.len() + && expected.iter().all(|(name, ty)| { + variants.iter().any(|variant| { + variant.name == *name + && !variant.error + && variant.fields.len() == 1 + && variant.fields[0].ty == *ty + && variant.fields[0].bounds.is_empty() + && !variant.fields[0].private + }) + }) + } + _ => false, + }; + if !valid { + return Err(Diagnostic::error( + "C005", + "WebSocket Message does not match its system contract", + span, + )); + } + Ok(Type::Named(id)) +} diff --git a/crates/dever-core/src/check/jobs.rs b/crates/dever-core/src/check/jobs.rs new file mode 100644 index 0000000..7aec33a --- /dev/null +++ b/crates/dever-core/src/check/jobs.rs @@ -0,0 +1,256 @@ +use std::collections::BTreeSet; + +use crate::diagnostic::Diagnostic; +use crate::hir::{Function, Job}; +use crate::model::ConnectionSelector; +use crate::source::SourceRole; +use crate::syntax::{Declaration, FunctionKind, InputKind, Package, Pattern}; +use crate::types::{Definition, DefinitionKind, Parameter, Shape, Type}; +use crate::wire::{Policy, Schema}; + +impl super::body::Body<'_, '_> { + pub(super) fn enqueue_job( + &mut self, + arguments: &[crate::syntax::Expression], + scheduled: bool, + span: crate::source::Span, + ) -> super::Checked<crate::hir::Expression> { + let caller = &self.context.symbols.packages[self.context.owner].layout; + if caller.role() != Some(SourceRole::App) { + return Err(Diagnostic::error( + "C006", + "only App may enqueue a Job", + span, + )); + } + let Some(crate::syntax::Expression { + kind: crate::syntax::ExpressionKind::Name(path), + .. + }) = arguments.first() + else { + return Err(Diagnostic::error( + "C005", + "enqueue requires a static Job target", + span, + )); + }; + let name = super::path_name(path); + let candidates = self.context.candidates(&name); + let targets = self + .context + .functions + .iter() + .enumerate() + .filter(|(_, function)| { + matches!(function.kind, FunctionKind::Job { .. }) + && candidates.contains(&function.name) + }) + .collect::<Vec<_>>(); + let [(target, entry)] = targets.as_slice() else { + return Err(Diagnostic::error( + "C004", + "enqueue target must name one declared Job", + span, + )); + }; + if self.context.symbols.packages[entry.owner].layout.domain() != caller.domain() { + return Err(Diagnostic::error( + "C006", + "App may enqueue only a Job in its own domain", + span, + )); + } + let mut expected = entry + .parameters + .iter() + .map(|parameter| match parameter { + Parameter::Value(ty) => Ok(ty.clone()), + _ => Err(Diagnostic::error( + "C005", + "Job payload must be a record", + span, + )), + }) + .collect::<Result<Vec<_>, _>>()?; + expected.push(Type::Text); + if scheduled { + expected.push(Type::DateTime); + } + if arguments.len() != expected.len() + 1 { + return Err(Diagnostic::error( + "C005", + "enqueue requires target, optional payload, key and optional run_at", + span, + )); + } + let arguments = arguments[1..] + .iter() + .zip(&expected) + .map(|(argument, ty)| self.expression(argument, Some(ty))) + .collect::<Result<Vec<_>, _>>()?; + Ok(crate::hir::Expression { + kind: crate::hir::ExpressionKind::JobEnqueue { + target: *target, + arguments, + scheduled, + }, + ty: Type::Id, + span, + }) + } +} + +pub(super) fn register( + packages: &[Package], + functions: &[Function], + types: &[Definition], + errors: &mut Vec<Diagnostic>, +) -> Vec<Job> { + let mut jobs = Vec::new(); + for (owner, package) in packages.iter().enumerate() { + let is_job = package.layout.role() == Some(SourceRole::Job); + let mut connection = None; + let mut scheduled = BTreeSet::new(); + for declaration in &package.declarations { + match declaration { + Declaration::Database(binding) if is_job => { + if connection + .replace(ConnectionSelector::Explicit(binding.name.text.clone())) + .is_some() + { + errors.push(Diagnostic::error( + "C002", + "duplicate Job database binding", + binding.span, + )); + } + } + Declaration::Function(clause) => { + let declared = matches!(clause.kind, FunctionKind::Job { .. }); + if is_job != declared { + errors.push(Diagnostic::error("C005", "job declarations belong only in job source; job source contains only Job entries", clause.span)); + } + } + Declaration::Schedule(_) if is_job => {} + _ if is_job => errors.push(Diagnostic::error( + "C005", + "Job files declare database, jobs and schedules; payload records belong in App", + crate::markdown::declaration_span(declaration), + )), + _ => {} + } + } + if !is_job { + continue; + } + let Some(connection) = connection else { + errors.push(Diagnostic::error( + "C014", + "Job source requires an explicit database binding", + package.span, + )); + continue; + }; + for (function, entry) in functions + .iter() + .enumerate() + .filter(|(_, function)| function.owner == owner) + { + let FunctionKind::Job { + attempts, + timeout_ms, + } = entry.kind + else { + continue; + }; + let clauses = package + .declarations + .iter() + .filter_map(|declaration| match declaration { + Declaration::Function(clause) + if entry.name.ends_with(&format!(".{}", clause.name.text)) => + { + Some(clause) + } + _ => None, + }) + .collect::<Vec<_>>(); + if clauses.len() != 1 || entry.parameters.len() > 1 || !entry.outputs.is_empty() + || clauses.iter().any(|clause| clause.pure || clause.recovery.is_some() || clause.fails.is_some() + || clause.inputs.iter().any(|input| !matches!(&input.kind, InputKind::Value(Pattern::Typed { bounds, .. }) if bounds.is_empty()))) + { + errors.push(Diagnostic::error("C005", "Job requires one plain zero-output declaration with zero inputs or one App record payload", entry.span)); + continue; + } + let payload = match entry.parameters.as_slice() { + [] => None, + [Parameter::Value(Type::Named(id))] + if types[*id].kind == DefinitionKind::Regular + && matches!(types[*id].shape, Shape::Record(_)) + && packages[types[*id].owner].layout.role() == Some(SourceRole::App) + && packages[types[*id].owner].layout.domain() + == package.layout.domain() => + { + match Schema::build(&Type::Named(*id), types, Policy::Job) { + Ok(schema) => Some(schema), + Err(reason) => { + errors.push(Diagnostic::error("C005", reason, entry.span)); + continue; + } + } + } + _ => { + errors.push(Diagnostic::error( + "C005", + "Job payload must be a wire-safe record owned by the same-domain App", + entry.span, + )); + continue; + } + }; + jobs.push(Job { + function, + connection: connection.clone(), + payload, + attempts, + timeout_ms, + schedule: None, + components: Vec::new(), + }); + } + for declaration in &package.declarations { + let Declaration::Schedule(schedule) = declaration else { + continue; + }; + let identity = format!( + "{}.{}", + super::path_name(&package.name), + schedule.target.text + ); + let Some(job) = jobs + .iter_mut() + .find(|job| functions[job.function].name == identity) + else { + errors.push(Diagnostic::error( + "C004", + "schedule target must name a Job declared in this source", + schedule.span, + )); + continue; + }; + if job.payload.is_some() || !scheduled.insert(job.function) { + errors.push(Diagnostic::error( + "C005", + "schedule requires a unique zero-input Job target", + schedule.span, + )); + continue; + } + match dever_runtime::cron::Cron::parse(&schedule.cron) { + Ok(_) => job.schedule = Some(schedule.cron.clone()), + Err(reason) => errors.push(Diagnostic::error("C005", reason, schedule.span)), + } + } + } + jobs +} diff --git a/crates/dever-core/src/check/layout.rs b/crates/dever-core/src/check/layout.rs new file mode 100644 index 0000000..e2546f7 --- /dev/null +++ b/crates/dever-core/src/check/layout.rs @@ -0,0 +1,131 @@ +use std::collections::BTreeMap; + +use crate::diagnostic::{Diagnostic, Label}; +use crate::source::{SourceLayout, SourceMap, SourceRole, Span}; + +#[derive(Default)] +struct RoleFiles { + root: Option<Span>, + topics: Vec<Span>, +} + +pub(super) fn validate(sources: &SourceMap) -> Vec<Diagnostic> { + let mut errors = Vec::new(); + let mut roles: BTreeMap<(String, String, SourceRole), RoleFiles> = BTreeMap::new(); + + for source in sources.files() { + let layout = match source.layout() { + Ok(layout) => layout, + Err(error) => { + errors.push(error); + continue; + } + }; + match layout { + SourceLayout::Main => {} + SourceLayout::Test { + component, + domain, + topic, + } => { + check_bucket_names( + [&component, &domain, &topic], + source.span(0, 0), + &mut errors, + ); + } + SourceLayout::Loose if sources.strict_layout() => errors.push(Diagnostic::error( + "C003", + "application source must be main.dever or <component>/<domain>/<app|domain|model|port|adapter|api>.dever, with an optional role topic directory", + source.span(0, 0), + )), + SourceLayout::Loose => {} + SourceLayout::Role { + component, + domain, + role, + topics, + } => { + check_bucket_names( + std::iter::once(&component) + .chain(std::iter::once(&domain)) + .chain(topics.iter()), + source.span(0, 0), + &mut errors, + ); + if role != SourceRole::Api && topics.len() > 1 { + errors.push(Diagnostic::error( + "C003", + format!( + "{} topic directories are flat; nested directories are supported only below api/", + role.name() + ), + source.span(0, 0), + )); + } + let files = roles.entry((component, domain, role)).or_default(); + if topics.is_empty() { + files.root = Some(source.span(0, 0)); + } else { + files.topics.push(source.span(0, 0)); + } + } + } + } + + for ((_, _, role), files) in roles { + // Model topics name persisted entities; API topics name route paths. + // Folding either into the root file would change its public identity. + if matches!(role, SourceRole::Model | SourceRole::Api) { + continue; + } + if let (Some(root), Some(topic)) = (files.root, files.topics.first().copied()) { + let mut error = Diagnostic::error( + "C003", + format!( + "{}.dever and its {}/ directory are mutually exclusive", + role.name(), + role.name() + ), + topic, + ); + error.related.push(Label { + span: root, + message: "role file declared here".into(), + }); + errors.push(error); + } else if files.root.is_none() && files.topics.len() == 1 { + errors.push(Diagnostic::error( + "C003", + format!( + "a {}/ directory needs at least two cohesive topic files; merge a single topic into {}.dever", + role.name(), + role.name() + ), + files.topics[0], + )); + } + } + errors +} + +fn check_bucket_names<'a>( + names: impl IntoIterator<Item = &'a String>, + span: Span, + errors: &mut Vec<Diagnostic>, +) { + for name in names { + if matches!( + name.as_str(), + "common" | "shared" | "utils" | "helper" | "helpers" | "base" | "service" | "services" + ) { + errors.push(Diagnostic::error( + "C003", + format!( + "'{name}' is a generic source bucket; name the business domain or cohesive topic instead" + ), + span, + )); + } + } +} diff --git a/crates/dever-core/src/check/model.rs b/crates/dever-core/src/check/model.rs new file mode 100644 index 0000000..0423e32 --- /dev/null +++ b/crates/dever-core/src/check/model.rs @@ -0,0 +1,2378 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use crate::diagnostic::Diagnostic; +use crate::model::{ + self, ConnectionSelector, ModelChoice, ModelField, ModelFieldType, ModelIndex, ModelMigration, + ModelMigrationSql, ModelRelation, ModelRelationKind, ModelSchema, ModelScope, ModelSql, + ModelSqlCardinality, ModelSqlParameter, ModelSqlResult, ModelValue, RestOwnerSource, SeedRow, +}; +use crate::source::{SourceLayout, SourceMap, SourceRole}; +use crate::syntax::{ + Declaration, Expression, ExpressionKind, Field, MigrationOperation, Package, TypeDeclaration, + TypeRef, TypeShape, UnaryOperator, +}; +use crate::types::{Definition, DefinitionKind, Field as CheckedField, Shape, Type}; + +use super::{Symbols, path_name}; + +#[derive(Clone, Copy, Debug)] +pub(super) struct Registration { + owner: usize, + record: usize, + id: usize, + page: usize, + cursor: usize, +} + +#[derive(Clone)] +pub(super) struct RestFieldSource { + pub field: usize, + pub private: bool, + pub span: crate::source::Span, + pub create: Option<Expression>, + pub replace: Option<Expression>, + pub search: Option<Expression>, +} + +pub(super) fn rest_sources( + schemas: &[ModelSchema], + packages: &[Package], + types: &[Definition], +) -> Vec<Vec<RestFieldSource>> { + schemas + .iter() + .map(|schema| { + let package = &packages[types[schema.record].owner]; + let record = package + .declarations + .iter() + .find_map(|declaration| match declaration { + Declaration::Type(record) if record.name.text == schema.name => Some(record), + _ => None, + }) + .expect("Model source record"); + let TypeShape::Record(fields) = &record.shape else { + unreachable!() + }; + fields + .iter() + .enumerate() + .filter_map(|(index, field)| { + let storage = &field.storage; + (storage.create.is_some() + || storage.replace.is_some() + || storage.search.is_some()) + .then(|| RestFieldSource { + field: index + 2, + private: field.private, + span: field.span, + create: storage.create.clone(), + replace: storage.replace.clone(), + search: storage.search.clone(), + }) + }) + .collect() + }) + .collect() +} + +pub(super) fn register<'a>( + packages: &'a [Package], + sources: &SourceMap, + symbols: &mut Symbols, + types: &mut Vec<Definition>, + type_sources: &mut Vec<Option<&'a TypeDeclaration>>, + errors: &mut Vec<Diagnostic>, +) -> Vec<Registration> { + let mut registrations = Vec::new(); + for (owner, package) in packages.iter().enumerate() { + let source = sources.get(package.span.source); + if !source.is_model() { + validate_regular_package(package, errors); + continue; + } + let Some(stem) = source.path().file_stem().and_then(|stem| stem.to_str()) else { + continue; + }; + let stem = if source.is_markdown() { + stem.strip_suffix(".dever").expect("Markdown source suffix") + } else { + stem + }; + let role_model = matches!(package.layout.role(), Some(SourceRole::Model)); + let model_name = match &package.layout { + SourceLayout::Role { domain, topics, .. } if topics.is_empty() => domain.as_str(), + _ => stem, + }; + let expected_name = model_type_name(model_name); + let records = package + .declarations + .iter() + .filter_map(|declaration| match declaration { + Declaration::Type(declaration) + if matches!(declaration.shape, TypeShape::Record(_)) => + { + Some(declaration) + } + _ => None, + }) + .collect::<Vec<_>>(); + let Some(record_source) = records + .iter() + .copied() + .find(|record| record.name.text == expected_name) + else { + if let [record] = records.as_slice() { + errors.push(Diagnostic::error( + "C014", + format!( + "Model record must be named '{expected_name}' to match '{}.dever'", + model_name + ), + record.name.span, + )); + } else { + errors.push(Diagnostic::error( + "C014", + format!("a Model file must declare record '{expected_name}'"), + package.span, + )); + } + continue; + }; + if record_source.global && !role_model { + errors.push(Diagnostic::error( + "C014", + "'global type' requires a Model source role", + record_source.span, + )); + } + if !role_model && !record_source.public { + errors.push(Diagnostic::error( + "C014", + "the Model record must be declared public", + record_source.name.span, + )); + } + validate_model_declarations(package, record_source, errors); + let package_name = path_name(&package.name); + let qualified_record = format!("{package_name}.{expected_name}"); + let Some(record) = symbols.types.get(&qualified_record).copied() else { + continue; + }; + types[record].kind = DefinitionKind::Model; + if role_model { + types[record].public = true; + } + for declaration in &package.declarations { + let Declaration::Type(choice) = declaration else { + continue; + }; + if !matches!(choice.shape, TypeShape::Choice(_)) { + continue; + } + let qualified = format!("{package_name}.{}", choice.name.text); + if let Some(id) = symbols.types.get(&qualified) { + types[*id].kind = DefinitionKind::ModelChoice; + if role_model { + types[*id].public = true; + } + } + } + let qualified_id = format!("{package_name}.id"); + if symbols.types.contains_key(&qualified_id) { + errors.push(Diagnostic::error( + "C002", + "Model ID type conflicts with an existing type", + record_source.span, + )); + continue; + } + let id = types.len(); + symbols.types.insert(qualified_id.clone(), id); + types.push(Definition { + name: qualified_id, + owner, + public: true, + shape: Shape::Record(Vec::new()), + span: record_source.span, + properties: Default::default(), + kind: DefinitionKind::ModelId, + }); + type_sources.push(None); + let Some(page) = register_result_type( + &package_name, + "Page", + DefinitionKind::ModelPage, + owner, + record_source, + symbols, + types, + type_sources, + errors, + ) else { + continue; + }; + let Some(cursor) = register_result_type( + &package_name, + "Cursor", + DefinitionKind::ModelCursor, + owner, + record_source, + symbols, + types, + type_sources, + errors, + ) else { + continue; + }; + registrations.push(Registration { + owner, + record, + id, + page, + cursor, + }); + } + registrations +} + +#[allow(clippy::too_many_arguments)] +fn register_result_type<'a>( + package: &str, + name: &str, + kind: DefinitionKind, + owner: usize, + source: &'a TypeDeclaration, + symbols: &mut Symbols, + types: &mut Vec<Definition>, + type_sources: &mut Vec<Option<&'a TypeDeclaration>>, + errors: &mut Vec<Diagnostic>, +) -> Option<usize> { + let qualified = format!("{package}.{name}"); + if symbols.types.contains_key(&qualified) { + errors.push(Diagnostic::error( + "C002", + format!("Model {name} type conflicts with an existing type"), + source.span, + )); + return None; + } + let id = types.len(); + symbols.types.insert(qualified.clone(), id); + types.push(Definition { + name: qualified, + owner, + public: true, + shape: Shape::Record(Vec::new()), + span: source.span, + properties: Default::default(), + kind, + }); + type_sources.push(None); + Some(id) +} + +pub(super) fn synthesize_fields( + registrations: &[Registration], + types: &mut [Definition], + errors: &mut Vec<Diagnostic>, +) { + for registration in registrations { + let Shape::Record(fields) = &mut types[registration.record].shape else { + unreachable!("registered Model is a record") + }; + if fields + .iter() + .any(|field| matches!(field.name.as_str(), "id" | "created_at")) + { + errors.push(Diagnostic::error( + "C014", + "'id' and 'created_at' are implicit Model fields", + types[registration.record].span, + )); + continue; + } + fields.insert( + 0, + CheckedField { + name: "created_at".into(), + ty: Type::DateTime, + private: false, + bounds: Vec::new(), + }, + ); + fields.insert( + 0, + CheckedField { + name: "id".into(), + ty: Type::Named(registration.id), + private: false, + bounds: Vec::new(), + }, + ); + let page_fields = vec![ + CheckedField { + name: "items".into(), + ty: Type::List(Box::new(Type::Named(registration.record))), + private: false, + bounds: Vec::new(), + }, + CheckedField { + name: "page".into(), + ty: Type::Int, + private: false, + bounds: Vec::new(), + }, + CheckedField { + name: "size".into(), + ty: Type::Int, + private: false, + bounds: Vec::new(), + }, + CheckedField { + name: "total".into(), + ty: Type::Int, + private: false, + bounds: Vec::new(), + }, + CheckedField { + name: "pages".into(), + ty: Type::Int, + private: false, + bounds: Vec::new(), + }, + ]; + types[registration.page].shape = Shape::Record(page_fields); + types[registration.cursor].shape = Shape::Record(vec![ + CheckedField { + name: "items".into(), + ty: Type::List(Box::new(Type::Named(registration.record))), + private: false, + bounds: Vec::new(), + }, + CheckedField { + name: "next".into(), + ty: Type::Named(registration.record).nullable(), + private: false, + bounds: Vec::new(), + }, + CheckedField { + name: "has_more".into(), + ty: Type::Bool, + private: false, + bounds: Vec::new(), + }, + ]); + } +} + +pub(super) fn synthesize_relation_fields( + schemas: &[ModelSchema], + types: &mut [Definition], + errors: &mut Vec<Diagnostic>, +) { + let records = schemas + .iter() + .map(|schema| (schema.package.as_str(), schema.record)) + .collect::<BTreeMap<_, _>>(); + for schema in schemas { + let Shape::Record(fields) = &mut types[schema.record].shape else { + unreachable!("Model is a record") + }; + for relation in &schema.relations { + if fields.iter().any(|field| field.name == relation.name) { + errors.push(Diagnostic::error( + "C002", + format!("relation '{}' conflicts with a Model field", relation.name), + types[schema.record].span, + )); + continue; + } + let Some(target) = records.get(relation.model.as_str()).copied() else { + continue; + }; + let payload = match relation.kind { + ModelRelationKind::ToOne => { + let nullable = schema + .fields + .iter() + .find(|field| field.name == relation.field) + .is_some_and(|field| field.nullable); + let target = Type::Named(target); + if nullable { target.nullable() } else { target } + } + ModelRelationKind::ToMany => Type::List(Box::new(Type::Named(target))), + }; + fields.push(CheckedField { + name: relation.name.clone(), + ty: Type::Related(Box::new(payload)), + private: false, + bounds: Vec::new(), + }); + } + } +} + +pub(super) fn schemas( + registrations: &[Registration], + packages: &[Package], + symbols: &Symbols, + types: &[Definition], + errors: &mut Vec<Diagnostic>, +) -> Vec<ModelSchema> { + let mut schemas = Vec::new(); + for registration in registrations { + let package = &packages[registration.owner]; + if let Some(schema) = schema(package, *registration, symbols, types, errors) { + schemas.push(schema); + } + } + validate_model_references(&mut schemas, packages, registrations, errors); + validate_relations(&mut schemas, packages, registrations, errors); + for schema in &mut schemas { + schema.revision = model::checksum(&schema_identity(schema)); + } + validate_table_names(&schemas, packages, registrations, errors); + schemas +} + +fn validate_regular_package(package: &Package, errors: &mut Vec<Diagnostic>) { + for declaration in &package.declarations { + match declaration { + Declaration::Type(declaration) => { + if declaration.global { + errors.push(Diagnostic::error( + "C014", + "'global type' is allowed only for the primary record in a Model source role", + declaration.span, + )); + } + match &declaration.shape { + TypeShape::Record(fields) => validate_regular_fields(fields, errors), + TypeShape::Choice(variants) => { + for variant in variants { + if variant.label.is_some() { + errors.push(Diagnostic::error( + "C014", + "choice display labels are only allowed in Model files", + variant.span, + )); + } + validate_regular_fields(&variant.payload, errors); + } + } + } + } + Declaration::Function(function) => { + for input in &function.inputs { + match &input.kind { + crate::syntax::InputKind::Value(crate::syntax::Pattern::Typed { + ty, + .. + }) if !ty.parameters.is_empty() => errors.push(Diagnostic::error( + "C014", + "bounded storage types are only allowed on Model fields", + ty.span, + )), + crate::syntax::InputKind::Handler(handler) => { + validate_regular_fields(&handler.inputs, errors); + validate_regular_fields(&handler.outputs, errors); + } + _ => {} + } + } + validate_regular_fields(&function.outputs, errors); + } + Declaration::Database(_) | Declaration::Schedule(_) + if package.layout.role() == Some(crate::source::SourceRole::Job) => {} + Declaration::Api(_) | Declaration::Rest(_) | Declaration::External(_) => {} + Declaration::Schedule(_) + | Declaration::Database(_) + | Declaration::ModelIndex(_) + | Declaration::Relation(_) + | Declaration::Seed(_) + | Declaration::Migration(_) + | Declaration::ModelSql(_) => errors.push(Diagnostic::error( + "C014", + "Model metadata is only allowed in a model source role", + declaration_span(declaration), + )), + } + } +} + +fn validate_regular_fields(fields: &[Field], errors: &mut Vec<Diagnostic>) { + for field in fields { + if !field.ty.parameters.is_empty() || field.storage != Default::default() { + errors.push(Diagnostic::error( + "C014", + "storage parameters and modifiers are only allowed on Model fields", + field.span, + )); + } + } +} + +fn validate_model_declarations( + package: &Package, + record: &TypeDeclaration, + errors: &mut Vec<Diagnostic>, +) { + let mut database = None; + let mut seed = None; + let mut migrations = BTreeSet::new(); + let mut sql = BTreeSet::new(); + for declaration in &package.declarations { + match declaration { + Declaration::Database(binding) => { + if database.replace(binding.span).is_some() { + errors.push(Diagnostic::error( + "C002", + "a Model may declare only one database binding", + binding.span, + )); + } + } + Declaration::Seed(value) => { + if seed.replace(value.span).is_some() { + errors.push(Diagnostic::error( + "C002", + "a Model may declare only one seed block", + value.span, + )); + } + } + Declaration::Migration(migration) => { + if !migrations.insert(&migration.name.text) { + errors.push(Diagnostic::error( + "C002", + "duplicate Model migration name", + migration.name.span, + )); + } + if migration.operations.is_empty() { + errors.push(Diagnostic::error( + "C014", + "a Model migration must contain at least one operation", + migration.span, + )); + } + } + Declaration::ModelSql(operation) => { + if !sql.insert(&operation.name.text) { + errors.push(Diagnostic::error( + "C002", + "duplicate Model SQL operation name", + operation.name.span, + )); + } + if matches!( + operation.name.text.as_str(), + "create" + | "create_many" + | "get" + | "first" + | "list" + | "cursor" + | "count" + | "exists" + | "stream" + | "update" + | "delete" + | "upsert" + ) || package.declarations.iter().any(|declaration| { + matches!(declaration, Declaration::Function(function) if function.name.text == operation.name.text) + }) { + errors.push(Diagnostic::error( + "C002", + "Model SQL operation name conflicts with an existing operation", + operation.name.span, + )); + } + } + Declaration::Type(declaration) if !std::ptr::eq(declaration, record) => { + if declaration.global { + errors.push(Diagnostic::error( + "C014", + "only the primary Model record may be declared global", + declaration.span, + )); + } + if let TypeShape::Record(fields) = &declaration.shape { + if declaration.public { + errors.push(Diagnostic::error( + "C014", + "a Model SQL result record must be private", + declaration.span, + )); + } + validate_regular_fields(fields, errors); + } + } + Declaration::Api(_) | Declaration::Rest(_) => errors.push(Diagnostic::error( + "C014", + "API declarations are only allowed in an API source role", + declaration_span(declaration), + )), + _ => {} + } + } + let TypeShape::Record(fields) = &record.shape else { + unreachable!() + }; + for field in fields { + if !field.bounds.is_empty() { + errors.push(Diagnostic::error( + "C014", + "Model constraints use storage type parameters, not function pattern bounds", + field.span, + )); + } + } +} + +fn schema( + package: &Package, + registration: Registration, + symbols: &Symbols, + types: &[Definition], + errors: &mut Vec<Diagnostic>, +) -> Option<ModelSchema> { + let package_name = path_name(&package.name); + let record_source = package + .declarations + .iter() + .find_map(|declaration| match declaration { + Declaration::Type(declaration) + if declaration.name.text + == types[registration.record].name.rsplit('.').next().unwrap() => + { + Some(declaration) + } + _ => None, + })?; + let TypeShape::Record(source_fields) = &record_source.shape else { + unreachable!() + }; + let choices = normalize_choices(package, errors); + let mut fields = vec![ + ModelField { + name: "id".into(), + ty: ModelFieldType::ModelId(package_name.clone()), + nullable: false, + generated: true, + default: None, + rename_from: None, + logical_reference: false, + owner: None, + }, + ModelField { + name: "created_at".into(), + ty: ModelFieldType::DateTime, + nullable: false, + generated: true, + default: None, + rename_from: None, + logical_reference: false, + owner: None, + }, + ]; + let mut source_by_name = BTreeMap::new(); + let mut rename_sources = BTreeSet::new(); + for field in source_fields { + source_by_name.insert(field.name.text.clone(), field); + let Some(ty) = normalize_field_type(&field.ty, &package_name, symbols, types, errors) + else { + continue; + }; + let nullable = field.ty.nullable; + let storage = &field.storage; + if storage.generated && (!matches!(ty, ModelFieldType::Uuid) || nullable) { + errors.push(Diagnostic::error( + "C014", + "'generated' requires a non-null Uuid field", + field.span, + )); + } + if storage.generated && storage.default.is_some() { + errors.push(Diagnostic::error( + "C014", + "a generated field cannot also declare a default", + field.span, + )); + } + if storage.index && storage.unique { + errors.push(Diagnostic::error( + "C014", + "'unique' already creates an index", + field.span, + )); + } + if let Some(from) = &storage.from + && (from.text == field.name.text || !rename_sources.insert(&from.text)) + { + errors.push(Diagnostic::error( + "C014", + "a rename source must be distinct and used by one field", + from.span, + )); + } + let default = storage + .default + .as_ref() + .and_then(|value| normalize_value(value, &ty, nullable, &choices, errors)); + let owner = storage + .owner + .as_ref() + .and_then(|source| normalize_owner_source(source, &ty, nullable, storage, errors)); + fields.push(ModelField { + name: field.name.text.clone(), + ty, + nullable, + generated: storage.generated, + default, + rename_from: storage.from.as_ref().map(|name| name.text.clone()), + logical_reference: false, + owner, + }); + } + + if fields.iter().filter(|field| field.owner.is_some()).count() > 1 { + errors.push(Diagnostic::error( + "C014", + "a Model may declare only one REST owner field", + record_source.span, + )); + } + + let indexes = normalize_indexes(package, &fields, &source_by_name, errors); + let seed = normalize_seed(package, &fields, &indexes, &choices, errors); + let migrations = normalize_migrations(package, &fields, errors); + let mut relations = fields + .iter() + .filter_map(|field| { + let ModelFieldType::ModelId(target) = &field.ty else { + return None; + }; + let name = field.name.strip_suffix("_id")?; + if name.is_empty() { + return None; + } + Some(ModelRelation { + name: name.into(), + model: target.clone(), + field: field.name.clone(), + kind: ModelRelationKind::ToOne, + logical: false, + }) + }) + .collect::<Vec<_>>(); + relations.extend( + package + .declarations + .iter() + .filter_map(|declaration| match declaration { + Declaration::Relation(relation) => { + let mut path = relation + .field + .iter() + .map(|name| name.text.clone()) + .collect::<Vec<_>>(); + let field = path.pop()?; + Some(ModelRelation { + name: relation.name.text.clone(), + model: path.join("."), + field, + kind: ModelRelationKind::ToMany, + logical: false, + }) + } + _ => None, + }), + ); + let connection = package + .declarations + .iter() + .find_map(|declaration| match declaration { + Declaration::Database(binding) => { + Some(ConnectionSelector::Explicit(binding.name.text.clone())) + } + _ => None, + }) + .unwrap_or_else(|| ConnectionSelector::PackageRoot(package.name[0].text.clone())); + let sql = normalize_sql(package, registration, symbols, types, errors); + let table = match &package.layout { + SourceLayout::Role { + domain, + role: SourceRole::Model, + topics, + .. + } => topics.last().unwrap_or(domain).clone(), + _ => package.name.last()?.text.clone(), + }; + let mut schema = ModelSchema { + record: registration.record, + page: registration.page, + cursor: registration.cursor, + package: package_name, + name: record_source.name.text.clone(), + scope: if record_source.global { + ModelScope::Global + } else { + ModelScope::Tenant + }, + table, + connection, + fields, + choices, + indexes, + relations, + sql, + seed, + migrations, + revision: String::new(), + seed_revision: String::new(), + }; + schema.seed_revision = model::checksum(&seed_identity(&schema.seed)); + let _ = registration; + Some(schema) +} + +fn normalize_owner_source( + source: &Expression, + ty: &ModelFieldType, + nullable: bool, + storage: &crate::syntax::FieldStorage, + errors: &mut Vec<Diagnostic>, +) -> Option<RestOwnerSource> { + let mut valid = true; + if nullable { + errors.push(Diagnostic::error( + "C014", + "a REST owner field must be non-null", + source.span, + )); + valid = false; + } + if !matches!(ty, ModelFieldType::Int | ModelFieldType::ModelId(_)) { + errors.push(Diagnostic::error( + "C014", + "a REST owner field must use Int or a Model id type", + source.span, + )); + valid = false; + } + if storage.generated || storage.default.is_some() { + errors.push(Diagnostic::error( + "C014", + "a REST owner field cannot also be generated or declare a default", + source.span, + )); + valid = false; + } + + let source = super::expressions::ungroup(source); + let owner = match &source.kind { + ExpressionKind::Call { + function, + arguments, + } if arguments.is_empty() => { + let function = super::expressions::ungroup(function); + let ExpressionKind::Name(path) = &function.kind else { + return invalid_owner_source(source, errors); + }; + match path_name(path).as_str() { + "dever.auth.user_id" => Some(RestOwnerSource::User), + "dever.auth.tenant_id" => Some(RestOwnerSource::Tenant), + _ => None, + } + } + _ => None, + }; + let Some(owner) = owner else { + return invalid_owner_source(source, errors); + }; + valid.then_some(owner) +} + +fn invalid_owner_source( + source: &Expression, + errors: &mut Vec<Diagnostic>, +) -> Option<RestOwnerSource> { + errors.push(Diagnostic::error( + "C014", + "a REST owner source must be dever.auth.user_id() or dever.auth.tenant_id()", + source.span, + )); + None +} + +fn normalize_choices(package: &Package, errors: &mut Vec<Diagnostic>) -> Vec<ModelChoice> { + package + .declarations + .iter() + .filter_map(|declaration| match declaration { + Declaration::Type(declaration) => match &declaration.shape { + TypeShape::Choice(variants) => { + let labelled = variants + .iter() + .filter(|variant| variant.label.is_some()) + .count(); + if labelled != 0 && labelled != variants.len() { + errors.push(Diagnostic::error( + "C014", + "choice variants must either all have display labels or all omit them", + declaration.span, + )); + } + if variants + .iter() + .any(|variant| variant.error || !variant.payload.is_empty()) + { + errors.push(Diagnostic::error( + "C014", + "Model choices cannot contain error or payload variants", + declaration.span, + )); + } + Some(ModelChoice { + name: format!("{}.{}", path_name(&package.name), declaration.name.text), + options: variants + .iter() + .map(|variant| (variant.name.text.clone(), variant.label.clone())) + .collect(), + }) + } + TypeShape::Record(_) => None, + }, + _ => None, + }) + .collect() +} + +fn normalize_sql( + package: &Package, + registration: Registration, + symbols: &Symbols, + types: &[Definition], + errors: &mut Vec<Diagnostic>, +) -> Vec<ModelSql> { + let package_name = path_name(&package.name); + let mut used_result_records = BTreeSet::new(); + let operations = package + .declarations + .iter() + .filter_map(|declaration| { + let Declaration::ModelSql(source) = declaration else { + return None; + }; + let mut names = BTreeSet::new(); + let mut parameters = Vec::new(); + for input in &source.inputs { + if input.private || !input.bounds.is_empty() || input.storage != Default::default() + { + errors.push(Diagnostic::error( + "C014", + "SQL parameters cannot use field modifiers or bounds", + input.span, + )); + continue; + } + if !names.insert(&input.name.text) { + errors.push(Diagnostic::error( + "C002", + "duplicate SQL parameter name", + input.name.span, + )); + continue; + } + let Some(ty) = + normalize_field_type(&input.ty, &package_name, symbols, types, errors) + else { + continue; + }; + parameters.push(ModelSqlParameter { + name: input.name.text.clone(), + ty, + nullable: input.ty.nullable, + }); + } + if parameters.len() != source.inputs.len() { + return None; + } + if source.outputs.len() != 1 { + errors.push(Diagnostic::error( + "C014", + "a SQL operation must declare exactly one result", + source.span, + )); + return None; + } + let output = &source.outputs[0]; + if output.private || !output.bounds.is_empty() || output.storage != Default::default() { + errors.push(Diagnostic::error( + "C014", + "a SQL result cannot use field modifiers or bounds", + output.span, + )); + return None; + } + let result = normalize_sql_result( + &output.ty, + registration, + &package_name, + symbols, + types, + errors, + )?; + if types[result.record].kind == DefinitionKind::Regular { + used_result_records.insert(result.record); + } + validate_sql_placeholders( + &source.sqlite.value, + '?', + '$', + parameters.len(), + source.sqlite.span, + errors, + ); + validate_sql_placeholders( + &source.postgres.value, + '$', + '?', + parameters.len(), + source.postgres.span, + errors, + ); + if source.sqlite.value.trim().is_empty() || source.postgres.value.trim().is_empty() { + errors.push(Diagnostic::error( + "C014", + "SQL dialect text cannot be empty", + source.span, + )); + } + Some(ModelSql { + name: source.name.text.clone(), + parameters, + result, + sqlite: source.sqlite.value.clone(), + postgres: source.postgres.value.clone(), + }) + }) + .collect::<Vec<_>>(); + + for declaration in &package.declarations { + let Declaration::Type(declaration) = declaration else { + continue; + }; + if declaration.name.text == types[registration.record].name.rsplit('.').next().unwrap() + || !matches!(declaration.shape, TypeShape::Record(_)) + { + continue; + } + let qualified = format!("{package_name}.{}", declaration.name.text); + let Some(record) = symbols.types.get(&qualified) else { + continue; + }; + if !used_result_records.contains(record) { + errors.push(Diagnostic::error( + "C014", + "an auxiliary record in a Model file must be used by a SQL result", + declaration.span, + )); + } + } + operations +} + +fn normalize_sql_result( + source: &TypeRef, + registration: Registration, + package: &str, + symbols: &Symbols, + types: &[Definition], + errors: &mut Vec<Diagnostic>, +) -> Option<ModelSqlResult> { + let (row, cardinality) = if path_name(&source.name) == "List" { + if source.arguments.len() != 1 + || source.nullable + || !source.parameters.is_empty() + || source.arguments[0].nullable + { + errors.push(Diagnostic::error( + "C014", + "SQL results use Record, Record? or List<Record>", + source.span, + )); + return None; + } + (&source.arguments[0], ModelSqlCardinality::Many) + } else { + ( + source, + if source.nullable { + ModelSqlCardinality::Optional + } else { + ModelSqlCardinality::One + }, + ) + }; + if !row.arguments.is_empty() || !row.parameters.is_empty() { + errors.push(Diagnostic::error( + "C014", + "SQL results use a concrete Model or record type", + row.span, + )); + return None; + } + let name = path_name(&row.name); + let context = super::Context { + symbols, + types, + functions: &[], + models: &[], + owner: registration.owner, + }; + let Some(record) = context + .candidates(&name) + .iter() + .find_map(|qualified| symbols.types.get(qualified).copied()) + else { + errors.push(Diagnostic::error( + "C004", + format!("unknown SQL result type '{name}' in '{package}'"), + row.span, + )); + return None; + }; + let Shape::Record(fields) = &types[record].shape else { + errors.push(Diagnostic::error( + "C014", + "SQL results must use a concrete Model or record type", + row.span, + )); + return None; + }; + let result_owner = &symbols.packages[types[record].owner].layout; + let model_owner = &symbols.packages[registration.owner].layout; + let app_view = types[record].public + && result_owner.role() == Some(SourceRole::App) + && result_owner.domain() == model_owner.domain(); + match types[record].kind { + DefinitionKind::Model => {} + DefinitionKind::Regular + if (types[record].owner == registration.owner && !types[record].public) || app_view => + { + for field in fields { + if (app_view && field.private) + || !field.bounds.is_empty() + || !sql_result_field(&field.ty, types) + { + errors.push(Diagnostic::error( + "C014", + format!( + "SQL result field '{}' must be visible, without bounds, and have a supported storage type, found {}", + field.name, + field.ty.label(types) + ), + types[record].span, + )); + } + } + } + _ => { + errors.push(Diagnostic::error( + "C014", + "a custom SQL result record must be private to its Model file or an owning-domain App View", + row.span, + )); + return None; + } + } + Some(ModelSqlResult { + record, + cardinality, + }) +} + +fn sql_result_field(ty: &Type, types: &[Definition]) -> bool { + match ty { + Type::Bool + | Type::Int + | Type::Decimal + | Type::Float + | Type::Text + | Type::Bytes + | Type::Uuid + | Type::DateTime + | Type::Date + | Type::Time + | Type::Duration + | Type::Json => true, + Type::Nullable(inner) => sql_result_field(inner, types), + Type::Named(id) => { + types[*id].kind == DefinitionKind::ModelId + || matches!(&types[*id].shape, Shape::Choice(variants) if variants.iter().all(|variant| !variant.error && variant.fields.is_empty())) + } + _ => false, + } +} + +fn validate_sql_placeholders( + sql: &str, + marker: char, + other_marker: char, + parameters: usize, + span: crate::source::Span, + errors: &mut Vec<Diagnostic>, +) { + validate_sql(sql, marker, other_marker, parameters, span, false, errors); +} + +fn validate_sql( + sql: &str, + marker: char, + other_marker: char, + parameters: usize, + span: crate::source::Span, + migration: bool, + errors: &mut Vec<Diagnostic>, +) { + let bytes = sql.as_bytes(); + let marker = marker as u8; + let other_marker = other_marker as u8; + let mut used = BTreeSet::new(); + let mut index = 0; + let mut started = false; + let mut terminated = false; + while index < bytes.len() { + let current = bytes[index]; + if current.is_ascii_whitespace() { + index += 1; + continue; + } + if current == b'-' && bytes.get(index + 1) == Some(&b'-') { + index = skip_sql_line_comment(bytes, index + 2); + continue; + } + if current == b'/' && bytes.get(index + 1) == Some(&b'*') { + index = skip_sql_block_comment(bytes, index + 2, marker == b'$'); + continue; + } + if migration { + if terminated { + errors.push(Diagnostic::error( + "C014", + "migration SQL must contain exactly one DML statement", + span, + )); + return; + } + if !started { + let end = bytes[index..] + .iter() + .position(|byte| !byte.is_ascii_alphabetic()) + .map_or(bytes.len(), |end| index + end); + if !matches!( + sql[index..end].to_ascii_uppercase().as_str(), + "INSERT" | "UPDATE" | "DELETE" + ) { + errors.push(Diagnostic::error( + "C014", + "migration SQL must start with INSERT, UPDATE or DELETE", + span, + )); + return; + } + started = true; + } + if current == b';' { + terminated = true; + index += 1; + continue; + } + if current.is_ascii_alphabetic() || current == b'_' || current >= 128 { + let end = bytes[index..] + .iter() + .position(|byte| { + !byte.is_ascii_alphanumeric() + && *byte != b'_' + && *byte < 128 + && !(marker == b'$' && *byte == b'$') + }) + .map_or(bytes.len(), |end| index + end); + if reserved_migration_identifier(&sql[index..end]) { + errors.push(Diagnostic::error( + "C014", + "migration SQL cannot reference reserved database objects", + span, + )); + return; + } + if marker == b'$' + && sql[index..end].eq_ignore_ascii_case("u") + && bytes[end..].starts_with(b"&\"") + { + errors.push(Diagnostic::error( + "C014", + "migration SQL does not accept Unicode-escaped identifiers", + span, + )); + return; + } + index = end; + continue; + } + } + if matches!(current, b'\'' | b'"' | b'`') { + let backslash_escape = marker == b'$' + && current == b'\'' + && index > 0 + && matches!(bytes[index - 1], b'e' | b'E') + && (index == 1 + || !matches!( + bytes[index - 2], + b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'_' + )); + let Some(end) = skip_sql_quoted(bytes, index, current, backslash_escape) else { + errors.push(Diagnostic::error( + "C014", + "SQL has an unclosed quoted value or identifier", + span, + )); + return; + }; + // SQLite also permits single-quoted table names. Reserved names belong + // in bind values when they are data, never in migration SQL text. + if migration && reserved_migration_identifier(&sql[index + 1..end - 1]) { + errors.push(Diagnostic::error( + "C014", + "migration SQL cannot reference reserved database objects", + span, + )); + return; + } + index = end; + continue; + } + if current == b'[' && marker == b'?' { + let Some(end) = skip_sql_bracket_identifier(bytes, index) else { + errors.push(Diagnostic::error( + "C014", + "SQL has an unclosed quoted identifier", + span, + )); + return; + }; + if migration && reserved_migration_identifier(&sql[index + 1..end - 1]) { + errors.push(Diagnostic::error( + "C014", + "migration SQL cannot reference reserved database objects", + span, + )); + return; + } + index = end; + continue; + } + if marker == b'$' + && current == b'$' + && (index == 0 || !postgres_identifier_byte(bytes[index - 1])) + { + if let Some(delimiter) = postgres_dollar_quote(bytes, index) { + let Some(end) = skip_postgres_dollar_quote(bytes, index, delimiter) else { + errors.push(Diagnostic::error( + "C014", + "SQL has an unclosed dollar-quoted value", + span, + )); + return; + }; + index = end; + continue; + } + if bytes + .get(index + 1) + .is_some_and(|byte| byte.is_ascii_alphabetic() || *byte == b'_' || *byte >= 128) + { + errors.push(Diagnostic::error( + "C014", + "SQL has an unclosed dollar-quote delimiter", + span, + )); + return; + } + } + if current == other_marker && bytes.get(index + 1).is_some_and(u8::is_ascii_digit) { + errors.push(Diagnostic::error( + "C014", + "SQL uses the other dialect's parameter marker", + span, + )); + return; + } + if current != marker || !bytes.get(index + 1).is_some_and(u8::is_ascii_digit) { + index += 1; + continue; + } + let start = index + 1; + index = start; + while bytes.get(index).is_some_and(u8::is_ascii_digit) { + index += 1; + } + let parameter = sql[start..index].parse::<usize>().unwrap_or(usize::MAX); + if parameter == 0 || parameter > parameters { + errors.push(Diagnostic::error( + "C014", + format!("SQL parameter {parameter} is outside the declared parameter list"), + span, + )); + return; + } + used.insert(parameter); + } + if migration && !started { + errors.push(Diagnostic::error( + "C014", + "migration SQL cannot be empty", + span, + )); + } + if let Some(parameter) = (1..=parameters).find(|parameter| !used.contains(parameter)) { + errors.push(Diagnostic::error( + "C014", + format!("SQL parameter {parameter} is declared but not bound in this dialect"), + span, + )); + } +} + +fn reserved_migration_identifier(identifier: &str) -> bool { + let identifier = identifier.to_ascii_lowercase(); + identifier.starts_with("_dever_") + || identifier.starts_with("sqlite_") + || identifier.starts_with("pg_") + || identifier == "information_schema" +} + +fn skip_sql_quoted( + bytes: &[u8], + mut index: usize, + quote: u8, + backslash_escape: bool, +) -> Option<usize> { + index += 1; + while index < bytes.len() { + if backslash_escape && bytes[index] == b'\\' { + index = (index + 2).min(bytes.len()); + } else if bytes[index] == quote { + if bytes.get(index + 1) == Some("e) { + index += 2; + } else { + return Some(index + 1); + } + } else { + index += 1; + } + } + None +} + +fn skip_sql_bracket_identifier(bytes: &[u8], index: usize) -> Option<usize> { + bytes[index + 1..] + .iter() + .position(|byte| *byte == b']') + .map(|end| index + end + 2) +} + +fn skip_sql_line_comment(bytes: &[u8], mut index: usize) -> usize { + while index < bytes.len() && !matches!(bytes[index], b'\n' | b'\r') { + index += 1; + } + index +} + +fn skip_sql_block_comment(bytes: &[u8], mut index: usize, nested: bool) -> usize { + let mut depth = 1usize; + while index < bytes.len() && depth > 0 { + if nested && bytes[index..].starts_with(b"/*") { + depth += 1; + index += 2; + } else if bytes[index..].starts_with(b"*/") { + depth -= 1; + index += 2; + } else { + index += 1; + } + } + index +} + +fn postgres_dollar_quote(bytes: &[u8], index: usize) -> Option<&[u8]> { + let mut end = index + 1; + if bytes.get(end) == Some(&b'$') { + return Some(&bytes[index..=end]); + } + if !bytes + .get(end) + .is_some_and(|byte| byte.is_ascii_alphabetic() || *byte == b'_' || *byte >= 128) + { + return None; + } + end += 1; + while bytes + .get(end) + .is_some_and(|byte| postgres_identifier_byte(*byte) && *byte != b'$') + { + end += 1; + } + (bytes.get(end) == Some(&b'$')).then(|| &bytes[index..=end]) +} + +fn postgres_identifier_byte(byte: u8) -> bool { + byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'$') || byte >= 128 +} + +fn skip_postgres_dollar_quote(bytes: &[u8], index: usize, delimiter: &[u8]) -> Option<usize> { + let content = index + delimiter.len(); + bytes[content..] + .windows(delimiter.len()) + .position(|candidate| candidate == delimiter) + .map(|end| content + end + delimiter.len()) +} + +fn normalize_field_type( + reference: &TypeRef, + package: &str, + symbols: &Symbols, + types: &[Definition], + errors: &mut Vec<Diagnostic>, +) -> Option<ModelFieldType> { + if !reference.arguments.is_empty() { + errors.push(Diagnostic::error( + "C014", + "List, Map and stream types cannot be stored as Model fields; use Json explicitly", + reference.span, + )); + return None; + } + let name = path_name(&reference.name); + let parameters = reference + .parameters + .iter() + .map(|parameter| parameter.value.parse::<u32>().ok()) + .collect::<Option<Vec<_>>>(); + let Some(parameters) = parameters else { + errors.push(Diagnostic::error( + "C014", + "storage type parameters must be unsigned integers", + reference.span, + )); + return None; + }; + let simple = match (name.as_str(), parameters.as_slice()) { + ("Bool", []) => Some(ModelFieldType::Bool), + ("Int", []) => Some(ModelFieldType::Int), + ("Float", []) => Some(ModelFieldType::Float), + ("Uuid", []) => Some(ModelFieldType::Uuid), + ("DateTime", []) => Some(ModelFieldType::DateTime), + ("Date", []) => Some(ModelFieldType::Date), + ("Time", []) => Some(ModelFieldType::Time), + ("Duration", []) => Some(ModelFieldType::Duration), + ("Json", []) => Some(ModelFieldType::Json), + ("Text", []) => Some(ModelFieldType::Text { + minimum: 0, + maximum: None, + }), + ("Text", [maximum]) if *maximum > 0 => Some(ModelFieldType::Text { + minimum: 0, + maximum: Some(*maximum), + }), + ("Text", [minimum, maximum]) if minimum <= maximum && *maximum > 0 => { + Some(ModelFieldType::Text { + minimum: *minimum, + maximum: Some(*maximum), + }) + } + ("Bytes", []) => Some(ModelFieldType::Bytes { + minimum: 0, + maximum: None, + }), + ("Bytes", [maximum]) if *maximum > 0 => Some(ModelFieldType::Bytes { + minimum: 0, + maximum: Some(*maximum), + }), + ("Bytes", [minimum, maximum]) if minimum <= maximum && *maximum > 0 => { + Some(ModelFieldType::Bytes { + minimum: *minimum, + maximum: Some(*maximum), + }) + } + ("Decimal", [precision, scale]) + if *precision > 0 && *precision <= 34 && scale <= precision => + { + Some(ModelFieldType::Decimal { + precision: *precision as u8, + scale: *scale as u8, + }) + } + ("Decimal", _) | ("Text", _) | ("Bytes", _) => { + errors.push(Diagnostic::error( + "C014", + "invalid storage type parameters", + reference.span, + )); + return None; + } + _ => None, + }; + if simple.is_some() { + return simple; + } + if !parameters.is_empty() { + errors.push(Diagnostic::error( + "C014", + "this Model field type does not accept storage parameters", + reference.span, + )); + return None; + } + let qualified = if name.contains('.') { + name.clone() + } else { + format!("{package}.{name}") + }; + let Some(id) = symbols.types.get(&qualified).copied() else { + errors.push(Diagnostic::error( + "C004", + format!("unknown Model field type '{qualified}'"), + reference.span, + )); + return None; + }; + match types[id].kind { + DefinitionKind::ModelId => Some(ModelFieldType::ModelId( + qualified.trim_end_matches(".id").into(), + )), + DefinitionKind::Regular | DefinitionKind::ModelChoice => match &types[id].shape { + Shape::Choice(variants) + if variants + .iter() + .all(|variant| !variant.error && variant.fields.is_empty()) => + { + Some(ModelFieldType::Choice(qualified)) + } + _ => { + errors.push(Diagnostic::error( + "C014", + "only no-payload choices can be stored as Model fields", + reference.span, + )); + None + } + }, + DefinitionKind::Model | DefinitionKind::ModelPage | DefinitionKind::ModelCursor => { + errors.push(Diagnostic::error( + "C014", + "Model relations must reference the target '.id' type", + reference.span, + )); + None + } + } +} + +fn normalize_value( + expression: &Expression, + ty: &ModelFieldType, + nullable: bool, + choices: &[ModelChoice], + errors: &mut Vec<Diagnostic>, +) -> Option<ModelValue> { + let value = match &expression.kind { + ExpressionKind::Literal(crate::syntax::Literal::Null) if nullable => ModelValue::Null, + ExpressionKind::Literal(crate::syntax::Literal::Bool(value)) + if matches!(ty, ModelFieldType::Bool) => + { + ModelValue::Bool(*value) + } + ExpressionKind::Literal(crate::syntax::Literal::Number(value)) + if matches!( + ty, + ModelFieldType::Int + | ModelFieldType::Float + | ModelFieldType::Decimal { .. } + | ModelFieldType::DateTime + | ModelFieldType::Date + | ModelFieldType::Time + | ModelFieldType::Duration + ) => + { + ModelValue::Number(value.clone()) + } + ExpressionKind::Unary { + operator: UnaryOperator::Negate, + value, + } if matches!( + ty, + ModelFieldType::Int + | ModelFieldType::Float + | ModelFieldType::Decimal { .. } + | ModelFieldType::Duration + ) => + { + match &value.kind { + ExpressionKind::Literal(crate::syntax::Literal::Number(value)) => { + ModelValue::Number(format!("-{value}")) + } + _ => return invalid_value(expression, errors), + } + } + ExpressionKind::Literal(crate::syntax::Literal::Text(value)) + if matches!(ty, ModelFieldType::Text { .. } | ModelFieldType::Uuid) => + { + if matches!(ty, ModelFieldType::Uuid) + && let Err(error) = dever_runtime::orm::Uuid::parse(value) + { + errors.push(Diagnostic::error( + "C014", + error.to_string(), + expression.span, + )); + return None; + } + ModelValue::Text(value.clone()) + } + ExpressionKind::Name(path) => { + let name = path_name(path); + let ModelFieldType::Choice(choice) = ty else { + return invalid_value(expression, errors); + }; + let valid = choices.iter().any(|candidate| { + candidate.name == *choice + && candidate.options.iter().any(|(variant, _)| { + name == format!("{}.{}", candidate.name, variant) + || name + == format!( + "{}.{}", + candidate.name.rsplit('.').next().unwrap(), + variant + ) + }) + }); + if !valid { + return invalid_value(expression, errors); + } + ModelValue::Choice(name) + } + _ => return invalid_value(expression, errors), + }; + if let Err(message) = validate_constant(&value, ty) { + errors.push(Diagnostic::error("C014", message, expression.span)); + return None; + } + Some(value) +} + +fn validate_constant(value: &ModelValue, ty: &ModelFieldType) -> Result<(), String> { + match (value, ty) { + (ModelValue::Number(value), ModelFieldType::Float) => value + .parse::<f64>() + .ok() + .filter(|value| value.is_finite()) + .map(|_| ()) + .ok_or_else(|| "Model Float constant must be finite".to_owned()), + (ModelValue::Number(value), ModelFieldType::Decimal { precision, scale }) => { + dever_runtime::orm::decimal_storage(value, *precision, *scale) + .map(|_| ()) + .map_err(|error| error.to_string()) + } + (ModelValue::Number(value), _) => value.parse::<i64>().map(|_| ()).map_err(|_| { + "Model integer constant must be an integer in the 64-bit range".to_owned() + }), + (ModelValue::Text(value), ModelFieldType::Text { minimum, maximum }) => { + let length = value.chars().count() as u64; + if length < u64::from(*minimum) + || maximum.is_some_and(|maximum| length > u64::from(maximum)) + { + Err( + "Model Text constant is outside the declared codepoint length bounds" + .to_owned(), + ) + } else { + Ok(()) + } + } + _ => Ok(()), + } +} + +fn invalid_value(expression: &Expression, errors: &mut Vec<Diagnostic>) -> Option<ModelValue> { + errors.push(Diagnostic::error( + "C014", + "Model defaults and Seed values must be constants matching the field type", + expression.span, + )); + None +} + +fn normalize_indexes( + package: &Package, + fields: &[ModelField], + source_by_name: &BTreeMap<String, &Field>, + errors: &mut Vec<Diagnostic>, +) -> Vec<ModelIndex> { + let known = fields + .iter() + .map(|field| field.name.as_str()) + .collect::<BTreeSet<_>>(); + let mut candidates = vec![ModelIndex { + fields: vec!["id".into()], + unique: true, + implicit: true, + }]; + for (name, field) in source_by_name { + if field.storage.index || field.storage.unique { + candidates.push(ModelIndex { + fields: vec![name.clone()], + unique: field.storage.unique, + implicit: false, + }); + } + if fields.iter().any(|model_field| { + model_field.name == *name + && (matches!(model_field.ty, ModelFieldType::ModelId(_)) + || model_field.owner.is_some()) + }) { + candidates.push(ModelIndex { + fields: vec![name.clone()], + unique: false, + implicit: true, + }); + } + } + let search_fields = source_by_name + .iter() + .filter(|(_, field)| field.storage.search.is_some()) + .map(|(name, _)| name.clone()) + .collect::<Vec<_>>(); + if let [search] = search_fields.as_slice() { + let mut indexed = fields + .iter() + .find(|field| field.owner.is_some()) + .map(|field| vec![field.name.clone()]) + .unwrap_or_default(); + indexed.push(search.clone()); + candidates.push(ModelIndex { + fields: indexed, + unique: false, + implicit: true, + }); + } + for declaration in &package.declarations { + let Declaration::ModelIndex(index) = declaration else { + continue; + }; + let names = index + .fields + .iter() + .map(|field| field.text.clone()) + .collect::<Vec<_>>(); + let mut unique_names = BTreeSet::new(); + for field in &index.fields { + if !known.contains(field.text.as_str()) { + errors.push(Diagnostic::error( + "C004", + format!("unknown Model index field '{}'", field.text), + field.span, + )); + } + if !unique_names.insert(&field.text) { + errors.push(Diagnostic::error( + "C002", + "an index cannot repeat a field", + field.span, + )); + } + } + candidates.push(ModelIndex { + fields: names, + unique: index.unique, + implicit: false, + }); + } + let mut normalized: BTreeMap<Vec<String>, ModelIndex> = BTreeMap::new(); + for index in candidates { + if let Some(existing) = normalized.get_mut(&index.fields) { + if existing.unique == index.unique && !existing.implicit && !index.implicit { + errors.push(Diagnostic::error( + "C002", + "duplicate Model index", + package.span, + )); + } + existing.unique |= index.unique; + existing.implicit &= index.implicit; + } else { + normalized.insert(index.fields.clone(), index); + } + } + normalized.into_values().collect() +} + +fn normalize_seed( + package: &Package, + fields: &[ModelField], + indexes: &[ModelIndex], + choices: &[ModelChoice], + errors: &mut Vec<Diagnostic>, +) -> Vec<SeedRow> { + let Some(seed) = package + .declarations + .iter() + .find_map(|declaration| match declaration { + Declaration::Seed(seed) => Some(seed), + _ => None, + }) + else { + return Vec::new(); + }; + let writable = fields + .iter() + .filter(|field| !field.generated) + .map(|field| (field.name.as_str(), field)) + .collect::<BTreeMap<_, _>>(); + let unique = indexes + .iter() + .filter(|index| { + index.unique + && index + .fields + .iter() + .all(|field| writable.contains_key(field.as_str())) + }) + .collect::<Vec<_>>(); + let mut rows = Vec::new(); + for row in &seed.rows { + let mut seen = BTreeSet::new(); + let mut normalized = Vec::new(); + for value in row { + let Some(field) = writable.get(value.name.text.as_str()) else { + errors.push(Diagnostic::error( + "C004", + format!("unknown or generated Seed field '{}'", value.name.text), + value.name.span, + )); + continue; + }; + if !seen.insert(value.name.text.as_str()) { + errors.push(Diagnostic::error( + "C002", + "duplicate Seed field", + value.name.span, + )); + continue; + } + if let Some(value) = + normalize_value(&value.value, &field.ty, field.nullable, choices, errors) + { + normalized.push((field.name.clone(), value)); + } + } + for field in writable.values() { + if !field.nullable && field.default.is_none() && !seen.contains(field.name.as_str()) { + errors.push(Diagnostic::error( + "C014", + format!("Seed row is missing required field '{}'", field.name), + seed.span, + )); + } + } + let identity = unique.iter().find(|index| { + index.fields.iter().all(|identity_field| { + normalized.iter().any(|(field, value)| { + field == identity_field && !matches!(value, ModelValue::Null) + }) + }) + }); + if identity.is_none() { + errors.push(Diagnostic::error( + "C014", + "each Seed row must provide non-null values for every field of a unique key", + seed.span, + )); + } + normalized.sort_by(|left, right| left.0.cmp(&right.0)); + rows.push(SeedRow { + fields: normalized, + identity: identity + .map(|index| index.fields.clone()) + .unwrap_or_default(), + }); + } + rows +} + +fn normalize_migrations( + package: &Package, + fields: &[ModelField], + errors: &mut Vec<Diagnostic>, +) -> Vec<ModelMigration> { + let current = fields + .iter() + .map(|field| field.name.as_str()) + .collect::<BTreeSet<_>>(); + package + .declarations + .iter() + .filter_map(|declaration| match declaration { + Declaration::Migration(migration) => { + let mut drops = Vec::new(); + let mut statements = Vec::new(); + let mut seen = BTreeSet::new(); + for operation in &migration.operations { + match operation { + MigrationOperation::Drop(field) => { + if current.contains(field.text.as_str()) { + errors.push(Diagnostic::error( + "C014", + "a migration cannot drop a field in the target schema", + field.span, + )); + } + if !seen.insert(&field.text) { + errors.push(Diagnostic::error( + "C002", + "duplicate migration operation", + field.span, + )); + } + drops.push(field.text.clone()); + } + MigrationOperation::Sql(sql) => { + let parameters = sql + .parameters + .iter() + .filter_map(|value| migration_parameter(value, errors)) + .collect::<Vec<_>>(); + for (text, marker, other) in + [(&sql.sqlite, '?', '$'), (&sql.postgres, '$', '?')] + { + validate_sql( + &text.value, + marker, + other, + sql.parameters.len(), + text.span, + true, + errors, + ); + } + statements.push(ModelMigrationSql { + phase: sql.phase, + sqlite: sql.sqlite.value.clone(), + postgres: sql.postgres.value.clone(), + parameters, + }); + } + } + } + Some(ModelMigration { + name: migration.name.text.clone(), + drops, + statements, + }) + } + _ => None, + }) + .collect() +} + +fn migration_parameter( + expression: &Expression, + errors: &mut Vec<Diagnostic>, +) -> Option<(ModelFieldType, ModelValue)> { + use crate::syntax::Literal; + let literal = match &expression.kind { + ExpressionKind::Literal(literal) => literal, + ExpressionKind::Unary { + operator: UnaryOperator::Negate, + value, + } => match &value.kind { + ExpressionKind::Literal(literal @ Literal::Number(_)) => literal, + _ => { + errors.push(Diagnostic::error( + "C014", + "migration parameters must be static literals", + expression.span, + )); + return None; + } + }, + _ => { + errors.push(Diagnostic::error( + "C014", + "migration parameters must be static literals", + expression.span, + )); + return None; + } + }; + let ty = match literal { + Literal::Null | Literal::Text(_) => ModelFieldType::Text { + minimum: 0, + maximum: None, + }, + Literal::Bool(_) => ModelFieldType::Bool, + Literal::Number(value) if value.contains(['e', 'E']) => ModelFieldType::Float, + Literal::Number(value) if value.contains('.') => { + errors.push(Diagnostic::error("C014", "migration Decimal parameters require an explicit SQL conversion from bound Text; use exponent notation only for Float", expression.span)); + return None; + } + Literal::Number(_) => ModelFieldType::Int, + }; + normalize_value(expression, &ty, true, &[], errors).map(|value| (ty, value)) +} + +fn validate_model_references( + schemas: &mut [ModelSchema], + packages: &[Package], + registrations: &[Registration], + errors: &mut Vec<Diagnostic>, +) { + let by_package = schemas + .iter() + .enumerate() + .map(|(index, schema)| (schema.package.clone(), index)) + .collect::<BTreeMap<_, _>>(); + for (index, registration) in registrations.iter().enumerate() { + let source_scope = schemas[index].scope; + for field_index in 0..schemas[index].fields.len() { + let ModelFieldType::ModelId(target) = &schemas[index].fields[field_index].ty else { + continue; + }; + let Some(target_index) = by_package.get(target).copied() else { + continue; + }; + let target_scope = schemas[target_index].scope; + if source_scope == ModelScope::Global && target_scope == ModelScope::Tenant { + errors.push(Diagnostic::error( + "C014", + format!( + "global Model field '{}' cannot reference tenant Model '{}'", + schemas[index].fields[field_index].name, target + ), + packages[registration.owner].span, + )); + } else if source_scope == ModelScope::Tenant && target_scope == ModelScope::Global { + schemas[index].fields[field_index].logical_reference = true; + } + } + } +} + +fn validate_relations( + schemas: &mut [ModelSchema], + packages: &[Package], + registrations: &[Registration], + errors: &mut Vec<Diagnostic>, +) { + let by_package = schemas + .iter() + .enumerate() + .map(|(index, schema)| (schema.package.clone(), index)) + .collect::<BTreeMap<_, _>>(); + for (index, registration) in registrations.iter().enumerate() { + let package = &packages[registration.owner]; + let mut names = BTreeSet::new(); + let source_scope = schemas[index].scope; + for relation_index in 0..schemas[index].relations.len() { + let relation = schemas[index].relations[relation_index].clone(); + if !names.insert(relation.name.clone()) { + errors.push(Diagnostic::error( + "C002", + "duplicate relation name", + package.span, + )); + } + let Some(target_index) = by_package.get(&relation.model).copied() else { + errors.push(Diagnostic::error( + "C004", + format!("unknown relation Model '{}'", relation.model), + package.span, + )); + continue; + }; + let target_scope = schemas[target_index].scope; + if source_scope == ModelScope::Global && target_scope == ModelScope::Tenant { + if relation.kind == ModelRelationKind::ToMany { + errors.push(Diagnostic::error( + "C014", + "a global Model cannot declare a relation to a tenant Model", + package.span, + )); + } + continue; + } + let logical = source_scope == ModelScope::Tenant && target_scope == ModelScope::Global; + let valid = match relation.kind { + ModelRelationKind::ToOne => schemas[index].fields.iter().any(|field| { + field.name == relation.field + && matches!(&field.ty, ModelFieldType::ModelId(model) if model == &schemas[target_index].package) + }), + ModelRelationKind::ToMany => schemas[target_index].fields.iter().any(|field| { + field.name == relation.field + && matches!(&field.ty, ModelFieldType::ModelId(model) if model == &schemas[index].package) + }), + }; + if !valid { + errors.push(Diagnostic::error( + "C014", + "a relation must name a foreign-key field that references this Model", + package.span, + )); + } else { + schemas[index].relations[relation_index].logical = logical; + } + } + } +} + +fn validate_table_names( + schemas: &[ModelSchema], + packages: &[Package], + registrations: &[Registration], + errors: &mut Vec<Diagnostic>, +) { + let mut names = BTreeMap::new(); + for (schema, registration) in schemas.iter().zip(registrations) { + let selector = match &schema.connection { + ConnectionSelector::Explicit(name) | ConnectionSelector::PackageRoot(name) => name, + }; + if names + .insert((selector, &schema.table), &schema.package) + .is_some() + { + errors.push(Diagnostic::error( + "C014", + "Models bound to the same logical connection cannot share a table name", + packages[registration.owner].span, + )); + } + } +} + +fn schema_identity(schema: &ModelSchema) -> String { + let mut fields = schema.fields.iter().collect::<Vec<_>>(); + fields.sort_by(|left, right| left.name.cmp(&right.name)); + let mut indexes = schema.indexes.iter().collect::<Vec<_>>(); + indexes.sort_by(|left, right| left.fields.cmp(&right.fields)); + let mut migrations = schema.migrations.iter().collect::<Vec<_>>(); + migrations.sort_by(|left, right| left.name.cmp(&right.name)); + let scope = match schema.scope { + // Keep existing tenant revisions stable when tenancy is not enabled. + ModelScope::Tenant => "", + ModelScope::Global => "global|", + }; + format!( + "{scope}{}|{}|{}|{}|{}|{}", + schema.package, + schema.table, + fields + .iter() + .map(|field| format!( + "{}:{}:{}:{}:{}:{}{}", + field.name, + model::field_type(&field.ty), + field.nullable, + field.generated, + field.default.as_ref().map(model::value).unwrap_or_default(), + field.rename_from.as_deref().unwrap_or_default(), + if field.logical_reference { + ":logical" + } else { + "" + } + )) + .collect::<Vec<_>>() + .join(";"), + indexes + .iter() + .map(|index| format!("{}:{}", index.unique, index.fields.join(","))) + .collect::<Vec<_>>() + .join(";"), + schema + .choices + .iter() + .map(|choice| format!( + "{}:{}", + choice.name, + choice + .options + .iter() + .map(|(name, _)| name.as_str()) + .collect::<Vec<_>>() + .join(",") + )) + .collect::<Vec<_>>() + .join(";"), + migrations + .iter() + .map(|migration| { + if migration.statements.is_empty() { + format!("{}:{}", migration.name, migration.drops.join(",")) + } else { + format!("{}:{}", migration.name, migration.revision()) + } + }) + .collect::<Vec<_>>() + .join(";") + ) +} + +fn seed_identity(rows: &[SeedRow]) -> String { + rows.iter() + .map(|row| { + format!( + "{}:{}", + row.identity.join(","), + row.fields + .iter() + .map(|(name, value)| format!("{name}={}", model::value(value))) + .collect::<Vec<_>>() + .join(",") + ) + }) + .collect::<Vec<_>>() + .join(";") +} + +fn model_type_name(stem: &str) -> String { + stem.split('_') + .map(|part| { + let mut chars = part.chars(); + chars + .next() + .map(|first| first.to_ascii_uppercase().to_string() + chars.as_str()) + .unwrap_or_default() + }) + .collect() +} + +fn declaration_span(declaration: &Declaration) -> crate::source::Span { + match declaration { + Declaration::Api(value) => value.span, + Declaration::Rest(value) => value.span, + Declaration::External(value) => value.span, + Declaration::Type(value) => value.span, + Declaration::Function(value) => value.span, + Declaration::Database(value) => value.span, + Declaration::Schedule(value) => value.span, + Declaration::ModelIndex(value) => value.span, + Declaration::Relation(value) => value.span, + Declaration::Seed(value) => value.span, + Declaration::Migration(value) => value.span, + Declaration::ModelSql(value) => value.span, + } +} diff --git a/crates/dever-core/src/check/names.rs b/crates/dever-core/src/check/names.rs new file mode 100644 index 0000000..9779b57 --- /dev/null +++ b/crates/dever-core/src/check/names.rs @@ -0,0 +1,188 @@ +use crate::diagnostic::Diagnostic; +use crate::syntax::{ + self, Declaration, InputKind, Name, Package, Pattern, StatementKind, TypeShape, +}; + +use super::Checked; + +pub(super) fn check(package: &Package) -> Checked<()> { + for name in &package.name { + snake(name)?; + } + for declaration in &package.declarations { + match declaration { + Declaration::Api(binding) => { + snake(&binding.action)?; + for segment in &binding.target { + snake(segment)?; + } + } + Declaration::Rest(rest) => { + for segment in rest.model.iter().flatten() { + snake(segment)?; + } + } + Declaration::External(external) => { + for capability in &external.capabilities { + snake(capability)?; + } + } + Declaration::Type(ty) => { + camel(&ty.name)?; + if matches!( + ty.name.text.as_str(), + "Bool" + | "Int" + | "Decimal" + | "Float" + | "Text" + | "Id" + | "Bytes" + | "Secret" + | "Uuid" + | "DateTime" + | "Date" + | "Time" + | "Duration" + | "Json" + | "List" + | "Map" + | "MapEntry" + | "Stream" + | "AsyncStream" + | "Channel" + | "Task" + | "Group" + ) { + return Err(Diagnostic::error( + "C002", + "type name is reserved by the language", + ty.name.span, + )); + } + match &ty.shape { + TypeShape::Record(fields) => check_fields(fields)?, + TypeShape::Choice(variants) => { + for variant in variants { + camel(&variant.name)?; + check_fields(&variant.payload)?; + } + } + } + } + Declaration::Function(function) => { + for segment in function.name.text.split('.') { + snake(&Name { + text: segment.into(), + span: function.name.span, + })?; + } + for input in &function.inputs { + snake(&input.name)?; + match &input.kind { + InputKind::Value(Pattern::Variant { bindings, .. }) => { + for binding in bindings { + if binding.text != "_" { + snake(binding)?; + } + } + } + InputKind::Handler(signature) => { + check_fields(&signature.inputs)?; + check_fields(&signature.outputs)?; + } + _ => {} + } + } + check_fields(&function.outputs)?; + for statement in &function.body { + if let StatementKind::Assign { target, .. } = &statement.kind { + for name in target { + snake(name)?; + } + } + } + } + Declaration::Database(binding) => snake(&binding.name)?, + Declaration::Schedule(schedule) => snake(&schedule.target)?, + Declaration::ModelIndex(index) => { + for field in &index.fields { + snake(field)?; + } + } + Declaration::Relation(relation) => { + snake(&relation.name)?; + for segment in &relation.field { + snake(segment)?; + } + } + Declaration::Seed(seed) => { + for row in &seed.rows { + for field in row { + snake(&field.name)?; + } + } + } + Declaration::Migration(migration) => { + snake(&migration.name)?; + for operation in &migration.operations { + match operation { + syntax::MigrationOperation::Drop(field) => snake(field)?, + syntax::MigrationOperation::Sql(_) => {} + } + } + } + Declaration::ModelSql(sql) => { + snake(&sql.name)?; + check_fields(&sql.inputs)?; + check_fields(&sql.outputs)?; + } + } + } + Ok(()) +} + +fn check_fields(fields: &[syntax::Field]) -> Checked<()> { + for field in fields { + snake(&field.name)?; + } + Ok(()) +} + +fn snake(name: &Name) -> Checked<()> { + if name + .text + .starts_with(|character: char| character.is_ascii_lowercase()) + && name.text.split('_').all(|word| { + !word.is_empty() + && word + .chars() + .all(|character| character.is_ascii_lowercase() || character.is_ascii_digit()) + }) + { + return Ok(()); + } + Err(Diagnostic::error( + "C010", + "name must use ASCII lower_snake_case", + name.span, + )) +} + +fn camel(name: &Name) -> Checked<()> { + if name + .text + .starts_with(|character: char| character.is_ascii_uppercase()) + && name + .text + .chars() + .all(|character| character.is_ascii_alphanumeric()) + { + return Ok(()); + } + Err(Diagnostic::error( + "C010", + "type and variant names must use ASCII UpperCamelCase", + name.span, + )) +} diff --git a/crates/dever-core/src/check/operators.rs b/crates/dever-core/src/check/operators.rs new file mode 100644 index 0000000..efedfab --- /dev/null +++ b/crates/dever-core/src/check/operators.rs @@ -0,0 +1,184 @@ +use super::{ + Checked, + body::Body, + constants, + expressions::{is_null, ungroup}, +}; +use crate::diagnostic::Diagnostic; +use crate::hir::{Constant, Expression as Value, ExpressionKind as Expr}; +use crate::source::Span; +use crate::syntax::{ + BinaryOperator as Op, Expression, ExpressionKind as Ast, Literal, UnaryOperator, +}; +use crate::types::Type; + +impl Body<'_, '_> { + pub(super) fn unary( + &mut self, + operator: UnaryOperator, + source: &Expression, + expected: Option<&Type>, + span: Span, + ) -> Checked<Value> { + if operator == UnaryOperator::Negate + && let Ast::Literal(Literal::Number(text)) = &ungroup(source).kind + { + return constants::literal(&Literal::Number(format!("-{text}")), expected, span); + } + let hint = if operator == UnaryOperator::Not { + Some(&Type::Bool) + } else { + expected.map(Type::base).filter(|ty| ty.numeric()) + }; + let value = self.expression(source, hint)?; + if (operator == UnaryOperator::Not && value.ty != Type::Bool) + || (operator == UnaryOperator::Negate && !value.ty.numeric()) + { + return Err(Diagnostic::error( + "C005", + "invalid unary operand type", + span, + )); + } + let ty = value.ty.clone(); + let kind = if let Expr::Constant(constant) = &value.kind { + Expr::Constant(constants::unary(operator, constant, span)?) + } else { + Expr::Unary { + operator, + value: Box::new(value), + } + }; + Ok(Value { kind, ty, span }) + } + + pub(super) fn binary( + &mut self, + left: &Expression, + operator: Op, + right: &Expression, + expected: Option<&Type>, + span: Span, + ) -> Checked<Value> { + let boolean = matches!(operator, Op::And | Op::Or); + let equality = matches!(operator, Op::Equal | Op::NotEqual); + let comparison = matches!( + operator, + Op::Less | Op::LessEqual | Op::Greater | Op::GreaterEqual + ); + let hint = if boolean { + Some(&Type::Bool) + } else if !equality && !comparison && expected.map(Type::base) == Some(&Type::Float) { + Some(&Type::Float) + } else { + None + }; + let left_null = is_null(left); + let (mut left, mut right) = if equality && left_null { + let right = self.expression(right, None)?; + (self.expression(left, Some(&right.ty))?, right) + } else if hint.is_none() && contextual_decimal_literal(left) { + let right = self.expression(right, None)?; + let hint = (right.ty == Type::Float).then_some(&Type::Float); + (self.expression(left, hint)?, right) + } else { + let left = self.expression(left, hint)?; + let right_hint = if equality && is_null(right) { + Some(&left.ty) + } else if left.ty == Type::Float && contextual_decimal_literal(right) { + Some(&Type::Float) + } else { + hint + }; + let right = self.expression(right, right_hint)?; + (left, right) + }; + let ty = if equality { + if left.ty != right.ty || !left.ty.comparable(self.context.types) { + return Err(Diagnostic::error( + "C005", + "equality requires the same comparable type; it does not promote numbers", + span, + )); + } + Type::Bool + } else if boolean { + Type::Bool + } else if operator == Op::Add && left.ty == Type::Text && right.ty == Type::Text { + Type::Text + } else { + if !left.ty.numeric() || !right.ty.numeric() { + return Err(Diagnostic::error( + "C005", + "arithmetic and ordering require numeric operands", + span, + )); + } + if matches!(operator, Op::IntegerDivide | Op::Remainder) { + if left.ty != Type::Int || right.ty != Type::Int { + return Err(Diagnostic::error( + "C005", + "// and % require Int operands", + span, + )); + } + } else { + let decimal = left.ty == Type::Decimal + || right.ty == Type::Decimal + || (operator == Op::Divide && left.ty == Type::Int && right.ty == Type::Int); + if decimal { + if left.ty == Type::Int { + left = constants::promote(left); + } + if right.ty == Type::Int { + right = constants::promote(right); + } + } + if left.ty != right.ty { + return Err(Diagnostic::error( + "C005", + "Float cannot implicitly mix with Int or Decimal", + span, + )); + } + } + if comparison { + Type::Bool + } else { + left.ty.clone() + } + }; + if matches!(operator, Op::Divide | Op::IntegerDivide | Op::Remainder) + && matches!(&right.kind, Expr::Constant(Constant::Int(0))) + { + return Err(Diagnostic::error("C008", "Int division by zero", span)); + } + if operator == Op::Divide + && matches!(&right.kind, Expr::Constant(Constant::Decimal(value)) if *value == dever_runtime::number::DecimalValue::ZERO) + { + return Err(Diagnostic::error("C008", "Decimal division by zero", span)); + } + let kind = match (&left.kind, &right.kind) { + (Expr::Constant(left), Expr::Constant(right)) => { + Expr::Constant(constants::binary(left, operator, right, span)?) + } + _ => Expr::Binary { + left: Box::new(left), + operator, + right: Box::new(right), + }, + }; + Ok(Value { kind, ty, span }) + } +} + +fn contextual_decimal_literal(expression: &Expression) -> bool { + match &ungroup(expression).kind { + Ast::Literal(Literal::Number(text)) => text.contains('.') && !text.contains(['e', 'E']), + Ast::Unary { + operator: UnaryOperator::Negate, + value, + } => contextual_decimal_literal(value), + _ => false, + } +} diff --git a/crates/dever-core/src/check/orm.rs b/crates/dever-core/src/check/orm.rs new file mode 100644 index 0000000..c839d84 --- /dev/null +++ b/crates/dever-core/src/check/orm.rs @@ -0,0 +1,944 @@ +use std::collections::BTreeSet; + +use crate::diagnostic::Diagnostic; +use crate::hir::{ + Expression as Value, ExpressionKind as Expr, ModelCondition, ModelOperation, ModelOrder, + QueryPlan, TextCondition, +}; +use crate::model::{ModelFieldType, ModelSchema, ModelSqlCardinality, ModelSqlParameter}; +use crate::source::{SourceLayout, SourceRole, Span}; +use crate::syntax::{BinaryOperator, Expression, ExpressionKind as Ast, MapEntry}; +use crate::types::{Shape, Type}; + +use super::{Checked, body::Body, path_name}; + +const MAX_CREATE_MANY_ROWS: usize = 256; +const MAX_QUERY_VALUES: usize = 100; +const MAX_SQLITE_PARAMETERS: usize = 999; + +#[derive(Clone, Copy, Eq, PartialEq)] +enum QueryKind { + Plain, + Included, + Cursor, + Stream, +} + +impl Body<'_, '_> { + pub(super) fn model_call( + &mut self, + name: &str, + arguments: &[Expression], + span: Span, + ) -> Checked<Option<Value>> { + let Some((model_name, operation_name)) = name.rsplit_once('.') else { + return Ok(None); + }; + let built_in = matches!( + operation_name, + "create" + | "create_many" + | "get" + | "first" + | "list" + | "cursor" + | "count" + | "exists" + | "stream" + | "update" + | "delete" + | "upsert" + ); + let Some((model_index, model)) = self.resolve_model(model_name, span)? else { + return Ok(None); + }; + let model = model.clone(); + let sql_operation = model + .sql + .iter() + .position(|operation| operation.name == operation_name); + if !built_in && sql_operation.is_none() { + return Ok(None); + } + let operation = match operation_name { + "create" => { + arity(name, arguments, 1, span)?; + ModelOperation::Create { + values: self.write_values(&model, &arguments[0], true)?, + } + } + "create_many" => { + arity(name, arguments, 1, span)?; + ModelOperation::CreateMany { + rows: self.create_many_values(&model, &arguments[0])?, + } + } + "get" => { + arity(name, arguments, 1, span)?; + let id = self.expression(&arguments[0], Some(&self.id_type(&model)))?; + ModelOperation::Get { id: Box::new(id) } + } + "first" | "list" | "cursor" | "count" | "exists" | "stream" => { + if arguments.len() > 1 { + return Err(Diagnostic::error( + "C005", + format!("'{name}' expects zero or one query block"), + span, + )); + } + let query = match arguments.first() { + Some(query) => self.query( + &model, + query, + match operation_name { + "first" | "list" => QueryKind::Included, + "cursor" => QueryKind::Cursor, + "stream" => QueryKind::Stream, + _ => QueryKind::Plain, + }, + )?, + None => default_query(span), + }; + match operation_name { + "first" => ModelOperation::First { query }, + "list" => ModelOperation::List { query }, + "cursor" => ModelOperation::Cursor { query }, + "count" => ModelOperation::Count { query }, + "exists" => ModelOperation::Exists { query }, + "stream" => ModelOperation::Stream { query }, + _ => unreachable!(), + } + } + "update" => { + arity(name, arguments, 2, span)?; + let query = self.target_query(&model, &arguments[0])?; + let values = self.write_values(&model, &arguments[1], false)?; + if values.is_empty() { + return Err(Diagnostic::error( + "C005", + "update requires at least one changed field", + arguments[1].span, + )); + } + ModelOperation::Update { query, values } + } + "delete" => { + arity(name, arguments, 1, span)?; + ModelOperation::Delete { + query: self.target_query(&model, &arguments[0])?, + } + } + "upsert" => { + arity(name, arguments, 3, span)?; + let key = self.write_values(&model, &arguments[0], false)?; + self.validate_upsert_key(&model, &key, arguments[0].span)?; + let create = self.write_values(&model, &arguments[1], false)?; + validate_distinct_fields(&key, &create, arguments[1].span)?; + self.validate_required_create( + &model, + key.iter().chain(&create), + arguments[1].span, + )?; + let update = self.write_values(&model, &arguments[2], false)?; + if update.is_empty() { + return Err(Diagnostic::error( + "C005", + "upsert requires at least one changed field", + arguments[2].span, + )); + } + ModelOperation::Upsert { + key, + create, + update, + } + } + _ => { + let operation = sql_operation.expect("resolved Model SQL operation"); + let declaration = &model.sql[operation]; + arity(name, arguments, declaration.parameters.len(), span)?; + let arguments = arguments + .iter() + .zip(&declaration.parameters) + .map(|(argument, parameter)| { + let expected = self.sql_parameter_type(parameter); + self.expression(argument, Some(&expected)) + }) + .collect::<Checked<Vec<_>>>()?; + ModelOperation::Sql { + operation, + arguments, + read_only: super::sql_read::proven(&model, declaration), + } + } + }; + let ty = match &operation { + ModelOperation::Create { .. } + | ModelOperation::Get { .. } + | ModelOperation::Upsert { .. } => Type::Named(model.record), + ModelOperation::First { .. } => Type::Named(model.record).nullable(), + ModelOperation::List { .. } => Type::Named(model.page), + ModelOperation::Cursor { .. } => Type::Named(model.cursor), + ModelOperation::Stream { .. } => Type::RowStream(Box::new(Type::Named(model.record))), + ModelOperation::CreateMany { .. } + | ModelOperation::Count { .. } + | ModelOperation::Update { .. } + | ModelOperation::Delete { .. } => Type::Int, + ModelOperation::Exists { .. } => Type::Bool, + ModelOperation::Sql { operation, .. } => { + let result = model.sql[*operation].result; + let row = Type::Named(result.record); + match result.cardinality { + ModelSqlCardinality::One => row, + ModelSqlCardinality::Optional => row.nullable(), + ModelSqlCardinality::Many => Type::List(Box::new(row)), + } + } + }; + Ok(Some(Value { + kind: Expr::ModelOperation { + model: model_index, + operation: Box::new(operation), + }, + ty, + span, + })) + } + + fn resolve_model(&self, name: &str, span: Span) -> Checked<Option<(usize, &ModelSchema)>> { + let caller = &self.context.symbols.packages[self.context.owner]; + let candidates = match &caller.layout { + SourceLayout::Role { + component, domain, .. + } + | SourceLayout::Test { + component, domain, .. + } => { + let local = if name == "model" { + Some(format!("{component}.{domain}.model")) + } else { + name.strip_prefix("model.") + .map(|rest| format!("{component}.{domain}.model.{rest}")) + }; + local + .into_iter() + .chain(std::iter::once(name.to_owned())) + .collect::<Vec<_>>() + } + SourceLayout::Main | SourceLayout::Loose => vec![name.to_owned()], + }; + let matches = self + .context + .models + .iter() + .enumerate() + .filter(|(_, model)| { + candidates.contains(&model.package) + || matches!(&caller.layout, SourceLayout::Loose) && model.table == name + }) + .collect::<Vec<_>>(); + match matches.as_slice() { + [] => Ok(None), + [(index, model)] => { + let target = &self.context.symbols.packages[self.context.types[model.record].owner]; + if !matches!(&caller.layout, SourceLayout::Loose) + && (caller.layout.role() != Some(SourceRole::App) + || caller.layout.domain() != target.layout.domain()) + { + return Err(Diagnostic::error( + "C006", + "Model operations are private to their owning domain App", + span, + )); + } + Ok(Some((*index, *model))) + } + _ => Err(Diagnostic::error( + "C004", + format!("Model name '{name}' is ambiguous; use its full package name"), + span, + )), + } + } + + fn write_values( + &mut self, + model: &ModelSchema, + source: &Expression, + creating: bool, + ) -> Checked<Vec<(usize, Value)>> { + let entries = map_entries(source, if creating { "create" } else { "update" })?; + let fields = self.model_fields(model).to_vec(); + let mut seen = BTreeSet::new(); + let mut values = Vec::new(); + for entry in entries { + let name = map_key(entry)?; + let Some((index, field)) = fields + .iter() + .enumerate() + .find(|(_, field)| field.name == name) + else { + return Err(Diagnostic::error( + "C004", + format!("unknown Model field '{name}'"), + entry.key.span, + )); + }; + let field_type = field.ty.clone(); + if model.fields[index].generated { + return Err(Diagnostic::error( + "C005", + format!("generated Model field '{name}' cannot be written"), + entry.key.span, + )); + } + if !seen.insert(index) { + return Err(Diagnostic::error( + "C002", + format!("duplicate Model field '{name}'"), + entry.span, + )); + } + values.push((index, self.expression(&entry.value, Some(&field_type))?)); + } + if creating { + self.validate_required_create(model, values.iter(), source.span)?; + } + Ok(values) + } + + fn create_many_values( + &mut self, + model: &ModelSchema, + source: &Expression, + ) -> Checked<Vec<Vec<(usize, Value)>>> { + let Ast::List(items) = &source.kind else { + return Err(Diagnostic::error( + "C005", + "create_many requires a static List of anonymous create blocks", + source.span, + )); + }; + if items.is_empty() || items.len() > MAX_CREATE_MANY_ROWS { + return Err(Diagnostic::error( + "C005", + format!("create_many requires between 1 and {MAX_CREATE_MANY_ROWS} rows"), + source.span, + )); + } + let mut rows = Vec::with_capacity(items.len()); + for item in items { + rows.push(self.write_values(model, item, true)?); + } + let fields = rows[0].iter().map(|(field, _)| *field).collect::<Vec<_>>(); + if rows + .iter() + .skip(1) + .any(|row| row.iter().map(|(field, _)| *field).collect::<Vec<_>>() != fields) + { + return Err(Diagnostic::error( + "C005", + "create_many rows must write the same fields in the same order", + source.span, + )); + } + let generated = model + .fields + .iter() + .filter(|field| { + field.generated && matches!(field.ty, crate::model::ModelFieldType::Uuid) + }) + .count(); + if items.len().saturating_mul(fields.len() + generated) > MAX_SQLITE_PARAMETERS { + return Err(Diagnostic::error( + "C005", + format!("create_many exceeds the {MAX_SQLITE_PARAMETERS}-parameter batch limit"), + source.span, + )); + } + Ok(rows) + } + + fn validate_required_create<'a>( + &self, + model: &ModelSchema, + values: impl Iterator<Item = &'a (usize, Value)>, + span: Span, + ) -> Checked<()> { + let present = values.map(|(field, _)| *field).collect::<BTreeSet<_>>(); + for (index, field) in model.fields.iter().enumerate() { + if !field.generated + && !field.nullable + && field.default.is_none() + && !present.contains(&index) + { + return Err(Diagnostic::error( + "C005", + format!("create is missing required Model field '{}'", field.name), + span, + )); + } + } + Ok(()) + } + + fn validate_upsert_key( + &self, + model: &ModelSchema, + key: &[(usize, Value)], + span: Span, + ) -> Checked<()> { + if key.is_empty() { + return Err(Diagnostic::error( + "C005", + "upsert key cannot be empty", + span, + )); + } + let names = key + .iter() + .map(|(field, _)| model.fields[*field].name.as_str()) + .collect::<BTreeSet<_>>(); + if !model.indexes.iter().any(|index| { + index.unique + && index.fields.len() == names.len() + && index + .fields + .iter() + .all(|field| names.contains(field.as_str())) + }) { + return Err(Diagnostic::error( + "C005", + "upsert key must exactly match a unique Model index", + span, + )); + } + Ok(()) + } + + fn target_query(&mut self, model: &ModelSchema, source: &Expression) -> Checked<QueryPlan> { + if matches!(source.kind, Ast::Map(_)) { + self.query(model, source, QueryKind::Plain) + } else { + let id = self.expression(source, Some(&self.id_type(model)))?; + Ok(QueryPlan { + condition: Some(ModelCondition::Compare { + field: 0, + operator: BinaryOperator::Equal, + value: id, + }), + ..default_query(source.span) + }) + } + } + + fn query( + &mut self, + model: &ModelSchema, + source: &Expression, + kind: QueryKind, + ) -> Checked<QueryPlan> { + let entries = map_entries(source, "query")?; + let mut query = default_query(source.span); + let mut seen = BTreeSet::new(); + for entry in entries { + let name = map_key(entry)?; + if !seen.insert(name.clone()) { + return Err(Diagnostic::error( + "C002", + format!("duplicate query option '{name}'"), + entry.span, + )); + } + match name.as_str() { + "where" => query.condition = Some(self.condition(model, &entry.value)?), + "order" => query.order = self.order(model, &entry.value)?, + "with" if matches!(kind, QueryKind::Included | QueryKind::Cursor) => { + query.relations = self.relations(model, &entry.value)? + } + "after" if kind == QueryKind::Cursor => { + query.after = Some( + self.expression(&entry.value, Some(&Type::Named(model.record).nullable()))?, + ) + } + "page" => query.page = self.expression(&entry.value, Some(&Type::Int))?, + "size" => query.size = self.expression(&entry.value, Some(&Type::Int))?, + _ => { + return Err(Diagnostic::error( + "C004", + format!("unknown query option '{name}'"), + entry.key.span, + )); + } + } + } + if !query.order.iter().any(|order| order.field == 0) { + query.order.push(ModelOrder { + field: 0, + descending: true, + }); + } + if kind == QueryKind::Cursor + && query + .order + .iter() + .any(|order| model.fields[order.field].nullable) + { + return Err(Diagnostic::error( + "C005", + "cursor ordering does not support nullable Model fields", + source.span, + )); + } + if kind == QueryKind::Stream && seen.contains("page") { + return Err(Diagnostic::error( + "C005", + "stream traverses the full query and does not accept page", + source.span, + )); + } + Ok(query) + } + + fn relations(&self, model: &ModelSchema, source: &Expression) -> Checked<Vec<usize>> { + let Ast::List(items) = &source.kind else { + return Err(Diagnostic::error( + "C005", + "with requires a static List of relation names", + source.span, + )); + }; + let mut seen = BTreeSet::new(); + let mut relations = Vec::new(); + for item in items { + let Ast::Name(path) = &item.kind else { + return Err(Diagnostic::error( + "C005", + "with entries must be bare relation names", + item.span, + )); + }; + if path.len() != 1 { + return Err(Diagnostic::error( + "C005", + "nested relation loading is not supported", + item.span, + )); + } + let Some(index) = model + .relations + .iter() + .position(|relation| relation.name == path[0].text) + else { + return Err(Diagnostic::error( + "C004", + format!("unknown Model relation '{}'", path[0].text), + item.span, + )); + }; + if !seen.insert(index) { + return Err(Diagnostic::error( + "C002", + format!("with cannot repeat relation '{}'", path[0].text), + item.span, + )); + } + if model.relations[index].logical { + return Err(Diagnostic::error( + "C014", + format!( + "logical cross-scope relation '{}' cannot be loaded with 'with'", + path[0].text + ), + item.span, + )); + } + relations.push(index); + } + Ok(relations) + } + + fn condition(&mut self, model: &ModelSchema, source: &Expression) -> Checked<ModelCondition> { + if let Ast::Group(value) = &source.kind { + return self.condition(model, value); + } + if let Ast::Unary { + operator: crate::syntax::UnaryOperator::Not, + value, + } = &source.kind + { + return Ok(ModelCondition::Not(Box::new(self.condition(model, value)?))); + } + if let Ast::Call { + function, + arguments, + } = &source.kind + { + let Ast::Name(path) = &function.kind else { + return Err(invalid_condition(source.span)); + }; + if path.len() == 1 { + return self.condition_call(model, &path[0].text, arguments, source.span); + } + return Err(invalid_condition(source.span)); + } + let Ast::Binary { + left, + operator, + right, + } = &source.kind + else { + return Err(invalid_condition(source.span)); + }; + if matches!(operator, BinaryOperator::And | BinaryOperator::Or) { + let left = Box::new(self.condition(model, left)?); + let right = Box::new(self.condition(model, right)?); + return Ok(if *operator == BinaryOperator::And { + ModelCondition::And(left, right) + } else { + ModelCondition::Or(left, right) + }); + } + if !matches!( + operator, + BinaryOperator::Equal + | BinaryOperator::NotEqual + | BinaryOperator::Less + | BinaryOperator::LessEqual + | BinaryOperator::Greater + | BinaryOperator::GreaterEqual + ) { + return Err(Diagnostic::error( + "C005", + "unsupported Model comparison", + source.span, + )); + } + let field = field_name(left).ok_or_else(|| { + Diagnostic::error( + "C005", + "the left side of a Model comparison must be a field", + left.span, + ) + })?; + let fields = self.model_fields(model).to_vec(); + let Some((index, definition)) = fields + .iter() + .enumerate() + .find(|(_, definition)| definition.name == field) + else { + return Err(Diagnostic::error( + "C004", + format!("unknown Model field '{field}'"), + left.span, + )); + }; + if matches!(right.kind, Ast::Literal(crate::syntax::Literal::Null)) + && !matches!(operator, BinaryOperator::Equal | BinaryOperator::NotEqual) + { + return Err(Diagnostic::error( + "C005", + "null only supports equality comparisons", + right.span, + )); + } + let expected = definition.ty.clone(); + let value = self.expression(right, Some(&expected))?; + Ok(ModelCondition::Compare { + field: index, + operator: *operator, + value, + }) + } + + fn condition_call( + &mut self, + model: &ModelSchema, + name: &str, + arguments: &[Expression], + span: Span, + ) -> Checked<ModelCondition> { + let expected = match name { + "in" | "contains" | "starts_with" | "ends_with" => 2, + "between" => 3, + _ => return Err(invalid_condition(span)), + }; + if arguments.len() != expected { + return Err(Diagnostic::error( + "C005", + format!("where '{name}' expects {expected} arguments"), + span, + )); + } + let field_name = field_name(&arguments[0]).ok_or_else(|| { + Diagnostic::error( + "C005", + format!("the first '{name}' argument must be a Model field"), + arguments[0].span, + ) + })?; + let fields = self.model_fields(model).to_vec(); + let Some((field, definition)) = fields + .iter() + .enumerate() + .find(|(_, definition)| definition.name == field_name) + else { + return Err(Diagnostic::error( + "C004", + format!("unknown Model field '{field_name}'"), + arguments[0].span, + )); + }; + let field_type = definition.ty.clone(); + match name { + "in" => { + let Ast::List(items) = &arguments[1].kind else { + return Err(Diagnostic::error( + "C005", + "where 'in' requires a static List as its second argument", + arguments[1].span, + )); + }; + if items.is_empty() || items.len() > MAX_QUERY_VALUES { + return Err(Diagnostic::error( + "C005", + format!("where 'in' requires between 1 and {MAX_QUERY_VALUES} values"), + arguments[1].span, + )); + } + let values = items + .iter() + .map(|item| self.expression(item, Some(&field_type))) + .collect::<Checked<Vec<_>>>()?; + Ok(ModelCondition::In { field, values }) + } + "between" => { + let lower = self.expression(&arguments[1], Some(&field_type))?; + let upper = self.expression(&arguments[2], Some(&field_type))?; + if !field_type.base().numeric() + && !matches!( + field_type.base(), + Type::DateTime | Type::Date | Type::Time | Type::Duration + ) + { + return Err(Diagnostic::error( + "C005", + "where 'between' requires an ordered numeric or time field", + arguments[0].span, + )); + } + Ok(ModelCondition::Between { + field, + lower, + upper, + }) + } + _ => { + if field_type.base() != &Type::Text { + return Err(Diagnostic::error( + "C005", + format!("where '{name}' requires a Text field"), + arguments[0].span, + )); + } + let value = self.expression(&arguments[1], Some(&Type::Text))?; + let operation = match name { + "contains" => TextCondition::Contains, + "starts_with" => TextCondition::StartsWith, + "ends_with" => TextCondition::EndsWith, + _ => unreachable!(), + }; + Ok(ModelCondition::Text { + field, + operation, + value, + }) + } + } + } + + fn order(&self, model: &ModelSchema, source: &Expression) -> Checked<Vec<ModelOrder>> { + let items: &[Expression] = match &source.kind { + Ast::List(items) => items, + _ => std::slice::from_ref(source), + }; + let fields = self.model_fields(model); + let mut seen = BTreeSet::new(); + let mut result = Vec::new(); + for item in items { + let Ast::Name(path) = &item.kind else { + return Err(Diagnostic::error( + "C005", + "order entries use 'field.asc' or 'field.desc'", + item.span, + )); + }; + if path.len() != 2 || !matches!(path[1].text.as_str(), "asc" | "desc") { + return Err(Diagnostic::error( + "C005", + "order entries use 'field.asc' or 'field.desc'", + item.span, + )); + } + let Some(index) = fields.iter().position(|field| field.name == path[0].text) else { + return Err(Diagnostic::error( + "C004", + format!("unknown Model field '{}'", path[0].text), + path[0].span, + )); + }; + if !seen.insert(index) { + return Err(Diagnostic::error( + "C002", + "order cannot repeat a Model field", + item.span, + )); + } + result.push(ModelOrder { + field: index, + descending: path[1].text == "desc", + }); + } + Ok(result) + } + + fn model_fields(&self, model: &ModelSchema) -> &[crate::types::Field] { + let Shape::Record(fields) = &self.context.types[model.record].shape else { + unreachable!("Model record type") + }; + fields + } + + fn id_type(&self, model: &ModelSchema) -> Type { + self.model_fields(model)[0].ty.clone() + } + + fn sql_parameter_type(&self, parameter: &ModelSqlParameter) -> Type { + let ty = match ¶meter.ty { + ModelFieldType::Bool => Type::Bool, + ModelFieldType::Int => Type::Int, + ModelFieldType::Float => Type::Float, + ModelFieldType::Decimal { .. } => Type::Decimal, + ModelFieldType::Text { .. } => Type::Text, + ModelFieldType::Bytes { .. } => Type::Bytes, + ModelFieldType::Uuid => Type::Uuid, + ModelFieldType::DateTime => Type::DateTime, + ModelFieldType::Date => Type::Date, + ModelFieldType::Time => Type::Time, + ModelFieldType::Duration => Type::Duration, + ModelFieldType::Json => Type::Json, + ModelFieldType::Choice(name) => Type::Named( + self.context + .types + .iter() + .position(|definition| definition.name == *name) + .expect("validated SQL choice type"), + ), + ModelFieldType::ModelId(model) => { + let id = format!("{model}.id"); + Type::Named( + self.context + .types + .iter() + .position(|definition| definition.name == id) + .expect("validated SQL Model ID type"), + ) + } + }; + if parameter.nullable { + ty.nullable() + } else { + ty + } + } +} + +fn arity(name: &str, arguments: &[Expression], expected: usize, span: Span) -> Checked<()> { + if arguments.len() == expected { + Ok(()) + } else { + Err(Diagnostic::error( + "C005", + format!("'{name}' expects {expected} argument(s)"), + span, + )) + } +} + +fn map_entries<'a>(source: &'a Expression, operation: &str) -> Checked<&'a [MapEntry]> { + match &source.kind { + Ast::Map(entries) => Ok(entries), + _ => Err(Diagnostic::error( + "C005", + format!("{operation} requires an anonymous block"), + source.span, + )), + } +} + +fn map_key(entry: &MapEntry) -> Checked<String> { + match &entry.key.kind { + Ast::Name(path) if path.len() == 1 => Ok(path_name(path)), + _ => Err(Diagnostic::error( + "C005", + "anonymous block keys must be bare names", + entry.key.span, + )), + } +} + +fn validate_distinct_fields( + left: &[(usize, Value)], + right: &[(usize, Value)], + span: Span, +) -> Checked<()> { + let left = left + .iter() + .map(|(field, _)| *field) + .collect::<BTreeSet<_>>(); + if right.iter().any(|(field, _)| left.contains(field)) { + return Err(Diagnostic::error( + "C002", + "upsert key and create blocks cannot repeat a Model field", + span, + )); + } + Ok(()) +} + +fn invalid_condition(span: Span) -> Diagnostic { + Diagnostic::error( + "C005", + "where requires Model comparisons, not/and/or, in, between or Text matching", + span, + ) +} + +fn field_name(source: &Expression) -> Option<&str> { + match &source.kind { + Ast::Name(path) if path.len() == 1 => Some(&path[0].text), + _ => None, + } +} + +fn default_query(span: Span) -> QueryPlan { + QueryPlan { + condition: None, + order: vec![ModelOrder { + field: 0, + descending: true, + }], + relations: Vec::new(), + after: None, + page: int(1, span), + size: int(20, span), + } +} + +fn int(value: i64, span: Span) -> Value { + Value { + kind: Expr::Constant(crate::hir::Constant::Int(value)), + ty: Type::Int, + span, + } +} diff --git a/crates/dever-core/src/check/ports.rs b/crates/dever-core/src/check/ports.rs new file mode 100644 index 0000000..6609105 --- /dev/null +++ b/crates/dever-core/src/check/ports.rs @@ -0,0 +1,1013 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use crate::diagnostic::Diagnostic; +use crate::hir::{ + Adapter, AdapterImplementation, ExternalAdapter, ExternalOperation, Failure, Function, + PortOperation, Program, +}; +use crate::source::{SourceLayout, SourceRole}; +use crate::syntax::{FunctionClause, FunctionKind, InputKind, Package, Pattern}; +use crate::types::{Definition, Shape, Type}; + +use super::{Context, Symbols}; + +pub(super) fn register( + packages: &[Package], + groups: &[Vec<&FunctionClause>], + symbols: &Symbols, + types: &[Definition], + functions: &mut Vec<Function>, + errors: &mut Vec<Diagnostic>, +) -> Vec<Adapter> { + register_operations(packages, groups, symbols, types, functions, errors); + let mut adapters = register_adapters(packages, groups, symbols, functions, errors); + register_external_adapters(packages, groups, types, functions, &mut adapters, errors); + validate_adapter_completeness(packages, functions, &adapters, errors); + adapters +} + +fn register_operations( + packages: &[Package], + groups: &[Vec<&FunctionClause>], + symbols: &Symbols, + types: &[Definition], + functions: &mut [Function], + errors: &mut Vec<Diagnostic>, +) { + for id in 0..functions.len() { + let source = groups[id][0]; + let owner = functions[id].owner; + let package = &packages[owner]; + if package.layout.role() == Some(SourceRole::Adapter) + && (source + .inputs + .iter() + .any(|input| input.name.text == "setting") + || source + .outputs + .iter() + .any(|field| field.name.text == "setting")) + { + errors.push(Diagnostic::error( + "C006", + "Adapter setting cannot be shadowed by a parameter or output", + source.span, + )); + } + if package.layout.role() != Some(SourceRole::Port) { + if source.bodyless || source.fails.is_some() { + errors.push(Diagnostic::error( + "C005", + "fails and bodyless contracts belong only to Port sources", + source.span, + )); + } + if functions[id].implementation + && package.layout.role() != Some(SourceRole::Adapter) + && !matches!(package.layout, SourceLayout::Test { .. }) + { + errors.push(Diagnostic::error( + "C005", + "qualified implementations belong only to Adapter or Test sources", + source.span, + )); + } + continue; + } + if groups[id].len() != 1 + || !source.bodyless + || source.fails.is_none() + || source.kind != FunctionKind::Ordinary + || source.pure + || source.recovery.is_some() + || functions[id].implementation + { + errors.push(Diagnostic::error( + "C005", + "Port operations require one ordinary bodyless signature with fails Choice", + source.span, + )); + continue; + } + if source.inputs.iter().any(|input| !matches!(&input.kind, InputKind::Value(Pattern::Typed { bounds, .. }) if bounds.is_empty())) + || source.outputs.iter().any(|field| !field.bounds.is_empty() || field.private) + { + errors.push(Diagnostic::error("C005", "Port signatures require typed value inputs and unbounded visible outputs", source.span)); + continue; + } + let context = Context { + symbols, + types, + functions, + models: &[], + owner, + }; + let failure_type = match context.resolve_type(source.fails.as_ref().unwrap()) { + Ok(Type::Named(id)) if matches!(types[id].shape, Shape::Choice(_)) => id, + Ok(_) => { + errors.push(Diagnostic::error( + "C005", + "fails requires an error choice", + source.span, + )); + continue; + } + Err(error) => { + errors.push(error); + continue; + } + }; + let Shape::Choice(variants) = &types[failure_type].shape else { + unreachable!() + }; + let failure_owner = &symbols.packages[types[failure_type].owner]; + if !types[failure_type].public + || (!failure_owner.bundled + && (failure_owner.layout.role() != Some(SourceRole::App) + || failure_owner.layout.domain() != package.layout.domain())) + { + errors.push(Diagnostic::error("C006", "Port fails must name a public App error choice in its domain or a public standard error choice", source.span)); + continue; + } + let failures: BTreeSet<_> = variants + .iter() + .enumerate() + .filter(|(_, variant)| variant.error) + .map(|(variant, _)| Failure { + ty: failure_type, + variant, + }) + .collect(); + if failures.is_empty() { + errors.push(Diagnostic::error( + "C005", + "Port failure choice must contain an error variant", + source.span, + )); + continue; + } + functions[id].port = Some(PortOperation { + identity: identity(package), + operation: source.name.text.clone(), + input_names: source + .inputs + .iter() + .map(|input| input.name.text.clone()) + .collect(), + failures, + implementations: Vec::new(), + }); + } +} + +fn register_adapters( + packages: &[Package], + groups: &[Vec<&FunctionClause>], + symbols: &Symbols, + functions: &mut [Function], + errors: &mut Vec<Diagnostic>, +) -> Vec<Adapter> { + let mut adapters: Vec<Adapter> = Vec::new(); + for (id, source_group) in groups.iter().enumerate() { + if !functions[id].implementation { + continue; + } + let source = source_group[0]; + let owner = functions[id].owner; + let package = &packages[owner]; + let Some(prefix) = package.layout.app_prefix() else { + continue; + }; + let target_name = format!("{prefix}.{}", source.name.text); + let target = symbols + .functions + .get(&(target_name, source.inputs.len())) + .copied(); + let Some(target) = target.filter(|target| functions[*target].port.is_some()) else { + errors.push(Diagnostic::error( + "C005", + "qualified implementation must target an operation of this domain's Port", + source.span, + )); + continue; + }; + let port_owner = functions[target].owner; + if !source.name.text.starts_with("port.") || source.kind != FunctionKind::Ordinary { + errors.push(Diagnostic::error( + "C005", + "Adapter implementation must use port.operation or port.topic.operation", + source.span, + )); + continue; + } + let expected = &functions[target]; + if functions[id].parameters != expected.parameters + || functions[id].outputs != expected.outputs + || source_group.iter().any(|clause| { + clause + .inputs + .iter() + .map(|input| &input.name.text) + .ne(groups[target][0] + .inputs + .iter() + .map(|input| &input.name.text)) + }) + { + errors.push(Diagnostic::error("C005", "implementation must preserve Port input names/types and ordered output names/types", source.span)); + continue; + } + let fake = matches!(package.layout, SourceLayout::Test { .. }); + let adapter_index = adapters.iter().position(|adapter| { + adapter.owner == owner && (!fake || adapter.port_owner == port_owner) + }); + let adapter_index = match adapter_index { + Some(index) if adapters[index].port_owner != port_owner => { + errors.push(Diagnostic::error( + "C005", + "one Adapter implementation owns exactly one Port identity", + source.span, + )); + continue; + } + Some(index) => index, + None => { + let name = match &package.layout { + SourceLayout::Role { topics, .. } if !topics.is_empty() => topics.join("."), + SourceLayout::Test { topic, .. } => topic.clone(), + _ => "default".into(), + }; + adapters.push(Adapter { + owner, + name, + port_owner, + identity: identity(&packages[port_owner]), + operations: BTreeMap::new(), + fake: matches!(package.layout, SourceLayout::Test { .. }), + setting: None, + implementation: AdapterImplementation::Dever, + }); + adapters.len() - 1 + } + }; + adapters[adapter_index].operations.insert(target, id); + if !adapters[adapter_index].fake { + functions[target] + .port + .as_mut() + .unwrap() + .implementations + .push(id); + } + } + adapters +} + +fn register_external_adapters( + packages: &[Package], + groups: &[Vec<&FunctionClause>], + types: &[Definition], + functions: &mut Vec<Function>, + adapters: &mut Vec<Adapter>, + errors: &mut Vec<Diagnostic>, +) { + for (owner, package) in packages.iter().enumerate() { + let declarations = package + .declarations + .iter() + .filter_map(|declaration| { + let crate::syntax::Declaration::External(external) = declaration else { + return None; + }; + Some(external) + }) + .collect::<Vec<_>>(); + let [source] = declarations.as_slice() else { + continue; + }; + if source.ecosystem == crate::syntax::ExternalEcosystem::Command + && (!source.libs.is_empty() + || package.declarations.iter().any(|declaration| { + matches!(declaration, crate::syntax::Declaration::Type(ty) if ty.setting) + })) + { + errors.push(Diagnostic::error( + "C005", + "external command does not accept setting or lib declarations", + source.span, + )); + continue; + } + if !valid_entry(&source.entry) { + errors.push(Diagnostic::error( + "C005", + "external entry must be a portable relative path without NUL, '.', '..', '\\', or ':' segments", + source.entry_span, + )); + continue; + } + let Some(domain) = package.layout.domain() else { + errors.push(Diagnostic::error( + "C005", + "external implementation belongs only to an Adapter source", + source.span, + )); + continue; + }; + let port_owners = functions + .iter() + .filter(|function| { + function.port.is_some() && packages[function.owner].layout.domain() == Some(domain) + }) + .map(|function| function.owner) + .collect::<BTreeSet<_>>(); + let port_owner = match port_owners.iter().copied().collect::<Vec<_>>().as_slice() { + [port_owner] => *port_owner, + [] => { + errors.push(Diagnostic::error( + "C005", + "external Adapter requires one Port identity in its domain", + source.span, + )); + continue; + } + _ => { + errors.push(Diagnostic::error( + "C005", + "external Adapter cannot choose between multiple Port identities in its domain", + source.span, + )); + continue; + } + }; + let Some(capabilities) = capabilities(source, errors) else { + continue; + }; + let mut lib_names = BTreeSet::new(); + let mut libs = Vec::new(); + for lib in &source.libs { + let request = match source.ecosystem { + crate::syntax::ExternalEcosystem::Exec => lib.text.clone(), + ecosystem if lib.text.starts_with(&format!("{}:", ecosystem.keyword())) => { + lib.text.clone() + } + ecosystem => format!("{}:{}", ecosystem.keyword(), lib.text), + }; + let valid = request + .split_once(':') + .is_some_and(|(ecosystem, name_version)| { + matches!(ecosystem, "pip" | "npm" | "go") + && name_version + .rsplit_once('@') + .is_some_and(|(name, version)| { + !name.is_empty() + && !name.contains(':') + && !version.is_empty() + && !version.contains('@') + }) + && !request.chars().any(char::is_whitespace) + }) + && (source.ecosystem == crate::syntax::ExternalEcosystem::Exec + || request.starts_with(&format!("{}:", source.ecosystem.keyword()))); + if !valid { + errors.push(Diagnostic::error( + "C005", + "external lib must use an exact name@version for its ecosystem", + lib.span, + )); + } else if !lib_names.insert(request.clone()) { + errors.push(Diagnostic::error( + "C002", + "duplicate external lib declaration", + lib.span, + )); + } else { + libs.push(crate::syntax::Name { + text: request, + span: lib.span, + }); + } + } + if libs.len() != source.libs.len() { + continue; + } + let contracts = functions + .iter() + .enumerate() + .filter(|(_, function)| function.owner == port_owner && function.port.is_some()) + .map(|(id, _)| id) + .collect::<Vec<_>>(); + let setting = package.declarations.iter().find_map(|declaration| { + let crate::syntax::Declaration::Type(setting) = declaration else { + return None; + }; + setting.setting.then(|| { + types + .iter() + .position(|ty| ty.owner == owner && ty.span == setting.span) + .expect("setting declaration registered as a private record") + }) + }); + let schema = match external_schema( + types, + functions, + groups, + &contracts, + setting, + &libs, + source.ecosystem, + ) { + Ok(schema) => schema, + Err((span, error)) => { + errors.push(Diagnostic::error("C005", error, span)); + continue; + } + }; + let name = match &package.layout { + SourceLayout::Role { topics, .. } if !topics.is_empty() => topics.join("."), + _ => "default".into(), + }; + let entry = external_entry(package, &source.entry); + let adapter_index = adapters.len(); + adapters.push(Adapter { + owner, + name, + port_owner, + identity: identity(&packages[port_owner]), + operations: BTreeMap::new(), + fake: false, + setting: None, + implementation: AdapterImplementation::External(ExternalAdapter { + ecosystem: source.ecosystem, + entry, + libs: libs.iter().map(|lib| lib.text.clone()).collect(), + capabilities, + schema, + }), + }); + for contract in contracts { + let implementation = functions.len(); + let operation = functions[contract] + .port + .as_ref() + .expect("Port contract") + .operation + .clone(); + functions.push(Function { + name: format!("{}.__external.{operation}", super::path_name(&package.name)), + owner, + public: false, + parameters: functions[contract].parameters.clone(), + outputs: functions[contract].outputs.clone(), + clauses: Vec::new(), + span: source.span, + suspends: true, + pure: false, + recovery: None, + kind: FunctionKind::Ordinary, + port: None, + implementation: true, + setting: None, + external: Some(ExternalOperation { + adapter: adapter_index, + contract, + }), + }); + adapters[adapter_index] + .operations + .insert(contract, implementation); + functions[contract] + .port + .as_mut() + .expect("Port contract") + .implementations + .push(implementation); + } + } +} + +fn validate_adapter_completeness( + packages: &[Package], + functions: &[Function], + adapters: &[Adapter], + errors: &mut Vec<Diagnostic>, +) { + for adapter in adapters { + if functions.iter().enumerate().any(|(id, function)| { + function.owner == adapter.port_owner + && function.port.is_some() + && !adapter.operations.contains_key(&id) + }) { + errors.push(Diagnostic::error( + "C005", + "Adapter must implement every operation of its Port", + packages[adapter.owner].span, + )); + } + } + for (owner, package) in packages.iter().enumerate() { + if package.layout.role() == Some(SourceRole::Adapter) + && !adapters.iter().any(|adapter| adapter.owner == owner) + { + errors.push(Diagnostic::error( + "C005", + "Adapter source must implement a Port", + package.span, + )); + } + } +} + +fn valid_entry(entry: &str) -> bool { + !entry.is_empty() + && !entry.starts_with('/') + && !entry.contains(['\\', ':', '\0']) + && entry + .split('/') + .all(|segment| !segment.is_empty() && !matches!(segment, "." | "..")) +} + +fn external_entry(package: &Package, entry: &str) -> String { + let directory = package.name[..package.name.len() - 1] + .iter() + .map(|name| name.text.as_str()) + .collect::<Vec<_>>() + .join("/"); + format!("module/{directory}/{entry}") +} + +fn capabilities( + source: &crate::syntax::ExternalAdapter, + errors: &mut Vec<Diagnostic>, +) -> Option<BTreeSet<&'static str>> { + let mut capabilities = BTreeSet::new(); + let mut valid = true; + for capability in &source.capabilities { + let known = match capability.text.as_str() { + "network" => "network", + "file" => "file", + "process" => "process", + "gpu" => "gpu", + _ => { + valid = false; + errors.push(Diagnostic::error( + "C005", + "unknown external capability", + capability.span, + )); + continue; + } + }; + if !capabilities.insert(known) { + valid = false; + errors.push(Diagnostic::error( + "C002", + "duplicate external capability", + capability.span, + )); + } + } + valid.then_some(capabilities) +} + +fn external_schema( + types: &[Definition], + functions: &[Function], + groups: &[Vec<&FunctionClause>], + contracts: &[usize], + setting: Option<usize>, + libs: &[crate::syntax::Name], + ecosystem: crate::syntax::ExternalEcosystem, +) -> Result<String, (crate::source::Span, String)> { + if ecosystem == crate::syntax::ExternalEcosystem::Command { + validate_command_contract(types, functions, contracts)?; + } + let policy = crate::wire::Policy::external(ecosystem); + let mut identity = String::from("dever-component-v1;"); + if ecosystem != crate::syntax::ExternalEcosystem::Exec { + identity.push_str("ecosystem="); + identity.push_str(ecosystem.keyword()); + identity.push(';'); + } + for lib in libs { + identity.push_str("lib="); + identity.push_str(&lib.text); + identity.push(';'); + } + match setting { + Some(setting) => { + let schema = crate::wire::Schema::build( + &Type::Named(setting), + types, + crate::wire::Policy::SettingInput, + ) + .map_err(|error| (types[setting].span, error))?; + identity.push_str("setting="); + identity.push_str(schema.fingerprint()); + identity.push(';'); + } + None => identity.push_str("setting=none;"), + } + for contract in contracts { + let function = &functions[*contract]; + let operation = function.port.as_ref().expect("Port contract"); + identity.push_str("operation="); + identity.push_str(&operation.operation); + identity.push(';'); + for (index, parameter) in function.parameters.iter().enumerate() { + let ty = parameter.value_type().expect("Port accepts value inputs"); + let field = crate::types::Field { + name: operation.input_names[index].clone(), + ty: ty.clone(), + private: false, + bounds: Vec::new(), + }; + let schema = crate::wire::Schema::from_field(&field, types, policy) + .map_err(|error| (groups[*contract][0].inputs[index].span, error))?; + identity.push_str("input="); + identity.push_str(&field.name); + identity.push(':'); + identity.push_str(schema.fingerprint()); + identity.push(';'); + } + for field in &function.outputs { + let schema = crate::wire::Schema::from_field(field, types, policy) + .map_err(|error| (function.span, error))?; + identity.push_str("output="); + identity.push_str(&field.name); + identity.push(':'); + identity.push_str(schema.fingerprint()); + identity.push(';'); + } + for failure in &operation.failures { + let Shape::Choice(variants) = &types[failure.ty].shape else { + unreachable!() + }; + let variant = &variants[failure.variant]; + identity.push_str(&types[failure.ty].name); + identity.push('.'); + identity.push_str(&variant.name); + identity.push(';'); + for field in &variant.fields { + let schema = crate::wire::Schema::from_field(field, types, policy) + .map_err(|error| (types[failure.ty].span, error))?; + identity.push_str("error-field="); + identity.push_str(&field.name); + identity.push(':'); + identity.push_str(schema.fingerprint()); + identity.push(';'); + } + } + } + Ok(crate::model::checksum(&identity)) +} + +fn validate_command_contract( + types: &[Definition], + functions: &[Function], + contracts: &[usize], +) -> Result<(), (crate::source::Span, String)> { + let function = &functions[contracts[0]]; + let fail = |message: &str| Err((function.span, message.to_owned())); + if contracts.len() != 1 { + return fail("external command requires exactly one Port operation"); + } + let port = function.port.as_ref().expect("Port contract"); + let parameters = function + .parameters + .iter() + .map(|parameter| parameter.value_type().expect("Port value parameter")) + .collect::<Vec<_>>(); + let args = Type::List(Box::new(Type::Text)); + let stdin = Type::Nullable(Box::new(Type::Bytes)); + if !(parameters.as_slice() == [&args] && port.input_names == ["args"] + || parameters.as_slice() == [&args, &stdin] && port.input_names == ["args", "stdin"]) + { + return fail("external command inputs must be args: List<Text> and optional stdin: Bytes?"); + } + if !matches!(function.outputs.as_slice(), [field] + if field.name == "output" + && matches!(field.ty, Type::Named(id) if types[id].name == "dever.process.Output")) + { + return fail("external command must return output: dever.process.Output"); + } + if port.failures.len() != 1 + || !port + .failures + .iter() + .all(|failure| types[failure.ty].name == "dever.process.Error") + { + return fail("external command must declare fails dever.process.Error"); + } + Ok(()) +} + +pub(super) fn expand_contract_types( + packages: &[Package], + symbols: &mut Symbols, + types: &[Definition], + sources: &[Option<&crate::syntax::TypeDeclaration>], +) { + // Discover the complete signature type graph before shape access checks. + // This grants only the App DTOs actually nested in a Port contract. + for (owner, package) in packages.iter().enumerate() { + if package.layout.role() != Some(SourceRole::Port) { + continue; + } + let mut context = Context { + symbols, + types, + functions: &[], + models: &[], + owner, + }; + let mut pending = Vec::new(); + for declaration in &package.declarations { + let crate::syntax::Declaration::Function(function) = declaration else { + continue; + }; + for input in &function.inputs { + if let InputKind::Value(Pattern::Typed { ty, .. }) = &input.kind { + contract_reference(ty, &context, &mut pending); + } + } + for field in &function.outputs { + contract_reference(&field.ty, &context, &mut pending); + } + if let Some(fails) = &function.fails { + contract_reference(fails, &context, &mut pending); + } + } + let mut visited = BTreeSet::new(); + while let Some(id) = pending.pop() { + if !visited.insert(id) { + continue; + } + let Some(source) = sources[id] else { + continue; + }; + context.owner = types[id].owner; + let fields: Vec<_> = match &source.shape { + crate::syntax::TypeShape::Record(fields) => fields.iter().collect(), + crate::syntax::TypeShape::Choice(variants) => variants + .iter() + .flat_map(|variant| &variant.payload) + .collect(), + }; + for field in fields { + contract_reference(&field.ty, &context, &mut pending); + } + } + symbols.packages[owner].contract_types = visited + .into_iter() + .map(|id| types[id].name.clone()) + .collect(); + } +} + +fn contract_reference( + reference: &crate::syntax::TypeRef, + context: &Context<'_>, + pending: &mut Vec<usize>, +) { + let name = super::path_name(&reference.name); + if let Some(id) = context + .candidates(&name) + .iter() + .find_map(|name| context.symbols.types.get(name)) + { + pending.push(*id); + } + for argument in &reference.arguments { + contract_reference(argument, context, pending); + } +} + +pub(super) fn settings( + packages: &[Package], + types: &[Definition], + functions: &mut Vec<Function>, + adapters: &mut [Adapter], + errors: &mut Vec<Diagnostic>, +) { + for (owner, package) in packages.iter().enumerate() { + for declaration in &package.declarations { + let crate::syntax::Declaration::Type(source) = declaration else { + continue; + }; + if !source.setting { + continue; + } + let Some(adapter) = adapters + .iter_mut() + .find(|adapter| adapter.owner == owner && !adapter.fake) + else { + errors.push(Diagnostic::error( + "C005", + "setting belongs to an Adapter implementing one Port", + source.span, + )); + continue; + }; + let ty = types + .iter() + .position(|ty| ty.owner == owner && ty.span == source.span) + .expect("setting declaration registered as a private record"); + if let Err(error) = crate::wire::Schema::build( + &Type::Named(ty), + types, + crate::wire::Policy::SettingInput, + ) { + errors.push(Diagnostic::error("C005", error, source.span)); + continue; + } + adapter.setting = Some(ty); + functions.push(Function { + name: format!("{}.__setting", super::path_name(&package.name)), + owner, + public: false, + parameters: Vec::new(), + outputs: vec![crate::types::Field { + name: "setting".into(), + ty: Type::Named(ty), + private: false, + bounds: Vec::new(), + }], + clauses: Vec::new(), + span: source.span, + suspends: false, + pure: true, + recovery: None, + kind: FunctionKind::Ordinary, + port: None, + implementation: false, + setting: Some(ty), + external: None, + }); + } + } +} + +fn identity(package: &Package) -> String { + let mut identity = package.layout.app_prefix().expect("Port has a domain"); + if let SourceLayout::Role { topics, .. } = &package.layout { + for topic in topics { + identity.push('.'); + identity.push_str(topic); + } + } + identity +} + +pub(super) fn validate_failures(program: &Program, errors: &mut Vec<Diagnostic>) { + let roots = program + .functions + .iter() + .enumerate() + .filter(|(_, function)| { + let package = &program.packages[function.owner]; + !package.bundled + && (function.public + || function.name == "main.main" + || package.role == Some(SourceRole::Api)) + }) + .map(|(function, _)| crate::specialize::Specialization { + function, + handlers: Vec::new(), + }) + .filter(|instance| { + program.functions[instance.function] + .parameters + .iter() + .all(|parameter| parameter.value_type().is_some()) + }); + if let Ok(reachable) = crate::specialize::reachable(&program.functions, roots) { + for instance in reachable { + let function = &program.functions[instance.function]; + if function + .port + .as_ref() + .is_some_and(|port| port.implementations.is_empty()) + { + errors.push(Diagnostic::error( + "C005", + "reachable Port requires a production Adapter", + function.span, + )); + } + } + } + for adapter in program.adapters.iter().filter(|adapter| !adapter.fake) { + validate_adapter_failures(program, adapter, errors); + } +} + +fn validate_adapter_failures(program: &Program, adapter: &Adapter, errors: &mut Vec<Diagnostic>) { + for (contract, implementation) in &adapter.operations { + let allowed = &program.functions[*contract].port.as_ref().unwrap().failures; + if !program.failures[*implementation].is_subset(allowed) + || !program.failure_handlers[*implementation].is_empty() + { + errors.push(Diagnostic::error( + "C012", + "Adapter failure set exceeds the declared Port contract", + program.functions[*implementation].span, + )); + } + } +} + +/// Checking and native emission use the same closed graph for one isolated case. +pub(crate) fn test_program(program: &Program, index: usize) -> Result<Program, Vec<Diagnostic>> { + let test = &program.tests[index]; + let owner = program.functions[test.function].owner; + let mut case = program.clone(); + for (id, function) in case.functions.iter_mut().enumerate() { + if let Some(port) = &mut function.port { + port.implementations = test.port_bindings.get(&id).copied().into_iter().collect(); + } + } + // Also check unused fake operations/helpers; completeness does not make them reachable. + let roots = case + .functions + .iter() + .enumerate() + .filter(|(_, function)| { + function.owner == owner + && function + .parameters + .iter() + .all(|parameter| parameter.value_type().is_some()) + }) + .map(|(function, _)| crate::specialize::Specialization { + function, + handlers: Vec::new(), + }); + let reachable = + crate::specialize::execution_reachable(&case.functions, roots).map_err(|span| { + vec![Diagnostic::error( + "C007", + "recursive calls through a test Port fake are not allowed", + span, + )] + })?; + if reachable.iter().any(|instance| { + case.functions[instance.function].port.is_some() + && !test.port_bindings.contains_key(&instance.function) + }) { + return Err(vec![Diagnostic::error( + "C005", + "each reachable Port requires a fake in this test file; production fallback is forbidden", + test.span, + )]); + } + // The static closure also retains uninstantiated handler helpers for ordinary + // declaration checks; runtime reachability still uses concrete specialization. + let functions = crate::contracts::Dependencies::new(&case).reachable( + case.functions + .iter() + .enumerate() + .filter(|(_, function)| function.owner == owner) + .map(|(id, _)| id) + .chain(reachable.iter().map(|instance| instance.function)), + ); + for (id, function) in case.functions.iter_mut().enumerate() { + if !functions.contains(&id) { + function.clauses.clear(); + if let Some(port) = &mut function.port { + port.implementations.clear(); + } + } + } + let mut errors = Vec::new(); + crate::contracts::check(&mut case, &mut errors).map_err(|error| vec![error])?; + for adapter in case + .adapters + .iter() + .filter(|adapter| adapter.fake && adapter.owner == owner) + { + validate_adapter_failures(&case, adapter, &mut errors); + } + if errors.is_empty() { + Ok(case) + } else { + Err(errors) + } +} + +pub(super) fn bind_tests(program: &mut Program) { + for test in &mut program.tests { + let owner = program.functions[test.function].owner; + test.port_bindings = program + .adapters + .iter() + .filter(|adapter| adapter.fake && adapter.owner == owner) + .flat_map(|adapter| { + adapter + .operations + .iter() + .map(|(port, implementation)| (*port, *implementation)) + }) + .collect(); + } +} diff --git a/crates/dever-core/src/check/sql_read.rs b/crates/dever-core/src/check/sql_read.rs new file mode 100644 index 0000000..3e2b623 --- /dev/null +++ b/crates/dever-core/src/check/sql_read.rs @@ -0,0 +1,173 @@ +//! A deliberately small proof of read-only SQL, not a general SQL parser. +//! Anything outside this complete single-table projection keeps its write effect. + +use crate::model::{ModelSchema, ModelSql}; + +pub(super) fn proven(model: &ModelSchema, operation: &ModelSql) -> bool { + dialect(model, &operation.sqlite, b'?') && dialect(model, &operation.postgres, b'$') +} + +fn dialect(model: &ModelSchema, sql: &str, marker: u8) -> bool { + let Some(tokens) = tokens(sql, marker) else { + return false; + }; + let mut query = Projection { + tokens: &tokens, + position: 0, + model, + }; + if !query.keyword("SELECT") || !query.fields(false) || !query.keyword("FROM") { + return false; + } + if !query.identifier(&model.table) { + return false; + } + if query.keyword("WHERE") { + loop { + if !query.field() || !query.take(Token::Equal) || !query.take(Token::Bind) { + return false; + } + if !query.keyword("AND") { + break; + } + } + } + if query.keyword("ORDER") && (!query.keyword("BY") || !query.fields(true)) { + return false; + } + if query.keyword("LIMIT") && !query.take(Token::Bind) { + return false; + } + query.take(Token::Terminator); + query.position == tokens.len() +} + +#[derive(Clone, Copy, Eq, PartialEq)] +enum Token<'a> { + Word(&'a str), + Quoted(&'a str), + Bind, + Comma, + Equal, + Terminator, +} + +impl Token<'_> { + fn identifier(self, expected: &str) -> bool { + match self { + Self::Word(name) => name.eq_ignore_ascii_case(expected), + Self::Quoted(name) => name == expected, + _ => false, + } + } +} + +fn tokens(sql: &str, marker: u8) -> Option<Vec<Token<'_>>> { + let bytes = sql.as_bytes(); + let mut tokens = Vec::new(); + let mut position = 0; + while position < bytes.len() { + let start = position; + let byte = bytes[position]; + position += 1; + let token = match byte { + b if b.is_ascii_whitespace() => continue, + b',' => Token::Comma, + b'=' => Token::Equal, + b';' => Token::Terminator, + b'"' => { + let end = bytes[position..].iter().position(|byte| *byte == b'"')? + position; + let name = &sql[position..end]; + position = end + 1; + Token::Quoted(name) + } + b if b == marker => { + let digits = position; + while bytes.get(position).is_some_and(u8::is_ascii_digit) { + position += 1; + } + if position == digits { + return None; + } + // Existing Model SQL validation owns index bounds and complete binding. + Token::Bind + } + b if b.is_ascii_alphabetic() || b == b'_' => { + while bytes + .get(position) + .is_some_and(|b| b.is_ascii_alphanumeric() || *b == b'_') + { + position += 1; + } + Token::Word(&sql[start..position]) + } + _ => return None, + }; + tokens.push(token); + } + Some(tokens) +} + +struct Projection<'a> { + tokens: &'a [Token<'a>], + position: usize, + model: &'a ModelSchema, +} + +impl Projection<'_> { + fn take(&mut self, token: Token<'_>) -> bool { + if self.tokens.get(self.position) != Some(&token) { + return false; + } + self.position += 1; + true + } + + fn keyword(&mut self, keyword: &str) -> bool { + if !matches!(self.tokens.get(self.position), Some(Token::Word(word)) if word.eq_ignore_ascii_case(keyword)) + { + return false; + } + self.position += 1; + true + } + + fn identifier(&mut self, expected: &str) -> bool { + if !self + .tokens + .get(self.position) + .is_some_and(|token| token.identifier(expected)) + { + return false; + } + self.position += 1; + true + } + + fn field(&mut self) -> bool { + if !self.tokens.get(self.position).is_some_and(|token| { + self.model + .fields + .iter() + .any(|field| token.identifier(&field.name)) + }) { + return false; + } + self.position += 1; + true + } + + fn fields(&mut self, ordered: bool) -> bool { + loop { + if !self.field() { + return false; + } + if ordered && !self.keyword("ASC") { + self.keyword("DESC"); + } + if !self.take(Token::Comma) { + return true; + } + } + } +} diff --git a/crates/dever-core/src/check/symbols.rs b/crates/dever-core/src/check/symbols.rs new file mode 100644 index 0000000..f6381d5 --- /dev/null +++ b/crates/dever-core/src/check/symbols.rs @@ -0,0 +1,694 @@ +use std::collections::BTreeSet; + +use crate::diagnostic::Diagnostic; +use crate::source::{SourceLayout, SourceRole, Span}; +use crate::syntax::{self, TypeRef}; +use crate::types::{Definition, DefinitionKind, Field, Shape, Type, Variant}; + +use super::{Checked, Context, cycles, path_name}; + +impl Context<'_> { + pub(super) fn owns_private_fields(&self, id: usize) -> bool { + let definition = &self.types[id]; + if definition.kind != crate::types::DefinitionKind::Model { + return definition.owner == self.owner; + } + let caller = &self.symbols.packages[self.owner].layout; + let owner = &self.symbols.packages[definition.owner].layout; + caller.role() == Some(crate::source::SourceRole::App) && caller.domain() == owner.domain() + } + pub(super) fn is_test(&self) -> bool { + matches!( + &self.symbols.packages[self.owner].layout, + SourceLayout::Test { .. } + ) + } + + fn fragment_main(&self) -> bool { + !self.symbols.strict_layout + && matches!( + &self.symbols.packages[self.owner].layout, + SourceLayout::Main + ) + } + + pub(super) fn candidates(&self, name: &str) -> Vec<String> { + let scope = &self.symbols.packages[self.owner]; + if !name.contains('.') { + return vec![match scope.layout.role() { + Some(SourceRole::App) => { + format!("{}.{name}", scope.layout.app_prefix().unwrap()) + } + _ => format!("{}.{name}", scope.name), + }]; + } + + let mut candidates = Vec::new(); + if let SourceLayout::Role { + component, domain, .. + } + | SourceLayout::Test { + component, domain, .. + } = &scope.layout + { + if let Some(rest) = name.strip_prefix("app.") { + candidates.push(format!("{component}.{domain}.{rest}")); + } else if matches!( + name.split('.').next(), + Some("domain" | "model" | "port" | "adapter" | "api" | "job") + ) { + candidates.push(format!("{component}.{domain}.{name}")); + } + candidates.push(name.to_owned()); + candidates.push(format!("{component}.{name}")); + } else { + candidates.push(name.to_owned()); + } + candidates.dedup(); + candidates + } + + pub(super) fn has_function(&self, name: &str, arity: usize) -> bool { + self.candidates(name).iter().any(|candidate| { + self.symbols + .functions + .contains_key(&(candidate.clone(), arity)) + }) + } + + pub(super) fn named_type(&self, name: &str, span: Span) -> Checked<usize> { + let (name, id) = self + .candidates(name) + .into_iter() + .find_map(|candidate| { + self.symbols + .types + .get(&candidate) + .copied() + .map(|id| (candidate, id)) + }) + .ok_or_else(|| Diagnostic::error("C004", format!("unknown type '{name}'"), span))?; + if !self.can_access_type(id) { + let target = &self.symbols.packages[self.types[id].owner]; + let legacy = matches!( + &self.symbols.packages[self.owner].layout, + SourceLayout::Loose + ) || matches!(&target.layout, SourceLayout::Loose) + || self.fragment_main(); + return Err(Diagnostic::error( + "C006", + if legacy { + format!("type '{name}' is private") + } else { + format!("type '{name}' is outside the current domain boundary") + }, + span, + )); + } + Ok(id) + } + + pub(super) fn variant(&self, name: &str, span: Span) -> Checked<Option<(usize, usize)>> { + let Some((type_name, variant_name)) = name.rsplit_once('.') else { + return Ok(None); + }; + let Some(id) = self + .candidates(type_name) + .into_iter() + .find_map(|name| self.symbols.types.get(&name).copied()) + else { + return Ok(None); + }; + self.named_type(type_name, span)?; + let Shape::Choice(variants) = &self.types[id].shape else { + return Ok(None); + }; + let variant = variants + .iter() + .position(|variant| variant.name == variant_name) + .ok_or_else(|| Diagnostic::error("C004", format!("unknown variant '{name}'"), span))?; + Ok(Some((id, variant))) + } + + pub(super) fn related_variant( + &self, + name: &str, + arguments: &[TypeRef], + span: Span, + ) -> Checked<Option<(Type, usize, Vec<Field>)>> { + let variant = match name { + "Related.Unloaded" => 0, + "Related.Loaded" => 1, + _ => return Ok(None), + }; + if arguments.len() != 1 { + return Err(Diagnostic::error( + "C005", + "Related variant patterns require one Model type argument", + span, + )); + } + let payload = self.resolve_type(&arguments[0])?; + if !related_payload(&payload, self.types) { + return Err(Diagnostic::error( + "C005", + "Related payload must be a Model, nullable Model or List<Model>", + arguments[0].span, + )); + } + let fields = if variant == 0 { + Vec::new() + } else { + vec![Field { + name: "value".into(), + ty: payload.clone(), + private: false, + bounds: Vec::new(), + }] + }; + Ok(Some((Type::Related(Box::new(payload)), variant, fields))) + } + + pub(super) fn function(&self, name: &str, arity: usize, span: Span) -> Checked<usize> { + let (qualified, id) = self + .candidates(name) + .into_iter() + .find_map(|candidate| { + self.symbols + .functions + .get(&(candidate.clone(), arity)) + .copied() + .map(|id| (candidate, id)) + }) + .ok_or_else(|| { + Diagnostic::error( + "C004", + format!("unknown function '{name}' with {arity} argument(s)"), + span, + ) + })?; + if !self.can_call(id) { + let target = &self.symbols.packages[self.functions[id].owner]; + let legacy = matches!( + &self.symbols.packages[self.owner].layout, + SourceLayout::Loose + ) || matches!(&target.layout, SourceLayout::Loose) + || self.fragment_main(); + return Err(Diagnostic::error( + "C006", + if legacy { + format!("function '{qualified}' is private") + } else { + format!("function '{qualified}' is not callable from this source role") + }, + span, + )); + } + Ok(id) + } + + fn can_call(&self, function: usize) -> bool { + if matches!( + self.functions[function].kind, + crate::syntax::FunctionKind::Job { .. } + ) { + return false; + } + if self.functions[function].implementation { + return false; + } + let target = &self.symbols.packages[self.functions[function].owner]; + let caller = &self.symbols.packages[self.owner]; + if caller.layout.role() == Some(SourceRole::Job) && target.bundled { + return false; + } + if matches!(&target.layout, SourceLayout::Test { .. }) { + return matches!(&caller.layout, SourceLayout::Test { .. }) + && self.functions[function].owner == self.owner; + } + if target.bundled { + return self.functions[function].owner == self.owner || self.functions[function].public; + } + if self.fragment_main() { + return self.functions[function].owner == self.owner || self.functions[function].public; + } + if target.layout.role() == Some(SourceRole::Api) { + return false; + } + if matches!(&caller.layout, SourceLayout::Loose) + || matches!(&target.layout, SourceLayout::Loose) + { + return self.functions[function].owner == self.owner || self.functions[function].public; + } + if matches!(&caller.layout, SourceLayout::Test { .. }) { + return match target.layout.role() { + Some(SourceRole::App) => true, + Some(SourceRole::Domain) => caller.layout.domain() == target.layout.domain(), + Some( + SourceRole::Model + | SourceRole::Port + | SourceRole::Adapter + | SourceRole::Api + | SourceRole::Job, + ) + | None => false, + }; + } + match caller.layout.role() { + None => target.layout.role() == Some(SourceRole::App), + Some(SourceRole::Api | SourceRole::Job) => { + target.layout.role() == Some(SourceRole::App) + && caller.layout.domain() == target.layout.domain() + } + Some(SourceRole::App) => { + target.layout.role() == Some(SourceRole::App) + || (matches!( + target.layout.role(), + Some(SourceRole::Domain | SourceRole::Port) + ) && caller.layout.domain() == target.layout.domain()) + } + Some(SourceRole::Domain) => { + target.layout.role() == Some(SourceRole::Domain) + && caller.layout.domain() == target.layout.domain() + } + Some(SourceRole::Adapter) => self.functions[function].owner == self.owner, + Some(SourceRole::Model | SourceRole::Port) => false, + } + } + + fn can_access_type(&self, definition: usize) -> bool { + let target = &self.symbols.packages[self.types[definition].owner]; + let caller = &self.symbols.packages[self.owner]; + if matches!(&target.layout, SourceLayout::Test { .. }) { + return matches!(&caller.layout, SourceLayout::Test { .. }) + && self.types[definition].owner == self.owner; + } + if target.bundled { + return self.types[definition].owner == self.owner || self.types[definition].public; + } + if self.fragment_main() { + return self.types[definition].owner == self.owner || self.types[definition].public; + } + if matches!(&caller.layout, SourceLayout::Loose) + || matches!(&target.layout, SourceLayout::Loose) + { + return self.types[definition].owner == self.owner || self.types[definition].public; + } + if target.layout.role() == Some(SourceRole::Model) { + return self.types[definition].public + && !matches!( + caller.layout.role(), + Some(SourceRole::Domain | SourceRole::Port | SourceRole::Adapter) + ); + } + if self.types[definition].owner == self.owner { + return true; + } + if matches!(&caller.layout, SourceLayout::Test { .. }) { + return match target.layout.role() { + Some(SourceRole::App) => self.types[definition].public, + Some(SourceRole::Domain | SourceRole::Port) => { + caller.layout.domain() == target.layout.domain() + } + Some(SourceRole::Model) => self.types[definition].public, + Some(SourceRole::Adapter | SourceRole::Api | SourceRole::Job) | None => false, + }; + } + if matches!( + caller.layout.role(), + Some(SourceRole::Port | SourceRole::Adapter) + ) { + if target.layout.role() == Some(SourceRole::App) + && self.types[definition].public + && caller.layout.domain() == target.layout.domain() + { + let name = &self.types[definition].name; + return if caller.layout.role() == Some(SourceRole::Port) { + caller.contract_types.contains(name) + } else { + self.symbols.packages.iter().any(|port| { + caller.implemented_ports.contains(&port.name) + && port.contract_types.contains(name) + }) + }; + } + return target.layout.role() == Some(SourceRole::Port) + && caller.implemented_ports.contains(&target.name); + } + match target.layout.role() { + Some(SourceRole::App) => self.types[definition].public, + Some(SourceRole::Domain | SourceRole::Port) => { + caller.layout.domain() == target.layout.domain() + && (caller.layout.role() == Some(SourceRole::App) + || caller.layout.role() == target.layout.role()) + } + Some(SourceRole::Adapter | SourceRole::Api | SourceRole::Model | SourceRole::Job) + | None => false, + } + } + + pub(super) fn resolve_type(&self, reference: &TypeRef) -> Checked<Type> { + let arguments = reference + .arguments + .iter() + .map(|ty| self.resolve_type(ty)) + .collect::<Checked<Vec<_>>>()?; + if arguments.iter().any(|ty| ty == &Type::Upload) { + return Err(Diagnostic::error( + "C005", + "Upload cannot be stored in a container", + reference.span, + )); + } + let name = path_name(&reference.name); + let ty = match (name.as_str(), arguments.as_slice()) { + ("Bool", []) => Type::Bool, + ("Int", []) => Type::Int, + ("Decimal", []) => Type::Decimal, + ("Float", []) => Type::Float, + ("Text", []) => Type::Text, + ("Id", []) => Type::Id, + ("Bytes", []) => Type::Bytes, + ("Secret", []) => Type::Secret, + ("Upload", []) => Type::Upload, + ("Uuid", []) => Type::Uuid, + ("DateTime", []) => Type::DateTime, + ("Date", []) => Type::Date, + ("Time", []) => Type::Time, + ("Duration", []) => Type::Duration, + ("Json", []) => Type::Json, + ("dever.system.File", []) => Type::File, + ("dever.system.Socket", []) => Type::Socket, + ("dever.system.Listener", []) => Type::Listener, + ("dever.system.HttpReply", []) => Type::HttpReply, + ("dever.system.WebSocket", []) => Type::WebSocket, + ("dever.system.ClientTls", []) => Type::ClientTls, + ("dever.system.ServerTls", []) => Type::ServerTls, + ("dever.system.HttpClient", []) => Type::HttpClient, + ("List", [element]) => Type::List(Box::new(element.clone())), + ("Map", [key, value]) => Type::Map(Box::new(key.clone()), Box::new(value.clone())), + ("MapEntry", [key, value]) => { + Type::MapEntry(Box::new(key.clone()), Box::new(value.clone())) + } + ("Stream", [element]) => Type::Stream(Box::new(element.clone())), + ("AsyncStream", [element]) => Type::AsyncStream(Box::new(element.clone())), + ("Channel", [element]) => Type::Channel(Box::new(element.clone())), + ("Related", [element]) if related_payload(element, self.types) => { + Type::Related(Box::new(element.clone())) + } + ("Related", [_]) => { + return Err(Diagnostic::error( + "C005", + "Related payload must be a Model, nullable Model or List<Model>", + reference.span, + )); + } + ("List" | "Map" | "MapEntry" | "Stream" | "AsyncStream" | "Channel" | "Related", _) => { + return Err(Diagnostic::error( + "C005", + "incorrect number of type arguments", + reference.span, + )); + } + (_, []) => Type::Named(self.named_type(&name, reference.span)?), + _ => { + return Err(Diagnostic::error( + "C005", + "user-defined generic types are not supported", + reference.span, + )); + } + }; + if reference.nullable && ty == Type::Upload { + return Err(Diagnostic::error( + "C005", + "Upload cannot be nullable", + reference.span, + )); + } + Ok(if reference.nullable { + ty.nullable() + } else { + ty + }) + } + + pub(super) fn fields(&self, fields: &[syntax::Field]) -> Checked<Vec<Field>> { + let mut names = BTreeSet::new(); + fields + .iter() + .map(|field| { + if !names.insert(&field.name.text) { + return Err(Diagnostic::error( + "C002", + "duplicate field or output name", + field.span, + )); + } + let ty = self.resolve_type(&field.ty)?; + Ok(Field { + name: field.name.text.clone(), + bounds: super::clauses::field_bounds(field, &ty, self)?, + ty, + private: field.private, + }) + }) + .collect() + } + + pub(super) fn shape(&self, shape: &syntax::TypeShape) -> Checked<Shape> { + match shape { + syntax::TypeShape::Record(fields) => Ok(Shape::Record(self.fields(fields)?)), + syntax::TypeShape::Choice(variants) => { + let mut names = BTreeSet::new(); + Ok(Shape::Choice( + variants + .iter() + .map(|variant| { + if !names.insert(&variant.name.text) { + return Err(Diagnostic::error( + "C002", + "duplicate choice variant", + variant.span, + )); + } + Ok(Variant { + name: variant.name.text.clone(), + fields: self.fields(&variant.payload)?, + error: variant.error, + label: variant.label.clone(), + }) + }) + .collect::<Checked<_>>()?, + )) + } + } + } + + pub(super) fn map_key(&self, ty: &Type) -> bool { + map_key(ty, self.types) + } + + pub(super) fn value_fields(&self, ty: &Type) -> Option<Vec<Field>> { + match ty { + Type::Named(id) => match &self.types[*id].shape { + Shape::Record(fields) => Some(fields.clone()), + _ => None, + }, + Type::MapEntry(key, value) => Some(vec![ + Field { + name: "key".into(), + ty: *key.clone(), + private: false, + bounds: Vec::new(), + }, + Field { + name: "value".into(), + ty: *value.clone(), + private: false, + bounds: Vec::new(), + }, + ]), + Type::Outputs(fields) => Some(fields.clone()), + _ => None, + } + } + + pub(super) fn model_choice(&self, name: &str, span: Span) -> Checked<Option<usize>> { + let Some(id) = self + .candidates(name) + .into_iter() + .find_map(|name| self.symbols.types.get(&name).copied()) + else { + return Ok(None); + }; + if self.types[id].kind != DefinitionKind::ModelChoice { + return Ok(None); + } + self.named_type(name, span).map(Some) + } +} + +fn map_key(ty: &Type, definitions: &[Definition]) -> bool { + match ty { + Type::Bool | Type::Int | Type::Text | Type::Id => true, + Type::Named(id) => { + definitions[*id].kind == DefinitionKind::ModelId + || matches!(&definitions[*id].shape, Shape::Choice(variants) if variants.iter().all(|variant| variant.fields.is_empty())) + } + _ => false, + } +} + +pub(super) fn referenced_types(ty: &Type, visit: &mut impl FnMut(usize)) { + match ty { + Type::Named(id) => visit(*id), + Type::List(element) + | Type::Nullable(element) + | Type::Stream(element) + | Type::AsyncStream(element) + | Type::RowStream(element) + | Type::Related(element) + | Type::Channel(element) => { + if !matches!(ty, Type::Related(_)) { + referenced_types(element, visit) + } + } + Type::Map(key, value) | Type::MapEntry(key, value) => { + referenced_types(key, visit); + referenced_types(value, visit); + } + Type::Outputs(fields) | Type::Task(fields) => { + for field in fields { + referenced_types(&field.ty, visit); + } + } + _ => {} + } +} + +pub(super) fn shape_fields(shape: &Shape) -> Vec<&Field> { + match shape { + Shape::Record(fields) => fields.iter().collect(), + Shape::Choice(variants) => variants + .iter() + .flat_map(|variant| &variant.fields) + .collect(), + } +} + +pub(super) fn validate_maps(ty: &Type, types: &[Definition], span: Span) -> Checked<()> { + match ty { + Type::Map(key, value) | Type::MapEntry(key, value) => { + if !map_key(key, types) { + return Err(Diagnostic::error("C005", "invalid Map key type", span)); + } + validate_maps(value, types, span) + } + Type::List(element) + | Type::Nullable(element) + | Type::Stream(element) + | Type::AsyncStream(element) + | Type::RowStream(element) + | Type::Related(element) + | Type::Channel(element) => validate_maps(element, types, span), + _ => Ok(()), + } +} + +pub(super) fn validate_type_constraints( + ty: &Type, + types: &[Definition], + span: Span, +) -> Checked<()> { + validate_maps(ty, types, span)?; + validate_transfer_containers(ty, types, span) +} + +fn validate_transfer_containers(ty: &Type, types: &[Definition], span: Span) -> Checked<()> { + match ty { + Type::AsyncStream(element) | Type::RowStream(element) | Type::Channel(element) => { + if !element.transferable(types) { + let container = if matches!(ty, Type::AsyncStream(_) | Type::RowStream(_)) { + "AsyncStream" + } else { + "Channel" + }; + return Err(Diagnostic::error( + "C005", + format!("{container} item type must be transferable between threads"), + span, + )); + } + validate_transfer_containers(element, types, span) + } + Type::List(element) + | Type::Nullable(element) + | Type::Related(element) + | Type::Stream(element) => validate_transfer_containers(element, types, span), + Type::Map(key, value) | Type::MapEntry(key, value) => { + validate_transfer_containers(key, types, span)?; + validate_transfer_containers(value, types, span) + } + _ => Ok(()), + } +} + +fn related_payload(ty: &Type, definitions: &[Definition]) -> bool { + match ty { + Type::Named(id) => definitions[*id].kind == DefinitionKind::Model, + Type::Nullable(inner) => { + matches!( + inner.as_ref(), + Type::Named(id) if definitions[*id].kind == DefinitionKind::Model + ) + } + Type::List(inner) => { + matches!( + inner.as_ref(), + Type::Named(id) if definitions[*id].kind == DefinitionKind::Model + ) + } + _ => false, + } +} + +pub(super) fn validate_types(types: &[Definition], errors: &mut Vec<Diagnostic>) { + let mut graph = vec![Vec::new(); types.len()]; + for (id, definition) in types.iter().enumerate() { + for field in shape_fields(&definition.shape) { + let mut upload = false; + crate::types::visit_type(&field.ty, &mut |ty| upload |= matches!(ty, Type::Upload)); + if upload { + errors.push(Diagnostic::error( + "C005", + "Upload cannot be stored in a record or choice", + definition.span, + )); + } + referenced_types(&field.ty, &mut |next| { + graph[id].push((next, definition.span)) + }); + if let Err(error) = validate_maps(&field.ty, types, definition.span) { + errors.push(error); + } + } + } + if let Some(span) = cycles::cycle(&graph) { + errors.push(Diagnostic::error( + "C007", + "recursive type definitions are not allowed", + span, + )); + } else if errors.is_empty() { + crate::types::cache_properties(types, &graph); + for definition in types { + for field in shape_fields(&definition.shape) { + if let Err(error) = validate_transfer_containers(&field.ty, types, definition.span) + { + errors.push(error); + } + } + } + } +} diff --git a/crates/dever-core/src/check/upload.rs b/crates/dever-core/src/check/upload.rs new file mode 100644 index 0000000..4b80927 --- /dev/null +++ b/crates/dever-core/src/check/upload.rs @@ -0,0 +1,147 @@ +use super::{Checked, body::Body}; +use crate::diagnostic::Diagnostic; +use crate::hir::{Expression, ExpressionKind}; +use crate::intrinsic::Intrinsic; +use crate::source::Span; +use crate::syntax::{Expression as SourceExpression, ExpressionKind as Ast}; +use crate::types::Type; + +pub(super) fn validate_signatures( + functions: &[crate::hir::Function], + routes: &[crate::hir::ApiRoute], + errors: &mut Vec<Diagnostic>, +) { + for (id, function) in functions.iter().enumerate() { + if crate::source::SourceMap::is_standard(function.span.source) { + continue; + } + let allowed = routes + .iter() + .any(|route| route.function == id && route.method == "POST") + || function.port.is_some() + || function.implementation; + for parameter in &function.parameters { + let Some(ty) = parameter.value_type() else { + if let crate::types::Parameter::Handler(handler) = parameter + && handler + .parameters + .iter() + .chain(handler.outputs.iter().map(|field| &field.ty)) + .any(|ty| ty == &Type::Upload) + { + errors.push(Diagnostic::error( + "C005", + "Upload cannot cross a handler boundary", + function.span, + )); + } + continue; + }; + let mut upload = false; + crate::types::visit_type(ty, &mut |ty| upload |= matches!(ty, Type::Upload)); + if upload && (!allowed || ty != &Type::Upload) { + errors.push(Diagnostic::error( + "C005", + "Upload requires a direct POST App, Port or Adapter input", + function.span, + )); + } + } + for field in &function.outputs { + let mut upload = false; + crate::types::visit_type(&field.ty, &mut |ty| upload |= matches!(ty, Type::Upload)); + if upload { + errors.push(Diagnostic::error( + "C005", + "Upload cannot be returned", + function.span, + )); + } + } + } +} + +impl Body<'_, '_> { + pub(super) fn upload_call( + &mut self, + name: &str, + arguments: &[SourceExpression], + span: Span, + ) -> Option<Checked<Expression>> { + let (operation, ty, consume) = match name { + "dever.api.upload_filename" | "dever.system.upload_filename" => { + (Intrinsic::UploadFilename, Type::Text, false) + } + "dever.api.upload_content_type" | "dever.system.upload_content_type" => { + (Intrinsic::UploadContentType, Type::Text, false) + } + "dever.api.upload_size" | "dever.system.upload_size" => { + (Intrinsic::UploadSize, Type::Int, false) + } + "dever.api.close_upload" | "dever.system.upload_close" => { + (Intrinsic::UploadClose, Type::Unit, true) + } + "dever.system.upload_store" => { + let result = self.context.named_type("dever.storage.PutResult", span); + match result { + Ok(result) => (Intrinsic::UploadStore, Type::Named(result), true), + Err(error) => return Some(Err(error)), + } + } + _ => return None, + }; + Some((|| { + if name.starts_with("dever.system.") + && !self.context.symbols.packages[self.context.owner].bundled + { + return Err(Diagnostic::error( + "C006", + "system upload primitives are reserved for the official library", + span, + )); + } + let [argument] = arguments else { + return Err(Diagnostic::error( + "C005", + "upload operation requires one Upload local", + span, + )); + }; + let Ast::Name(path) = &argument.kind else { + return Err(Diagnostic::error( + "C005", + "upload operation requires an Upload local", + span, + )); + }; + if path.len() != 1 { + return Err(Diagnostic::error( + "C005", + "upload operation requires an Upload local", + span, + )); + } + let (slot, input) = self.affine_local(&path[0].text, argument.span, consume)?; + if input != Type::Upload { + return Err(Diagnostic::error( + "C005", + "upload operation requires Upload", + span, + )); + } + Ok(Expression { + kind: ExpressionKind::Intrinsic { + operation, + handler: None, + arguments: vec![Expression { + kind: ExpressionKind::Local(slot), + ty: input, + span: argument.span, + }], + }, + ty, + span, + }) + })()) + } +} diff --git a/crates/dever-core/src/check/values.rs b/crates/dever-core/src/check/values.rs new file mode 100644 index 0000000..3f9d758 --- /dev/null +++ b/crates/dever-core/src/check/values.rs @@ -0,0 +1,313 @@ +use std::collections::BTreeSet; + +use super::{ + Checked, + body::Body, + expressions::{is_null, needs_context}, + path_name, +}; +use crate::diagnostic::Diagnostic; +use crate::hir::{Expression as Value, ExpressionKind as Expr}; +use crate::source::Span; +use crate::syntax::{self, Expression}; +use crate::types::{DefinitionKind, Shape, Type}; + +impl Body<'_, '_> { + pub(super) fn record( + &mut self, + name: &syntax::Path, + fields: &[syntax::RecordField], + span: Span, + ) -> Checked<Value> { + let id = self.context.named_type(&path_name(name), span)?; + if matches!( + self.context.types[id].kind, + DefinitionKind::ModelId | DefinitionKind::ModelPage | DefinitionKind::ModelCursor + ) { + return Err(Diagnostic::error( + "C005", + "generated Model values are created only by database operations", + span, + )); + } + let Shape::Record(expected) = &self.context.types[id].shape else { + return Err(Diagnostic::error( + "C005", + "choice values use a type-qualified variant constructor", + span, + )); + }; + let expected = expected.clone(); + if !self.context.owns_private_fields(id) && expected.iter().any(|field| field.private) { + return Err(Diagnostic::error( + "C006", + "record with private fields can only be constructed by its owner (Model: owning domain App)", + span, + )); + } + let mut seen = BTreeSet::new(); + let mut values = Vec::new(); + for field in fields { + let Some((index, expected)) = expected + .iter() + .enumerate() + .find(|(_, expected)| expected.name == field.name.text) + else { + return Err(Diagnostic::error( + "C004", + "unknown constructor field", + field.name.span, + )); + }; + if !seen.insert(index) { + return Err(Diagnostic::error( + "C002", + "duplicate constructor field", + field.span, + )); + } + values.push((index, self.expression(&field.value, Some(&expected.ty))?)); + } + if seen.len() != expected.len() { + return Err(Diagnostic::error( + "C005", + "record construction must initialize every field, including nullable fields", + span, + )); + } + Ok(Value { + kind: Expr::Record { fields: values }, + ty: Type::Named(id), + span, + }) + } + + pub(super) fn construct_variant( + &mut self, + id: usize, + variant: usize, + arguments: &[Expression], + span: Span, + ) -> Checked<Value> { + self.construct_variant_with_kind(id, variant, arguments, span, false) + } + + pub(super) fn construct_failure_variant( + &mut self, + id: usize, + variant: usize, + arguments: &[Expression], + span: Span, + ) -> Checked<Value> { + self.construct_variant_with_kind(id, variant, arguments, span, true) + } + + fn construct_variant_with_kind( + &mut self, + id: usize, + variant: usize, + arguments: &[Expression], + span: Span, + failure: bool, + ) -> Checked<Value> { + let Shape::Choice(variants) = &self.context.types[id].shape else { + unreachable!() + }; + if failure && !variants[variant].error { + return Err(Diagnostic::error( + "C005", + "fail requires an error choice variant", + span, + )); + } + if !failure && variants[variant].error { + return Err(Diagnostic::error( + "C005", + "error choice variants can only be used with fail", + span, + )); + } + let fields = variants[variant].fields.clone(); + if fields.len() != arguments.len() { + return Err(Diagnostic::error( + "C005", + "choice constructor payload arity mismatch", + span, + )); + } + let arguments = arguments + .iter() + .zip(fields) + .map(|(argument, field)| self.expression(argument, Some(&field.ty))) + .collect::<Checked<_>>()?; + Ok(Value { + kind: Expr::Variant { variant, arguments }, + ty: Type::Named(id), + span, + }) + } + + pub(super) fn list( + &mut self, + elements: &[Expression], + expected: Option<&Type>, + span: Span, + ) -> Checked<Value> { + let hint = match expected.map(Type::base) { + Some(Type::List(element)) => Some(element.as_ref()), + _ => None, + }; + let (values, element) = self.homogeneous(elements, hint, span)?; + Ok(Value { + kind: Expr::List(values), + ty: Type::List(Box::new(element)), + span, + }) + } + + pub(super) fn map( + &mut self, + entries: &[syntax::MapEntry], + expected: Option<&Type>, + span: Span, + ) -> Checked<Value> { + let (key_hint, value_hint) = match expected.map(Type::base) { + Some(Type::Map(key, value)) => (Some(key.as_ref()), Some(value.as_ref())), + _ => (None, None), + }; + let keys: Vec<_> = entries.iter().map(|entry| entry.key.clone()).collect(); + let values: Vec<_> = entries.iter().map(|entry| entry.value.clone()).collect(); + let (keys, key) = self.homogeneous(&keys, key_hint, span)?; + let (values, value) = self.homogeneous(&values, value_hint, span)?; + if !self.context.map_key(&key) { + return Err(Diagnostic::error("C005", "invalid Map key type", span)); + } + let mut seen: Vec<&Value> = Vec::new(); + for key in &keys { + if seen.iter().any(|previous| same_key(previous, key)) { + return Err(Diagnostic::error( + "C002", + "duplicate Map literal key", + key.span, + )); + } + seen.push(key); + } + Ok(Value { + kind: Expr::Map(keys.into_iter().zip(values).collect()), + ty: Type::Map(Box::new(key), Box::new(value)), + span, + }) + } + + fn homogeneous( + &mut self, + sources: &[Expression], + hint: Option<&Type>, + span: Span, + ) -> Checked<(Vec<Value>, Type)> { + if let Some(hint) = hint { + return Ok(( + sources + .iter() + .map(|source| self.expression(source, Some(hint))) + .collect::<Checked<_>>()?, + hint.clone(), + )); + } + let seed = sources + .iter() + .position(|source| !needs_context(source)) + .ok_or_else(|| { + Diagnostic::error( + "C005", + "empty or null-only collection requires a type context", + span, + ) + })?; + let first = self.expression(&sources[seed], None)?; + let mut ty = first.ty.clone(); + if !ty.value() { + return Err(Diagnostic::error( + "C005", + "collections cannot contain actions or named multi-output results", + span, + )); + } + let mut values: Vec<Option<Value>> = vec![None; sources.len()]; + values[seed] = Some(first); + for (index, source) in sources.iter().enumerate() { + if index == seed || needs_context(source) { + continue; + } + let value = self.expression(source, None)?; + ty = common_type(&ty, &value.ty).ok_or_else(|| { + Diagnostic::error( + "C005", + "collection elements must have one stable type", + source.span, + ) + })?; + values[index] = Some(value); + } + if sources.iter().any(is_null) { + ty = ty.nullable(); + } + let values = sources + .iter() + .zip(values) + .map(|(source, value)| match value { + Some(value) => self.coerce(value, &ty), + None => self.expression(source, Some(&ty)), + }) + .collect::<Checked<_>>()?; + Ok((values, ty)) + } +} + +fn same_key(left: &Value, right: &Value) -> bool { + if left.ty != right.ty { + return false; + } + match (&left.kind, &right.kind) { + (Expr::Constant(left), Expr::Constant(right)) => left == right, + ( + Expr::Variant { + variant: left, + arguments: left_args, + }, + Expr::Variant { + variant: right, + arguments: right_args, + }, + ) => left == right && left_args.is_empty() && right_args.is_empty(), + (Expr::Local(left), Expr::Local(right)) => left == right, + ( + Expr::Field { + value: left, + index: left_index, + }, + Expr::Field { + value: right, + index: right_index, + }, + ) => left_index == right_index && same_key(left, right), + _ => false, + } +} + +fn common_type(left: &Type, right: &Type) -> Option<Type> { + if left == right { + return Some(left.clone()); + } + if matches!( + (left, right), + (Type::Int, Type::Decimal) | (Type::Decimal, Type::Int) + ) { + return Some(Type::Decimal); + } + if left.base() == right.base() { + return Some(left.base().clone().nullable()); + } + None +} diff --git a/crates/dever-core/src/contracts.rs b/crates/dever-core/src/contracts.rs new file mode 100644 index 0000000..e8be1b6 --- /dev/null +++ b/crates/dever-core/src/contracts.rs @@ -0,0 +1,89 @@ +use crate::diagnostic::Diagnostic; +use crate::hir::Program; +use crate::syntax::MAX_SYNTAX_DEPTH; +use crate::types::{Shape, Type}; + +mod dependencies; +pub(crate) use dependencies::Dependencies; +mod effects; +mod error_effects; +pub(crate) use error_effects::failure_names; +// A bounded proof must reject excessive work, never treat it as a successful proof. +const MAX_PROOF_WORK: usize = 16_384; +mod facts; +mod failures; +mod redundancy; +pub(crate) use facts::bounds_subset; +pub(crate) use redundancy::application_errors; + +pub(crate) fn check(program: &mut Program, errors: &mut Vec<Diagnostic>) -> Result<(), Diagnostic> { + if let Some(error) = type_depth_error(program) { + // 结构无效时尚未生成分析结果，调用方必须停止依赖这些结果的后续检查。 + return Err(error); + } + let dependencies = dependencies::Dependencies::new(program); + effects::check(program, &dependencies, errors); + error_effects::check(program, &dependencies, errors); + facts::check(program, &dependencies, errors); + failures::check(program, &dependencies, errors); + redundancy::check(program); + Ok(()) +} + +fn type_depth_error(program: &Program) -> Option<Diagnostic> { + // Nominal references can form a deep DAG despite shallow declaration syntax. + // Resolve depths iteratively so the limit itself cannot overflow the host stack. + let mut depths = vec![0; program.types.len()]; + loop { + let mut changed = false; + for (id, definition) in program.types.iter().enumerate() { + let depth = 1 + match &definition.shape { + Shape::Record(fields) => fields + .iter() + .map(|field| type_depth(&field.ty, &depths)) + .max() + .unwrap_or(0), + Shape::Choice(variants) => variants + .iter() + .flat_map(|variant| &variant.fields) + .map(|field| type_depth(&field.ty, &depths)) + .max() + .unwrap_or(0), + }; + if depth > MAX_SYNTAX_DEPTH { + return Some(Diagnostic::error( + "C015", + "contract type nesting exceeds the structural depth limit", + definition.span, + )); + } + changed |= depths[id] != depth; + depths[id] = depth; + } + if !changed { + return None; + } + } +} + +fn type_depth(ty: &Type, depths: &[usize]) -> usize { + match ty { + Type::Named(id) => depths[*id], + Type::List(inner) + | Type::Stream(inner) + | Type::AsyncStream(inner) + | Type::RowStream(inner) + | Type::Nullable(inner) => 1 + type_depth(inner, depths), + Type::Map(key, value) | Type::MapEntry(key, value) => { + 1 + type_depth(key, depths).max(type_depth(value, depths)) + } + Type::Outputs(fields) => { + 1 + fields + .iter() + .map(|field| type_depth(&field.ty, depths)) + .max() + .unwrap_or(0) + } + _ => 0, + } +} diff --git a/crates/dever-core/src/contracts/dependencies.rs b/crates/dever-core/src/contracts/dependencies.rs new file mode 100644 index 0000000..e22fbdf --- /dev/null +++ b/crates/dever-core/src/contracts/dependencies.rs @@ -0,0 +1,119 @@ +use std::collections::{BTreeSet, VecDeque}; + +use crate::hir::{CallArgument, CallTarget, HandlerTarget, Program}; + +pub(crate) struct Dependencies { + callees: Vec<BTreeSet<usize>>, + callers: Vec<Vec<usize>>, + order: Vec<usize>, +} + +impl Dependencies { + pub fn new(program: &Program) -> Self { + let mut dependencies = vec![BTreeSet::new(); program.functions.len()]; + for (caller, function) in program.functions.iter().enumerate() { + if let Some(port) = &function.port { + dependencies[caller].extend(&port.implementations); + } + crate::check::visit(function, |expression| { + if let Some((target, arguments)) = expression.kind.static_call() { + if let CallTarget::Function(id) + | CallTarget::Handler(HandlerTarget::Function(id)) = target + { + dependencies[caller].insert(id); + } + for argument in arguments { + if let CallArgument::Handler(HandlerTarget::Function(id)) = argument { + dependencies[caller].insert(*id); + } + } + return; + } + if let Some(HandlerTarget::Function(id)) = expression.kind.callback() { + dependencies[caller].insert(id); + } + }); + } + let mut callers = vec![Vec::new(); program.functions.len()]; + let mut pending: Vec<_> = dependencies.iter().map(BTreeSet::len).collect(); + for (caller, callees) in dependencies.iter().enumerate() { + for callee in callees { + callers[*callee].push(caller); + } + } + let mut ready: BTreeSet<_> = pending + .iter() + .enumerate() + .filter_map(|(id, count)| (*count == 0).then_some(id)) + .collect(); + let mut order = Vec::new(); + while let Some(id) = ready.pop_first() { + order.push(id); + for caller in &callers[id] { + pending[*caller] -= 1; + if pending[*caller] == 0 { + ready.insert(*caller); + } + } + } + // 未使用的 handler 引用可能形成摘要依赖环，但没有运行时递归。 + // Facts 沿用声明保证；单调摘要通过工作队列继续收敛。 + order.extend( + pending + .iter() + .enumerate() + .filter_map(|(id, count)| (*count != 0).then_some(id)), + ); + Self { + callees: dependencies, + callers, + order, + } + } + + pub fn reachable(&self, roots: impl IntoIterator<Item = usize>) -> BTreeSet<usize> { + let mut reachable = BTreeSet::new(); + let mut pending: Vec<_> = roots.into_iter().collect(); + while let Some(function) = pending.pop() { + if reachable.insert(function) { + pending.extend(&self.callees[function]); + } + } + reachable + } + + pub(super) fn order(&self) -> &[usize] { + &self.order + } + + pub(super) fn work(&self) -> Work<'_> { + Work { + dependencies: self, + ready: self.order.iter().copied().collect(), + queued: vec![true; self.order.len()], + } + } +} + +pub(super) struct Work<'a> { + dependencies: &'a Dependencies, + ready: VecDeque<usize>, + queued: Vec<bool>, +} + +impl Work<'_> { + pub fn next(&mut self) -> Option<usize> { + let id = self.ready.pop_front()?; + self.queued[id] = false; + Some(id) + } + + pub fn changed(&mut self, id: usize) { + for caller in &self.dependencies.callers[id] { + if !self.queued[*caller] { + self.queued[*caller] = true; + self.ready.push_back(*caller); + } + } + } +} diff --git a/crates/dever-core/src/contracts/effects.rs b/crates/dever-core/src/contracts/effects.rs new file mode 100644 index 0000000..a8767f4 --- /dev/null +++ b/crates/dever-core/src/contracts/effects.rs @@ -0,0 +1,760 @@ +use std::collections::BTreeSet; + +use crate::diagnostic::{Diagnostic, Label}; +use crate::hir::{ + CallArgument, CallTarget, CollectionOp, Expression, ExpressionKind, HandlerTarget, Program, + SequenceKind, +}; +use crate::intrinsic::Intrinsic; +use crate::types::{Parameter, Type}; + +use super::dependencies::Dependencies; + +#[derive(Clone, Default, PartialEq, Eq)] +struct Effects { + known: BTreeSet<&'static str>, + handlers: BTreeSet<usize>, + recoveries: BTreeSet<String>, + databases: BTreeSet<crate::model::DatabaseOwner>, + blocking: bool, + suspends: bool, + unmediated_io: bool, + starts_api: bool, +} + +impl Effects { + fn include(&mut self, other: &Self) { + self.known.extend(&other.known); + self.handlers.extend(&other.handlers); + self.recoveries.extend(other.recoveries.iter().cloned()); + self.databases.extend(&other.databases); + self.blocking |= other.blocking; + self.suspends |= other.suspends; + self.unmediated_io |= other.unmediated_io; + self.starts_api |= other.starts_api; + } +} + +pub(super) fn check(program: &mut Program, graph: &Dependencies, errors: &mut Vec<Diagnostic>) { + let summaries = infer(program, graph); + for (function, summary) in program.functions.iter_mut().zip(&summaries) { + function.suspends = summary.suspends; + } + program.suspension_handlers = summaries + .iter() + .map(|summary| summary.handlers.clone()) + .collect(); + program.recoveries = summaries + .iter() + .map(|summary| summary.recoveries.clone()) + .collect(); + program.effects = summaries + .iter() + .map(|summary| { + let mut effects = summary.known.clone(); + if !summary.handlers.is_empty() { + effects.insert("handler"); + } + effects + }) + .collect(); + program.database_effects = summaries + .iter() + .map(|summary| summary.databases.clone()) + .collect(); + for (id, function) in program.functions.iter().enumerate() { + if !function.pure || program.effects[id].is_empty() { + continue; + } + let mut diagnostic = Diagnostic::error( + "C013", + format!( + "pure function '{}' has effects: {}", + function.name, + program.effects[id] + .iter() + .copied() + .collect::<Vec<_>>() + .join(", ") + ), + function.span, + ); + crate::check::visit(function, |expression| { + let effects = expression_effects(expression, &summaries); + if diagnostic.related.is_empty() + && (!effects.known.is_empty() || !effects.handlers.is_empty()) + { + diagnostic.related.push(Label { + span: expression.span, + message: + "this operation introduces an effect or invokes an unconstrained handler" + .into(), + }); + } + }); + errors.push(diagnostic); + } + validate_blocking_calls(program, &summaries, errors); + validate_role_effects(program, &summaries, errors); + validate_concurrency_calls(program, &summaries, errors); + validate_suspending_signatures(program, &summaries, errors); + validate_parallel_each_handlers(program, &summaries, errors); + validate_suspending_blocking_boundaries(program, &summaries, errors); + validate_transactions(program, &summaries, errors); + crate::check::check_suspensions(program, errors); +} + +fn validate_role_effects(program: &Program, summaries: &[Effects], errors: &mut Vec<Diagnostic>) { + use crate::source::SourceRole; + for (id, function) in program.functions.iter().enumerate() { + let package = &program.packages[function.owner]; + if package.bundled { + continue; + } + if matches!(package.role, Some(SourceRole::App | SourceRole::Domain)) + && summaries[id].unmediated_io + { + errors.push(Diagnostic::error( + "C006", + "App and Domain external I/O must pass through a Port", + function.span, + )); + } + if package.role.is_some() && summaries[id].starts_api { + errors.push(Diagnostic::error( + "C006", + "only main may start the application API or Job worker", + function.span, + )); + } + } + if let Ok(instances) = crate::specialize::concrete(program) { + for instance in instances { + let function = &program.functions[instance.function]; + if !matches!( + program.packages[function.owner].role, + Some(crate::source::SourceRole::App | crate::source::SourceRole::Domain) + ) { + continue; + } + let bindings = crate::specialize::handler_bindings(function, &instance); + if summaries[instance.function] + .handlers + .iter() + .any(|parameter| summaries[bindings[parameter]].unmediated_io) + { + errors.push(Diagnostic::error( + "C006", + "App and Domain handler I/O must pass through a Port", + function.span, + )); + } + } + } +} + +fn infer(program: &Program, dependencies: &Dependencies) -> Vec<Effects> { + let mut summaries = vec![Effects::default(); program.functions.len()]; + let mut work = dependencies.work(); + while let Some(id) = work.next() { + let function = &program.functions[id]; + let mut next = summaries[id].clone(); + if let Some(external) = function.external { + let crate::hir::AdapterImplementation::External(adapter) = + &program.adapters[external.adapter].implementation + else { + unreachable!("external operation belongs to an external Adapter") + }; + next.known.extend(&adapter.capabilities); + next.known.insert("concurrency"); + next.suspends = true; + next.unmediated_io = adapter.capabilities.contains("file") + || adapter.capabilities.contains("network") + || adapter.capabilities.contains("process"); + } + if let Some(port) = &function.port { + for implementation in &port.implementations { + next.include(&summaries[*implementation]); + } + next.unmediated_io = false; + } + if let Some(reason) = &function.recovery { + next.recoveries.insert(reason.clone()); + } + crate::check::visit(function, |expression| { + next.include(&expression_effects(expression, &summaries)) + }); + if next != summaries[id] { + summaries[id] = next; + work.changed(id); + } + } + summaries +} + +fn handler_effects(target: HandlerTarget, summaries: &[Effects]) -> Effects { + match target { + HandlerTarget::Function(id) => summaries[id].clone(), + HandlerTarget::Parameter(id) => Effects { + handlers: BTreeSet::from([id]), + ..Effects::default() + }, + } +} + +fn expression_effects(expression: &Expression, summaries: &[Effects]) -> Effects { + let mut result = Effects::default(); + match &expression.kind { + ExpressionKind::ModelOperation { model, operation } => { + result.known.insert("database"); + result + .databases + .insert(crate::model::DatabaseOwner::Model(*model)); + result.suspends = true; + if matches!( + operation.as_ref(), + crate::hir::ModelOperation::Stream { .. } + ) { + result.known.insert("stream"); + result.known.insert("concurrency"); + } + } + ExpressionKind::JobEnqueue { target, .. } => { + result.known.insert("database"); + result + .databases + .insert(crate::model::DatabaseOwner::Job(*target)); + result.suspends = true; + } + ExpressionKind::Intrinsic { + operation, handler, .. + } => { + if let Some(handler) = handler { + result.include(&handler_effects(*handler, summaries)); + } + if let Some(effect) = intrinsic_effect(*operation) { + result.known.insert(effect); + result.unmediated_io |= matches!(effect, "file" | "network"); + } + result.starts_api |= matches!(operation, Intrinsic::ApiServe | Intrinsic::JobServe); + if operation.is_async() { + if !matches!(operation, Intrinsic::UploadStore) { + result.known.insert("concurrency"); + } + result.suspends = true; + } + result.blocking |= intrinsic_blocking(*operation); + } + ExpressionKind::Collection { + operation, + handler, + sequence, + .. + } => { + if let Some(handler) = handler { + result.include(&handler_effects(*handler, summaries)); + } + if matches!( + sequence, + Some(SequenceKind::Stream | SequenceKind::AsyncStream | SequenceKind::RowStream) + ) || *operation == CollectionOp::Close + { + result.known.insert("stream"); + } + if *operation == CollectionOp::ParallelEach + || (*sequence == Some(SequenceKind::AsyncStream) + && *operation != CollectionOp::Close) + || (*sequence == Some(SequenceKind::RowStream) && *operation != CollectionOp::Close) + { + result.known.insert("concurrency"); + } else if *sequence == Some(SequenceKind::Stream) { + result.blocking = true; + } + if matches!( + sequence, + Some(SequenceKind::AsyncStream | SequenceKind::RowStream) + ) && *operation != CollectionOp::Close + { + result.suspends = true; + } + } + _ => { + if let Some((target, arguments)) = expression.kind.static_call() { + result.include(&call_effects(target, arguments, summaries)); + if matches!( + expression.kind, + ExpressionKind::RunCall { .. } + | ExpressionKind::ParallelCall { .. } + | ExpressionKind::BlockingCall { .. } + ) { + result.known.insert("concurrency"); + result.suspends = true; + } + } else if matches!( + expression.kind, + ExpressionKind::AwaitTask(_) + | ExpressionKind::StopTask(_) + | ExpressionKind::Group(_) + | ExpressionKind::AwaitGroup(_) + | ExpressionKind::StopGroup(_) + | ExpressionKind::Channel { .. } + | ExpressionKind::ChannelSend { .. } + | ExpressionKind::ChannelReceive(_) + | ExpressionKind::ChannelClose(_) + ) { + result.known.insert("concurrency"); + result.suspends |= !matches!( + expression.kind, + ExpressionKind::Group(_) | ExpressionKind::Channel { .. } + ); + } + } + } + result +} + +fn call_effects(target: CallTarget, arguments: &[CallArgument], summaries: &[Effects]) -> Effects { + match target { + CallTarget::Handler(handler) => handler_effects(handler, summaries), + CallTarget::Function(id) => { + let mut result = Effects { + known: summaries[id].known.clone(), + recoveries: summaries[id].recoveries.clone(), + databases: summaries[id].databases.clone(), + blocking: summaries[id].blocking, + suspends: summaries[id].suspends, + unmediated_io: summaries[id].unmediated_io, + starts_api: summaries[id].starts_api, + ..Effects::default() + }; + for parameter in &summaries[id].handlers { + if let CallArgument::Handler(handler) = arguments[*parameter] { + result.include(&handler_effects(handler, summaries)); + } + } + result + } + } +} + +fn validate_blocking_calls(program: &Program, summaries: &[Effects], errors: &mut Vec<Diagnostic>) { + for function in &program.functions { + crate::check::visit(function, |expression| { + let ExpressionKind::BlockingCall { target, arguments } = &expression.kind else { + return; + }; + let effects = call_effects(*target, arguments, summaries); + if call_may_suspend(&effects) { + errors.push(Diagnostic::error( + "C005", + "blocking requires a non-suspending function call", + expression.span, + )); + } else if !effects.blocking { + errors.push(Diagnostic::error( + "C005", + "blocking requires a synchronous call with a transitive blocking system effect", + expression.span, + )); + } else if !effects.handlers.is_empty() { + errors.push(Diagnostic::error( + "C005", + "blocking cannot wrap a call with unconstrained handler effects", + expression.span, + )); + } else if effects.known.contains("concurrency") { + errors.push(Diagnostic::error( + "C005", + "blocking cannot wrap a call that uses parallel_each; call parallel_each directly from suspending code", + expression.span, + )); + } + }); + } +} + +fn validate_concurrency_calls( + program: &Program, + summaries: &[Effects], + errors: &mut Vec<Diagnostic>, +) { + for function in &program.functions { + crate::check::visit(function, |expression| { + if let ExpressionKind::ParallelCall { target, arguments } = &expression.kind + && call_may_suspend(&call_effects(*target, arguments, summaries)) + { + errors.push(Diagnostic::error( + "C005", + "parallel requires a non-suspending pure function call", + expression.span, + )); + } + }); + } +} + +fn validate_suspending_signatures( + program: &Program, + summaries: &[Effects], + errors: &mut Vec<Diagnostic>, +) { + for (id, function) in program.functions.iter().enumerate() { + if !call_may_suspend(&summaries[id]) { + continue; + } + let inputs_transfer = function.parameters.iter().all(|parameter| match parameter { + // Upload is owned by the same request across sequential suspension; + // detached calls and containers still require transferable(), which is false. + Parameter::Value(ty) => ty == &Type::Upload || ty.transferable(&program.types), + Parameter::Handler(_) => true, + }); + let outputs_transfer = function + .outputs + .iter() + .all(|field| field.ty.transferable(&program.types)); + if !inputs_transfer || !outputs_transfer { + errors.push(Diagnostic::error( + "C005", + "suspending function inputs and outputs must be transferable between threads", + function.span, + )); + } + } +} + +fn validate_parallel_each_handlers( + program: &Program, + summaries: &[Effects], + errors: &mut Vec<Diagnostic>, +) { + for (id, function) in program.functions.iter().enumerate() { + if !call_may_suspend(&summaries[id]) { + continue; + } + crate::check::visit(function, |expression| { + let ExpressionKind::Collection { + operation: CollectionOp::ParallelEach, + handler: Some(handler), + sequence, + .. + } = &expression.kind + else { + return; + }; + if matches!( + sequence, + Some(SequenceKind::AsyncStream | SequenceKind::RowStream) + ) || handler_may_suspend(*handler, summaries) + { + return; + } + if handler_effects(*handler, summaries) + .known + .contains("concurrency") + { + errors.push(Diagnostic::error( + "C005", + "a suspending parallel_each handler cannot transitively use parallel_each or concurrency", + expression.span, + )); + } + }); + } +} + +fn validate_suspending_blocking_boundaries( + program: &Program, + summaries: &[Effects], + errors: &mut Vec<Diagnostic>, +) { + for (id, function) in program.functions.iter().enumerate() { + if !call_may_suspend(&summaries[id]) { + continue; + } + if let Some(port) = &function.port { + for implementation in &port.implementations { + let effects = &summaries[*implementation]; + if !call_may_suspend(effects) && requires_worker_boundary(effects) { + errors.push(Diagnostic::error( + "C005", + "a suspending Port cannot dispatch a blocking implementation inline; use an explicit worker boundary in the implementation", + program.functions[*implementation].span, + )); + } + } + } + crate::check::visit(function, |expression| { + let blocks_runtime = match &expression.kind { + ExpressionKind::Call { target, arguments } + | ExpressionKind::CaptureResult { + target, arguments, .. + } => { + let effects = call_effects(*target, arguments, summaries); + (!call_may_suspend(&effects) && requires_worker_boundary(&effects)) + || bound_handler_requires_worker_boundary(*target, arguments, summaries) + } + ExpressionKind::Intrinsic { + operation, handler, .. + } => { + intrinsic_blocking(*operation) + || handler.is_some_and(|handler| { + !handler_may_suspend(handler, summaries) + && requires_worker_boundary(&handler_effects(handler, summaries)) + }) + } + ExpressionKind::Collection { + operation, + sequence, + handler, + .. + } => { + if *operation == CollectionOp::ParallelEach { + false + } else if matches!( + sequence, + Some(SequenceKind::AsyncStream | SequenceKind::RowStream) + ) || handler + .is_some_and(|handler| handler_may_suspend(handler, summaries)) + { + handler.is_some_and(|handler| { + !handler_may_suspend(handler, summaries) + && requires_worker_boundary(&handler_effects(handler, summaries)) + }) + } else { + let effects = expression_effects(expression, summaries); + requires_worker_boundary(&effects) + } + } + _ => false, + }; + if blocks_runtime { + errors.push(Diagnostic::error( + "C005", + "a blocking or parallel operation cannot run inline inside a suspending function", + expression.span, + )); + } + }); + } +} + +fn requires_worker_boundary(effects: &Effects) -> bool { + effects.blocking || effects.known.contains("concurrency") || !effects.handlers.is_empty() +} + +fn bound_handler_requires_worker_boundary( + target: CallTarget, + arguments: &[CallArgument], + summaries: &[Effects], +) -> bool { + let CallTarget::Function(function) = target else { + return false; + }; + summaries[function].handlers.iter().any(|parameter| { + let CallArgument::Handler(handler) = arguments[*parameter] else { + unreachable!("inferred handler effect belongs to a handler parameter") + }; + let effects = handler_effects(handler, summaries); + !call_may_suspend(&effects) && requires_worker_boundary(&effects) + }) +} + +fn call_may_suspend(effects: &Effects) -> bool { + effects.suspends || !effects.handlers.is_empty() +} + +fn validate_transactions(program: &Program, summaries: &[Effects], errors: &mut Vec<Diagnostic>) { + for (id, function) in program.functions.iter().enumerate() { + // Case-specific checking removes unreachable bodies after production + // contracts were checked; those signatures are not empty transactions. + if function.kind != crate::syntax::FunctionKind::Transaction || function.clauses.is_empty() + { + continue; + } + if summaries[id].databases.is_empty() && summaries[id].handlers.is_empty() { + errors.push(Diagnostic::error( + "C014", + "transaction functions must perform a database operation", + function.span, + )); + } + let instance = crate::specialize::Specialization { + function: id, + handlers: Vec::new(), + }; + let concurrent = crate::specialize::function_starts_transaction_unsafe_work(program, id) + || (summaries[id].handlers.is_empty() + && crate::specialize::starts_transaction_unsafe_work(program, &instance)); + if concurrent { + errors.push(Diagnostic::error( + "C014", + "transaction functions cannot start concurrent work", + function.span, + )); + } + let password_crypto = summaries[id].handlers.is_empty() + && (crate::specialize::uses_intrinsic( + program, + &instance, + crate::intrinsic::Intrinsic::CryptoPasswordHash, + ) || crate::specialize::uses_intrinsic( + program, + &instance, + crate::intrinsic::Intrinsic::CryptoPasswordVerify, + )); + if password_crypto { + errors.push(Diagnostic::error( + "C014", + "transaction functions cannot perform password hashing or verification", + function.span, + )); + } + let explicit = summaries[id] + .databases + .iter() + .filter_map(|owner| match program.database_binding(*owner).0 { + crate::model::ConnectionSelector::Explicit(name) => Some(name), + _ => None, + }) + .collect::<BTreeSet<_>>(); + if explicit.len() > 1 { + errors.push(Diagnostic::error( + "C014", + "transaction spans multiple explicit database connections", + function.span, + )); + } + } + let Ok(instances) = crate::specialize::concrete(program) else { + return; + }; + let mut missing_database = BTreeSet::new(); + let mut concurrent = BTreeSet::new(); + let mut password_crypto = BTreeSet::new(); + for instance in instances { + let function = &program.functions[instance.function]; + if function.kind != crate::syntax::FunctionKind::Transaction { + continue; + } + if !summaries[instance.function].handlers.is_empty() + && (crate::specialize::uses_intrinsic( + program, + &instance, + crate::intrinsic::Intrinsic::CryptoPasswordHash, + ) || crate::specialize::uses_intrinsic( + program, + &instance, + crate::intrinsic::Intrinsic::CryptoPasswordVerify, + )) + { + password_crypto.insert(instance.function); + } + if summaries[instance.function].handlers.is_empty() { + continue; + } + if crate::specialize::database_effects(program, &instance).is_empty() { + missing_database.insert(instance.function); + } + if crate::specialize::starts_transaction_unsafe_work(program, &instance) { + concurrent.insert(instance.function); + } + } + for function in missing_database { + errors.push(Diagnostic::error( + "C014", + "transaction functions must perform a database operation", + program.functions[function].span, + )); + } + for function in concurrent { + errors.push(Diagnostic::error( + "C014", + "transaction functions cannot start concurrent work", + program.functions[function].span, + )); + } + for function in password_crypto { + errors.push(Diagnostic::error( + "C014", + "transaction functions cannot perform password hashing or verification", + program.functions[function].span, + )); + } +} + +fn handler_may_suspend(handler: HandlerTarget, summaries: &[Effects]) -> bool { + match handler { + HandlerTarget::Function(id) => call_may_suspend(&summaries[id]), + HandlerTarget::Parameter(_) => true, + } +} + +fn intrinsic_blocking(operation: Intrinsic) -> bool { + use Intrinsic::*; + matches!( + operation, + TimeSleep + | CryptoPasswordHash + | CryptoPasswordVerify + | StdoutWrite + | LogDebug + | LogInfo + | LogWarn + | LogError + | FileOpen + | FileCreate + | FileRead + | FileWrite + | FileClose + | FileChunks + ) +} + +// Exhaustive classification makes every new system primitive choose its effect boundary. +fn intrinsic_effect(operation: Intrinsic) -> Option<&'static str> { + use Intrinsic::*; + match operation { + JobServe | TestJobDrain => Some("database"), + TestClockAdvance => Some("time"), + TaskTicks | TaskTimeout | TaskRace | StreamOf => Some("concurrency"), + ProcessArguments => Some("process"), + CryptoToken | CryptoPasswordHash => Some("random"), + TimeNow | TimeUnixMillis | TimeMonotonicNanos | TimeSleep | TaskSleep => Some("time"), + StdoutWrite => Some("stdout"), + LogDebug | LogInfo | LogWarn | LogError => Some("log"), + FileOpen | FileCreate | FileRead | FileWrite | FileClose | FileChunks => Some("file"), + AuthIssue | AuthIssueCookie | AuthClearCookie | AuthOwnsUser | AuthId | AuthSession + | AuthUserId | AuthTenantId | AuthPermissions | AuthSaveRole | AuthGrantRole + | AuthRevokeRole | AuthDisableRole | SiteKey => Some("identity"), + ApiRequestId + | ApiMethod + | ApiPath + | ApiClientAddress + | ApiHeader + | ApiCookie + | ApiSecretCookie + | ApiResponseHeader + | ApiResponseCookie + | ApiResponseSecretCookie => Some("request"), + UploadFilename | UploadContentType | UploadSize | UploadClose => Some("request"), + UploadStore => Some("file"), + TcpConnect | TcpConnectTimeout | TcpListen | TcpAccept | TcpPort | TcpRead | TcpWrite + | TcpTimeout | TcpClose | TcpCloseListener | TcpChunks | TcpConnections | HttpServe + | ApiServe | HttpSend | HttpServeLive | HttpRespond | HttpStart | HttpWrite + | HttpFinish | SseStart | SseSend | WsAccept | WsConnect | WsSend | WsReceive + | WsMessages | WsClose | TlsSystem | TlsClient | TlsServer | HttpClientNew + | HttpRequest | HttpOpen | HttpOpenStream | HttpCloseClient | HttpServeTls + | HttpServeLiveTls | WsOpen => Some("network"), + TestAssert | TestAssertEq | TestSecret | CryptoPasswordVerify | CryptoSha256 + | CryptoHmacSha256 | CryptoConstantTimeEq | TextTrim | TextLower | TextUpper + | TextCodepoint | TextFromCodepoint | TextAt | TextSlice | TextIndexOf | TextContains + | TextStartsWith | TextEndsWith | TextSplit | TextReplace | IntParse | IntToText + | DecimalParse | DecimalToText | DecimalFromInt | FloatParse | FloatToText + | DecimalRound | FloatFromInt | FloatSqrt | FloatSin | FloatCos | FloatLog | FloatPow + | FloatIsNan | FloatIsFinite | FloatIsInfinite | IdFromText | IdToText | UuidParse + | UuidToText | BytesFromText | BytesToText | BytesFromInts | BytesLength | BytesAt + | BytesSlice | BytesConcat | TimeParseDateTime | TimeFormatDateTime | TimeParseDate + | TimeFormatDate | TimeParseTime | TimeFormatTime | TimeDuration | TimeAdd + | TimeSubtract | TimeDifference => None, + } +} diff --git a/crates/dever-core/src/contracts/error_effects.rs b/crates/dever-core/src/contracts/error_effects.rs new file mode 100644 index 0000000..c50dba4 --- /dev/null +++ b/crates/dever-core/src/contracts/error_effects.rs @@ -0,0 +1,268 @@ +use std::collections::BTreeSet; + +use crate::diagnostic::Diagnostic; +use crate::hir::{ + CallArgument, CallTarget, Expression, ExpressionKind, Failure, HandlerTarget, Program, +}; +use crate::types::{Shape, Type}; + +use super::dependencies::Dependencies; + +#[derive(Clone, Default, Eq, PartialEq)] +struct FailureSummary { + known: BTreeSet<Failure>, + handlers: BTreeSet<usize>, +} + +impl FailureSummary { + fn include(&mut self, other: &Self) { + self.known.extend(&other.known); + self.handlers.extend(&other.handlers); + } +} + +pub(super) fn check( + program: &mut Program, + dependencies: &Dependencies, + errors: &mut Vec<Diagnostic>, +) { + let summaries = infer(program, dependencies); + validate_captures(program, &summaries, errors); + validate_payloads(program, &summaries, errors); + program.failures = summaries + .iter() + .map(|summary| summary.known.clone()) + .collect(); + program.failure_handlers = summaries + .iter() + .map(|summary| summary.handlers.clone()) + .collect(); +} + +fn infer(program: &Program, dependencies: &Dependencies) -> Vec<FailureSummary> { + let mut summaries = vec![FailureSummary::default(); program.functions.len()]; + let mut work = dependencies.work(); + while let Some(id) = work.next() { + let mut next = FailureSummary::default(); + if let Some(external) = program.functions[id].external { + next.known.extend( + &program.functions[external.contract] + .port + .as_ref() + .expect("external operation targets a Port") + .failures, + ); + } + if let Some(port) = &program.functions[id].port { + next.known.extend(&port.failures); + } + crate::check::visit(&program.functions[id], |expression| { + next.include(&expression_failures(expression, &summaries, program)); + }); + if program.functions[id].kind == crate::syntax::FunctionKind::Transaction { + // Begin/commit can fail even when the body handles every query error. + next.known + .extend(crate::capture::database_errors(&program.types)); + } + if next != summaries[id] { + summaries[id] = next; + work.changed(id); + } + } + summaries +} + +fn expression_failures( + expression: &Expression, + summaries: &[FailureSummary], + program: &Program, +) -> FailureSummary { + match &expression.kind { + ExpressionKind::Fail(error) => { + let (ty, variant) = failure_variant(error); + FailureSummary { + known: BTreeSet::from([Failure { ty, variant }]), + handlers: BTreeSet::new(), + } + } + ExpressionKind::CaptureResult { .. } => FailureSummary::default(), + ExpressionKind::ModelOperation { .. } | ExpressionKind::JobEnqueue { .. } => { + FailureSummary { + known: crate::capture::database_errors(&program.types) + .into_iter() + .collect(), + handlers: BTreeSet::new(), + } + } + ExpressionKind::Collection { + operation, + handler, + arguments, + .. + } => { + let mut result = handler + .map(|target| handler_failures(target, summaries)) + .unwrap_or_default(); + if *operation != crate::hir::CollectionOp::Close + && arguments + .first() + .is_some_and(|argument| matches!(argument.ty, Type::RowStream(_))) + { + result + .known + .extend(crate::capture::database_errors(&program.types)); + } + result + } + ExpressionKind::Intrinsic { + handler: Some(handler), + .. + } => handler_failures(*handler, summaries), + _ => expression + .kind + .static_call() + .map(|(target, arguments)| call_failures(target, arguments, summaries)) + .unwrap_or_default(), + } +} + +fn failure_variant(error: &Expression) -> (usize, usize) { + let Type::Named(ty) = error.ty else { + unreachable!("checked fail value is a choice variant") + }; + let ExpressionKind::Variant { variant, .. } = error.kind else { + unreachable!("checked fail value is a direct error variant") + }; + (ty, variant) +} + +fn handler_failures(target: HandlerTarget, summaries: &[FailureSummary]) -> FailureSummary { + match target { + HandlerTarget::Function(id) => summaries[id].clone(), + HandlerTarget::Parameter(id) => FailureSummary { + known: BTreeSet::new(), + handlers: BTreeSet::from([id]), + }, + } +} + +fn call_failures( + target: CallTarget, + arguments: &[CallArgument], + summaries: &[FailureSummary], +) -> FailureSummary { + match target { + CallTarget::Handler(handler) => handler_failures(handler, summaries), + CallTarget::Function(id) => { + let mut result = FailureSummary { + known: summaries[id].known.clone(), + handlers: BTreeSet::new(), + }; + for parameter in &summaries[id].handlers { + let CallArgument::Handler(handler) = arguments[*parameter] else { + unreachable!("failure handler index refers to a handler argument") + }; + result.include(&handler_failures(handler, summaries)); + } + result + } + } +} + +fn validate_captures( + program: &Program, + summaries: &[FailureSummary], + errors: &mut Vec<Diagnostic>, +) { + for function in &program.functions { + crate::check::visit(function, |expression| { + let ExpressionKind::CaptureResult { + target, + arguments, + choice, + .. + } = &expression.kind + else { + return; + }; + let actual = call_failures(*target, arguments, summaries); + if !actual.handlers.is_empty() { + errors.push(Diagnostic::error( + "C012", + "result(call) cannot capture an unconstrained handler error set", + expression.span, + )); + return; + } + let targets = match crate::capture::targets(&program.types, *choice, &actual.known) { + Ok(targets) => targets, + Err(message) => { + errors.push(Diagnostic::error("C012", message, expression.span)); + return; + } + }; + let expected = targets.keys().copied().collect::<BTreeSet<_>>(); + if actual.known.is_empty() { + errors.push(Diagnostic::error( + "C012", + "result(call) requires a call with a recoverable error set", + expression.span, + )); + } else if actual.known != expected { + errors.push(Diagnostic::error( + "C012", + format!( + "result capture errors do not match the call: expected [{}], found [{}]", + failure_names(program, &expected), + failure_names(program, &actual.known), + ), + expression.span, + )); + } + }); + } +} + +fn validate_payloads( + program: &Program, + summaries: &[FailureSummary], + errors: &mut Vec<Diagnostic>, +) { + let failures = summaries + .iter() + .flat_map(|summary| summary.known.iter()) + .copied() + .collect::<BTreeSet<_>>(); + for failure in failures { + let Shape::Choice(variants) = &program.types[failure.ty].shape else { + unreachable!("failure type is a choice") + }; + if variants[failure.variant] + .fields + .iter() + .any(|field| !field.ty.transferable(&program.types)) + { + errors.push(Diagnostic::error( + "C005", + "error payloads must be transferable across execution boundaries", + program.types[failure.ty].span, + )); + } + } +} + +pub(crate) fn failure_names(program: &Program, failures: &BTreeSet<Failure>) -> String { + failures + .iter() + .map(|failure| { + let Shape::Choice(variants) = &program.types[failure.ty].shape else { + unreachable!("failure type is a choice") + }; + format!( + "{}.{}", + program.types[failure.ty].name, variants[failure.variant].name + ) + }) + .collect::<Vec<_>>() + .join(", ") +} diff --git a/crates/dever-core/src/contracts/facts.rs b/crates/dever-core/src/contracts/facts.rs new file mode 100644 index 0000000..e2d3326 --- /dev/null +++ b/crates/dever-core/src/contracts/facts.rs @@ -0,0 +1,815 @@ +//! Conservative value guarantees shared by construction, assignment and call checks. + +use std::cell::Cell; +use std::collections::BTreeMap; +use std::rc::Rc; + +use dever_runtime::number::DecimalValue; + +use crate::diagnostic::{Diagnostic, Label}; +use crate::hir::{ + Atom, CallArgument, CallTarget, Clause, CollectionOp, Domain, Expression, ExpressionKind, + Function, HandlerTarget, Program, Projection, Statement, +}; +use crate::source::Span; +use crate::syntax::BinaryOperator as Binary; +use crate::types::{Field, Parameter, Shape, Type}; + +mod domain; + +use super::MAX_PROOF_WORK; +use super::dependencies::Dependencies; +pub(crate) use domain::bounds_subset; +use domain::{binary, constant_atom, disjoint, subset, unary, union}; + +type FactFields = BTreeMap<usize, Rc<Fact>>; + +#[derive(Default)] +struct ProofWork { + visits: Cell<usize>, +} + +impl ProofWork { + fn spend(&self) -> bool { + let visits = self.visits.get(); + if visits > MAX_PROOF_WORK { + return false; + } + self.visits.set(visits + 1); + visits < MAX_PROOF_WORK + } + + fn exhausted(&self) -> bool { + self.visits.get() > MAX_PROOF_WORK + } +} + +#[derive(Clone, Debug, PartialEq)] +struct InputOrigin { + parameter: usize, + fields: Vec<usize>, +} + +#[derive(Clone, Debug)] +struct Fact { + domain: Domain, + fields: Rc<FactFields>, + origin: Option<InputOrigin>, + evidence: Option<Span>, +} + +impl Fact { + fn domain(domain: Domain) -> Self { + Self { + domain, + fields: Rc::new(BTreeMap::new()), + origin: None, + evidence: None, + } + } + + fn merge(&self, other: &Self, work: &ProofWork) -> Self { + self.merge_cached(other, work, &mut BTreeMap::new()) + } + + fn merge_cached( + &self, + other: &Self, + work: &ProofWork, + cache: &mut BTreeMap<(usize, usize), Rc<FactFields>>, + ) -> Self { + if !work.spend() { + return Self::domain(vec![Atom::Any]); + } + let key = ( + Rc::as_ptr(&self.fields) as usize, + Rc::as_ptr(&other.fields) as usize, + ); + let fields = if Rc::ptr_eq(&self.fields, &other.fields) { + Rc::clone(&self.fields) + } else if let Some(fields) = cache.get(&key) { + Rc::clone(fields) + } else { + let fields = Rc::new( + self.fields + .iter() + .filter_map(|(index, fact)| { + other + .fields + .get(index) + .map(|other| (*index, Rc::new(fact.merge_cached(other, work, cache)))) + }) + .collect(), + ); + cache.insert(key, Rc::clone(&fields)); + fields + }; + Self { + domain: union(&self.domain, &other.domain), + fields, + origin: (self.origin == other.origin) + .then(|| self.origin.clone()) + .flatten(), + evidence: (self.evidence == other.evidence) + .then_some(self.evidence) + .flatten(), + } + } +} + +pub(super) fn check( + program: &mut Program, + dependencies: &Dependencies, + errors: &mut Vec<Diagnostic>, +) { + let mut analyzer = Analyzer { + program, + summaries: vec![None; program.functions.len()], + errors: Vec::new(), + warnings: Vec::new(), + work: ProofWork::default(), + }; + for id in dependencies.order() { + analyzer.analyze_function(*id); + } + errors.extend(analyzer.errors); + program.warnings.extend(analyzer.warnings); +} + +struct Analyzer<'a> { + program: &'a Program, + summaries: Vec<Option<Vec<Fact>>>, + errors: Vec<Diagnostic>, + warnings: Vec<Diagnostic>, + work: ProofWork, +} + +impl Analyzer<'_> { + fn unknown(&self, ty: &Type) -> Fact { + let domain = match ty { + Type::Bool => vec![Atom::Bool(false), Atom::Bool(true)], + Type::Int => vec![Atom::Int(i64::MIN, i64::MAX)], + Type::Decimal => vec![Atom::Decimal( + DecimalValue::minimum(), + DecimalValue::maximum(), + )], + Type::Nullable(inner) => { + let mut domain = self.unknown(inner).domain; + domain.push(Atom::Null); + domain + } + Type::Named(id) => match &self.program.types[*id].shape { + Shape::Choice(variants) => (0..variants.len()).map(Atom::Variant).collect(), + Shape::Record(_) => vec![Atom::Any], + }, + _ => vec![Atom::Any], + }; + Fact::domain(domain) + } + + fn field_fact(&self, field: &Field) -> Fact { + if field.bounds.is_empty() { + self.unknown(&field.ty) + } else { + Fact::domain(field.bounds.clone()) + } + } + + fn fields<'a>(&'a self, ty: &'a Type) -> Option<&'a [Field]> { + match ty.base() { + Type::Named(id) => match &self.program.types[*id].shape { + Shape::Record(fields) => Some(fields), + Shape::Choice(_) => None, + }, + Type::Outputs(fields) => Some(fields), + _ => None, + } + } + + fn project(&self, fact: &Fact, ty: &Type, index: usize) -> Fact { + if let Some(field) = fact.fields.get(&index) { + return field.as_ref().clone(); + } + let mut field = self + .fields(ty) + .map(|fields| self.field_fact(&fields[index])) + .unwrap_or_else(|| Fact::domain(vec![Atom::Any])); + field.origin = fact.origin.clone().map(|mut origin| { + origin.fields.push(index); + origin + }); + field.evidence = fact.evidence; + field + } + + fn function(&self, id: usize) -> Vec<Fact> { + self.summaries[id].clone().unwrap_or_else(|| { + self.program.functions[id] + .outputs + .iter() + .map(|field| self.field_fact(field)) + .collect() + }) + } + + fn analyze_function(&mut self, id: usize) { + self.work.visits.set(0); + let function = self.program.functions[id].clone(); + let mut summary: Option<Vec<Fact>> = None; + for clause in &function.clauses { + let facts = self.clause(&function, clause); + summary = Some(match summary { + None => facts, + Some(previous) => previous + .iter() + .zip(&facts) + .map(|(left, right)| left.merge(right, &self.work)) + .collect(), + }); + } + let summary = summary.unwrap_or_else(|| { + function + .outputs + .iter() + .map(|field| self.field_fact(field)) + .collect() + }); + self.summaries[id] = Some(summary); + if self.work.exhausted() { + self.errors.push(Diagnostic::error("C015", "value contract analysis exceeds its proof-work limit; simplify the value or contract flow", function.span)); + } + } + + fn clause(&mut self, function: &Function, clause: &Clause) -> Vec<Fact> { + let mut locals: Vec<_> = clause.locals.iter().map(|ty| self.unknown(ty)).collect(); + let mut axis = 0; + let inputs: Vec<_> = function + .parameters + .iter() + .enumerate() + .map(|(parameter, ty)| { + if let Parameter::Value(_) = ty { + let mut fact = Fact::domain(clause.patterns[axis].clone()); + axis += 1; + fact.origin = Some(InputOrigin { + parameter, + fields: Vec::new(), + }); + fact.evidence = Some(function.span); + Some(fact) + } else { + None + } + }) + .collect(); + for binding in &clause.bindings { + let input = inputs[binding.input].as_ref().expect("value binding"); + locals[binding.slot] = match binding.projection { + Projection::Whole | Projection::NonNull => input.clone(), + Projection::Payload { variant, field } => { + let ty = function.parameters[binding.input] + .value_type() + .expect("value binding"); + if let Type::Related(payload) = ty.base() { + debug_assert_eq!(variant, 1); + debug_assert_eq!(field, 0); + let mut fact = self.unknown(payload); + fact.evidence = Some(function.span); + locals[binding.slot] = fact; + continue; + } + let Type::Named(id) = ty.base() else { + unreachable!("choice binding") + }; + let Shape::Choice(variants) = &self.program.types[*id].shape else { + unreachable!("choice binding") + }; + let mut fact = self.field_fact(&variants[variant].fields[field]); + fact.evidence = Some(self.program.types[*id].span); + fact + } + }; + } + for statement in &clause.body { + match statement { + Statement::Assign { + slot, + fields, + value, + } => { + let fact = self.expression(value, &locals, function); + if fields.is_empty() { + if let Some(output) = + clause.outputs.iter().position(|output| output == slot) + { + self.require( + &fact, + &function.outputs[output].bounds, + value.span, + "named output", + function.span, + ); + } + locals[*slot] = fact; + } else { + self.write( + &mut locals[*slot], + &clause.locals[*slot], + fields, + fact, + value.span, + ); + } + } + Statement::Call(value) => { + self.expression(value, &locals, function); + } + } + } + clause + .outputs + .iter() + .map(|slot| locals[*slot].clone()) + .collect() + } + + fn write(&mut self, host: &mut Fact, ty: &Type, path: &[usize], value: Fact, span: Span) { + if !self.work.spend() { + return; + } + let field = self.fields(ty).expect("checked assignment path")[path[0]].clone(); + if path.len() == 1 { + self.require(&value, &field.bounds, span, "field assignment", span); + Rc::make_mut(&mut host.fields).insert(path[0], Rc::new(value)); + } else { + let mut child = self.project(host, ty, path[0]); + self.write(&mut child, &field.ty, &path[1..], value, span); + Rc::make_mut(&mut host.fields).insert(path[0], Rc::new(child)); + } + // Value aliases keep their own snapshot. This aggregate no longer equals + // the original parameter after any of its fields change. + host.origin = None; + host.evidence = Some(span); + } + + fn require( + &mut self, + fact: &Fact, + bounds: &Domain, + span: Span, + context: &str, + declaration: Span, + ) { + if bounds.is_empty() || subset(&fact.domain, bounds) { + return; + } + let mut diagnostic = Diagnostic::error( + "C011", + format!("cannot prove {context} satisfies its numeric bounds"), + span, + ); + diagnostic.related.push(Label { + span: declaration, + message: format!("required bounds: {bounds:?}"), + }); + if let Some(evidence) = fact.evidence.filter(|evidence| *evidence != span) { + diagnostic.related.push(Label { + span: evidence, + message: format!("known value domain: {:?}", fact.domain), + }); + } + self.errors.push(diagnostic); + } + + fn expression(&mut self, expression: &Expression, locals: &[Fact], owner: &Function) -> Fact { + if !self.work.spend() { + return self.unknown(&expression.ty); + } + let mut result = match &expression.kind { + ExpressionKind::Constant(value) => Fact::domain(vec![constant_atom(value)]), + ExpressionKind::ChoiceOptions { .. } => self.unknown(&expression.ty), + ExpressionKind::ModelOperation { operation, .. } => { + for value in operation.expressions() { + self.expression(value, locals, owner); + } + self.unknown(&expression.ty) + } + ExpressionKind::JobEnqueue { arguments, .. } => { + for value in arguments { + self.expression(value, locals, owner); + } + self.unknown(&expression.ty) + } + ExpressionKind::Local(slot) => return locals[*slot].clone(), + ExpressionKind::Field { value, index } => { + let fact = self.expression(value, locals, owner); + self.project(&fact, &value.ty, *index) + } + ExpressionKind::Some(value) => self.expression(value, locals, owner), + ExpressionKind::Promote(value) => { + let fact = self.expression(value, locals, owner); + if expression.ty == Type::Decimal { + Fact::domain( + fact.domain + .iter() + .map(|atom| match atom { + Atom::Int(lo, hi) => Atom::Decimal( + DecimalValue::from_int(*lo), + DecimalValue::from_int(*hi), + ), + _ => Atom::Any, + }) + .collect(), + ) + } else { + self.unknown(&expression.ty) + } + } + ExpressionKind::Record { fields } => { + let definitions = self + .fields(&expression.ty) + .expect("checked record") + .to_vec(); + let mut result = self.unknown(&expression.ty); + for (index, value) in fields { + let fact = self.expression(value, locals, owner); + self.require( + &fact, + &definitions[*index].bounds, + value.span, + "record field", + expression.span, + ); + Rc::make_mut(&mut result.fields).insert(*index, Rc::new(fact)); + } + result + } + ExpressionKind::Variant { variant, arguments } => { + let Type::Named(id) = expression.ty.base() else { + unreachable!("choice expression") + }; + let Shape::Choice(variants) = &self.program.types[*id].shape else { + unreachable!("choice expression") + }; + let fields = variants[*variant].fields.clone(); + let mut result = Fact::domain(vec![Atom::Variant(*variant)]); + for (index, value) in arguments.iter().enumerate() { + let fact = self.expression(value, locals, owner); + self.require( + &fact, + &fields[index].bounds, + value.span, + "choice payload", + self.program.types[*id].span, + ); + // Payload projections are resolved only by a selected clause; + // they must not be confused with record/output field indices. + } + Rc::make_mut(&mut result.fields).clear(); + result + } + ExpressionKind::Fail(error) => { + self.expression(error, locals, owner); + self.unknown(&expression.ty) + } + ExpressionKind::Call { target, arguments } + | ExpressionKind::ParallelCall { target, arguments } + | ExpressionKind::BlockingCall { target, arguments } => { + let values: Vec<_> = arguments + .iter() + .map(|argument| match argument { + CallArgument::Value(value) => Some(self.expression(value, locals, owner)), + CallArgument::Handler(_) => None, + }) + .collect(); + self.call(*target, arguments, &values, expression, owner) + } + ExpressionKind::CaptureResult { + target, arguments, .. + } => { + let values: Vec<_> = arguments + .iter() + .map(|argument| match argument { + CallArgument::Value(value) => Some(self.expression(value, locals, owner)), + CallArgument::Handler(_) => None, + }) + .collect(); + self.call(*target, arguments, &values, expression, owner); + self.unknown(&expression.ty) + } + ExpressionKind::RunCall { + group, + target, + arguments, + } => { + if let Some(group) = group { + self.expression(group, locals, owner); + } + let values: Vec<_> = arguments + .iter() + .map(|argument| match argument { + CallArgument::Value(value) => Some(self.expression(value, locals, owner)), + CallArgument::Handler(_) => None, + }) + .collect(); + self.call(*target, arguments, &values, expression, owner) + } + ExpressionKind::AwaitTask(value) => self.expression(value, locals, owner), + ExpressionKind::StopTask(value) + | ExpressionKind::Group(value) + | ExpressionKind::AwaitGroup(value) + | ExpressionKind::StopGroup(value) + | ExpressionKind::ChannelReceive(value) + | ExpressionKind::ChannelClose(value) => { + self.expression(value, locals, owner); + self.unknown(&expression.ty) + } + ExpressionKind::Channel { capacity, .. } => { + self.expression(capacity, locals, owner); + self.unknown(&expression.ty) + } + ExpressionKind::ChannelSend { channel, value } => { + self.expression(channel, locals, owner); + self.expression(value, locals, owner); + self.unknown(&expression.ty) + } + ExpressionKind::Unary { operator, value } => { + let fact = self.expression(value, locals, owner); + unary(*operator, &fact).unwrap_or_else(|| self.unknown(&expression.ty)) + } + ExpressionKind::Binary { + left, + operator, + right, + } => { + let left_fact = self.expression(left, locals, owner); + let right_fact = self.expression(right, locals, owner); + let inferred = binary(*operator, &left_fact, &right_fact); + if let Some(Fact { domain, .. }) = &inferred + && matches!(domain.as_slice(), [Atom::Bool(_)]) + && matches!( + operator, + Binary::Equal + | Binary::NotEqual + | Binary::Less + | Binary::LessEqual + | Binary::Greater + | Binary::GreaterEqual + ) + && (!matches!(left.kind, ExpressionKind::Constant(_)) + || !matches!(right.kind, ExpressionKind::Constant(_))) + { + self.warning( + expression.span, + "comparison is determined by known input/value bounds", + left_fact.evidence, + ); + } + inferred.unwrap_or_else(|| self.unknown(&expression.ty)) + } + ExpressionKind::Collection { + operation, + handler, + arguments, + .. + } => { + let facts: Vec<_> = arguments + .iter() + .map(|value| self.expression(value, locals, owner)) + .collect(); + self.collection(*operation, *handler, arguments, &facts, expression, owner) + } + ExpressionKind::Intrinsic { arguments, .. } | ExpressionKind::List(arguments) => { + for value in arguments { + self.expression(value, locals, owner); + } + self.unknown(&expression.ty) + } + ExpressionKind::Map(entries) => { + for (key, value) in entries { + self.expression(key, locals, owner); + self.expression(value, locals, owner); + } + self.unknown(&expression.ty) + } + }; + if result.evidence.is_none() { + result.evidence = Some(expression.span); + } + result + } + + fn call( + &mut self, + target: CallTarget, + arguments: &[CallArgument], + values: &[Option<Fact>], + expression: &Expression, + owner: &Function, + ) -> Fact { + let outputs = match target { + CallTarget::Function(id) | CallTarget::Handler(HandlerTarget::Function(id)) => { + let callee = self.program.functions[id].clone(); + let summary = self.function(id); + for (parameter, argument) in callee.parameters.iter().zip(arguments) { + if let ( + Parameter::Handler(signature), + CallArgument::Handler(HandlerTarget::Function(handler)), + ) = (parameter, argument) + { + let output = self.function(*handler); + let handler_function = &self.program.functions[*handler]; + let inputs: Vec<_> = signature + .parameters + .iter() + .zip(&signature.bounds) + .map(|(ty, bounds)| { + Some(if bounds.is_empty() { + self.unknown(ty) + } else { + Fact::domain(bounds.clone()) + }) + }) + .collect(); + for (fact, field) in output.iter().zip(&signature.outputs) { + let fact = self.instantiate(fact, handler_function, &inputs); + self.require( + &fact, + &field.bounds, + expression.span, + "handler output", + callee.span, + ); + } + } + } + let domains: Vec<_> = values.iter().flatten().map(|fact| &fact.domain).collect(); + let excluded = callee + .clauses + .iter() + .filter(|clause| { + clause + .patterns + .iter() + .zip(&domains) + .any(|(pattern, domain)| disjoint(pattern, domain)) + }) + .count(); + if excluded > 0 && arguments.iter().any(|argument| matches!(argument, CallArgument::Value(value) if !matches!(value.kind, ExpressionKind::Constant(_)))) { + self.warning(expression.span, &format!("{excluded} clause(s) of '{}' cannot run for the known argument domains", callee.name), values.iter().flatten().find_map(|fact| fact.evidence)); + } + summary + .iter() + .map(|fact| self.instantiate(fact, &callee, values)) + .collect() + } + CallTarget::Handler(HandlerTarget::Parameter(parameter)) => { + let Parameter::Handler(signature) = &owner.parameters[parameter] else { + unreachable!("handler parameter") + }; + for (fact, bounds) in values.iter().flatten().zip(&signature.bounds) { + self.require(fact, bounds, expression.span, "handler input", owner.span); + } + signature + .outputs + .iter() + .map(|field| self.field_fact(field)) + .collect() + } + }; + output_fact(outputs) + } + + fn instantiate(&self, fact: &Fact, callee: &Function, arguments: &[Option<Fact>]) -> Fact { + self.instantiate_cached(fact, callee, arguments, &mut BTreeMap::new()) + } + + fn instantiate_cached( + &self, + fact: &Fact, + callee: &Function, + arguments: &[Option<Fact>], + cache: &mut BTreeMap<usize, Rc<FactFields>>, + ) -> Fact { + if !self.work.spend() { + return Fact::domain(vec![Atom::Any]); + } + if let Some(origin) = &fact.origin { + let mut transferred = arguments[origin.parameter] + .as_ref() + .expect("value origin") + .clone(); + let mut ty = callee.parameters[origin.parameter] + .value_type() + .expect("value origin") + .clone(); + for index in &origin.fields { + transferred = self.project(&transferred, &ty, *index); + let Some(fields) = self.fields(&ty) else { + return fact.clone(); + }; + ty = fields[*index].ty.clone(); + } + return transferred; + } + let mut instantiated = fact.clone(); + let key = Rc::as_ptr(&fact.fields) as usize; + instantiated.fields = if let Some(fields) = cache.get(&key) { + Rc::clone(fields) + } else { + let fields = Rc::new( + fact.fields + .iter() + .map(|(index, field)| { + ( + *index, + Rc::new(self.instantiate_cached(field, callee, arguments, cache)), + ) + }) + .collect(), + ); + cache.insert(key, Rc::clone(&fields)); + fields + }; + instantiated + } + + fn collection( + &mut self, + operation: CollectionOp, + handler: Option<HandlerTarget>, + arguments: &[Expression], + facts: &[Fact], + expression: &Expression, + owner: &Function, + ) -> Fact { + if let Some(HandlerTarget::Parameter(parameter)) = handler { + let Parameter::Handler(signature) = &owner.parameters[parameter] else { + unreachable!("handler parameter") + }; + let mut inputs: Vec<_> = signature + .parameters + .iter() + .map(|ty| self.unknown(ty)) + .collect(); + // Traversals receive one unknown element plus context/state. Bounds + // on an element are not implied by a List<Int> or Stream<Int> type. + if inputs.len() > 1 { + let context = if operation == CollectionOp::ParallelEach { + 2 + } else { + 1 + }; + if let Some(fact) = facts.get(context) { + inputs[1] = fact.clone(); + } + } + if matches!(operation, CollectionOp::Reduce | CollectionOp::ReduceUntil) + && let Some(state) = inputs.get_mut(1) + { + *state = state.merge(&self.field_fact(&signature.outputs[0]), &self.work); + } + for (fact, bounds) in inputs.iter().zip(&signature.bounds) { + self.require( + fact, + bounds, + expression.span, + "sequence handler input", + owner.span, + ); + } + } else if let Some(HandlerTarget::Function(id)) = handler { + self.function(id); + } + if operation == CollectionOp::Length && !arguments.is_empty() { + return Fact::domain(vec![Atom::Int(0, i64::MAX)]); + } + self.unknown(&expression.ty) + } + + fn warning(&mut self, span: Span, message: &str, evidence: Option<Span>) { + let mut diagnostic = Diagnostic::error("W002", message, span); + if let Some(span) = evidence { + diagnostic.related.push(Label { + span, + message: "value guarantee established here".into(), + }); + } + self.warnings.push(diagnostic); + } +} + +fn output_fact(mut outputs: Vec<Fact>) -> Fact { + if outputs.len() == 1 { + return outputs.remove(0); + } + let mut result = Fact::domain(vec![Atom::Any]); + result.fields = Rc::new( + outputs + .into_iter() + .enumerate() + .map(|(index, fact)| (index, Rc::new(fact))) + .collect(), + ); + result +} diff --git a/crates/dever-core/src/contracts/facts/domain.rs b/crates/dever-core/src/contracts/facts/domain.rs new file mode 100644 index 0000000..8054991 --- /dev/null +++ b/crates/dever-core/src/contracts/facts/domain.rs @@ -0,0 +1,188 @@ +//! Finite abstract-domain operations. Unknowns never establish a numeric bound. + +use crate::hir::{Atom, Constant, Domain}; +use crate::syntax::{BinaryOperator as Binary, UnaryOperator}; +use dever_runtime::number::DecimalValue; + +use super::Fact; + +pub(super) fn constant_atom(constant: &Constant) -> Atom { + match constant { + Constant::Null => Atom::Null, + Constant::Bool(value) => Atom::Bool(*value), + Constant::Int(value) => Atom::Int(*value, *value), + Constant::Decimal(value) => Atom::Decimal(*value, *value), + _ => Atom::Literal(constant.clone()), + } +} + +pub(super) fn subset(domain: &Domain, bounds: &Domain) -> bool { + !domain.is_empty() + && domain.iter().all(|value| { + bounds.iter().any(|bound| match (value, bound) { + (_, Atom::Any) => true, + (Atom::Int(lo, hi), Atom::Int(min, max)) => min <= lo && hi <= max, + (Atom::Decimal(lo, hi), Atom::Decimal(min, max)) => min <= lo && hi <= max, + _ => value == bound, + }) + }) +} + +/// Empty contract bounds mean the complete type domain, not an empty value set. +pub(crate) fn bounds_subset(actual: &Domain, required: &Domain) -> bool { + required.is_empty() || (!actual.is_empty() && subset(actual, required)) +} + +pub(super) fn disjoint(left: &Domain, right: &Domain) -> bool { + left.iter().all(|left| { + right.iter().all(|right| match (left, right) { + (Atom::Any | Atom::Remainder(_), _) | (_, Atom::Any | Atom::Remainder(_)) => false, + (Atom::Int(lo, hi), Atom::Int(min, max)) => hi < min || max < lo, + (Atom::Decimal(lo, hi), Atom::Decimal(min, max)) => hi < min || max < lo, + _ => left != right, + }) + }) +} + +pub(super) fn union(left: &Domain, right: &Domain) -> Domain { + let mut result = left.clone(); + for atom in right { + if !result.contains(atom) { + result.push(atom.clone()); + } + } + if result.iter().any(|atom| matches!(atom, Atom::Any)) { + return vec![Atom::Any]; + } + if result.iter().all(|atom| matches!(atom, Atom::Int(..))) { + let (lo, hi) = result.iter().fold((i64::MAX, i64::MIN), |(lo, hi), atom| { + let Atom::Int(min, max) = atom else { + unreachable!() + }; + (lo.min(*min), hi.max(*max)) + }); + return vec![Atom::Int(lo, hi)]; + } + if result.iter().all(|atom| matches!(atom, Atom::Decimal(..))) { + let (lo, hi) = result.iter().fold( + (DecimalValue::maximum(), DecimalValue::minimum()), + |(lo, hi), atom| { + let Atom::Decimal(min, max) = atom else { + unreachable!() + }; + (lo.min(*min), hi.max(*max)) + }, + ); + return vec![Atom::Decimal(lo, hi)]; + } + // Widen large finite unions rather than growing summaries with every caller. + if result.len() > 32 { + vec![Atom::Any] + } else { + result + } +} + +pub(super) fn unary(operator: UnaryOperator, value: &Fact) -> Option<Fact> { + let domain = value + .domain + .iter() + .map(|atom| match (operator, atom) { + (UnaryOperator::Not, Atom::Bool(value)) => Some(Atom::Bool(!value)), + (UnaryOperator::Negate, Atom::Int(lo, hi)) => { + Some(Atom::Int(hi.checked_neg()?, lo.checked_neg()?)) + } + (UnaryOperator::Negate, Atom::Decimal(lo, hi)) => Some(Atom::Decimal( + hi.checked_neg().ok()?, + lo.checked_neg().ok()?, + )), + _ => None, + }) + .collect::<Option<Domain>>()?; + Some(Fact::domain(domain)) +} + +pub(super) fn binary(operator: Binary, left: &Fact, right: &Fact) -> Option<Fact> { + if left.domain.len() != 1 || right.domain.len() != 1 { + return None; + } + let atom = match (&left.domain[0], &right.domain[0]) { + (Atom::Int(lo, hi), Atom::Int(min, max)) => match operator { + Binary::Add => Atom::Int(lo.checked_add(*min)?, hi.checked_add(*max)?), + Binary::Subtract => Atom::Int(lo.checked_sub(*max)?, hi.checked_sub(*min)?), + Binary::Multiply => { + let products = [ + lo.checked_mul(*min)?, + lo.checked_mul(*max)?, + hi.checked_mul(*min)?, + hi.checked_mul(*max)?, + ]; + Atom::Int(*products.iter().min()?, *products.iter().max()?) + } + _ => Atom::Bool(compare(operator, lo, hi, min, max)?), + }, + (Atom::Decimal(lo, hi), Atom::Decimal(min, max)) => match operator { + Binary::Add => Atom::Decimal(lo.checked_add(*min).ok()?, hi.checked_add(*max).ok()?), + Binary::Subtract => { + Atom::Decimal(lo.checked_sub(*max).ok()?, hi.checked_sub(*min).ok()?) + } + Binary::Multiply => { + let products = [ + lo.checked_mul(*min).ok()?, + lo.checked_mul(*max).ok()?, + hi.checked_mul(*min).ok()?, + hi.checked_mul(*max).ok()?, + ]; + Atom::Decimal(*products.iter().min()?, *products.iter().max()?) + } + _ => Atom::Bool(compare(operator, lo, hi, min, max)?), + }, + (Atom::Bool(left), Atom::Bool(right)) => Atom::Bool(match operator { + Binary::And => *left && *right, + Binary::Or => *left || *right, + Binary::Equal => left == right, + Binary::NotEqual => left != right, + _ => return None, + }), + // A choice tag proves inequality across variants, but equal payload-bearing + // tags do not prove equality of the complete values. + (Atom::Variant(left), Atom::Variant(right)) if left != right => { + Atom::Bool(match operator { + Binary::Equal => false, + Binary::NotEqual => true, + _ => return None, + }) + } + (Atom::Null, Atom::Null) => Atom::Bool(match operator { + Binary::Equal => true, + Binary::NotEqual => false, + _ => return None, + }), + (Atom::Null, other) | (other, Atom::Null) + if !matches!(other, Atom::Any | Atom::Remainder(_)) => + { + Atom::Bool(match operator { + Binary::Equal => false, + Binary::NotEqual => true, + _ => return None, + }) + } + _ => return None, + }; + Some(Fact::domain(vec![atom])) +} + +fn compare<T: Ord>(operator: Binary, lo: &T, hi: &T, min: &T, max: &T) -> Option<bool> { + match operator { + Binary::Less if hi < min => Some(true), + Binary::Less if lo >= max => Some(false), + Binary::LessEqual if hi <= min => Some(true), + Binary::LessEqual if lo > max => Some(false), + Binary::Greater => compare(Binary::Less, min, max, lo, hi), + Binary::GreaterEqual => compare(Binary::LessEqual, min, max, lo, hi), + Binary::Equal if hi < min || max < lo => Some(false), + Binary::Equal if lo == hi && min == max => Some(lo == min), + Binary::NotEqual => compare(Binary::Equal, lo, hi, min, max).map(|value| !value), + _ => None, + } +} diff --git a/crates/dever-core/src/contracts/failures.rs b/crates/dever-core/src/contracts/failures.rs new file mode 100644 index 0000000..738cb5c --- /dev/null +++ b/crates/dever-core/src/contracts/failures.rs @@ -0,0 +1,692 @@ +use std::cell::OnceCell; +use std::collections::{BTreeMap, BTreeSet}; +use std::rc::Rc; + +use crate::diagnostic::{Diagnostic, Label}; +use crate::hir::{ + Atom, CallArgument, CallTarget, Clause, CollectionOp, Domain, Expression, + ExpressionKind as Expr, Function, Program, Projection, Statement, +}; +use crate::source::Span; +use crate::syntax::BinaryOperator; +use crate::types::{Parameter, Shape, Type, output_type}; + +use super::MAX_PROOF_WORK; + +/// Origins stay attached to values; reading or renaming a result is not handling it. +#[derive(Clone, Default)] +struct Value { + own: BTreeSet<usize>, + fields: Vec<Rc<Value>>, + definite_error: bool, + fields_guaranteed: bool, + list_length: Option<usize>, + origins_cache: OnceCell<BTreeSet<usize>>, + error_cache: OnceCell<bool>, +} + +impl Value { + fn origins(&self) -> &BTreeSet<usize> { + self.origins_cache.get_or_init(|| { + let mut origins = self.own.clone(); + for field in &self.fields { + origins.extend(field.origins()); + } + origins + }) + } + + fn carries_error(&self) -> bool { + *self.error_cache.get_or_init(|| { + self.definite_error + || (self.fields_guaranteed && self.fields.iter().any(|field| field.carries_error())) + }) + } + + fn invalidate(&mut self) { + self.origins_cache.take(); + self.error_cache.take(); + } +} + +struct Flow<'a> { + program: &'a Program, + summaries: &'a [Vec<bool>], + error_types: &'a [bool], + expanded: usize, + exhausted: Option<Span>, + pending: BTreeMap<usize, Span>, + next_origin: usize, + controls: BTreeSet<usize>, + locals: Vec<Value>, +} + +pub(super) fn check( + program: &Program, + dependencies: &super::dependencies::Dependencies, + errors: &mut Vec<Diagnostic>, +) { + let error_types = error_types(program); + // Monotone finite summaries describe guaranteed failure outputs, not guessed success. + let mut summaries: Vec<Vec<bool>> = program + .functions + .iter() + .map(|f| vec![false; f.outputs.len()]) + .collect(); + let mut pending = vec![Vec::new(); program.functions.len()]; + let mut work = dependencies.work(); + while let Some(id) = work.next() { + let function = &program.functions[id]; + let mut next = vec![true; function.outputs.len()]; + pending[id].clear(); + for clause in &function.clauses { + let analysis = analyze(program, &summaries, &error_types, function, clause); + if let Some(span) = analysis.exhausted { + errors.push(Diagnostic::error("C015", "failure analysis exceeds its structural limit; simplify the nested failure-bearing value", span)); + return; + } + pending[id].push(analysis.pending.values().next().copied()); + for (guarantee, value) in next.iter_mut().zip(&analysis.outputs) { + *guarantee &= value.carries_error(); + } + } + if next != summaries[id] { + summaries[id] = next; + work.changed(id); + } + } + for (function, clauses) in program.functions.iter().zip(pending) { + if function.recovery.is_some() { + continue; + } + for span in clauses.into_iter().flatten() { + let mut diagnostic = Diagnostic::error( + "C012", + "failure is not handled: propagate it, return an explicit error, or declare recover(\"reason\")", + span, + ); + diagnostic.related.push(Label { span: function.span, message: format!("function '{}' leaves a failure obligation unresolved; logging or reading it is not recovery", function.name) }); + errors.push(diagnostic); + } + } +} + +fn error_types(program: &Program) -> Vec<bool> { + let mut known = vec![false; program.types.len()]; + loop { + let mut changed = false; + for (id, definition) in program.types.iter().enumerate() { + if known[id] { + continue; + } + let contains = match &definition.shape { + Shape::Record(fields) => fields.iter().any(|field| has_error(&field.ty, &known)), + Shape::Choice(variants) => variants.iter().any(|variant| { + variant.error + || variant + .fields + .iter() + .any(|field| has_error(&field.ty, &known)) + }), + }; + if contains { + known[id] = true; + changed = true; + } + } + if !changed { + return known; + } + } +} + +fn has_error(ty: &Type, known: &[bool]) -> bool { + match ty { + Type::Named(id) => known[*id], + Type::Nullable(inner) | Type::List(inner) => has_error(inner, known), + Type::Map(key, value) | Type::MapEntry(key, value) => { + has_error(key, known) || has_error(value, known) + } + Type::Outputs(fields) | Type::Task(fields) => { + fields.iter().any(|field| has_error(&field.ty, known)) + } + _ => false, + } +} + +struct Analysis { + outputs: Vec<Value>, + pending: BTreeMap<usize, Span>, + exhausted: Option<Span>, +} + +fn analyze( + program: &Program, + summaries: &[Vec<bool>], + error_types: &[bool], + function: &Function, + clause: &Clause, +) -> Analysis { + let mut flow = Flow { + program, + summaries, + error_types, + expanded: 0, + exhausted: None, + pending: BTreeMap::new(), + next_origin: 0, + controls: BTreeSet::new(), + locals: vec![Value::default(); clause.locals.len()], + }; + let mut inputs = BTreeMap::new(); + let mut domains = clause.patterns.iter(); + for (index, parameter) in function.parameters.iter().enumerate() { + if let Parameter::Value(ty) = parameter { + let domain = domains.next().expect("value input domain"); + let value = flow.unknown(ty, Some(domain), function.span); + flow.controls.extend(value.origins()); + inputs.insert(index, value); + } + } + for binding in &clause.bindings { + let value = &inputs[&binding.input]; + flow.locals[binding.slot] = match binding.projection { + Projection::Whole | Projection::NonNull => value.clone(), + Projection::Payload { field, .. } => value + .fields + .get(field) + .map(|value| value.as_ref().clone()) + .unwrap_or_default(), + }; + } + for statement in &clause.body { + match statement { + Statement::Assign { + slot, + fields, + value, + } => { + let assigned = flow.expression(value); + let mut target = &mut flow.locals[*slot]; + for index in fields { + target.invalidate(); + // Type checking guarantees the path; unknown containers get their declared shape. + if target.fields.len() <= *index { + target.fields.resize_with(index + 1, Default::default); + } + target = Rc::make_mut(&mut target.fields[*index]); + } + *target = assigned; + } + Statement::Call(value) => { + flow.expression(value); + } + } + } + let outputs: Vec<_> = clause + .outputs + .iter() + .map(|slot| flow.locals[*slot].clone()) + .collect(); + for output in &outputs { + flow.consume(output); + } + if outputs.iter().any(Value::carries_error) { + for control in &flow.controls { + flow.pending.remove(control); + } + } + Analysis { + outputs, + pending: flow.pending, + exhausted: flow.exhausted, + } +} + +impl Flow<'_> { + fn origin(&mut self, span: Span) -> usize { + let origin = self.next_origin; + self.next_origin += 1; + self.pending.insert(origin, span); + origin + } + + fn consume(&mut self, value: &Value) { + for origin in value.origins() { + self.pending.remove(origin); + } + } + + fn unknown(&mut self, ty: &Type, domain: Option<&Domain>, span: Span) -> Value { + if !has_error(ty, self.error_types) { + return Value::default(); + } + if self.expanded >= MAX_PROOF_WORK { + self.exhausted = Some(span); + return Value::default(); + } + self.expanded += 1; + match ty { + Type::Named(id) => match &self.program.types[*id].shape { + Shape::Record(fields) => Value { + fields: fields + .iter() + .map(|field| Rc::new(self.unknown(&field.ty, None, span))) + .collect(), + fields_guaranteed: true, + ..Value::default() + }, + Shape::Choice(variants) => { + let selected: Vec<_> = variants + .iter() + .enumerate() + .filter(|(index, _)| { + domain.is_none_or(|atoms| { + atoms.iter().any(|atom| { + matches!(atom, Atom::Any) + || matches!(atom, Atom::Variant(id) if id == index) + }) + }) + }) + .collect(); + let possible_error = selected.iter().any(|(_, variant)| variant.error); + let definite_error = + !selected.is_empty() && selected.iter().all(|(_, variant)| variant.error); + let fields = if selected.len() == 1 { + selected[0] + .1 + .fields + .iter() + .map(|field| Rc::new(self.unknown(&field.ty, None, span))) + .collect() + } else { + // Distinct alternatives have no common payload projection. Keep their obligations. + selected + .iter() + .flat_map(|(_, variant)| &variant.fields) + .map(|field| Rc::new(self.unknown(&field.ty, None, span))) + .collect() + }; + Value { + own: if possible_error { + BTreeSet::from([self.origin(span)]) + } else { + BTreeSet::new() + }, + fields, + definite_error, + fields_guaranteed: selected.len() == 1, + list_length: None, + ..Value::default() + } + } + }, + Type::Nullable(inner) => { + if domain.is_some_and(|atoms| atoms.iter().all(|atom| matches!(atom, Atom::Null))) { + Value::default() + } else { + let mut value = self.unknown(inner, domain, span); + if domain + .is_none_or(|atoms| atoms.iter().any(|atom| matches!(atom, Atom::Null))) + { + value.definite_error = false; + value.fields_guaranteed = false; + } + value + } + } + Type::List(inner) => Value { + fields: vec![Rc::new(self.unknown(inner, None, span))], + ..Value::default() + }, + Type::Map(key, value) | Type::MapEntry(key, value) => Value { + fields: vec![ + Rc::new(self.unknown(key, None, span)), + Rc::new(self.unknown(value, None, span)), + ], + fields_guaranteed: matches!(ty, Type::MapEntry(..)), + ..Value::default() + }, + Type::Outputs(fields) | Type::Task(fields) => Value { + fields: fields + .iter() + .map(|field| Rc::new(self.unknown(&field.ty, None, span))) + .collect(), + fields_guaranteed: true, + ..Value::default() + }, + // An unread Stream has no yielded failure. Its handler checks each yielded value. + _ => Value::default(), + } + } + + fn expression(&mut self, expression: &Expression) -> Value { + match &expression.kind { + Expr::Local(slot) => self.locals[*slot].clone(), + Expr::Field { value, index } => self + .expression(value) + .fields + .get(*index) + .map(|value| value.as_ref().clone()) + .unwrap_or_default(), + Expr::Record { fields } => { + let empty = Rc::new(Value::default()); + let mut values = vec![empty; fields.len()]; + for (index, value) in fields { + values[*index] = Rc::new(self.expression(value)); + } + Value { + fields: values, + fields_guaranteed: true, + ..Value::default() + } + } + Expr::Variant { variant, arguments } => { + let fields = arguments + .iter() + .map(|value| Rc::new(self.expression(value))) + .collect(); + let error = matches!(&expression.ty, Type::Named(id) if matches!(&self.program.types[*id].shape, Shape::Choice(variants) if variants[*variant].error)); + Value { + fields, + definite_error: error, + fields_guaranteed: true, + own: if error { + BTreeSet::from([self.origin(expression.span)]) + } else { + BTreeSet::new() + }, + list_length: None, + ..Value::default() + } + } + Expr::Fail(error) => { + self.expression(error); + // fail terminates the clause with an explicit propagated or converted + // failure, so no previously observed obligation can be silently lost. + self.pending.clear(); + Value::default() + } + Expr::CaptureResult { arguments, .. } => { + for argument in arguments { + if let CallArgument::Value(value) = argument { + let value = self.expression(value); + self.consume(&value); + } + } + self.unknown(&expression.ty, None, expression.span) + } + Expr::ModelOperation { operation, .. } => { + for value in operation.expressions() { + let value = self.expression(value); + self.consume(&value); + } + self.unknown(&expression.ty, None, expression.span) + } + Expr::JobEnqueue { arguments, .. } => { + for value in arguments { + let value = self.expression(value); + self.consume(&value); + } + self.unknown(&expression.ty, None, expression.span) + } + Expr::List(elements) => Value { + fields: elements + .iter() + .map(|value| Rc::new(self.expression(value))) + .collect(), + fields_guaranteed: true, + list_length: Some(elements.len()), + ..Value::default() + }, + Expr::Map(entries) => { + let mut fields = Vec::new(); + for (index, (key, value)) in entries.iter().enumerate() { + let key_value = self.expression(key); + let value = self.expression(value); + // Later equal keys replace earlier entries. Unknown keys cannot + // prove that a failure survives construction of the Map. + if entries[index + 1..].iter().all(|(later, _)| matches!((&key.kind, &later.kind), (Expr::Constant(left), Expr::Constant(right)) if left != right)) { + fields.extend([Rc::new(key_value), Rc::new(value)]); + } + } + Value { + fields, + fields_guaranteed: true, + ..Value::default() + } + } + Expr::Some(value) | Expr::Promote(value) => self.expression(value), + Expr::Unary { value, .. } => { + self.expression(value); + Value::default() + } + Expr::Binary { + left, + operator, + right, + } => { + self.expression(left); + let short_circuit = match operator { + BinaryOperator::And => Some(false), + BinaryOperator::Or => Some(true), + _ => None, + }; + if let Some(skip_when) = short_circuit { + if let Expr::Constant(crate::hir::Constant::Bool(value)) = left.kind { + if value == skip_when { + return Value::default(); + } + } else { + let pending = self.pending.clone(); + self.expression(right); + // The right operand may never execute, so it cannot discharge + // obligations that already existed before the conditional call. + self.pending.extend(pending); + return Value::default(); + } + } + self.expression(right); + Value::default() + } + Expr::Call { target, arguments } + | Expr::ParallelCall { target, arguments } + | Expr::BlockingCall { target, arguments } => { + self.call_value(*target, arguments, &expression.ty, expression.span) + } + Expr::RunCall { + group, + target, + arguments, + } => { + if let Some(group) = group { + self.expression(group); + } + let Type::Task(outputs) = &expression.ty else { + self.call_value(*target, arguments, &Type::Unit, expression.span); + return Value::default(); + }; + let result = + self.call_value(*target, arguments, &output_type(outputs), expression.span); + match outputs.as_slice() { + [] => Value::default(), + [_] => Value { + fields: vec![Rc::new(result)], + fields_guaranteed: true, + ..Value::default() + }, + _ => result, + } + } + Expr::AwaitTask(task) => { + let result = self.expression(task); + match &task.ty { + Type::Task(outputs) if outputs.len() == 1 => result + .fields + .first() + .map(|value| value.as_ref().clone()) + .unwrap_or_default(), + Type::Task(_) => result, + _ => unreachable!("await task HIR carries a Task"), + } + } + Expr::StopTask(task) + | Expr::Group(task) + | Expr::AwaitGroup(task) + | Expr::StopGroup(task) + | Expr::ChannelReceive(task) + | Expr::ChannelClose(task) => { + self.expression(task); + self.unknown(&expression.ty, None, expression.span) + } + Expr::Channel { capacity, .. } => { + self.expression(capacity); + Value::default() + } + Expr::ChannelSend { channel, value } => { + self.expression(channel); + let value = self.expression(value); + self.consume(&value); + Value::default() + } + Expr::Intrinsic { + operation: crate::intrinsic::Intrinsic::StreamOf, + arguments, + .. + } => self.expression(&arguments[0]), + Expr::Intrinsic { arguments, .. } => { + // Typed primitive boundaries consume task/context/upload inputs; + // recoverable failures remain explicit in their checked outputs. + for value in arguments { + let value = self.expression(value); + self.consume(&value); + } + self.unknown(&expression.ty, None, expression.span) + } + Expr::Collection { + operation, + handler, + arguments, + .. + } => self.collection(expression, operation, handler.is_some(), arguments), + Expr::Constant(_) | Expr::ChoiceOptions { .. } => Value::default(), + } + } + + fn call_value( + &mut self, + target: CallTarget, + arguments: &[CallArgument], + output: &Type, + span: Span, + ) -> Value { + for argument in arguments { + if let CallArgument::Value(value) = argument { + let value = self.expression(value); + self.consume(&value); + } + } + let mut result = self.unknown(output, None, span); + let callee = match target { + CallTarget::Function(id) + | CallTarget::Handler(crate::hir::HandlerTarget::Function(id)) => Some(id), + CallTarget::Handler(crate::hir::HandlerTarget::Parameter(_)) => None, + }; + if let Some(id) = callee { + match self.summaries[id].as_slice() { + [guarantee] => result.definite_error = *guarantee, + guarantees => { + for (field, guarantee) in result.fields.iter_mut().zip(guarantees) { + let field = Rc::make_mut(field); + field.invalidate(); + field.definite_error = *guarantee; + } + } + } + } + result + } + + fn collection( + &mut self, + expression: &Expression, + operation: &CollectionOp, + has_handler: bool, + arguments: &[Expression], + ) -> Value { + let values: Vec<_> = arguments + .iter() + .map(|value| self.expression(value)) + .collect(); + match operation { + CollectionOp::Append if values[0].list_length.is_some() => { + let mut result = values[0].clone(); + result.invalidate(); + result.fields.push(Rc::new(values[1].clone())); + result.list_length = result.list_length.map(|length| length + 1); + result + } + CollectionOp::Append | CollectionOp::Entries => { + // Copying/repacking errors keeps their identities and obligations. + Value { + fields: values.into_iter().map(Rc::new).collect(), + fields_guaranteed: true, + ..Value::default() + } + } + CollectionOp::Each + | CollectionOp::ParallelEach + | CollectionOp::Reduce + | CollectionOp::Filter + | CollectionOp::Sum + if has_handler => + { + if values[0].list_length == Some(0) { + return if *operation == CollectionOp::Reduce { + values[1].clone() + } else { + Value { + list_length: Some(0), + ..Value::default() + } + }; + } + self.consume(&values[0]); + // Context is not passed anywhere when a traversal is empty. + if *operation == CollectionOp::Reduce + || values[0].list_length.is_some_and(|length| length > 0) + { + for value in &values[1..] { + self.consume(value); + } + } + self.unknown(&expression.ty, None, expression.span) + } + CollectionOp::ReduceUntil => { + // Only the state is wholly transferred; a finite source may stop early. + if let Some(state) = values.get(1) { + self.consume(state); + } + self.unknown(&expression.ty, None, expression.span) + } + CollectionOp::First if values[0].list_length == Some(1) => { + values[0].fields[0].as_ref().clone() + } + CollectionOp::First if values[0].list_length == Some(0) => Value::default(), + CollectionOp::Put => { + // The replaced entry may contain an unhandled failure. Only the + // inserted key/value are guaranteed to survive this operation. + Value { + fields: values[1..].iter().cloned().map(Rc::new).collect(), + fields_guaranteed: true, + ..Value::default() + } + } + CollectionOp::First | CollectionOp::Get | CollectionOp::Find | CollectionOp::Remove => { + // A projection cannot silently dispose of errors in the unselected remainder. + self.unknown(&expression.ty, None, expression.span) + } + _ => Value::default(), + } + } +} diff --git a/crates/dever-core/src/contracts/redundancy.rs b/crates/dever-core/src/contracts/redundancy.rs new file mode 100644 index 0000000..7dd5879 --- /dev/null +++ b/crates/dever-core/src/contracts/redundancy.rs @@ -0,0 +1,486 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt::Write; + +use crate::diagnostic::{Diagnostic, Label}; +use crate::hir::{ + CallArgument, CallTarget, Clause, Expression, ExpressionKind as Expr, Function, Program, + Projection, Statement, +}; +use crate::source::Span; +use crate::types::{Parameter, Type}; + +pub(crate) fn check(program: &mut Program) { + let mut warnings = Vec::new(); + let mut implementations = BTreeMap::new(); + for (id, function) in program.functions.iter().enumerate() { + if function.port.is_some() || function.setting.is_some() { + continue; + } + if !function.public { + let key = (function.owner, function_key(function)); + if let Some(previous) = implementations.get(&key).copied() { + let previous: &Function = &program.functions[previous]; + warnings.push(related("W003", format!("private function '{}' has the same implementation and contract as '{}'; consider sharing one implementation", function.name, previous.name), function.span, previous.span)); + } else { + implementations.insert(key, id); + if let Some(target) = forwarding_target(function, program) { + warnings.push(related("W005", format!("private function '{}' only forwards unchanged inputs to '{}' without adding a contract; review whether this wrapper is needed", function.name, target.name), function.span, target.span)); + } + } + } + for clause in &function.clauses { + repeated_calls(clause, program, &mut warnings); + } + } + program.warnings.extend(warnings); +} + +fn related(code: &'static str, message: String, primary: Span, previous: Span) -> Diagnostic { + let mut warning = Diagnostic::error(code, message, primary); + warning.related.push(Label { + span: previous, + message: "previous implementation or computation".into(), + }); + warning +} + +fn function_key(function: &Function) -> String { + let mut key = format!( + "{:?}|{:?}|{}|{}|{:?}", + function.parameters, function.outputs, function.suspends, function.pure, function.recovery + ); + for clause in &function.clauses { + write!( + key, + "clause({:?}|{:?}|{:?}|{:?})", + clause.patterns, clause.bindings, clause.locals, clause.outputs + ) + .expect("string formatting"); + for statement in &clause.body { + match statement { + Statement::Assign { + slot, + fields, + value, + } => { + write!(key, "assign({slot},{fields:?})").expect("string formatting"); + expression_key(value, &[], &mut key); + } + Statement::Call(value) => { + key.push_str("action"); + expression_key(value, &[], &mut key); + } + } + } + } + key +} + +/// Structural encoding has explicit node delimiters and never includes source positions. +fn expression_key(expression: &Expression, versions: &[usize], key: &mut String) { + write!(key, "({:?}:", expression.ty).expect("string formatting"); + match &expression.kind { + Expr::Constant(value) => write!(key, "constant{value:?}").expect("string formatting"), + Expr::Local(slot) => write!( + key, + "local{slot}@{}", + versions.get(*slot).copied().unwrap_or_default() + ) + .expect("string formatting"), + Expr::Field { value, index } => { + write!(key, "field{index}").expect("string formatting"); + expression_key(value, versions, key); + } + Expr::ChoiceOptions { choice } => { + write!(key, "choice_options{choice}").expect("string formatting"); + } + Expr::ModelOperation { model, operation } => { + write!(key, "model{model}/{}", model_operation_name(operation)) + .expect("string formatting"); + for value in operation.expressions() { + expression_key(value, versions, key); + } + } + Expr::JobEnqueue { + target, + arguments, + scheduled, + } => { + write!(key, "enqueue{target}/{scheduled}").expect("string formatting"); + for value in arguments { + expression_key(value, versions, key); + } + } + Expr::Call { target, arguments } => { + write!(key, "call{target:?}").expect("string formatting"); + call_arguments_key(arguments, versions, key); + } + Expr::Fail(error) => { + key.push_str("fail"); + expression_key(error, versions, key); + } + Expr::CaptureResult { + target, + arguments, + choice, + success, + } => { + write!(key, "capture{target:?}/{choice}/{success}").expect("string formatting"); + call_arguments_key(arguments, versions, key); + } + Expr::RunCall { + group, + target, + arguments, + } => { + write!(key, "run_call{target:?}").expect("string formatting"); + if let Some(group) = group { + expression_key(group, versions, key); + } + call_arguments_key(arguments, versions, key); + } + Expr::ParallelCall { target, arguments } => { + write!(key, "parallel_call{target:?}").expect("string formatting"); + call_arguments_key(arguments, versions, key); + } + Expr::BlockingCall { target, arguments } => { + write!(key, "blocking_call{target:?}").expect("string formatting"); + call_arguments_key(arguments, versions, key); + } + Expr::AwaitTask(value) => { + key.push_str("await_task"); + expression_key(value, versions, key); + } + Expr::StopTask(value) => { + key.push_str("stop_task"); + expression_key(value, versions, key); + } + Expr::Group(value) => { + key.push_str("group"); + expression_key(value, versions, key); + } + Expr::AwaitGroup(value) => { + key.push_str("await_group"); + expression_key(value, versions, key); + } + Expr::StopGroup(value) => { + key.push_str("stop_group"); + expression_key(value, versions, key); + } + Expr::Channel { element, capacity } => { + write!(key, "channel{element:?}").expect("string formatting"); + expression_key(capacity, versions, key); + } + Expr::ChannelSend { channel, value } => { + key.push_str("channel_send"); + expression_key(channel, versions, key); + expression_key(value, versions, key); + } + Expr::ChannelReceive(value) => { + key.push_str("channel_receive"); + expression_key(value, versions, key); + } + Expr::ChannelClose(value) => { + key.push_str("channel_close"); + expression_key(value, versions, key); + } + Expr::Intrinsic { + operation, + handler, + arguments, + } => { + write!(key, "intrinsic{operation:?}/{handler:?}").expect("string formatting"); + arguments_key(arguments, versions, key); + } + Expr::Collection { + operation, + handler, + sequence, + arguments, + } => { + write!(key, "collection{operation:?}/{handler:?}/{sequence:?}") + .expect("string formatting"); + arguments_key(arguments, versions, key); + } + Expr::Record { fields } => { + key.push_str("record"); + for (index, value) in fields { + write!(key, "field{index}").expect("string formatting"); + expression_key(value, versions, key); + } + } + Expr::Variant { variant, arguments } => { + write!(key, "variant{variant}").expect("string formatting"); + arguments_key(arguments, versions, key); + } + Expr::List(values) => { + key.push_str("list"); + arguments_key(values, versions, key); + } + Expr::Map(entries) => { + key.push_str("map"); + for (entry_key, value) in entries { + expression_key(entry_key, versions, key); + expression_key(value, versions, key); + } + } + Expr::Unary { operator, value } => { + write!(key, "unary{operator:?}").expect("string formatting"); + expression_key(value, versions, key); + } + Expr::Binary { + left, + operator, + right, + } => { + write!(key, "binary{operator:?}").expect("string formatting"); + expression_key(left, versions, key); + expression_key(right, versions, key); + } + Expr::Promote(value) => { + key.push_str("promote"); + expression_key(value, versions, key); + } + Expr::Some(value) => { + key.push_str("some"); + expression_key(value, versions, key); + } + } + key.push(')'); +} + +fn model_operation_name(operation: &crate::hir::ModelOperation) -> String { + use crate::hir::ModelOperation; + match operation { + ModelOperation::Create { .. } => "create".into(), + ModelOperation::CreateMany { .. } => "create_many".into(), + ModelOperation::Get { .. } => "get".into(), + ModelOperation::First { .. } => "first".into(), + ModelOperation::List { .. } => "list".into(), + ModelOperation::Cursor { .. } => "cursor".into(), + ModelOperation::Count { .. } => "count".into(), + ModelOperation::Exists { .. } => "exists".into(), + ModelOperation::Stream { .. } => "stream".into(), + ModelOperation::Update { .. } => "update".into(), + ModelOperation::Delete { .. } => "delete".into(), + ModelOperation::Upsert { .. } => "upsert".into(), + ModelOperation::Sql { operation, .. } => format!("sql{operation}"), + } +} + +fn arguments_key(arguments: &[Expression], versions: &[usize], key: &mut String) { + for argument in arguments { + expression_key(argument, versions, key); + } +} + +fn call_arguments_key(arguments: &[CallArgument], versions: &[usize], key: &mut String) { + for argument in arguments { + match argument { + CallArgument::Value(value) => expression_key(value, versions, key), + CallArgument::Handler(target) => { + write!(key, "handler{target:?}").expect("string formatting"); + } + } + } +} + +struct Computation { + span: Span, + result_slot: usize, + result_version: usize, +} + +fn repeated_calls(clause: &Clause, program: &Program, warnings: &mut Vec<Diagnostic>) { + let mut versions = vec![0; clause.locals.len()]; + let mut previous = BTreeMap::<String, Computation>::new(); + for statement in &clause.body { + let Statement::Assign { + slot, + fields, + value, + } = statement + else { + continue; + }; + let candidate = fields.is_empty() && reusable_call(value, program); + let mut key = String::new(); + if candidate { + expression_key(value, &versions, &mut key); + if let Some(computation) = previous.get(&key) + && versions[computation.result_slot] == computation.result_version + { + warnings.push(related("W004", "this pure call repeats unchanged inputs while the earlier result is still available; consider reusing that result".into(), value.span, computation.span)); + } + } + // A field write invalidates every projection from the containing value. + versions[*slot] += 1; + if candidate { + previous.insert( + key, + Computation { + span: value.span, + result_slot: *slot, + result_version: versions[*slot], + }, + ); + } + } +} + +fn reusable_call(expression: &Expression, program: &Program) -> bool { + let Expr::Call { + target: CallTarget::Function(id), + arguments, + } = &expression.kind + else { + return false; + }; + program.effects[*id].is_empty() + && expression.ty.movable(&program.types) + && arguments.iter().all(|argument| match argument { + CallArgument::Value(value) => stable_argument(value, program), + CallArgument::Handler(_) => true, + }) +} + +fn stable_argument(expression: &Expression, program: &Program) -> bool { + if !expression.ty.movable(&program.types) { + return false; + } + match &expression.kind { + Expr::Constant(_) | Expr::Local(_) => true, + Expr::Field { value, .. } => stable_argument(value, program), + _ => false, + } +} + +pub(crate) fn application_errors( + program: &Program, + reachable: &BTreeSet<usize>, +) -> Vec<Diagnostic> { + let mut errors = Vec::new(); + for (id, function) in program.functions.iter().enumerate() { + let package = &program.packages[function.owner]; + if package.bundled || function.port.is_some() || function.setting.is_some() { + continue; + } + let app = package.role == Some(crate::source::SourceRole::App); + let domain = package.role == Some(crate::source::SourceRole::Domain); + if !app && !domain { + continue; + } + if !reachable.contains(&id) { + errors.push(Diagnostic::error( + "C016", + format!( + "function '{}' is unreachable from API, CMD, or Job production entries", + function.name + ), + function.span, + )); + } else if !function.public { + if let Some(target) = forwarding_target(function, program) { + errors.push(related("C016", format!("private function '{}' only forwards unchanged inputs to '{}' without adding a contract", function.name, target.name), function.span, target.span)); + } + } else if app && trivial_model_wrapper(function) { + errors.push(Diagnostic::error("C016", format!("App function '{}' only forwards a Model CRUD operation; declare rest for plain CRUD", function.name), function.span)); + } + } + errors.sort_by_key(|error| (error.primary.source, error.primary.start)); + errors +} + +fn trivial_model_wrapper(function: &Function) -> bool { + if function.recovery.is_some() || function.clauses.len() != 1 { + return false; + } + let clause = &function.clauses[0]; + if clause + .patterns + .iter() + .any(|pattern| !matches!(pattern.as_slice(), [crate::hir::Atom::Any])) + { + return false; + } + let [ + Statement::Assign { + slot, + fields, + value, + }, + ] = clause.body.as_slice() + else { + return false; + }; + if !fields.is_empty() || clause.outputs.as_slice() != [*slot] { + return false; + } + let Expr::ModelOperation { operation, .. } = &value.kind else { + return false; + }; + if !matches!( + operation.as_ref(), + crate::hir::ModelOperation::Get { .. } | crate::hir::ModelOperation::Create { .. } + ) { + return false; + } + operation.expressions().into_iter().all(|value| matches!(value.kind, Expr::Local(slot) if clause.bindings.iter().any(|binding| binding.slot == slot && matches!(binding.projection, Projection::Whole)))) +} + +fn forwarding_target<'a>(function: &Function, program: &'a Program) -> Option<&'a Function> { + if function.recovery.is_some() || function.clauses.len() != 1 { + return None; + } + if function + .parameters + .iter() + .any(|parameter| !matches!(parameter, Parameter::Value(ty) if plain_scalar(ty))) + || function + .outputs + .iter() + .any(|field| !field.bounds.is_empty() || !plain_scalar(&field.ty)) + { + return None; + } + let clause = &function.clauses[0]; + let expression = match clause.body.as_slice() { + [ + Statement::Assign { + slot, + fields, + value, + }, + ] if fields.is_empty() && clause.outputs.as_slice() == [*slot] => value, + [Statement::Call(value)] if clause.outputs.is_empty() => value, + _ => return None, + }; + let Expr::Call { + target: CallTarget::Function(id), + arguments, + } = &expression.kind + else { + return None; + }; + let target = &program.functions[*id]; + if function.owner != target.owner + || function.parameters != target.parameters + || function.outputs != target.outputs + || function.pure != target.pure + || target.recovery.is_some() + || !program.effects[*id].is_empty() + { + return None; + } + let unchanged = arguments.iter().enumerate().all(|(input, argument)| { + matches!(argument, CallArgument::Value(Expression { kind: Expr::Local(slot), .. }) if clause.bindings.iter().any(|binding| binding.input == input && binding.slot == *slot && matches!(binding.projection, Projection::Whole))) + }); + unchanged.then_some(target) +} + +fn plain_scalar(ty: &Type) -> bool { + matches!( + ty, + Type::Bool | Type::Int | Type::Decimal | Type::Float | Type::Text | Type::Id | Type::Bytes + ) +} diff --git a/crates/dever-core/src/diagnostic.rs b/crates/dever-core/src/diagnostic.rs new file mode 100644 index 0000000..710c310 --- /dev/null +++ b/crates/dever-core/src/diagnostic.rs @@ -0,0 +1,69 @@ +use std::fmt::Write; + +use crate::source::{SourceMap, Span}; + +#[derive(Clone, Debug, PartialEq)] +pub struct Label { + pub span: Span, + pub message: String, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Diagnostic { + pub code: &'static str, + pub message: String, + pub primary: Span, + pub related: Vec<Label>, +} + +impl Diagnostic { + pub(crate) fn error(code: &'static str, message: impl Into<String>, primary: Span) -> Self { + Self { + code, + message: message.into(), + primary, + related: Vec::new(), + } + } + + pub fn render(&self, sources: &SourceMap) -> String { + let mut rendered = String::new(); + render_label( + &mut rendered, + sources, + self.primary, + &format!( + "{}[{}]: {}", + if self.code.starts_with('W') { + "warning" + } else { + "error" + }, + self.code, + self.message + ), + ); + for related in &self.related { + render_label( + &mut rendered, + sources, + related.span, + &format!("note: {}", related.message), + ); + } + rendered + } +} + +fn render_label(output: &mut String, sources: &SourceMap, span: Span, message: &str) { + let source = sources.get(span.source); + let (line, column) = source.position(span.start); + let line_text = source.line(line); + writeln!( + output, + "{}:{line}:{column}: {message}", + source.path().display() + ) + .expect("string formatting"); + writeln!(output, " | {line_text}").expect("string formatting"); +} diff --git a/crates/dever-core/src/format.rs b/crates/dever-core/src/format.rs new file mode 100644 index 0000000..8cc6e5e --- /dev/null +++ b/crates/dever-core/src/format.rs @@ -0,0 +1,820 @@ +//! Syntax-only formatting. Source token positions attach comments; the AST owns layout. + +use crate::diagnostic::Diagnostic; +use crate::source::SourceFile; +use crate::syntax::*; +use crate::token::{Kind, Token}; + +mod document; +mod markdown; + +use document::{Document as Doc, join}; + +/// Formats syntactically valid source without requiring package or type resolution. +pub fn format(source: &SourceFile) -> Result<String, Vec<Diagnostic>> { + let parsed = crate::parse_source(source)?; + if let Some(blocks) = &parsed.blocks { + return Ok(markdown::format( + source, + &parsed.package, + &parsed.tokens, + blocks, + )); + } + Ok(format_fragment( + source, + parsed.tokens, + &parsed.package.declarations, + )) +} + +fn format_fragment( + source: &SourceFile, + tokens: Vec<Token>, + declarations: &[Declaration], +) -> String { + let mut annotated: Vec<SourceToken> = Vec::new(); + let mut leading = Vec::new(); + for token in tokens { + match token.kind { + Kind::Newline => {} + Kind::Comment(text) => { + let comment = text.trim_end().to_owned(); + if let Some(previous) = annotated.last_mut().filter(|previous| { + source.position(previous.end).0 == source.position(token.span.start).0 + }) { + previous.trailing.push(comment); + } else { + leading.push(comment); + } + } + kind => annotated.push(SourceToken { + kind, + end: token.span.end, + leading: std::mem::take(&mut leading), + trailing: Vec::new(), + }), + } + } + let mut formatter = Formatter { + tokens: annotated.into_iter().peekable(), + }; + formatter.fragment(declarations).render() +} + +struct SourceToken { + kind: Kind, + end: usize, + leading: Vec<String>, + trailing: Vec<String>, +} + +struct Formatter { + tokens: std::iter::Peekable<std::vec::IntoIter<SourceToken>>, +} + +impl Formatter { + fn token(&mut self, spelling: impl Into<String>) -> Doc { + let token = self + .tokens + .next() + .expect("AST follows parsed source tokens"); + let mut parts = Vec::new(); + for comment in token.leading { + parts.extend([Doc::Line, Doc::Text(comment), Doc::Line]); + } + parts.push(Doc::Text(spelling.into())); + for comment in token.trailing { + parts.extend([Doc::space(), Doc::Text(comment), Doc::Line]); + } + Doc::sequence(parts) + } + + fn at(&mut self, kind: Kind) -> bool { + self.tokens.peek().is_some_and(|token| token.kind == kind) + } + + fn path(&mut self, path: &Path) -> Doc { + let mut parts = Vec::new(); + for (index, name) in path.iter().enumerate() { + if index != 0 { + parts.push(self.token(".")); + } + parts.push(self.token(&name.text)); + } + Doc::sequence(parts) + } + + fn fragment(&mut self, declarations: &[Declaration]) -> Doc { + let mut parts = Vec::new(); + for declaration in declarations { + if !parts.is_empty() { + parts.push(Doc::BlankLine); + } + parts.push(match declaration { + Declaration::Api(binding) => { + let mut parts = Vec::new(); + if binding.anonymous { + parts.extend([self.token("public"), Doc::space()]); + } + parts.extend(vec![ + self.token(binding.kind.keyword()), + Doc::space(), + self.token(&binding.action.text), + Doc::space(), + self.token("="), + Doc::space(), + self.path(&binding.target), + ]); + Doc::sequence(parts) + } + Declaration::Rest(rest) => { + let mut parts = vec![self.token("rest")]; + if let Some(model) = &rest.model { + parts.extend([Doc::space(), self.path(model)]); + } + Doc::sequence(parts) + } + Declaration::External(external) => { + let mut entries = external + .libs + .iter() + .map(|lib| { + Doc::sequence(vec![ + self.token("lib"), + Doc::space(), + self.token(quoted_text(&lib.text)), + ]) + }) + .collect::<Vec<_>>(); + entries.extend(external.capabilities.iter().map(|capability| { + Doc::sequence(vec![ + self.token("allow"), + Doc::space(), + self.token(&capability.text), + ]) + })); + Doc::sequence(vec![ + self.token("external"), + Doc::space(), + self.token(external.ecosystem.keyword()), + Doc::space(), + self.token(quoted_text(&external.entry)), + Doc::space(), + self.lines("{", "}", &entries, |_, entry| entry.clone()), + ]) + } + Declaration::Type(declaration) => self.type_declaration(declaration), + Declaration::Function(function) => self.function(function), + Declaration::Schedule(schedule) => Doc::sequence(vec![ + self.token("schedule"), + Doc::space(), + self.token(&schedule.target.text), + Doc::space(), + self.token("="), + Doc::space(), + self.token(quoted_text(&schedule.cron)), + ]), + Declaration::Database(binding) => Doc::sequence(vec![ + self.token("database"), + Doc::space(), + self.token(&binding.name.text), + ]), + Declaration::ModelIndex(index) => self.model_index(index), + Declaration::Relation(relation) => Doc::sequence(vec![ + self.token("relation"), + Doc::space(), + self.token(&relation.name.text), + Doc::space(), + self.token("="), + Doc::space(), + self.path(&relation.field), + ]), + Declaration::Seed(seed) => self.seed(seed), + Declaration::Migration(migration) => self.migration(migration), + Declaration::ModelSql(sql) => self.model_sql(sql), + }); + } + // End carries standalone comments after the last declaration, including comment-only tails. + parts.push(Doc::Line); + parts.push(self.token("")); + debug_assert!(self.tokens.next().is_none()); + Doc::sequence(parts) + } + + fn type_declaration(&mut self, declaration: &TypeDeclaration) -> Doc { + let mut prefix = Vec::new(); + if declaration.public { + prefix.extend([self.token("public"), Doc::space()]); + } + if declaration.setting { + prefix.extend([self.token("setting"), Doc::space()]); + } else { + if declaration.global { + prefix.extend([self.token("global"), Doc::space()]); + } + prefix.extend([ + self.token("type"), + Doc::space(), + self.token(&declaration.name.text), + Doc::space(), + ]); + } + let body = match &declaration.shape { + TypeShape::Record(fields) => self.lines("{", "}", fields, Self::field), + TypeShape::Choice(variants) => self.lines("{", "}", variants, |this, variant| { + let mut parts = Vec::new(); + if variant.error { + parts.extend([this.token("error"), Doc::space()]); + } + let name = this.token(&variant.name.text); + let payload = if this.at(Kind::LeftParen) { + this.positional("(", ")", &variant.payload, Self::field) + } else { + Doc::sequence(Vec::new()) + }; + parts.extend([name, payload]); + if let Some(label) = &variant.label { + parts.extend([ + Doc::space(), + this.token("="), + Doc::space(), + this.token(quoted_text(label)), + ]); + } + Doc::sequence(parts) + }), + }; + Doc::sequence(vec![Doc::sequence(prefix), body]) + } + + fn field(&mut self, field: &Field) -> Doc { + let mut parts = Vec::new(); + if field.private { + parts.extend([self.token("private"), Doc::space()]); + } + if field.storage.owner.is_some() { + parts.extend([self.token("owner"), Doc::space()]); + } + parts.extend([ + self.token(&field.name.text), + self.token(":"), + Doc::space(), + self.type_ref(&field.ty), + ]); + parts.push(self.bounds(&field.bounds)); + while matches!( + self.tokens.peek(), + Some(token) + if token.end <= field.span.end + && matches!(&token.kind, Kind::Name(name) + if matches!(name.as_str(), "generated" | "default" | "index" | "unique" | "from" | "create" | "replace" | "search")) + ) { + let Kind::Name(modifier) = self + .tokens + .peek() + .map(|token| token.kind.clone()) + .expect("field modifier token") + else { + unreachable!() + }; + parts.extend([Doc::space(), self.token(&modifier)]); + match modifier.as_str() { + "default" => parts.extend([ + Doc::space(), + self.expression(field.storage.default.as_ref().expect("parsed default")), + ]), + "from" => parts.extend([ + Doc::space(), + self.token( + &field + .storage + .from + .as_ref() + .expect("parsed rename source") + .text, + ), + ]), + "create" => parts.extend([ + Doc::space(), + self.token("="), + Doc::space(), + self.expression( + field + .storage + .create + .as_ref() + .expect("parsed create binding"), + ), + ]), + "replace" => parts.extend([ + Doc::space(), + self.token("="), + Doc::space(), + self.expression( + field + .storage + .replace + .as_ref() + .expect("parsed replace binding"), + ), + ]), + "search" => parts.extend([ + Doc::space(), + self.token("="), + Doc::space(), + self.expression( + field + .storage + .search + .as_ref() + .expect("parsed search binding"), + ), + ]), + _ => {} + } + } + if let Some(owner) = &field.storage.owner { + parts.extend([ + Doc::space(), + self.token("="), + Doc::space(), + self.expression(owner), + ]); + } + Doc::sequence(parts) + } + + fn type_ref(&mut self, ty: &TypeRef) -> Doc { + let mut parts = vec![self.path(&ty.name)]; + // A following '<' can also begin a numeric pattern bound outside this type span. + if self + .tokens + .peek() + .is_some_and(|token| token.kind == Kind::Less && token.end <= ty.span.end) + { + parts.push(self.positional("<", ">", &ty.arguments, Self::type_ref)); + } + if !ty.parameters.is_empty() { + parts.push( + self.positional("(", ")", &ty.parameters, |this, parameter| { + this.number(¶meter.value) + }), + ); + } + if ty.nullable { + parts.push(self.token("?")); + } + Doc::sequence(parts) + } + + fn function(&mut self, function: &FunctionClause) -> Doc { + let mut parts = Vec::new(); + if function.public { + parts.extend([self.token("public"), Doc::space()]); + } + if function.kind == FunctionKind::Transaction { + parts.extend([self.token("transaction"), Doc::space()]); + } + if matches!(function.kind, FunctionKind::Job { .. }) { + parts.extend([self.token("job"), Doc::space()]); + } + for (index, segment) in function.name.text.split('.').enumerate() { + if index != 0 { + parts.push(self.token(".")); + } + parts.push(self.token(segment)); + } + parts.extend([ + self.positional("(", ")", &function.inputs, Self::input), + Doc::space(), + self.positional("(", ")", &function.outputs, Self::field), + ]); + // Consume source order before arranging contract documents canonically; + if let FunctionKind::Job { + attempts, + timeout_ms, + } = function.kind + { + for (name, value) in [("retry", attempts), ("timeout", timeout_ms)] { + parts.extend([ + Doc::space(), + self.token(name), + self.token("("), + self.token(value.to_string()), + self.token(")"), + ]); + } + } + // each marker keeps its attached comments when the order changes. + let mut pure = None; + let mut recovery = None; + while self.at(Kind::Name("pure".into())) || self.at(Kind::Name("recover".into())) { + if self.at(Kind::Name("pure".into())) { + pure = Some(self.token("pure")); + } else { + recovery = Some(Doc::sequence(vec![ + self.token("recover"), + self.token("("), + self.token(quoted_text( + function + .recovery + .as_ref() + .expect("parsed recovery contract"), + )), + self.token(")"), + ])); + } + } + for contract in [pure, recovery].into_iter().flatten() { + parts.extend([Doc::space(), contract]); + } + if let Some(fails) = &function.fails { + parts.extend([ + Doc::space(), + self.token("fails"), + Doc::space(), + self.type_ref(fails), + ]); + } + if function.bodyless { + return Doc::sequence(parts); + } + parts.extend([ + Doc::space(), + self.lines("{", "}", &function.body, Self::statement), + ]); + Doc::sequence(parts) + } + + fn model_index(&mut self, index: &ModelIndex) -> Doc { + Doc::sequence(vec![ + self.token(if index.unique { "unique" } else { "index" }), + self.positional("(", ")", &index.fields, |this, field| { + this.token(&field.text) + }), + ]) + } + + fn seed(&mut self, seed: &Seed) -> Doc { + Doc::sequence(vec![ + self.token("seed"), + Doc::space(), + self.lines("{", "}", &seed.rows, |this, row| { + this.lines("{", "}", row, |this, field| { + Doc::sequence(vec![ + this.token(&field.name.text), + Doc::space(), + this.token("="), + Doc::space(), + this.expression(&field.value), + ]) + }) + }), + ]) + } + + fn migration(&mut self, migration: &Migration) -> Doc { + Doc::sequence(vec![ + self.token("migrate"), + Doc::space(), + self.token(&migration.name.text), + Doc::space(), + self.lines( + "{", + "}", + &migration.operations, + |this, operation| match operation { + MigrationOperation::Drop(field) => Doc::sequence(vec![ + this.token("drop"), + Doc::space(), + this.token(&field.text), + ]), + MigrationOperation::Sql(sql) => this.migration_sql(sql), + }, + ), + ]) + } + + fn migration_sql(&mut self, sql: &MigrationSql) -> Doc { + let phase = match sql.phase { + MigrationPhase::Before => "before", + MigrationPhase::After => "after", + }; + Doc::sequence(vec![ + self.token(phase), + Doc::space(), + self.lines("{", "}", &[(), (), ()], |this, _| { + let Kind::Name(field) = this.tokens.peek().expect("SQL field token").kind.clone() + else { + unreachable!("parsed migration SQL field") + }; + let prefix = vec![ + this.token(&field), + Doc::space(), + this.token("="), + Doc::space(), + ]; + let value = match field.as_str() { + "sqlite" => this.token(quoted_text(&sql.sqlite.value)), + "postgres" => this.token(quoted_text(&sql.postgres.value)), + "parameters" => this.positional("[", "]", &sql.parameters, Self::expression), + _ => unreachable!("parsed migration SQL field"), + }; + Doc::sequence(vec![Doc::sequence(prefix), value]) + }), + ]) + } + + fn model_sql(&mut self, sql: &ModelSql) -> Doc { + Doc::sequence(vec![ + self.token("sql"), + Doc::space(), + self.token(&sql.name.text), + self.positional("(", ")", &sql.inputs, Self::field), + Doc::space(), + self.positional("(", ")", &sql.outputs, Self::field), + Doc::space(), + self.lines( + "{", + "}", + &[(&sql.sqlite, "sqlite"), (&sql.postgres, "postgres")], + |this, (text, dialect)| { + Doc::sequence(vec![ + this.token(*dialect), + Doc::space(), + this.token("="), + Doc::space(), + this.token(quoted_text(&text.value)), + ]) + }, + ), + ]) + } + + fn input(&mut self, input: &Input) -> Doc { + let mut parts = vec![self.token(&input.name.text), self.token(":"), Doc::space()]; + parts.push(match &input.kind { + InputKind::Value(pattern) => self.pattern(pattern), + InputKind::Handler(signature) => { + let parts = vec![ + self.token("handler"), + self.positional("(", ")", &signature.inputs, Self::field), + Doc::space(), + self.positional("(", ")", &signature.outputs, Self::field), + ]; + Doc::sequence(parts) + } + }); + Doc::sequence(parts) + } + + fn pattern(&mut self, pattern: &Pattern) -> Doc { + match pattern { + Pattern::Typed { ty, bounds } => { + Doc::sequence(vec![self.type_ref(ty), self.bounds(bounds)]) + } + Pattern::Variant { + name, + arguments, + bindings, + .. + } => Doc::sequence(vec![ + self.path(name), + if arguments.is_empty() { + Doc::sequence(Vec::new()) + } else { + self.positional("<", ">", arguments, Self::type_ref) + }, + self.positional("(", ")", bindings, |this, name| this.token(&name.text)), + ]), + Pattern::Literal { value, .. } => self.literal(value), + Pattern::Other(_) => self.token("other"), + } + } + + fn bounds(&mut self, bounds: &[Bound]) -> Doc { + let mut parts = Vec::new(); + for (index, bound) in bounds.iter().enumerate() { + if index != 0 { + parts.extend([Doc::space(), self.token("and")]); + } + parts.extend([ + Doc::space(), + self.token(operator_text(bound.comparison)), + Doc::space(), + self.number(&bound.number), + ]); + } + Doc::sequence(parts) + } + + fn statement(&mut self, statement: &Statement) -> Doc { + match &statement.kind { + StatementKind::Assign { target, value } => Doc::sequence(vec![ + self.path(target), + Doc::space(), + self.token("="), + Doc::space(), + self.expression(value), + ]), + StatementKind::Call(call) => self.expression(call), + } + } + + fn expression(&mut self, expression: &Expression) -> Doc { + match &expression.kind { + ExpressionKind::Literal(literal) => self.literal(literal), + ExpressionKind::Name(path) => self.path(path), + // Keeping explicit groups preserves associativity and gives comments a stable home. + ExpressionKind::Group(value) => { + self.positional("(", ")", &[value.as_ref()], |this, value| { + this.expression(value) + }) + } + ExpressionKind::Field { value, name } => Doc::sequence(vec![ + self.expression(value), + self.token("."), + self.token(&name.text), + ]), + ExpressionKind::Call { + function, + arguments, + } => Doc::sequence(vec![ + self.expression(function), + self.positional("(", ")", arguments, Self::expression), + ]), + ExpressionKind::Fail(error) => Doc::sequence(vec![ + self.token("fail"), + self.positional("(", ")", &[error.as_ref()], |this, value| { + this.expression(value) + }), + ]), + ExpressionKind::CaptureResult(call) => Doc::sequence(vec![ + self.token("result"), + self.positional("(", ")", &[call.as_ref()], |this, value| { + this.expression(value) + }), + ]), + ExpressionKind::Contextual(ContextualExpression::Channel { element, capacity }) => { + Doc::sequence(vec![ + self.token("channel"), + self.token("("), + self.type_ref(element), + self.token(","), + Doc::Break(" "), + self.expression(capacity), + self.token(")"), + ]) + .grouped() + } + ExpressionKind::Record { name, fields } => Doc::sequence(vec![ + self.path(name), + Doc::space(), + self.lines("{", "}", fields, |this, field| { + Doc::sequence(vec![ + this.token(&field.name.text), + Doc::space(), + this.token("="), + Doc::space(), + this.expression(&field.value), + ]) + }), + ]), + ExpressionKind::List(values) => self.positional("[", "]", values, Self::expression), + ExpressionKind::Map(entries) => self.lines("{", "}", entries, |this, entry| { + Doc::sequence(vec![ + this.expression(&entry.key), + Doc::space(), + this.token("="), + Doc::space(), + this.expression(&entry.value), + ]) + }), + ExpressionKind::Unary { operator, value } => { + let (spelling, space) = match operator { + UnaryOperator::Negate => ("-", ""), + UnaryOperator::Not => ("not", " "), + }; + Doc::sequence(vec![ + self.token(spelling), + Doc::Text(space.into()), + self.expression(value), + ]) + } + ExpressionKind::Binary { + left, + operator, + right, + } => Doc::sequence(vec![ + self.expression(left), + Doc::space(), + self.token(operator_text(*operator)), + Doc::space(), + self.expression(right), + ]), + } + } + + fn literal(&mut self, literal: &Literal) -> Doc { + match literal { + Literal::Bool(value) => self.token(if *value { "true" } else { "false" }), + Literal::Null => self.token("null"), + Literal::Number(number) => self.number(number), + Literal::Text(text) => self.token(quoted_text(text)), + } + } + + fn number(&mut self, spelling: &str) -> Doc { + if let Some(magnitude) = spelling.strip_prefix('-') { + Doc::sequence(vec![self.token("-"), self.token(magnitude)]) + } else { + self.token(spelling) + } + } + + fn positional<T>( + &mut self, + open: &str, + close: &str, + entries: &[T], + format: impl Fn(&mut Self, &T) -> Doc, + ) -> Doc { + let opening = self.token(open); + let mut members = Vec::new(); + for (index, entry) in entries.iter().enumerate() { + if index != 0 { + members.extend([self.token(","), Doc::Break(" ")]); + } + members.push(format(self, entry)); + } + Doc::sequence(vec![ + opening, + Doc::sequence(vec![Doc::Break(""), Doc::sequence(members)]).indented(), + Doc::Break(""), + self.token(close), + ]) + .grouped() + } + + fn lines<T>( + &mut self, + open: &str, + close: &str, + entries: &[T], + format: impl Fn(&mut Self, &T) -> Doc, + ) -> Doc { + let opening = self.token(open); + let members = entries.iter().map(|entry| format(self, entry)).collect(); + let closing = self.token(close); + if entries.is_empty() { + return Doc::sequence(vec![opening, closing]); + } + Doc::sequence(vec![ + opening, + Doc::sequence(vec![Doc::Line, join(members, Doc::Line)]).indented(), + Doc::Line, + closing, + ]) + } +} + +fn quoted_text(text: &str) -> String { + let mut quoted = String::from("\""); + for character in text.chars() { + match character { + '"' => quoted.push_str("\\\""), + '\\' => quoted.push_str("\\\\"), + '\n' => quoted.push_str("\\n"), + '\r' => quoted.push_str("\\r"), + '\t' => quoted.push_str("\\t"), + control if control.is_control() => { + use std::fmt::Write; + write!(quoted, "\\u{{{:x}}}", u32::from(control)) + .expect("writing a Text literal to a String"); + } + other => quoted.push(other), + } + } + quoted.push('"'); + quoted +} + +fn operator_text(operator: BinaryOperator) -> &'static str { + match operator { + BinaryOperator::Add => "+", + BinaryOperator::Subtract => "-", + BinaryOperator::Multiply => "*", + BinaryOperator::Divide => "/", + BinaryOperator::IntegerDivide => "//", + BinaryOperator::Remainder => "%", + BinaryOperator::Equal => "==", + BinaryOperator::NotEqual => "!=", + BinaryOperator::Less => "<", + BinaryOperator::LessEqual => "<=", + BinaryOperator::Greater => ">", + BinaryOperator::GreaterEqual => ">=", + BinaryOperator::And => "and", + BinaryOperator::Or => "or", + } +} diff --git a/crates/dever-core/src/format/document.rs b/crates/dever-core/src/format/document.rs new file mode 100644 index 0000000..1e4b130 --- /dev/null +++ b/crates/dever-core/src/format/document.rs @@ -0,0 +1,126 @@ +#[derive(Clone)] +pub(super) enum Document { + Text(String), + Line, + BlankLine, + Break(&'static str), + Sequence(Vec<Document>), + Indented(Box<Document>), + Group(Box<Document>), +} + +impl Document { + pub(super) fn sequence(parts: Vec<Self>) -> Self { + Self::Sequence(parts) + } + + pub(super) fn space() -> Self { + Self::Text(" ".into()) + } + + pub(super) fn indented(self) -> Self { + Self::Indented(Box::new(self)) + } + + pub(super) fn grouped(self) -> Self { + Self::Group(Box::new(self)) + } + + fn flat_width(&self) -> Option<usize> { + match self { + Self::Text(text) => Some(text.chars().count()), + Self::Break(text) => Some(text.len()), + Self::Line | Self::BlankLine => None, + Self::Sequence(parts) => parts + .iter() + .try_fold(0, |width, part| Some(width + part.flat_width()?)), + Self::Indented(inner) | Self::Group(inner) => inner.flat_width(), + } + } + + pub(super) fn render(&self) -> String { + let mut writer = Writer { + output: String::new(), + column: 0, + }; + writer.document(self, 0, false); + writer.output.truncate(writer.output.trim_end().len()); + writer.output.push('\n'); + writer.output + } +} + +pub(super) fn join(parts: Vec<Document>, separator: Document) -> Document { + let mut joined = Vec::new(); + for (index, part) in parts.into_iter().enumerate() { + if index != 0 { + joined.push(separator.clone()); + } + joined.push(part); + } + Document::sequence(joined) +} + +struct Writer { + output: String, + column: usize, +} + +impl Writer { + fn document(&mut self, document: &Document, indent: usize, flat: bool) { + match document { + Document::Text(text) => self.text(text, indent), + Document::Line => self.newline(), + Document::BlankLine => { + self.newline(); + if !self.output.is_empty() && !self.output.ends_with("\n\n") { + self.output.push('\n'); + } + } + Document::Break(text) => { + if flat { + self.text(text, indent); + } else { + self.newline(); + } + } + Document::Sequence(parts) => { + for part in parts { + self.document(part, indent, flat); + } + } + Document::Indented(inner) => self.document(inner, indent + 2, flat), + Document::Group(inner) => { + let column = if self.column == 0 { + indent + } else { + self.column + }; + let fits = inner.flat_width().is_some_and(|width| column + width <= 88); + self.document(inner, indent, flat || fits); + } + } + } + + fn text(&mut self, text: &str, indent: usize) { + if text.is_empty() || (self.column == 0 && text == " ") { + return; + } + if self.column == 0 { + self.output.extend(std::iter::repeat_n(' ', indent)); + self.column = indent; + } + self.output.push_str(text); + self.column += text.chars().count(); + } + + fn newline(&mut self) { + while self.output.ends_with(' ') { + self.output.pop(); + } + if !self.output.is_empty() && !self.output.ends_with('\n') { + self.output.push('\n'); + } + self.column = 0; + } +} diff --git a/crates/dever-core/src/format/markdown.rs b/crates/dever-core/src/format/markdown.rs new file mode 100644 index 0000000..92cb239 --- /dev/null +++ b/crates/dever-core/src/format/markdown.rs @@ -0,0 +1,48 @@ +use crate::markdown::{ProgramBlock, declaration_span}; +use crate::source::SourceFile; +use crate::syntax::Package; +use crate::token::{Kind, Token}; + +pub(super) fn format( + source: &SourceFile, + package: &Package, + tokens: &[Token], + blocks: &[ProgramBlock], +) -> String { + let mut output = String::new(); + let mut copied = 0; + for block in blocks { + output.push_str(&source.text()[copied..block.content.start]); + let first = tokens.partition_point(|token| token.span.start < block.content.start); + let last = tokens.partition_point(|token| token.span.start < block.content.end); + let mut block_tokens = tokens[first..last].to_vec(); + if block_tokens + .iter() + .any(|token| !matches!(token.kind, Kind::Newline)) + { + block_tokens.push(Token { + kind: Kind::End, + span: source.span(block.content.end, block.content.end), + }); + let declarations = &package.declarations; + let first = declarations.partition_point(|declaration| { + declaration_span(declaration).start < block.content.start + }); + let last = declarations.partition_point(|declaration| { + declaration_span(declaration).start < block.content.end + }); + let formatted = + super::format_fragment(source, block_tokens, &declarations[first..last]); + for line in formatted.split_terminator('\n') { + if !line.is_empty() { + output.push_str(&block.indentation); + } + output.push_str(line); + output.push_str(block.newline); + } + } + copied = block.content.end; + } + output.push_str(&source.text()[copied..]); + output +} diff --git a/crates/dever-core/src/hir.rs b/crates/dever-core/src/hir.rs new file mode 100644 index 0000000..648bc12 --- /dev/null +++ b/crates/dever-core/src/hir.rs @@ -0,0 +1,908 @@ +use std::collections::BTreeMap; + +use dever_runtime::number::DecimalValue; + +use crate::diagnostic::Diagnostic; +use crate::intrinsic::Intrinsic; +use crate::source::Span; +use crate::syntax::{BinaryOperator, UnaryOperator}; +use crate::types::{Definition, Field, Parameter, Type}; + +/// Only semantic checking can construct a program for native emission. +#[derive(Clone, Debug)] +pub struct Program { + pub(crate) types: Vec<Definition>, + pub(crate) functions: Vec<Function>, + pub(crate) warnings: Vec<Diagnostic>, + pub(crate) packages: Vec<PackageContract>, + pub(crate) effects: Vec<std::collections::BTreeSet<&'static str>>, + pub(crate) recoveries: Vec<std::collections::BTreeSet<String>>, + pub(crate) suspension_handlers: Vec<std::collections::BTreeSet<usize>>, + pub(crate) failures: Vec<std::collections::BTreeSet<Failure>>, + pub(crate) failure_handlers: Vec<std::collections::BTreeSet<usize>>, + pub(crate) database_effects: Vec<std::collections::BTreeSet<crate::model::DatabaseOwner>>, + pub(crate) models: Vec<crate::model::ModelSchema>, + pub(crate) jobs: Vec<Job>, + pub(crate) api_routes: Vec<ApiRoute>, + pub(crate) api_commands: Vec<ApiCommand>, + pub(crate) api_rest: Vec<ApiRest>, + pub(crate) permissions: Vec<Permission>, + pub(crate) tenant_components: Vec<String>, + pub(crate) tests: Vec<TestCase>, + pub(crate) adapters: Vec<Adapter>, + pub(crate) auth: Vec<AuthHook>, +} + +#[derive(Clone, Debug)] +pub(crate) struct AuthHook { + pub provider: String, + pub function: usize, + pub claims: usize, + pub identity: usize, +} + +#[derive(Clone, Debug)] +pub(crate) struct Job { + pub function: usize, + pub connection: crate::model::ConnectionSelector, + pub payload: Option<crate::wire::Schema>, + pub attempts: u32, + pub timeout_ms: u32, + pub schedule: Option<String>, + pub components: Vec<String>, +} + +#[derive(Clone, Debug)] +pub struct TestCase { + pub(crate) name: String, + pub(crate) function: usize, + pub(crate) span: Span, + pub(crate) database_connections: Vec<String>, + pub(crate) port_bindings: std::collections::BTreeMap<usize, usize>, +} + +#[derive(Clone, Debug)] +pub(crate) struct Adapter { + pub owner: usize, + pub name: String, + pub port_owner: usize, + pub identity: String, + pub operations: std::collections::BTreeMap<usize, usize>, + pub fake: bool, + pub setting: Option<usize>, + pub implementation: AdapterImplementation, +} + +#[derive(Clone, Debug)] +pub(crate) enum AdapterImplementation { + Dever, + External(ExternalAdapter), +} + +#[derive(Clone, Debug)] +pub(crate) struct ExternalAdapter { + pub ecosystem: crate::syntax::ExternalEcosystem, + pub entry: String, + pub libs: Vec<String>, + pub capabilities: std::collections::BTreeSet<&'static str>, + pub schema: String, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ExternalWorkerContract { + pub port: String, + pub adapter: String, + pub ecosystem: String, + pub entry: String, + pub schema: String, + pub capabilities: Vec<String>, + pub operations: Vec<String>, + pub libs: Vec<String>, + pub setting: Option<serde_json::Value>, + pub inputs: std::collections::BTreeMap<String, serde_json::Value>, + pub outputs: std::collections::BTreeMap<String, serde_json::Value>, + pub errors: std::collections::BTreeMap<String, serde_json::Value>, +} + +impl ExternalWorkerContract { + pub fn sdk_manifest(&self) -> serde_json::Value { + serde_json::json!({ + "port": self.port, "adapter": self.adapter, "ecosystem": self.ecosystem, + "entry": self.entry, "schema": self.schema, "libs": self.libs, + "capabilities": self.capabilities, "operations": self.operations, + "setting": self.setting, "inputs": self.inputs, "outputs": self.outputs, + "errors": self.errors, + }) + } +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct ExternalOperation { + pub adapter: usize, + pub contract: usize, +} + +#[derive(Clone, Debug)] +pub(crate) struct PortOperation { + pub identity: String, + pub operation: String, + pub input_names: Vec<String>, + pub failures: std::collections::BTreeSet<Failure>, + pub implementations: Vec<usize>, +} + +impl TestCase { + pub fn name(&self) -> &str { + &self.name + } + + pub fn uses_database(&self) -> bool { + !self.database_connections.is_empty() + } + + pub fn database_connections(&self) -> &[String] { + &self.database_connections + } +} + +#[derive(Clone, Debug)] +pub(crate) struct ApiRoute { + pub multipart: bool, + pub method: &'static str, + pub path: String, + pub function: usize, + pub inputs: Vec<String>, + pub directory: Vec<String>, + pub component: String, + pub domain: String, + pub action: String, + pub anonymous: bool, + pub auth: Option<usize>, + pub permission: Option<usize>, + pub components: Vec<String>, + pub span: Span, +} + +#[derive(Clone, Debug)] +pub(crate) struct ApiCommand { + pub name: String, + pub function: usize, + pub inputs: Vec<String>, + pub components: Vec<String>, +} + +#[derive(Clone, Debug)] +pub(crate) struct ApiRest { + pub path: String, + pub model: usize, + pub span: Span, + pub directory: Vec<String>, + pub component: String, + pub domain: String, + pub auth: Option<usize>, + pub permissions: RestPermissions, + pub components: Vec<String>, + pub contract: crate::model::RestContract, +} + +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct RestPermissions { + pub read: Option<usize>, + pub create: Option<usize>, + pub replace: Option<usize>, + pub delete: Option<usize>, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct Permission { + pub key: String, + pub component: String, + pub domain: String, + pub site: String, + pub action: String, + pub method: &'static str, +} + +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub(crate) struct Failure { + pub ty: usize, + pub variant: usize, +} + +#[derive(Clone, Debug)] +pub(crate) struct PackageContract { + pub bundled: bool, + pub domain: Option<(String, String)>, + pub role: Option<crate::source::SourceRole>, +} + +#[derive(Clone, Debug)] +pub(crate) struct Function { + pub name: String, + pub owner: usize, + pub public: bool, + pub parameters: Vec<Parameter>, + pub outputs: Vec<Field>, + pub clauses: Vec<Clause>, + pub span: Span, + pub suspends: bool, + pub pure: bool, + pub recovery: Option<String>, + pub kind: crate::syntax::FunctionKind, + pub port: Option<PortOperation>, + pub implementation: bool, + pub setting: Option<usize>, + pub external: Option<ExternalOperation>, +} + +#[derive(Clone, Debug)] +pub(crate) struct Clause { + pub patterns: Vec<Domain>, + pub bindings: Vec<Binding>, + pub locals: Vec<Type>, + pub outputs: Vec<usize>, + pub body: Vec<Statement>, + pub terminates: bool, +} + +#[derive(Clone, Debug)] +pub(crate) struct Binding { + pub slot: usize, + pub input: usize, + pub projection: Projection, +} + +#[derive(Clone, Debug)] +pub(crate) enum Projection { + Whole, + NonNull, + Payload { variant: usize, field: usize }, +} + +#[derive(Clone, Debug, PartialEq)] +pub(crate) enum Atom { + Null, + Any, + Bool(bool), + Variant(usize), + Int(i64, i64), + Decimal(DecimalValue, DecimalValue), + Literal(Constant), + Remainder(Vec<Constant>), +} + +pub(crate) type Domain = Vec<Atom>; + +#[derive(Clone, Debug)] +pub(crate) enum Statement { + Assign { + slot: usize, + fields: Vec<usize>, + value: Expression, + }, + Call(Expression), +} + +#[derive(Clone, Debug)] +pub(crate) struct Expression { + pub kind: ExpressionKind, + pub ty: Type, + pub span: Span, +} + +#[derive(Clone, Debug)] +pub(crate) enum ExpressionKind { + Constant(Constant), + Local(usize), + Field { + value: Box<Expression>, + index: usize, + }, + ChoiceOptions { + choice: usize, + }, + ModelOperation { + model: usize, + operation: Box<ModelOperation>, + }, + JobEnqueue { + target: usize, + arguments: Vec<Expression>, + scheduled: bool, + }, + Call { + target: CallTarget, + arguments: Vec<CallArgument>, + }, + Fail(Box<Expression>), + CaptureResult { + target: CallTarget, + arguments: Vec<CallArgument>, + choice: usize, + success: usize, + }, + RunCall { + group: Option<Box<Expression>>, + target: CallTarget, + arguments: Vec<CallArgument>, + }, + AwaitTask(Box<Expression>), + StopTask(Box<Expression>), + Group(Box<Expression>), + AwaitGroup(Box<Expression>), + StopGroup(Box<Expression>), + ParallelCall { + target: CallTarget, + arguments: Vec<CallArgument>, + }, + BlockingCall { + target: CallTarget, + arguments: Vec<CallArgument>, + }, + Channel { + element: Type, + capacity: Box<Expression>, + }, + ChannelSend { + channel: Box<Expression>, + value: Box<Expression>, + }, + ChannelReceive(Box<Expression>), + ChannelClose(Box<Expression>), + Intrinsic { + operation: Intrinsic, + handler: Option<HandlerTarget>, + arguments: Vec<Expression>, + }, + Collection { + operation: CollectionOp, + handler: Option<HandlerTarget>, + sequence: Option<SequenceKind>, + arguments: Vec<Expression>, + }, + Record { + fields: Vec<(usize, Expression)>, + }, + Variant { + variant: usize, + arguments: Vec<Expression>, + }, + List(Vec<Expression>), + Map(Vec<(Expression, Expression)>), + Unary { + operator: UnaryOperator, + value: Box<Expression>, + }, + Binary { + left: Box<Expression>, + operator: BinaryOperator, + right: Box<Expression>, + }, + Promote(Box<Expression>), + Some(Box<Expression>), +} + +#[derive(Clone, Debug)] +pub(crate) enum ModelOperation { + Create { + values: Vec<(usize, Expression)>, + }, + CreateMany { + rows: Vec<Vec<(usize, Expression)>>, + }, + Get { + id: Box<Expression>, + }, + First { + query: QueryPlan, + }, + List { + query: QueryPlan, + }, + Cursor { + query: QueryPlan, + }, + Count { + query: QueryPlan, + }, + Exists { + query: QueryPlan, + }, + Stream { + query: QueryPlan, + }, + Update { + query: QueryPlan, + values: Vec<(usize, Expression)>, + }, + Delete { + query: QueryPlan, + }, + Upsert { + key: Vec<(usize, Expression)>, + create: Vec<(usize, Expression)>, + update: Vec<(usize, Expression)>, + }, + Sql { + operation: usize, + arguments: Vec<Expression>, + read_only: bool, + }, +} + +#[derive(Clone, Debug)] +pub(crate) struct QueryPlan { + pub condition: Option<ModelCondition>, + pub order: Vec<ModelOrder>, + /// Static indexes into ModelSchema::relations. + pub relations: Vec<usize>, + pub after: Option<Expression>, + pub page: Expression, + pub size: Expression, +} + +#[derive(Clone, Debug)] +pub(crate) enum ModelCondition { + Compare { + field: usize, + operator: BinaryOperator, + value: Expression, + }, + In { + field: usize, + values: Vec<Expression>, + }, + Between { + field: usize, + lower: Expression, + upper: Expression, + }, + Text { + field: usize, + operation: TextCondition, + value: Expression, + }, + Not(Box<ModelCondition>), + And(Box<ModelCondition>, Box<ModelCondition>), + Or(Box<ModelCondition>, Box<ModelCondition>), +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum TextCondition { + Contains, + StartsWith, + EndsWith, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct ModelOrder { + pub field: usize, + pub descending: bool, +} + +impl ModelOperation { + /// SQL outside the checker's proven projection subset remains a possible write. + pub(crate) fn writes_database(&self) -> bool { + matches!( + self, + Self::Create { .. } + | Self::CreateMany { .. } + | Self::Update { .. } + | Self::Delete { .. } + | Self::Upsert { .. } + | Self::Sql { + read_only: false, + .. + } + ) + } + + pub(crate) fn expressions(&self) -> Vec<&Expression> { + let mut values = Vec::new(); + self.visit(&mut |value| values.push(value)); + values + } + + pub(crate) fn visit<'a>(&'a self, apply: &mut impl FnMut(&'a Expression)) { + match self { + Self::Create { values } => visit_write(values, apply), + Self::CreateMany { rows } => { + for values in rows { + visit_write(values, apply); + } + } + Self::Get { id } => apply(id), + Self::First { query } + | Self::List { query } + | Self::Cursor { query } + | Self::Count { query } + | Self::Exists { query } + | Self::Stream { query } + | Self::Delete { query } => query.visit(apply), + Self::Update { query, values } => { + query.visit(apply); + visit_write(values, apply); + } + Self::Upsert { + key, + create, + update, + } => { + visit_write(key, apply); + visit_write(create, apply); + visit_write(update, apply); + } + Self::Sql { arguments, .. } => arguments.iter().for_each(apply), + } + } + + pub(crate) fn visit_mut(&mut self, apply: &mut impl FnMut(&mut Expression)) { + match self { + Self::Create { values } => visit_write_mut(values, apply), + Self::CreateMany { rows } => { + for values in rows { + visit_write_mut(values, apply); + } + } + Self::Get { id } => apply(id), + Self::First { query } + | Self::List { query } + | Self::Cursor { query } + | Self::Count { query } + | Self::Exists { query } + | Self::Stream { query } + | Self::Delete { query } => query.visit_mut(apply), + Self::Update { query, values } => { + query.visit_mut(apply); + visit_write_mut(values, apply); + } + Self::Upsert { + key, + create, + update, + } => { + visit_write_mut(key, apply); + visit_write_mut(create, apply); + visit_write_mut(update, apply); + } + Self::Sql { arguments, .. } => arguments.iter_mut().for_each(apply), + } + } +} + +impl QueryPlan { + fn visit<'a>(&'a self, apply: &mut impl FnMut(&'a Expression)) { + if let Some(condition) = &self.condition { + condition.visit(apply); + } + if let Some(after) = &self.after { + apply(after); + } + apply(&self.page); + apply(&self.size); + } + + fn visit_mut(&mut self, apply: &mut impl FnMut(&mut Expression)) { + if let Some(condition) = &mut self.condition { + condition.visit_mut(apply); + } + if let Some(after) = &mut self.after { + apply(after); + } + apply(&mut self.page); + apply(&mut self.size); + } +} + +impl ModelCondition { + fn visit<'a>(&'a self, apply: &mut impl FnMut(&'a Expression)) { + match self { + Self::Compare { value, .. } => apply(value), + Self::In { values, .. } => values.iter().for_each(apply), + Self::Between { lower, upper, .. } => { + apply(lower); + apply(upper); + } + Self::Text { value, .. } => apply(value), + Self::Not(value) => value.visit(apply), + Self::And(left, right) | Self::Or(left, right) => { + left.visit(apply); + right.visit(apply); + } + } + } + + fn visit_mut(&mut self, apply: &mut impl FnMut(&mut Expression)) { + match self { + Self::Compare { value, .. } => apply(value), + Self::In { values, .. } => values.iter_mut().for_each(apply), + Self::Between { lower, upper, .. } => { + apply(lower); + apply(upper); + } + Self::Text { value, .. } => apply(value), + Self::Not(value) => value.visit_mut(apply), + Self::And(left, right) | Self::Or(left, right) => { + left.visit_mut(apply); + right.visit_mut(apply); + } + } + } +} + +fn visit_write<'a>(values: &'a [(usize, Expression)], apply: &mut impl FnMut(&'a Expression)) { + for (_, value) in values { + apply(value); + } +} + +fn visit_write_mut(values: &mut [(usize, Expression)], apply: &mut impl FnMut(&mut Expression)) { + for (_, value) in values { + apply(value); + } +} + +impl ExpressionKind { + pub(crate) fn callback(&self) -> Option<HandlerTarget> { + match self { + Self::Collection { handler, .. } | Self::Intrinsic { handler, .. } => *handler, + _ => None, + } + } + + pub(crate) fn static_call(&self) -> Option<(CallTarget, &[CallArgument])> { + match self { + Self::Call { target, arguments } + | Self::CaptureResult { + target, arguments, .. + } + | Self::RunCall { + target, arguments, .. + } + | Self::ParallelCall { target, arguments } + | Self::BlockingCall { target, arguments } => Some((*target, arguments)), + _ => None, + } + } +} + +#[derive(Clone, Debug)] +pub(crate) enum CallArgument { + Value(Expression), + Handler(HandlerTarget), +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum HandlerTarget { + Function(usize), + Parameter(usize), +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum CallTarget { + Function(usize), + Handler(HandlerTarget), +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum CollectionOp { + Each, + ParallelEach, + Reduce, + ReduceUntil, + Filter, + Find, + Sum, + Append, + First, + Get, + Put, + Remove, + Entries, + Length, + Close, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum SequenceKind { + List, + Bytes, + Stream, + AsyncStream, + RowStream, +} + +#[derive(Clone, Debug, PartialEq)] +pub(crate) enum Constant { + Null, + Bool(bool), + Int(i64), + Decimal(DecimalValue), + Float(f64), + Text(String), +} + +impl Program { + pub fn warnings(&self) -> &[Diagnostic] { + &self.warnings + } + + pub fn tests(&self) -> &[TestCase] { + &self.tests + } + + /// Returns whether the checked program contains a production external + /// Adapter. The CLI uses this only for lock/preparation validation; the + /// external Port contract remains owned by the compiler HIR. + pub fn has_external_adapters(&self) -> bool { + self.functions + .iter() + .any(|function| function.external.is_some()) + } + + pub fn external_worker_entries(&self) -> Vec<String> { + let mut entries = self + .adapters + .iter() + .filter_map(|adapter| match &adapter.implementation { + AdapterImplementation::External(external) + if !adapter.fake + && external.ecosystem != crate::syntax::ExternalEcosystem::Command => + { + Some(external.entry.clone()) + } + _ => None, + }) + .collect::<Vec<_>>(); + entries.sort(); + entries.dedup(); + entries + } + + pub fn external_worker_contracts(&self) -> Vec<ExternalWorkerContract> { + self.external_contracts(false) + } + + pub fn external_command_contracts(&self) -> Vec<ExternalWorkerContract> { + self.external_contracts(true) + } + + fn external_contracts(&self, command: bool) -> Vec<ExternalWorkerContract> { + let mut contracts = self + .adapters + .iter() + .filter_map(|adapter| { + let AdapterImplementation::External(external) = &adapter.implementation else { + return None; + }; + if adapter.fake + || (external.ecosystem == crate::syntax::ExternalEcosystem::Command) != command + { + return None; + } + Some(self.external_contract(adapter, external)) + }) + .collect::<Vec<_>>(); + contracts.sort_by(|left, right| { + left.port + .cmp(&right.port) + .then(left.adapter.cmp(&right.adapter)) + }); + contracts + } + + fn external_contract( + &self, + adapter: &Adapter, + external: &ExternalAdapter, + ) -> ExternalWorkerContract { + let policy = crate::wire::Policy::external(external.ecosystem); + let record = |fields: &[crate::types::Field]| { + let fields = fields + .iter() + .map(|field| { + let schema = crate::wire::Schema::from_field(field, &self.types, policy) + .expect("checked external wire field"); + (field.name.clone(), schema.worker_type()) + }) + .collect::<BTreeMap<_, _>>(); + serde_json::json!({"type": "record", "fields": fields}) + }; + let mut inputs = BTreeMap::new(); + let mut outputs = BTreeMap::new(); + let mut errors = BTreeMap::new(); + for contract in adapter.operations.keys() { + let function = &self.functions[*contract]; + let port = function.port.as_ref().expect("checked Port operation"); + let fields = function + .parameters + .iter() + .enumerate() + .map(|(index, parameter)| crate::types::Field { + name: port.input_names[index].clone(), + ty: parameter + .value_type() + .expect("Port value parameter") + .clone(), + private: false, + bounds: Vec::new(), + }) + .collect::<Vec<_>>(); + inputs.insert(port.operation.clone(), record(&fields)); + outputs.insert(port.operation.clone(), record(&function.outputs)); + for failure in &port.failures { + let crate::types::Shape::Choice(variants) = &self.types[failure.ty].shape else { + unreachable!() + }; + let variant = &variants[failure.variant]; + let identity = format!("{}.{}", self.types[failure.ty].name, variant.name); + errors.insert(identity, record(&variant.fields)); + } + } + ExternalWorkerContract { + port: adapter.identity.clone(), + adapter: adapter.name.clone(), + ecosystem: external.ecosystem.keyword().into(), + entry: external.entry.clone(), + schema: external.schema.clone(), + capabilities: external + .capabilities + .iter() + .map(|value| (*value).to_owned()) + .collect(), + operations: inputs.keys().cloned().collect(), + libs: external.libs.clone(), + setting: adapter.setting.map(|ty| { + crate::wire::Schema::build( + &crate::types::Type::Named(ty), + &self.types, + crate::wire::Policy::SettingInput, + ) + .expect("checked external setting") + .worker_type() + }), + inputs, + outputs, + errors, + } + } + + /// Exact Lib requests declared by external Adapters. The compiler keeps + /// them opaque; the CLI owns ecosystem parsing and resolution. + pub fn external_lib_requests(&self) -> Vec<String> { + let mut requests = self + .adapters + .iter() + .filter_map(|adapter| match &adapter.implementation { + AdapterImplementation::External(external) => Some(external.libs.clone()), + AdapterImplementation::Dever => None, + }) + .flatten() + .collect::<Vec<_>>(); + requests.sort(); + requests.dedup(); + requests + } + + pub(crate) fn entry(&self, name: &str) -> Result<usize, String> { + let (id, function) = self + .functions + .iter() + .enumerate() + .find(|(_, function)| function.name == name && function.parameters.is_empty()) + .ok_or_else(|| format!("entry '{name}' must name a zero-input function"))?; + if !function.public && name != "main.main" { + return Err(format!("entry '{name}' must be public")); + } + if function + .outputs + .iter() + .any(|field| !field.ty.observable(&self.types)) + { + return Err(format!( + "entry '{name}' cannot render Secret or private Model data" + )); + } + Ok(id) + } +} diff --git a/crates/dever-core/src/intrinsic.rs b/crates/dever-core/src/intrinsic.rs new file mode 100644 index 0000000..8b6693f --- /dev/null +++ b/crates/dever-core/src/intrinsic.rs @@ -0,0 +1,354 @@ +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum Intrinsic { + TestAssert, + TestAssertEq, + TestSecret, + JobServe, + TestJobDrain, + TestClockAdvance, + CryptoToken, + CryptoPasswordHash, + CryptoPasswordVerify, + CryptoSha256, + CryptoHmacSha256, + CryptoConstantTimeEq, + TextTrim, + TextLower, + TextUpper, + TextCodepoint, + TextFromCodepoint, + TextAt, + TextSlice, + TextIndexOf, + TextContains, + TextStartsWith, + TextEndsWith, + TextSplit, + TextReplace, + IntParse, + IntToText, + DecimalParse, + DecimalToText, + DecimalFromInt, + FloatParse, + FloatToText, + ProcessArguments, + TimeUnixMillis, + TimeNow, + TimeParseDateTime, + TimeFormatDateTime, + TimeParseDate, + TimeFormatDate, + TimeParseTime, + TimeFormatTime, + TimeDuration, + TimeAdd, + TimeSubtract, + TimeDifference, + TimeMonotonicNanos, + TimeSleep, + TaskSleep, + TaskTicks, + TaskTimeout, + TaskRace, + StreamOf, + TlsSystem, + TlsClient, + TlsServer, + HttpClientNew, + HttpRequest, + HttpOpen, + HttpOpenStream, + HttpCloseClient, + HttpServeTls, + HttpServeLiveTls, + WsOpen, + DecimalRound, + FloatFromInt, + FloatSqrt, + FloatSin, + FloatCos, + FloatLog, + FloatPow, + FloatIsNan, + FloatIsFinite, + FloatIsInfinite, + StdoutWrite, + LogDebug, + LogInfo, + LogWarn, + LogError, + IdFromText, + IdToText, + UuidParse, + UuidToText, + BytesFromText, + BytesToText, + BytesFromInts, + BytesLength, + BytesAt, + BytesSlice, + BytesConcat, + FileOpen, + FileCreate, + FileRead, + FileWrite, + FileClose, + FileChunks, + TcpConnect, + TcpConnectTimeout, + TcpListen, + TcpAccept, + TcpPort, + TcpRead, + TcpWrite, + TcpTimeout, + TcpClose, + TcpCloseListener, + TcpChunks, + TcpConnections, + HttpServe, + ApiServe, + UploadFilename, + UploadContentType, + UploadSize, + UploadClose, + UploadStore, + ApiRequestId, + ApiMethod, + ApiPath, + ApiClientAddress, + ApiHeader, + ApiCookie, + ApiSecretCookie, + ApiResponseHeader, + ApiResponseCookie, + ApiResponseSecretCookie, + AuthIssue, + AuthIssueCookie, + AuthClearCookie, + AuthOwnsUser, + AuthId, + AuthSession, + AuthUserId, + AuthTenantId, + AuthPermissions, + AuthSaveRole, + AuthGrantRole, + AuthRevokeRole, + AuthDisableRole, + SiteKey, + HttpSend, + HttpServeLive, + HttpRespond, + HttpStart, + HttpWrite, + HttpFinish, + SseStart, + SseSend, + WsAccept, + WsConnect, + WsSend, + WsReceive, + WsMessages, + WsClose, +} + +impl Intrinsic { + pub fn is_async(self) -> bool { + matches!( + self, + Self::UploadStore + | Self::HttpClientNew + | Self::HttpRequest + | Self::HttpOpen + | Self::HttpOpenStream + | Self::HttpCloseClient + | Self::HttpServeTls + | Self::HttpServeLiveTls + | Self::WsOpen + | Self::TaskTicks + | Self::TaskSleep + | Self::TaskTimeout + | Self::TaskRace + | Self::TcpConnect + | Self::TcpConnectTimeout + | Self::TcpListen + | Self::TcpAccept + | Self::TcpRead + | Self::TcpWrite + | Self::HttpServe + | Self::ApiServe + | Self::JobServe + | Self::TestJobDrain + | Self::HttpSend + | Self::HttpServeLive + | Self::HttpRespond + | Self::HttpStart + | Self::HttpWrite + | Self::HttpFinish + | Self::SseStart + | Self::SseSend + | Self::AuthPermissions + | Self::AuthSaveRole + | Self::AuthGrantRole + | Self::AuthRevokeRole + | Self::AuthDisableRole + | Self::WsAccept + | Self::WsConnect + | Self::WsSend + | Self::WsReceive + | Self::WsClose + ) + } + + pub fn from_name(name: &str) -> Option<Self> { + match name { + "dever.test.drain_jobs" => return Some(Self::TestJobDrain), + "dever.test.advance_clock" => return Some(Self::TestClockAdvance), + "dever.auth.owns_user" => return Some(Self::AuthOwnsUser), + _ => {} + } + Some(match name.strip_prefix("dever.system.")? { + "job_serve" => Self::JobServe, + "text_trim" => Self::TextTrim, + "text_lower" => Self::TextLower, + "text_upper" => Self::TextUpper, + "text_codepoint" => Self::TextCodepoint, + "text_from_codepoint" => Self::TextFromCodepoint, + "text_at" => Self::TextAt, + "text_slice" => Self::TextSlice, + "text_index_of" => Self::TextIndexOf, + "text_contains" => Self::TextContains, + "text_starts_with" => Self::TextStartsWith, + "text_ends_with" => Self::TextEndsWith, + "text_split" => Self::TextSplit, + "text_replace" => Self::TextReplace, + "int_parse" => Self::IntParse, + "int_to_text" => Self::IntToText, + "decimal_parse" => Self::DecimalParse, + "decimal_to_text" => Self::DecimalToText, + "decimal_from_int" => Self::DecimalFromInt, + "float_parse" => Self::FloatParse, + "float_to_text" => Self::FloatToText, + "process_arguments" => Self::ProcessArguments, + "crypto_token" => Self::CryptoToken, + "crypto_password_hash" => Self::CryptoPasswordHash, + "crypto_password_verify" => Self::CryptoPasswordVerify, + "crypto_sha256" => Self::CryptoSha256, + "crypto_hmac_sha256" => Self::CryptoHmacSha256, + "crypto_constant_time_eq" => Self::CryptoConstantTimeEq, + "time_unix_millis" => Self::TimeUnixMillis, + "time_now" => Self::TimeNow, + "time_parse_datetime" => Self::TimeParseDateTime, + "time_format_datetime" => Self::TimeFormatDateTime, + "time_parse_date" => Self::TimeParseDate, + "time_format_date" => Self::TimeFormatDate, + "time_parse_time" => Self::TimeParseTime, + "time_format_time" => Self::TimeFormatTime, + "time_duration" => Self::TimeDuration, + "time_add" => Self::TimeAdd, + "time_subtract" => Self::TimeSubtract, + "time_difference" => Self::TimeDifference, + "time_monotonic_nanos" => Self::TimeMonotonicNanos, + "time_sleep" => Self::TimeSleep, + "task_sleep" => Self::TaskSleep, + "task_ticks" => Self::TaskTicks, + "tls_system" => Self::TlsSystem, + "tls_client" => Self::TlsClient, + "tls_server" => Self::TlsServer, + "http_client" => Self::HttpClientNew, + "http_request" => Self::HttpRequest, + "http_open" => Self::HttpOpen, + "http_open_stream" => Self::HttpOpenStream, + "http_close_client" => Self::HttpCloseClient, + "http_serve_tls" => Self::HttpServeTls, + "http_serve_live_tls" => Self::HttpServeLiveTls, + "ws_open" => Self::WsOpen, + "decimal_round" => Self::DecimalRound, + "float_from_int" => Self::FloatFromInt, + "float_sqrt" => Self::FloatSqrt, + "float_sin" => Self::FloatSin, + "float_cos" => Self::FloatCos, + "float_log" => Self::FloatLog, + "float_pow" => Self::FloatPow, + "float_is_nan" => Self::FloatIsNan, + "float_is_finite" => Self::FloatIsFinite, + "float_is_infinite" => Self::FloatIsInfinite, + "stdout_write" => Self::StdoutWrite, + "log_debug" => Self::LogDebug, + "log_info" => Self::LogInfo, + "log_warn" => Self::LogWarn, + "log_error" => Self::LogError, + "id_from_text" => Self::IdFromText, + "id_to_text" => Self::IdToText, + "uuid_parse" => Self::UuidParse, + "uuid_to_text" => Self::UuidToText, + "bytes_from_text" => Self::BytesFromText, + "bytes_to_text" => Self::BytesToText, + "bytes_from_ints" => Self::BytesFromInts, + "bytes_length" => Self::BytesLength, + "bytes_at" => Self::BytesAt, + "bytes_slice" => Self::BytesSlice, + "bytes_concat" => Self::BytesConcat, + "file_open" => Self::FileOpen, + "file_create" => Self::FileCreate, + "file_read" => Self::FileRead, + "file_write" => Self::FileWrite, + "file_close" => Self::FileClose, + "file_chunks" => Self::FileChunks, + "tcp_connect" => Self::TcpConnect, + "tcp_connect_timeout" => Self::TcpConnectTimeout, + "tcp_listen" => Self::TcpListen, + "tcp_accept" => Self::TcpAccept, + "tcp_port" => Self::TcpPort, + "tcp_read" => Self::TcpRead, + "tcp_write" => Self::TcpWrite, + "tcp_timeout" => Self::TcpTimeout, + "tcp_close" => Self::TcpClose, + "tcp_close_listener" => Self::TcpCloseListener, + "tcp_chunks" => Self::TcpChunks, + "tcp_connections" => Self::TcpConnections, + "http_serve" => Self::HttpServe, + "api_serve" => Self::ApiServe, + "api_request_id" => Self::ApiRequestId, + "api_method" => Self::ApiMethod, + "api_path" => Self::ApiPath, + "api_client_address" => Self::ApiClientAddress, + "api_header" => Self::ApiHeader, + "api_cookie" => Self::ApiCookie, + "api_secret_cookie" => Self::ApiSecretCookie, + "api_response_header" => Self::ApiResponseHeader, + "api_response_cookie" => Self::ApiResponseCookie, + "api_response_secret_cookie" => Self::ApiResponseSecretCookie, + "auth_issue" => Self::AuthIssue, + "auth_issue_cookie" => Self::AuthIssueCookie, + "auth_clear_cookie" => Self::AuthClearCookie, + "auth_id" => Self::AuthId, + "auth_session" => Self::AuthSession, + "auth_user_id" => Self::AuthUserId, + "auth_tenant_id" => Self::AuthTenantId, + "auth_permissions" => Self::AuthPermissions, + "auth_save_role" => Self::AuthSaveRole, + "auth_grant_role" => Self::AuthGrantRole, + "auth_revoke_role" => Self::AuthRevokeRole, + "auth_disable_role" => Self::AuthDisableRole, + "site_key" => Self::SiteKey, + "http_send" => Self::HttpSend, + "http_serve_live" => Self::HttpServeLive, + "http_respond" => Self::HttpRespond, + "http_start" => Self::HttpStart, + "http_write" => Self::HttpWrite, + "http_finish" => Self::HttpFinish, + "sse_start" => Self::SseStart, + "sse_send" => Self::SseSend, + "ws_accept" => Self::WsAccept, + "ws_connect" => Self::WsConnect, + "ws_send" => Self::WsSend, + "ws_receive" => Self::WsReceive, + "ws_messages" => Self::WsMessages, + "ws_close" => Self::WsClose, + _ => return None, + }) + } +} diff --git a/crates/dever-core/src/lexer.rs b/crates/dever-core/src/lexer.rs new file mode 100644 index 0000000..dd2b6c6 --- /dev/null +++ b/crates/dever-core/src/lexer.rs @@ -0,0 +1,276 @@ +use crate::diagnostic::Diagnostic; +use crate::source::SourceFile; +use crate::token::{Kind, Token}; +use std::ops::Range; + +pub(crate) fn lex(source: &SourceFile, range: Range<usize>) -> Result<Vec<Token>, Vec<Diagnostic>> { + let mut lexer = Lexer { + source, + offset: range.start, + end: range.end, + tokens: Vec::new(), + errors: Vec::new(), + }; + while let Some(ch) = lexer.peek() { + let start = lexer.offset; + match ch { + ' ' | '\t' => { + lexer.advance(); + } + '\n' | '\r' => { + lexer.advance(); + if ch == '\r' && lexer.peek() == Some('\n') { + lexer.advance(); + } + lexer.push(start, Kind::Newline); + } + '#' => lexer.comment(), + '"' => lexer.text(), + '0'..='9' => lexer.number(), + 'a'..='z' | 'A'..='Z' | '_' => lexer.name(), + _ => lexer.symbol(), + } + } + lexer.push(lexer.offset, Kind::End); + if lexer.errors.is_empty() { + Ok(lexer.tokens) + } else { + Err(lexer.errors) + } +} + +struct Lexer<'a> { + source: &'a SourceFile, + offset: usize, + end: usize, + tokens: Vec<Token>, + errors: Vec<Diagnostic>, +} + +impl Lexer<'_> { + fn peek(&self) -> Option<char> { + self.source.text()[self.offset..self.end].chars().next() + } + + fn advance(&mut self) -> Option<char> { + let ch = self.peek()?; + self.offset += ch.len_utf8(); + Some(ch) + } + + fn push(&mut self, start: usize, kind: Kind) { + self.tokens.push(Token { + kind, + span: self.source.span(start, self.offset), + }); + } + + fn error(&mut self, start: usize, code: &'static str, message: impl Into<String>) { + self.errors.push(Diagnostic::error( + code, + message, + self.source.span(start, self.offset), + )); + } + + fn comment(&mut self) { + let start = self.offset; + while self.peek().is_some_and(|ch| ch != '\n' && ch != '\r') { + self.advance(); + } + self.push( + start, + Kind::Comment(self.source.text()[start..self.offset].into()), + ); + } + + fn name(&mut self) { + let start = self.offset; + while self + .peek() + .is_some_and(|ch| ch.is_ascii_alphanumeric() || ch == '_') + { + self.advance(); + } + let spelling = &self.source.text()[start..self.offset]; + let kind = match spelling { + "package" => Kind::Package, + "exposes" => Kind::Exposes, + "public" => Kind::Public, + "type" => Kind::Type, + "true" => Kind::True, + "false" => Kind::False, + "null" => Kind::Null, + "other" => Kind::Other, + "and" => Kind::And, + "or" => Kind::Or, + "not" => Kind::Not, + "if" | "else" | "switch" | "match" | "when" | "by" | "for" | "while" | "break" + | "continue" | "iterate" | "recur" | "repeat" | "return" | "function" | "throw" + | "catch" | "try" | "import" | "use" | "class" | "interface" | "unsafe" | "macro" => { + self.error( + start, + "L003", + format!("'{spelling}' is not supported in Core 0.1"), + ); + return; + } + _ => Kind::Name(spelling.into()), + }; + self.push(start, kind); + } + + fn digits(&mut self) { + while self.peek().is_some_and(|ch| ch.is_ascii_digit()) { + self.advance(); + } + } + + fn number(&mut self) { + let start = self.offset; + self.digits(); + if self.peek() == Some('.') + && self.source.text()[self.offset + 1..self.end] + .starts_with(|ch: char| ch.is_ascii_digit()) + { + self.advance(); + self.digits(); + } + if matches!(self.peek(), Some('e' | 'E')) { + self.advance(); + if matches!(self.peek(), Some('+' | '-')) { + self.advance(); + } + let exponent_start = self.offset; + self.digits(); + if exponent_start == self.offset { + self.error(start, "L004", "expected digits in numeric exponent"); + } + } + if self + .peek() + .is_some_and(|ch| ch.is_alphanumeric() || ch == '_') + { + while self + .peek() + .is_some_and(|ch| ch.is_alphanumeric() || ch == '_') + { + self.advance(); + } + self.error(start, "L004", "numeric literals cannot have a suffix"); + } + self.push( + start, + Kind::Number(self.source.text()[start..self.offset].into()), + ); + } + + fn text(&mut self) { + let start = self.offset; + self.advance(); + let mut value = String::new(); + loop { + let escape_start = self.offset; + match self.advance() { + Some('"') => { + self.push(start, Kind::Text(value)); + return; + } + Some('\n' | '\r') | None => { + self.error( + start, + "L005", + "unterminated Text literal; use an escape for a newline", + ); + return; + } + Some('\\') => match self.advance() { + Some('\\') => value.push('\\'), + Some('"') => value.push('"'), + Some('n') => value.push('\n'), + Some('r') => value.push('\r'), + Some('t') => value.push('\t'), + Some('u') => match self.unicode_escape() { + Some(ch) => value.push(ch), + None => self.error( + escape_start, + "L006", + "expected a Unicode scalar escape such as \\u{41}", + ), + }, + _ => self.error(escape_start, "L006", "invalid Text escape"), + }, + Some(ch) => value.push(ch), + } + } + } + + fn unicode_escape(&mut self) -> Option<char> { + if self.peek() != Some('{') { + return None; + } + self.advance(); + let start = self.offset; + while self.peek().is_some_and(|ch| ch.is_ascii_hexdigit()) { + self.advance(); + } + let digits = &self.source.text()[start..self.offset]; + if self.peek() != Some('}') { + return None; + } + self.advance(); + if digits.is_empty() || digits.len() > 6 { + return None; + } + u32::from_str_radix(digits, 16) + .ok() + .and_then(char::from_u32) + } + + fn symbol(&mut self) { + let start = self.offset; + let ch = self.advance().expect("symbol starts at a character"); + let pair = match (ch, self.peek()) { + ('/', Some('/')) => Some(Kind::IntegerDivide), + ('=', Some('=')) => Some(Kind::Equal), + ('!', Some('=')) => Some(Kind::NotEqual), + ('<', Some('=')) => Some(Kind::LessEqual), + ('>', Some('=')) => Some(Kind::GreaterEqual), + _ => None, + }; + if let Some(kind) = pair { + self.advance(); + self.push(start, kind); + return; + } + let kind = match ch { + '(' => Kind::LeftParen, + ')' => Kind::RightParen, + '{' => Kind::LeftBrace, + '}' => Kind::RightBrace, + '[' => Kind::LeftBracket, + ']' => Kind::RightBracket, + ':' => Kind::Colon, + ',' => Kind::Comma, + '.' => Kind::Dot, + '?' => Kind::Question, + '=' => Kind::Assign, + '+' => Kind::Plus, + '-' => Kind::Minus, + '*' => Kind::Star, + '/' => Kind::Slash, + '%' => Kind::Percent, + '<' => Kind::Less, + '>' => Kind::Greater, + _ => { + self.error( + start, + "L001", + format!("unexpected character '{ch}'; identifiers must use ASCII"), + ); + return; + } + }; + self.push(start, kind); + } +} diff --git a/crates/dever-core/src/lib.rs b/crates/dever-core/src/lib.rs new file mode 100644 index 0000000..6e1de31 --- /dev/null +++ b/crates/dever-core/src/lib.rs @@ -0,0 +1,103 @@ +pub mod diagnostic; +pub mod format; +pub mod hir; +pub mod llvm; +pub mod native; +#[cfg(feature = "reference")] +pub mod reference; +pub mod source; +pub mod syntax; +pub mod wire; + +mod api; +mod capture; +mod check; +mod contracts; +mod intrinsic; +mod lexer; +mod markdown; +mod model; +mod parser; +mod specialize; +mod token; +mod types; + +pub use check::{check, check_with_bindings, check_with_settings}; + +use diagnostic::Diagnostic; +use source::SourceFile; +use syntax::Package; + +/// Parses source syntax only. Name resolution and type checking are separate phases. +pub fn parse(source: &SourceFile) -> Result<Package, Vec<Diagnostic>> { + if source::SourceMap::is_standard(source.id()) { + return source::parsed_standard(source.id()); + } + parse_uncached(source) +} + +fn parse_uncached(source: &SourceFile) -> Result<Package, Vec<Diagnostic>> { + parse_source(source) + .map(|parsed| parsed.package) + .map_err(|mut diagnostics| { + diagnostics.sort_by_key(|diagnostic| { + ( + diagnostic.primary.start, + diagnostic.primary.end, + diagnostic.code, + ) + }); + diagnostics + }) +} + +struct ParsedSource { + package: Package, + tokens: Vec<token::Token>, + blocks: Option<Vec<markdown::ProgramBlock>>, +} + +// Parsing and formatting share extraction and block-boundary validation. +fn parse_source(source: &SourceFile) -> Result<ParsedSource, Vec<Diagnostic>> { + let blocks = source + .is_markdown() + .then(|| markdown::blocks(source)) + .transpose()?; + let mut tokens = Vec::new(); + if let Some(blocks) = &blocks { + let mut errors = Vec::new(); + for block in blocks { + match lexer::lex(source, block.content.clone()) { + Ok(mut block_tokens) => { + block_tokens.pop(); // Each block's End is replaced by the package's End. + tokens.extend(block_tokens); + } + Err(block_errors) => errors.extend(block_errors), + } + } + if !errors.is_empty() { + return Err(errors); + } + let end = blocks + .last() + .expect("Markdown has program blocks") + .content + .end; + tokens.push(token::Token { + kind: token::Kind::End, + span: source.span(end, end), + }); + } else { + tokens = lexer::lex(source, 0..source.text().len())?; + } + let package = parser::parse(tokens.clone(), source)?; + if let Some(blocks) = &blocks { + markdown::validate_declarations(&package, blocks)?; + markdown::validate_structure(source, &package, blocks)?; + } + Ok(ParsedSource { + package, + tokens, + blocks, + }) +} diff --git a/crates/dever-core/src/llvm.rs b/crates/dever-core/src/llvm.rs new file mode 100644 index 0000000..e5dbe3d --- /dev/null +++ b/crates/dever-core/src/llvm.rs @@ -0,0 +1,1517 @@ +//! Typed LLVM lowering for checked function kernels and switched-resume coroutines. +//! +//! Scalar kernels remain freestanding; managed values call the versioned +//! internal runtime ABI. Applications have a separate generated entry from +//! ordinary kernels. Unsupported reachable HIR is rejected with its +//! source location before any IR is returned. + +use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::fmt::Write; + +use crate::hir::{ + Atom, CallArgument, CallTarget, CollectionOp, Constant, Expression, ExpressionKind, Program, + Projection, SequenceKind, Statement, +}; +use crate::intrinsic::Intrinsic; +use crate::source::{SourceMap, Span}; +use crate::specialize::{self, Specialization}; +use crate::syntax::{BinaryOperator, FunctionKind, UnaryOperator}; +use crate::types::{DefinitionKind, Parameter, Shape, Type, output_type}; + +pub use application::{ + BinaryResource, ResourceModule, emit_application, emit_application_with_resources, + emit_executable, emit_executable_with_resources, +}; +use ownership::{TypeOwner, collect_owned_types, contains_owned}; +pub use testing::{emit_test_executable, emit_test_suite}; + +#[repr(i32)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum FaultCode { + Overflow = 1, + DivisionByZero = 2, + RuntimeAbi = 3, + Business = 4, +} + +/// Lower one checked, public, zero-input function into a directly callable +/// `i32 @dever_entry(ptr out, ptr fault)` kernel. `out` has the LLVM type +/// returned by `output_type`; the entry initializes both writable pointers. +/// Callers release transferred outputs and fault messages through the emitted +/// `dever_outputs_release` and `dever_fault_release` functions. +pub fn emit_kernel(program: &Program, sources: &SourceMap, entry: &str) -> Result<String, String> { + let root = program.entry(entry)?; + let application_span = program + .api_routes + .first() + .map(|route| route.span) + .or_else(|| program.api_rest.first().map(|route| route.span)) + .or_else(|| { + program + .api_commands + .first() + .map(|command| program.functions[command.function].span) + }) + .or_else(|| { + program + .jobs + .first() + .map(|job| program.functions[job.function].span) + }); + if let Some(span) = application_span { + return Err(located( + sources, + span, + "LLVM function kernels do not implement API, CMD, REST, or Job application entries", + )); + } + let root = Specialization { + function: root, + handlers: Vec::new(), + }; + let instances = specialize::reachable(&program.functions, [root.clone()]) + .map_err(|span| located(sources, span, "recursive specialized call"))?; + for instance in &instances { + validate_function(program, sources, instance, false)?; + } + let mut module = Module::new(program, sources, instances); + module.validate_async_boundaries(sources)?; + module.emit_types()?; + module.emit_functions()?; + module.emit_entry(&root); + Ok(module.finish()) +} + +fn located(sources: &SourceMap, span: Span, message: &str) -> String { + let source = sources.get(span.source); + let (line, column) = source.position(span.start); + format!("{}:{line}:{column}: {message}", source.path().display()) +} + +fn validate_function( + program: &Program, + sources: &SourceMap, + instance: &Specialization, + application: bool, +) -> Result<(), String> { + let function = &program.functions[instance.function]; + if !application { + let mut application_effect = function + .parameters + .iter() + .any(|parameter| parameter.value_type() == Some(&Type::Upload)) + .then_some(function.span); + crate::check::visit(function, |expression| { + if expression.ty == Type::Upload { + application_effect.get_or_insert(expression.span); + } + if let ExpressionKind::Intrinsic { operation, .. } = expression.kind + && api_context::supported(operation) + { + application_effect.get_or_insert(expression.span); + } + }); + if let Some(span) = application_effect { + return Err(located( + sources, + span, + "LLVM function kernel does not support application context", + )); + } + } + if (!matches!( + function.kind, + FunctionKind::Ordinary | FunctionKind::Transaction | FunctionKind::Job { .. } + ) || (!application && function.kind == FunctionKind::Transaction)) + || (!application && !specialize::database_effects(program, instance).is_empty()) + || (!application && function.port.is_some()) + || (!application && function.setting.is_some()) + || (!application && function.external.is_some()) + { + return Err(located( + sources, + function.span, + "LLVM function kernel requires an ordinary function without application effects", + )); + } + for clause in &function.clauses { + for statement in &clause.body { + let expression = match statement { + Statement::Assign { value, .. } | Statement::Call(value) => value, + }; + validate_expression(sources, expression)?; + } + } + Ok(()) +} + +fn validate_expression(sources: &SourceMap, expression: &Expression) -> Result<(), String> { + let mut unsupported = None; + crate::check::visit_expression(expression, |value| { + let supported = match &value.kind { + ExpressionKind::Intrinsic { operation, .. } => intrinsics::supported(*operation), + ExpressionKind::Collection { + operation, + sequence, + .. + } => collections::supported(*operation, *sequence), + ExpressionKind::Constant(_) + | ExpressionKind::Local(_) + | ExpressionKind::Field { .. } + | ExpressionKind::Some(_) + | ExpressionKind::Unary { .. } + | ExpressionKind::Promote(_) + | ExpressionKind::List(_) + | ExpressionKind::Map(_) + | ExpressionKind::Call { .. } + | ExpressionKind::CaptureResult { .. } + | ExpressionKind::Fail(_) + | ExpressionKind::RunCall { .. } + | ExpressionKind::ParallelCall { .. } + | ExpressionKind::BlockingCall { .. } + | ExpressionKind::AwaitTask(_) + | ExpressionKind::StopTask(_) + | ExpressionKind::Group(_) + | ExpressionKind::AwaitGroup(_) + | ExpressionKind::StopGroup(_) + | ExpressionKind::ChannelReceive(_) + | ExpressionKind::ChannelClose(_) + | ExpressionKind::Channel { .. } + | ExpressionKind::ChannelSend { .. } + | ExpressionKind::JobEnqueue { .. } + | ExpressionKind::Record { .. } + | ExpressionKind::Variant { .. } + | ExpressionKind::Binary { .. } + | ExpressionKind::ModelOperation { .. } => true, + _ => false, + }; + if !supported { + unsupported.get_or_insert(value.span); + } + }); + unsupported.map_or(Ok(()), |span| { + Err(located( + sources, + span, + "LLVM function kernel does not support this effect or expression", + )) + }) +} + +struct Module<'a> { + program: &'a Program, + sources: &'a SourceMap, + instances: BTreeSet<Specialization>, + names: BTreeMap<Specialization, String>, + declarations: String, + functions: String, + locations: Vec<Span>, + type_ids: BTreeSet<usize>, + value_types: Vec<Type>, + next_literal: usize, + read_events: BTreeSet<usize>, + parallel_functions: BTreeSet<Specialization>, + http_handlers: BTreeSet<Specialization>, + websocket_events: BTreeSet<usize>, + upload_decoders: BTreeSet<usize>, + application: Option<application::Application>, + wire_codecs: BTreeMap<String, String>, + database_errors: BTreeSet<(String, usize)>, + database_decoders: BTreeSet<usize>, + related_renderers: BTreeSet<usize>, + api_permission_types: BTreeSet<usize>, + rest_encoders: BTreeSet<(usize, bool)>, +} + +impl<'a> Module<'a> { + fn new( + program: &'a Program, + sources: &'a SourceMap, + instances: BTreeSet<Specialization>, + ) -> Self { + let names = instances + .iter() + .cloned() + .map(|instance| { + let suffix = instance + .handlers + .iter() + .map(usize::to_string) + .collect::<Vec<_>>() + .join("h"); + let name = format!("f{}h{suffix}", instance.function); + (instance, name) + }) + .collect(); + let mut type_ids = BTreeSet::new(); + let mut value_types = Vec::new(); + for instance in &instances { + let function = &program.functions[instance.function]; + for parameter in &function.parameters { + if let Parameter::Value(ty) = parameter { + collect_types(program, ty, &mut type_ids); + collect_owned_types(program, ty, &mut value_types); + } + } + for field in &function.outputs { + collect_types(program, &field.ty, &mut type_ids); + collect_owned_types(program, &field.ty, &mut value_types); + } + let output = output_type(&function.outputs); + collect_types(program, &output, &mut type_ids); + collect_owned_types(program, &output, &mut value_types); + collect_owned_types( + program, + &asynchronous::input_type(function), + &mut value_types, + ); + // A Port may declare failures that no implementation constructs. + // Its typed fault ABI still needs their layouts and owner callbacks. + for failure in specialize::failures(program, instance) { + let ty = Type::Named(failure.ty); + collect_types(program, &ty, &mut type_ids); + collect_owned_types(program, &ty, &mut value_types); + } + for clause in &function.clauses { + for ty in &clause.locals { + collect_types(program, ty, &mut type_ids); + collect_owned_types(program, ty, &mut value_types); + } + } + crate::check::visit(function, |expression| { + collect_types(program, &expression.ty, &mut type_ids); + collect_owned_types(program, &expression.ty, &mut value_types); + }); + } + Self { + program, + sources, + instances, + names, + declarations: String::new(), + functions: String::new(), + locations: Vec::new(), + type_ids, + value_types, + next_literal: 0, + read_events: BTreeSet::new(), + parallel_functions: BTreeSet::new(), + http_handlers: BTreeSet::new(), + websocket_events: BTreeSet::new(), + upload_decoders: BTreeSet::new(), + application: None, + wire_codecs: BTreeMap::new(), + database_errors: BTreeSet::new(), + database_decoders: BTreeSet::new(), + related_renderers: BTreeSet::new(), + api_permission_types: BTreeSet::new(), + rest_encoders: BTreeSet::new(), + } + } + + fn location(&mut self, span: Span) -> usize { + if let Some(index) = self.locations.iter().position(|item| *item == span) { + return index; + } + let index = self.locations.len(); + self.locations.push(span); + index + } + + fn ty(&self, ty: &Type) -> String { + match ty { + Type::Unit => "{}".into(), + Type::Bool => "i1".into(), + Type::Int | Type::DateTime | Type::Date | Type::Time | Type::Duration => "i64".into(), + Type::Float => "double".into(), + Type::Decimal => "{ i64, i64 }".into(), + Type::Text + | Type::Json + | Type::Id + | Type::Secret + | Type::Uuid + | Type::ClientTls + | Type::ServerTls + | Type::HttpClient + | Type::HttpReply + | Type::WebSocket + | Type::Upload + | Type::Bytes + | Type::List(_) + | Type::Map(_, _) + | Type::File + | Type::Stream(_) + | Type::AsyncStream(_) + | Type::RowStream(_) + | Type::Related(_) + | Type::Socket + | Type::Listener + | Type::Task(_) + | Type::Group + | Type::Channel(_) => "ptr".into(), + Type::Nullable(inner) => format!("{{ i1, {} }}", self.ty(inner)), + Type::Named(id) if self.program.types[*id].kind == DefinitionKind::ModelId => { + "i64".into() + } + Type::Named(id) => format!("%T{id}"), + Type::MapEntry(key, value) => format!("{{ {}, {} }}", self.ty(key), self.ty(value)), + Type::Outputs(fields) => format!( + "{{ {} }}", + fields + .iter() + .map(|field| self.ty(&field.ty)) + .collect::<Vec<_>>() + .join(", ") + ), + } + } + + fn bytes(&self, ty: &Type) -> usize { + match ty { + Type::Unit => 0, + Type::Bool => 1, + Type::Int + | Type::DateTime + | Type::Date + | Type::Time + | Type::Duration + | Type::Secret + | Type::Uuid + | Type::ClientTls + | Type::ServerTls + | Type::HttpClient + | Type::HttpReply + | Type::WebSocket + | Type::Upload + | Type::Float + | Type::Text + | Type::Json + | Type::Id + | Type::Bytes + | Type::List(_) + | Type::Map(_, _) + | Type::File + | Type::Stream(_) + | Type::AsyncStream(_) + | Type::RowStream(_) + | Type::Related(_) + | Type::Socket + | Type::Listener + | Type::Task(_) + | Type::Group + | Type::Channel(_) => 8, + Type::Decimal => 16, + Type::Nullable(inner) => align( + align(1, self.alignment(inner)) + self.bytes(inner), + self.alignment(inner), + ), + Type::Outputs(fields) => self.fields_bytes(fields), + Type::MapEntry(key, value) => self.pair_bytes(key, value), + Type::Named(id) if self.program.types[*id].kind == DefinitionKind::ModelId => 8, + Type::Named(id) => match &self.program.types[*id].shape { + Shape::Record(fields) => self.fields_bytes(fields), + Shape::Choice(variants) => { + 8 + variants + .iter() + .map(|variant| align(self.fields_bytes(&variant.fields), 8)) + .max() + .unwrap_or(0) + } + }, + } + } + + fn alignment(&self, ty: &Type) -> usize { + match ty { + Type::Unit | Type::Bool => 1, + Type::Int + | Type::DateTime + | Type::Date + | Type::Time + | Type::Duration + | Type::Secret + | Type::Uuid + | Type::ClientTls + | Type::ServerTls + | Type::HttpClient + | Type::HttpReply + | Type::WebSocket + | Type::Upload + | Type::Float + | Type::Decimal + | Type::Text + | Type::Json + | Type::Id + | Type::Bytes + | Type::List(_) + | Type::Map(_, _) + | Type::File + | Type::Stream(_) + | Type::AsyncStream(_) + | Type::RowStream(_) + | Type::Related(_) + | Type::Socket + | Type::Listener + | Type::Task(_) + | Type::Group + | Type::Channel(_) => 8, + Type::Nullable(inner) => self.alignment(inner), + Type::MapEntry(key, value) => self.alignment(key).max(self.alignment(value)), + Type::Outputs(fields) => fields + .iter() + .map(|field| self.alignment(&field.ty)) + .max() + .unwrap_or(1), + Type::Named(id) if self.program.types[*id].kind == DefinitionKind::ModelId => 8, + Type::Named(id) => match &self.program.types[*id].shape { + Shape::Record(fields) => fields + .iter() + .map(|field| self.alignment(&field.ty)) + .max() + .unwrap_or(1), + Shape::Choice(_) => 8, + }, + } + } + + fn fields_bytes(&self, fields: &[crate::types::Field]) -> usize { + let mut size = 0; + let mut max_align = 1; + for field in fields { + let alignment = self.alignment(&field.ty); + max_align = max_align.max(alignment); + size = align(size, alignment) + self.bytes(&field.ty); + } + align(size, max_align) + } + + fn pair_bytes(&self, first: &Type, second: &Type) -> usize { + let second_start = align(self.bytes(first), self.alignment(second)); + align( + second_start + self.bytes(second), + self.alignment(first).max(self.alignment(second)), + ) + } + + fn type_index(&self, ty: &Type) -> usize { + self.value_types + .iter() + .position(|known| known == ty) + .expect("reachable type collected before emission") + } + + fn uses_runtime(&self) -> bool { + self.instances + .iter() + .any(|instance| specialize::suspends(self.program, instance)) + || self + .value_types + .iter() + .any(|ty| matches!(ty, Type::Decimal) || contains_owned(self.program, ty)) + } + + fn emit_types(&mut self) -> Result<(), String> { + let capacity = self.instances.len() + 1; + let failure_words = self + .instances + .iter() + .flat_map(|instance| specialize::failures(self.program, instance)) + .map(|failure| align(self.bytes(&Type::Named(failure.ty)), 8) / 8) + .max() + .unwrap_or(0); + writeln!( + self.declarations, + "%dever.fault = type {{ i32, i32, i32, [{capacity} x i32], {{ ptr, i64 }}, i32, i32, [{failure_words} x i64], ptr }}" + ) + .unwrap(); + writeln!( + self.declarations, + "@dever_fault_capacity = constant i32 {capacity}" + ) + .unwrap(); + for id in self.type_ids.iter().copied() { + let definition = &self.program.types[id]; + if definition.kind == DefinitionKind::ModelId { + continue; + } + match &definition.shape { + Shape::Record(fields) => { + let body = fields + .iter() + .map(|field| self.ty(&field.ty)) + .collect::<Vec<_>>() + .join(", "); + writeln!(self.declarations, "%T{id} = type {{ {body} }}").unwrap(); + } + Shape::Choice(variants) => { + let words = variants + .iter() + .map(|variant| align(self.fields_bytes(&variant.fields), 8) / 8) + .max() + .unwrap_or(0); + writeln!( + self.declarations, + "%T{id} = type {{ i32, [{words} x i64] }}" + ) + .unwrap(); + for (variant, definition) in variants.iter().enumerate() { + let body = definition + .fields + .iter() + .map(|field| self.ty(&field.ty)) + .collect::<Vec<_>>() + .join(", "); + writeln!(self.declarations, "%T{id}V{variant} = type {{ {body} }}") + .unwrap(); + } + } + } + } + self.declarations + .push_str("declare { i64, i1 } @llvm.sadd.with.overflow.i64(i64, i64)\n"); + self.declarations + .push_str("declare { i64, i1 } @llvm.ssub.with.overflow.i64(i64, i64)\n"); + self.declarations + .push_str("declare { i64, i1 } @llvm.smul.with.overflow.i64(i64, i64)\n"); + if self.uses_runtime() { + self.declarations + .push_str("declare void @dever_rt_v1_buffer_free(ptr, i64)\n"); + for kind in [ + "text", + "bytes", + "list", + "map", + "file", + "stream", + "secret", + "uuid", + "client_tls", + "server_tls", + "http_client", + "http_reply", + "websocket", + "upload", + ] { + writeln!( + self.declarations, + "declare ptr @dever_rt_v1_{kind}_retain(ptr)\ndeclare void @dever_rt_v1_{kind}_release(ptr)" + ) + .unwrap(); + } + self.declarations.push_str(abi::DECLARATIONS); + self.declarations.push_str(system::DECLARATIONS); + self.declarations.push_str(http::DECLARATIONS); + } + self.declarations + .push_str("%dever.type = type { i64, i64, ptr, ptr, ptr, ptr }\n"); + let helpers = self + .value_types + .iter() + .enumerate() + .map(|(index, ty)| TypeOwner::new(self).emit(index, ty)) + .collect::<String>(); + self.declarations.push_str(&helpers); + for (index, ty) in self.value_types.clone().iter().enumerate() { + let equal = self.emit_type_equal(index, ty)?; + let hash_function = self.emit_type_hash(index, ty); + self.declarations.push_str(&equal); + self.declarations.push_str(&hash_function); + let hash = if ownership::hashable_key(self.program, ty) { + format!("@dever_hash_{index}") + } else { + "null".into() + }; + writeln!( + self.declarations, + "@dever_type_{index} = private constant %dever.type {{ i64 {}, i64 {}, ptr @dever_clone_{index}, ptr @dever_drop_{index}, ptr @dever_equal_{index}, ptr {hash} }}", + self.bytes(ty), self.alignment(ty), + ) + .unwrap(); + } + self.emit_async_types(); + Ok(()) + } + + fn emit_functions(&mut self) -> Result<(), String> { + for instance in self.instances.clone() { + if let Some(index) = self.application.as_ref().and_then(|app| { + app.job_handlers + .iter() + .position(|function| *function == instance.function) + }) { + self.emit_job_body(&instance, index)?; + continue; + } + if let Some(handler) = self + .application + .as_ref() + .and_then(|app| { + app.rest_handlers + .iter() + .find(|handler| handler.function == instance.function) + }) + .cloned() + { + self.emit_rest_body(&instance, &handler)?; + continue; + } + if let Some(index) = self.application.as_ref().and_then(|app| { + app.command_handlers + .iter() + .position(|function| *function == instance.function) + }) { + self.emit_command_body(&instance, index)?; + continue; + } + if let Some(index) = self.application.as_ref().and_then(|app| { + app.api_handlers + .iter() + .position(|function| *function == instance.function) + }) { + self.emit_api_body(&instance, index)?; + continue; + } + if self + .application + .as_ref() + .is_some_and(|app| app.root == instance.function) + { + if self.application.as_ref().unwrap().test.is_some() { + self.emit_test_body(&instance)?; + } else { + self.emit_application_body(&instance)?; + } + continue; + } + let emitter = FunctionEmitter::new(self, &instance); + let body = emitter.emit()?; + self.functions.push_str(&body); + } + Ok(()) + } + + fn emit_entry(&mut self, root: &Specialization) { + self.emit_named_entry(root, "dever_entry", false); + } + + fn emit_named_entry(&mut self, root: &Specialization, name: &str, internal: bool) { + let function = &self.program.functions[root.function]; + let output = output_type(&function.outputs); + let linkage = if internal { "internal " } else { "" }; + writeln!( + self.functions, + "define {linkage}i32 @{name}(ptr %out, ptr %fault) {{\nentry:" + ) + .unwrap(); + writeln!( + self.functions, + " store {} zeroinitializer, ptr %out", + self.ty(&output) + ) + .unwrap(); + self.functions + .push_str(" store %dever.fault zeroinitializer, ptr %fault\n"); + if specialize::suspends(self.program, root) { + self.emit_async_entry(root); + } else { + writeln!( + self.functions, + " %status = call i32 @{}(ptr %out, ptr %fault)", + self.names[root] + ) + .unwrap(); + } + let uses_log = self.instances.iter().any(|instance| { + let mut found = false; + crate::check::visit(&self.program.functions[instance.function], |expression| { + found |= matches!( + expression.kind, + ExpressionKind::Intrinsic { + operation: Intrinsic::LogDebug + | Intrinsic::LogInfo + | Intrinsic::LogWarn + | Intrinsic::LogError + | Intrinsic::HttpServe + | Intrinsic::HttpServeTls + | Intrinsic::HttpServeLive + | Intrinsic::HttpServeLiveTls, + .. + } + ); + }); + found + }); + if uses_log { + self.functions + .push_str(" call void @dever_rt_v1_log_flush()\n"); + } + self.functions.push_str(" ret i32 %status\n}\n"); + self.functions + .push_str("define void @dever_outputs_release(ptr %out) {\nentry:\n"); + if contains_owned(self.program, &output) { + writeln!( + self.functions, + " call void @dever_drop_{}(ptr %out)", + self.type_index(&output) + ) + .unwrap(); + } + self.functions.push_str(" ret void\n}\n"); + self.functions + .push_str("define void @dever_fault_release(ptr %fault) {\nentry:\n"); + let failures = self + .instances + .iter() + .flat_map(|instance| specialize::failures(self.program, instance)) + .map(|failure| failure.ty) + .collect::<BTreeSet<_>>(); + if !failures.is_empty() { + self.functions.push_str(" %kind = getelementptr %dever.fault, ptr %fault, i32 0, i32 5\n %failure_type = load i32, ptr %kind\n"); + let arms = failures + .iter() + .map(|id| format!("i32 {}, label %drop_failure_{id}", id + 1)) + .collect::<Vec<_>>() + .join(" "); + writeln!( + self.functions, + " switch i32 %failure_type, label %message_release [{arms}]" + ) + .unwrap(); + for id in failures { + writeln!(self.functions, "drop_failure_{id}:").unwrap(); + if contains_owned(self.program, &Type::Named(id)) { + writeln!(self.functions, " %payload_{id} = getelementptr %dever.fault, ptr %fault, i32 0, i32 7\n call void @dever_drop_{}(ptr %payload_{id})", self.type_index(&Type::Named(id))).unwrap(); + } + self.functions.push_str(" br label %message_release\n"); + } + self.functions.push_str("message_release:\n"); + } + if self.uses_runtime() { + self.functions.push_str(concat!( + " %message = getelementptr %dever.fault, ptr %fault, i32 0, i32 4\n", + " %buffer = load { ptr, i64 }, ptr %message\n", + " %bytes = extractvalue { ptr, i64 } %buffer, 0\n", + " %length = extractvalue { ptr, i64 } %buffer, 1\n", + " call void @dever_rt_v1_buffer_free(ptr %bytes, i64 %length)\n", + " %cause_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 8\n", + " %cause = load ptr, ptr %cause_ptr\n", + " call void @dever_rt_v1_text_release(ptr %cause)\n", + )); + } + self.functions + .push_str(" store %dever.fault zeroinitializer, ptr %fault\n ret void\n}\n"); + debug_assert!(function.parameters.is_empty()); + } + + fn finish(mut self) -> String { + if !self.http_handlers.is_empty() + || self + .application + .as_ref() + .is_some_and(|app| app.test.is_some() || app.executable) + { + self.emit_location_renderer(); + } + let mut ir = String::from("; Dever checked typed function kernel\n"); + ir.push_str(&self.declarations); + writeln!( + ir, + "@dever_source_span_count = constant i32 {}", + self.locations.len() + ) + .unwrap(); + let spans = self + .locations + .iter() + .map(|span| { + format!( + "{{ i64, i64, i64 }} {{ i64 {}, i64 {}, i64 {} }}", + span.source.0, span.start, span.end + ) + }) + .collect::<Vec<_>>() + .join(", "); + writeln!( + ir, + "@dever_source_spans = constant [{} x {{ i64, i64, i64 }}] [{spans}]", + self.locations.len() + ) + .unwrap(); + ir.push_str(&self.functions); + match self.application.as_ref().and_then(|app| app.test) { + Some(index) => testing::namespace_module(&ir, index), + None => ir, + } + } +} + +fn align(size: usize, alignment: usize) -> usize { + (size + alignment - 1) & !(alignment - 1) +} + +fn collect_types(program: &Program, ty: &Type, ids: &mut BTreeSet<usize>) { + match ty { + Type::Named(id) if ids.insert(*id) => match &program.types[*id].shape { + Shape::Record(fields) => fields + .iter() + .for_each(|field| collect_types(program, &field.ty, ids)), + Shape::Choice(variants) => variants + .iter() + .flat_map(|variant| &variant.fields) + .for_each(|field| collect_types(program, &field.ty, ids)), + }, + Type::Nullable(inner) + | Type::List(inner) + | Type::Stream(inner) + | Type::AsyncStream(inner) + | Type::RowStream(inner) + | Type::Related(inner) + | Type::Channel(inner) => collect_types(program, inner, ids), + Type::Map(key, value) | Type::MapEntry(key, value) => { + collect_types(program, key, ids); + collect_types(program, value, ids); + } + Type::Outputs(fields) | Type::Task(fields) => fields + .iter() + .for_each(|field| collect_types(program, &field.ty, ids)), + _ => {} + } +} + +struct FunctionEmitter<'a, 'b> { + module: &'a mut Module<'b>, + instance: &'a Specialization, + prologue: String, + body: String, + next: usize, + block: String, + locals: Vec<(Type, String)>, + local_guards: Vec<Option<usize>>, + guards: Vec<OwnedSlot>, + moves: HashSet<*const Expression>, + bindings: BTreeMap<usize, usize>, + asynchronous: bool, + parameters: BTreeMap<usize, String>, + database_context: String, + database_errors_override: Option<String>, + protocol_callback: bool, + transactions: Vec<transaction::TransactionScope>, +} + +struct OwnedSlot { + kind: OwnedKind, + pointer: String, + live: String, +} + +enum OwnedKind { + Fault, + Value(Type), + Output(Type), + Protocol(&'static str), + ListCursor, + MapCursor, + AsyncOp, + AsyncStreamScope, + DatabaseStreamScope, +} + +impl<'a, 'b> FunctionEmitter<'a, 'b> { + fn new(module: &'a mut Module<'b>, instance: &'a Specialization) -> Self { + let function = &module.program.functions[instance.function]; + let bindings = specialize::handler_bindings(function, instance); + let asynchronous = specialize::suspends(module.program, instance); + Self { + module, + instance, + prologue: String::new(), + body: String::new(), + next: 0, + block: String::new(), + locals: Vec::new(), + local_guards: Vec::new(), + guards: Vec::new(), + moves: HashSet::new(), + bindings, + asynchronous, + parameters: BTreeMap::new(), + database_context: "null".into(), + database_errors_override: None, + protocol_callback: false, + transactions: Vec::new(), + } + } + + fn temp(&mut self) -> String { + let id = self.next; + self.next += 1; + format!("%v{id}") + } + fn label(&mut self, prefix: &str) -> String { + let id = self.next; + self.next += 1; + format!("{prefix}{id}") + } + fn start(&mut self, label: &str) { + self.block = label.to_owned(); + if label != "entry" { + writeln!(self.body, "{label}:").unwrap(); + } + } + fn line(&mut self, line: impl AsRef<str>) { + writeln!(self.body, " {}", line.as_ref()).unwrap(); + } + + fn exit(&mut self, status: &str) { + self.line(format!("store i32 {status}, ptr %exit_status")); + self.line("br label %cleanup"); + } + + fn entry_slot(&mut self, ty: &Type) -> String { + self.entry_slot_ir(&self.module.ty(ty)) + } + + fn entry_slot_ir(&mut self, ty: &str) -> String { + let pointer = self.temp(); + writeln!(self.prologue, " {pointer} = alloca {ty}").unwrap(); + pointer + } + + fn register_guard(&mut self, ty: &Type, pointer: String) -> usize { + self.register_owned(OwnedKind::Value(ty.clone()), pointer) + } + + fn register_owned(&mut self, kind: OwnedKind, pointer: String) -> usize { + let live = self.temp(); + writeln!( + self.prologue, + " {live} = alloca i1\n store i1 0, ptr {live}" + ) + .unwrap(); + let index = self.guards.len(); + self.guards.push(OwnedSlot { + kind, + pointer, + live, + }); + index + } + + fn mark_live(&mut self, guard: usize) { + self.line(format!("store i1 1, ptr {}", self.guards[guard].live)); + } + + fn own_value(&mut self, ty: &Type, value: &str, borrowed: bool) -> usize { + let pointer = self.entry_slot(ty); + let guard = self.register_guard(ty, pointer.clone()); + if borrowed { + let source = self.entry_slot(ty); + self.line(format!( + "store {} {value}, ptr {source}", + self.module.ty(ty) + )); + self.line(format!( + "call void @dever_clone_{}(ptr {source}, ptr {pointer})", + self.module.type_index(ty) + )); + } else { + self.line(format!( + "store {} {value}, ptr {pointer}", + self.module.ty(ty) + )); + } + self.mark_live(guard); + guard + } + + fn load_owned_value(&mut self, ty: &Type, guard: usize) -> String { + let value = self.temp(); + self.line(format!( + "{value} = load {}, ptr {}", + self.module.ty(ty), + self.guards[guard].pointer + )); + value + } + + fn disarm(&mut self, guard: usize) { + self.line(format!("store i1 0, ptr {}", self.guards[guard].live)); + } + + fn release_guard(&mut self, guard: usize) { + let live_ptr = self.guards[guard].live.clone(); + let pointer = self.guards[guard].pointer.clone(); + let live = self.temp(); + self.line(format!("{live} = load i1, ptr {live_ptr}")); + let release = self.label("release_owned"); + let done = self.label("release_done"); + self.line(format!("br i1 {live}, label %{release}, label %{done}")); + self.start(&release); + match &self.guards[guard].kind { + OwnedKind::Fault => self.line(format!("call void @dever_fault_release(ptr {pointer})")), + OwnedKind::Protocol(kind) => { + let kind = *kind; + let handle = self.temp(); + self.line(format!("{handle} = load ptr, ptr {pointer}")); + self.line(format!( + "call void @dever_rt_v1_{kind}_release(ptr {handle})" + )); + } + OwnedKind::Value(ty) => self.line(format!( + "call void @dever_drop_{}(ptr {pointer})", + self.module.type_index(ty) + )), + OwnedKind::Output(ty) => { + let ty = ty.clone(); + self.line(format!( + "call void @dever_drop_{}(ptr {pointer})", + self.module.type_index(&ty) + )); + self.line(format!( + "store {} zeroinitializer, ptr {pointer}", + self.module.ty(&ty) + )); + } + OwnedKind::ListCursor | OwnedKind::MapCursor => { + let kind = if matches!(self.guards[guard].kind, OwnedKind::ListCursor) { + "list" + } else { + "map" + }; + let cursor = self.temp(); + self.line(format!("{cursor} = load ptr, ptr {pointer}")); + self.line(format!( + "call void @dever_rt_v1_{kind}_cursor_release(ptr {cursor})" + )); + } + OwnedKind::AsyncOp => { + let operation = self.temp(); + self.line(format!("{operation} = load ptr, ptr {pointer}")); + self.line(format!( + "call void @dever_rt_v1_async_op_release(ptr {operation})" + )); + } + OwnedKind::AsyncStreamScope | OwnedKind::DatabaseStreamScope => { + let kind = if matches!(self.guards[guard].kind, OwnedKind::DatabaseStreamScope) { + "db_stream" + } else { + "async_stream" + }; + let stream = self.temp(); + self.line(format!("{stream} = load ptr, ptr {pointer}")); + let output = self.entry_slot_ir("i8"); + let error = self.entry_slot_ir("{ ptr, i64 }"); + // Closing a compiler-owned valid stream is infallible and must + // not replace the original fault during structured cleanup. + self.line(format!( + "call i32 @dever_rt_v1_{kind}_close(ptr {stream}, ptr {output}, ptr {error})" + )); + } + } + self.line(format!("store i1 0, ptr {live_ptr}")); + self.line(format!("br label %{done}")); + self.start(&done); + } + + fn release_from(&mut self, first: usize) { + for guard in (first..self.guards.len()).rev() { + self.release_guard(guard); + } + } + + fn release_between(&mut self, first: usize, end: usize) { + for guard in (first..end).rev() { + self.release_guard(guard); + } + } + + fn emit(mut self) -> Result<String, String> { + let function = self.module.program.functions[self.instance.function].clone(); + let name = self.module.names[self.instance].clone(); + let parameters = function + .parameters + .iter() + .enumerate() + .filter_map(|(index, parameter)| { + parameter + .value_type() + .map(|ty| format!("{} %p{index}", self.module.ty(ty))) + }) + .collect::<Vec<_>>() + .join(", "); + let mut parameters = if parameters.is_empty() { + String::new() + } else { + format!("{parameters}, ") + }; + if self.module.database_function(self.instance) { + parameters.push_str("ptr %database, "); + self.database_context = "%database".into(); + } + self.start("entry"); + self.prologue + .push_str(" %exit_status = alloca i32\n store i32 0, ptr %exit_status\n"); + if self.asynchronous { + self.begin_coroutine(&function); + } + self.begin_transaction(&function); + if self.emit_external_body(&function)? { + return Ok(self.finish_function(&name, ¶meters)); + } + if self.emit_port_body(&function)? { + return Ok(self.finish_function(&name, ¶meters)); + } + self.line("br label %clause0"); + for (index, clause) in function.clauses.iter().enumerate() { + self.moves = crate::native::liveness::last_uses(clause); + self.start(&format!("clause{index}")); + let guard = self.clause_guard(clause)?; + self.line(format!( + "br i1 {guard}, label %body{index}, label %clause{}", + index + 1 + )); + self.start(&format!("body{index}")); + self.locals = clause + .locals + .iter() + .map(|ty| (ty.clone(), String::new())) + .collect(); + self.local_guards.clear(); + for (slot, ty) in clause.locals.iter().enumerate() { + let pointer = self.entry_slot(ty); + let guard = contains_owned(self.module.program, ty) + .then(|| self.register_guard(ty, pointer.clone())); + self.local_guards.push(guard); + self.locals[slot].1 = pointer; + } + for binding in &clause.bindings { + self.emit_binding(binding, &function)?; + } + for statement in &clause.body { + self.emit_statement(statement)?; + } + if !clause.terminates { + let output = output_type(&function.outputs); + if !matches!(output, Type::Unit) { + let result = self.outputs(clause, &output)?; + self.line(format!( + "store {} {result}, ptr %out", + self.module.ty(&output) + )); + for slot in &clause.outputs { + if let Some(guard) = self.local_guards[*slot] { + self.disarm(guard); + } + } + } + self.exit("0"); + } else { + self.line("unreachable"); + } + } + self.start(&format!("clause{}", function.clauses.len())); + self.line("unreachable"); + Ok(self.finish_function(&name, ¶meters)) + } + + fn finish_function(mut self, name: &str, parameters: &str) -> String { + self.start("cleanup"); + self.finish_transaction(); + self.release_from(0); + let status = self.temp(); + self.line(format!("{status} = load i32, ptr %exit_status")); + if self.asynchronous { + self.end_coroutine(&status); + return format!( + "define ptr @{name}({parameters}ptr %out, ptr %fault, ptr %state) presplitcoroutine {{\nentry:\n{}{} }}\n", + self.prologue, self.body + ); + } + self.line(format!("ret i32 {status}")); + format!( + "define i32 @{name}({parameters}ptr %out, ptr %fault) {{\nentry:\n{}{} }}\n", + self.prologue, self.body + ) + } + + fn clause_guard(&mut self, clause: &crate::hir::Clause) -> Result<String, String> { + let parameters = &self.module.program.functions[self.instance.function].parameters; + let mut guards = Vec::new(); + for ((index, parameter), domain) in parameters + .iter() + .enumerate() + .filter(|(_, p)| p.value_type().is_some()) + .zip(&clause.patterns) + { + let ty = parameter.value_type().expect("filtered value parameter"); + let mut alternatives = Vec::new(); + for atom in domain { + let parameter = self.parameter(index); + alternatives.push(self.atom(atom, ty, ¶meter)?); + } + guards.push(self.reduce_bool("or", alternatives, "0")); + } + Ok(self.reduce_bool("and", guards, "1")) + } + + fn reduce_bool(&mut self, op: &str, values: Vec<String>, empty: &str) -> String { + let mut values = values.into_iter(); + let Some(mut result) = values.next() else { + return empty.into(); + }; + for value in values { + let next = self.temp(); + self.line(format!("{next} = {op} i1 {result}, {value}")); + result = next; + } + result + } + + fn emit_binding( + &mut self, + binding: &crate::hir::Binding, + function: &crate::hir::Function, + ) -> Result<(), String> { + let ty = function.parameters[binding.input] + .value_type() + .expect("checked value binding"); + let source = self.parameter(binding.input); + if let (Projection::Payload { .. }, Type::Related(inner)) = (&binding.projection, ty.base()) + { + let related = if matches!(ty, Type::Nullable(_)) { + let payload = self.temp(); + self.line(format!( + "{payload} = extractvalue {} {source}, 1", + self.module.ty(ty) + )); + payload + } else { + source + }; + let value = self.runtime_call( + "dever_rt_v1_db_related_get", + vec![format!("ptr {related}")], + inner, + false, + None, + function.span, + ); + let pointer = self.locals[binding.slot].1.clone(); + self.line(format!( + "store {} {value}, ptr {pointer}", + self.module.ty(inner) + )); + if let Some(guard) = self.local_guards[binding.slot] { + self.mark_live(guard); + } + return Ok(()); + } + let (value, projected) = match binding.projection { + Projection::Whole => (source, ty.clone()), + Projection::NonNull => { + let Type::Nullable(inner) = ty else { + unreachable!("checked non-null binding") + }; + let value = self.temp(); + self.line(format!( + "{value} = extractvalue {} {source}, 1", + self.module.ty(ty) + )); + (value, *inner.clone()) + } + Projection::Payload { variant, field } => { + let nominal = ty.base(); + let Type::Named(id) = nominal else { + unreachable!("checked choice binding") + }; + let Shape::Choice(variants) = &self.module.program.types[*id].shape else { + unreachable!() + }; + let projected = variants[variant].fields[field].ty.clone(); + let value = if matches!(ty, Type::Nullable(_)) { + let temp = self.temp(); + self.line(format!( + "{temp} = extractvalue {} {source}, 1", + self.module.ty(ty) + )); + temp + } else { + source + }; + let alloca = self.temp(); + self.line(format!("{alloca} = alloca %T{id}")); + self.line(format!("store %T{id} {value}, ptr {alloca}")); + let payload = self.temp(); + self.line(format!( + "{payload} = getelementptr %T{id}, ptr {alloca}, i32 0, i32 1" + )); + let field_ptr = self.temp(); + self.line(format!("{field_ptr} = getelementptr %T{id}V{variant}, ptr {payload}, i32 0, i32 {field}")); + let result = self.temp(); + self.line(format!( + "{result} = load {}, ptr {field_ptr}", + self.module.ty(&projected) + )); + (result, projected) + } + }; + debug_assert_eq!(projected, self.locals[binding.slot].0); + let destination = self.locals[binding.slot].1.clone(); + if let Some(guard) = self.local_guards[binding.slot] { + let source = self.entry_slot(&projected); + self.line(format!( + "store {} {value}, ptr {source}", + self.module.ty(&projected) + )); + self.line(format!( + "call void @dever_clone_{}(ptr {source}, ptr {destination})", + self.module.type_index(&projected) + )); + self.line(format!("store i1 1, ptr {}", self.guards[guard].live)); + } else { + self.line(format!( + "store {} {value}, ptr {destination}", + self.module.ty(&projected) + )); + } + Ok(()) + } + + fn outputs(&mut self, clause: &crate::hir::Clause, output: &Type) -> Result<String, String> { + if let [slot] = clause.outputs.as_slice() { + return Ok(self.load_local(*slot)); + } + let Type::Outputs(fields) = output else { + unreachable!("checked output arity") + }; + let mut aggregate = "undef".to_owned(); + for (index, (slot, field)) in clause.outputs.iter().zip(fields).enumerate() { + let value = self.load_local(*slot); + let next = self.temp(); + self.line(format!( + "{next} = insertvalue {} {aggregate}, {} {value}, {index}", + self.module.ty(output), + self.module.ty(&field.ty) + )); + aggregate = next; + } + Ok(aggregate) + } + + fn load_local(&mut self, slot: usize) -> String { + let ty = self.module.ty(&self.locals[slot].0); + let value = self.temp(); + self.line(format!("{value} = load {ty}, ptr {}", self.locals[slot].1)); + value + } + + fn field_pointer(&mut self, slot: usize, fields: &[usize]) -> (String, Type) { + let mut pointer = self.locals[slot].1.clone(); + let mut ty = self.locals[slot].0.clone(); + for index in fields { + let element_ty = match &ty { + Type::Named(id) => match &self.module.program.types[*id].shape { + Shape::Record(fields) => fields[*index].ty.clone(), + Shape::Choice(_) => unreachable!("checked field place"), + }, + Type::MapEntry(key, value) => { + if *index == 0 { + key.as_ref().clone() + } else { + value.as_ref().clone() + } + } + Type::Outputs(fields) => fields[*index].ty.clone(), + _ => unreachable!("checked field place"), + }; + let next = self.temp(); + self.line(format!( + "{next} = getelementptr {}, ptr {pointer}, i32 0, i32 {index}", + self.module.ty(&ty) + )); + pointer = next; + ty = element_ty; + } + (pointer, ty) + } + + fn emit_statement(&mut self, statement: &Statement) -> Result<(), String> { + match statement { + Statement::Assign { + slot, + fields, + value, + } => { + let result = self.expression(value)?; + let result_guard = + contains_owned(self.module.program, &value.ty).then(|| self.guards.len() - 1); + let (pointer, ty) = self.field_pointer(*slot, fields); + if fields.is_empty() { + if let Some(guard) = self.local_guards[*slot] { + self.release_guard(guard); + } + } else if contains_owned(self.module.program, &ty) { + self.line(format!( + "call void @dever_drop_{}(ptr {pointer})", + self.module.type_index(&ty) + )); + } + self.line(format!( + "store {} {result}, ptr {pointer}", + self.module.ty(&ty) + )); + if let Some(guard) = result_guard { + self.disarm(guard); + } + if fields.is_empty() + && let Some(guard) = self.local_guards[*slot] + { + self.line(format!("store i1 1, ptr {}", self.guards[guard].live)); + } + } + Statement::Call(value) => { + let first = self.guards.len(); + self.expression(value)?; + self.release_from(first); + } + } + Ok(()) + } +} + +mod abi; +mod api; +mod api_context; +mod application; +mod application_commands; +mod asynchronous; +mod collections; +mod concurrency; +mod database; +mod database_rows; +mod database_schema; +mod expressions; +mod external; +mod failures; +mod http; +mod http_handlers; +mod http_values; +mod intrinsics; +mod jobs; +mod network; +mod orm; +mod orm_read; +mod orm_relations; +mod orm_stream; +mod ownership; +mod parallel; +mod ports; +mod process_entry; +mod render; +mod resources; +mod rest; +mod system; +mod testing; +mod transaction; +mod traversal; +mod wire; diff --git a/crates/dever-core/src/llvm/abi.rs b/crates/dever-core/src/llvm/abi.rs new file mode 100644 index 0000000..8479e4b --- /dev/null +++ b/crates/dever-core/src/llvm/abi.rs @@ -0,0 +1,319 @@ +use super::*; + +enum AbiOutput<'a> { + Direct(Option<&'a str>), + Optional(Option<&'a Type>), +} + +pub(super) const DECLARATIONS: &str = "\ +declare i32 @dever_rt_v1_time_sleep(i64, ptr, ptr) +declare i32 @dever_rt_v1_text_new(ptr, i64, ptr, ptr) +declare i8 @dever_rt_v1_text_equal(ptr, ptr) +declare i64 @dever_rt_v1_text_hash(ptr) +declare i32 @dever_rt_v1_text_compare(ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_text_concat(ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_text_length(ptr, ptr, ptr) +declare i32 @dever_rt_v1_text_trim(ptr, ptr, ptr) +declare i32 @dever_rt_v1_text_lower(ptr, ptr, ptr) +declare i32 @dever_rt_v1_text_upper(ptr, ptr, ptr) +declare i32 @dever_rt_v1_text_codepoint(ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_text_from_codepoint(i64, ptr, ptr, ptr) +declare i32 @dever_rt_v1_text_at(ptr, i64, ptr, ptr, ptr) +declare i32 @dever_rt_v1_text_slice(ptr, i64, i64, ptr, ptr, ptr) +declare i32 @dever_rt_v1_text_index_of(ptr, ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_text_contains(ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_text_starts_with(ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_text_ends_with(ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_text_replace(ptr, ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_text_split(ptr, ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_int_parse(ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_float_parse(ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_decimal_parse(ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_int_to_text(i64, ptr) +declare i32 @dever_rt_v1_float_to_text(double, ptr) +declare i32 @dever_rt_v1_decimal_to_text(i64, i64, ptr) +declare i32 @dever_rt_v1_decimal_from_int(i64, ptr, ptr) +declare i32 @dever_rt_v1_decimal_add(i64, i64, i64, i64, ptr, ptr) +declare i32 @dever_rt_v1_decimal_sub(i64, i64, i64, i64, ptr, ptr) +declare i32 @dever_rt_v1_decimal_mul(i64, i64, i64, i64, ptr, ptr) +declare i32 @dever_rt_v1_decimal_div(i64, i64, i64, i64, ptr, ptr) +declare i32 @dever_rt_v1_decimal_neg(i64, i64, ptr, ptr) +declare i32 @dever_rt_v1_decimal_round(i64, i64, i64, ptr, ptr) +declare i32 @dever_rt_v1_decimal_compare(i64, i64, i64, i64, ptr, ptr) +declare i8 @dever_rt_v1_decimal_equal(i64, i64, i64, i64) +declare i8 @dever_rt_v1_bytes_equal(ptr, ptr) +declare i32 @dever_rt_v1_bytes_from_text(ptr, ptr, ptr) +declare i32 @dever_rt_v1_bytes_length(ptr, ptr, ptr) +declare i32 @dever_rt_v1_bytes_at(ptr, i64, ptr, ptr, ptr) +declare i32 @dever_rt_v1_bytes_concat(ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_bytes_slice(ptr, i64, i64, ptr, ptr) +declare i32 @dever_rt_v1_bytes_to_text_handle(ptr, ptr, ptr) +declare i32 @dever_rt_v1_bytes_from_ints(ptr, ptr, ptr) +declare i32 @dever_rt_v1_list_new(ptr, ptr, i64, ptr, ptr) +declare i32 @dever_rt_v1_list_append_take(ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_list_first_take(ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_list_length(ptr, ptr, ptr) +declare i8 @dever_rt_v1_list_equal(ptr, ptr) +declare i32 @dever_rt_v1_list_cursor_take(ptr, ptr, ptr) +declare i32 @dever_rt_v1_list_cursor_next(ptr, ptr, ptr, ptr) +declare void @dever_rt_v1_list_cursor_release(ptr) +declare i32 @dever_rt_v1_map_new(ptr, ptr, ptr, ptr, i64, ptr, ptr) +declare i32 @dever_rt_v1_map_put_take(ptr, ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_map_remove_take(ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_map_get_take(ptr, ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_map_length(ptr, ptr, ptr) +declare i8 @dever_rt_v1_map_equal(ptr, ptr) +declare i32 @dever_rt_v1_map_cursor_take(ptr, ptr, ptr) +declare i32 @dever_rt_v1_map_cursor_next(ptr, ptr, ptr, ptr, ptr) +declare void @dever_rt_v1_map_cursor_release(ptr) +declare i32 @dever_rt_v1_file_open(ptr, ptr, ptr) +declare i32 @dever_rt_v1_file_create(ptr, ptr, ptr) +declare i32 @dever_rt_v1_file_read(ptr, i64, ptr, ptr, ptr) +declare i32 @dever_rt_v1_file_write(ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_file_close(ptr, ptr, ptr) +declare i32 @dever_rt_v1_file_chunks(ptr, i64, ptr, ptr, ptr) +declare i32 @dever_rt_v1_stream_pull(ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_stream_close(ptr, ptr, ptr) +declare double @llvm.sqrt.f64(double) +declare double @llvm.sin.f64(double) +declare double @llvm.cos.f64(double) +declare double @llvm.log.f64(double) +declare double @llvm.pow.f64(double, double) +"; + +impl Module<'_> { + pub(super) fn text_literal(&mut self, value: &str) -> (String, usize) { + let len = value.len(); + if len == 0 { + return ("null".into(), 0); + } + let name = format!("@dever_text_literal_{}", self.next_literal); + self.next_literal += 1; + let bytes = value + .as_bytes() + .iter() + .map(|byte| format!("\\{byte:02X}")) + .collect::<String>(); + writeln!( + self.declarations, + "{name} = private constant [{len} x i8] c\"{bytes}\"" + ) + .unwrap(); + (name, len) + } +} + +impl FunctionEmitter<'_, '_> { + pub(super) fn check_runtime_status(&mut self, status: &str, error: &str, span: Span) { + let failed = self.temp(); + self.line(format!("{failed} = icmp ne i32 {status}, 0")); + let fault = self.label("runtime_fault"); + let success = self.label("runtime_ok"); + self.line(format!("br i1 {failed}, label %{fault}, label %{success}")); + self.start(&fault); + let message = self.temp(); + self.line(format!("{message} = load {{ ptr, i64 }}, ptr {error}")); + if self.database_errors_override.is_some() { + self.line(format!("store {{ ptr, i64 }} {message}, ptr %error")); + self.exit("2"); + self.start(&success); + return; + } + let destination = self.temp(); + self.line(format!( + "{destination} = getelementptr %dever.fault, ptr %fault, i32 0, i32 4" + )); + self.line(format!("store {{ ptr, i64 }} {message}, ptr {destination}")); + if self.protocol_callback { + self.exit(status); + } else { + self.write_fault(&(FaultCode::RuntimeAbi as i32).to_string(), span); + } + self.start(&success); + } + + pub(super) fn runtime_call( + &mut self, + symbol: &str, + arguments: Vec<String>, + result_type: &Type, + nullable: bool, + physical_type: Option<&str>, + span: Span, + ) -> String { + let output = if nullable { + assert!( + physical_type.is_none(), + "optional ABI output uses its concrete row type" + ); + AbiOutput::Optional(None) + } else { + AbiOutput::Direct(physical_type) + }; + self.runtime_call_with_output(symbol, arguments, result_type, output, span) + } + + pub(super) fn runtime_optional_row( + &mut self, + symbol: &str, + arguments: Vec<String>, + row_type: &Type, + result_type: &Type, + span: Span, + ) -> String { + self.runtime_call_with_output( + symbol, + arguments, + result_type, + AbiOutput::Optional(Some(row_type)), + span, + ) + } + + fn runtime_call_with_output( + &mut self, + symbol: &str, + mut arguments: Vec<String>, + result_type: &Type, + output_kind: AbiOutput<'_>, + span: Span, + ) -> String { + let (value_type, physical_type, nullable) = match output_kind { + AbiOutput::Direct(physical) => (result_type, physical, false), + AbiOutput::Optional(row) => { + let Type::Nullable(inner) = result_type else { + unreachable!("nullable ABI result type") + }; + (row.unwrap_or(inner.as_ref()), None, true) + } + }; + let output_type = physical_type + .map(str::to_owned) + .unwrap_or_else(|| self.module.ty(value_type)); + let output = self.entry_slot_ir(&output_type); + arguments.push(format!("ptr {output}")); + let present = nullable.then(|| { + let pointer = self.temp(); + writeln!(self.prologue, " {pointer} = alloca i8").unwrap(); + arguments.push(format!("ptr {pointer}")); + pointer + }); + let error = self.temp(); + writeln!(self.prologue, " {error} = alloca {{ ptr, i64 }}").unwrap(); + arguments.push(format!("ptr {error}")); + let status = self.temp(); + self.line(format!( + "{status} = call i32 @{symbol}({})", + arguments.join(", ") + )); + self.check_runtime_status(&status, &error, span); + if let Some(present) = present { + self.optional_row(result_type, value_type, &output, &present) + } else { + let result = self.temp(); + self.line(format!("{result} = load {output_type}, ptr {output}")); + if output_type == "i8" && matches!(result_type, Type::Bool) { + let boolean = self.temp(); + self.line(format!("{boolean} = icmp ne i8 {result}, 0")); + boolean + } else { + result + } + } + } + + /// Presence guards the concrete row; a nullable row is transferred intact. + /// Collection lookups and Channel receive use the same flattening contract. + pub(super) fn optional_row( + &mut self, + result: &Type, + row_type: &Type, + row: &str, + present: &str, + ) -> String { + let slot = self.entry_slot(result); + self.line(format!( + "store {} zeroinitializer, ptr {slot}", + self.module.ty(result) + )); + let flag = self.temp(); + self.line(format!("{flag} = load i8, ptr {present}")); + let has_value = self.temp(); + self.line(format!("{has_value} = icmp ne i8 {flag}, 0")); + let some = self.label("runtime_some"); + let done = self.label("runtime_optional_done"); + self.line(format!("br i1 {has_value}, label %{some}, label %{done}")); + self.start(&some); + let value = self.temp(); + self.line(format!( + "{value} = load {}, ptr {row}", + self.module.ty(row_type) + )); + if row_type == result { + self.line(format!( + "store {} {value}, ptr {slot}", + self.module.ty(result) + )); + } else { + let flag_pointer = self.temp(); + self.line(format!( + "{flag_pointer} = getelementptr {}, ptr {slot}, i32 0, i32 0", + self.module.ty(result) + )); + self.line(format!("store i1 1, ptr {flag_pointer}")); + let payload = self.temp(); + self.line(format!( + "{payload} = getelementptr {}, ptr {slot}, i32 0, i32 1", + self.module.ty(result) + )); + self.line(format!( + "store {} {value}, ptr {payload}", + self.module.ty(row_type) + )); + } + self.line(format!("br label %{done}")); + self.start(&done); + let value = self.temp(); + self.line(format!( + "{value} = load {}, ptr {slot}", + self.module.ty(result) + )); + value + } + + pub(super) fn render_text( + &mut self, + symbol: &str, + arguments: Vec<String>, + span: Span, + ) -> String { + let buffer = self.entry_slot_ir("{ ptr, i64 }"); + let mut arguments = arguments; + arguments.push(format!("ptr {buffer}")); + let status = self.temp(); + self.line(format!( + "{status} = call i32 @{symbol}({})", + arguments.join(", ") + )); + self.check_runtime_status(&status, &buffer, span); + let bytes = self.temp(); + self.line(format!("{bytes} = load {{ ptr, i64 }}, ptr {buffer}")); + let pointer = self.temp(); + self.line(format!( + "{pointer} = extractvalue {{ ptr, i64 }} {bytes}, 0" + )); + let length = self.temp(); + self.line(format!("{length} = extractvalue {{ ptr, i64 }} {bytes}, 1")); + let output = self.entry_slot(&Type::Text); + let error = self.entry_slot_ir("{ ptr, i64 }"); + let created = self.temp(); + self.line(format!("{created} = call i32 @dever_rt_v1_text_new(ptr {pointer}, i64 {length}, ptr {output}, ptr {error})")); + self.line(format!( + "call void @dever_rt_v1_buffer_free(ptr {pointer}, i64 {length})" + )); + self.check_runtime_status(&created, &error, span); + let result = self.temp(); + self.line(format!("{result} = load ptr, ptr {output}")); + result + } +} diff --git a/crates/dever-core/src/llvm/api.rs b/crates/dever-core/src/llvm/api.rs new file mode 100644 index 0000000..446d3bc --- /dev/null +++ b/crates/dever-core/src/llvm/api.rs @@ -0,0 +1,570 @@ +//! Static request adapters call concrete App functions inside the runtime request scope. +use super::*; + +struct RouteDescriptor<'a> { + method: &'a str, + path: &'a str, + directory: &'a [String], + components: &'a [String], + permission: Option<usize>, + auth: Option<usize>, + handler: &'a str, + anonymous: bool, + multipart: bool, + detail: bool, +} + +pub(super) const DECLARATIONS: &str = "\ +%dever.api_handler = type { ptr, ptr, ptr, ptr, ptr, ptr } +%dever.api_auth = type { ptr, ptr, ptr, ptr, ptr, ptr } +%dever.api_permission = type { { ptr, i64 }, { ptr, i64 }, { ptr, i64 }, { ptr, i64 }, { ptr, i64 }, { ptr, i64 } } +%dever.api_route = type { { ptr, i64 }, { ptr, i64 }, ptr, i64, ptr, i64, ptr, ptr, ptr, i8, i8, i8 } +declare i32 @dever_rt_v1_api_input_field(ptr, ptr, i8, i32, ptr, ptr, ptr) +declare i32 @dever_rt_v1_api_input_upload(ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_api_input_raw_field(ptr, ptr, i8, ptr, ptr, ptr) +declare i32 @dever_rt_v1_api_text_bounds(ptr, ptr, i64, i64, i8, ptr) +declare i32 @dever_rt_v1_api_pagination(i64, i64, i64, ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_api_session_new(ptr, i64, ptr, i64, ptr, ptr) +declare i32 @dever_rt_v1_api_session_database(ptr, ptr, ptr) +declare void @dever_rt_v1_api_session_release(ptr) +declare i32 @dever_rt_v1_api_application_root(ptr, ptr, ptr, ptr, ptr, ptr, ptr) +declare ptr @dever_rt_v1_api_initialize(ptr, ptr, ptr, i64, i8, ptr) +declare void @dever_rt_v1_api_cause_append(ptr, ptr) +declare ptr @dever_rt_v1_api_serve(ptr, ptr, i64, ptr, i64, ptr) +declare i32 @dever_rt_v1_api_has_tenant(ptr, ptr, ptr) +declare ptr @dever_rt_v1_api_tenant_scope(i64, ptr, ptr, ptr) +declare ptr @dever_rt_v1_api_require_components(ptr, ptr, i64, ptr, i64, ptr) +declare ptr @dever_rt_v1_api_tenant_owner(ptr, i64, ptr, i64, ptr) +declare ptr @dever_rt_v1_api_tenant_component(ptr, i64, ptr, i8, ptr, i64, ptr) +@dever_api_session = private global ptr null +"; + +impl Module<'_> { + pub(super) fn api_enabled(&self) -> bool { + if self + .application + .as_ref() + .is_some_and(|app| app.test.is_some()) + { + return self.jobs_enabled(); + } + self.application.is_some() + && (!self.program.jobs.is_empty() + || !self.program.api_routes.is_empty() + || !self.program.api_rest.is_empty() + || !self.program.auth.is_empty() + || !self.program.tenant_components.is_empty() + || self.instances.iter().any(|instance| { + let mut scoped_capability = false; + crate::check::visit(&self.program.functions[instance.function], |expression| { + if let ExpressionKind::Intrinsic { operation, .. } = expression.kind { + // Logs read invocation Settings even without HTTP routes. + scoped_capability |= api_context::supported(operation) + || matches!( + operation, + Intrinsic::LogDebug + | Intrinsic::LogInfo + | Intrinsic::LogWarn + | Intrinsic::LogError + ); + } + }); + scoped_capability + })) + } + + pub(super) fn api_name(&mut self, value: &str) -> String { + let (pointer, length) = self.text_literal(value); + format!("{{ ptr, i64 }} {{ ptr {pointer}, i64 {length} }}") + } + + pub(super) fn emit_api_descriptors(&mut self) { + if !self.database_enabled() { + self.declarations + .push_str("%dever.db_binding = type { ptr, { ptr, i64 }, i8 }\n"); + } + self.declarations.push_str("define internal void @dever_api_append_cause(ptr %fault, ptr %message) {\nentry:\n %cause = getelementptr %dever.fault, ptr %fault, i32 0, i32 8\n call void @dever_rt_v1_api_cause_append(ptr %cause, ptr %message)\n ret void\n}\n"); + self.api_fault_status(false); + self.api_fault_status(true); + self.rest_fault_message(); + for (index, hook) in self.program.auth.clone().iter().enumerate() { + self.emit_auth_descriptor(index, hook); + } + let permissions = self.program.permissions.clone(); + let mut entries = Vec::new(); + for (index, permission) in permissions.iter().enumerate() { + let fields = [ + &permission.key, + &permission.component, + &permission.domain, + &permission.site, + &permission.action, + permission.method, + ] + .iter() + .map(|value| self.api_name(value)) + .collect::<Vec<_>>() + .join(", "); + writeln!( + self.declarations, + "@dever_permission_{index} = private constant %dever.api_permission {{ {fields} }}" + ) + .unwrap(); + entries.push(format!("%dever.api_permission {{ {fields} }}")); + } + writeln!( + self.declarations, + "@dever_permissions = private constant [{} x %dever.api_permission] [{}]", + entries.len(), + entries.join(", ") + ) + .unwrap(); + let mut routes = Vec::new(); + for (index, route) in self.program.api_routes.clone().iter().enumerate() { + let handler = self.api_handler(index); + routes.push(self.api_route_value(RouteDescriptor { + method: route.method, + path: &route.path, + directory: &route.directory, + components: &route.components, + permission: route.permission, + auth: route.auth, + handler: &handler, + anonymous: route.anonymous, + multipart: route.multipart, + detail: false, + })); + } + for (index, handler) in self + .application + .as_ref() + .unwrap() + .rest_handlers + .clone() + .iter() + .enumerate() + { + let route = self.program.api_rest[handler.route].clone(); + let permission = match handler.method { + "GET" => route.permissions.read, + "POST" => route.permissions.create, + "PUT" => route.permissions.replace, + "DELETE" => route.permissions.delete, + _ => unreachable!(), + }; + let descriptor = self.rest_handler(index, handler); + routes.push(self.api_route_value(RouteDescriptor { + method: handler.method, + path: &route.path, + directory: &route.directory, + components: &route.components, + permission, + auth: route.auth, + handler: &descriptor, + anonymous: false, + multipart: false, + detail: handler.detail, + })); + } + writeln!( + self.declarations, + "@dever_api_routes = private constant [{} x %dever.api_route] [{}]", + routes.len(), + routes.join(", ") + ) + .unwrap(); + } + + fn api_route_value(&mut self, route: RouteDescriptor<'_>) -> String { + let RouteDescriptor { + method, + path, + directory, + components, + permission, + auth, + handler, + anonymous, + multipart, + detail, + } = route; + let method = self.api_name(method); + let path = self.api_name(path); + let names = self.wire_names(directory); + let components_pointer = self.wire_names(components); + let permission = permission.map_or_else( + || "null".into(), + |index| format!("@dever_permission_{index}"), + ); + let auth = auth.map_or_else(|| "null".into(), |index| format!("@dever_api_auth_{index}")); + format!( + "%dever.api_route {{ {method}, {path}, ptr {names}, i64 {}, ptr {components_pointer}, i64 {}, ptr {permission}, ptr {auth}, ptr {handler}, i8 {}, i8 {}, i8 {} }}", + directory.len(), + components.len(), + u8::from(anonymous), + u8::from(multipart), + u8::from(detail) + ) + } + + fn emit_auth_descriptor(&mut self, index: usize, hook: &crate::hir::AuthHook) { + let instance = Specialization { + function: hook.function, + handlers: Vec::new(), + }; + let input = asynchronous::input_type(&self.program.functions[hook.function]); + let mut emitter = FunctionEmitter::new(self, &instance); + emitter.asynchronous = false; + emitter.protocol_callback = true; + emitter.start("entry"); + let Shape::Record(fields) = emitter.module.program.types[hook.claims].shape.clone() else { + unreachable!(); + }; + let mut record = "zeroinitializer".to_owned(); + for (field, pointer) in ["%subject", "%session", "%tenant", "%site"] + .iter() + .enumerate() + { + let value = if field == 2 { + let present = emitter.temp(); + emitter.line(format!("{present} = icmp ne ptr {pointer}, null")); + let flag = emitter.temp(); + let row = emitter.temp(); + emitter.line(format!( + "{flag} = insertvalue {{ i1, ptr }} zeroinitializer, i1 {present}, 0" + )); + emitter.line(format!( + "{row} = insertvalue {{ i1, ptr }} {flag}, ptr {pointer}, 1" + )); + row + } else { + pointer.to_string() + }; + let row = emitter.temp(); + emitter.line(format!( + "{row} = insertvalue %T{} {record}, {} {value}, {field}", + hook.claims, + emitter.module.ty(&fields[field].ty) + )); + record = row; + } + let source = emitter.row_pointer(&Type::Named(hook.claims), &record); + emitter.line(format!( + "call void @dever_clone_{}(ptr {source}, ptr %out)", + emitter.module.type_index(&Type::Named(hook.claims)) + )); + let callback = format!("dever_auth_pack_{index}"); + let body = emitter.finish_protocol_callback( + &callback, + "ptr %subject, ptr %session, ptr %tenant, ptr %site, ptr %out, ptr %error", + ); + self.declarations.push_str(&body); + // Claims is the only input field, so its layout equals the one-field + // input tuple; all owned Text fields were cloned by the typed helper. + let mut unpack = format!( + "define internal i32 @dever_auth_unpack_{index}(ptr %input, ptr %id, ptr %user, ptr %user_present, ptr %tenant, ptr %tenant_present, ptr %error) {{\nentry:\n %identity = load %T{}, ptr %input\n %subject = extractvalue %T{} %identity, 0\n %owned = call ptr @dever_rt_v1_text_retain(ptr %subject)\n store ptr %owned, ptr %id\n", + hook.identity, hook.identity + ); + for (field, name) in [(1, "user"), (2, "tenant")] { + writeln!(unpack, " %{name}_value = extractvalue %T{} %identity, {field}\n %{name}_flag = extractvalue {{ i1, i64 }} %{name}_value, 0\n %{name}_number = extractvalue {{ i1, i64 }} %{name}_value, 1\n %{name}_byte = zext i1 %{name}_flag to i8\n store i64 %{name}_number, ptr %{name}\n store i8 %{name}_byte, ptr %{name}_present", hook.identity).unwrap(); + } + unpack.push_str(" store { ptr, i64 } zeroinitializer, ptr %error\n ret i32 0\n}\n"); + self.declarations.push_str(&unpack); + let name = &self.names[&instance]; + let (asynchronous, synchronous) = if specialize::suspends(self.program, &instance) { + (format!("@dever_async_{name}"), "null".into()) + } else { + ("null".into(), format!("@dever_sync_{name}")) + }; + writeln!(self.declarations, "@dever_api_auth_{index} = private constant %dever.api_auth {{ ptr {asynchronous}, ptr {synchronous}, ptr @dever_owned_{}, ptr @{callback}, ptr @dever_auth_unpack_{index}, ptr @dever_api_fault_status }}", self.type_index(&input)).unwrap(); + } + pub(super) fn emit_api_body( + &mut self, + instance: &Specialization, + index: usize, + ) -> Result<(), String> { + let route = self.program.api_routes[index].clone(); + let wrapper = self.program.functions[instance.function].clone(); + let app = self.program.functions[route.function].clone(); + let target = Specialization { + function: route.function, + handlers: Vec::new(), + }; + let mut emitter = FunctionEmitter::new(self, instance); + let parameters = emitter.begin_generated_function(&wrapper); + let password = [ + Intrinsic::CryptoPasswordHash, + Intrinsic::CryptoPasswordVerify, + ] + .iter() + .any(|operation| specialize::uses_intrinsic(emitter.module.program, &target, *operation)); + let transaction = !password && emitter.begin_command_transaction(&target, route.span); + let arguments = app + .parameters + .iter() + .enumerate() + .map(|(index, parameter)| { + ( + parameter.value_type().unwrap().clone(), + emitter.parameter(index), + ) + }) + .collect::<Vec<_>>(); + let output = output_type(&app.outputs); + let (status, row) = emitter.invoke_values(&target, &arguments, &output, route.span); + emitter.propagate_status(&status, route.span, false); + let value = emitter.temp(); + emitter.line(format!( + "{value} = load {}, ptr {row}", + emitter.module.ty(&output) + )); + emitter.own_value(&output, &value, false); + let encoded = emitter.encode_api_output(&output, &row, route.span); + let guard = emitter.own_value(&Type::Text, &encoded, false); + if transaction { + emitter.commit_command_transaction(); + } + emitter.line(format!("store ptr {encoded}, ptr %out")); + emitter.disarm(guard); + emitter.exit("0"); + let name = emitter.module.names[instance].clone(); + let body = emitter.finish_function(&name, ¶meters); + self.functions.push_str(&body); + Ok(()) + } + + pub(super) fn api_fault_status(&mut self, rest: bool) -> String { + let name = if rest { + "dever_rest_fault_status" + } else { + "dever_api_fault_status" + }; + let mut cases = Vec::new(); + for (id, definition) in self.program.types.iter().enumerate() { + let Shape::Choice(variants) = &definition.shape else { + continue; + }; + for (index, variant) in variants.iter().enumerate() { + let status = match (definition.name.as_str(), variant.name.as_str()) { + ("dever.api.Error", "Invalid") => 400, + ("dever.api.Error", "Unauthorized") => 401, + ("dever.api.Error", "Forbidden") => 403, + ("dever.api.Error", "NotFound") => 404, + ("dever.api.Error", "Conflict") => 409, + ("dever.api.Error", "TooManyRequests") => 429, + ("dever.database.Error", "NotFound") if rest => 404, + ("<generated REST Error>", "Invalid") => 400, + ("<generated REST Error>", "Forbidden") => 403, + ("<generated REST Error>", "NotFound") => 404, + _ => continue, + }; + cases.push((id + 1, index, status)); + } + } + let mut body = format!( + "define internal i64 @{name}(ptr %fault) {{\nentry:\n %code_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 0\n %code = load i32, ptr %code_ptr\n %business = icmp eq i32 %code, 4\n br i1 %business, label %identity, label %internal\nidentity:\n %type_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 5\n %type = load i32, ptr %type_ptr\n %variant_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 6\n %variant = load i32, ptr %variant_ptr\n" + ); + for (index, (id, variant, status)) in cases.iter().enumerate() { + writeln!(body, " %type{index} = icmp eq i32 %type, {id}\n %variant{index} = icmp eq i32 %variant, {variant}\n %matches{index} = and i1 %type{index}, %variant{index}\n br i1 %matches{index}, label %status{index}, label %next{index}\nstatus{index}:\n ret i64 {status}\nnext{index}:").unwrap(); + } + body.push_str(" br label %internal\ninternal:\n ret i64 500\n}\n"); + self.declarations.push_str(&body); + format!("@{name}") + } + + pub(super) fn api_handler(&mut self, index: usize) -> String { + let route = self.program.api_routes[index].clone(); + let function = self.application.as_ref().unwrap().api_handlers[index]; + let instance = Specialization { + function, + handlers: Vec::new(), + }; + let input = asynchronous::input_type(&self.program.functions[function]); + let parameters = self.program.functions[function].parameters.clone(); + let fields = route + .inputs + .iter() + .zip(parameters) + .map(|(name, parameter)| { + let ty = parameter.value_type().unwrap().clone(); + let callback = if ty == Type::Upload { + String::new() + } else { + self.wire_callback(&ty, crate::wire::Policy::ApiInput, false) + }; + (name.clone(), ty, callback) + }) + .collect::<Vec<_>>(); + let mut emitter = FunctionEmitter::new(self, &instance); + emitter.asynchronous = false; + emitter.protocol_callback = true; + emitter.start("entry"); + let value = emitter.decode_fields(&input, &fields, "%inputs", wire::FieldInput::Api); + emitter.line(format!( + "store {} {value}, ptr %out", + emitter.module.ty(&input) + )); + let callback = format!("dever_api_decode_{index}"); + let body = emitter.finish_protocol_callback( + &callback, + "ptr %inputs, i64 %detail_id, ptr %out, ptr %error", + ); + self.declarations.push_str(&body); + let name = &self.names[&instance]; + let descriptor = format!("@dever_api_handler_{index}"); + writeln!(self.declarations, "{descriptor} = private constant %dever.api_handler {{ ptr @dever_async_{name}, ptr null, ptr @dever_owned_{}, ptr @{callback}, ptr @dever_api_fault_status, ptr null }}", self.type_index(&input)).unwrap(); + descriptor + } +} + +impl FunctionEmitter<'_, '_> { + pub(super) fn initialize_api( + &mut self, + bindings: &str, + binding_count: usize, + transactions: &str, + transaction_count: usize, + service_start: &str, + span: Span, + ) -> String { + let session = self.protocol_call( + "dever_rt_v1_api_session_new", + vec![ + format!("ptr {bindings}"), + format!("i64 {binding_count}"), + format!("ptr {transactions}"), + format!("i64 {transaction_count}"), + ], + "ptr", + span, + ); + self.line(format!("store ptr {session}, ptr @dever_api_session")); + if self.module.jobs_enabled() && self.module.application.as_ref().unwrap().test.is_none() { + let configure = self.label("application_configure_services"); + let ready = self.label("application_resources_ready"); + self.line(format!( + "br i1 {service_start}, label %{configure}, label %{ready}" + )); + self.start(&configure); + self.wire_write( + "dever_rt_v1_job_configure", + vec![ + format!("ptr {session}"), + format!( + "i8 {}", + u8::from( + !self.module.program.api_routes.is_empty() + || !self.module.program.api_rest.is_empty() + ) + ), + "i8 1".into(), + ], + ); + self.line(format!("br label %{ready}")); + self.start(&ready); + } + self.register_jobs(span); + let tenant_models = self + .module + .program + .models + .iter() + .enumerate() + .filter(|(_, model)| model.scope == crate::model::ModelScope::Tenant) + .collect::<Vec<_>>(); + let fingerprint = + crate::native::orm::tenant_fingerprint(&tenant_models, self.module.program); + let fingerprint = self.module.wire_names(&[fingerprint]); + let service = self.temp(); + self.line(format!("{service} = zext i1 {service_start} to i8")); + let operation = self.async_operation( + "api_initialize", + vec![ + format!("ptr {session}"), + format!("ptr {fingerprint}"), + "ptr @dever_permissions".into(), + format!("i64 {}", self.module.program.permissions.len()), + format!("i8 {service}"), + ], + span, + ); + self.await_operation(&operation, &Type::Unit, span, true); + session + } + + pub(super) fn serve_api(&mut self, span: Span) { + let session = self.temp(); + self.line(format!("{session} = load ptr, ptr @dever_api_session")); + let components = self + .module + .wire_names(&self.module.program.tenant_components.clone()); + let mut arguments = vec![ + format!("ptr {session}"), + "ptr @dever_api_routes".into(), + format!( + "i64 {}", + self.module.program.api_routes.len() + + self + .module + .application + .as_ref() + .unwrap() + .rest_handlers + .len() + ), + format!("ptr {components}"), + format!("i64 {}", self.module.program.tenant_components.len()), + ]; + let symbol = if self.module.jobs_enabled() { + let jobs = self.temp(); + self.line(format!("{jobs} = load ptr, ptr @dever_job_session")); + arguments.push(format!("ptr {jobs}")); + "api_serve_with_jobs" + } else { + "api_serve" + }; + let operation = self.async_operation(symbol, arguments, span); + self.await_operation(&operation, &Type::Unit, span, true); + } + + pub(super) fn begin_generated_function(&mut self, function: &crate::hir::Function) -> String { + let mut parameters = function + .parameters + .iter() + .enumerate() + .map(|(index, parameter)| { + format!( + "{} %p{index}, ", + self.module.ty(parameter.value_type().unwrap()) + ) + }) + .collect::<String>(); + if self.module.database_function(self.instance) { + parameters.push_str("ptr %database, "); + self.database_context = "%database".into(); + } + self.start("entry"); + self.prologue + .push_str(" %exit_status = alloca i32\n store i32 0, ptr %exit_status\n"); + self.begin_coroutine(function); + parameters + } + + pub(super) fn encode_api_output(&mut self, ty: &Type, row: &str, span: Span) -> String { + let encoder = self + .module + .wire_callback(ty, crate::wire::Policy::Output, true); + self.runtime_call( + "dever_rt_v1_wire_encode", + vec![ + format!("ptr {row}"), + format!("ptr @dever_type_{}", self.module.type_index(ty)), + format!("ptr @{encoder}"), + ], + &Type::Text, + false, + None, + span, + ) + } +} diff --git a/crates/dever-core/src/llvm/api_context.rs b/crates/dever-core/src/llvm/api_context.rs new file mode 100644 index 0000000..b4332bd --- /dev/null +++ b/crates/dever-core/src/llvm/api_context.rs @@ -0,0 +1,326 @@ +//! Request-only capabilities keep context and affine resources inside the runtime scope. +use super::*; + +pub(super) const DECLARATIONS: &str = "\ +%dever.api_cookie_options = type { ptr, ptr, i64, i8, i8, i64, i8 } +declare i32 @dever_rt_v1_api_request_id(ptr, ptr) +declare i32 @dever_rt_v1_api_method(ptr, ptr) +declare i32 @dever_rt_v1_api_path(ptr, ptr) +declare i32 @dever_rt_v1_api_client_address(ptr, ptr, ptr) +declare i32 @dever_rt_v1_api_header(ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_api_cookie(ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_api_secret_cookie(ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_api_response_header(ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_api_response_cookie(ptr, ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_api_response_secret_cookie(ptr, ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_auth_site_key(ptr, ptr) +declare i32 @dever_rt_v1_auth_id(ptr, ptr) +declare i32 @dever_rt_v1_auth_session(ptr, ptr) +declare i32 @dever_rt_v1_auth_user_id(ptr, ptr, ptr) +declare i32 @dever_rt_v1_auth_tenant_id(ptr, ptr, ptr) +declare i32 @dever_rt_v1_auth_owns_user(i64, ptr, ptr) +declare i32 @dever_rt_v1_auth_issue(ptr, ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_auth_issue_cookie(ptr, ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_auth_clear_cookie(ptr, ptr) +declare i32 @dever_rt_v1_upload_filename(ptr, ptr, ptr) +declare i32 @dever_rt_v1_upload_content_type(ptr, ptr, ptr) +declare i32 @dever_rt_v1_upload_size(ptr, ptr, ptr) +declare i32 @dever_rt_v1_upload_close_take(ptr, ptr, ptr) +declare ptr @dever_rt_v1_upload_store_take(ptr, ptr) +declare ptr @dever_rt_v1_auth_permissions(ptr, ptr, ptr) +declare ptr @dever_rt_v1_auth_save_role(ptr, ptr, i8, ptr, ptr) +declare ptr @dever_rt_v1_auth_grant_role(i64, ptr, ptr) +declare ptr @dever_rt_v1_auth_revoke_role(i64, ptr, ptr) +declare ptr @dever_rt_v1_auth_disable_role(ptr, ptr) +"; + +pub(super) fn supported(operation: Intrinsic) -> bool { + use Intrinsic::*; + matches!( + operation, + ApiServe + | ApiRequestId + | ApiMethod + | ApiPath + | ApiClientAddress + | ApiHeader + | ApiCookie + | ApiSecretCookie + | ApiResponseHeader + | ApiResponseCookie + | ApiResponseSecretCookie + | AuthIssue + | AuthIssueCookie + | AuthClearCookie + | AuthOwnsUser + | AuthId + | AuthSession + | AuthUserId + | AuthTenantId + | AuthPermissions + | AuthSaveRole + | AuthGrantRole + | AuthRevokeRole + | AuthDisableRole + | SiteKey + | UploadFilename + | UploadContentType + | UploadSize + | UploadClose + | UploadStore + ) +} + +impl FunctionEmitter<'_, '_> { + pub(super) fn api_intrinsic( + &mut self, + operation: Intrinsic, + arguments: &[Expression], + values: &[String], + result: &Expression, + ) -> String { + use Intrinsic::*; + let mut args = arguments + .iter() + .zip(values) + .map(|(argument, value)| format!("{} {value}", self.module.ty(&argument.ty))) + .collect::<Vec<_>>(); + let symbol = match operation { + ApiServe => { + self.serve_api(result.span); + return "zeroinitializer".into(); + } + ApiRequestId => "api_request_id", + ApiMethod => "api_method", + ApiPath => "api_path", + ApiClientAddress => "api_client_address", + ApiHeader => "api_header", + ApiCookie => "api_cookie", + ApiSecretCookie => "api_secret_cookie", + ApiResponseHeader => "api_response_header", + ApiResponseCookie | ApiResponseSecretCookie => { + let options = self.api_cookie_options(&arguments[2].ty, &values[2]); + args[2] = format!("ptr {options}"); + if operation == ApiResponseCookie { + "api_response_cookie" + } else { + "api_response_secret_cookie" + } + } + AuthIssue | AuthIssueCookie => { + let tenant = self.nullable_handle(&arguments[2].ty, &values[2]); + args[2] = format!("ptr {tenant}"); + if operation == AuthIssue { + "auth_issue" + } else { + "auth_issue_cookie" + } + } + AuthClearCookie => "auth_clear_cookie", + AuthOwnsUser => "auth_owns_user", + AuthId => "auth_id", + AuthSession => "auth_session", + AuthUserId => "auth_user_id", + AuthTenantId => "auth_tenant_id", + SiteKey => "auth_site_key", + UploadFilename => "upload_filename", + UploadContentType => "upload_content_type", + UploadSize => "upload_size", + UploadClose | UploadStore => { + let owned = self.temp(); + self.line(format!( + "{owned} = call ptr @dever_rt_v1_upload_retain(ptr {})", + values[0] + )); + args[0] = format!("ptr {owned}"); + if operation == UploadStore { + let operation = self.async_operation( + "upload_store_take", + vec![format!("ptr {owned}")], + result.span, + ); + return self.async_resource_result(&operation, result, "Stored", None); + } + "upload_close_take" + } + AuthPermissions | AuthSaveRole | AuthGrantRole | AuthRevokeRole | AuthDisableRole => { + return self.api_authorization_intrinsic(operation, arguments, values, result); + } + _ => unreachable!("checked request intrinsic"), + }; + let value = self.runtime_call( + &format!("dever_rt_v1_{symbol}"), + args, + &result.ty, + matches!(result.ty, Type::Nullable(_)), + matches!(result.ty, Type::Bool | Type::Unit).then_some("i8"), + result.span, + ); + if result.ty == Type::Unit { + "zeroinitializer".into() + } else { + value + } + } + + fn api_authorization_intrinsic( + &mut self, + operation: Intrinsic, + arguments: &[Expression], + values: &[String], + result: &Expression, + ) -> String { + use Intrinsic::*; + let mut args = arguments + .iter() + .zip(values) + .map(|(argument, value)| format!("{} {value}", self.module.ty(&argument.ty))) + .collect::<Vec<_>>(); + let symbol = match operation { + AuthPermissions => { + let Type::List(element) = &result.ty else { + unreachable!("checked permission list"); + }; + let Type::Named(id) = element.as_ref() else { + unreachable!(); + }; + let callback = self.module.api_permission_pack(*id); + args.extend([ + format!("ptr @dever_type_{}", self.module.type_index(element)), + format!("ptr {callback}"), + ]); + "auth_permissions" + } + AuthSaveRole => { + let flag = self.temp(); + self.line(format!("{flag} = zext i1 {} to i8", values[2])); + args[2] = format!("i8 {flag}"); + "auth_save_role" + } + AuthGrantRole => "auth_grant_role", + AuthRevokeRole => "auth_revoke_role", + AuthDisableRole => "auth_disable_role", + _ => unreachable!("authorization intrinsic"), + }; + let operation = self.async_operation(symbol, args, result.span); + self.await_operation(&operation, &result.ty, result.span, true) + } + + pub(super) fn nullable_handle(&mut self, ty: &Type, value: &str) -> String { + let present = self.temp(); + let handle = self.temp(); + let result = self.temp(); + self.line(format!( + "{present} = extractvalue {} {value}, 0", + self.module.ty(ty) + )); + self.line(format!( + "{handle} = extractvalue {} {value}, 1", + self.module.ty(ty) + )); + self.line(format!( + "{result} = select i1 {present}, ptr {handle}, ptr null" + )); + result + } + + fn api_cookie_options(&mut self, ty: &Type, value: &str) -> String { + let Type::Named(id) = ty else { + unreachable!("checked CookieOptions"); + }; + let Shape::Record(fields) = self.module.program.types[*id].shape.clone() else { + unreachable!(); + }; + let mut values = Vec::new(); + for (index, field) in fields.iter().enumerate() { + let part = self.temp(); + self.line(format!("{part} = extractvalue %T{id} {value}, {index}")); + values.push((field.ty.clone(), part)); + } + let domain = self.nullable_handle(&values[1].0, &values[1].1); + let Type::Named(same_site) = values[2].0 else { + unreachable!(); + }; + let Shape::Choice(variants) = self.module.program.types[same_site].shape.clone() else { + unreachable!(); + }; + let tag = self.temp(); + self.line(format!( + "{tag} = extractvalue %T{same_site} {}, 0", + values[2].1 + )); + let mut same_site = "0".to_owned(); + for (name, kind) in [("Lax", 1), ("None", 2)] { + let index = variants + .iter() + .position(|variant| variant.name == name) + .unwrap(); + let matches = self.temp(); + let selected = self.temp(); + self.line(format!("{matches} = icmp eq i32 {tag}, {index}")); + self.line(format!( + "{selected} = select i1 {matches}, i64 {kind}, i64 {same_site}" + )); + same_site = selected; + } + let secure = self.temp(); + let http_only = self.temp(); + let max_present = self.temp(); + let max_age = self.temp(); + let max_flag = self.temp(); + self.line(format!("{secure} = zext i1 {} to i8", values[3].1)); + self.line(format!("{http_only} = zext i1 {} to i8", values[4].1)); + self.line(format!( + "{max_present} = extractvalue {{ i1, i64 }} {}, 0", + values[5].1 + )); + self.line(format!( + "{max_age} = extractvalue {{ i1, i64 }} {}, 1", + values[5].1 + )); + self.line(format!("{max_flag} = zext i1 {max_present} to i8")); + let slot = self.entry_slot_ir("%dever.api_cookie_options"); + let mut row = "zeroinitializer".to_owned(); + for (index, field) in [ + format!("ptr {}", values[0].1), + format!("ptr {domain}"), + format!("i64 {same_site}"), + format!("i8 {secure}"), + format!("i8 {http_only}"), + format!("i64 {max_age}"), + format!("i8 {max_flag}"), + ] + .iter() + .enumerate() + { + let next = self.temp(); + self.line(format!( + "{next} = insertvalue %dever.api_cookie_options {row}, {field}, {index}" + )); + row = next; + } + self.line(format!("store %dever.api_cookie_options {row}, ptr {slot}")); + slot + } +} + +impl Module<'_> { + fn api_permission_pack(&mut self, id: usize) -> String { + let name = format!("@dever_permission_pack_{id}"); + // The compiler-owned callback is reused by every call of permissions(). + if self.api_permission_types.insert(id) { + let mut body = format!( + "define internal void {name}(ptr %key, ptr %component, ptr %domain, ptr %site, ptr %action, ptr %method, ptr %out) {{\nentry:\n" + ); + for (index, field) in ["key", "component", "domain", "site", "action", "method"] + .iter() + .enumerate() + { + writeln!(body, " %owned{index} = call ptr @dever_rt_v1_text_retain(ptr %{field})\n %field{index} = getelementptr %T{id}, ptr %out, i32 0, i32 {index}\n store ptr %owned{index}, ptr %field{index}").unwrap(); + } + body.push_str(" ret void\n}\n"); + self.declarations.push_str(&body); + } + name + } +} diff --git a/crates/dever-core/src/llvm/application.rs b/crates/dever-core/src/llvm/application.rs new file mode 100644 index 0000000..d8cd7ff --- /dev/null +++ b/crates/dever-core/src/llvm/application.rs @@ -0,0 +1,798 @@ +//! Compiler-owned application roots. Settings live until the root Scope drains. +use super::*; + +/// Textual code and borrowed binary initializers form one compilation input. +pub struct ResourceModule<'a> { + pub ir: String, + pub resources: Vec<BinaryResource<'a>>, +} + +pub struct BinaryResource<'a> { + pub symbol: String, + pub bytes: &'a [u8], +} + +#[derive(Clone)] +pub(super) struct Application { + pub root: usize, + pub ports: Vec<usize>, + pub adapters: Vec<usize>, + pub api_handlers: Vec<usize>, + pub command_handlers: Vec<usize>, + pub rest_handlers: Vec<rest::Handler>, + pub rest_error: Option<usize>, + pub job_handlers: Vec<usize>, + pub test: Option<usize>, + pub resources: Option<(String, usize)>, + pub executable: bool, +} + +/// Emit the checked production application entry, independently of ordinary kernels. +/// The caller supplies Unit output and the module's fault slot, then releases +/// both using the same generated ownership exports as other LLVM entries. +pub fn emit_application(program: &Program, sources: &SourceMap) -> Result<String, String> { + emit_application_with_resources(program, sources, &[]).map(|module| module.ir) +} + +/// Embed already prepared Worker resources; registry resolution stays with the packager. +pub fn emit_application_with_resources<'a>( + program: &Program, + sources: &SourceMap, + resources: &'a [crate::native::EmbeddedResource], +) -> Result<ResourceModule<'a>, String> { + emit_program_with_resources(program, sources, None, resources, false) +} + +/// Emit a complete production executable with the C ABI process entry. +pub fn emit_executable(program: &Program, sources: &SourceMap) -> Result<String, String> { + emit_executable_with_resources(program, sources, &[]).map(|module| module.ir) +} + +/// The same application entry with already verified, immutable Worker resources. +pub fn emit_executable_with_resources<'a>( + program: &Program, + sources: &SourceMap, + resources: &'a [crate::native::EmbeddedResource], +) -> Result<ResourceModule<'a>, String> { + emit_program_with_resources(program, sources, None, resources, true) +} + +pub(super) fn emit_program( + program: &Program, + sources: &SourceMap, + test: Option<usize>, +) -> Result<String, String> { + emit_program_with_resources(program, sources, test, &[], false).map(|module| module.ir) +} + +fn emit_program_with_resources<'a>( + program: &Program, + sources: &SourceMap, + test: Option<usize>, + resources: &'a [crate::native::EmbeddedResource], + executable: bool, +) -> Result<ResourceModule<'a>, String> { + crate::native::validate_embedded_resources(resources)?; + validate_http_bindings(program, sources)?; + let span = test + .map(|index| program.tests[index].span) + .or_else(|| { + program + .api_routes + .first() + .map(|route| route.span) + .or_else(|| program.api_rest.first().map(|route| route.span)) + .or_else(|| { + program + .api_commands + .first() + .map(|command| program.functions[command.function].span) + }) + .or_else(|| { + program + .jobs + .first() + .map(|job| program.functions[job.function].span) + }) + }) + .ok_or_else(|| "LLVM application has no API, REST, CMD, or Job entry".to_owned())?; + let mut checked = program.clone(); + let (rest_handlers, rest_error) = rest::prepare(&mut checked); + let roots = program + .api_commands + .iter() + .map(|command| command.function) + .chain(program.api_routes.iter().map(|route| route.function)) + .chain(program.auth.iter().map(|hook| hook.function)) + .chain(rest_handlers.iter().map(|handler| handler.function)) + .chain( + program + .jobs + .iter() + .filter(|_| test.is_none()) + .map(|job| job.function), + ) + .chain(test.map(|index| program.tests[index].function)) + .map(|function| Specialization { + function, + handlers: Vec::new(), + }) + .collect::<Vec<_>>(); + let mut instances = specialize::execution_reachable(&checked.functions, roots) + .map_err(|span| located(sources, span, "recursive specialized call"))?; + checked.jobs.retain(|job| { + instances + .iter() + .any(|instance| instance.function == job.function) + }); + for instance in &instances { + if instance.function < program.functions.len() { + validate_function(&checked, sources, instance, true)?; + } + } + let ports = instances + .iter() + .filter_map(|instance| { + checked.functions[instance.function] + .port + .as_ref() + .map(|_| instance.function) + }) + .collect::<Vec<_>>(); + let owners = ports + .iter() + .map(|port| program.functions[*port].owner) + .collect::<BTreeSet<_>>(); + let adapters = program + .adapters + .iter() + .enumerate() + .filter_map(|(id, adapter)| { + (test.is_none() && !adapter.fake && owners.contains(&adapter.port_owner)).then_some(id) + }) + .collect::<Vec<_>>(); + for owner in owners { + if test.is_some() { + continue; + } + if !adapters + .iter() + .any(|id| program.adapters[*id].port_owner == owner) + { + return Err(located( + sources, + span, + "reachable Port has no production Adapter", + )); + } + } + // The synthetic root has no source-callable identity. Its body is generated + // below; the existing descriptors still own all concrete entry callees. + let mut command_handlers = Vec::new(); + for (index, command) in program.api_commands.iter().enumerate() { + let original = &program.functions[command.function]; + let mut function = generated_function(format!("<generated CMD {index}>"), original.span); + function.parameters = vec![Parameter::Value(Type::Text)]; + let id = append_handler(&mut checked, command.function, function); + command_handlers.push(id); + instances.insert(Specialization { + function: id, + handlers: Vec::new(), + }); + } + let mut api_handlers = Vec::new(); + for (index, route) in program.api_routes.iter().enumerate() { + let original = &program.functions[route.function]; + let mut function = generated_function(format!("<generated API {index}>"), route.span); + function.parameters = original.parameters.clone(); + function.outputs = vec![crate::types::Field { + name: "encoded".into(), + ty: Type::Text, + private: false, + bounds: Vec::new(), + }]; + let id = append_handler(&mut checked, route.function, function); + api_handlers.push(id); + instances.insert(Specialization { + function: id, + handlers: Vec::new(), + }); + } + let mut job_handlers = Vec::new(); + for job in checked.jobs.clone() { + let original = checked.functions[job.function].clone(); + let mut function = + generated_function(format!("<generated Job {}>", job.function), original.span); + function.parameters = original.parameters.clone(); + let id = append_handler(&mut checked, job.function, function); + job_handlers.push(id); + instances.insert(Specialization { + function: id, + handlers: Vec::new(), + }); + } + let root = append_generated( + &mut checked, + generated_function("<generated application entry>".into(), span), + ); + checked.failures[root] = instances + .iter() + .flat_map(|instance| specialize::failures(&checked, instance)) + .collect(); + let database = test.map_or( + !checked.models.is_empty() || !checked.jobs.is_empty(), + |index| checked.tests[index].uses_database(), + ); + if database { + checked.failures[root].extend(crate::capture::database_errors(&checked.types)); + } + let root_instance = Specialization { + function: root, + handlers: Vec::new(), + }; + instances.insert(root_instance.clone()); + let mut module = Module::new(&checked, sources, instances); + module.collect_external_types(); + for handler in &rest_handlers { + let decoded = Type::Outputs( + handler.inputs[usize::from(handler.detail)..] + .iter() + .enumerate() + .map(|(index, input)| crate::types::Field { + name: format!("p{index}"), + ty: input.ty.clone(), + private: false, + bounds: Vec::new(), + }) + .collect(), + ); + collect_owned_types(&checked, &decoded, &mut module.value_types); + } + for model in checked.models.iter().filter(|_| database) { + for id in [model.record, model.page, model.cursor] + .into_iter() + .chain(model.sql.iter().map(|sql| sql.result.record)) + { + collect_types(&checked, &Type::Named(id), &mut module.type_ids); + collect_owned_types(&checked, &Type::Named(id), &mut module.value_types); + } + for ty in [ + Type::List(Box::new(Type::Named(model.record))), + Type::Nullable(Box::new(Type::Named(model.record))), + Type::Map( + Box::new(Type::Int), + Box::new(Type::List(Box::new(Type::Named(model.record)))), + ), + ] { + collect_owned_types(&checked, &ty, &mut module.value_types); + } + } + for ty in [ + Type::Text, + Type::Json, + Type::Bool, + Type::Int, + Type::List(Box::new(Type::Text)), + Type::Nullable(Box::new(Type::Text)), + ] { + collect_owned_types(&checked, &ty, &mut module.value_types); + } + for adapter in &adapters { + if let Some(id) = checked.adapters[*adapter].setting { + let ty = Type::Named(id); + collect_types(&checked, &ty, &mut module.type_ids); + collect_owned_types(&checked, &ty, &mut module.value_types); + } + } + module.application = Some(Application { + root, + ports, + adapters, + api_handlers, + command_handlers, + rest_handlers, + rest_error, + job_handlers, + test, + resources: None, + executable, + }); + module.validate_async_boundaries(sources)?; + module.emit_types()?; + module.declarations.push_str(wire::DECLARATIONS); + module.declarations.push_str(ports::DECLARATIONS); + let inputs = crate::native::resource_inputs(resources); + let attachments = inputs + .unique + .iter() + .enumerate() + .map(|(index, resource)| BinaryResource { + symbol: format!("dever_external_bytes_{index}"), + bytes: resource.bytes.as_slice(), + }) + .collect::<Vec<_>>(); + module.emit_external_descriptors(resources, &inputs.indices, &attachments); + if module.api_enabled() { + module.declarations.push_str(api::DECLARATIONS); + module.declarations.push_str(api_context::DECLARATIONS); + module.emit_api_descriptors(); + } + if module.database_enabled() { + module.declarations.push_str(database::DECLARATIONS); + module.emit_database_slots(); + } + module.emit_job_descriptors(); + module.declarations.push_str("declare i32 @dever_rt_v1_text_copy_utf8(ptr, i64, ptr)\ndeclare i32 @dever_rt_v1_text_export(ptr, ptr)\n"); + module.emit_port_slots(); + module.emit_functions()?; + module.emit_named_entry(&root_instance, "dever_application_run", true); + module.emit_application_entry(&root_instance); + if test.is_some() { + module.emit_test_exports(&root_instance); + } + if executable { + module.emit_process_entry(&root_instance); + } + Ok(ResourceModule { + ir: module.finish(), + resources: attachments, + }) +} + +fn validate_http_bindings(program: &Program, sources: &SourceMap) -> Result<(), String> { + for route in &program.api_routes { + if route.auth.is_none() || (!route.anonymous && route.permission.is_none()) { + return Err(located( + sources, + route.span, + "LLVM API application requires a configured site, auth hook, and protected-route permission; check the application with settings", + )); + } + } + for route in &program.api_rest { + if route.auth.is_none() + || [ + route.permissions.read, + route.permissions.create, + route.permissions.replace, + route.permissions.delete, + ] + .iter() + .any(Option::is_none) + { + return Err(located( + sources, + route.span, + "LLVM REST application requires a configured site, auth hook, and all four permissions; check the application with settings", + )); + } + } + Ok(()) +} + +pub(super) fn generated_function(name: String, span: Span) -> crate::hir::Function { + crate::hir::Function { + name, + owner: 0, + public: false, + parameters: Vec::new(), + outputs: Vec::new(), + clauses: Vec::new(), + span, + suspends: true, + pure: false, + recovery: None, + kind: FunctionKind::Ordinary, + port: None, + implementation: false, + setting: None, + external: None, + } +} + +pub(super) fn append_generated(program: &mut Program, function: crate::hir::Function) -> usize { + let id = program.functions.len(); + program.functions.push(function); + program.effects.push(BTreeSet::new()); + program.recoveries.push(BTreeSet::new()); + program.suspension_handlers.push(BTreeSet::new()); + program.failure_handlers.push(BTreeSet::new()); + program.database_effects.push(BTreeSet::new()); + program.failures.push(BTreeSet::new()); + id +} + +fn append_handler( + program: &mut Program, + target: usize, + mut function: crate::hir::Function, +) -> usize { + let original = &program.functions[target]; + function.owner = original.owner; + // Generated bodies keep their original checked callee's transitive + // Send, failure, and database effects through this analysis-only edge. + function.clauses = generated_call_edge(original, target); + let id = append_generated(program, function); + program.effects[id] = program.effects[target].clone(); + program.failures[id] = program.failures[target].clone(); + program.database_effects[id] = program.database_effects[target].clone(); + id +} + +fn generated_call_edge(function: &crate::hir::Function, target: usize) -> Vec<crate::hir::Clause> { + vec![crate::hir::Clause { + patterns: Vec::new(), + bindings: Vec::new(), + locals: Vec::new(), + outputs: Vec::new(), + terminates: false, + body: vec![Statement::Call(Expression { + kind: ExpressionKind::Call { + target: CallTarget::Function(target), + arguments: function + .parameters + .iter() + .enumerate() + .map(|(index, parameter)| { + CallArgument::Value(Expression { + kind: ExpressionKind::Local(index), + ty: parameter.value_type().expect("entry value input").clone(), + span: function.span, + }) + }) + .collect(), + }, + ty: output_type(&function.outputs), + span: function.span, + })], + }] +} + +impl Module<'_> { + pub(super) fn emit_command_body( + &mut self, + instance: &Specialization, + index: usize, + ) -> Result<(), String> { + let function = self.program.functions[instance.function].clone(); + let command = self.program.api_commands[index].clone(); + let mut emitter = FunctionEmitter::new(self, instance); + let parameters = emitter.begin_generated_function(&function); + let body = emitter.parameter(0); + emitter.emit_command(&command, index, &body)?; + let name = emitter.module.names[instance].clone(); + let body = emitter.finish_function(&name, ¶meters); + self.functions.push_str(&body); + Ok(()) + } + pub(super) fn emit_application_body( + &mut self, + instance: &Specialization, + ) -> Result<(), String> { + let function = self.program.functions[instance.function].clone(); + let commands = self.program.api_commands.clone(); + let mut emitter = FunctionEmitter::new(self, instance); + emitter.start("entry"); + emitter + .prologue + .push_str(" %exit_status = alloca i32\n store i32 0, ptr %exit_status\n"); + emitter.begin_coroutine(&function); + let span = function.span; + emitter.initialize_clock(span); + emitter.initialize_external_resources(span); + let arguments_ty = Type::List(Box::new(Type::Text)); + let arguments = emitter.runtime_call( + "dever_rt_v1_process_arguments", + vec![format!( + "ptr @dever_type_{}", + emitter.module.type_index(&Type::Text) + )], + &arguments_ty, + false, + None, + span, + ); + emitter.own_value(&arguments_ty, &arguments, false); + let length = emitter.runtime_call( + "dever_rt_v1_list_length", + vec![format!("ptr {arguments}")], + &Type::Int, + false, + None, + span, + ); + let empty = emitter.temp(); + emitter.line(format!("{empty} = icmp eq i64 {length}, 0")); + let migration = emitter.initialize_database(span, &empty); + emitter.line(format!( + "br i1 {empty}, label %command_empty, label %command_check_arity" + )); + emitter.start("command_empty"); + emitter.initialize_ports(span); + if !emitter.module.program.api_routes.is_empty() + || !emitter.module.program.api_rest.is_empty() + || emitter.module.jobs_enabled() + { + emitter.serve_api(span); + emitter.exit("0"); + } else { + emitter.application_fault( + "application has no service entry; select a CMD and JSON input", + span, + ); + } + emitter.start("command_check_arity"); + let retained = emitter.temp(); + emitter.line(format!( + "{retained} = call ptr @dever_rt_v1_list_retain(ptr {arguments})" + )); + let cursor = emitter.protocol_call( + "dever_rt_v1_list_cursor_take", + vec![format!("ptr {retained}")], + "ptr", + span, + ); + let cursor_slot = emitter.entry_slot_ir("ptr"); + emitter.line(format!("store ptr {cursor}, ptr {cursor_slot}")); + let guard = emitter.register_owned(OwnedKind::ListCursor, cursor_slot); + emitter.mark_live(guard); + let first = emitter.application_argument(&cursor, span); + let (name, body, tenant) = emitter.application_command_arguments( + &first, + &cursor, + &length, + migration.as_ref(), + span, + )?; + emitter.release_guard(guard); + emitter.initialize_ports(span); + for (index, command) in commands.iter().enumerate() { + let command_name = emitter.application_text(&command.name, span); + let equal = emitter.equals(&Type::Text, &name, &command_name)?; + emitter.line(format!( + "br i1 {equal}, label %command_{index}, label %command_next_{index}" + )); + emitter.start(&format!("command_{index}")); + let function = emitter + .module + .application + .as_ref() + .unwrap() + .command_handlers[index]; + let target = Specialization { + function, + handlers: Vec::new(), + }; + emitter.application_invoke_command(command, &target, &body, &tenant, span); + emitter.exit("0"); + emitter.start(&format!("command_next_{index}")); + } + let prefix = emitter.application_text("unknown CMD '", span); + let suffix = emitter.application_text("'", span); + let message = emitter.application_concat(&prefix, &name, span); + let message = emitter.application_concat(&message, &suffix, span); + emitter.application_fault_text(&message, span); + let name = emitter.module.names[instance].clone(); + let body = emitter.finish_function(&name, ""); + self.functions.push_str(&body); + Ok(()) + } + + fn emit_application_entry(&mut self, root: &Specialization) { + let app = self.application.clone().unwrap(); + let gated = !app.ports.is_empty() + || self.database_enabled() + || self.api_enabled() + || self.clock_enabled(); + let span = self.program.functions[root.function].span; + self.functions.push_str("define i32 @dever_application_entry(ptr %out, ptr %fault) {\nentry:\n store {} zeroinitializer, ptr %out\n store %dever.fault zeroinitializer, ptr %fault\n"); + if gated { + self.declarations + .push_str("@dever_application_active = private global i8 0\n"); + let (message, length) = self.text_literal("application entry is already active"); + let location = self.location(span); + writeln!(self.functions, " %gate = cmpxchg ptr @dever_application_active, i8 0, i8 1 acq_rel acquire\n %entered = extractvalue {{ i8, i1 }} %gate, 1\n br i1 %entered, label %run, label %busy\nbusy:\n %message = getelementptr %dever.fault, ptr %fault, i32 0, i32 4\n %copied = call i32 @dever_rt_v1_text_copy_utf8(ptr {message}, i64 {length}, ptr %message)\n %code = getelementptr %dever.fault, ptr %fault, i32 0, i32 0\n store i32 3, ptr %code\n %origin = getelementptr %dever.fault, ptr %fault, i32 0, i32 1\n store i32 {location}, ptr %origin\n ret i32 1\nrun:").unwrap(); + } + self.functions + .push_str(" %status = call i32 @dever_application_run(ptr %out, ptr %fault)\n"); + // async_root has drained every child before these immutable globals can + // be destroyed or another entry can publish a new Adapter selection. + for adapter in app.adapters { + let adapter = &self.program.adapters[adapter]; + if let Some(id) = adapter.setting { + let owner = adapter.owner; + let ty = self.type_index(&Type::Named(id)); + writeln!(self.functions, " %live{owner} = load i1, ptr @dever_setting_live_{owner}\n br i1 %live{owner}, label %drop_setting{owner}, label %setting_done{owner}\ndrop_setting{owner}:\n call void @dever_drop_{ty}(ptr @dever_setting_{owner})\n store %T{id} zeroinitializer, ptr @dever_setting_{owner}\n store i1 false, ptr @dever_setting_live_{owner}\n br label %setting_done{owner}\nsetting_done{owner}:").unwrap(); + } + } + for port in app.ports { + writeln!(self.functions, " store i64 0, ptr @dever_port_{port}").unwrap(); + } + if self.clock_enabled() { + self.functions.push_str(" %job_clock = load ptr, ptr @dever_job_clock\n call void @dever_rt_v1_job_clock_release(ptr %job_clock)\n store ptr null, ptr @dever_job_clock\n"); + } + if self.jobs_enabled() { + self.functions.push_str(" %jobs = load ptr, ptr @dever_job_session\n call void @dever_rt_v1_job_session_release(ptr %jobs)\n store ptr null, ptr @dever_job_session\n"); + } + if self.database_enabled() { + for index in 0..self.program.models.len() { + writeln!(self.functions, " %database{index} = load ptr, ptr @dever_database_{index}\n call void @dever_rt_v1_db_release(ptr %database{index})\n store ptr null, ptr @dever_database_{index}").unwrap(); + } + self.functions.push_str(" %session = load ptr, ptr @dever_database_session\n call void @dever_rt_v1_db_session_release(ptr %session)\n store ptr null, ptr @dever_database_session\n"); + } + if self.api_enabled() { + self.functions.push_str(" %api_session = load ptr, ptr @dever_api_session\n call void @dever_rt_v1_api_session_release(ptr %api_session)\n store ptr null, ptr @dever_api_session\n"); + } + if gated { + self.functions + .push_str(" store atomic i8 0, ptr @dever_application_active release, align 1\n"); + } + self.functions.push_str(" ret i32 %status\n}\n"); + } +} + +impl FunctionEmitter<'_, '_> { + pub(super) fn application_text(&mut self, text: &str, span: Span) -> String { + let (pointer, length) = self.module.text_literal(text); + let value = self.runtime_call( + "dever_rt_v1_text_new", + vec![format!("ptr {pointer}"), format!("i64 {length}")], + &Type::Text, + false, + None, + span, + ); + self.own_value(&Type::Text, &value, false); + value + } + + fn application_concat(&mut self, left: &str, right: &str, span: Span) -> String { + let value = self.runtime_call( + "dever_rt_v1_text_concat", + vec![format!("ptr {left}"), format!("ptr {right}")], + &Type::Text, + false, + None, + span, + ); + self.own_value(&Type::Text, &value, false); + value + } + + pub(super) fn application_fault(&mut self, message: &str, span: Span) { + let message = self.application_text(message, span); + self.application_fault_text(&message, span); + } + + pub(super) fn application_fault_text(&mut self, message: &str, span: Span) { + let buffer = self.entry_slot_ir("{ ptr, i64 }"); + let status = self.temp(); + self.line(format!( + "{status} = call i32 @dever_rt_v1_text_export(ptr {message}, ptr {buffer})" + )); + self.check_runtime_status(&status, &buffer, span); + self.check_runtime_status("1", &buffer, span); + self.line("unreachable"); + } + + pub(super) fn application_argument(&mut self, cursor: &str, span: Span) -> String { + let value = self.runtime_call( + "dever_rt_v1_list_cursor_next", + vec![format!("ptr {cursor}")], + &Type::Nullable(Box::new(Type::Text)), + true, + None, + span, + ); + let argument = self.temp(); + self.line(format!( + "{argument} = extractvalue {{ i1, ptr }} {value}, 1" + )); + self.own_value(&Type::Text, &argument, false); + argument + } + + fn emit_command( + &mut self, + command: &crate::hir::ApiCommand, + index: usize, + body: &str, + ) -> Result<(), String> { + let instance = Specialization { + function: command.function, + handlers: Vec::new(), + }; + let function = self.module.program.functions[command.function].clone(); + let span = function.span; + self.application_require_components(&command.components, span); + let input_ty = asynchronous::input_type(&function); + let mut fields = Vec::new(); + for (name, parameter) in command.inputs.iter().zip(&function.parameters) { + let ty = parameter + .value_type() + .expect("checked CMD value parameter") + .clone(); + let callback = self + .module + .wire_callback(&ty, crate::wire::Policy::ApiInput, false); + fields.push((name.clone(), ty, callback)); + } + let callback_name = format!("dever_command_decode_{index}"); + let mut decoder = FunctionEmitter::new(self.module, &instance); + decoder.asynchronous = false; + decoder.start("entry"); + let row = decoder.decode_fields(&input_ty, &fields, "%inputs", wire::FieldInput::Command); + decoder.line(format!( + "store {} {row}, ptr %out", + decoder.module.ty(&input_ty) + )); + let callback = + decoder.finish_protocol_callback(&callback_name, "ptr %inputs, ptr %out, ptr %error"); + self.module.declarations.push_str(&callback); + let input = self.runtime_call( + "dever_rt_v1_cmd_decode", + vec![ + format!("ptr {body}"), + format!("ptr @dever_type_{}", self.module.type_index(&input_ty)), + format!("ptr @{callback_name}"), + ], + &input_ty, + false, + None, + span, + ); + self.own_value(&input_ty, &input, false); + let mut args = Vec::new(); + for (index, (_, ty, _)) in fields.iter().enumerate() { + let value = self.temp(); + self.line(format!( + "{value} = extractvalue {} {input}, {index}", + self.module.ty(&input_ty) + )); + args.push((ty.clone(), value)); + } + let output_ty = output_type(&function.outputs); + let transactional = self.begin_command_transaction(&instance, span); + let (status, row) = self.invoke_values(&instance, &args, &output_ty, span); + self.propagate_status(&status, span, false); + let value = self.temp(); + self.line(format!( + "{value} = load {}, ptr {row}", + self.module.ty(&output_ty) + )); + self.own_value(&output_ty, &value, false); + let encoder = self + .module + .wire_callback(&output_ty, crate::wire::Policy::Output, true); + let encoded = self.runtime_call( + "dever_rt_v1_wire_encode", + vec![ + format!("ptr {row}"), + format!("ptr @dever_type_{}", self.module.type_index(&output_ty)), + format!("ptr @{encoder}"), + ], + &Type::Text, + false, + None, + span, + ); + self.own_value(&Type::Text, &encoded, false); + if transactional { + self.commit_command_transaction(); + } + let prefix = self.application_text("{\"code\":0,\"message\":\"ok\",\"data\":", span); + let suffix = self.application_text("}\n", span); + let text = self.application_concat(&prefix, &encoded, span); + let text = self.application_concat(&text, &suffix, span); + self.runtime_call( + "dever_rt_v1_stdout_write", + vec![format!("ptr {text}")], + &Type::Bool, + false, + Some("i8"), + span, + ); + self.exit("0"); + Ok(()) + } +} diff --git a/crates/dever-core/src/llvm/application_commands.rs b/crates/dever-core/src/llvm/application_commands.rs new file mode 100644 index 0000000..02c07c8 --- /dev/null +++ b/crates/dever-core/src/llvm/application_commands.rs @@ -0,0 +1,340 @@ +//! Normalize the existing application CLI into one typed CMD dispatch path. +use super::*; + +impl FunctionEmitter<'_, '_> { + pub(super) fn application_command_arguments( + &mut self, + first: &str, + cursor: &str, + length: &str, + migration: Option<&database_schema::Migration>, + span: Span, + ) -> Result<(String, String, String), String> { + let name_slot = self.entry_slot(&Type::Text); + let body_slot = self.entry_slot(&Type::Text); + let tenant_slot = self.entry_slot(&Type::Int); + self.line(format!("store i64 0, ptr {tenant_slot}")); + if self.module.api_enabled() { + self.application_management(first, cursor, length, migration, span)?; + let flag = self.application_text("--dever-tenant-cmd", span); + let matches = self.equals(&Type::Text, first, &flag)?; + self.line(format!( + "br i1 {matches}, label %command_tenant, label %command_plain" + )); + self.start("command_tenant"); + self.application_arity(length, 4, "expected --dever-tenant-cmd, one positive tenant id, CMD name, and one JSON object", span); + let tenant = self.application_positive_argument( + cursor, + "tenant id must be a positive integer", + span, + ); + let name = self.application_argument(cursor, span); + let body = self.application_argument(cursor, span); + self.line(format!("store ptr {name}, ptr {name_slot}")); + self.line(format!("store ptr {body}, ptr {body_slot}")); + self.line(format!("store i64 {tenant}, ptr {tenant_slot}")); + self.line("br label %command_dispatch"); + self.start("command_plain"); + } + self.application_arity(length, 2, "expected CMD name and one JSON object", span); + let body = self.application_argument(cursor, span); + self.line(format!("store ptr {first}, ptr {name_slot}")); + self.line(format!("store ptr {body}, ptr {body_slot}")); + self.line("br label %command_dispatch"); + self.start("command_dispatch"); + let name = self.temp(); + let body = self.temp(); + let tenant = self.temp(); + self.line(format!("{name} = load ptr, ptr {name_slot}")); + self.line(format!("{body} = load ptr, ptr {body_slot}")); + self.line(format!("{tenant} = load i64, ptr {tenant_slot}")); + Ok((name, body, tenant)) + } + + fn application_management( + &mut self, + first: &str, + cursor: &str, + length: &str, + migration: Option<&database_schema::Migration>, + span: Span, + ) -> Result<(), String> { + for (flag, arity) in [ + ("--dever-tenant-migrate", 2), + ("--dever-tenant-owner", 4), + ("--dever-tenant-component", 4), + ] { + if flag == "--dever-tenant-owner" && self.module.program.permissions.is_empty() { + continue; + } + let expected = self.application_text(flag, span); + let matches = self.equals(&Type::Text, first, &expected)?; + let matched = self.label("management"); + let next = self.label("management_next"); + self.line(format!("br i1 {matches}, label %{matched}, label %{next}")); + self.start(&matched); + let usage = match flag { + "--dever-tenant-migrate" => { + "expected --dever-tenant-migrate and one positive tenant id" + } + "--dever-tenant-owner" => { + "expected --dever-tenant-owner, one positive tenant id, one site, and one positive user id" + } + _ => { + "expected --dever-tenant-component, one positive tenant id, enable or disable, and one component" + } + }; + self.application_arity(length, arity, usage, span); + let tenant = self.application_positive_argument( + cursor, + "tenant id must be a positive integer", + span, + ); + let session = self.application_session(); + match flag { + "--dever-tenant-migrate" => { + if let Some(migration) = migration { + let mut arguments = vec![ + format!("ptr {session}"), + format!("ptr {}", migration.bindings), + format!("ptr {}", migration.models), + format!("ptr {}", migration.errors), + format!("i64 {}", migration.count), + ]; + let symbol = if self.module.jobs_enabled() { + let errors = self.database_errors(span); + arguments.push(format!("ptr {errors}")); + "api_migrate_application" + } else { + "api_migrate_models" + }; + arguments.push(format!("i64 {tenant}")); + let operation = self.async_operation(symbol, arguments, span); + self.await_operation(&operation, &Type::Unit, span, false); + self.exit("0"); + } else { + self.application_fault("application has no database runtime", span); + } + } + "--dever-tenant-owner" => { + let site = self.application_argument(cursor, span); + let user = self.application_positive_argument( + cursor, + "user id must be a positive integer", + span, + ); + let operation = self.async_operation( + "api_tenant_owner", + vec![ + format!("ptr {session}"), + format!("i64 {tenant}"), + format!("ptr {site}"), + format!("i64 {user}"), + ], + span, + ); + self.await_operation(&operation, &Type::Unit, span, false); + self.exit("0"); + } + _ => { + if self.module.program.tenant_components.is_empty() { + self.application_fault("application has no tenant components", span); + } else { + let action = self.application_argument(cursor, span); + let component = self.application_argument(cursor, span); + let enable = self.application_text("enable", span); + let disable = self.application_text("disable", span); + let enabled = self.equals(&Type::Text, &action, &enable)?; + let disabled = self.equals(&Type::Text, &action, &disable)?; + let valid = self.temp(); + self.line(format!("{valid} = or i1 {enabled}, {disabled}")); + self.application_require( + &valid, + "operation must be enable or disable", + span, + ); + let byte = self.temp(); + self.line(format!("{byte} = zext i1 {enabled} to i8")); + let manifest = self + .module + .wire_names(&self.module.program.tenant_components.clone()); + let operation = self.async_operation( + "api_tenant_component", + vec![ + format!("ptr {session}"), + format!("i64 {tenant}"), + format!("ptr {component}"), + format!("i8 {byte}"), + format!("ptr {manifest}"), + format!("i64 {}", self.module.program.tenant_components.len()), + ], + span, + ); + self.await_operation(&operation, &Type::Unit, span, false); + self.exit("0"); + } + } + } + self.start(&next); + } + Ok(()) + } + + pub(super) fn application_invoke_command( + &mut self, + command: &crate::hir::ApiCommand, + target: &Specialization, + body: &str, + tenant: &str, + span: Span, + ) { + if self.module.api_enabled() { + let selected = self.temp(); + self.line(format!("{selected} = icmp sgt i64 {tenant}, 0")); + let scoped = self.label("command_scoped"); + let plain = self.label("command_unscoped"); + self.line(format!("br i1 {selected}, label %{scoped}, label %{plain}")); + self.start(&scoped); + if command.components.is_empty() { + self.application_fault( + "selected CMD does not use tenant storage; remove --tenant", + span, + ); + } else { + let configured = self.application_has_tenant(span); + self.application_require(&configured, "tenant storage is not configured", span); + let input = + asynchronous::input_type(&self.module.program.functions[target.function]); + let row = self.temp(); + self.line(format!( + "{row} = insertvalue {} zeroinitializer, ptr {body}, 0", + self.module.ty(&input) + )); + let pointer = self.row_pointer(&input, &row); + if self.module.jobs_enabled() { + self.invoke_system(target, &pointer, tenant, span); + } else { + let function = self.module.names[target].clone(); + let operation = self.async_operation( + "api_tenant_scope", + vec![ + format!("i64 {tenant}"), + format!("ptr @dever_async_{function}"), + format!("ptr {pointer}"), + ], + span, + ); + self.await_operation(&operation, &Type::Unit, span, false); + } + self.exit("0"); + } + self.start(&plain); + if !command.components.is_empty() { + let configured = self.application_has_tenant(span); + let ordinary = self.temp(); + self.line(format!("{ordinary} = xor i1 {configured}, true")); + self.application_require( + &ordinary, + "selected CMD requires an explicit tenant; use dever run --tenant", + span, + ); + } + } + if self.module.jobs_enabled() { + let input = asynchronous::input_type(&self.module.program.functions[target.function]); + let row = self.temp(); + self.line(format!( + "{row} = insertvalue {} zeroinitializer, ptr {body}, 0", + self.module.ty(&input) + )); + let pointer = self.row_pointer(&input, &row); + self.invoke_system(target, &pointer, "0", span); + } else { + let (status, _) = + self.invoke_values(target, &[(Type::Text, body.into())], &Type::Unit, span); + self.propagate_status(&status, span, false); + } + } + + pub(super) fn application_require_components(&mut self, components: &[String], span: Span) { + if !self.module.api_enabled() || components.is_empty() { + return; + } + let session = self.application_session(); + let names = self.module.wire_names(components); + let manifest = self + .module + .wire_names(&self.module.program.tenant_components.clone()); + let operation = self.async_operation( + "api_require_components", + vec![ + format!("ptr {session}"), + format!("ptr {names}"), + format!("i64 {}", components.len()), + format!("ptr {manifest}"), + format!("i64 {}", self.module.program.tenant_components.len()), + ], + span, + ); + self.await_operation(&operation, &Type::Unit, span, false); + } + + fn application_session(&mut self) -> String { + let session = self.temp(); + self.line(format!("{session} = load ptr, ptr @dever_api_session")); + session + } + + fn application_has_tenant(&mut self, span: Span) -> String { + let session = self.application_session(); + self.runtime_call( + "dever_rt_v1_api_has_tenant", + vec![format!("ptr {session}")], + &Type::Bool, + false, + Some("i8"), + span, + ) + } + + fn application_positive_argument(&mut self, cursor: &str, message: &str, span: Span) -> String { + let text = self.application_argument(cursor, span); + let optional = self.runtime_call( + "dever_rt_v1_int_parse", + vec![format!("ptr {text}")], + &Type::Nullable(Box::new(Type::Int)), + true, + None, + span, + ); + let present = self.temp(); + let number = self.temp(); + let positive = self.temp(); + let valid = self.temp(); + self.line(format!( + "{present} = extractvalue {{ i1, i64 }} {optional}, 0" + )); + self.line(format!( + "{number} = extractvalue {{ i1, i64 }} {optional}, 1" + )); + self.line(format!("{positive} = icmp sgt i64 {number}, 0")); + self.line(format!("{valid} = and i1 {present}, {positive}")); + self.application_require(&valid, message, span); + number + } + + fn application_arity(&mut self, length: &str, expected: i64, message: &str, span: Span) { + let valid = self.temp(); + self.line(format!("{valid} = icmp eq i64 {length}, {expected}")); + self.application_require(&valid, message, span); + } + + pub(super) fn application_require(&mut self, valid: &str, message: &str, span: Span) { + let success = self.label("argument_valid"); + let invalid = self.label("argument_invalid"); + self.line(format!("br i1 {valid}, label %{success}, label %{invalid}")); + self.start(&invalid); + self.application_fault(message, span); + self.start(&success); + } +} diff --git a/crates/dever-core/src/llvm/asynchronous.rs b/crates/dever-core/src/llvm/asynchronous.rs new file mode 100644 index 0000000..208b68a --- /dev/null +++ b/crates/dever-core/src/llvm/asynchronous.rs @@ -0,0 +1,677 @@ +//! Coroutine frames, typed async descriptors, and polling at actual suspension points. +use super::*; + +pub(super) fn input_type(function: &crate::hir::Function) -> Type { + Type::Outputs( + function + .parameters + .iter() + .enumerate() + .filter_map(|(index, parameter)| { + parameter.value_type().map(|ty| crate::types::Field { + name: format!("p{index}"), + ty: ty.clone(), + private: false, + bounds: Vec::new(), + }) + }) + .collect(), + ) +} + +const DECLARATIONS: &str = "\ +%dever.owned_type = type { i64, i64, ptr, ptr, i8 } +%dever.async_function = type { ptr, ptr, ptr, ptr, ptr, ptr, i8 } +%dever.sync_function = type { ptr, ptr, ptr, ptr, i8 } +declare token @llvm.coro.id(i32, ptr, ptr, ptr) +declare i64 @llvm.coro.size.i64() +declare i64 @llvm.coro.align.i64() +declare ptr @llvm.coro.begin(token, ptr) +declare i8 @llvm.coro.suspend(token, i1) +declare ptr @llvm.coro.free(token, ptr) +declare i1 @llvm.coro.end(ptr, i1, token) +declare void @llvm.coro.resume(ptr) +declare void @llvm.coro.destroy(ptr) +declare i1 @llvm.coro.done(ptr) +declare ptr @dever_rt_v1_async_frame_alloc(i64, i64) +declare void @dever_rt_v1_async_frame_free(ptr, i64, i64) +declare i32 @dever_rt_v1_async_root(ptr, ptr, ptr, ptr, ptr) +declare void @dever_rt_v1_async_complete(ptr, i32) +declare i32 @dever_rt_v1_async_op_poll(ptr, ptr, ptr, ptr, ptr, ptr) +declare void @dever_rt_v1_async_op_release(ptr) +declare ptr @dever_rt_v1_async_call(ptr, ptr, ptr) +declare ptr @dever_rt_v1_async_sleep(i64) +declare ptr @dever_rt_v1_async_blocking(ptr, ptr, ptr) +declare ptr @dever_rt_v1_async_parallel(ptr, ptr, ptr) +declare ptr @dever_rt_v1_async_task_run(ptr, ptr, ptr) +declare ptr @dever_rt_v1_async_task_run_sync(ptr, ptr, ptr) +declare ptr @dever_rt_v1_async_task_wait(ptr) +declare ptr @dever_rt_v1_async_task_stop(ptr) +declare ptr @dever_rt_v1_async_task_retain(ptr) +declare void @dever_rt_v1_async_task_release(ptr) +declare i32 @dever_rt_v1_async_group_new(i64, ptr, ptr) +declare ptr @dever_rt_v1_async_group_run(ptr, ptr, ptr, ptr) +declare ptr @dever_rt_v1_async_group_run_sync(ptr, ptr, ptr, ptr) +declare ptr @dever_rt_v1_async_group_wait(ptr) +declare ptr @dever_rt_v1_async_group_stop(ptr) +declare ptr @dever_rt_v1_async_group_retain(ptr) +declare void @dever_rt_v1_async_group_release(ptr) +declare i32 @dever_rt_v1_async_channel_new(i64, ptr, ptr) +declare ptr @dever_rt_v1_async_channel_retain(ptr) +declare void @dever_rt_v1_async_channel_release(ptr) +declare ptr @dever_rt_v1_async_channel_send(ptr, ptr, ptr, ptr) +declare ptr @dever_rt_v1_async_channel_receive(ptr) +declare ptr @dever_rt_v1_async_channel_close(ptr) +"; + +impl Module<'_> { + pub(super) fn validate_async_boundaries(&self, sources: &SourceMap) -> Result<(), String> { + for instance in &self.instances { + let function = &self.program.functions[instance.function]; + if matches!(function.kind, FunctionKind::Job { .. }) && !self.send_safe(instance) { + return Err(located( + sources, + function.span, + "LLVM Job boundary requires transferable input, frame, and typed failures", + )); + } + let bindings = specialize::handler_bindings(function, instance); + let mut rejected = None; + crate::check::visit(function, |expression| { + let target = match &expression.kind { + ExpressionKind::RunCall { + target, arguments, .. + } + | ExpressionKind::ParallelCall { target, arguments } + | ExpressionKind::BlockingCall { target, arguments } => match target { + CallTarget::Function(function) => Some(specialize::call_specialization( + *function, arguments, &bindings, + )), + CallTarget::Handler(handler) => Some(Specialization { + function: specialize::resolve_handler(*handler, &bindings), + handlers: Vec::new(), + }), + }, + ExpressionKind::Collection { + operation: CollectionOp::ParallelEach, + handler: Some(handler), + .. + } => Some(Specialization { + function: specialize::resolve_handler(*handler, &bindings), + handlers: Vec::new(), + }), + ExpressionKind::Intrinsic { + operation: + Intrinsic::HttpServe + | Intrinsic::HttpServeTls + | Intrinsic::HttpServeLive + | Intrinsic::HttpServeLiveTls, + handler: Some(handler), + .. + } => Some(Specialization { + function: specialize::resolve_handler(*handler, &bindings), + handlers: Vec::new(), + }), + _ => None, + }; + if let Some(target) = target + && !self.send_safe(&target) + { + rejected.get_or_insert(expression.span); + } + }); + if let Some(span) = rejected { + return Err(located( + sources, + span, + "LLVM task or blocking boundary requires a transferable frame, outputs, and typed failures", + )); + } + } + Ok(()) + } + + pub(super) fn emit_async_types(&mut self) { + let parallel = self.instances.iter().any(|instance| { + let mut found = false; + crate::check::visit(&self.program.functions[instance.function], |expression| { + found |= matches!( + expression.kind, + ExpressionKind::Collection { + operation: CollectionOp::ParallelEach, + .. + } + ); + }); + found + }); + if !self + .instances + .iter() + .any(|instance| specialize::suspends(self.program, instance)) + && !self.value_types.iter().any(|ty| { + matches!( + ty, + Type::Task(_) + | Type::Group + | Type::Channel(_) + | Type::AsyncStream(_) + | Type::Socket + | Type::Listener + ) + }) + && !parallel + { + return; + } + self.declarations.push_str(DECLARATIONS); + self.declarations.push_str(super::network::DECLARATIONS); + self.declarations.push_str(super::parallel::DECLARATIONS); + self.declarations.push_str(concat!( + "define internal void @dever_coro_resume(ptr %frame) {\nentry:\n call void @llvm.coro.resume(ptr %frame)\n ret void\n}\n", + "define internal void @dever_coro_destroy(ptr %frame) {\nentry:\n call void @llvm.coro.destroy(ptr %frame)\n ret void\n}\n", + "define internal i8 @dever_coro_done(ptr %frame) {\nentry:\n %done = call i1 @llvm.coro.done(ptr %frame)\n %result = zext i1 %done to i8\n ret i8 %result\n}\n", + )); + for (index, ty) in self.value_types.iter().enumerate() { + let llvm = self.ty(ty); + writeln!(self.declarations, "define internal void @dever_move_{index}(ptr %source, ptr %destination) {{\nentry:\n %value = load {llvm}, ptr %source\n store {llvm} %value, ptr %destination\n store {llvm} zeroinitializer, ptr %source\n ret void\n}}").unwrap(); + writeln!(self.declarations, "@dever_owned_{index} = private constant %dever.owned_type {{ i64 {}, i64 {}, ptr @dever_move_{index}, ptr @dever_drop_{index}, i8 {} }}", self.bytes(ty), self.alignment(ty), u8::from(self.boundary_transferable(ty))).unwrap(); + } + self.emit_fault_move(); + for instance in self.instances.clone() { + self.emit_async_descriptor(&instance); + } + } + + // Fault payloads retain their checked nominal type. Never memcpy an erased + // payload or treat a business error as its diagnostic text. + fn emit_fault_move(&mut self) { + self.declarations.push_str("define internal void @dever_fault_move(ptr %source, ptr %destination) {\nentry:\n store %dever.fault zeroinitializer, ptr %destination\n"); + let capacity = self.instances.len() + 1; + for (index, ty) in [ + "i32".into(), + "i32".into(), + "i32".into(), + format!("[{capacity} x i32]"), + "{ ptr, i64 }".into(), + "i32".into(), + "i32".into(), + ] + .iter() + .enumerate() + { + writeln!(self.declarations, " %s{index} = getelementptr %dever.fault, ptr %source, i32 0, i32 {index}\n %d{index} = getelementptr %dever.fault, ptr %destination, i32 0, i32 {index}\n %v{index} = load {ty}, ptr %s{index}\n store {ty} %v{index}, ptr %d{index}").unwrap(); + } + self.declarations.push_str(" %scause = getelementptr %dever.fault, ptr %source, i32 0, i32 8\n %dcause = getelementptr %dever.fault, ptr %destination, i32 0, i32 8\n %cause = load ptr, ptr %scause\n store ptr %cause, ptr %dcause\n"); + let failures = self + .instances + .iter() + .flat_map(|instance| specialize::failures(self.program, instance)) + .map(|failure| failure.ty) + .collect::<BTreeSet<_>>(); + let arms = failures + .iter() + .map(|id| format!("i32 {}, label %payload_{id}", id + 1)) + .collect::<Vec<_>>() + .join(" "); + writeln!(self.declarations, " switch i32 %v5, label %moved [{arms}]").unwrap(); + for id in failures { + writeln!(self.declarations, "payload_{id}:\n %sp{id} = getelementptr %dever.fault, ptr %source, i32 0, i32 7\n %dp{id} = getelementptr %dever.fault, ptr %destination, i32 0, i32 7\n %p{id} = load %T{id}, ptr %sp{id}\n store %T{id} %p{id}, ptr %dp{id}\n br label %moved").unwrap(); + } + self.declarations + .push_str("moved:\n store %dever.fault zeroinitializer, ptr %source\n ret void\n}\n"); + } + + fn fault_transferable(&self, instance: &Specialization) -> bool { + specialize::failures(self.program, instance) + .iter() + .all(|failure| { + let Shape::Choice(variants) = &self.program.types[failure.ty].shape else { + unreachable!() + }; + variants[failure.variant] + .fields + .iter() + .all(|field| field.ty.transferable(&self.program.types)) + }) + } + + fn frame_send(&self, ty: &Type) -> bool { + match ty { + Type::Unit + | Type::Task(_) + | Type::Group + | Type::Socket + | Type::Listener + | Type::Upload => true, + Type::Stream(_) => false, + Type::Related(_) | Type::RowStream(_) => ty.transferable(&self.program.types), + Type::List(inner) + | Type::Nullable(inner) + | Type::Channel(inner) + | Type::AsyncStream(inner) => self.frame_send(inner), + Type::Map(key, value) | Type::MapEntry(key, value) => { + self.frame_send(key) && self.frame_send(value) + } + Type::Outputs(fields) => fields.iter().all(|field| self.frame_send(&field.ty)), + Type::Named(id) => match &self.program.types[*id].shape { + Shape::Record(fields) => fields.iter().all(|field| self.frame_send(&field.ty)), + Shape::Choice(variants) => variants + .iter() + .flat_map(|variant| &variant.fields) + .all(|field| self.frame_send(&field.ty)), + }, + _ => ty.transferable(&self.program.types), + } + } + + fn boundary_transferable(&self, ty: &Type) -> bool { + // Unit and Outputs are compiler-internal ABI rows. HTTP may transfer + // its affine Upload owner; source spawn checks still forbid Upload. + match ty { + Type::Unit | Type::Upload => true, + Type::Outputs(fields) => fields + .iter() + .all(|field| self.boundary_transferable(&field.ty)), + _ => ty.transferable(&self.program.types), + } + } + + fn send_safe(&self, instance: &Specialization) -> bool { + let function = &self.program.functions[instance.function]; + let boundary = function + .parameters + .iter() + .filter_map(Parameter::value_type) + .all(|ty| self.boundary_transferable(ty)) + && function + .outputs + .iter() + .all(|field| field.ty.transferable(&self.program.types)) + && self.fault_transferable(instance); + if !boundary || !specialize::suspends(self.program, instance) { + return boundary; + } + // A Send caller also carries the futures of ordinary async callees. + specialize::reachable(&self.program.functions, [instance.clone()]) + .expect("validated acyclic calls") + .iter() + .all(|callee| { + if !specialize::suspends(self.program, callee) { + return true; + } + let function = &self.program.functions[callee.function]; + let mut expressions_send = true; + crate::check::visit(function, |expression| { + expressions_send &= self.frame_send(&expression.ty) + }); + expressions_send + && function + .parameters + .iter() + .filter_map(Parameter::value_type) + .all(|ty| self.frame_send(ty)) + && function + .clauses + .iter() + .flat_map(|clause| &clause.locals) + .all(|ty| self.frame_send(ty)) + && function + .outputs + .iter() + .all(|field| self.frame_send(&field.ty)) + && self.fault_transferable(callee) + }) + } + + fn emit_async_descriptor(&mut self, instance: &Specialization) { + let function = &self.program.functions[instance.function]; + let name = self.names[instance].clone(); + let input = input_type(function); + let input_id = self.type_index(&input); + let output_id = self.type_index(&output_type(&function.outputs)); + writeln!(self.declarations, "@dever_fault_owned_{name} = private constant %dever.owned_type {{ i64 ptrtoint (ptr getelementptr (%dever.fault, ptr null, i32 1) to i64), i64 8, ptr @dever_fault_move, ptr @dever_fault_release, i8 {} }}", u8::from(self.fault_transferable(instance))).unwrap(); + let asynchronous = specialize::suspends(self.program, instance); + let mut wrapper = format!( + "define internal {} @dever_{}_create_{name}({}) {{\nentry:\n", + if asynchronous { "ptr" } else { "i32" }, + if asynchronous { "async" } else { "sync" }, + if asynchronous { + "ptr %state, ptr %input, ptr %out, ptr %fault" + } else { + "ptr %input, ptr %out, ptr %fault" + } + ); + wrapper.push_str(" store %dever.fault zeroinitializer, ptr %fault\n"); + let mut args = Vec::new(); + for (position, (index, ty)) in function + .parameters + .iter() + .enumerate() + .filter_map(|(index, parameter)| parameter.value_type().map(|ty| (index, ty))) + .enumerate() + { + writeln!(wrapper, " %a{index} = getelementptr {}, ptr %input, i32 0, i32 {position}\n %p{index} = load {}, ptr %a{index}", self.ty(&input), self.ty(ty)).unwrap(); + args.push(format!("{} %p{index}", self.ty(ty))); + } + let database = self.database_function(instance); + let direct_arguments = args.clone(); + if database { + args.push("ptr null".into()); + } + args.extend(["ptr %out".into(), "ptr %fault".into()]); + if asynchronous { + args.push("ptr %state".into()); + } + writeln!( + wrapper, + " %result = call {} @{name}({})", + if asynchronous { "ptr" } else { "i32" }, + args.join(", ") + ) + .unwrap(); + writeln!( + wrapper, + " ret {} %result\n}}", + if asynchronous { "ptr" } else { "i32" } + ) + .unwrap(); + self.declarations.push_str(&wrapper); + let send_safe = u8::from(self.send_safe(instance)); + if asynchronous { + writeln!(self.declarations, "@dever_async_{name} = private constant %dever.async_function {{ ptr @dever_owned_{output_id}, ptr @dever_fault_owned_{name}, ptr @dever_async_create_{name}, ptr @dever_coro_resume, ptr @dever_coro_destroy, ptr @dever_coro_done, i8 {send_safe} }}").unwrap(); + if database { + let layout = format!("{{ {}, ptr }}", self.ty(&input)); + writeln!(self.declarations, "define internal ptr @dever_async_context_create_{name}(ptr %state, ptr %input, ptr %out, ptr %fault) {{\nentry:\n %context_ptr = getelementptr {layout}, ptr %input, i32 0, i32 1\n %context = load ptr, ptr %context_ptr\n store %dever.fault zeroinitializer, ptr %fault").unwrap(); + for (position, (index, ty)) in function + .parameters + .iter() + .enumerate() + .filter_map(|(index, parameter)| parameter.value_type().map(|ty| (index, ty))) + .enumerate() + { + writeln!(self.declarations, " %a{index} = getelementptr {}, ptr %input, i32 0, i32 {position}\n %p{index} = load {}, ptr %a{index}", self.ty(&input), self.ty(ty)).unwrap(); + } + let mut arguments = direct_arguments; + arguments.extend([ + "ptr %context".into(), + "ptr %out".into(), + "ptr %fault".into(), + "ptr %state".into(), + ]); + writeln!(self.declarations, " %result = call ptr @{name}({})\n ret ptr %result\n}}\n@dever_async_context_{name} = private constant %dever.async_function {{ ptr @dever_owned_{output_id}, ptr @dever_fault_owned_{name}, ptr @dever_async_context_create_{name}, ptr @dever_coro_resume, ptr @dever_coro_destroy, ptr @dever_coro_done, i8 {send_safe} }}", arguments.join(", ")).unwrap(); + } + } else { + writeln!(self.declarations, "@dever_sync_{name} = private constant %dever.sync_function {{ ptr @dever_owned_{input_id}, ptr @dever_owned_{output_id}, ptr @dever_fault_owned_{name}, ptr @dever_sync_create_{name}, i8 {send_safe} }}").unwrap(); + } + } + + pub(super) fn emit_async_entry(&mut self, root: &Specialization) { + let name = self.names[root].clone(); + let location = self.location(self.program.functions[root.function].span); + let (symbol, extra) = if self.api_enabled() { + ( + "api_application_root", + ", ptr @dever_api_session, ptr @dever_api_append_cause".into(), + ) + } else if self.database_enabled() { + let errors = self.database_error(root, self.program.functions[root.function].span); + ( + "db_application_root", + format!(", ptr @dever_database_session, ptr {errors}"), + ) + } else if self.external_enabled() { + ("external_root", ", ptr @dever_external_append_cause".into()) + } else { + ("async_root", String::new()) + }; + writeln!(self.functions, " %error = alloca {{ ptr, i64 }}\n %input = alloca {{}}\n %async_status = call i32 @dever_rt_v1_{symbol}(ptr @dever_async_{name}, ptr %input, ptr %out, ptr %fault{extra}, ptr %error)\n switch i32 %async_status, label %root_error [i32 1, label %root_success i32 2, label %root_fault]\nroot_error:\n %message = load {{ ptr, i64 }}, ptr %error\n %message_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 4\n store {{ ptr, i64 }} %message, ptr %message_ptr\n %code = getelementptr %dever.fault, ptr %fault, i32 0, i32 0\n store i32 3, ptr %code\n %origin = getelementptr %dever.fault, ptr %fault, i32 0, i32 1\n store i32 {location}, ptr %origin\n br label %root_fault\nroot_success:\n br label %root_return\nroot_fault:\n br label %root_return\nroot_return:\n %status = phi i32 [0, %root_success], [1, %root_fault]").unwrap(); + } +} + +impl FunctionEmitter<'_, '_> { + pub(super) fn parameter(&self, index: usize) -> String { + self.parameters + .get(&index) + .cloned() + .unwrap_or_else(|| format!("%p{index}")) + } + + pub(super) fn begin_coroutine(&mut self, function: &crate::hir::Function) { + self.prologue.push_str(concat!( + " %cancelling = alloca i1\n store i1 0, ptr %cancelling\n", + " %coro_id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null)\n", + " %coro_size = call i64 @llvm.coro.size.i64()\n %coro_align = call i64 @llvm.coro.align.i64()\n", + " %coro_memory = call ptr @dever_rt_v1_async_frame_alloc(i64 %coro_size, i64 %coro_align)\n", + " %coro_frame = call ptr @llvm.coro.begin(token %coro_id, ptr %coro_memory)\n", + )); + // create() receives borrowed arguments. Acquire frame ownership before + // the initial suspend returns to its caller, not on the first poll. + for (index, parameter) in function.parameters.iter().enumerate() { + let Some(ty) = parameter.value_type() else { + continue; + }; + if contains_owned(self.module.program, ty) { + let guard = self.own_value(ty, &format!("%p{index}"), true); + let owned = self.load_owned_value(ty, guard); + self.parameters.insert(index, owned); + } + } + if self.database_context != "null" { + let retained = self.temp(); + self.line(format!( + "{retained} = call ptr @dever_rt_v1_db_transaction_retain(ptr {})", + self.database_context + )); + self.protocol_owner("db_transaction", &retained); + self.database_context = retained; + } + self.suspend("async_start", false); + self.start("async_start"); + } + + fn suspend(&mut self, resume: &str, final_suspend: bool) { + let result = self.temp(); + self.line(format!( + "{result} = call i8 @llvm.coro.suspend(token none, i1 {})", + u8::from(final_suspend) + )); + let arms = if final_suspend { + "i8 1, label %free_frame".into() + } else { + format!("i8 0, label %{resume} i8 1, label %destroy_frame") + }; + self.line(format!( + "switch i8 {result}, label %suspend_return [{arms}]" + )); + } + + pub(super) fn end_coroutine(&mut self, status: &str) { + let cancelling = self.temp(); + self.line(format!("{cancelling} = load i1, ptr %cancelling")); + self.line(format!( + "br i1 {cancelling}, label %free_frame, label %complete_frame" + )); + self.start("complete_frame"); + let okay = self.temp(); + self.line(format!("{okay} = icmp eq i32 {status}, 0")); + let completion = self.temp(); + self.line(format!("{completion} = select i1 {okay}, i32 1, i32 2")); + self.line(format!( + "call void @dever_rt_v1_async_complete(ptr %state, i32 {completion})" + )); + self.suspend("", true); + self.start("destroy_frame"); + self.line("store i1 1, ptr %cancelling"); + self.line("br label %cleanup"); + self.start("free_frame"); + self.line("%allocation = call ptr @llvm.coro.free(token %coro_id, ptr %coro_frame)"); + self.line("call void @dever_rt_v1_async_frame_free(ptr %allocation, i64 %coro_size, i64 %coro_align)"); + self.line("br label %suspend_return"); + self.start("suspend_return"); + self.line("call i1 @llvm.coro.end(ptr %coro_frame, i1 false, token none)"); + self.line("ret ptr %coro_frame"); + } + + pub(super) fn pack_input( + &mut self, + instance: &Specialization, + values: &[(Type, String)], + owned: bool, + ) -> (String, Option<usize>) { + let ty = input_type(&self.module.program.functions[instance.function]); + let source = self.entry_slot(&ty); + self.line(format!( + "store {} zeroinitializer, ptr {source}", + self.module.ty(&ty) + )); + for (index, (field, value)) in values.iter().enumerate() { + let pointer = self.temp(); + self.line(format!( + "{pointer} = getelementptr {}, ptr {source}, i32 0, i32 {index}", + self.module.ty(&ty) + )); + self.line(format!( + "store {} {value}, ptr {pointer}", + self.module.ty(field) + )); + } + if !owned { + return (source, None); + } + let destination = self.entry_slot(&ty); + self.line(format!( + "call void @dever_clone_{}(ptr {source}, ptr {destination})", + self.module.type_index(&ty) + )); + let guard = self.register_guard(&ty, destination.clone()); + self.mark_live(guard); + (destination, Some(guard)) + } + + pub(super) fn async_operation( + &mut self, + symbol: &str, + mut arguments: Vec<String>, + span: Span, + ) -> String { + let error = self.entry_slot_ir("{ ptr, i64 }"); + arguments.push(format!("ptr {error}")); + let operation = self.temp(); + self.line(format!( + "{operation} = call ptr @dever_rt_v1_{symbol}({})", + arguments.join(", ") + )); + let missing = self.temp(); + self.line(format!("{missing} = icmp eq ptr {operation}, null")); + let failed = self.label("operation_error"); + let ready = self.label("operation_created"); + self.line(format!("br i1 {missing}, label %{failed}, label %{ready}")); + self.start(&failed); + self.check_runtime_status("1", &error, span); + self.line("unreachable"); + self.start(&ready); + operation + } + + pub(super) fn poll_raw( + &mut self, + operation: &str, + row: &Type, + ) -> (String, String, String, String) { + self.poll_raw_layout(operation, &self.module.ty(row)) + } + + pub(super) fn poll_raw_layout( + &mut self, + operation: &str, + layout: &str, + ) -> (String, String, String, String) { + debug_assert!( + self.asynchronous, + "only checked suspension points poll operations" + ); + let operation_slot = self.entry_slot_ir("ptr"); + let guard = self.register_owned(OwnedKind::AsyncOp, operation_slot.clone()); + self.line(format!("store ptr {operation}, ptr {operation_slot}")); + self.mark_live(guard); + let output = self.entry_slot_ir(layout); + self.line(format!("store {layout} zeroinitializer, ptr {output}")); + let present = self.entry_slot_ir("i8"); + let error = self.entry_slot_ir("{ ptr, i64 }"); + let poll = self.label("poll_operation"); + let waiting = self.label("pending_operation"); + let ready = self.label("ready_operation"); + self.line(format!("br label %{poll}")); + self.start(&poll); + let status = self.temp(); + self.line(format!("{status} = call i32 @dever_rt_v1_async_op_poll(ptr %state, ptr {operation}, ptr {output}, ptr {present}, ptr %fault, ptr {error})")); + let pending = self.temp(); + self.line(format!("{pending} = icmp eq i32 {status}, 0")); + self.line(format!("br i1 {pending}, label %{waiting}, label %{ready}")); + self.start(&waiting); + self.suspend(&poll, false); + self.start(&ready); + self.release_guard(guard); + (status, output, present, error) + } + + pub(super) fn poll_operation( + &mut self, + operation: &str, + output: &Type, + span: Span, + ) -> (String, String) { + let (status, value, _, error) = self.poll_raw(operation, output); + let legacy = self.poll_status(&status, &error, span); + (legacy, value) + } + + pub(super) fn poll_status(&mut self, status: &str, error: &str, span: Span) -> String { + let error_status = self.temp(); + self.line(format!("{error_status} = icmp ugt i32 {status}, 2")); + let failed = self.label("poll_error"); + let done = self.label("poll_done"); + self.line(format!( + "br i1 {error_status}, label %{failed}, label %{done}" + )); + self.start(&failed); + self.check_runtime_status(status, error, span); + self.line("unreachable"); + self.start(&done); + let legacy = self.temp(); + self.line(format!("{legacy} = sub i32 {status}, 1")); + legacy + } + + pub(super) fn await_operation( + &mut self, + operation: &str, + output: &Type, + span: Span, + trace: bool, + ) -> String { + let (status, value) = self.poll_operation(operation, output, span); + self.propagate_status(&status, span, trace); + let result = self.temp(); + self.line(format!( + "{result} = load {}, ptr {value}", + self.module.ty(output) + )); + result + } + + pub(super) fn propagate_status(&mut self, status: &str, span: Span, trace: bool) { + let failed = self.temp(); + self.line(format!("{failed} = icmp ne i32 {status}, 0")); + let error = self.label("await_fault"); + let done = self.label("await_done"); + self.line(format!("br i1 {failed}, label %{error}, label %{done}")); + self.start(&error); + if trace { + self.append_frame(span); + } + self.exit(status); + self.start(&done); + } +} diff --git a/crates/dever-core/src/llvm/collections.rs b/crates/dever-core/src/llvm/collections.rs new file mode 100644 index 0000000..93ca2da --- /dev/null +++ b/crates/dever-core/src/llvm/collections.rs @@ -0,0 +1,367 @@ +use super::*; + +pub(super) fn supported(operation: CollectionOp, sequence: Option<SequenceKind>) -> bool { + let direct = matches!( + operation, + CollectionOp::Append + | CollectionOp::First + | CollectionOp::Get + | CollectionOp::Put + | CollectionOp::Remove + | CollectionOp::Length + | CollectionOp::Entries + | CollectionOp::Close + ); + let traversal = matches!( + operation, + CollectionOp::Each + | CollectionOp::ParallelEach + | CollectionOp::Filter + | CollectionOp::Find + | CollectionOp::Reduce + | CollectionOp::ReduceUntil + | CollectionOp::Sum + ) && matches!( + sequence, + Some( + SequenceKind::List + | SequenceKind::Bytes + | SequenceKind::Stream + | SequenceKind::AsyncStream + | SequenceKind::RowStream + ) + ); + direct || traversal +} + +impl FunctionEmitter<'_, '_> { + pub(super) fn row_pointer(&mut self, ty: &Type, value: &str) -> String { + let pointer = self.entry_slot(ty); + self.line(format!( + "store {} {value}, ptr {pointer}", + self.module.ty(ty) + )); + pointer + } + + pub(super) fn pointer_array(&mut self, pointers: &[String]) -> String { + if pointers.is_empty() { + return "null".into(); + } + let array = self.entry_slot_ir(&format!("[{} x ptr]", pointers.len())); + for (index, pointer) in pointers.iter().enumerate() { + let element = self.temp(); + self.line(format!( + "{element} = getelementptr [{} x ptr], ptr {array}, i32 0, i32 {index}", + pointers.len() + )); + self.line(format!("store ptr {pointer}, ptr {element}")); + } + array + } + + pub(super) fn list_literal(&mut self, values: &[Expression], result: &Expression) -> String { + let Type::List(element) = &result.ty else { + unreachable!("checked List literal") + }; + let mut pointers = Vec::with_capacity(values.len()); + for value in values { + let row = self.expression(value).expect("validated List element"); + pointers.push(self.row_pointer(element, &row)); + } + let array = self.pointer_array(&pointers); + self.runtime_call( + "dever_rt_v1_list_new", + vec![ + format!("ptr @dever_type_{}", self.module.type_index(element)), + format!("ptr {array}"), + format!("i64 {}", values.len()), + ], + &result.ty, + false, + None, + result.span, + ) + } + + pub(super) fn map_literal( + &mut self, + entries: &[(Expression, Expression)], + result: &Expression, + ) -> String { + let Type::Map(key_ty, value_ty) = &result.ty else { + unreachable!("checked Map literal") + }; + let mut keys = Vec::with_capacity(entries.len()); + let mut values = Vec::with_capacity(entries.len()); + for (key, value) in entries { + let key_value = self.expression(key).expect("validated Map key"); + keys.push(self.row_pointer(key_ty, &key_value)); + let mapped = self.expression(value).expect("validated Map value"); + values.push(self.row_pointer(value_ty, &mapped)); + } + let key_array = self.pointer_array(&keys); + let value_array = self.pointer_array(&values); + self.runtime_call( + "dever_rt_v1_map_new", + vec![ + format!("ptr @dever_type_{}", self.module.type_index(key_ty)), + format!("ptr @dever_type_{}", self.module.type_index(value_ty)), + format!("ptr {key_array}"), + format!("ptr {value_array}"), + format!("i64 {}", entries.len()), + ], + &result.ty, + false, + None, + result.span, + ) + } + + pub(super) fn collection( + &mut self, + operation: CollectionOp, + _handler: Option<crate::hir::HandlerTarget>, + _sequence: Option<SequenceKind>, + arguments: &[Expression], + result: &Expression, + ) -> String { + if operation == CollectionOp::ParallelEach { + return self.parallel_each( + _handler.expect("checked parallel handler"), + _sequence.expect("checked parallel sequence"), + arguments, + result, + ); + } + if matches!( + operation, + CollectionOp::Each + | CollectionOp::Filter + | CollectionOp::Find + | CollectionOp::Reduce + | CollectionOp::ReduceUntil + | CollectionOp::Sum + ) { + return self.traverse( + operation, + _handler, + _sequence.expect("checked sequence"), + arguments, + result, + ); + } + if operation == CollectionOp::Entries { + return self.map_entries(&arguments[0], result); + } + let receiver = self + .expression(&arguments[0]) + .expect("validated collection input"); + let receiver_guard = + contains_owned(self.module.program, &arguments[0].ty).then(|| self.guards.len() - 1); + let mut values = vec![receiver.clone()]; + for argument in &arguments[1..] { + values.push( + self.expression(argument) + .expect("validated collection argument"), + ); + } + let receiver_ty = &arguments[0].ty; + if operation == CollectionOp::Close { + let symbol = if matches!(receiver_ty, Type::AsyncStream(_)) { + "async_stream_close" + } else if matches!(receiver_ty, Type::RowStream(_)) { + "db_stream_close" + } else { + "stream_close" + }; + self.runtime_call( + &format!("dever_rt_v1_{symbol}"), + vec![format!("ptr {receiver}")], + &Type::Unit, + false, + Some("i8"), + result.span, + ); + return "zeroinitializer".into(); + } + let mut typed_args = Vec::new(); + if matches!( + operation, + CollectionOp::Append + | CollectionOp::First + | CollectionOp::Get + | CollectionOp::Put + | CollectionOp::Remove + ) && let Some(guard) = receiver_guard + { + self.disarm(guard); + } + typed_args.push(format!("ptr {receiver}")); + let symbol = match operation { + CollectionOp::Append => { + typed_args.push(format!( + "ptr {}", + self.row_pointer(&arguments[1].ty, &values[1]) + )); + "list_append_take" + } + CollectionOp::First => "list_first_take", + CollectionOp::Get | CollectionOp::Put | CollectionOp::Remove => { + typed_args.push(format!( + "ptr {}", + self.row_pointer(&arguments[1].ty, &values[1]) + )); + if operation == CollectionOp::Put { + typed_args.push(format!( + "ptr {}", + self.row_pointer(&arguments[2].ty, &values[2]) + )); + } + match operation { + CollectionOp::Get => "map_get_take", + CollectionOp::Put => "map_put_take", + CollectionOp::Remove => "map_remove_take", + _ => unreachable!(), + } + } + CollectionOp::Length => match receiver_ty { + Type::Text => "text_length", + Type::Bytes => "bytes_length", + Type::List(_) => "list_length", + Type::Map(_, _) => "map_length", + _ => unreachable!("checked Length receiver"), + }, + _ => unreachable!("preflighted collection"), + }; + let symbol = format!("dever_rt_v1_{symbol}"); + let row_type = match (operation, receiver_ty) { + (CollectionOp::First, Type::List(element)) => Some(element.as_ref()), + (CollectionOp::Get, Type::Map(_, value)) => Some(value.as_ref()), + _ => None, + }; + if let Some(row_type) = row_type { + self.runtime_optional_row(&symbol, typed_args, row_type, &result.ty, result.span) + } else { + self.runtime_call(&symbol, typed_args, &result.ty, false, None, result.span) + } + } + + fn map_entries(&mut self, map: &Expression, result: &Expression) -> String { + let Type::Map(key_ty, value_ty) = &map.ty else { + unreachable!("checked entries input") + }; + let Type::List(entry_ty) = &result.ty else { + unreachable!("checked entries output") + }; + let input = self.expression(map).expect("validated Map input"); + self.disarm(self.guards.len() - 1); + let cursor = self.runtime_call( + "dever_rt_v1_map_cursor_take", + vec![format!("ptr {input}")], + &Type::Unit, + false, + Some("ptr"), + result.span, + ); + let cursor_pointer = self.entry_slot_ir("ptr"); + self.line(format!("store ptr {cursor}, ptr {cursor_pointer}")); + let cursor_guard = self.register_owned(OwnedKind::MapCursor, cursor_pointer); + self.mark_live(cursor_guard); + + let list = self.runtime_call( + "dever_rt_v1_list_new", + vec![ + format!("ptr @dever_type_{}", self.module.type_index(entry_ty)), + "ptr null".into(), + "i64 0".into(), + ], + &result.ty, + false, + None, + result.span, + ); + let list_pointer = self.entry_slot(&result.ty); + self.line(format!("store ptr {list}, ptr {list_pointer}")); + let list_guard = self.register_guard(&result.ty, list_pointer.clone()); + self.mark_live(list_guard); + + let key_pointer = self.entry_slot(key_ty); + let value_pointer = self.entry_slot(value_ty); + let key_guard = contains_owned(self.module.program, key_ty) + .then(|| self.register_guard(key_ty, key_pointer.clone())); + let value_guard = contains_owned(self.module.program, value_ty) + .then(|| self.register_guard(value_ty, value_pointer.clone())); + let present_pointer = self.entry_slot_ir("i8"); + let error = self.entry_slot_ir("{ ptr, i64 }"); + let loop_label = self.label("entries_next"); + let body = self.label("entries_body"); + let done = self.label("entries_done"); + self.line(format!("br label %{loop_label}")); + self.start(&loop_label); + let status = self.temp(); + self.line(format!("{status} = call i32 @dever_rt_v1_map_cursor_next(ptr {cursor}, ptr {key_pointer}, ptr {value_pointer}, ptr {present_pointer}, ptr {error})")); + self.check_runtime_status(&status, &error, result.span); + let present = self.temp(); + self.line(format!("{present} = load i8, ptr {present_pointer}")); + let has_entry = self.temp(); + self.line(format!("{has_entry} = icmp ne i8 {present}, 0")); + self.line(format!("br i1 {has_entry}, label %{body}, label %{done}")); + self.start(&body); + if let Some(guard) = key_guard { + self.mark_live(guard); + } + if let Some(guard) = value_guard { + self.mark_live(guard); + } + let key = self.temp(); + self.line(format!( + "{key} = load {}, ptr {key_pointer}", + self.module.ty(key_ty) + )); + let value = self.temp(); + self.line(format!( + "{value} = load {}, ptr {value_pointer}", + self.module.ty(value_ty) + )); + let first = self.temp(); + self.line(format!( + "{first} = insertvalue {} undef, {} {key}, 0", + self.module.ty(entry_ty), + self.module.ty(key_ty) + )); + let row = self.temp(); + self.line(format!( + "{row} = insertvalue {} {first}, {} {value}, 1", + self.module.ty(entry_ty), + self.module.ty(value_ty) + )); + let row_pointer = self.row_pointer(entry_ty, &row); + let current = self.temp(); + self.line(format!("{current} = load ptr, ptr {list_pointer}")); + self.disarm(list_guard); + let appended = self.runtime_call( + "dever_rt_v1_list_append_take", + vec![format!("ptr {current}"), format!("ptr {row_pointer}")], + &result.ty, + false, + None, + result.span, + ); + self.line(format!("store ptr {appended}, ptr {list_pointer}")); + self.mark_live(list_guard); + if let Some(guard) = value_guard { + self.release_guard(guard); + } + if let Some(guard) = key_guard { + self.release_guard(guard); + } + self.line(format!("br label %{loop_label}")); + self.start(&done); + self.release_guard(cursor_guard); + let output = self.temp(); + self.line(format!("{output} = load ptr, ptr {list_pointer}")); + self.disarm(list_guard); + output + } +} diff --git a/crates/dever-core/src/llvm/concurrency.rs b/crates/dever-core/src/llvm/concurrency.rs new file mode 100644 index 0000000..24f64f6 --- /dev/null +++ b/crates/dever-core/src/llvm/concurrency.rs @@ -0,0 +1,292 @@ +//! Checked Task/Group/Channel and blocking boundaries share coroutine polling. +use super::*; + +impl FunctionEmitter<'_, '_> { + pub(super) fn task_selection( + &mut self, + operation: Intrinsic, + handler: Option<crate::hir::HandlerTarget>, + arguments: &[Expression], + result: &Expression, + ) -> String { + let mut values = Vec::with_capacity(arguments.len()); + let mut task_guards = Vec::new(); + for argument in arguments { + values.push( + self.expression(argument) + .expect("checked task selection argument"), + ); + if matches!(argument.ty, Type::Task(_)) { + task_guards.push(self.guards.len() - 1); + } + } + if operation == Intrinsic::TaskRace { + let tasks = self.pointer_array(&values); + let operation = self.async_operation( + "async_task_race", + vec![format!("ptr {tasks}"), format!("i64 {}", values.len())], + result.span, + ); + // A rejected race leaves every wrapper caller-owned. + for guard in task_guards { + self.disarm(guard); + } + return self.await_operation(&operation, &result.ty, result.span, true); + } + // Argument evaluation may fail; transfer only after every argument exists. + for guard in task_guards { + self.disarm(guard); + } + let operation = self.temp(); + self.line(format!( + "{operation} = call ptr @dever_rt_v1_async_task_wait_timeout(ptr {}, i64 {})", + values[0], values[1] + )); + let (status, output, present, error) = self.poll_raw(&operation, &result.ty); + let legacy = self.poll_status(&status, &error, result.span); + self.propagate_status(&legacy, result.span, true); + let flag = self.temp(); + self.line(format!("{flag} = load i8, ptr {present}")); + let completed = self.temp(); + self.line(format!("{completed} = icmp ne i8 {flag}, 0")); + let fallback = self.label("timeout_fallback"); + let done = self.label("timeout_done"); + self.line(format!( + "br i1 {completed}, label %{done}, label %{fallback}" + )); + self.start(&fallback); + let (value, ty) = + self.invoke_handler(handler.expect("checked timeout fallback"), &[], result.span); + self.line(format!( + "store {} {value}, ptr {output}", + self.module.ty(&ty) + )); + self.line(format!("br label %{done}")); + self.start(&done); + let value = self.temp(); + self.line(format!( + "{value} = load {}, ptr {output}", + self.module.ty(&result.ty) + )); + value + } + + fn concurrency_arguments( + &mut self, + arguments: &[CallArgument], + ) -> Result<Vec<(Type, String)>, String> { + arguments + .iter() + .filter_map(|argument| match argument { + CallArgument::Value(value) => Some( + self.expression(value) + .map(|result| (value.ty.clone(), result)), + ), + CallArgument::Handler(_) => None, + }) + .collect() + } + + pub(super) fn concurrency(&mut self, expression: &Expression) -> Result<String, String> { + let span = expression.span; + let result = match &expression.kind { + ExpressionKind::RunCall { + group, + target, + arguments, + } => { + let group = group + .as_ref() + .map(|group| self.expression(group)) + .transpose()?; + let instance = self.call_instance(*target, arguments); + let asynchronous = specialize::suspends(self.module.program, &instance); + let values = self.concurrency_arguments(arguments)?; + let (input, guard) = self.pack_input(&instance, &values, !asynchronous); + let mut args = Vec::new(); + if let Some(group) = &group { + args.push(format!("ptr {group}")); + } + let descriptor = if asynchronous { "async" } else { "sync" }; + args.extend([ + format!("ptr @dever_{descriptor}_{}", self.module.names[&instance]), + format!("ptr {input}"), + ]); + let symbol = match (group.is_some(), asynchronous) { + (false, true) => "async_task_run", + (false, false) => "async_task_run_sync", + (true, true) => "async_group_run", + (true, false) => "async_group_run_sync", + }; + let operation = self.async_operation(symbol, args, span); + if let Some(guard) = guard { + self.disarm(guard); + } + self.await_operation(&operation, &expression.ty, span, group.is_some()) + } + ExpressionKind::ParallelCall { target, arguments } + | ExpressionKind::BlockingCall { target, arguments } => { + let instance = self.call_instance(*target, arguments); + let values = self.concurrency_arguments(arguments)?; + let symbol = if matches!(expression.kind, ExpressionKind::ParallelCall { .. }) { + "async_parallel" + } else { + "async_blocking" + }; + self.invoke_sync_boundary(&instance, &values, symbol, &expression.ty, span) + } + ExpressionKind::AwaitTask(value) + | ExpressionKind::StopTask(value) + | ExpressionKind::AwaitGroup(value) + | ExpressionKind::StopGroup(value) => { + let handle = self.expression(value)?; + // The ABI consumes this one wrapper even when polling fails. + self.disarm(self.guards.len() - 1); + let symbol = match expression.kind { + ExpressionKind::AwaitTask(_) => "async_task_wait", + ExpressionKind::StopTask(_) => "async_task_stop", + ExpressionKind::AwaitGroup(_) => "async_group_wait", + _ => "async_group_stop", + }; + let operation = self.temp(); + self.line(format!( + "{operation} = call ptr @dever_rt_v1_{symbol}(ptr {handle})" + )); + self.await_operation(&operation, &expression.ty, span, true) + } + ExpressionKind::Group(limit) => { + let limit = self.expression(limit)?; + self.runtime_call( + "dever_rt_v1_async_group_new", + vec![format!("i64 {limit}")], + &Type::Group, + false, + None, + span, + ) + } + ExpressionKind::Channel { capacity, .. } => { + let capacity = self.expression(capacity)?; + self.runtime_call( + "dever_rt_v1_async_channel_new", + vec![format!("i64 {capacity}")], + &expression.ty, + false, + None, + span, + ) + } + ExpressionKind::ChannelSend { channel, value } => { + let channel = self.expression(channel)?; + let value_ir = self.expression(value)?; + let guard = self.own_value( + &value.ty, + &value_ir, + contains_owned(self.module.program, &value.ty), + ); + let input = self.guards[guard].pointer.clone(); + let operation = self.async_operation( + "async_channel_send", + vec![ + format!("ptr {channel}"), + format!("ptr @dever_owned_{}", self.module.type_index(&value.ty)), + format!("ptr {input}"), + ], + span, + ); + self.disarm(guard); + self.await_operation(&operation, &Type::Unit, span, false) + } + ExpressionKind::ChannelReceive(channel) => { + let Type::Channel(element) = &channel.ty else { + unreachable!("checked receive") + }; + let handle = self.expression(channel)?; + let operation = self.temp(); + self.line(format!( + "{operation} = call ptr @dever_rt_v1_async_channel_receive(ptr {handle})" + )); + let (status, row, present, error) = self.poll_raw(&operation, element); + let legacy = self.poll_status(&status, &error, span); + self.propagate_status(&legacy, span, false); + self.optional_row(&expression.ty, element, &row, &present) + } + ExpressionKind::ChannelClose(channel) => { + let channel = self.expression(channel)?; + let operation = self.temp(); + self.line(format!( + "{operation} = call ptr @dever_rt_v1_async_channel_close(ptr {channel})" + )); + self.await_operation(&operation, &Type::Unit, span, false) + } + _ => unreachable!("checked concurrency expression"), + }; + Ok(result) + } + + pub(super) fn task_sleep(&mut self, duration: &str, expression: &Expression) -> String { + let operation = self.temp(); + self.line(format!( + "{operation} = call ptr @dever_rt_v1_async_sleep(i64 {duration})" + )); + let (status, _, _, error) = self.poll_raw(&operation, &Type::Unit); + let Type::Named(id) = expression.ty else { + unreachable!("checked sleep result") + }; + let Shape::Choice(variants) = &self.module.program.types[id].shape else { + unreachable!() + }; + let done_tag = variants + .iter() + .position(|variant| variant.name == "Done") + .expect("sleep Done"); + let failed_tag = variants + .iter() + .position(|variant| variant.name == "Failed") + .expect("sleep Failed"); + let slot = self.entry_slot(&expression.ty); + let done = self.label("sleep_done"); + let failed = self.label("sleep_failed"); + let invalid = self.label("sleep_invalid"); + let result = self.label("sleep_result"); + self.line(format!( + "switch i32 {status}, label %{invalid} [i32 1, label %{done} i32 3, label %{failed}]" + )); + self.start(&done); + self.write_variant(id, done_tag, &[], &slot); + self.line(format!("br label %{result}")); + self.start(&failed); + let message = self.error_text(&error, expression.span); + self.write_variant(id, failed_tag, &[(Type::Text, message)], &slot); + self.line(format!("br label %{result}")); + self.start(&invalid); + let legacy = self.poll_status(&status, &error, expression.span); + self.propagate_status(&legacy, expression.span, false); + self.line("unreachable"); + self.start(&result); + let value = self.temp(); + self.line(format!("{value} = load %T{id}, ptr {slot}")); + value + } + + pub(super) fn invoke_sync_boundary( + &mut self, + instance: &Specialization, + values: &[(Type, String)], + symbol: &str, + output: &Type, + span: Span, + ) -> String { + let (input, guard) = self.pack_input(instance, values, true); + let operation = self.async_operation( + symbol, + vec![ + format!("ptr @dever_sync_{}", self.module.names[instance]), + format!("ptr {input}"), + ], + span, + ); + self.disarm(guard.expect("owned worker input")); + self.await_operation(&operation, output, span, true) + } +} diff --git a/crates/dever-core/src/llvm/database.rs b/crates/dever-core/src/llvm/database.rs new file mode 100644 index 0000000..e7b2a76 --- /dev/null +++ b/crates/dever-core/src/llvm/database.rs @@ -0,0 +1,238 @@ +//! Typed database boundary. SQL values never enter source-language evaluation. +use super::*; + +pub(super) const DECLARATIONS: &str = "\ +%dever.db_error = type { ptr, ptr, ptr } +%dever.db_value = type { i32, i32, i32, i64, double, { i64, i64 }, ptr } +%dever.db_sql = type { { ptr, i64 }, { ptr, i64 } } +declare void @dever_rt_v1_db_cause_append(ptr, ptr) +declare i32 @dever_rt_v1_db_application_root(ptr, ptr, ptr, ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_db_session_new(ptr, i64, ptr, i64, ptr, ptr) +declare void @dever_rt_v1_db_session_release(ptr) +declare ptr @dever_rt_v1_db_prepare(ptr, ptr, ptr) +declare ptr @dever_rt_v1_db_resolve_scoped(ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_db_select(ptr, ptr, ptr, ptr, ptr, ptr) +declare ptr @dever_rt_v1_db_retain(ptr) +declare void @dever_rt_v1_db_release(ptr) +declare i32 @dever_rt_v1_db_max_page_size(ptr, ptr, ptr) +declare ptr @dever_rt_v1_db_begin(ptr, ptr, ptr) +declare ptr @dever_rt_v1_db_transaction_retain(ptr) +declare void @dever_rt_v1_db_transaction_release(ptr) +declare ptr @dever_rt_v1_db_finish(ptr, i8, ptr, ptr) +declare ptr @dever_rt_v1_db_rollback_take(ptr, ptr, ptr, ptr, ptr) +declare ptr @dever_rt_v1_db_execute(ptr, ptr, ptr, ptr, i64, ptr, ptr) +declare ptr @dever_rt_v1_db_query(ptr, ptr, ptr, ptr, i64, i64, ptr, ptr) +declare ptr @dever_rt_v1_db_relation_query(ptr, ptr, ptr, ptr, ptr, ptr, i64, i64, ptr, ptr) +declare ptr @dever_rt_v1_db_stream(ptr, ptr, ptr, ptr, i64, i64, ptr, ptr, ptr) +declare ptr @dever_rt_v1_db_stream_retain(ptr) +declare void @dever_rt_v1_db_stream_release(ptr) +declare i32 @dever_rt_v1_db_stream_close(ptr, ptr, ptr) +declare ptr @dever_rt_v1_db_stream_pull(ptr, ptr, ptr) +declare void @dever_rt_v1_db_rows_release(ptr) +declare i32 @dever_rt_v1_db_rows_len(ptr, ptr, ptr) +declare i32 @dever_rt_v1_db_rows_at(ptr, i64, ptr, ptr) +declare i32 @dever_rt_v1_db_required_row(ptr, ptr, ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_db_optional_row(ptr, ptr, ptr, ptr, ptr, ptr, ptr) +declare void @dever_rt_v1_db_row_release(ptr) +declare i32 @dever_rt_v1_db_row_len(ptr, ptr, ptr) +declare i32 @dever_rt_v1_db_related_new(ptr, ptr, ptr, ptr) +declare ptr @dever_rt_v1_db_related_retain(ptr) +declare void @dever_rt_v1_db_related_release(ptr) +declare i8 @dever_rt_v1_db_related_equal(ptr, ptr) +declare i32 @dever_rt_v1_db_related_get(ptr, ptr, ptr) +declare i32 @dever_rt_v1_db_row_null(ptr, i64, ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_db_row_bool(ptr, i64, ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_db_row_int(ptr, i64, ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_db_row_float(ptr, i64, ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_db_row_decimal(ptr, i64, ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_db_row_text(ptr, i64, ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_db_row_bytes(ptr, i64, ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_db_row_uuid(ptr, i64, ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_db_uuid_new(ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_db_error(i32, ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_db_pagination(i64, i64, i64, ptr, ptr, ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_db_cursor_size(i64, i64, ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_db_stream_capacity(i64, i64, ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_db_relation_limit(i64, ptr, ptr, ptr, ptr) +"; + +impl Module<'_> { + pub(super) fn database_enabled(&self) -> bool { + self.application.as_ref().is_some_and(|app| match app.test { + Some(index) => self.program.tests[index].uses_database(), + None => !self.program.models.is_empty() || !self.program.jobs.is_empty(), + }) + } + + pub(super) fn database_function(&self, instance: &Specialization) -> bool { + !specialize::database_effects(self.program, instance).is_empty() + } + + pub(super) fn database_error(&mut self, instance: &Specialization, span: Span) -> String { + let location = self.location(span); + let name = self.names[instance].clone(); + let symbol = format!("@dever_db_error_{name}_{location}"); + if !self.database_errors.insert((name.clone(), location)) { + return symbol; + } + let id = crate::capture::database_errors(&self.program.types)[0].ty; + let Shape::Choice(variants) = &self.program.types[id].shape else { + unreachable!() + }; + let kinds = [ + "Pool", + "PoolExhausted", + "Connection", + "Timeout", + "Cancelled", + "Database", + "Constraint", + "NotFound", + "InvalidData", + "Migration", + ]; + let variants = kinds + .iter() + .map(|name| { + variants + .iter() + .position(|variant| variant.name == *name) + .expect("bundled database error kind") + }) + .collect::<Vec<_>>(); + writeln!(self.declarations, "{symbol} = private constant %dever.db_error {{ ptr @dever_fault_owned_{name}, ptr @dever_db_pack_{name}_{location}, ptr @dever_db_append_cause }}").unwrap(); + writeln!(self.declarations, "define internal void @dever_db_pack_{name}_{location}(i32 %kind, ptr %message, ptr %fault) {{\nentry:\n store %dever.fault zeroinitializer, ptr %fault\n %owned = call ptr @dever_rt_v1_text_retain(ptr %message)\n %payload = getelementptr %dever.fault, ptr %fault, i32 0, i32 7").unwrap(); + for (index, value) in [(0, 4), (1, location), (5, id + 1)] { + writeln!(self.declarations, " %f{index} = getelementptr %dever.fault, ptr %fault, i32 0, i32 {index}\n store i32 {value}, ptr %f{index}").unwrap(); + } + let arms = variants + .iter() + .enumerate() + .map(|(kind, variant)| format!("i32 {kind}, label %kind{variant}")) + .collect::<Vec<_>>() + .join(" "); + writeln!( + self.declarations, + " switch i32 %kind, label %invalid [{arms}]\ninvalid:\n unreachable" + ) + .unwrap(); + for variant in variants { + writeln!(self.declarations, "kind{variant}:\n %tag{variant} = getelementptr %T{id}, ptr %payload, i32 0, i32 0\n store i32 {variant}, ptr %tag{variant}\n %body{variant} = getelementptr %T{id}, ptr %payload, i32 0, i32 1\n store ptr %owned, ptr %body{variant}\n %variant{variant} = getelementptr %dever.fault, ptr %fault, i32 0, i32 6\n store i32 {variant}, ptr %variant{variant}\n ret void").unwrap(); + } + self.declarations.push_str("}\n"); + symbol + } + + pub(super) fn database_sql(&mut self, sqlite: &str, postgres: &str) -> String { + let (sqlite, slen) = self.text_literal(sqlite); + let (postgres, plen) = self.text_literal(postgres); + let name = format!("@dever_db_sql_{}", self.next_literal); + self.next_literal += 1; + writeln!(self.declarations, "{name} = private constant %dever.db_sql {{ {{ ptr, i64 }} {{ ptr {sqlite}, i64 {slen} }}, {{ ptr, i64 }} {{ ptr {postgres}, i64 {plen} }} }}").unwrap(); + name + } + + pub(super) fn planned_sql(&mut self, sql: &str) -> String { + let (sqlite, postgres) = crate::native::orm::sql_dialects(sql, sql); + self.database_sql(&sqlite, &postgres) + } +} + +impl FunctionEmitter<'_, '_> { + pub(super) fn await_database_handle(&mut self, operation: &str, span: Span) -> String { + let (status, output, _, error) = self.poll_raw_layout(operation, "ptr"); + let status = self.poll_status(&status, &error, span); + self.propagate_status(&status, span, false); + let handle = self.temp(); + self.line(format!("{handle} = load ptr, ptr {output}")); + handle + } + pub(super) fn database_errors(&mut self, span: Span) -> String { + self.database_errors_override + .clone() + .unwrap_or_else(|| self.module.database_error(self.instance, span)) + } + pub(super) fn database_status(&mut self, status: &str, error: &str, span: Span) { + let typed = self.temp(); + let failed = self.label("database_fault"); + let checked = self.label("database_status"); + self.line(format!("{typed} = icmp eq i32 {status}, 1")); + self.line(format!("br i1 {typed}, label %{failed}, label %{checked}")); + self.start(&failed); + self.exit("1"); + self.start(&checked); + self.check_runtime_status(status, error, span); + } + + pub(super) fn database_call( + &mut self, + symbol: &str, + mut arguments: Vec<String>, + layout: &str, + span: Span, + ) -> String { + let output = self.entry_slot_ir(layout); + let error = self.entry_slot_ir("{ ptr, i64 }"); + let errors = self.database_errors(span); + arguments.extend([ + format!("ptr {output}"), + format!("ptr {errors}"), + "ptr %fault".into(), + format!("ptr {error}"), + ]); + let status = self.temp(); + self.line(format!( + "{status} = call i32 @dever_rt_v1_{symbol}({})", + arguments.join(", ") + )); + self.database_status(&status, &error, span); + let value = self.temp(); + self.line(format!("{value} = load {layout}, ptr {output}")); + value + } + + pub(super) fn database_operation( + &mut self, + symbol: &str, + mut arguments: Vec<String>, + span: Span, + ) -> String { + let errors = self.database_errors(span); + arguments.push(format!("ptr {errors}")); + self.async_operation(symbol, arguments, span) + } + + pub(super) fn database_fault(&mut self, kind: u32, message: &str, span: Span) { + let text = self.application_text(message, span); + let errors = self.database_errors(span); + let error = self.entry_slot_ir("{ ptr, i64 }"); + let status = self.temp(); + self.line(format!("{status} = call i32 @dever_rt_v1_db_error(i32 {kind}, ptr {text}, ptr {errors}, ptr %fault, ptr {error})")); + self.database_status(&status, &error, span); + self.line("unreachable"); + } + + pub(super) fn model_database(&mut self, model: usize) -> String { + if self.module.api_enabled() { + let span = self.module.program.functions[self.instance.function].span; + let session = self.temp(); + self.line(format!("{session} = load ptr, ptr @dever_database_session")); + let binding = self.module.database_binding(model); + let slot = self.entry_slot_ir("%dever.db_binding"); + self.line(format!("store %dever.db_binding {binding}, ptr {slot}")); + let operation = self.database_operation( + "db_resolve_scoped", + vec![format!("ptr {session}"), format!("ptr {slot}")], + span, + ); + let database = self.await_database_handle(&operation, span); + self.protocol_owner("db", &database); + return database; + } + let database = self.temp(); + self.line(format!( + "{database} = load ptr, ptr @dever_database_{model}" + )); + database + } +} diff --git a/crates/dever-core/src/llvm/database_rows.rs b/crates/dever-core/src/llvm/database_rows.rs new file mode 100644 index 0000000..fb6d463 --- /dev/null +++ b/crates/dever-core/src/llvm/database_rows.rs @@ -0,0 +1,356 @@ +//! Concrete Model/SQL row codecs and borrowed persistence parameter arrays. +use super::*; +use crate::model::ModelFieldType; + +impl FunctionEmitter<'_, '_> { + pub(super) fn database_parameters( + &mut self, + values: &[(Type, ModelFieldType, String)], + span: Span, + ) -> String { + let count = values.len(); + let pointer = self.entry_slot_ir(&format!("[{count} x %dever.db_value]")); + for (index, (ty, storage, value)) in values.iter().enumerate() { + let encoded = self.database_parameter(ty, storage, value, span); + let destination = self.temp(); + self.line(format!("{destination} = getelementptr [{count} x %dever.db_value], ptr {pointer}, i32 0, i64 {index}")); + self.line(format!( + "store %dever.db_value {encoded}, ptr {destination}" + )); + } + pointer + } + + fn database_parameter( + &mut self, + ty: &Type, + storage: &ModelFieldType, + value: &str, + span: Span, + ) -> String { + if let Type::Nullable(inner) = ty { + let result = self.entry_slot_ir("%dever.db_value"); + self.line(format!( + "store %dever.db_value zeroinitializer, ptr {result}" + )); + let present = self.temp(); + self.line(format!( + "{present} = extractvalue {} {value}, 0", + self.module.ty(ty) + )); + let some = self.label("database_parameter_some"); + let done = self.label("database_parameter_done"); + self.line(format!("br i1 {present}, label %{some}, label %{done}")); + self.start(&some); + let payload = self.temp(); + self.line(format!( + "{payload} = extractvalue {} {value}, 1", + self.module.ty(ty) + )); + let encoded = self.database_parameter(inner, storage, &payload, span); + self.line(format!("store %dever.db_value {encoded}, ptr {result}")); + self.line(format!("br label %{done}")); + self.start(&done); + let encoded = self.temp(); + self.line(format!("{encoded} = load %dever.db_value, ptr {result}")); + return encoded; + } + let (kind, field, layout, value) = match ty { + Type::Bool => { + let integer = self.temp(); + self.line(format!("{integer} = zext i1 {value} to i64")); + (1, 3, "i64", integer) + } + Type::Float => (3, 4, "double", value.into()), + Type::Decimal => (4, 5, "{ i64, i64 }", value.into()), + Type::Text | Type::Json => (5, 6, "ptr", value.into()), + Type::Bytes => (6, 6, "ptr", value.into()), + Type::Uuid => (7, 6, "ptr", value.into()), + Type::Named(id) if self.module.program.types[*id].kind != DefinitionKind::ModelId => { + let Shape::Choice(variants) = self.module.program.types[*id].shape.clone() else { + unreachable!() + }; + let slot = self.entry_slot(&Type::Text); + let tag = self.temp(); + self.line(format!("{tag} = extractvalue %T{id} {value}, 0")); + let done = self.label("database_choice_done"); + let arms = variants + .iter() + .enumerate() + .map(|(index, _)| (index, self.label("database_choice"))) + .collect::<Vec<_>>(); + let cases = arms + .iter() + .map(|(index, block)| format!("i32 {index}, label %{block}")) + .collect::<Vec<_>>() + .join(" "); + let invalid = self.label("database_choice_invalid"); + self.line(format!("switch i32 {tag}, label %{invalid} [{cases}]")); + self.start(&invalid); + self.line("unreachable"); + for (index, block) in arms { + self.start(&block); + let text = self.application_text(&variants[index].name, span); + self.line(format!("store ptr {text}, ptr {slot}")); + self.line(format!("br label %{done}")); + } + self.start(&done); + let text = self.temp(); + self.line(format!("{text} = load ptr, ptr {slot}")); + (5, 6, "ptr", text) + } + _ => (2, 3, "i64", value.into()), + }; + let tagged = self.temp(); + self.line(format!( + "{tagged} = insertvalue %dever.db_value zeroinitializer, i32 {kind}, 0" + )); + let mut result = self.temp(); + self.line(format!( + "{result} = insertvalue %dever.db_value {tagged}, {layout} {value}, {field}" + )); + if let ModelFieldType::Decimal { precision, scale } = storage { + for (field, value) in [(1, precision), (2, scale)] { + let next = self.temp(); + self.line(format!( + "{next} = insertvalue %dever.db_value {result}, i32 {value}, {field}" + )); + result = next; + } + } + result + } + + pub(super) fn database_scalar( + &mut self, + row: &str, + index: usize, + ty: &Type, + span: Span, + ) -> String { + let arguments = vec![format!("ptr {row}"), format!("i64 {index}")]; + if let Type::Nullable(inner) = ty { + let null = self.database_call("db_row_null", arguments, "i8", span); + let absent = self.temp(); + self.line(format!("{absent} = icmp ne i8 {null}, 0")); + let slot = self.entry_slot(ty); + self.line(format!( + "store {} zeroinitializer, ptr {slot}", + self.module.ty(ty) + )); + let some = self.label("database_column_some"); + let done = self.label("database_column_done"); + self.line(format!("br i1 {absent}, label %{done}, label %{some}")); + self.start(&some); + let value = self.database_scalar(row, index, inner, span); + let present = self.temp(); + self.line(format!( + "{present} = insertvalue {} zeroinitializer, i1 1, 0", + self.module.ty(ty) + )); + let wrapped = self.temp(); + self.line(format!( + "{wrapped} = insertvalue {} {present}, {} {value}, 1", + self.module.ty(ty), + self.module.ty(inner) + )); + self.line(format!( + "store {} {wrapped}, ptr {slot}", + self.module.ty(ty) + )); + self.line(format!("br label %{done}")); + self.start(&done); + let value = self.temp(); + self.line(format!("{value} = load {}, ptr {slot}", self.module.ty(ty))); + return value; + } + let kind = match ty { + Type::Bool => "bool", + Type::Float => "float", + Type::Decimal => "decimal", + Type::Text | Type::Json => "text", + Type::Bytes => "bytes", + Type::Uuid => "uuid", + Type::Named(id) if self.module.program.types[*id].kind != DefinitionKind::ModelId => { + "text" + } + _ => "int", + }; + let choice = matches!(ty, Type::Named(id) if self.module.program.types[*id].kind != DefinitionKind::ModelId); + let layout = if *ty == Type::Bool { + "i8".into() + } else if choice { + "ptr".into() + } else { + self.module.ty(ty) + }; + let value = self.database_call(&format!("db_row_{kind}"), arguments, &layout, span); + if *ty == Type::Bool { + let boolean = self.temp(); + self.line(format!("{boolean} = icmp ne i8 {value}, 0")); + return boolean; + } + if choice { + let Type::Named(id) = ty else { unreachable!() }; + let Shape::Choice(variants) = self.module.program.types[*id].shape.clone() else { + unreachable!() + }; + let owner = self.own_value(&Type::Text, &value, false); + let output = self.entry_slot(ty); + let done = self.label("database_choice_read"); + for (index, variant) in variants.iter().enumerate() { + let expected = self.application_text(&variant.name, span); + let equal = self.temp(); + self.line(format!( + "{equal} = call i8 @dever_rt_v1_text_equal(ptr {value}, ptr {expected})" + )); + let matched = self.temp(); + self.line(format!("{matched} = icmp ne i8 {equal}, 0")); + let success = self.label("database_choice_found"); + let next = self.label("database_choice_next"); + self.line(format!("br i1 {matched}, label %{success}, label %{next}")); + self.start(&success); + self.write_variant(*id, index, &[], &output); + self.line(format!("br label %{done}")); + self.start(&next); + } + let prefix = self.application_text("invalid database choice '", span); + let text = self.runtime_call( + "dever_rt_v1_text_concat", + vec![format!("ptr {prefix}"), format!("ptr {value}")], + &Type::Text, + false, + None, + span, + ); + self.own_value(&Type::Text, &text, false); + let suffix = self.application_text("'", span); + let message = self.runtime_call( + "dever_rt_v1_text_concat", + vec![format!("ptr {text}"), format!("ptr {suffix}")], + &Type::Text, + false, + None, + span, + ); + self.own_value(&Type::Text, &message, false); + let errors = self.database_errors(span); + let error = self.entry_slot_ir("{ ptr, i64 }"); + let status = self.temp(); + self.line(format!("{status} = call i32 @dever_rt_v1_db_error(i32 8, ptr {message}, ptr {errors}, ptr %fault, ptr {error})")); + self.database_status(&status, &error, span); + self.line("unreachable"); + self.start(&done); + self.release_guard(owner); + let decoded = self.temp(); + self.line(format!("{decoded} = load %T{id}, ptr {output}")); + return decoded; + } + value + } + + pub(super) fn database_record( + &mut self, + row: &str, + id: usize, + offset: usize, + exact: bool, + span: Span, + ) -> String { + let Shape::Record(fields) = self.module.program.types[id].shape.clone() else { + unreachable!() + }; + let slot = self.entry_slot(&Type::Named(id)); + self.line(format!("store %T{id} zeroinitializer, ptr {slot}")); + let guard = self.register_guard(&Type::Named(id), slot.clone()); + self.mark_live(guard); + let actual = self.protocol_call( + "dever_rt_v1_db_row_len", + vec![format!("ptr {row}")], + "i64", + span, + ); + let record_name = self + .module + .program + .models + .iter() + .find(|model| model.record == id) + .map(|model| model.package.clone()) + .unwrap_or_else(|| self.module.program.types[id].name.clone()); + let mut column = offset; + for (index, field) in fields.iter().enumerate() { + if matches!(field.ty, Type::Related(_)) { + continue; + } + self.database_column_present( + &actual, + column, + &format!("{record_name}.{}", field.name), + span, + ); + let value = self.database_scalar(row, column, &field.ty, span); + let destination = self.temp(); + self.line(format!( + "{destination} = getelementptr %T{id}, ptr {slot}, i32 0, i32 {index}" + )); + self.line(format!( + "store {} {value}, ptr {destination}", + self.module.ty(&field.ty) + )); + column += 1; + } + if exact { + self.database_row_length( + row, + column, + &format!("database row for {record_name} has extra columns"), + span, + ); + } + let value = self.temp(); + self.line(format!("{value} = load %T{id}, ptr {slot}")); + self.disarm(guard); + value + } + + pub(super) fn database_column_present( + &mut self, + actual: &str, + index: usize, + name: &str, + span: Span, + ) { + let present = self.temp(); + self.line(format!("{present} = icmp sgt i64 {actual}, {index}")); + let found = self.label("database_column_found"); + let missing = self.label("database_column_missing"); + self.line(format!("br i1 {present}, label %{found}, label %{missing}")); + self.start(&missing); + self.database_fault(8, &format!("database row is missing column '{name}'"), span); + self.start(&found); + } + + pub(super) fn database_row_length( + &mut self, + row: &str, + length: usize, + message: &str, + span: Span, + ) { + let actual = self.protocol_call( + "dever_rt_v1_db_row_len", + vec![format!("ptr {row}")], + "i64", + span, + ); + let equal = self.temp(); + self.line(format!("{equal} = icmp eq i64 {actual}, {length}")); + let okay = self.label("database_row_exact"); + let invalid = self.label("database_row_extra"); + self.line(format!("br i1 {equal}, label %{okay}, label %{invalid}")); + self.start(&invalid); + self.database_fault(8, message, span); + self.start(&okay); + } +} diff --git a/crates/dever-core/src/llvm/database_schema.rs b/crates/dever-core/src/llvm/database_schema.rs new file mode 100644 index 0000000..fe07b01 --- /dev/null +++ b/crates/dever-core/src/llvm/database_schema.rs @@ -0,0 +1,548 @@ +//! Static schema metadata shares the native backend's SQL/DDL owner. +use super::*; +use crate::model::{DatabaseOwner, ModelFieldType, ModelSchema, ModelScope, ModelValue}; +use crate::native::orm as sql; + +pub(super) struct Migration { + pub bindings: String, + pub models: String, + pub errors: String, + pub count: usize, +} + +pub(super) const DECLARATIONS: &str = "\ +%dever.db_binding = type { ptr, { ptr, i64 }, i8 } +%dever.db_field = type { { ptr, i64 }, { ptr, i64 }, i8, i8, ptr, ptr } +%dever.db_index = type { { ptr, i64 }, ptr, i64, i8 } +%dever.db_migration = type { { ptr, i64 }, { ptr, i64 }, ptr, i64 } +%dever.db_seed = type { %dever.db_sql, ptr, i64 } +%dever.db_data_migration = type { { ptr, i64 }, i32, %dever.db_seed } +%dever.db_pg_column = type { { ptr, i64 }, { ptr, i64 }, { ptr, i64 }, ptr } +%dever.db_pg_constraint = type { { ptr, i64 }, { ptr, i64 }, i8 } +%dever.db_model = type { { ptr, i64 }, { ptr, i64 }, { ptr, i64 }, { ptr, i64 }, ptr, i64, ptr, i64, ptr, i64, %dever.db_sql, { ptr, i64 }, ptr, i64, ptr, i64, ptr, i64, ptr, i64, ptr, i64 } +declare ptr @dever_rt_v1_db_migrate_models(ptr, ptr, ptr, i64, ptr) +declare ptr @dever_rt_v1_api_migrate_models(ptr, ptr, ptr, ptr, i64, i64, ptr) +"; + +impl Module<'_> { + fn database_name(&mut self, name: &str) -> String { + let (pointer, length) = self.text_literal(name); + format!("{{ ptr, i64 }} {{ ptr {pointer}, i64 {length} }}") + } + + fn database_optional_name(&mut self, name: Option<&str>) -> String { + name.map(|name| self.wire_names(&[name.into()])) + .unwrap_or_else(|| "null".into()) + } + + fn database_array(&mut self, ty: &str, values: &[String]) -> String { + if values.is_empty() { + return "null".into(); + } + let name = format!("@dever_db_metadata_{}", self.next_literal); + self.next_literal += 1; + let body = values + .iter() + .map(|value| format!("{ty} {value}")) + .collect::<Vec<_>>() + .join(", "); + writeln!( + self.declarations, + "{name} = private constant [{} x {ty}] [{body}]", + values.len() + ) + .unwrap(); + name + } + + pub(super) fn database_binding(&mut self, index: usize) -> String { + let (explicit, root) = self.program.database_selector(DatabaseOwner::Model(index)); + let explicit = explicit.map(str::to_owned); + let root = root.to_owned(); + let explicit = self.database_optional_name(explicit.as_deref()); + let root = self.database_name(&root); + let tenant = u8::from(self.program.models[index].scope == ModelScope::Tenant); + format!("{{ ptr {explicit}, {root}, i8 {tenant} }}") + } + + pub(super) fn emit_database_slots(&mut self) { + self.declarations.push_str(DECLARATIONS); + self.declarations.push_str("@dever_database_session = private global ptr null\ndefine internal void @dever_db_append_cause(ptr %fault, ptr %message) {\nentry:\n %cause = getelementptr %dever.fault, ptr %fault, i32 0, i32 8\n call void @dever_rt_v1_db_cause_append(ptr %cause, ptr %message)\n ret void\n}\n"); + for index in 0..self.program.models.len() { + writeln!( + self.declarations, + "@dever_database_{index} = private global ptr null" + ) + .unwrap(); + } + } + + fn database_model(&mut self, model: &ModelSchema) -> String { + let fields = model + .fields + .iter() + .map(|field| { + let name = self.database_name(&field.name); + let ty = self.database_name(&crate::model::field_type(&field.ty)); + let default = field.default.as_ref().map(crate::model::value); + let default = self.database_optional_name(default.as_deref()); + let rename = self.database_optional_name(field.rename_from.as_deref()); + format!( + "{{ {name}, {ty}, i8 {}, i8 {}, ptr {default}, ptr {rename} }}", + u8::from(field.nullable), + u8::from(field.generated) + ) + }) + .collect::<Vec<_>>(); + let fields = self.database_array("%dever.db_field", &fields); + let indexes = model + .indexes + .iter() + .filter(|index| index.fields.as_slice() != ["id"]) + .map(|index| { + let name = self.database_name(&sql::index_name(model, index)); + let fields = self.wire_names(&index.fields); + format!( + "{{ {name}, ptr {fields}, i64 {}, i8 {} }}", + index.fields.len(), + u8::from(index.unique) + ) + }) + .collect::<Vec<_>>(); + let index_count = indexes.len(); + let indexes = self.database_array("%dever.db_index", &indexes); + let migrations = model + .migrations + .iter() + .map(|migration| { + let name = self.database_name(&migration.name); + let revision = self.database_name(&migration.revision()); + let drops = self.wire_names(&migration.drops); + format!( + "{{ {name}, {revision}, ptr {drops}, i64 {} }}", + migration.drops.len() + ) + }) + .collect::<Vec<_>>(); + let migrations = self.database_array("%dever.db_migration", &migrations); + let constraints = sql::postgres_constraints(model, &self.program.models); + let pg_table = sql::postgres_table(model, &constraints, &model.table); + let sqlite_table = sql::sqlite_table(model, &self.program.models, &model.table); + let sqlite_table = self.database_name(&sqlite_table); + let pg_table = self.database_name(&pg_table); + let temporary = format!("_dever_{}_{}", model.table, &model.revision[..8]); + let temporary = sql::sqlite_table(model, &self.program.models, &temporary); + let temporary = self.database_name(&temporary); + let create_indexes = sql::sqlite_indexes(model) + .iter() + .map(|query| { + let name = self.database_name(query); + format!("{{ {name}, {name} }}") + }) + .collect::<Vec<_>>(); + let create_index_count = create_indexes.len(); + let create_indexes = self.database_array("%dever.db_sql", &create_indexes); + let columns = model + .fields + .iter() + .map(|field| { + let name = self.database_name(&field.name); + let definition = self.database_name(&sql::postgres_column(field)); + let ty = self.database_name(&sql::postgres_type(&field.ty)); + let default = sql::postgres_field_default(field); + let default = self.database_optional_name(default.as_deref()); + format!("{{ {name}, {definition}, {ty}, ptr {default} }}") + }) + .collect::<Vec<_>>(); + let columns = self.database_array("%dever.db_pg_column", &columns); + let constraints = constraints + .iter() + .map(|(name, definition, foreign)| { + let name = self.database_name(name); + let definition = self.database_name(definition); + format!("{{ {name}, {definition}, i8 {} }}", u8::from(*foreign)) + }) + .collect::<Vec<_>>(); + let constraint_count = constraints.len(); + let constraints = self.database_array("%dever.db_pg_constraint", &constraints); + let name = self.database_name(&model.package); + let table = self.database_name(&model.table); + let revision = self.database_name(&model.revision); + let seed_revision = self.database_name(&model.seed_revision); + format!( + "{{ {name}, {table}, {revision}, {seed_revision}, ptr {fields}, i64 {}, ptr {indexes}, i64 {index_count}, ptr {migrations}, i64 {}, %dever.db_sql {{ {sqlite_table}, {pg_table} }}, {temporary}, ptr {create_indexes}, i64 {create_index_count}, ptr {columns}, i64 {}, ptr {constraints}, i64 {constraint_count}, ptr null, i64 0, ptr null, i64 0 }}", + model.fields.len(), + model.migrations.len(), + model.fields.len() + ) + } +} + +impl FunctionEmitter<'_, '_> { + pub(super) fn initialize_database( + &mut self, + span: Span, + service_start: &str, + ) -> Option<Migration> { + if !self.module.database_enabled() { + if self.module.api_enabled() { + self.initialize_api("null", 0, "null", 0, service_start, span); + } + return None; + } + let models = self.module.program.models.clone(); + let mut bindings = (0..models.len()) + .map(|index| self.module.database_binding(index)) + .collect::<Vec<_>>(); + for job in self.module.program.jobs.clone() { + bindings.push(self.module.owner_binding(DatabaseOwner::Job(job.function))); + } + let binding_count = bindings.len(); + let bindings = self.module.database_array("%dever.db_binding", &bindings); + let mut transactions = self + .module + .instances + .iter() + .filter(|instance| { + self.module.program.functions[instance.function].kind == FunctionKind::Transaction + }) + .cloned() + .collect::<Vec<_>>(); + transactions.extend( + self.module + .program + .api_commands + .iter() + .map(|command| Specialization { + function: command.function, + handlers: Vec::new(), + }) + .filter(|instance| specialize::writes_database(self.module.program, instance)), + ); + transactions.extend( + self.module + .program + .api_routes + .iter() + .filter(|route| matches!(route.method, "POST" | "PUT" | "DELETE")) + .map(|route| Specialization { + function: route.function, + handlers: Vec::new(), + }) + .filter(|instance| specialize::writes_database(self.module.program, instance)), + ); + if let Some(application) = &self.module.application { + transactions.extend( + application + .rest_handlers + .iter() + .filter(|handler| handler.method != "GET") + .map(|handler| Specialization { + function: handler.function, + handlers: Vec::new(), + }), + ); + } + transactions.extend( + self.module + .program + .jobs + .iter() + .map(|job| Specialization { + function: job.function, + handlers: Vec::new(), + }) + .filter(|instance| specialize::writes_database(self.module.program, instance)), + ); + let groups = transactions + .iter() + .map(|instance| { + let owners = specialize::database_effects(self.module.program, instance); + let values = owners + .iter() + .map(|owner| self.module.owner_binding(*owner)) + .collect::<Vec<_>>(); + let pointer = self.module.database_array("%dever.db_binding", &values); + format!("{{ ptr {pointer}, i64 {} }}", values.len()) + }) + .collect::<Vec<_>>(); + let transactions = self.module.database_array("{ ptr, i64 }", &groups); + let session = if self.module.api_enabled() { + let api_session = self.initialize_api( + &bindings, + binding_count, + &transactions, + groups.len(), + service_start, + span, + ); + self.protocol_call( + "dever_rt_v1_api_session_database", + vec![format!("ptr {api_session}")], + "ptr", + span, + ) + } else { + self.protocol_call( + "dever_rt_v1_db_session_new", + vec![ + format!("ptr {bindings}"), + format!("i64 {binding_count}"), + format!("ptr {transactions}"), + format!("i64 {}", groups.len()), + ], + "ptr", + span, + ) + }; + self.line(format!("store ptr {session}, ptr @dever_database_session")); + self.initialize_jobs(span); + if !self.module.api_enabled() { + let prepare = + self.database_operation("db_prepare", vec![format!("ptr {session}")], span); + self.await_operation(&prepare, &Type::Unit, span, false); + } + let database_array = self.entry_slot_ir(&format!("[{} x ptr]", models.len())); + let model_array = self.entry_slot_ir(&format!("[{} x %dever.db_model]", models.len())); + let error_array = self.entry_slot_ir(&format!("[{} x ptr]", models.len())); + for (index, model) in models.iter().enumerate() { + let span = self.module.program.types[model.record].span; + let binding = self.temp(); + self.line(format!("{binding} = getelementptr [{} x %dever.db_binding], ptr {bindings}, i32 0, i64 {index}", models.len())); + let database = if self.module.api_enabled() { + "null".into() + } else { + self.database_call( + "db_select", + vec![format!("ptr {session}"), format!("ptr {binding}")], + "ptr", + span, + ) + }; + self.line(format!("store ptr {database}, ptr @dever_database_{index}")); + let database_slot = self.temp(); + self.line(format!("{database_slot} = getelementptr [{} x ptr], ptr {database_array}, i32 0, i64 {index}", models.len())); + self.line(format!("store ptr {database}, ptr {database_slot}")); + let model_slot = self.temp(); + self.line(format!("{model_slot} = getelementptr [{} x %dever.db_model], ptr {model_array}, i32 0, i64 {index}", models.len())); + let value = self.module.database_model(model); + self.line(format!("store %dever.db_model {value}, ptr {model_slot}")); + self.database_seeds(model, &model_slot, span); + let errors = self.database_errors(span); + let error_slot = self.temp(); + self.line(format!( + "{error_slot} = getelementptr [{} x ptr], ptr {error_array}, i32 0, i64 {index}", + models.len() + )); + self.line(format!("store ptr {errors}, ptr {error_slot}")); + } + let operation = if self.module.api_enabled() { + let api_session = self.temp(); + self.line(format!("{api_session} = load ptr, ptr @dever_api_session")); + let mut arguments = vec![ + format!("ptr {api_session}"), + format!("ptr {bindings}"), + format!("ptr {model_array}"), + format!("ptr {error_array}"), + format!("i64 {}", models.len()), + ]; + let symbol = if self.module.jobs_enabled() { + let errors = self.database_errors(span); + arguments.push(format!("ptr {errors}")); + "api_migrate_application" + } else { + "api_migrate_models" + }; + arguments.push("i64 0".into()); + self.async_operation(symbol, arguments, span) + } else { + self.async_operation( + "db_migrate_models", + vec![ + format!("ptr {database_array}"), + format!("ptr {model_array}"), + format!("ptr {error_array}"), + format!("i64 {}", models.len()), + ], + span, + ) + }; + self.await_operation(&operation, &Type::Unit, span, false); + Some(Migration { + bindings, + models: model_array, + errors: error_array, + count: models.len(), + }) + } + + fn database_seed_value( + &mut self, + storage: &ModelFieldType, + value: &ModelValue, + span: Span, + ) -> (Type, ModelFieldType, String) { + let (ty, value) = match value { + ModelValue::Null => ( + Type::Nullable(Box::new(Type::Int)), + "zeroinitializer".into(), + ), + ModelValue::Bool(value) => (Type::Bool, u8::from(*value).to_string()), + ModelValue::Number(value) => match storage { + ModelFieldType::Float => ( + Type::Float, + format!( + "0x{:016X}", + value.parse::<f64>().expect("checked seed Float").to_bits() + ), + ), + ModelFieldType::Decimal { .. } => { + let decimal = + dever_runtime::number::parse_decimal(value).expect("checked seed Decimal"); + let bytes = decimal.to_bytes(); + let low = i64::from_le_bytes(bytes[..8].try_into().unwrap()); + let high = i64::from_le_bytes(bytes[8..].try_into().unwrap()); + (Type::Decimal, format!("{{ i64 {low}, i64 {high} }}")) + } + _ => (Type::Int, value.clone()), + }, + ModelValue::Text(value) | ModelValue::Choice(value) => { + let value = if matches!(storage, ModelFieldType::Choice(_)) { + value.rsplit('.').next().unwrap() + } else { + value.as_str() + }; + let text = self.application_text(value, span); + if matches!(storage, ModelFieldType::Uuid) { + let uuid = self.uuid_from_literal_text(&text, span); + self.own_value(&Type::Uuid, &uuid, false); + (Type::Uuid, uuid) + } else { + (Type::Text, text) + } + } + }; + (ty, storage.clone(), value) + } + + fn database_seeds(&mut self, model: &ModelSchema, model_slot: &str, span: Span) { + let mut seeds = Vec::new(); + for seed in &model.seed { + let mut columns = Vec::new(); + let mut values = Vec::new(); + for (name, value) in &seed.fields { + let storage = &model + .fields + .iter() + .find(|field| field.name == *name) + .unwrap() + .ty; + columns.push(sql::quoted(name)); + values.push(self.database_seed_value(storage, value, span)); + } + for field in model + .fields + .iter() + .filter(|field| field.generated && field.ty == ModelFieldType::Uuid) + { + columns.push(sql::quoted(&field.name)); + let uuid = self.database_call("db_uuid_new", vec![], "ptr", span); + self.own_value(&Type::Uuid, &uuid, false); + values.push((Type::Uuid, ModelFieldType::Uuid, uuid)); + } + let placeholders = values + .iter() + .enumerate() + .map(|(index, (_, storage, _))| sql::parameter_sql(index + 1, storage)) + .collect::<Vec<_>>() + .join(", "); + let conflict = seed + .identity + .iter() + .map(|name| sql::quoted(name)) + .collect::<Vec<_>>() + .join(", "); + let statement = format!( + "INSERT INTO {} ({}) VALUES ({placeholders}) ON CONFLICT ({conflict}) DO NOTHING", + sql::quoted(&model.table), + columns.join(", ") + ); + let query = self.module.planned_sql(&statement); + seeds.push(self.database_seed(&query, &values, span)); + } + self.database_schema_array(model_slot, 18, "%dever.db_seed", &seeds); + let mut migrations = Vec::new(); + for migration in &model.migrations { + for statement in &migration.statements { + let values = statement + .parameters + .iter() + .map(|(storage, value)| self.database_seed_value(storage, value, span)) + .collect::<Vec<_>>(); + let query = self + .module + .database_sql(&statement.sqlite, &statement.postgres); + let seed = self.database_seed(&query, &values, span); + let name = self.module.database_name(&migration.name); + let phase = u8::from(matches!( + statement.phase, + crate::syntax::MigrationPhase::After + )); + let value = self.temp(); + self.line(format!("{value} = insertvalue %dever.db_data_migration {{ {name}, i32 {phase}, %dever.db_seed zeroinitializer }}, %dever.db_seed {seed}, 2")); + migrations.push(value); + } + } + self.database_schema_array(model_slot, 20, "%dever.db_data_migration", &migrations); + } + + fn database_seed( + &mut self, + query: &str, + values: &[(Type, ModelFieldType, String)], + span: Span, + ) -> String { + let parameters = self.database_parameters(values, span); + let sql = self.temp(); + self.line(format!("{sql} = load %dever.db_sql, ptr {query}")); + let mut result = "zeroinitializer".to_string(); + for (index, ty, value) in [ + (0, "%dever.db_sql", sql), + (1, "ptr", parameters), + (2, "i64", values.len().to_string()), + ] { + let next = self.temp(); + self.line(format!( + "{next} = insertvalue %dever.db_seed {result}, {ty} {value}, {index}" + )); + result = next; + } + result + } + + fn database_schema_array(&mut self, model: &str, field: usize, ty: &str, values: &[String]) { + if values.is_empty() { + return; + } + let count = values.len(); + let array = self.entry_slot_ir(&format!("[{count} x {ty}]")); + for (index, value) in values.iter().enumerate() { + let slot = self.temp(); + self.line(format!( + "{slot} = getelementptr [{count} x {ty}], ptr {array}, i32 0, i64 {index}" + )); + self.line(format!("store {ty} {value}, ptr {slot}")); + } + let pointer = self.temp(); + self.line(format!( + "{pointer} = getelementptr %dever.db_model, ptr {model}, i32 0, i32 {field}" + )); + self.line(format!("store ptr {array}, ptr {pointer}")); + let length = self.temp(); + self.line(format!( + "{length} = getelementptr %dever.db_model, ptr {model}, i32 0, i32 {}", + field + 1 + )); + self.line(format!("store i64 {count}, ptr {length}")); + } +} diff --git a/crates/dever-core/src/llvm/expressions.rs b/crates/dever-core/src/llvm/expressions.rs new file mode 100644 index 0000000..6fb5562 --- /dev/null +++ b/crates/dever-core/src/llvm/expressions.rs @@ -0,0 +1,984 @@ +use super::*; + +impl FunctionEmitter<'_, '_> { + pub(super) fn atom(&mut self, atom: &Atom, ty: &Type, value: &str) -> Result<String, String> { + if matches!(atom, Atom::Null) { + let present = self.temp(); + self.line(format!( + "{present} = extractvalue {} {value}, 0", + self.module.ty(ty) + )); + let absent = self.temp(); + self.line(format!("{absent} = xor i1 {present}, 1")); + return Ok(absent); + } + if let Type::Nullable(inner) = ty { + let present = self.temp(); + self.line(format!( + "{present} = extractvalue {} {value}, 0", + self.module.ty(ty) + )); + let payload = self.temp(); + self.line(format!( + "{payload} = extractvalue {} {value}, 1", + self.module.ty(ty) + )); + let matched = self.atom(atom, inner, &payload)?; + let result = self.temp(); + self.line(format!("{result} = and i1 {present}, {matched}")); + return Ok(result); + } + Ok(match atom { + Atom::Any => "1".into(), + Atom::Bool(expected) => { + self.compare("icmp eq", ty, value, if *expected { "1" } else { "0" }) + } + Atom::Variant(variant) => { + if matches!(ty, Type::Related(_)) { + let matched = self.temp(); + self.line(format!( + "{matched} = icmp {} ptr {value}, null", + if *variant == 0 { "eq" } else { "ne" } + )); + return Ok(matched); + } + let tag = self.temp(); + self.line(format!( + "{tag} = extractvalue {} {value}, 0", + self.module.ty(ty) + )); + let result = self.temp(); + self.line(format!("{result} = icmp eq i32 {tag}, {variant}")); + result + } + Atom::Int(low, high) => { + let mut tests = Vec::new(); + if *low != i64::MIN { + tests.push(self.compare("icmp sge", ty, value, &low.to_string())); + } + if *high != i64::MAX { + tests.push(self.compare("icmp sle", ty, value, &high.to_string())); + } + self.reduce_bool("and", tests, "1") + } + Atom::Literal(literal) => { + let first = self.guards.len(); + let expected = self.atom_constant(literal, ty); + let equal = self.equals(ty, value, &expected)?; + self.release_from(first); + equal + } + Atom::Remainder(literals) => { + let mut tests = Vec::new(); + for literal in literals { + let first = self.guards.len(); + let expected = self.atom_constant(literal, ty); + let equal = self.equals(ty, value, &expected)?; + self.release_from(first); + let different = self.temp(); + self.line(format!("{different} = xor i1 {equal}, 1")); + tests.push(different); + } + self.reduce_bool("and", tests, "1") + } + Atom::Decimal(low, high) => { + let low = self.constant(&Constant::Decimal(*low)); + let high = self.constant(&Constant::Decimal(*high)); + let below = self.decimal_compare( + value, + &low, + BinaryOperator::GreaterEqual, + self.module.program.functions[self.instance.function].span, + ); + let above = self.decimal_compare( + value, + &high, + BinaryOperator::LessEqual, + self.module.program.functions[self.instance.function].span, + ); + self.reduce_bool("and", vec![below, above], "1") + } + Atom::Null => unreachable!("validated kernel domain"), + }) + } + + fn atom_constant(&mut self, value: &Constant, ty: &Type) -> String { + if let Constant::Text(text) = value { + let (literal, length) = self.module.text_literal(text); + let result = self.runtime_call( + "dever_rt_v1_text_new", + vec![format!("ptr {literal}"), format!("i64 {length}")], + &Type::Text, + false, + None, + self.module.program.functions[self.instance.function].span, + ); + self.own_value(&Type::Text, &result, false); + if *ty == Type::Uuid { + let uuid = self.uuid_from_literal_text( + &result, + self.module.program.functions[self.instance.function].span, + ); + self.own_value(&Type::Uuid, &uuid, false); + uuid + } else { + result + } + } else { + self.constant(value) + } + } + + fn constant(&self, constant: &Constant) -> String { + match constant { + Constant::Null => "zeroinitializer".into(), + Constant::Bool(value) => u8::from(*value).to_string(), + Constant::Int(value) => value.to_string(), + Constant::Float(value) => format!("0x{:016X}", value.to_bits()), + Constant::Decimal(value) => { + let bytes = value.to_bytes(); + let low = i64::from_le_bytes(bytes[..8].try_into().unwrap()); + let high = i64::from_le_bytes(bytes[8..].try_into().unwrap()); + format!("{{ i64 {low}, i64 {high} }}") + } + Constant::Text(_) => unreachable!("Text needs an owned runtime handle"), + } + } + + pub(super) fn compare( + &mut self, + operation: &str, + ty: &Type, + left: &str, + right: &str, + ) -> String { + let result = self.temp(); + self.line(format!( + "{result} = {operation} {} {left}, {right}", + self.module.ty(ty) + )); + result + } + + pub(super) fn expression(&mut self, expression: &Expression) -> Result<String, String> { + let first = self.guards.len(); + let mut result = self.expression_raw(expression)?; + if contains_owned(self.module.program, &expression.ty) { + let field_move = matches!(expression.kind, ExpressionKind::Field { .. }) + && self.moves.contains(&(expression as *const Expression)) + && crate::native::updates::place(expression).is_some(); + let borrowed = matches!( + expression.kind, + ExpressionKind::Field { .. } + | ExpressionKind::Record { .. } + | ExpressionKind::Variant { .. } + | ExpressionKind::Some(_) + ) && !field_move + || matches!(expression.kind, ExpressionKind::Local(_)) + && !self.moves.contains(&(expression as *const Expression)); + let owner = self.own_value(&expression.ty, &result, borrowed); + if borrowed { + // A retain may return a different handle. Return the owned row + // before releasing the temporaries from which it was cloned. + result = self.load_owned_value(&expression.ty, owner); + } + self.release_between(first, owner); + } else { + self.release_from(first); + } + Ok(result) + } + + fn expression_raw(&mut self, expression: &Expression) -> Result<String, String> { + let result = match &expression.kind { + ExpressionKind::Constant(Constant::Text(value)) => { + let (literal, length) = self.module.text_literal(value); + let text = self.runtime_call( + "dever_rt_v1_text_new", + vec![format!("ptr {literal}"), format!("i64 {length}")], + &Type::Text, + false, + None, + expression.span, + ); + if expression.ty == Type::Uuid { + self.own_value(&Type::Text, &text, false); + self.uuid_from_literal_text(&text, expression.span) + } else { + text + } + } + ExpressionKind::Constant(value) => self.constant(value), + ExpressionKind::Local(slot) => { + let value = self.load_local(*slot); + if self.moves.contains(&(expression as *const Expression)) + && let Some(guard) = self.local_guards[*slot] + { + self.disarm(guard); + } + value + } + ExpressionKind::Field { value, index } => { + if self.moves.contains(&(expression as *const Expression)) + && contains_owned(self.module.program, &expression.ty) + && let Some((slot, fields)) = crate::native::updates::place(expression) + { + let (pointer, ty) = self.field_pointer(slot, &fields); + debug_assert_eq!(ty, expression.ty); + let result = self.temp(); + self.line(format!( + "{result} = load {}, ptr {pointer}", + self.module.ty(&ty) + )); + self.line(format!( + "store {} zeroinitializer, ptr {pointer}", + self.module.ty(&ty) + )); + return Ok(result); + } + let parent = self.expression(value)?; + let result = self.temp(); + self.line(format!( + "{result} = extractvalue {} {parent}, {index}", + self.module.ty(&value.ty) + )); + result + } + ExpressionKind::Call { target, arguments } => { + self.call(*target, arguments, expression)? + } + ExpressionKind::RunCall { .. } + | ExpressionKind::AwaitTask(_) + | ExpressionKind::StopTask(_) + | ExpressionKind::Group(_) + | ExpressionKind::AwaitGroup(_) + | ExpressionKind::StopGroup(_) + | ExpressionKind::ParallelCall { .. } + | ExpressionKind::BlockingCall { .. } + | ExpressionKind::Channel { .. } + | ExpressionKind::ChannelSend { .. } + | ExpressionKind::ChannelReceive(_) + | ExpressionKind::ChannelClose(_) => self.concurrency(expression)?, + ExpressionKind::CaptureResult { + target, + arguments, + choice, + success, + } => self.capture_result(*target, arguments, *choice, *success, expression)?, + ExpressionKind::Fail(error) => self.fail(error, expression.span)?, + ExpressionKind::Record { fields } => { + let mut values = Vec::with_capacity(fields.len()); + for (index, value) in fields { + values.push((*index, self.expression(value)?)); + } + let Type::Named(id) = expression.ty else { + unreachable!("checked record") + }; + let Shape::Record(definition) = &self.module.program.types[id].shape else { + unreachable!() + }; + let types = definition + .iter() + .map(|field| field.ty.clone()) + .collect::<Vec<_>>(); + let mut aggregate = "undef".to_owned(); + for (index, value) in values { + let next = self.temp(); + self.line(format!( + "{next} = insertvalue %T{id} {aggregate}, {} {value}, {index}", + self.module.ty(&types[index]) + )); + aggregate = next; + } + aggregate + } + ExpressionKind::Variant { variant, arguments } => { + if let Type::Related(inner) = &expression.ty { + if *variant == 0 { + return Ok("null".into()); + } + let value = self.expression(&arguments[0])?; + let input = self.entry_slot(inner); + self.line(format!( + "store {} {value}, ptr {input}", + self.module.ty(inner) + )); + let related = self.protocol_call( + "dever_rt_v1_db_related_new", + vec![ + format!("ptr @dever_type_{}", self.module.type_index(inner)), + format!("ptr {input}"), + ], + "ptr", + expression.span, + ); + self.own_value(&expression.ty, &related, false); + return Ok(related); + } + let mut values = Vec::with_capacity(arguments.len()); + for value in arguments { + values.push(self.expression(value)?); + } + let Type::Named(id) = expression.ty else { + unreachable!("checked variant") + }; + let Shape::Choice(variants) = &self.module.program.types[id].shape else { + unreachable!() + }; + let types = variants[*variant] + .fields + .iter() + .map(|field| field.ty.clone()) + .collect::<Vec<_>>(); + let alloca = self.temp(); + self.line(format!("{alloca} = alloca %T{id}")); + self.line(format!("store %T{id} zeroinitializer, ptr {alloca}")); + let tag = self.temp(); + self.line(format!( + "{tag} = getelementptr %T{id}, ptr {alloca}, i32 0, i32 0" + )); + self.line(format!("store i32 {variant}, ptr {tag}")); + let payload = self.temp(); + self.line(format!( + "{payload} = getelementptr %T{id}, ptr {alloca}, i32 0, i32 1" + )); + for (index, (ty, value)) in types.iter().zip(values).enumerate() { + let field = self.temp(); + self.line(format!("{field} = getelementptr %T{id}V{variant}, ptr {payload}, i32 0, i32 {index}")); + self.line(format!("store {} {value}, ptr {field}", self.module.ty(ty))); + } + let result = self.temp(); + self.line(format!("{result} = load %T{id}, ptr {alloca}")); + result + } + ExpressionKind::Some(value) => { + let inner = self.expression(value)?; + let first = self.temp(); + self.line(format!( + "{first} = insertvalue {} undef, i1 1, 0", + self.module.ty(&expression.ty) + )); + let result = self.temp(); + self.line(format!( + "{result} = insertvalue {} {first}, {} {inner}, 1", + self.module.ty(&expression.ty), + self.module.ty(&value.ty) + )); + result + } + ExpressionKind::Unary { operator, value } => { + let input = self.expression(value)?; + match (operator, &value.ty) { + (UnaryOperator::Not, Type::Bool) => { + let result = self.temp(); + self.line(format!("{result} = xor i1 {input}, 1")); + result + } + (UnaryOperator::Negate, Type::Float) => { + let result = self.temp(); + self.line(format!("{result} = fneg double {input}")); + result + } + (UnaryOperator::Negate, Type::Int) => { + self.checked_integer("ssub", "0", &input, expression.span) + } + (UnaryOperator::Negate, Type::Decimal) => { + let parts = self.decimal_parts(&input); + self.runtime_call( + "dever_rt_v1_decimal_neg", + parts, + &Type::Decimal, + false, + None, + expression.span, + ) + } + _ => unreachable!("checked unary type"), + } + } + ExpressionKind::Binary { + left, + operator, + right, + } => self.binary(left, *operator, right, expression.span)?, + ExpressionKind::Promote(value) => { + let input = self.expression(value)?; + self.runtime_call( + "dever_rt_v1_decimal_from_int", + vec![format!("i64 {input}")], + &Type::Decimal, + false, + None, + expression.span, + ) + } + ExpressionKind::Intrinsic { + operation, + handler, + arguments, + } => self.intrinsic(*operation, *handler, arguments, expression), + ExpressionKind::List(values) => self.list_literal(values, expression), + ExpressionKind::Map(entries) => self.map_literal(entries, expression), + ExpressionKind::Collection { + operation, + handler, + sequence, + arguments, + } => self.collection(*operation, *handler, *sequence, arguments, expression), + ExpressionKind::ModelOperation { model, operation } => { + self.model_operation(*model, operation, expression)? + } + ExpressionKind::JobEnqueue { + target, + arguments, + scheduled, + } => self.enqueue_job(*target, arguments, *scheduled, expression.span)?, + _ => unreachable!("validated kernel expression"), + }; + Ok(result) + } + + fn call( + &mut self, + target: CallTarget, + arguments: &[CallArgument], + expression: &Expression, + ) -> Result<String, String> { + let instance = self.call_instance(target, arguments); + let (status, result) = + self.invoke(&instance, arguments, &expression.ty, expression.span)?; + let failed = self.temp(); + self.line(format!("{failed} = icmp ne i32 {status}, 0")); + let fault = self.label("call_fault"); + let success = self.label("call_ok"); + self.line(format!("br i1 {failed}, label %{fault}, label %{success}")); + self.start(&fault); + self.append_frame(expression.span); + self.exit(&status); + self.start(&success); + let value = self.temp(); + self.line(format!( + "{value} = load {}, ptr {result}", + self.module.ty(&expression.ty) + )); + Ok(value) + } + + pub(super) fn append_frame(&mut self, span: Span) { + let location = self.module.location(span); + let depth_ptr = self.temp(); + self.line(format!( + "{depth_ptr} = getelementptr %dever.fault, ptr %fault, i32 0, i32 2" + )); + let depth = self.temp(); + self.line(format!("{depth} = load i32, ptr {depth_ptr}")); + let frame = self.temp(); + self.line(format!( + "{frame} = getelementptr %dever.fault, ptr %fault, i32 0, i32 3, i32 {depth}" + )); + self.line(format!("store i32 {location}, ptr {frame}")); + let next = self.temp(); + self.line(format!("{next} = add i32 {depth}, 1")); + self.line(format!("store i32 {next}, ptr {depth_ptr}")); + } + + pub(super) fn write_fault(&mut self, code: &str, span: Span) { + let location = self.module.location(span); + for (index, value) in [ + (0, code.to_owned()), + (1, location.to_string()), + (2, "0".into()), + ] { + let pointer = self.temp(); + self.line(format!( + "{pointer} = getelementptr %dever.fault, ptr %fault, i32 0, i32 {index}" + )); + self.line(format!("store i32 {value}, ptr {pointer}")); + } + self.exit("1"); + } + + pub(super) fn checked_integer( + &mut self, + operation: &str, + left: &str, + right: &str, + span: Span, + ) -> String { + let pair = self.temp(); + self.line(format!("{pair} = call {{ i64, i1 }} @llvm.{operation}.with.overflow.i64(i64 {left}, i64 {right})")); + let value = self.temp(); + self.line(format!("{value} = extractvalue {{ i64, i1 }} {pair}, 0")); + let overflow = self.temp(); + self.line(format!("{overflow} = extractvalue {{ i64, i1 }} {pair}, 1")); + let fault = self.label("overflow"); + let success = self.label("arithmetic_ok"); + self.line(format!( + "br i1 {overflow}, label %{fault}, label %{success}" + )); + self.start(&fault); + self.write_fault(&(FaultCode::Overflow as i32).to_string(), span); + self.start(&success); + value + } + + fn integer_division(&mut self, operation: &str, left: &str, right: &str, span: Span) -> String { + let zero = self.compare("icmp eq", &Type::Int, right, "0"); + let minimum = self.compare("icmp eq", &Type::Int, left, &i64::MIN.to_string()); + let negative_one = self.compare("icmp eq", &Type::Int, right, "-1"); + let overflow = self.temp(); + self.line(format!("{overflow} = and i1 {minimum}, {negative_one}")); + let invalid = self.temp(); + self.line(format!("{invalid} = or i1 {zero}, {overflow}")); + let fault = self.label("division_fault"); + let success = self.label("division_ok"); + self.line(format!("br i1 {invalid}, label %{fault}, label %{success}")); + self.start(&fault); + let code = self.temp(); + self.line(format!( + "{code} = select i1 {zero}, i32 {}, i32 {}", + FaultCode::DivisionByZero as i32, + FaultCode::Overflow as i32 + )); + self.write_fault(&code, span); + self.start(&success); + let result = self.temp(); + self.line(format!("{result} = {operation} i64 {left}, {right}")); + result + } + + fn decimal_compare( + &mut self, + left: &str, + right: &str, + operation: BinaryOperator, + span: Span, + ) -> String { + let mut arguments = self.decimal_parts(left); + arguments.extend(self.decimal_parts(right)); + let order = self.runtime_call( + "dever_rt_v1_decimal_compare", + arguments, + &Type::Int, + false, + Some("i32"), + span, + ); + let comparison = match operation { + BinaryOperator::Less => "slt", + BinaryOperator::LessEqual => "sle", + BinaryOperator::Greater => "sgt", + BinaryOperator::GreaterEqual => "sge", + _ => unreachable!("ordered Decimal comparison"), + }; + let result = self.temp(); + self.line(format!("{result} = icmp {comparison} i32 {order}, 0")); + result + } + + fn binary( + &mut self, + left: &Expression, + operation: BinaryOperator, + right: &Expression, + span: Span, + ) -> Result<String, String> { + let lhs = self.expression(left)?; + if matches!(operation, BinaryOperator::And | BinaryOperator::Or) { + let first = self.block.clone(); + let rhs_block = self.label("short_rhs"); + let done = self.label("short_done"); + if operation == BinaryOperator::And { + self.line(format!("br i1 {lhs}, label %{rhs_block}, label %{done}")); + } else { + self.line(format!("br i1 {lhs}, label %{done}, label %{rhs_block}")); + } + self.start(&rhs_block); + let rhs = self.expression(right)?; + let last = self.block.clone(); + self.line(format!("br label %{done}")); + self.start(&done); + let result = self.temp(); + self.line(format!( + "{result} = phi i1 [{lhs}, %{first}], [{rhs}, %{last}]" + )); + return Ok(result); + } + let rhs = self.expression(right)?; + let result = match operation { + BinaryOperator::Equal | BinaryOperator::NotEqual => { + let equal = self.equals(&left.ty, &lhs, &rhs)?; + if operation == BinaryOperator::Equal { + equal + } else { + let different = self.temp(); + self.line(format!("{different} = xor i1 {equal}, 1")); + different + } + } + BinaryOperator::Less + | BinaryOperator::LessEqual + | BinaryOperator::Greater + | BinaryOperator::GreaterEqual => { + if matches!(left.ty, Type::Decimal) { + return Ok(self.decimal_compare(&lhs, &rhs, operation, span)); + } + if matches!(left.ty, Type::Text) { + let order = self.runtime_call( + "dever_rt_v1_text_compare", + vec![format!("ptr {lhs}"), format!("ptr {rhs}")], + &Type::Int, + false, + Some("i32"), + span, + ); + let comparison = match operation { + BinaryOperator::Less => "slt", + BinaryOperator::LessEqual => "sle", + BinaryOperator::Greater => "sgt", + BinaryOperator::GreaterEqual => "sge", + _ => unreachable!(), + }; + let result = self.temp(); + self.line(format!("{result} = icmp {comparison} i32 {order}, 0")); + return Ok(result); + } + let comparison = match (&left.ty, operation) { + ( + Type::Int | Type::DateTime | Type::Date | Type::Time | Type::Duration, + BinaryOperator::Less, + ) => "icmp slt", + ( + Type::Int | Type::DateTime | Type::Date | Type::Time | Type::Duration, + BinaryOperator::LessEqual, + ) => "icmp sle", + ( + Type::Int | Type::DateTime | Type::Date | Type::Time | Type::Duration, + BinaryOperator::Greater, + ) => "icmp sgt", + ( + Type::Int | Type::DateTime | Type::Date | Type::Time | Type::Duration, + BinaryOperator::GreaterEqual, + ) => "icmp sge", + (Type::Float, BinaryOperator::Less) => "fcmp olt", + (Type::Float, BinaryOperator::LessEqual) => "fcmp ole", + (Type::Float, BinaryOperator::Greater) => "fcmp ogt", + (Type::Float, BinaryOperator::GreaterEqual) => "fcmp oge", + _ => unreachable!("checked ordered comparison"), + }; + self.compare(comparison, &left.ty, &lhs, &rhs) + } + BinaryOperator::Add + | BinaryOperator::Subtract + | BinaryOperator::Multiply + | BinaryOperator::Divide + | BinaryOperator::IntegerDivide + | BinaryOperator::Remainder => match &left.ty { + Type::Int => match operation { + BinaryOperator::Add => self.checked_integer("sadd", &lhs, &rhs, span), + BinaryOperator::Subtract => self.checked_integer("ssub", &lhs, &rhs, span), + BinaryOperator::Multiply => self.checked_integer("smul", &lhs, &rhs, span), + BinaryOperator::IntegerDivide => { + self.integer_division("sdiv", &lhs, &rhs, span) + } + BinaryOperator::Remainder => self.integer_division("srem", &lhs, &rhs, span), + _ => unreachable!("checked Int operation"), + }, + Type::Float => { + let operation = match operation { + BinaryOperator::Add => "fadd", + BinaryOperator::Subtract => "fsub", + BinaryOperator::Multiply => "fmul", + BinaryOperator::Divide => "fdiv", + BinaryOperator::Remainder => "frem", + _ => unreachable!("checked Float operation"), + }; + let result = self.temp(); + self.line(format!("{result} = {operation} double {lhs}, {rhs}")); + result + } + Type::Decimal => { + let symbol = match operation { + BinaryOperator::Add => "decimal_add", + BinaryOperator::Subtract => "decimal_sub", + BinaryOperator::Multiply => "decimal_mul", + BinaryOperator::Divide => "decimal_div", + _ => unreachable!("checked Decimal operation"), + }; + let mut arguments = self.decimal_parts(&lhs); + arguments.extend(self.decimal_parts(&rhs)); + self.runtime_call( + &format!("dever_rt_v1_{symbol}"), + arguments, + &Type::Decimal, + false, + None, + span, + ) + } + Type::Text => self.runtime_call( + "dever_rt_v1_text_concat", + vec![format!("ptr {lhs}"), format!("ptr {rhs}")], + &Type::Text, + false, + None, + span, + ), + _ => unreachable!("checked arithmetic type"), + }, + BinaryOperator::And | BinaryOperator::Or => unreachable!(), + }; + Ok(result) + } + + pub(super) fn equals(&mut self, ty: &Type, left: &str, right: &str) -> Result<String, String> { + match ty { + Type::Bool | Type::Int | Type::DateTime | Type::Date | Type::Time | Type::Duration => { + Ok(self.compare("icmp eq", ty, left, right)) + } + Type::Float => Ok(self.compare("fcmp oeq", ty, left, right)), + Type::Decimal => { + let mut arguments = self.decimal_parts(left); + arguments.extend(self.decimal_parts(right)); + let raw = self.temp(); + self.line(format!( + "{raw} = call i8 @dever_rt_v1_decimal_equal({})", + arguments.join(", ") + )); + let equal = self.temp(); + self.line(format!("{equal} = icmp ne i8 {raw}, 0")); + Ok(equal) + } + Type::Text + | Type::Json + | Type::Id + | Type::Uuid + | Type::Bytes + | Type::List(_) + | Type::Related(_) + | Type::Map(_, _) => { + let kind = match ty { + Type::Text | Type::Json | Type::Id => "text", + Type::Uuid => "uuid", + Type::Bytes => "bytes", + Type::List(_) => "list", + Type::Related(_) => "db_related", + Type::Map(_, _) => "map", + _ => unreachable!(), + }; + let raw = self.temp(); + self.line(format!( + "{raw} = call i8 @dever_rt_v1_{kind}_equal(ptr {left}, ptr {right})" + )); + let equal = self.temp(); + self.line(format!("{equal} = icmp ne i8 {raw}, 0")); + Ok(equal) + } + Type::Nullable(inner) => { + let left_present = self.temp(); + self.line(format!( + "{left_present} = extractvalue {} {left}, 0", + self.module.ty(ty) + )); + let right_present = self.temp(); + self.line(format!( + "{right_present} = extractvalue {} {right}, 0", + self.module.ty(ty) + )); + let same_presence = + self.compare("icmp eq", &Type::Bool, &left_present, &right_present); + let both = self.temp(); + self.line(format!("{both} = and i1 {left_present}, {right_present}")); + let skip = self.block.clone(); + let compare = self.label("some_eq"); + let done = self.label("nullable_eq"); + self.line(format!("br i1 {both}, label %{compare}, label %{done}")); + self.start(&compare); + let lhs = self.temp(); + self.line(format!( + "{lhs} = extractvalue {} {left}, 1", + self.module.ty(ty) + )); + let rhs = self.temp(); + self.line(format!( + "{rhs} = extractvalue {} {right}, 1", + self.module.ty(ty) + )); + let equal = self.equals(inner, &lhs, &rhs)?; + let compared = self.block.clone(); + self.line(format!("br label %{done}")); + self.start(&done); + let result = self.temp(); + self.line(format!( + "{result} = phi i1 [{same_presence}, %{skip}], [{equal}, %{compared}]" + )); + Ok(result) + } + Type::Named(id) if self.module.program.types[*id].kind == DefinitionKind::ModelId => { + Ok(self.compare("icmp eq", ty, left, right)) + } + Type::Named(id) => match &self.module.program.types[*id].shape { + Shape::Record(fields) => { + let types = fields + .iter() + .map(|field| field.ty.clone()) + .collect::<Vec<_>>(); + self.equals_fields(&types, ty, left, right) + } + Shape::Choice(variants) => { + let types = variants + .iter() + .map(|variant| { + variant + .fields + .iter() + .map(|field| field.ty.clone()) + .collect::<Vec<_>>() + }) + .collect::<Vec<_>>(); + self.equals_choice(*id, &types, left, right) + } + }, + Type::Outputs(fields) => { + let types = fields + .iter() + .map(|field| field.ty.clone()) + .collect::<Vec<_>>(); + self.equals_fields(&types, ty, left, right) + } + Type::MapEntry(key, value) => self.equals_fields( + &[key.as_ref().clone(), value.as_ref().clone()], + ty, + left, + right, + ), + Type::Unit => Ok("1".into()), + // Resources are not source-comparable. Internal collection descriptors + // still need a total callback for their concrete handle storage. + Type::File + | Type::Secret + | Type::ClientTls + | Type::ServerTls + | Type::HttpClient + | Type::HttpReply + | Type::WebSocket + | Type::Upload + | Type::Stream(_) + | Type::AsyncStream(_) + | Type::RowStream(_) + | Type::Socket + | Type::Listener + | Type::Task(_) + | Type::Group + | Type::Channel(_) => Ok(self.compare("icmp eq", ty, left, right)), + } + } + + fn equals_fields( + &mut self, + fields: &[Type], + ty: &Type, + left: &str, + right: &str, + ) -> Result<String, String> { + let mut checks = Vec::new(); + for (index, field) in fields.iter().enumerate() { + let lhs = self.temp(); + self.line(format!( + "{lhs} = extractvalue {} {left}, {index}", + self.module.ty(ty) + )); + let rhs = self.temp(); + self.line(format!( + "{rhs} = extractvalue {} {right}, {index}", + self.module.ty(ty) + )); + checks.push(self.equals(field, &lhs, &rhs)?); + } + Ok(self.reduce_bool("and", checks, "1")) + } + + fn equals_choice( + &mut self, + id: usize, + variants: &[Vec<Type>], + left: &str, + right: &str, + ) -> Result<String, String> { + let left_tag = self.temp(); + self.line(format!("{left_tag} = extractvalue %T{id} {left}, 0")); + let right_tag = self.temp(); + self.line(format!("{right_tag} = extractvalue %T{id} {right}, 0")); + let same = self.temp(); + self.line(format!("{same} = icmp eq i32 {left_tag}, {right_tag}")); + let mismatch = self.label("choice_mismatch"); + let dispatch = self.label("choice_dispatch"); + let done = self.label("choice_done"); + self.line(format!( + "br i1 {same}, label %{dispatch}, label %{mismatch}" + )); + self.start(&mismatch); + self.line(format!("br label %{done}")); + self.start(&dispatch); + let arms = variants + .iter() + .enumerate() + .map(|(index, _)| format!("i32 {index}, label %choice{id}_v{index}_{}", self.next)) + .collect::<Vec<_>>(); + let base = self.next; + self.next += 1; + self.line(format!( + "switch i32 {left_tag}, label %{mismatch} [{}]", + arms.join(" ") + )); + let mut results = vec![("0".to_owned(), mismatch)]; + for (variant, fields) in variants.iter().enumerate() { + self.start(&format!("choice{id}_v{variant}_{base}")); + let lhs = self.temp(); + self.line(format!("{lhs} = alloca %T{id}")); + self.line(format!("store %T{id} {left}, ptr {lhs}")); + let rhs = self.temp(); + self.line(format!("{rhs} = alloca %T{id}")); + self.line(format!("store %T{id} {right}, ptr {rhs}")); + let lhs_payload = self.temp(); + self.line(format!( + "{lhs_payload} = getelementptr %T{id}, ptr {lhs}, i32 0, i32 1" + )); + let rhs_payload = self.temp(); + self.line(format!( + "{rhs_payload} = getelementptr %T{id}, ptr {rhs}, i32 0, i32 1" + )); + let mut checks = Vec::new(); + for (index, ty) in fields.iter().enumerate() { + let left_ptr = self.temp(); + self.line(format!("{left_ptr} = getelementptr %T{id}V{variant}, ptr {lhs_payload}, i32 0, i32 {index}")); + let right_ptr = self.temp(); + self.line(format!("{right_ptr} = getelementptr %T{id}V{variant}, ptr {rhs_payload}, i32 0, i32 {index}")); + let left_value = self.temp(); + self.line(format!( + "{left_value} = load {}, ptr {left_ptr}", + self.module.ty(ty) + )); + let right_value = self.temp(); + self.line(format!( + "{right_value} = load {}, ptr {right_ptr}", + self.module.ty(ty) + )); + checks.push(self.equals(ty, &left_value, &right_value)?); + } + let equal = self.reduce_bool("and", checks, "1"); + let end = self.block.clone(); + self.line(format!("br label %{done}")); + results.push((equal, end)); + } + self.start(&done); + let result = self.temp(); + let incoming = results + .iter() + .map(|(value, block)| format!("[{value}, %{block}]")) + .collect::<Vec<_>>() + .join(", "); + self.line(format!("{result} = phi i1 {incoming}")); + Ok(result) + } +} diff --git a/crates/dever-core/src/llvm/external.rs b/crates/dever-core/src/llvm/external.rs new file mode 100644 index 0000000..26bf73e --- /dev/null +++ b/crates/dever-core/src/llvm/external.rs @@ -0,0 +1,436 @@ +//! Typed external operation envelopes over the shared invocation-owned supervisor. +use super::*; +use crate::hir::AdapterImplementation; +use crate::native::EmbeddedResource; +use crate::types::Field; + +const DECLARATIONS: &str = "\ +%dever.external_resource = type { { ptr, i64 }, ptr, i64, { ptr, i64 }, i8 } +%dever.external_definition = type { { ptr, i64 }, { ptr, i64 }, { ptr, i64 }, { ptr, i64 }, { ptr, i64 }, { ptr, i64 }, ptr, i64, ptr, i64, i64 } +declare i32 @dever_rt_v1_external_prepare(ptr, ptr, i64, ptr) +declare ptr @dever_rt_v1_external_start(ptr, ptr, ptr) +declare ptr @dever_rt_v1_external_call(ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_external_reply_take(ptr, ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_external_root(ptr, ptr, ptr, ptr, ptr, ptr) +declare void @dever_rt_v1_external_cause_append(ptr, ptr) +"; + +fn input_fields(program: &Program, function: usize) -> Vec<Field> { + let function = &program.functions[function]; + let external = function.external.expect("external operation"); + let contract = program.functions[external.contract].port.as_ref().unwrap(); + contract + .input_names + .iter() + .zip(&function.parameters) + .map(|(name, parameter)| Field { + name: name.clone(), + ty: parameter + .value_type() + .expect("checked Port value input") + .clone(), + private: false, + bounds: Vec::new(), + }) + .collect() +} + +impl Module<'_> { + pub(super) fn external_enabled(&self) -> bool { + self.application + .as_ref() + .is_some_and(|app| app.resources.is_some()) + || self + .instances + .iter() + .any(|instance| self.program.functions[instance.function].external.is_some()) + } + + pub(super) fn collect_external_types(&mut self) { + for instance in self.instances.clone() { + let function = &self.program.functions[instance.function]; + let Some(external) = function.external else { + continue; + }; + let mut rows = vec![ + input_fields(self.program, instance.function), + function.outputs.clone(), + ]; + let contract = self.program.functions[external.contract] + .port + .as_ref() + .unwrap(); + for failure in &contract.failures { + let Shape::Choice(variants) = &self.program.types[failure.ty].shape else { + unreachable!() + }; + rows.push(variants[failure.variant].fields.clone()); + } + for fields in rows { + collect_owned_types(self.program, &Type::Outputs(fields), &mut self.value_types); + } + } + } + + pub(super) fn emit_external_descriptors( + &mut self, + resources: &[EmbeddedResource], + indices: &[usize], + attachments: &[BinaryResource<'_>], + ) { + if !self.external_enabled() && resources.is_empty() { + return; + } + self.declarations.push_str(DECLARATIONS); + self.declarations.push_str("define internal void @dever_external_append_cause(ptr %fault, ptr %message) {\nentry:\n %cause = getelementptr %dever.fault, ptr %fault, i32 0, i32 8\n call void @dever_rt_v1_external_cause_append(ptr %cause, ptr %message)\n ret void\n}\n"); + let adapters = self.application.as_ref().unwrap().adapters.clone(); + for id in adapters { + let adapter = self.program.adapters[id].clone(); + let AdapterImplementation::External(external) = &adapter.implementation else { + continue; + }; + let names = [ + format!("{}:{}", adapter.identity, adapter.name), + external.ecosystem.keyword().to_owned(), + external.entry.clone(), + adapter.identity.clone(), + adapter.name.clone(), + external.schema.clone(), + ]; + let metadata = names + .iter() + .map(|name| self.api_name(name)) + .collect::<Vec<_>>() + .join(", "); + let capabilities = self.wire_names( + &external + .capabilities + .iter() + .map(|value| (*value).to_owned()) + .collect::<Vec<_>>(), + ); + let operations = adapter + .operations + .keys() + .map(|port| { + self.program.functions[*port] + .port + .as_ref() + .unwrap() + .operation + .clone() + }) + .collect::<Vec<_>>(); + let operation_names = self.wire_names(&operations); + writeln!(self.declarations, "@dever_external_{id} = private constant %dever.external_definition {{ {metadata}, ptr {capabilities}, i64 {}, ptr {operation_names}, i64 {}, i64 30000 }}", external.capabilities.len(), operations.len()).unwrap(); + } + if resources.is_empty() { + return; + } + for resource in attachments { + writeln!( + self.declarations, + "@{} = external constant [{} x i8]", + resource.symbol, + resource.bytes.len() + ) + .unwrap(); + } + let mut entries = Vec::new(); + for (resource, index) in resources.iter().zip(indices) { + let path = self.api_name(&resource.path); + let digest = self.api_name(&resource.sha256); + entries.push(format!("%dever.external_resource {{ {path}, ptr @dever_external_bytes_{index}, i64 {}, {digest}, i8 {} }}", resource.bytes.len(), u8::from(resource.executable))); + } + writeln!( + self.declarations, + "@dever_external_resources = private constant [{} x %dever.external_resource] [{}]", + resources.len(), + entries.join(", ") + ) + .unwrap(); + let digest = self.wire_names(&[crate::native::embedded_bundle_digest(resources)]); + self.application.as_mut().unwrap().resources = Some((digest, resources.len())); + } +} + +impl FunctionEmitter<'_, '_> { + fn external_wire_policy(&self) -> crate::wire::Policy { + let function = &self.module.program.functions[self.instance.function]; + let operation = function.external.expect("external operation"); + let AdapterImplementation::External(external) = + &self.module.program.adapters[operation.adapter].implementation + else { + unreachable!("external operation Adapter") + }; + crate::wire::Policy::external(external.ecosystem) + } + + pub(super) fn initialize_external_resources(&mut self, span: Span) { + let Some((digest, count)) = self + .module + .application + .as_ref() + .and_then(|app| app.resources.clone()) + else { + return; + }; + self.wire_write_at( + "dever_rt_v1_external_prepare", + vec![ + format!("ptr {digest}"), + "ptr @dever_external_resources".into(), + format!("i64 {count}"), + ], + span, + ); + } + + pub(super) fn start_external_adapter( + &mut self, + adapter: &crate::hir::Adapter, + setting: &str, + span: Span, + ) { + if !matches!(adapter.implementation, AdapterImplementation::External(_)) { + return; + } + let id = self + .module + .program + .adapters + .iter() + .position(|value| value.owner == adapter.owner) + .unwrap(); + let operation = self.async_operation( + "external_start", + vec![ + format!("ptr @dever_external_{id}"), + format!("ptr {setting}"), + ], + span, + ); + self.await_operation(&operation, &Type::Unit, span, false); + } + + pub(super) fn emit_external_body( + &mut self, + function: &crate::hir::Function, + ) -> Result<bool, String> { + let Some(external) = function.external else { + return Ok(false); + }; + let span = function.span; + let adapter = self.module.program.adapters[external.adapter].clone(); + let contract = self.module.program.functions[external.contract] + .port + .clone() + .unwrap(); + let payload = self.encode_external_inputs(span); + let key = self.application_text(&format!("{}:{}", adapter.identity, adapter.name), span); + let operation = self.application_text(&contract.operation, span); + let (tag, identity, body) = self.call_external(&key, &operation, &payload, span); + let success = self.label("external_result"); + let failure = self.label("external_failure"); + let ok = self.temp(); + self.line(format!("{ok} = icmp eq i8 {tag}, 0")); + self.line(format!("br i1 {ok}, label %{success}, label %{failure}")); + self.start(&success); + self.return_external_result(&function.outputs, &body, span); + self.start(&failure); + self.fail_external(&contract.failures, &identity, &body, span)?; + Ok(true) + } + + fn encode_external_inputs(&mut self, span: Span) -> String { + let fields = input_fields(self.module.program, self.instance.function); + let input_ty = Type::Outputs(fields.clone()); + let mut input = "zeroinitializer".to_owned(); + for (index, field) in fields.iter().enumerate() { + let value = self.temp(); + self.line(format!( + "{value} = insertvalue {} {input}, {} {}, {index}", + self.module.ty(&input_ty), + self.module.ty(&field.ty), + self.parameter(index) + )); + input = value; + } + let row = self.row_pointer(&input_ty, &input); + let callback = self.module.wire_fields_callback( + &fields, + &format!("dever_external_inputs_{}", self.instance.function), + self.external_wire_policy(), + true, + ); + let payload = self.runtime_call( + "dever_rt_v1_wire_encode", + vec![ + format!("ptr {row}"), + format!("ptr @dever_type_{}", self.module.type_index(&input_ty)), + format!("ptr @{callback}"), + ], + &Type::Text, + false, + None, + span, + ); + self.own_value(&Type::Text, &payload, false); + payload + } + + fn call_external( + &mut self, + key: &str, + operation_name: &str, + payload: &str, + span: Span, + ) -> (String, String, String) { + let operation = self.async_operation( + "external_call", + vec![ + format!("ptr {key}"), + format!("ptr {operation_name}"), + format!("ptr {payload}"), + ], + span, + ); + let (status, reply_row, _, error) = self.poll_raw_layout(&operation, "ptr"); + let status = self.poll_status(&status, &error, span); + self.propagate_status(&status, span, false); + let reply = self.temp(); + self.line(format!("{reply} = load ptr, ptr {reply_row}")); + let kind = self.entry_slot_ir("i8"); + let identity = self.entry_slot(&Type::Text); + let body = self.entry_slot(&Type::Text); + self.wire_write_at( + "dever_rt_v1_external_reply_take", + vec![ + format!("ptr {reply}"), + format!("ptr {kind}"), + format!("ptr {identity}"), + format!("ptr {body}"), + ], + span, + ); + let identity_value = self.temp(); + let body_value = self.temp(); + self.line(format!("{identity_value} = load ptr, ptr {identity}")); + self.line(format!("{body_value} = load ptr, ptr {body}")); + self.own_value(&Type::Text, &identity_value, false); + self.own_value(&Type::Text, &body_value, false); + let tag = self.temp(); + self.line(format!("{tag} = load i8, ptr {kind}")); + (tag, identity_value, body_value) + } + + fn return_external_result(&mut self, fields: &[Field], body: &str, span: Span) { + let value = self.decode_external_fields(fields, body, "outputs", span); + let output = output_type(fields); + let result = match fields.len() { + 0 => "zeroinitializer".into(), + 1 => { + let result = self.temp(); + self.line(format!( + "{result} = extractvalue {} {value}, 0", + self.module.ty(&Type::Outputs(fields.to_vec())) + )); + result + } + _ => value, + }; + self.line(format!( + "store {} {result}, ptr %out", + self.module.ty(&output) + )); + self.exit("0"); + } + + fn fail_external( + &mut self, + failures: &BTreeSet<crate::hir::Failure>, + identity: &str, + body: &str, + span: Span, + ) -> Result<(), String> { + for declared in failures { + let definition = self.module.program.types[declared.ty].clone(); + let Shape::Choice(variants) = &definition.shape else { + unreachable!() + }; + let variant = &variants[declared.variant]; + let expected = + self.application_text(&format!("{}.{}", definition.name, variant.name), span); + let matches = self.equals(&Type::Text, identity, &expected)?; + let matched = self.label("external_declared_error"); + let next = self.label("external_next_error"); + self.line(format!("br i1 {matches}, label %{matched}, label %{next}")); + self.start(&matched); + let values = self.decode_external_fields( + &variant.fields, + body, + &format!("error_{}_{}", declared.ty, declared.variant), + span, + ); + let row_ty = Type::Outputs(variant.fields.clone()); + let mut fields = Vec::new(); + for (index, field) in variant.fields.iter().enumerate() { + let value = self.temp(); + self.line(format!( + "{value} = extractvalue {} {values}, {index}", + self.module.ty(&row_ty) + )); + fields.push((field.ty.clone(), value)); + } + let payload = self.temp(); + self.line(format!( + "{payload} = getelementptr %dever.fault, ptr %fault, i32 0, i32 7" + )); + self.write_variant(declared.ty, declared.variant, &fields, &payload); + for (index, value) in [(5, declared.ty + 1), (6, declared.variant)] { + let pointer = self.temp(); + self.line(format!( + "{pointer} = getelementptr %dever.fault, ptr %fault, i32 0, i32 {index}" + )); + self.line(format!("store i32 {value}, ptr {pointer}")); + } + self.write_fault(&(FaultCode::Business as i32).to_string(), span); + self.start(&next); + } + self.application_fault( + "external Adapter returned an undeclared business error", + span, + ); + Ok(()) + } + + /// The wire callback transfers a complete row; callers immediately move it + /// into their output or fault, with no intervening fallible operation. + fn decode_external_fields( + &mut self, + fields: &[Field], + body: &str, + suffix: &str, + span: Span, + ) -> String { + let ty = Type::Outputs(fields.to_vec()); + let callback = self.module.wire_fields_callback( + fields, + &format!("dever_external_{suffix}_{}", self.instance.function), + self.external_wire_policy(), + false, + ); + self.runtime_call( + "dever_rt_v1_wire_decode", + vec![ + format!("ptr {body}"), + format!("ptr @dever_type_{}", self.module.type_index(&ty)), + format!("ptr @{callback}"), + ], + &ty, + false, + None, + span, + ) + } +} diff --git a/crates/dever-core/src/llvm/failures.rs b/crates/dever-core/src/llvm/failures.rs new file mode 100644 index 0000000..d50cbd2 --- /dev/null +++ b/crates/dever-core/src/llvm/failures.rs @@ -0,0 +1,310 @@ +use super::*; + +impl FunctionEmitter<'_, '_> { + pub(super) fn call_instance( + &self, + target: CallTarget, + arguments: &[CallArgument], + ) -> Specialization { + match target { + CallTarget::Function(function) => { + specialize::call_specialization(function, arguments, &self.bindings) + } + CallTarget::Handler(target) => Specialization { + function: specialize::resolve_handler(target, &self.bindings), + handlers: Vec::new(), + }, + } + } + + pub(super) fn invoke( + &mut self, + instance: &Specialization, + arguments: &[CallArgument], + output: &Type, + span: Span, + ) -> Result<(String, String), String> { + let mut values = Vec::new(); + for argument in arguments { + if let CallArgument::Value(value) = argument { + let result = self.expression(value)?; + values.push((value.ty.clone(), result)); + } + } + Ok(self.invoke_values(instance, &values, output, span)) + } + + pub(super) fn invoke_values( + &mut self, + instance: &Specialization, + arguments: &[(Type, String)], + output: &Type, + span: Span, + ) -> (String, String) { + if specialize::suspends(self.module.program, instance) { + let (input, _) = self.pack_input(instance, arguments, false); + let (input, descriptor) = if self.module.database_function(instance) { + let ty = + asynchronous::input_type(&self.module.program.functions[instance.function]); + let layout = format!("{{ {}, ptr }}", self.module.ty(&ty)); + let extended = self.entry_slot_ir(&layout); + let value = self.temp(); + self.line(format!( + "{value} = load {}, ptr {input}", + self.module.ty(&ty) + )); + self.line(format!( + "store {} {value}, ptr {extended}", + self.module.ty(&ty) + )); + let context = self.temp(); + self.line(format!( + "{context} = getelementptr {layout}, ptr {extended}, i32 0, i32 1" + )); + self.line(format!( + "store ptr {}, ptr {context}", + self.database_context + )); + ( + extended, + format!("@dever_async_context_{}", self.module.names[instance]), + ) + } else { + ( + input, + format!("@dever_async_{}", self.module.names[instance]), + ) + }; + let operation = self.async_operation( + "async_call", + vec![format!("ptr {descriptor}"), format!("ptr {input}")], + span, + ); + return self.poll_operation(&operation, output, span); + } + let result = self.entry_slot(output); + self.line(format!( + "store {} zeroinitializer, ptr {result}", + self.module.ty(output) + )); + let mut values = arguments + .iter() + .map(|(ty, value)| format!("{} {value}", self.module.ty(ty))) + .collect::<Vec<_>>(); + if self.module.database_function(instance) { + values.push(format!("ptr {}", self.database_context)); + } + values.push(format!("ptr {result}")); + values.push("ptr %fault".into()); + let status = self.temp(); + self.line(format!( + "{status} = call i32 @{}({})", + self.module.names[instance], + values.join(", ") + )); + (status, result) + } + + pub(super) fn fail(&mut self, error: &Expression, span: Span) -> Result<String, String> { + let Type::Named(id) = error.ty else { + unreachable!("checked fail choice") + }; + let ExpressionKind::Variant { variant, .. } = error.kind else { + unreachable!("checked direct failure variant") + }; + let value = self.expression(error)?; + let payload = self.temp(); + self.line(format!( + "{payload} = getelementptr %dever.fault, ptr %fault, i32 0, i32 7" + )); + self.line(format!("store %T{id} {value}, ptr {payload}")); + if contains_owned(self.module.program, &error.ty) { + self.disarm(self.guards.len() - 1); + } + for (index, number) in [(5, id + 1), (6, variant)] { + let pointer = self.temp(); + self.line(format!( + "{pointer} = getelementptr %dever.fault, ptr %fault, i32 0, i32 {index}" + )); + self.line(format!("store i32 {number}, ptr {pointer}")); + } + self.write_fault(&(FaultCode::Business as i32).to_string(), span); + let dead = self.label("after_fail"); + self.start(&dead); + Ok("zeroinitializer".into()) + } + + pub(super) fn capture_result( + &mut self, + target: CallTarget, + arguments: &[CallArgument], + choice: usize, + success: usize, + expression: &Expression, + ) -> Result<String, String> { + let instance = self.call_instance(target, arguments); + let function = &self.module.program.functions[instance.function]; + let output = output_type(&function.outputs); + let failures = specialize::failures(self.module.program, &instance); + let targets = crate::capture::targets(&self.module.program.types, choice, &failures) + .expect("checked result capture mapping"); + let (status, output_slot) = self.invoke(&instance, arguments, &output, expression.span)?; + let result_slot = self.entry_slot(&expression.ty); + let success_block = self.label("capture_success"); + let failure_block = self.label("capture_failure"); + let done = self.label("capture_done"); + let ok = self.temp(); + self.line(format!("{ok} = icmp eq i32 {status}, 0")); + self.line(format!( + "br i1 {ok}, label %{success_block}, label %{failure_block}" + )); + + self.start(&success_block); + let success_fields = match &self.module.program.types[choice].shape { + Shape::Choice(variants) => variants[success].fields.clone(), + _ => unreachable!("checked capture choice"), + }; + let values = success_fields + .iter() + .enumerate() + .map(|(index, field)| { + let value = self.temp(); + if success_fields.len() == 1 { + self.line(format!( + "{value} = load {}, ptr {output_slot}", + self.module.ty(&field.ty) + )); + } else { + let aggregate = self.temp(); + self.line(format!( + "{aggregate} = load {}, ptr {output_slot}", + self.module.ty(&output) + )); + self.line(format!( + "{value} = extractvalue {} {aggregate}, {index}", + self.module.ty(&output) + )); + } + (field.ty.clone(), value) + }) + .collect::<Vec<_>>(); + self.write_variant(choice, success, &values, &result_slot); + self.line(format!("br label %{done}")); + + self.start(&failure_block); + let kind_ptr = self.temp(); + self.line(format!( + "{kind_ptr} = getelementptr %dever.fault, ptr %fault, i32 0, i32 0" + )); + let kind = self.temp(); + self.line(format!("{kind} = load i32, ptr {kind_ptr}")); + let business = self.temp(); + self.line(format!( + "{business} = icmp eq i32 {kind}, {}", + FaultCode::Business as i32 + )); + let dispatch = self.label("capture_dispatch"); + let propagate = self.label("capture_propagate"); + self.line(format!( + "br i1 {business}, label %{dispatch}, label %{propagate}" + )); + self.start(&dispatch); + let type_ptr = self.temp(); + self.line(format!( + "{type_ptr} = getelementptr %dever.fault, ptr %fault, i32 0, i32 5" + )); + let type_id = self.temp(); + self.line(format!("{type_id} = load i32, ptr {type_ptr}")); + let variant_ptr = self.temp(); + self.line(format!( + "{variant_ptr} = getelementptr %dever.fault, ptr %fault, i32 0, i32 6" + )); + let variant_id = self.temp(); + self.line(format!("{variant_id} = load i32, ptr {variant_ptr}")); + for (failure, target) in targets { + if !failures.contains(&failure) { + continue; + } + let matches_type = self.temp(); + self.line(format!( + "{matches_type} = icmp eq i32 {type_id}, {}", + failure.ty + 1 + )); + let matches_variant = self.temp(); + self.line(format!( + "{matches_variant} = icmp eq i32 {variant_id}, {}", + failure.variant + )); + let matches = self.temp(); + self.line(format!( + "{matches} = and i1 {matches_type}, {matches_variant}" + )); + let captured = self.label("capture_match"); + let next = self.label("capture_next"); + self.line(format!("br i1 {matches}, label %{captured}, label %{next}")); + self.start(&captured); + let payload = self.temp(); + self.line(format!( + "{payload} = getelementptr %dever.fault, ptr %fault, i32 0, i32 7" + )); + let inner = self.temp(); + self.line(format!("{inner} = load %T{}, ptr {payload}", failure.ty)); + if target.wrapped { + self.write_variant( + choice, + target.variant, + &[(Type::Named(failure.ty), inner)], + &result_slot, + ); + } else { + self.line(format!("store %T{choice} {inner}, ptr {result_slot}")); + } + if self.module.uses_runtime() { + let cause_ptr = self.temp(); + self.line(format!( + "{cause_ptr} = getelementptr %dever.fault, ptr %fault, i32 0, i32 8" + )); + let cause = self.temp(); + self.line(format!("{cause} = load ptr, ptr {cause_ptr}")); + self.line(format!("call void @dever_rt_v1_text_release(ptr {cause})")); + } + self.line("store %dever.fault zeroinitializer, ptr %fault"); + self.line(format!("br label %{done}")); + self.start(&next); + } + self.line(format!("br label %{propagate}")); + self.start(&propagate); + self.append_frame(expression.span); + self.exit(&status); + self.start(&done); + let result = self.temp(); + self.line(format!("{result} = load %T{choice}, ptr {result_slot}")); + Ok(result) + } + + pub(super) fn write_variant( + &mut self, + id: usize, + variant: usize, + fields: &[(Type, String)], + slot: &str, + ) { + self.line(format!("store %T{id} zeroinitializer, ptr {slot}")); + let tag = self.temp(); + self.line(format!( + "{tag} = getelementptr %T{id}, ptr {slot}, i32 0, i32 0" + )); + self.line(format!("store i32 {variant}, ptr {tag}")); + let payload = self.temp(); + self.line(format!( + "{payload} = getelementptr %T{id}, ptr {slot}, i32 0, i32 1" + )); + for (index, (ty, value)) in fields.iter().enumerate() { + let field = self.temp(); + self.line(format!( + "{field} = getelementptr %T{id}V{variant}, ptr {payload}, i32 0, i32 {index}" + )); + self.line(format!("store {} {value}, ptr {field}", self.module.ty(ty))); + } + } +} diff --git a/crates/dever-core/src/llvm/http.rs b/crates/dever-core/src/llvm/http.rs new file mode 100644 index 0000000..13ccf9b --- /dev/null +++ b/crates/dever-core/src/llvm/http.rs @@ -0,0 +1,227 @@ +//! Protocol ABI conversion is field-name based, independent of source layout. +use super::*; + +pub(super) const DECLARATIONS: &str = "\ +%dever.http_handler = type { ptr, ptr, ptr, ptr, ptr, ptr, ptr } +declare ptr @dever_rt_v1_http_headers_retain(ptr) +declare void @dever_rt_v1_http_headers_release(ptr) +declare void @dever_rt_v1_http_request_release(ptr) +declare ptr @dever_rt_v1_http_request_retain(ptr) +declare void @dever_rt_v1_http_response_release(ptr) +declare void @dever_rt_v1_http_stream_request_release(ptr) +declare void @dever_rt_v1_http_stream_response_release(ptr) +declare void @dever_rt_v1_ws_message_release(ptr) +declare i32 @dever_rt_v1_http_headers_new(ptr, ptr) +declare i32 @dever_rt_v1_http_headers_push(ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_http_headers_length(ptr, ptr, ptr) +declare i32 @dever_rt_v1_http_headers_get(ptr, i64, ptr, ptr) +declare i32 @dever_rt_v1_http_request_new(ptr, ptr, ptr) +declare i32 @dever_rt_v1_http_request_fields(ptr, ptr, ptr) +declare i32 @dever_rt_v1_http_response_new(ptr, ptr, ptr) +declare i32 @dever_rt_v1_http_response_fields(ptr, ptr, ptr) +declare i32 @dever_rt_v1_http_stream_request_new(ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_http_stream_response_fields(ptr, ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_tls_system(ptr, ptr) +declare i32 @dever_rt_v1_tls_client(ptr, ptr, ptr) +declare i32 @dever_rt_v1_tls_server(ptr, ptr, ptr, ptr) +declare ptr @dever_rt_v1_async_http_send(ptr, i64, ptr, ptr) +declare ptr @dever_rt_v1_async_http_client(ptr, ptr, ptr, ptr) +declare ptr @dever_rt_v1_async_http_request(ptr, ptr) +declare ptr @dever_rt_v1_async_http_open(ptr, ptr) +declare ptr @dever_rt_v1_async_http_open_stream(ptr, ptr) +declare ptr @dever_rt_v1_async_http_close_client(ptr) +declare ptr @dever_rt_v1_async_http_serve(ptr, ptr, ptr, ptr, ptr, ptr, ptr) +declare ptr @dever_rt_v1_async_http_respond(ptr, ptr) +declare ptr @dever_rt_v1_async_http_start(ptr, i64, ptr) +declare ptr @dever_rt_v1_async_http_write(ptr, ptr) +declare ptr @dever_rt_v1_async_http_finish(ptr) +declare ptr @dever_rt_v1_async_sse_start(ptr, ptr) +declare ptr @dever_rt_v1_async_sse_send(ptr, ptr) +declare ptr @dever_rt_v1_async_ws_accept(ptr, ptr) +declare ptr @dever_rt_v1_async_ws_connect(ptr, i64, ptr, ptr) +declare ptr @dever_rt_v1_async_ws_open(ptr, ptr, ptr) +declare i32 @dever_rt_v1_ws_message_new(i32, ptr, ptr, ptr) +declare i32 @dever_rt_v1_ws_message_fields(ptr, ptr, ptr) +declare ptr @dever_rt_v1_async_ws_send(ptr, ptr) +declare ptr @dever_rt_v1_async_ws_receive(ptr) +declare i32 @dever_rt_v1_ws_messages(ptr, ptr, ptr, ptr, ptr) +declare ptr @dever_rt_v1_async_ws_close(ptr, i64, ptr) +"; + +pub(super) fn supported(operation: Intrinsic) -> bool { + use Intrinsic::*; + matches!( + operation, + TlsSystem + | TlsClient + | TlsServer + | HttpSend + | HttpClientNew + | HttpRequest + | HttpOpen + | HttpOpenStream + | HttpCloseClient + | HttpServe + | HttpServeTls + | HttpServeLive + | HttpServeLiveTls + | HttpRespond + | HttpStart + | HttpWrite + | HttpFinish + | SseStart + | SseSend + | WsAccept + | WsConnect + | WsOpen + | WsSend + | WsReceive + | WsMessages + | WsClose + ) +} + +impl FunctionEmitter<'_, '_> { + pub(super) fn http_intrinsic( + &mut self, + operation: Intrinsic, + handler: Option<crate::hir::HandlerTarget>, + arguments: &[Expression], + values: &[String], + result: &Expression, + ) -> String { + use Intrinsic::*; + let mut args = arguments + .iter() + .zip(values) + .map(|(argument, value)| format!("{} {value}", self.module.ty(&argument.ty))) + .collect::<Vec<_>>(); + match operation { + TlsSystem => { + return self.runtime_call( + "dever_rt_v1_tls_system", + args, + &result.ty, + false, + None, + result.span, + ); + } + TlsClient | TlsServer => { + return self.recoverable_call( + if operation == TlsClient { + "dever_rt_v1_tls_client" + } else { + "dever_rt_v1_tls_server" + }, + args, + result, + "Ready", + None, + ); + } + HttpServe | HttpServeTls | HttpServeLive | HttpServeLiveTls => { + return self.http_serve( + operation, + handler.expect("checked HTTP handler"), + arguments, + values, + result, + ); + } + HttpSend => { + args[2] = format!( + "ptr {}", + self.protocol_to_runtime(&arguments[2].ty, &values[2], result.span) + ); + args[3] = format!("ptr {}", self.http_limits(&arguments[3].ty, &values[3])); + } + HttpClientNew => { + args[2] = format!("ptr {}", self.http_limits(&arguments[2].ty, &values[2])); + args[3] = format!( + "ptr {}", + self.protocol_limits( + &arguments[3].ty, + &values[3], + &["idle_ms", "chunk_bytes", "read_ms"] + ) + ); + } + HttpRequest | HttpOpen | HttpOpenStream | HttpRespond | WsSend => { + args[1] = format!( + "ptr {}", + self.protocol_to_runtime(&arguments[1].ty, &values[1], result.span) + ); + } + HttpStart | SseStart => { + let index = if operation == HttpStart { 2 } else { 1 }; + args[index] = format!( + "ptr {}", + self.headers_to_runtime(&arguments[index].ty, &values[index], result.span) + ); + } + SseSend => args[1] = format!("ptr {}", self.sse_event(&arguments[1].ty, &values[1])), + WsAccept | WsConnect | WsOpen => { + let index = args.len() - 1; + args[index] = format!( + "ptr {}", + self.protocol_limits( + &arguments[index].ty, + &values[index], + &["message_bytes", "idle_ms", "write_ms"] + ) + ); + } + WsMessages => { + let Type::AsyncStream(event) = &result.ty else { + unreachable!() + }; + let Type::Named(id) = event.as_ref() else { + unreachable!() + }; + let descriptor = self.module.websocket_event(*id); + args.push(format!("ptr {descriptor}")); + args.push(format!( + "ptr @dever_owned_{}", + self.module.type_index(event) + )); + return self.runtime_call( + "dever_rt_v1_ws_messages", + args, + &result.ty, + false, + None, + result.span, + ); + } + _ => {} + } + let (symbol, success, end) = match operation { + HttpSend => ("http_send", "Read", None), + HttpClientNew => ("http_client", "Ready", None), + HttpRequest => ("http_request", "Read", None), + HttpOpen => ("http_open", "Read", None), + HttpOpenStream => ("http_open_stream", "Read", None), + HttpCloseClient => ("http_close_client", "Closed", None), + HttpRespond => ("http_respond", "Done", None), + HttpStart => ("http_start", "Done", None), + HttpWrite => ("http_write", "Done", None), + HttpFinish => ("http_finish", "Closed", None), + SseStart => ("sse_start", "Done", None), + SseSend => ("sse_send", "Done", None), + WsAccept => ("ws_accept", "Connected", None), + WsConnect => ("ws_connect", "Connected", None), + WsOpen => ("ws_open", "Connected", None), + WsSend => ("ws_send", "Done", None), + WsReceive => ("ws_receive", "Read", Some("End")), + WsClose => ("ws_close", "Closed", None), + _ => unreachable!("checked HTTP operation"), + }; + let operation = self.temp(); + self.line(format!( + "{operation} = call ptr @dever_rt_v1_async_{symbol}({})", + args.join(", ") + )); + self.async_resource_result(&operation, result, success, end) + } +} diff --git a/crates/dever-core/src/llvm/http_handlers.rs b/crates/dever-core/src/llvm/http_handlers.rs new file mode 100644 index 0000000..cfa1948 --- /dev/null +++ b/crates/dever-core/src/llvm/http_handlers.rs @@ -0,0 +1,272 @@ +//! Static HTTP handlers and typed upload/download callbacks. +use super::*; + +impl Module<'_> { + pub(super) fn upload_decoder(&mut self, id: usize) -> String { + let name = format!("@dever_upload_decode_{id}"); + if !self.upload_decoders.insert(id) { + return name; + } + let Shape::Choice(variants) = &self.program.types[id].shape else { + unreachable!() + }; + let mut callback = format!( + "define internal i32 {name}(ptr %event, ptr %out) {{\nentry:\n %tag_ptr = getelementptr %T{id}, ptr %event, i32 0, i32 0\n %tag = load i32, ptr %tag_ptr\n %payload_ptr = getelementptr %T{id}, ptr %event, i32 0, i32 1\n %payload = load ptr, ptr %payload_ptr\n switch i32 %tag, label %invalid [" + ); + for (index, _) in variants.iter().enumerate() { + write!(callback, "i32 {index}, label %variant{index} ").unwrap(); + } + callback.push_str("]\ninvalid:\n unreachable\n"); + for (index, variant) in variants.iter().enumerate() { + let (kind, status) = match variant.name.as_str() { + "Chunk" => ("bytes", 0), + "Failed" => ("text", 1), + _ => unreachable!(), + }; + writeln!(callback, "variant{index}:\n %owned{index} = call ptr @dever_rt_v1_{kind}_retain(ptr %payload)\n store ptr %owned{index}, ptr %out\n ret i32 {status}").unwrap(); + } + callback.push_str("}\n"); + self.declarations.push_str(&callback); + name + } + + pub(super) fn websocket_event(&mut self, id: usize) -> String { + let descriptor = format!("@dever_ws_event_{id}"); + if !self.websocket_events.insert(id) { + return descriptor; + } + let Shape::Choice(variants) = self.program.types[id].shape.clone() else { + unreachable!() + }; + let read = variants + .iter() + .position(|variant| variant.name == "Read") + .unwrap(); + let failed = variants + .iter() + .position(|variant| variant.name == "Failed") + .unwrap(); + let Type::Named(message) = variants[read].fields[0].ty else { + unreachable!() + }; + let Shape::Choice(messages) = self.program.types[message].shape.clone() else { + unreachable!() + }; + let instance = self.instances.first().unwrap().clone(); + let mut emitter = FunctionEmitter::new(self, &instance); + emitter.start("entry"); + emitter.line("switch i32 %kind, label %invalid [i32 0, label %kind0 i32 1, label %kind1 i32 2, label %kind2 i32 3, label %kind3]"); + emitter.start("invalid"); + emitter.line("unreachable"); + for (kind, name) in ["Text", "Binary", "Ping", "Pong"].iter().enumerate() { + emitter.start(&format!("kind{kind}")); + let (variant, definition) = messages + .iter() + .enumerate() + .find(|(_, variant)| variant.name == *name) + .unwrap(); + let symbol = if kind == 0 { "text" } else { "bytes" }; + let owned = emitter.temp(); + emitter.line(format!( + "{owned} = call ptr @dever_rt_v1_{symbol}_retain(ptr %payload)" + )); + let slot = emitter.entry_slot(&Type::Named(message)); + emitter.write_variant( + message, + variant, + &[(definition.fields[0].ty.clone(), owned)], + &slot, + ); + let value = emitter.temp(); + emitter.line(format!("{value} = load %T{message}, ptr {slot}")); + emitter.write_variant(id, read, &[(Type::Named(message), value)], "%out"); + emitter.line("ret void"); + } + let callback = format!( + "define internal void @dever_ws_read_{id}(i32 %kind, ptr %payload, ptr %out) {{\nentry:\n{}{} }}\n", + emitter.prologue, emitter.body + ); + self.declarations.push_str(&callback); + let mut emitter = FunctionEmitter::new(self, &instance); + emitter.start("entry"); + let text = emitter.temp(); + emitter.line(format!( + "{text} = call ptr @dever_rt_v1_text_retain(ptr %payload)" + )); + emitter.write_variant(id, failed, &[(Type::Text, text)], "%out"); + emitter.line("ret void"); + let callback = format!( + "define internal void @dever_ws_failed_{id}(ptr %payload, ptr %out) {{\nentry:\n{}{} }}\n", + emitter.prologue, emitter.body + ); + self.declarations.push_str(&callback); + writeln!(self.declarations, "{descriptor} = private constant {{ ptr, ptr, ptr }} {{ ptr @dever_type_{}, ptr @dever_ws_read_{id}, ptr @dever_ws_failed_{id} }}", self.type_index(&Type::Named(id))).unwrap(); + descriptor + } + + fn http_handler(&mut self, instance: &Specialization, live: bool) -> String { + let name = self.names[instance].clone(); + let descriptor = format!("@dever_http_handler_{name}"); + if !self.http_handlers.insert(instance.clone()) { + return descriptor; + } + let function = self.program.functions[instance.function].clone(); + let input = asynchronous::input_type(&function); + let Type::Outputs(fields) = &input else { + unreachable!() + }; + let context_index = if live { 2 } else { 1 }; + let context = fields.get(context_index).map_or("null".into(), |field| { + format!("@dever_type_{}", self.type_index(&field.ty)) + }); + let mut emitter = FunctionEmitter::new(self, instance); + emitter.asynchronous = false; + emitter.start("entry"); + let request = emitter.temp(); + emitter.line(format!( + "{request} = call ptr @dever_rt_v1_http_request_retain(ptr %request)" + )); + let value = emitter.protocol_to_language(&fields[0].ty, &request, function.span); + let request_guard = emitter.own_value(&fields[0].ty, &value, false); + let request_row = emitter.guards[request_guard].pointer.clone(); + for (index, field) in fields.iter().enumerate() { + let target = emitter.temp(); + emitter.line(format!( + "{target} = getelementptr {}, ptr %out, i32 0, i32 {index}", + emitter.module.ty(&input) + )); + if index == 0 { + emitter.line(format!( + "call void @dever_clone_{}(ptr {request_row}, ptr {target})", + emitter.module.type_index(&field.ty) + )); + } else if live && index == 1 { + let reply = emitter.temp(); + emitter.line(format!( + "{reply} = call ptr @dever_rt_v1_http_reply_retain(ptr %reply)" + )); + emitter.line(format!("store ptr {reply}, ptr {target}")); + } else { + emitter.line(format!( + "call void @dever_clone_{}(ptr %context, ptr {target})", + emitter.module.type_index(&field.ty) + )); + } + } + let callback = emitter.finish_protocol_callback( + &format!("dever_http_pack_{name}"), + "ptr %request, ptr %reply, ptr %context, ptr %out, ptr %error", + ); + self.declarations.push_str(&callback); + let response = if live { + "null".into() + } else { + let mut emitter = FunctionEmitter::new(self, instance); + emitter.asynchronous = false; + emitter.start("entry"); + let output_ty = &function.outputs[0].ty; + let output = emitter.temp(); + emitter.line(format!( + "{output} = load {}, ptr %input", + emitter.module.ty(output_ty) + )); + let handle = emitter.protocol_to_runtime(output_ty, &output, function.span); + emitter.line(format!("store ptr {handle}, ptr %out")); + emitter.disarm(emitter.guards.len() - 1); + let callback = emitter.finish_protocol_callback( + &format!("dever_http_response_{name}"), + "ptr %input, ptr %out, ptr %error", + ); + self.declarations.push_str(&callback); + format!("@dever_http_response_{name}") + }; + let message = self.http_fault_message(instance); + let (asynchronous, synchronous) = if specialize::suspends(self.program, instance) { + (format!("@dever_async_{name}"), "null".into()) + } else { + ("null".into(), format!("@dever_sync_{name}")) + }; + writeln!(self.declarations, "{descriptor} = private constant %dever.http_handler {{ ptr {asynchronous}, ptr {synchronous}, ptr @dever_owned_{}, ptr {context}, ptr @dever_http_pack_{name}, ptr {response}, ptr {message} }}", self.type_index(&input)).unwrap(); + descriptor + } +} + +impl FunctionEmitter<'_, '_> { + pub(super) fn finish_protocol_callback(mut self, name: &str, arguments: &str) -> String { + self.exit("0"); + self.start("cleanup"); + self.release_from(0); + let status = self.temp(); + self.line(format!("{status} = load i32, ptr %exit_status")); + let message = self.temp(); + self.line(format!( + "{message} = getelementptr %dever.fault, ptr %fault, i32 0, i32 4" + )); + let buffer = self.temp(); + self.line(format!("{buffer} = load {{ ptr, i64 }}, ptr {message}")); + self.line(format!("store {{ ptr, i64 }} {buffer}, ptr %error")); + self.line(format!("ret i32 {status}")); + format!( + "define internal i32 @{name}({arguments}) {{\nentry:\n %exit_status = alloca i32\n store i32 0, ptr %exit_status\n %fault = alloca %dever.fault\n store %dever.fault zeroinitializer, ptr %fault\n{}{} }}\n", + self.prologue, self.body + ) + } + + pub(super) fn http_serve( + &mut self, + operation: Intrinsic, + handler: crate::hir::HandlerTarget, + arguments: &[Expression], + values: &[String], + result: &Expression, + ) -> String { + let live = matches!( + operation, + Intrinsic::HttpServeLive | Intrinsic::HttpServeLiveTls + ); + let tls = matches!( + operation, + Intrinsic::HttpServeTls | Intrinsic::HttpServeLiveTls + ); + let instance = Specialization { + function: specialize::resolve_handler(handler, &self.bindings), + handlers: Vec::new(), + }; + let descriptor = self.module.http_handler(&instance, live); + let limits = self.http_limits(&arguments[1].ty, &values[1]); + let live_limits = if live { + self.protocol_limits( + &arguments[2].ty, + &values[2], + &["chunk_bytes", "idle_ms", "heartbeat_ms"], + ) + } else { + "null".into() + }; + let tls_index = if live { 3 } else { 2 }; + let tls_value = if tls { + values[tls_index].clone() + } else { + "null".into() + }; + let context_index = tls_index + usize::from(tls); + let context = arguments + .get(context_index) + .map_or("null".into(), |argument| { + self.row_pointer(&argument.ty, &values[context_index]) + }); + let operation = self.async_operation( + "async_http_serve", + vec![ + format!("ptr {}", values[0]), + format!("ptr {limits}"), + format!("ptr {live_limits}"), + format!("ptr {tls_value}"), + format!("ptr {descriptor}"), + format!("ptr {context}"), + ], + result.span, + ); + self.await_operation(&operation, &Type::Unit, result.span, true) + } +} diff --git a/crates/dever-core/src/llvm/http_values.rs b/crates/dever-core/src/llvm/http_values.rs new file mode 100644 index 0000000..f954fa1 --- /dev/null +++ b/crates/dever-core/src/llvm/http_values.rs @@ -0,0 +1,549 @@ +//! Owned protocol records and canonical ABI fields. +use super::*; + +impl FunctionEmitter<'_, '_> { + pub(super) fn protocol_to_runtime(&mut self, ty: &Type, value: &str, span: Span) -> String { + let Type::Named(id) = ty else { unreachable!() }; + let name = self.module.program.types[*id].name.clone(); + if name == "dever.websocket.Message" { + return self.websocket_to_runtime(*id, value, span); + } + let request = matches!( + name.as_str(), + "dever.http.Request" | "dever.http.StreamRequest" + ); + let stream = name == "dever.http.StreamRequest"; + let mut fields = Vec::new(); + for name in if request { + &["method", "target"][..] + } else { + &["status"][..] + } { + fields.push(( + if request { "ptr" } else { "i64" }, + self.record_field(ty, value, name).1, + )); + } + let (headers_ty, headers) = self.record_field(ty, value, "headers"); + fields.push(("ptr", self.headers_to_runtime(&headers_ty, &headers, span))); + let (body_ty, body) = self.record_field(ty, value, "body"); + fields.push(("ptr", body)); + let fields = self.protocol_struct(&fields); + let kind = if stream { + "http_stream_request" + } else if request { + "http_request" + } else { + "http_response" + }; + let mut args = vec![format!("ptr {fields}")]; + if stream { + let Type::AsyncStream(event) = body_ty else { + unreachable!() + }; + let Type::Named(id) = event.as_ref() else { + unreachable!() + }; + args.push(format!("ptr {}", self.module.upload_decoder(*id))); + } + let handle = self.protocol_call(&format!("dever_rt_v1_{kind}_new"), args, "ptr", span); + self.protocol_owner(kind, &handle); + handle + } + + pub(super) fn protocol_to_language(&mut self, ty: &Type, handle: &str, span: Span) -> String { + let Type::Named(id) = ty else { unreachable!() }; + let definition = self.module.program.types[*id].clone(); + if definition.name == "dever.websocket.Message" { + self.protocol_owner("ws_message", handle); + return self.websocket_to_language(*id, handle, span); + } + let Shape::Record(fields) = definition.shape else { + unreachable!() + }; + let request = definition.name == "dever.http.Request"; + let stream = definition.name == "dever.http.StreamResponse"; + let kind = if request { + "http_request" + } else if stream { + "http_stream_response" + } else { + "http_response" + }; + self.protocol_owner(kind, handle); + let mut args = vec![format!("ptr {handle}")]; + if stream { + let Type::AsyncStream(event) = + &fields.iter().find(|field| field.name == "body").unwrap().ty + else { + unreachable!() + }; + let Type::Named(id) = event.as_ref() else { + unreachable!() + }; + args.push(format!("ptr {}", self.module.read_event(*id))); + args.push(format!( + "ptr @dever_owned_{}", + self.module.type_index(event) + )); + } + let layout = if request { + "{ ptr, ptr, ptr, ptr }" + } else { + "{ i64, ptr, ptr }" + }; + let decoded = self.protocol_call(&format!("dever_rt_v1_{kind}_fields"), args, layout, span); + let names = if request { + &["method", "target", "headers", "body"][..] + } else { + &["status", "headers", "body"][..] + }; + let mut values = Vec::new(); + let mut transfers = Vec::new(); + for (index, name) in names.iter().enumerate() { + let field_ty = &fields.iter().find(|field| field.name == *name).unwrap().ty; + let value = self.temp(); + self.line(format!( + "{value} = extractvalue {layout} {decoded}, {index}" + )); + if *name == "headers" { + self.protocol_owner("http_headers", &value); + values.push((*name, value)); + } else { + if contains_owned(self.module.program, field_ty) { + transfers.push(self.own_value(field_ty, &value, false)); + } + values.push((*name, value)); + } + } + // Every getter field is owned before a subsequent conversion can fail. + let header_index = values + .iter() + .position(|(name, _)| *name == "headers") + .unwrap(); + let header_ty = &fields + .iter() + .find(|field| field.name == "headers") + .unwrap() + .ty; + let headers = self.headers_to_language(header_ty, &values[header_index].1, span); + transfers.push(self.own_value(header_ty, &headers, false)); + values[header_index].1 = headers; + let result = self.record_value(ty, &values); + for guard in transfers { + self.disarm(guard); + } + result + } + + pub(super) fn sse_event(&mut self, ty: &Type, value: &str) -> String { + let event = self.record_field(ty, value, "event").1; + let data = self.record_field(ty, value, "data").1; + let (id_ty, id) = self.record_field(ty, value, "id"); + let id = self.optional_payload(&id_ty, &id, "null").1; + let (retry_ty, retry) = self.record_field(ty, value, "retry_ms"); + let (present, retry) = self.optional_payload(&retry_ty, &retry, "0"); + let flag = self.temp(); + self.line(format!("{flag} = zext i1 {present} to i8")); + self.protocol_struct(&[ + ("ptr", event), + ("ptr", data), + ("ptr", id), + ("i64", retry), + ("i8", flag), + ]) + } + + fn optional_payload(&mut self, ty: &Type, value: &str, absent: &str) -> (String, String) { + let Type::Nullable(inner) = ty else { + unreachable!() + }; + let present = self.temp(); + self.line(format!( + "{present} = extractvalue {} {value}, 0", + self.module.ty(ty) + )); + let payload = self.temp(); + self.line(format!( + "{payload} = extractvalue {} {value}, 1", + self.module.ty(ty) + )); + let selected = self.temp(); + self.line(format!( + "{selected} = select i1 {present}, {} {payload}, {} {absent}", + self.module.ty(inner), + self.module.ty(inner) + )); + (present, selected) + } + + fn websocket_to_runtime(&mut self, id: usize, value: &str, span: Span) -> String { + let Shape::Choice(variants) = self.module.program.types[id].shape.clone() else { + unreachable!() + }; + let source = self.row_pointer(&Type::Named(id), value); + let tag = self.temp(); + self.line(format!("{tag} = extractvalue %T{id} {value}, 0")); + let output = self.entry_slot_ir("ptr"); + let done = self.label("ws_encoded"); + let invalid = self.label("ws_invalid_tag"); + let labels = variants + .iter() + .map(|_| self.label("ws_encode")) + .collect::<Vec<_>>(); + self.line(format!( + "switch i32 {tag}, label %{invalid} [{}]", + labels + .iter() + .enumerate() + .map(|(i, label)| format!("i32 {i}, label %{label}")) + .collect::<Vec<_>>() + .join(" ") + )); + self.start(&invalid); + self.line("unreachable"); + for (index, variant) in variants.iter().enumerate() { + self.start(&labels[index]); + let pointer = self.temp(); + self.line(format!( + "{pointer} = getelementptr %T{id}, ptr {source}, i32 0, i32 1" + )); + let payload = self.temp(); + self.line(format!("{payload} = load ptr, ptr {pointer}")); + let kind = ["Text", "Binary", "Ping", "Pong"] + .iter() + .position(|name| *name == variant.name) + .unwrap(); + let handle = self.protocol_call( + "dever_rt_v1_ws_message_new", + vec![format!("i32 {kind}"), format!("ptr {payload}")], + "ptr", + span, + ); + self.line(format!("store ptr {handle}, ptr {output}")); + self.line(format!("br label %{done}")); + } + self.start(&done); + let handle = self.temp(); + self.line(format!("{handle} = load ptr, ptr {output}")); + self.protocol_owner("ws_message", &handle); + handle + } + + fn websocket_to_language(&mut self, id: usize, handle: &str, span: Span) -> String { + let fields = self.protocol_call( + "dever_rt_v1_ws_message_fields", + vec![format!("ptr {handle}")], + "{ i32, ptr }", + span, + ); + let tag = self.temp(); + self.line(format!("{tag} = extractvalue {{ i32, ptr }} {fields}, 0")); + let payload = self.temp(); + self.line(format!( + "{payload} = extractvalue {{ i32, ptr }} {fields}, 1" + )); + let result = self.entry_slot(&Type::Named(id)); + let Shape::Choice(variants) = self.module.program.types[id].shape.clone() else { + unreachable!() + }; + let done = self.label("ws_decoded"); + let invalid = self.label("ws_invalid_kind"); + let labels = (0..4).map(|_| self.label("ws_decode")).collect::<Vec<_>>(); + self.line(format!( + "switch i32 {tag}, label %{invalid} [{}]", + labels + .iter() + .enumerate() + .map(|(i, label)| format!("i32 {i}, label %{label}")) + .collect::<Vec<_>>() + .join(" ") + )); + self.start(&invalid); + self.line("unreachable"); + for (kind, name) in ["Text", "Binary", "Ping", "Pong"].iter().enumerate() { + self.start(&labels[kind]); + let (variant, definition) = variants + .iter() + .enumerate() + .find(|(_, variant)| variant.name == *name) + .unwrap(); + self.write_variant( + id, + variant, + &[(definition.fields[0].ty.clone(), payload.clone())], + &result, + ); + self.line(format!("br label %{done}")); + } + self.start(&done); + let value = self.temp(); + self.line(format!("{value} = load %T{id}, ptr {result}")); + value + } + pub(super) fn protocol_owner(&mut self, kind: &'static str, handle: &str) -> usize { + let slot = self.entry_slot_ir("ptr"); + self.line(format!("store ptr {handle}, ptr {slot}")); + let guard = self.register_owned(OwnedKind::Protocol(kind), slot); + self.mark_live(guard); + guard + } + + pub(super) fn protocol_call( + &mut self, + symbol: &str, + args: Vec<String>, + layout: &str, + span: Span, + ) -> String { + self.runtime_call(symbol, args, &Type::Unit, false, Some(layout), span) + } + + fn record_field(&mut self, ty: &Type, value: &str, name: &str) -> (Type, String) { + let Type::Named(id) = ty else { + unreachable!("checked protocol record") + }; + let Shape::Record(fields) = &self.module.program.types[*id].shape else { + unreachable!() + }; + let (index, field) = fields + .iter() + .enumerate() + .find(|(_, field)| field.name == name) + .expect("checked protocol field"); + let field_ty = field.ty.clone(); + let extracted = self.temp(); + self.line(format!( + "{extracted} = extractvalue {} {value}, {index}", + self.module.ty(ty) + )); + (field_ty, extracted) + } + + fn record_value(&mut self, ty: &Type, fields: &[(&str, String)]) -> String { + let Type::Named(id) = ty else { unreachable!() }; + let Shape::Record(definition) = &self.module.program.types[*id].shape else { + unreachable!() + }; + let definition = definition.clone(); + let mut value = "zeroinitializer".to_string(); + for (index, field) in definition.iter().enumerate() { + let (_, content) = fields + .iter() + .find(|(name, _)| *name == field.name) + .expect("complete protocol fields"); + let next = self.temp(); + self.line(format!( + "{next} = insertvalue {} {value}, {} {content}, {index}", + self.module.ty(ty), + self.module.ty(&field.ty) + )); + value = next; + } + value + } + + fn protocol_struct(&mut self, fields: &[(&str, String)]) -> String { + let layout = format!( + "{{ {} }}", + fields + .iter() + .map(|(ty, _)| *ty) + .collect::<Vec<_>>() + .join(", ") + ); + let slot = self.entry_slot_ir(&layout); + let mut value = "zeroinitializer".to_string(); + for (index, (ty, content)) in fields.iter().enumerate() { + let next = self.temp(); + self.line(format!( + "{next} = insertvalue {layout} {value}, {ty} {content}, {index}" + )); + value = next; + } + self.line(format!("store {layout} {value}, ptr {slot}")); + slot + } + + pub(super) fn protocol_limits(&mut self, ty: &Type, value: &str, names: &[&str]) -> String { + let mut fields = Vec::new(); + for name in names { + fields.push(("i64", self.record_field(ty, value, name).1)); + } + self.protocol_struct(&fields) + } + + pub(super) fn http_limits(&mut self, ty: &Type, value: &str) -> String { + let mut fields = ["header_bytes", "body_bytes", "timeout_ms", "connections"] + .iter() + .map(|name| ("i64", self.record_field(ty, value, name).1)) + .collect::<Vec<_>>(); + let (http2_ty, http2) = self.record_field(ty, value, "http2"); + let Type::Nullable(inner) = &http2_ty else { + unreachable!() + }; + let present = self.temp(); + self.line(format!( + "{present} = extractvalue {} {http2}, 0", + self.module.ty(&http2_ty) + )); + let pointer = self.entry_slot_ir("ptr"); + self.line(format!("store ptr null, ptr {pointer}")); + let some = self.label("http2_limits"); + let done = self.label("http_limits"); + self.line(format!("br i1 {present}, label %{some}, label %{done}")); + self.start(&some); + let content = self.temp(); + self.line(format!( + "{content} = extractvalue {} {http2}, 1", + self.module.ty(&http2_ty) + )); + let limits = self.protocol_limits( + inner, + &content, + &["streams", "stream_window_bytes", "connection_window_bytes"], + ); + self.line(format!("store ptr {limits}, ptr {pointer}")); + self.line(format!("br label %{done}")); + self.start(&done); + let http2 = self.temp(); + self.line(format!("{http2} = load ptr, ptr {pointer}")); + fields.push(("ptr", http2)); + self.protocol_struct(&fields) + } + + pub(super) fn headers_to_runtime(&mut self, ty: &Type, value: &str, span: Span) -> String { + let Type::List(element) = ty else { + unreachable!() + }; + let headers = self.protocol_call("dever_rt_v1_http_headers_new", vec![], "ptr", span); + self.protocol_owner("http_headers", &headers); + let retained = self.temp(); + self.line(format!( + "{retained} = call ptr @dever_rt_v1_list_retain(ptr {value})" + )); + let cursor = self.protocol_call( + "dever_rt_v1_list_cursor_take", + vec![format!("ptr {retained}")], + "ptr", + span, + ); + let cursor_slot = self.entry_slot_ir("ptr"); + self.line(format!("store ptr {cursor}, ptr {cursor_slot}")); + let cursor_guard = self.register_owned(OwnedKind::ListCursor, cursor_slot); + self.mark_live(cursor_guard); + let row = self.entry_slot(element); + let guard = self.register_guard(element, row.clone()); + let present = self.entry_slot_ir("i8"); + let error = self.entry_slot_ir("{ ptr, i64 }"); + let next = self.label("header_next"); + let body = self.label("header_value"); + let done = self.label("headers_ready"); + self.line(format!("br label %{next}")); + self.start(&next); + let status = self.temp(); + self.line(format!("{status} = call i32 @dever_rt_v1_list_cursor_next(ptr {cursor}, ptr {row}, ptr {present}, ptr {error})")); + self.check_runtime_status(&status, &error, span); + let flag = self.temp(); + self.line(format!("{flag} = load i8, ptr {present}")); + let exists = self.temp(); + self.line(format!("{exists} = icmp ne i8 {flag}, 0")); + self.line(format!("br i1 {exists}, label %{body}, label %{done}")); + self.start(&body); + self.mark_live(guard); + let value = self.temp(); + self.line(format!( + "{value} = load {}, ptr {row}", + self.module.ty(element) + )); + let name = self.record_field(element, &value, "name").1; + let value = self.record_field(element, &value, "value").1; + let status = self.temp(); + self.line(format!("{status} = call i32 @dever_rt_v1_http_headers_push(ptr {headers}, ptr {name}, ptr {value}, ptr {error})")); + self.check_runtime_status(&status, &error, span); + self.release_guard(guard); + self.line(format!("br label %{next}")); + self.start(&done); + self.release_guard(cursor_guard); + headers + } + + fn headers_to_language(&mut self, ty: &Type, headers: &str, span: Span) -> String { + let Type::List(element) = ty else { + unreachable!() + }; + let list = self.runtime_call( + "dever_rt_v1_list_new", + vec![ + format!("ptr @dever_type_{}", self.module.type_index(element)), + "ptr null".into(), + "i64 0".into(), + ], + ty, + false, + None, + span, + ); + let list_guard = self.own_value(ty, &list, false); + let list_slot = self.guards[list_guard].pointer.clone(); + let length = self.runtime_call( + "dever_rt_v1_http_headers_length", + vec![format!("ptr {headers}")], + &Type::Int, + false, + None, + span, + ); + let index = self.entry_slot_ir("i64"); + self.line(format!("store i64 0, ptr {index}")); + let next = self.label("header_decode_next"); + let body = self.label("header_decode"); + let done = self.label("headers_decoded"); + self.line(format!("br label %{next}")); + self.start(&next); + let position = self.temp(); + self.line(format!("{position} = load i64, ptr {index}")); + let exists = self.temp(); + self.line(format!("{exists} = icmp slt i64 {position}, {length}")); + self.line(format!("br i1 {exists}, label %{body}, label %{done}")); + self.start(&body); + let first = self.guards.len(); + let fields = self.protocol_call( + "dever_rt_v1_http_headers_get", + vec![format!("ptr {headers}"), format!("i64 {position}")], + "{ ptr, ptr }", + span, + ); + let name = self.temp(); + self.line(format!("{name} = extractvalue {{ ptr, ptr }} {fields}, 0")); + self.own_value(&Type::Text, &name, false); + let value = self.temp(); + self.line(format!("{value} = extractvalue {{ ptr, ptr }} {fields}, 1")); + self.own_value(&Type::Bytes, &value, false); + let header = self.record_value(element, &[("name", name), ("value", value)]); + let row = self.row_pointer(element, &header); + let previous = self.temp(); + self.line(format!("{previous} = load ptr, ptr {list_slot}")); + self.disarm(list_guard); + let appended = self.runtime_call( + "dever_rt_v1_list_append_take", + vec![format!("ptr {previous}"), format!("ptr {row}")], + ty, + false, + None, + span, + ); + self.line(format!("store ptr {appended}, ptr {list_slot}")); + self.mark_live(list_guard); + self.release_from(first); + let incremented = self.temp(); + self.line(format!("{incremented} = add i64 {position}, 1")); + self.line(format!("store i64 {incremented}, ptr {index}")); + self.line(format!("br label %{next}")); + self.start(&done); + let list = self.temp(); + self.line(format!("{list} = load ptr, ptr {list_slot}")); + self.disarm(list_guard); + list + } +} diff --git a/crates/dever-core/src/llvm/intrinsics.rs b/crates/dever-core/src/llvm/intrinsics.rs new file mode 100644 index 0000000..7fc0eb4 --- /dev/null +++ b/crates/dever-core/src/llvm/intrinsics.rs @@ -0,0 +1,291 @@ +use super::*; + +pub(super) fn supported(operation: Intrinsic) -> bool { + use Intrinsic::*; + super::system::supported(operation) + || super::testing::supported(operation) + || super::http::supported(operation) + || super::api_context::supported(operation) + || matches!( + operation, + TextTrim + | TextLower + | TextUpper + | TextCodepoint + | TextFromCodepoint + | TextAt + | TextSlice + | TextIndexOf + | TextContains + | TextStartsWith + | TextEndsWith + | TextReplace + | TextSplit + | IntParse + | IntToText + | DecimalParse + | DecimalToText + | DecimalFromInt + | DecimalRound + | FloatParse + | FloatToText + | FloatFromInt + | FloatSqrt + | FloatSin + | FloatCos + | FloatLog + | FloatPow + | FloatIsNan + | FloatIsFinite + | FloatIsInfinite + | IdFromText + | IdToText + | BytesFromText + | BytesLength + | BytesAt + | BytesConcat + | BytesToText + | BytesFromInts + | BytesSlice + | FileOpen + | FileCreate + | FileRead + | FileWrite + | FileClose + | FileChunks + | TimeSleep + | TaskSleep + | TaskTicks + | TaskTimeout + | TaskRace + | StreamOf + | TcpConnect + | TcpConnectTimeout + | TcpListen + | TcpAccept + | TcpPort + | TcpRead + | TcpWrite + | TcpTimeout + | TcpClose + | TcpCloseListener + | TcpChunks + | TcpConnections + ) +} + +impl FunctionEmitter<'_, '_> { + pub(super) fn decimal_parts(&mut self, value: &str) -> Vec<String> { + (0..2) + .map(|index| { + let part = self.temp(); + self.line(format!( + "{part} = extractvalue {{ i64, i64 }} {value}, {index}" + )); + format!("i64 {part}") + }) + .collect() + } + + pub(super) fn intrinsic( + &mut self, + operation: Intrinsic, + handler: Option<crate::hir::HandlerTarget>, + arguments: &[Expression], + result: &Expression, + ) -> String { + use Intrinsic::*; + if matches!(operation, TaskTimeout | TaskRace) { + return self.task_selection(operation, handler, arguments, result); + } + let values = arguments + .iter() + .map(|argument| { + self.expression(argument) + .expect("validated intrinsic argument") + }) + .collect::<Vec<_>>(); + let args = arguments + .iter() + .zip(&values) + .map(|(argument, value)| format!("{} {value}", self.module.ty(&argument.ty))) + .collect::<Vec<_>>(); + let nullable = matches!(result.ty, Type::Nullable(_)); + let span = result.span; + if super::testing::supported(operation) { + return self.test_intrinsic(operation, arguments, &values, span); + } + if super::api_context::supported(operation) { + return self.api_intrinsic(operation, arguments, &values, result); + } + if super::http::supported(operation) { + return self.http_intrinsic(operation, handler, arguments, &values, result); + } + if super::system::supported(operation) { + return self.system_intrinsic(operation, args, &values, result); + } + if matches!( + operation, + StreamOf + | TaskTicks + | TcpConnect + | TcpConnectTimeout + | TcpListen + | TcpAccept + | TcpPort + | TcpRead + | TcpWrite + | TcpTimeout + | TcpClose + | TcpCloseListener + | TcpChunks + | TcpConnections + ) { + return self.network_intrinsic(operation, arguments, args, result); + } + if matches!(operation, BytesToText | BytesFromInts | BytesSlice) { + return self.bytes_result(operation, args, result); + } + if matches!( + operation, + FileOpen | FileCreate | FileRead | FileWrite | FileClose | FileChunks + ) { + return self.file_result(operation, args, result); + } + if operation == TaskSleep { + return self.task_sleep(&values[0], result); + } + if operation == TimeSleep { + return self.recoverable_call("dever_rt_v1_time_sleep", args, result, "Done", None); + } + let (symbol, args, physical) = match operation { + TextTrim => ("text_trim", args, None), + TextLower => ("text_lower", args, None), + TextUpper => ("text_upper", args, None), + TextCodepoint => ("text_codepoint", args, None), + TextFromCodepoint => ("text_from_codepoint", args, None), + TextAt => ("text_at", args, None), + TextSlice => ("text_slice", args, None), + TextIndexOf => ("text_index_of", args, None), + TextContains => ("text_contains", args, Some("i8")), + TextStartsWith => ("text_starts_with", args, Some("i8")), + TextEndsWith => ("text_ends_with", args, Some("i8")), + TextReplace => ("text_replace", args, None), + TextSplit => { + let mut args = args; + args.push(format!( + "ptr @dever_type_{}", + self.module.type_index(&Type::Text) + )); + ("text_split", args, None) + } + IntParse => ("int_parse", args, None), + DecimalParse => ("decimal_parse", args, None), + FloatParse => ("float_parse", args, None), + DecimalFromInt => ("decimal_from_int", args, None), + DecimalRound => { + let mut parts = self.decimal_parts(&values[0]); + parts.push(format!("i64 {}", values[1])); + ("decimal_round", parts, None) + } + BytesFromText => ("bytes_from_text", args, None), + BytesLength => ("bytes_length", args, None), + BytesAt => ("bytes_at", args, None), + BytesConcat => ("bytes_concat", args, None), + IntToText => return self.render_text("dever_rt_v1_int_to_text", args, span), + FloatToText => return self.render_text("dever_rt_v1_float_to_text", args, span), + DecimalToText => { + let parts = self.decimal_parts(&values[0]); + return self.render_text("dever_rt_v1_decimal_to_text", parts, span); + } + FloatFromInt => { + let converted = self.temp(); + self.line(format!("{converted} = sitofp i64 {} to double", values[0])); + return converted; + } + FloatSqrt | FloatSin | FloatCos | FloatLog | FloatPow => { + let name = match operation { + FloatSqrt => "sqrt", + FloatSin => "sin", + FloatCos => "cos", + FloatLog => "log", + FloatPow => "pow", + _ => unreachable!(), + }; + let converted = self.temp(); + self.line(format!( + "{converted} = call double @llvm.{name}.f64({})", + args.join(", ") + )); + return converted; + } + FloatIsNan => { + let converted = self.temp(); + self.line(format!( + "{converted} = fcmp uno double {}, {}", + values[0], values[0] + )); + return converted; + } + FloatIsFinite | FloatIsInfinite => { + let bits = self.temp(); + self.line(format!("{bits} = bitcast double {} to i64", values[0])); + let exponent = self.temp(); + self.line(format!("{exponent} = and i64 {bits}, 9218868437227405312")); + let all_ones = self.temp(); + self.line(format!( + "{all_ones} = icmp eq i64 {exponent}, 9218868437227405312" + )); + if operation == FloatIsFinite { + let finite = self.temp(); + self.line(format!("{finite} = xor i1 {all_ones}, 1")); + return finite; + } + let fraction = self.temp(); + self.line(format!("{fraction} = and i64 {bits}, 4503599627370495")); + let no_fraction = self.temp(); + self.line(format!("{no_fraction} = icmp eq i64 {fraction}, 0")); + let infinite = self.temp(); + self.line(format!("{infinite} = and i1 {all_ones}, {no_fraction}")); + return infinite; + } + IdFromText | IdToText => { + let retained = self.temp(); + self.line(format!( + "{retained} = call ptr @dever_rt_v1_text_retain(ptr {})", + values[0] + )); + return retained; + } + _ => unreachable!("preflighted intrinsic"), + }; + self.runtime_call( + &format!("dever_rt_v1_{symbol}"), + args, + &result.ty, + nullable, + physical, + span, + ) + } + + fn bytes_result( + &mut self, + operation: Intrinsic, + args: Vec<String>, + result: &Expression, + ) -> String { + let success_name = if operation == Intrinsic::BytesToText { + "Decoded" + } else { + "Ready" + }; + let symbol = match operation { + Intrinsic::BytesToText => "dever_rt_v1_bytes_to_text_handle", + Intrinsic::BytesFromInts => "dever_rt_v1_bytes_from_ints", + Intrinsic::BytesSlice => "dever_rt_v1_bytes_slice", + _ => unreachable!(), + }; + self.recoverable_call(symbol, args, result, success_name, None) + } +} diff --git a/crates/dever-core/src/llvm/jobs.rs b/crates/dever-core/src/llvm/jobs.rs new file mode 100644 index 0000000..14962fd --- /dev/null +++ b/crates/dever-core/src/llvm/jobs.rs @@ -0,0 +1,390 @@ +//! Durable Job metadata and concrete payload/transaction adapters. +use super::*; +use crate::model::{DatabaseOwner, ModelScope}; + +const CLOCK_DECLARATIONS: &str = "\ +declare i32 @dever_rt_v1_job_clock_new(i8, ptr, ptr) +declare i32 @dever_rt_v1_job_clock_now(ptr, ptr, ptr) +declare i32 @dever_rt_v1_job_clock_advance(ptr, i64, ptr) +declare void @dever_rt_v1_job_clock_release(ptr) +@dever_job_clock = private global ptr null +"; + +const DECLARATIONS: &str = "\ +%dever.job = type { { ptr, i64 }, { ptr, i64 }, i32, i32, ptr, ptr, ptr, i64, ptr } +%dever.job_auth = type { { ptr, i64 }, ptr } +declare i32 @dever_rt_v1_job_session_new(ptr, ptr, ptr, i64, ptr, i64, ptr, i64, ptr, i64, ptr, ptr) +declare void @dever_rt_v1_job_session_release(ptr) +declare i32 @dever_rt_v1_job_configure(ptr, i8, i8, ptr) +declare i32 @dever_rt_v1_job_decode_unit(ptr, ptr) +declare ptr @dever_rt_v1_job_initialize(ptr, ptr, ptr) +declare ptr @dever_rt_v1_job_enqueue(ptr, i64, ptr, ptr, ptr, ptr, i64, ptr, ptr) +declare ptr @dever_rt_v1_job_drain(ptr, i64, ptr) +declare ptr @dever_rt_v1_job_system_scope(i64, i8, ptr, ptr, ptr) +declare ptr @dever_rt_v1_api_serve_with_jobs(ptr, ptr, i64, ptr, i64, ptr, ptr) +declare ptr @dever_rt_v1_api_migrate_application(ptr, ptr, ptr, ptr, i64, ptr, i64, ptr) +@dever_job_session = private global ptr null +"; + +impl Module<'_> { + pub(super) fn jobs_enabled(&self) -> bool { + self.application.is_some() && !self.program.jobs.is_empty() + } + + pub(super) fn clock_enabled(&self) -> bool { + self.jobs_enabled() + || self + .application + .as_ref() + .is_some_and(|app| app.test.is_some()) + } + + pub(super) fn owner_binding(&mut self, owner: DatabaseOwner) -> String { + if let DatabaseOwner::Model(model) = owner { + return self.database_binding(model); + } + let DatabaseOwner::Job(target) = owner else { + unreachable!() + }; + let (explicit, root) = self.program.database_selector(owner); + let explicit = explicit.map(str::to_owned); + let root = root.to_owned(); + let explicit = explicit + .map(|value| self.wire_names(&[value])) + .unwrap_or_else(|| "null".into()); + let root = self.api_name(&root); + let tenant = + u8::from(crate::native::jobs::scope(self.program, target) == ModelScope::Tenant); + format!("{{ ptr {explicit}, {root}, i8 {tenant} }}") + } + + pub(super) fn emit_job_descriptors(&mut self) { + if self.clock_enabled() { + self.declarations.push_str(CLOCK_DECLARATIONS); + } + if !self.jobs_enabled() { + return; + } + self.declarations.push_str(DECLARATIONS); + let jobs = self.program.jobs.clone(); + let mut descriptors = Vec::new(); + for (index, job) in jobs.iter().enumerate() { + let binding = self.owner_binding(DatabaseOwner::Job(job.function)); + writeln!( + self.declarations, + "@dever_job_binding_{index} = private constant %dever.db_binding {binding}" + ) + .unwrap(); + let target = self.api_name(&self.program.functions[job.function].name.clone()); + let schema = self.api_name( + job.payload + .as_ref() + .map(|schema| schema.fingerprint()) + .unwrap_or("wire-v1:Job:Unit"), + ); + let schedule = job + .schedule + .as_ref() + .map(|cron| self.wire_names(std::slice::from_ref(cron))) + .unwrap_or_else(|| "null".into()); + let components = self.wire_names(&job.components); + let handler = self.job_handler(index); + descriptors.push(format!("%dever.job {{ {target}, {schema}, i32 {}, i32 {}, ptr {schedule}, ptr @dever_job_binding_{index}, ptr {components}, i64 {}, ptr {handler} }}", job.attempts, job.timeout_ms, job.components.len())); + } + writeln!( + self.declarations, + "@dever_jobs = private constant [{} x %dever.job] [{}]", + jobs.len(), + descriptors.join(", ") + ) + .unwrap(); + let mut hooks = Vec::new(); + for (index, hook) in self.program.auth.clone().iter().enumerate() { + let provider = self.api_name(&hook.provider); + hooks.push(format!( + "%dever.job_auth {{ {provider}, ptr @dever_api_auth_{index} }}" + )); + } + writeln!( + self.declarations, + "@dever_job_auth = private constant [{} x %dever.job_auth] [{}]", + hooks.len(), + hooks.join(", ") + ) + .unwrap(); + } + + fn job_handler(&mut self, index: usize) -> String { + let instance = Specialization { + function: self.application.as_ref().unwrap().job_handlers[index], + handlers: Vec::new(), + }; + let function = self.program.functions[instance.function].clone(); + let input = asynchronous::input_type(&function); + let span = function.span; + let mut emitter = FunctionEmitter::new(self, &instance); + emitter.asynchronous = false; + emitter.protocol_callback = true; + emitter.start("entry"); + if let Some(parameter) = function.parameters.first() { + let ty = parameter.value_type().unwrap(); + let decoder = emitter + .module + .wire_callback(ty, crate::wire::Policy::Job, false); + let payload = emitter.runtime_call( + "dever_rt_v1_wire_decode", + vec![ + "ptr %payload".into(), + format!("ptr @dever_type_{}", emitter.module.type_index(ty)), + format!("ptr @{decoder}"), + ], + ty, + false, + None, + span, + ); + let row = emitter.temp(); + emitter.line(format!( + "{row} = insertvalue {} zeroinitializer, {} {payload}, 0", + emitter.module.ty(&input), + emitter.module.ty(ty) + )); + emitter.line(format!( + "store {} {row}, ptr %out", + emitter.module.ty(&input) + )); + } else { + emitter.wire_write("dever_rt_v1_job_decode_unit", vec!["ptr %payload".into()]); + } + let decode = format!("dever_job_decode_{index}"); + let body = emitter + .finish_protocol_callback(&decode, "ptr %payload, i64 %unused, ptr %out, ptr %error"); + self.declarations.push_str(&body); + let descriptor = format!("@dever_job_handler_{index}"); + writeln!(self.declarations, "{descriptor} = private constant %dever.api_handler {{ ptr @dever_async_{}, ptr null, ptr @dever_owned_{}, ptr @{decode}, ptr @dever_api_fault_status, ptr null }}", self.names[&instance], self.type_index(&input)).unwrap(); + descriptor + } + + pub(super) fn emit_job_body( + &mut self, + instance: &Specialization, + index: usize, + ) -> Result<(), String> { + let function = self.program.functions[instance.function].clone(); + let target = Specialization { + function: self.program.jobs[index].function, + handlers: Vec::new(), + }; + let mut emitter = FunctionEmitter::new(self, instance); + let parameters = emitter.begin_generated_function(&function); + let arguments = function + .parameters + .iter() + .enumerate() + .map(|(index, parameter)| { + ( + parameter.value_type().unwrap().clone(), + emitter.parameter(index), + ) + }) + .collect::<Vec<_>>(); + let transaction = emitter.begin_command_transaction(&target, function.span); + if specialize::suspends(emitter.module.program, &target) { + let (status, _) = + emitter.invoke_values(&target, &arguments, &Type::Unit, function.span); + emitter.propagate_status(&status, function.span, false); + } else { + emitter.invoke_sync_boundary( + &target, + &arguments, + "async_blocking", + &Type::Unit, + function.span, + ); + } + if transaction { + emitter.commit_command_transaction(); + } + emitter.exit("0"); + let name = emitter.module.names[instance].clone(); + let body = emitter.finish_function(&name, ¶meters); + self.functions.push_str(&body); + Ok(()) + } +} + +impl FunctionEmitter<'_, '_> { + pub(super) fn initialize_clock(&mut self, span: Span) { + if !self.module.clock_enabled() { + return; + } + let test = u8::from(self.module.application.as_ref().unwrap().test.is_some()); + let clock = self.protocol_call( + "dever_rt_v1_job_clock_new", + vec![format!("i8 {test}")], + "ptr", + span, + ); + self.line(format!("store ptr {clock}, ptr @dever_job_clock")); + } + + pub(super) fn register_jobs(&mut self, span: Span) { + if !self.module.jobs_enabled() { + return; + } + let session = self.temp(); + self.line(format!("{session} = load ptr, ptr @dever_api_session")); + let clock = self.temp(); + self.line(format!("{clock} = load ptr, ptr @dever_job_clock")); + let manifest = self + .module + .wire_names(&self.module.program.tenant_components.clone()); + let jobs = self.protocol_call( + "dever_rt_v1_job_session_new", + vec![ + format!("ptr {session}"), + format!("ptr {clock}"), + "ptr @dever_jobs".into(), + format!("i64 {}", self.module.program.jobs.len()), + "ptr @dever_job_auth".into(), + format!("i64 {}", self.module.program.auth.len()), + "ptr @dever_permissions".into(), + format!("i64 {}", self.module.program.permissions.len()), + format!("ptr {manifest}"), + format!("i64 {}", self.module.program.tenant_components.len()), + ], + "ptr", + span, + ); + self.line(format!("store ptr {jobs}, ptr @dever_job_session")); + } + + pub(super) fn initialize_jobs(&mut self, span: Span) { + if !self.module.jobs_enabled() { + return; + } + let jobs = self.temp(); + self.line(format!("{jobs} = load ptr, ptr @dever_job_session")); + let operation = + self.database_operation("job_initialize", vec![format!("ptr {jobs}")], span); + self.await_operation(&operation, &Type::Unit, span, false); + } + + pub(super) fn owner_database(&mut self, owner: DatabaseOwner, span: Span) -> String { + if let DatabaseOwner::Model(model) = owner { + return self.model_database(model); + } + let session = self.temp(); + self.line(format!("{session} = load ptr, ptr @dever_database_session")); + let binding = self.module.owner_binding(owner); + let slot = self.entry_slot_ir("%dever.db_binding"); + self.line(format!("store %dever.db_binding {binding}, ptr {slot}")); + let operation = self.database_operation( + "db_resolve_scoped", + vec![format!("ptr {session}"), format!("ptr {slot}")], + span, + ); + let database = self.await_database_handle(&operation, span); + self.protocol_owner("db", &database); + database + } + + pub(super) fn enqueue_job( + &mut self, + target: usize, + arguments: &[Expression], + scheduled: bool, + span: Span, + ) -> Result<String, String> { + let index = self + .module + .program + .jobs + .iter() + .position(|job| job.function == target) + .expect("reachable Job"); + let has_payload = self.module.program.jobs[index].payload.is_some(); + let values = arguments + .iter() + .map(|argument| self.expression(argument)) + .collect::<Result<Vec<_>, _>>()?; + let payload = if has_payload { + let ty = &arguments[0].ty; + let encoder = self + .module + .wire_callback(ty, crate::wire::Policy::Job, true); + let row = self.row_pointer(ty, &values[0]); + let payload = self.runtime_call( + "dever_rt_v1_wire_encode", + vec![ + format!("ptr {row}"), + format!("ptr @dever_type_{}", self.module.type_index(ty)), + format!("ptr @{encoder}"), + ], + &Type::Text, + false, + None, + span, + ); + self.own_value(&Type::Text, &payload, false); + payload + } else { + self.application_text("null", span) + }; + let key = usize::from(has_payload); + let run_at = if scheduled { + values[key + 1].clone() + } else { + let clock = self.temp(); + self.line(format!("{clock} = load ptr, ptr @dever_job_clock")); + self.protocol_call( + "dever_rt_v1_job_clock_now", + vec![format!("ptr {clock}")], + "i64", + span, + ) + }; + let database = self.owner_database(DatabaseOwner::Job(target), span); + let jobs = self.temp(); + self.line(format!("{jobs} = load ptr, ptr @dever_job_session")); + let operation = self.database_operation( + "job_enqueue", + vec![ + format!("ptr {jobs}"), + format!("i64 {index}"), + format!("ptr {database}"), + format!("ptr {}", self.database_context), + format!("ptr {payload}"), + format!("ptr {}", values[key]), + format!("i64 {run_at}"), + ], + span, + ); + Ok(self.await_operation(&operation, &Type::Id, span, false)) + } + + pub(super) fn invoke_system( + &mut self, + target: &Specialization, + input: &str, + tenant: &str, + span: Span, + ) { + let present = self.temp(); + self.line(format!("{present} = icmp sgt i64 {tenant}, 0")); + let flag = self.temp(); + self.line(format!("{flag} = zext i1 {present} to i8")); + let name = self.module.names[target].clone(); + let operation = self.async_operation( + "job_system_scope", + vec![ + format!("i64 {tenant}"), + format!("i8 {flag}"), + format!("ptr @dever_async_{name}"), + format!("ptr {input}"), + ], + span, + ); + self.await_operation(&operation, &Type::Unit, span, false); + } +} diff --git a/crates/dever-core/src/llvm/network.rs b/crates/dever-core/src/llvm/network.rs new file mode 100644 index 0000000..dc9fcc1 --- /dev/null +++ b/crates/dever-core/src/llvm/network.rs @@ -0,0 +1,207 @@ +//! Typed stream and TCP operations reuse coroutine polling and resource choices. +use super::resources::ResourceRow; +use super::*; + +pub(super) const DECLARATIONS: &str = "\ +declare i32 @dever_rt_v1_async_stream_from_list(ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_async_stream_from_bytes(ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_async_channel_stream(ptr, ptr, ptr) +declare i32 @dever_rt_v1_async_ticks(i64, ptr, ptr, ptr) +declare ptr @dever_rt_v1_async_stream_retain(ptr) +declare void @dever_rt_v1_async_stream_release(ptr) +declare ptr @dever_rt_v1_async_stream_pull(ptr) +declare i32 @dever_rt_v1_async_stream_close(ptr, ptr, ptr) +declare ptr @dever_rt_v1_async_net_connect(ptr, i64) +declare ptr @dever_rt_v1_async_net_connect_timeout(ptr, i64, i64) +declare ptr @dever_rt_v1_async_net_listen(ptr, i64) +declare ptr @dever_rt_v1_async_net_accept(ptr) +declare ptr @dever_rt_v1_async_net_read(ptr, i64) +declare ptr @dever_rt_v1_async_net_write(ptr, ptr) +declare ptr @dever_rt_v1_socket_retain(ptr) +declare void @dever_rt_v1_socket_release(ptr) +declare ptr @dever_rt_v1_listener_retain(ptr) +declare void @dever_rt_v1_listener_release(ptr) +declare i32 @dever_rt_v1_net_port(ptr, ptr, ptr) +declare i32 @dever_rt_v1_net_timeout(ptr, i64, ptr, ptr) +declare i32 @dever_rt_v1_socket_close(ptr, ptr, ptr) +declare i32 @dever_rt_v1_listener_close(ptr, ptr, ptr) +declare i32 @dever_rt_v1_net_chunks(ptr, i64, ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_net_connections(ptr, ptr, ptr, ptr, ptr) +declare ptr @dever_rt_v1_async_task_wait_timeout(ptr, i64) +declare ptr @dever_rt_v1_async_task_race(ptr, i64, ptr) +"; + +impl FunctionEmitter<'_, '_> { + pub(super) fn network_intrinsic( + &mut self, + operation: Intrinsic, + arguments: &[Expression], + mut values: Vec<String>, + result: &Expression, + ) -> String { + use Intrinsic::*; + if matches!(operation, StreamOf | TaskTicks) { + let Type::AsyncStream(element) = &result.ty else { + unreachable!("checked async stream") + }; + let symbol = match operation { + TaskTicks => "async_ticks", + StreamOf => match arguments[0].ty { + Type::List(_) => "async_stream_from_list", + Type::Bytes => "async_stream_from_bytes", + Type::Channel(_) => "async_channel_stream", + _ => unreachable!("checked stream source"), + }, + _ => unreachable!(), + }; + if !matches!(arguments[0].ty, Type::Channel(_)) { + values.push(format!( + "ptr @dever_owned_{}", + self.module.type_index(element) + )); + } + return self.runtime_call( + &format!("dever_rt_v1_{symbol}"), + values, + &result.ty, + false, + None, + result.span, + ); + } + if matches!(operation, TcpChunks | TcpConnections) { + let stream_type = if operation == TcpChunks { + let Type::Named(id) = result.ty else { + unreachable!() + }; + let Shape::Choice(variants) = &self.module.program.types[id].shape else { + unreachable!() + }; + variants + .iter() + .find(|variant| variant.name == "Streaming") + .expect("checked Streaming") + .fields[0] + .ty + .clone() + } else { + result.ty.clone() + }; + let Type::AsyncStream(element) = stream_type else { + unreachable!() + }; + let Type::Named(event) = element.as_ref() else { + unreachable!() + }; + let descriptor = self.module.read_event(*event); + values.push(format!("ptr {descriptor}")); + values.push(format!( + "ptr @dever_owned_{}", + self.module.type_index(&element) + )); + return if operation == TcpChunks { + self.recoverable_call("dever_rt_v1_net_chunks", values, result, "Streaming", None) + } else { + self.runtime_call( + "dever_rt_v1_net_connections", + values, + &result.ty, + false, + None, + result.span, + ) + }; + } + let (symbol, success, end, asynchronous) = match operation { + TcpConnect => ("async_net_connect", "Connected", None, true), + TcpConnectTimeout => ("async_net_connect_timeout", "Connected", None, true), + TcpListen => ("async_net_listen", "Listening", None, true), + TcpAccept => ("async_net_accept", "Connected", None, true), + TcpRead => ("async_net_read", "Read", Some("End"), true), + TcpWrite => ("async_net_write", "Done", None, true), + TcpPort => ("net_port", "Bound", None, false), + TcpTimeout => ("net_timeout", "Done", None, false), + TcpClose => ("socket_close", "Closed", None, false), + TcpCloseListener => ("listener_close", "Closed", None, false), + _ => unreachable!("checked network operation"), + }; + if !asynchronous { + return self.recoverable_call( + &format!("dever_rt_v1_{symbol}"), + values, + result, + success, + end, + ); + } + let operation = self.temp(); + self.line(format!( + "{operation} = call ptr @dever_rt_v1_{symbol}({})", + values.join(", ") + )); + self.async_resource_result(&operation, result, success, end) + } + + pub(super) fn async_resource_result( + &mut self, + operation: &str, + result: &Expression, + success: &str, + end: Option<&str>, + ) -> String { + let Type::Named(id) = result.ty else { + unreachable!("checked resource result") + }; + let Shape::Choice(variants) = &self.module.program.types[id].shape else { + unreachable!() + }; + let mut row = variants + .iter() + .find(|variant| variant.name == success) + .expect("checked success") + .fields + .first() + .map_or(Type::Bool, |field| field.ty.clone()); + let protocol_output = matches!(&row, Type::Named(id) if matches!(self.module.program.types[*id].name.as_str(), "dever.http.Response" | "dever.http.StreamResponse" | "dever.websocket.Message")); + if protocol_output { + row = Type::Text; + } + let (status, output, present, error) = self.poll_raw(operation, &row); + let typed_fault = self.temp(); + self.line(format!("{typed_fault} = icmp eq i32 {status}, 2")); + let fault = self.label("resource_typed_fault"); + let ready = self.label("resource_polled"); + self.line(format!( + "br i1 {typed_fault}, label %{fault}, label %{ready}" + )); + self.start(&fault); + self.append_frame(result.span); + self.exit("1"); + self.start(&ready); + // Poll and synchronous ABI statuses have different numeric contracts. + let okay = self.temp(); + self.line(format!("{okay} = icmp eq i32 {status}, 1")); + let failed = self.temp(); + self.line(format!("{failed} = icmp eq i32 {status}, 3")); + let failure_status = self.temp(); + self.line(format!( + "{failure_status} = select i1 {failed}, i32 1, i32 2" + )); + let status = self.temp(); + self.line(format!( + "{status} = select i1 {okay}, i32 0, i32 {failure_status}" + )); + self.resource_row( + result, + success, + end, + ResourceRow { + status, + output, + present: end.map(|_| present), + error, + protocol_output, + }, + ) + } +} diff --git a/crates/dever-core/src/llvm/orm.rs b/crates/dever-core/src/llvm/orm.rs new file mode 100644 index 0000000..b82626a --- /dev/null +++ b/crates/dever-core/src/llvm/orm.rs @@ -0,0 +1,512 @@ +//! Checked Model operations lower to static SQL and concrete parameter/row code. +use super::*; +use crate::hir::{ModelOperation, QueryPlan}; +use crate::model::{ModelFieldType, ModelSchema, ModelSqlCardinality}; +use crate::native::orm as sql; + +pub(super) type DatabaseValues = Vec<(Type, ModelFieldType, String)>; +type ModelWrite = [(usize, Expression)]; + +impl FunctionEmitter<'_, '_> { + pub(super) fn model_operation( + &mut self, + index: usize, + operation: &ModelOperation, + result: &Expression, + ) -> Result<String, String> { + let model = self.module.program.models[index].clone(); + match operation { + ModelOperation::Create { values } => { + self.model_insert(index, &model, values, None, result) + } + ModelOperation::Upsert { + key, + create, + update, + } => { + let insert = key.iter().chain(create).cloned().collect::<Vec<_>>(); + self.model_insert(index, &model, &insert, Some((key, update)), result) + } + ModelOperation::CreateMany { rows } => { + self.model_insert_many(index, &model, rows, result) + } + ModelOperation::Get { id } => { + let value = self.expression(id)?; + let values = vec![(id.ty.clone(), model.fields[0].ty.clone(), value)]; + let query = self.module.planned_sql(&format!( + "SELECT {} FROM {} WHERE {} = ?1 LIMIT 1", + sql::select_columns(&model), + sql::quoted(&model.table), + sql::quoted("id") + )); + let rows = self.model_query(index, &query, &values, "0", result.span); + Ok(self.model_first( + &rows, + &model, + None, + Some((7, format!("{} was not found", model.package))), + result.span, + )) + } + ModelOperation::Update { query, values } => { + let mut expressions = values + .iter() + .map(|(field, value)| (value, &model.fields[*field].ty)) + .collect::<Vec<_>>(); + let assignments = + sql::update_assignments(&model, values.iter().map(|(field, _)| *field), 1); + let condition = + sql::condition_sql(&model, query.condition.as_ref(), &mut expressions); + let values = self.model_bind(&expressions)?; + let statement = self.module.planned_sql(&format!( + "UPDATE {} SET {assignments}{condition}", + sql::quoted(&model.table) + )); + Ok(self.model_execute(index, &statement, &values, result.span)) + } + ModelOperation::Delete { query } => { + let mut expressions = Vec::new(); + let condition = + sql::condition_sql(&model, query.condition.as_ref(), &mut expressions); + let values = self.model_bind(&expressions)?; + let statement = self.module.planned_sql(&format!( + "DELETE FROM {}{condition}", + sql::quoted(&model.table) + )); + Ok(self.model_execute(index, &statement, &values, result.span)) + } + ModelOperation::Count { query } | ModelOperation::Exists { query } => { + let mut expressions = Vec::new(); + let condition = + sql::condition_sql(&model, query.condition.as_ref(), &mut expressions); + let values = self.model_bind(&expressions)?; + let statement = if matches!(operation, ModelOperation::Count { .. }) { + format!( + "SELECT COUNT(*) FROM {}{condition}", + sql::quoted(&model.table) + ) + } else { + format!( + "SELECT EXISTS(SELECT 1 FROM {}{condition})", + sql::quoted(&model.table) + ) + }; + let statement = self.module.planned_sql(&statement); + let rows = self.model_query(index, &statement, &values, "0", result.span); + Ok(self.model_scalar_rows(&rows, &result.ty, result.span)) + } + ModelOperation::Sql { + operation, + arguments, + .. + } => { + let operation = &model.sql[*operation]; + let expressions = arguments + .iter() + .zip(&operation.parameters) + .map(|(argument, parameter)| (argument, ¶meter.ty)) + .collect::<Vec<_>>(); + let values = self.model_bind(&expressions)?; + let statement = self + .module + .database_sql(&operation.sqlite, &operation.postgres); + let maximum = if operation.result.cardinality == ModelSqlCardinality::Many { + self.model_maximum(index, result.span) + } else { + "1".into() + }; + let rows = self.model_query(index, &statement, &values, &maximum, result.span); + if operation.result.cardinality == ModelSqlCardinality::Many { + return Ok(self.model_rows_list( + index, + &rows, + operation.result.record, + None, + None, + result.span, + )); + } + let name = self + .module + .wire_names(&[format!("{}.{}", model.package, operation.name)]); + let nullable = operation.result.cardinality == ModelSqlCardinality::Optional; + let slot = self.entry_slot_ir("ptr"); + let present = self.entry_slot_ir("i8"); + let error = self.entry_slot_ir("{ ptr, i64 }"); + let errors = self.database_errors(result.span); + let status = self.temp(); + let symbol = if nullable { "optional" } else { "required" }; + let extra = if nullable { + format!(", ptr {present}") + } else { + String::new() + }; + self.line(format!("{status} = call i32 @dever_rt_v1_db_{symbol}_row(ptr {rows}, ptr {name}, ptr {slot}{extra}, ptr {errors}, ptr %fault, ptr {error})")); + self.database_status(&status, &error, result.span); + let output = self.entry_slot(&result.ty); + self.line(format!( + "store {} zeroinitializer, ptr {output}", + self.module.ty(&result.ty) + )); + let done = self.label("database_sql_done"); + if nullable { + let found = self.temp(); + self.line(format!("{found} = load i8, ptr {present}")); + let has_row = self.temp(); + self.line(format!("{has_row} = icmp ne i8 {found}, 0")); + let decode = self.label("database_sql_decode"); + self.line(format!("br i1 {has_row}, label %{decode}, label %{done}")); + self.start(&decode); + } + let row = self.temp(); + self.line(format!("{row} = load ptr, ptr {slot}")); + let owner = self.protocol_owner("db_row", &row); + let value = + self.database_record(&row, operation.result.record, 0, true, result.span); + let value = if nullable { + self.model_some(&Type::Named(operation.result.record), &value) + } else { + value + }; + self.line(format!( + "store {} {value}, ptr {output}", + self.module.ty(&result.ty) + )); + self.release_guard(owner); + self.line(format!("br label %{done}")); + self.start(&done); + let value = self.temp(); + self.line(format!( + "{value} = load {}, ptr {output}", + self.module.ty(&result.ty) + )); + Ok(value) + } + ModelOperation::First { query } => { + let (statement, values) = self.model_select(&model, query, " LIMIT 1")?; + let rows = self.model_query(index, &statement, &values, "0", result.span); + Ok(self.model_first(&rows, &model, Some(query), None, result.span)) + } + ModelOperation::List { query } => self.model_page(index, &model, query, result), + ModelOperation::Cursor { query } => self.model_cursor(index, &model, query, result), + ModelOperation::Stream { query } => self.model_stream(index, &model, query, result), + } + } + + pub(super) fn model_bind( + &mut self, + expressions: &[(&Expression, &ModelFieldType)], + ) -> Result<DatabaseValues, String> { + expressions + .iter() + .map(|(expression, storage)| { + Ok(( + expression.ty.clone(), + (*storage).clone(), + self.expression(expression)?, + )) + }) + .collect() + } + + pub(super) fn model_maximum(&mut self, model: usize, span: Span) -> String { + let database = self.model_database(model); + self.protocol_call( + "dever_rt_v1_db_max_page_size", + vec![format!("ptr {database}")], + "i64", + span, + ) + } + + pub(super) fn model_query( + &mut self, + model: usize, + query: &str, + values: &[(Type, ModelFieldType, String)], + maximum: &str, + span: Span, + ) -> String { + let database = self.model_database(model); + let parameters = self.database_parameters(values, span); + let operation = self.database_operation( + "db_query", + vec![ + format!("ptr {database}"), + format!("ptr {}", self.database_context), + format!("ptr {query}"), + format!("ptr {parameters}"), + format!("i64 {}", values.len()), + format!("i64 {maximum}"), + ], + span, + ); + let rows = self.await_database_handle(&operation, span); + self.protocol_owner("db_rows", &rows); + rows + } + + fn model_execute( + &mut self, + model: usize, + query: &str, + values: &[(Type, ModelFieldType, String)], + span: Span, + ) -> String { + let database = self.model_database(model); + let parameters = self.database_parameters(values, span); + let operation = self.database_operation( + "db_execute", + vec![ + format!("ptr {database}"), + format!("ptr {}", self.database_context), + format!("ptr {query}"), + format!("ptr {parameters}"), + format!("i64 {}", values.len()), + ], + span, + ); + self.await_operation(&operation, &Type::Int, span, false) + } + + fn model_insert( + &mut self, + index: usize, + model: &ModelSchema, + insert: &[(usize, Expression)], + upsert: Option<(&ModelWrite, &ModelWrite)>, + result: &Expression, + ) -> Result<String, String> { + let mut expressions = insert + .iter() + .map(|(field, expression)| (expression, &model.fields[*field].ty)) + .collect::<Vec<_>>(); + if let Some((_, update)) = upsert { + expressions.extend( + update + .iter() + .map(|(field, expression)| (expression, &model.fields[*field].ty)), + ); + } + let mut values = self.model_bind(&expressions)?; + let mut columns = insert + .iter() + .map(|(field, _)| sql::quoted(&model.fields[*field].name)) + .collect::<Vec<_>>(); + let generated = model + .fields + .iter() + .filter(|field| field.generated && field.ty == ModelFieldType::Uuid) + .collect::<Vec<_>>(); + for (offset, field) in generated.iter().enumerate() { + let uuid = self.database_call("db_uuid_new", vec![], "ptr", result.span); + self.own_value(&Type::Uuid, &uuid, false); + values.insert( + insert.len() + offset, + (Type::Uuid, ModelFieldType::Uuid, uuid), + ); + columns.push(sql::quoted(&field.name)); + } + let insert_count = insert.len() + generated.len(); + let placeholders = values[..insert_count] + .iter() + .enumerate() + .map(|(index, (_, storage, _))| sql::parameter_sql(index + 1, storage)) + .collect::<Vec<_>>() + .join(", "); + let body = if columns.is_empty() { + "DEFAULT VALUES".into() + } else { + format!("({}) VALUES ({placeholders})", columns.join(", ")) + }; + let conflict = if let Some((key, update)) = upsert { + let keys = key + .iter() + .map(|(field, _)| sql::quoted(&model.fields[*field].name)) + .collect::<Vec<_>>() + .join(", "); + let assignments = sql::update_assignments( + model, + update.iter().map(|(field, _)| *field), + insert_count + 1, + ); + format!(" ON CONFLICT ({keys}) DO UPDATE SET {assignments}") + } else { + String::new() + }; + let query = self.module.planned_sql(&format!( + "INSERT INTO {} {body}{conflict} RETURNING {}", + sql::quoted(&model.table), + sql::select_columns(model) + )); + let rows = self.model_query(index, &query, &values, "0", result.span); + let message = if upsert.is_some() { + "database UPSERT RETURNING produced no row" + } else { + "database INSERT RETURNING produced no row" + }; + Ok(self.model_first(&rows, model, None, Some((8, message.into())), result.span)) + } + + fn model_insert_many( + &mut self, + index: usize, + model: &ModelSchema, + rows: &[Vec<(usize, Expression)>], + result: &Expression, + ) -> Result<String, String> { + let expressions = rows + .iter() + .flatten() + .map(|(field, value)| (value, &model.fields[*field].ty)) + .collect::<Vec<_>>(); + let mut values = self.model_bind(&expressions)?; + let generated = model + .fields + .iter() + .filter(|field| field.generated && field.ty == ModelFieldType::Uuid) + .collect::<Vec<_>>(); + for _ in 0..rows.len() * generated.len() { + let uuid = self.database_call("db_uuid_new", vec![], "ptr", result.span); + self.own_value(&Type::Uuid, &uuid, false); + values.push((Type::Uuid, ModelFieldType::Uuid, uuid)); + } + let columns = rows[0] + .iter() + .map(|(field, _)| sql::quoted(&model.fields[*field].name)) + .chain(generated.iter().map(|field| sql::quoted(&field.name))) + .collect::<Vec<_>>(); + let table = sql::quoted(&model.table); + let (sqlite, postgres) = if columns.is_empty() { + ( + format!( + "INSERT INTO {table} ({}) VALUES {}", + sql::quoted("id"), + vec!["(NULL)"; rows.len()].join(", ") + ), + format!( + "INSERT INTO {table} ({}) VALUES {}", + sql::quoted("created_at"), + vec!["(DEFAULT)"; rows.len()].join(", ") + ), + ) + } else { + let explicit = expressions.len(); + let groups = rows + .iter() + .enumerate() + .map(|(row, fields)| { + let placeholders = fields + .iter() + .enumerate() + .map(|(column, (field, _))| { + sql::parameter_sql( + row * fields.len() + column + 1, + &model.fields[*field].ty, + ) + }) + .chain(generated.iter().enumerate().map(|(column, field)| { + sql::parameter_sql( + explicit + row * generated.len() + column + 1, + &field.ty, + ) + })) + .collect::<Vec<_>>() + .join(", "); + format!("({placeholders})") + }) + .collect::<Vec<_>>() + .join(", "); + let query = format!( + "INSERT INTO {table} ({}) VALUES {groups}", + columns.join(", ") + ); + (query.clone(), query) + }; + let (sqlite, postgres) = sql::sql_dialects(&sqlite, &postgres); + let query = self.module.database_sql(&sqlite, &postgres); + Ok(self.model_execute(index, &query, &values, result.span)) + } + + pub(super) fn model_select( + &mut self, + model: &ModelSchema, + query: &QueryPlan, + suffix: &str, + ) -> Result<(String, DatabaseValues), String> { + let mut expressions = Vec::new(); + let alias = sql::has_to_one(model, query).then_some("_dever_parent"); + let condition = + sql::condition_sql_qualified(model, query.condition.as_ref(), &mut expressions, alias); + let (projection, source) = sql::select_source(model, query, &self.module.program.models); + let order = sql::order_sql_qualified(model, &query.order, alias); + let statement = self.module.planned_sql(&format!( + "SELECT {projection} FROM {source}{condition} ORDER BY {order}{suffix}" + )); + Ok((statement, self.model_bind(&expressions)?)) + } + + pub(super) fn model_some(&mut self, ty: &Type, value: &str) -> String { + let optional = self.module.ty(&Type::Nullable(Box::new(ty.clone()))); + let present = self.temp(); + self.line(format!( + "{present} = insertvalue {optional} zeroinitializer, i1 1, 0" + )); + let wrapped = self.temp(); + self.line(format!( + "{wrapped} = insertvalue {optional} {present}, {} {value}, 1", + self.module.ty(ty) + )); + wrapped + } + + pub(super) fn model_scalar_rows(&mut self, rows: &str, ty: &Type, span: Span) -> String { + let length = self.protocol_call( + "dever_rt_v1_db_rows_len", + vec![format!("ptr {rows}")], + "i64", + span, + ); + self.model_require_nonempty(&length, 8, "database scalar query returned no row", span); + let row = self.protocol_call( + "dever_rt_v1_db_rows_at", + vec![format!("ptr {rows}"), "i64 0".into()], + "ptr", + span, + ); + let owner = self.protocol_owner("db_row", &row); + let actual = self.protocol_call( + "dever_rt_v1_db_row_len", + vec![format!("ptr {row}")], + "i64", + span, + ); + self.database_column_present(&actual, 0, "scalar", span); + let value = self.database_scalar(&row, 0, ty, span); + self.database_row_length( + &row, + 1, + "database scalar query returned more than one column", + span, + ); + self.release_guard(owner); + value + } + + pub(super) fn model_require_nonempty( + &mut self, + length: &str, + kind: u32, + message: &str, + span: Span, + ) { + let present = self.temp(); + self.line(format!("{present} = icmp sgt i64 {length}, 0")); + let found = self.label("database_rows_present"); + let missing = self.label("database_rows_empty"); + self.line(format!("br i1 {present}, label %{found}, label %{missing}")); + self.start(&missing); + self.database_fault(kind, message, span); + self.start(&found); + } +} diff --git a/crates/dever-core/src/llvm/orm_read.rs b/crates/dever-core/src/llvm/orm_read.rs new file mode 100644 index 0000000..7f61d56 --- /dev/null +++ b/crates/dever-core/src/llvm/orm_read.rs @@ -0,0 +1,534 @@ +//! Query result assembly, pagination, and association loading use concrete rows. +use super::*; +use crate::hir::QueryPlan; +use crate::model::{ModelFieldType, ModelRelationKind, ModelSchema}; +use crate::native::orm as sql; + +impl FunctionEmitter<'_, '_> { + pub(super) fn model_first( + &mut self, + rows: &str, + model: &ModelSchema, + query: Option<&QueryPlan>, + required: Option<(u32, String)>, + span: Span, + ) -> String { + let index = self + .module + .program + .models + .iter() + .position(|candidate| candidate.record == model.record) + .unwrap(); + let list = self.model_rows_list(index, rows, model.record, query, None, span); + let ty = Type::Nullable(Box::new(Type::Named(model.record))); + let value = self.runtime_call( + "dever_rt_v1_list_first_take", + vec![format!("ptr {list}")], + &ty, + true, + None, + span, + ); + let Some((kind, message)) = required else { + return value; + }; + let present = self.temp(); + self.line(format!( + "{present} = extractvalue {} {value}, 0", + self.module.ty(&ty) + )); + let found = self.label("database_first_found"); + let missing = self.label("database_first_missing"); + self.line(format!("br i1 {present}, label %{found}, label %{missing}")); + self.start(&missing); + self.database_fault(kind, &message, span); + self.start(&found); + let record = self.temp(); + self.line(format!( + "{record} = extractvalue {} {value}, 1", + self.module.ty(&ty) + )); + record + } + + pub(super) fn model_rows_list( + &mut self, + model: usize, + rows: &str, + record: usize, + query: Option<&QueryPlan>, + limit: Option<&str>, + span: Span, + ) -> String { + let ty = Type::List(Box::new(Type::Named(record))); + let list = self.runtime_call( + "dever_rt_v1_list_new", + vec![ + format!( + "ptr @dever_type_{}", + self.module.type_index(&Type::Named(record)) + ), + "ptr null".into(), + "i64 0".into(), + ], + &ty, + false, + None, + span, + ); + let owner = self.own_value(&ty, &list, false); + let list_slot = self.guards[owner].pointer.clone(); + let length = self.protocol_call( + "dever_rt_v1_db_rows_len", + vec![format!("ptr {rows}")], + "i64", + span, + ); + let index_slot = self.entry_slot(&Type::Int); + self.line(format!("store i64 0, ptr {index_slot}")); + let check = self.label("database_rows_check"); + let body = self.label("database_rows_decode"); + let done = self.label("database_rows_done"); + self.line(format!("br label %{check}")); + self.start(&check); + let index = self.temp(); + self.line(format!("{index} = load i64, ptr {index_slot}")); + let more = self.temp(); + self.line(format!("{more} = icmp slt i64 {index}, {length}")); + self.line(format!("br i1 {more}, label %{body}, label %{done}")); + self.start(&body); + let first = self.guards.len(); + let row = self.protocol_call( + "dever_rt_v1_db_rows_at", + vec![format!("ptr {rows}"), format!("i64 {index}")], + "ptr", + span, + ); + self.protocol_owner("db_row", &row); + let value = if let Some(query) = query { + self.model_included_row(&row, model, query, span) + } else { + self.database_record(&row, record, 0, true, span) + }; + self.own_value(&Type::Named(record), &value, false); + let advance = self.label("database_rows_advance"); + if let Some(limit) = limit { + let keep = self.temp(); + self.line(format!("{keep} = icmp slt i64 {index}, {limit}")); + let append = self.label("database_rows_append"); + self.line(format!("br i1 {keep}, label %{append}, label %{advance}")); + self.start(&append); + } + let pointer = self.row_pointer(&Type::Named(record), &value); + let list = self.temp(); + self.line(format!("{list} = load ptr, ptr {list_slot}")); + self.disarm(owner); + let appended = self.runtime_call( + "dever_rt_v1_list_append_take", + vec![format!("ptr {list}"), format!("ptr {pointer}")], + &ty, + false, + None, + span, + ); + self.line(format!("store ptr {appended}, ptr {list_slot}")); + self.mark_live(owner); + self.line(format!("br label %{advance}")); + self.start(&advance); + self.release_from(first); + let next = self.temp(); + self.line(format!("{next} = add i64 {index}, 1")); + self.line(format!("store i64 {next}, ptr {index_slot}")); + self.line(format!("br label %{check}")); + self.start(&done); + if let Some(query) = query { + self.model_load_many(model, query, &list_slot, span); + } + let value = self.temp(); + self.line(format!("{value} = load ptr, ptr {list_slot}")); + self.disarm(owner); + value + } + + fn model_included_row( + &mut self, + row: &str, + index: usize, + query: &QueryPlan, + span: Span, + ) -> String { + let model = self.module.program.models[index].clone(); + let mut value = self.database_record(row, model.record, 0, false, span); + let owner = self.own_value(&Type::Named(model.record), &value, false); + let record_slot = self.guards[owner].pointer.clone(); + let mut offset = model.fields.len(); + let Shape::Record(fields) = self.module.program.types[model.record].shape.clone() else { + unreachable!() + }; + for relation_index in &query.relations { + let relation = &model.relations[*relation_index]; + if relation.kind != ModelRelationKind::ToOne { + continue; + } + let target = self + .module + .program + .models + .iter() + .find(|target| target.package == relation.model) + .unwrap() + .clone(); + let field = model.fields.len() + relation_index; + let Type::Related(inner) = &fields[field].ty else { + unreachable!() + }; + let payload = self.entry_slot(inner); + self.line(format!( + "store {} zeroinitializer, ptr {payload}", + self.module.ty(inner) + )); + let done = self.label("database_relation_decoded"); + if matches!(inner.as_ref(), Type::Nullable(_)) { + let actual = self.protocol_call( + "dever_rt_v1_db_row_len", + vec![format!("ptr {row}")], + "i64", + span, + ); + self.database_column_present( + &actual, + offset, + &format!("{}.{}", target.package, target.fields[0].name), + span, + ); + let absent = self.database_call( + "db_row_null", + vec![format!("ptr {row}"), format!("i64 {offset}")], + "i8", + span, + ); + let absent_bool = self.temp(); + self.line(format!("{absent_bool} = icmp ne i8 {absent}, 0")); + let decode = self.label("database_relation_decode"); + let null = self.label("database_relation_null"); + self.line(format!( + "br i1 {absent_bool}, label %{null}, label %{decode}" + )); + self.start(&null); + self.database_column_present( + &actual, + offset + target.fields.len() - 1, + &format!("{}.{}", target.package, relation.name), + span, + ); + self.line(format!("br label %{done}")); + self.start(&decode); + } + let decoded = self.database_record(row, target.record, offset, false, span); + let decoded = if matches!(inner.as_ref(), Type::Nullable(_)) { + self.model_some(&Type::Named(target.record), &decoded) + } else { + decoded + }; + self.line(format!( + "store {} {decoded}, ptr {payload}", + self.module.ty(inner) + )); + self.line(format!("br label %{done}")); + self.start(&done); + let temporary = self.register_guard(inner, payload.clone()); + self.mark_live(temporary); + let related = self.protocol_call( + "dever_rt_v1_db_related_new", + vec![ + format!("ptr @dever_type_{}", self.module.type_index(inner)), + format!("ptr {payload}"), + ], + "ptr", + span, + ); + let destination = self.temp(); + self.line(format!( + "{destination} = getelementptr %T{}, ptr {record_slot}, i32 0, i32 {field}", + model.record + )); + self.line(format!("store ptr {related}, ptr {destination}")); + self.release_guard(temporary); + offset += target.fields.len(); + } + self.database_row_length( + row, + offset, + &format!("database row for {} has extra columns", model.package), + span, + ); + value = self.temp(); + self.line(format!( + "{value} = load %T{}, ptr {record_slot}", + model.record + )); + self.disarm(owner); + value + } + + pub(super) fn model_page( + &mut self, + index: usize, + model: &ModelSchema, + query: &QueryPlan, + result: &Expression, + ) -> Result<String, String> { + let mut expressions = Vec::new(); + let condition = sql::condition_sql(model, query.condition.as_ref(), &mut expressions); + let mut values = self.model_bind(&expressions)?; + let page = self.expression(&query.page)?; + let size = self.expression(&query.size)?; + let maximum = self.model_maximum(index, result.span); + let page_slot = self.entry_slot(&Type::Int); + let size_slot = self.entry_slot(&Type::Int); + let offset_slot = self.entry_slot(&Type::Int); + let error = self.entry_slot_ir("{ ptr, i64 }"); + let errors = self.database_errors(result.span); + let status = self.temp(); + self.line(format!("{status} = call i32 @dever_rt_v1_db_pagination(i64 {page}, i64 {size}, i64 {maximum}, ptr {page_slot}, ptr {size_slot}, ptr {offset_slot}, ptr {errors}, ptr %fault, ptr {error})")); + self.database_status(&status, &error, result.span); + let offset = self.temp(); + self.line(format!("{offset} = load i64, ptr {offset_slot}")); + let count_sql = self.module.planned_sql(&format!( + "SELECT COUNT(*) FROM {}{condition}", + sql::quoted(&model.table) + )); + let count_rows = self.model_query(index, &count_sql, &values, "0", result.span); + let total = self.model_scalar_rows(&count_rows, &Type::Int, result.span); + let alias = sql::has_to_one(model, query).then_some("_dever_parent"); + let mut ignored = Vec::new(); + let condition = + sql::condition_sql_qualified(model, query.condition.as_ref(), &mut ignored, alias); + let (projection, source) = sql::select_source(model, query, &self.module.program.models); + let order = sql::order_sql_qualified(model, &query.order, alias); + let statement = self.module.planned_sql(&format!( + "SELECT {projection} FROM {source}{condition} ORDER BY {order} LIMIT ?{} OFFSET ?{}", + values.len() + 1, + values.len() + 2 + )); + values.extend([ + (Type::Int, ModelFieldType::Int, size.clone()), + (Type::Int, ModelFieldType::Int, offset), + ]); + let rows = self.model_query(index, &statement, &values, "0", result.span); + let items = + self.model_rows_list(index, &rows, model.record, Some(query), None, result.span); + let quotient = self.temp(); + let remainder = self.temp(); + let extra = self.temp(); + let extra_integer = self.temp(); + let pages = self.temp(); + self.line(format!("{quotient} = sdiv i64 {total}, {size}")); + self.line(format!("{remainder} = srem i64 {total}, {size}")); + self.line(format!("{extra} = icmp ne i64 {remainder}, 0")); + self.line(format!("{extra_integer} = zext i1 {extra} to i64")); + self.line(format!("{pages} = add i64 {quotient}, {extra_integer}")); + Ok(self.model_aggregate( + model.page, + &[ + ("ptr", items), + ("i64", page), + ("i64", size), + ("i64", total), + ("i64", pages), + ], + )) + } + + pub(super) fn model_cursor( + &mut self, + index: usize, + model: &ModelSchema, + query: &QueryPlan, + result: &Expression, + ) -> Result<String, String> { + let mut expressions = Vec::new(); + let alias = sql::has_to_one(model, query).then_some("_dever_parent"); + let mut condition = + sql::condition_sql_qualified(model, query.condition.as_ref(), &mut expressions, alias); + let mut values = self.model_bind(&expressions)?; + if let Some(after) = &query.after { + let cursor = sql::cursor_condition(model, &query.order, values.len() + 1, alias); + condition.push_str(if condition.is_empty() { + " WHERE " + } else { + " AND " + }); + condition.push_str(&cursor); + let after_value = self.expression(after)?; + let present = self.temp(); + self.line(format!( + "{present} = extractvalue {} {after_value}, 0", + self.module.ty(&after.ty) + )); + let Shape::Record(fields) = self.module.program.types[model.record].shape.clone() + else { + unreachable!() + }; + for order in &query.order { + let field = &fields[order.field]; + let ty = Type::Nullable(Box::new(field.ty.clone())); + let slot = self.entry_slot(&ty); + self.line(format!( + "store {} zeroinitializer, ptr {slot}", + self.module.ty(&ty) + )); + let some = self.label("database_cursor_after"); + let done = self.label("database_cursor_bound"); + self.line(format!("br i1 {present}, label %{some}, label %{done}")); + self.start(&some); + let record = self.temp(); + self.line(format!( + "{record} = extractvalue {} {after_value}, 1", + self.module.ty(&after.ty) + )); + let value = self.temp(); + self.line(format!( + "{value} = extractvalue %T{} {record}, {}", + model.record, order.field + )); + let wrapped = self.model_some(&field.ty, &value); + self.line(format!( + "store {} {wrapped}, ptr {slot}", + self.module.ty(&ty) + )); + self.line(format!("br label %{done}")); + self.start(&done); + let value = self.temp(); + self.line(format!( + "{value} = load {}, ptr {slot}", + self.module.ty(&ty) + )); + values.push((ty, model.fields[order.field].ty.clone(), value)); + } + } + let size = self.expression(&query.size)?; + let maximum = self.model_maximum(index, result.span); + let size = self.database_call( + "db_cursor_size", + vec![format!("i64 {size}"), format!("i64 {maximum}")], + "i64", + result.span, + ); + let limit = self.temp(); + self.line(format!("{limit} = add i64 {size}, 1")); + let (projection, source) = sql::select_source(model, query, &self.module.program.models); + let order = sql::order_sql_qualified(model, &query.order, alias); + let statement = self.module.planned_sql(&format!( + "SELECT {projection} FROM {source}{condition} ORDER BY {order} LIMIT ?{}", + values.len() + 1 + )); + values.push((Type::Int, ModelFieldType::Int, limit)); + let rows = self.model_query(index, &statement, &values, "0", result.span); + let length = self.protocol_call( + "dever_rt_v1_db_rows_len", + vec![format!("ptr {rows}")], + "i64", + result.span, + ); + let more = self.temp(); + self.line(format!("{more} = icmp sgt i64 {length}, {size}")); + let items = self.model_rows_list( + index, + &rows, + model.record, + Some(query), + Some(&size), + result.span, + ); + let owner = self.own_value( + &Type::List(Box::new(Type::Named(model.record))), + &items, + false, + ); + let next_ty = Type::Nullable(Box::new(Type::Named(model.record))); + let next = self.entry_slot(&next_ty); + self.line(format!( + "store {} zeroinitializer, ptr {next}", + self.module.ty(&next_ty) + )); + let some = self.label("database_cursor_next"); + let done = self.label("database_cursor_ready"); + self.line(format!("br i1 {more}, label %{some}, label %{done}")); + self.start(&some); + self.model_last(&items, model.record, &next, result.span); + self.line(format!("br label %{done}")); + self.start(&done); + let next_value = self.temp(); + self.line(format!( + "{next_value} = load {}, ptr {next}", + self.module.ty(&next_ty) + )); + self.disarm(owner); + let layout = self.module.ty(&next_ty); + Ok(self.model_aggregate( + model.cursor, + &[("ptr", items), (&layout, next_value), ("i1", more)], + )) + } + + fn model_last(&mut self, list: &str, record: usize, output: &str, span: Span) { + let retained = self.temp(); + self.line(format!( + "{retained} = call ptr @dever_rt_v1_list_retain(ptr {list})" + )); + let cursor = self.protocol_call( + "dever_rt_v1_list_cursor_take", + vec![format!("ptr {retained}")], + "ptr", + span, + ); + let slot = self.entry_slot_ir("ptr"); + self.line(format!("store ptr {cursor}, ptr {slot}")); + let owner = self.register_owned(OwnedKind::ListCursor, slot); + self.mark_live(owner); + let ty = Type::Nullable(Box::new(Type::Named(record))); + let last_owner = self.register_guard(&ty, output.into()); + self.mark_live(last_owner); + let check = self.label("database_cursor_last"); + let found = self.label("database_cursor_last_found"); + let done = self.label("database_cursor_last_done"); + self.line(format!("br label %{check}")); + self.start(&check); + let row = self.runtime_call( + "dever_rt_v1_list_cursor_next", + vec![format!("ptr {cursor}")], + &ty, + true, + None, + span, + ); + let present = self.temp(); + self.line(format!( + "{present} = extractvalue {} {row}, 0", + self.module.ty(&ty) + )); + self.line(format!("br i1 {present}, label %{found}, label %{done}")); + self.start(&found); + self.release_guard(last_owner); + self.line(format!("store {} {row}, ptr {output}", self.module.ty(&ty))); + self.mark_live(last_owner); + self.line(format!("br label %{check}")); + self.start(&done); + self.release_guard(owner); + self.disarm(last_owner); + } + + fn model_aggregate(&mut self, id: usize, fields: &[(&str, String)]) -> String { + let mut value = "zeroinitializer".to_string(); + for (index, (ty, field)) in fields.iter().enumerate() { + let next = self.temp(); + self.line(format!( + "{next} = insertvalue %T{id} {value}, {ty} {field}, {index}" + )); + value = next; + } + value + } +} diff --git a/crates/dever-core/src/llvm/orm_relations.rs b/crates/dever-core/src/llvm/orm_relations.rs new file mode 100644 index 0000000..81481c4 --- /dev/null +++ b/crates/dever-core/src/llvm/orm_relations.rs @@ -0,0 +1,470 @@ +//! To-many loading batches 998 parent IDs and groups concrete child rows. +use super::*; +use crate::hir::QueryPlan; +use crate::model::{ModelRelationKind, ModelSchema}; +use crate::native::orm as sql; + +struct RelationBatch<'a> { + ids: &'a str, + count: &'a str, + limit: &'a str, +} + +impl FunctionEmitter<'_, '_> { + fn database_empty_list(&mut self, record: usize, span: Span) -> String { + self.runtime_call( + "dever_rt_v1_list_new", + vec![ + format!( + "ptr @dever_type_{}", + self.module.type_index(&Type::Named(record)) + ), + "ptr null".into(), + "i64 0".into(), + ], + &Type::List(Box::new(Type::Named(record))), + false, + None, + span, + ) + } + + fn database_list_cursor(&mut self, list: &str, span: Span) -> (String, usize) { + let retained = self.temp(); + self.line(format!( + "{retained} = call ptr @dever_rt_v1_list_retain(ptr {list})" + )); + let cursor = self.protocol_call( + "dever_rt_v1_list_cursor_take", + vec![format!("ptr {retained}")], + "ptr", + span, + ); + let slot = self.entry_slot_ir("ptr"); + self.line(format!("store ptr {cursor}, ptr {slot}")); + let owner = self.register_owned(OwnedKind::ListCursor, slot); + self.mark_live(owner); + (cursor, owner) + } + + fn database_for_each( + &mut self, + list: &str, + record: usize, + span: Span, + mut apply: impl FnMut(&mut Self, &str), + ) { + let (cursor, owner) = self.database_list_cursor(list, span); + let check = self.label("relation_rows_next"); + let body = self.label("relation_rows_body"); + let done = self.label("relation_rows_done"); + self.line(format!("br label %{check}")); + self.start(&check); + let ty = Type::Nullable(Box::new(Type::Named(record))); + let value = self.runtime_call( + "dever_rt_v1_list_cursor_next", + vec![format!("ptr {cursor}")], + &ty, + true, + None, + span, + ); + let present = self.temp(); + self.line(format!( + "{present} = extractvalue {} {value}, 0", + self.module.ty(&ty) + )); + self.line(format!("br i1 {present}, label %{body}, label %{done}")); + self.start(&body); + let first = self.guards.len(); + let row = self.temp(); + self.line(format!( + "{row} = extractvalue {} {value}, 1", + self.module.ty(&ty) + )); + self.own_value(&Type::Named(record), &row, false); + apply(self, &row); + self.release_from(first); + self.line(format!("br label %{check}")); + self.start(&done); + self.release_guard(owner); + } + + fn database_append(&mut self, list_slot: &str, record: usize, row: &str, span: Span) { + let list = self.temp(); + self.line(format!("{list} = load ptr, ptr {list_slot}")); + let owner = self + .guards + .iter() + .position(|guard| guard.pointer == list_slot) + .expect("owned relation list slot"); + let pointer = self.row_pointer(&Type::Named(record), row); + self.disarm(owner); + let next = self.runtime_call( + "dever_rt_v1_list_append_take", + vec![format!("ptr {list}"), format!("ptr {pointer}")], + &Type::List(Box::new(Type::Named(record))), + false, + None, + span, + ); + self.line(format!("store ptr {next}, ptr {list_slot}")); + self.mark_live(owner); + } + + pub(super) fn model_load_many( + &mut self, + index: usize, + query: &QueryPlan, + list_slot: &str, + span: Span, + ) { + let model = self.module.program.models[index].clone(); + for relation_index in &query.relations { + let relation = &model.relations[*relation_index]; + if relation.kind != ModelRelationKind::ToMany { + continue; + } + let target_index = self + .module + .program + .models + .iter() + .position(|candidate| candidate.package == relation.model) + .unwrap(); + let target = self.module.program.models[target_index].clone(); + let foreign_key = target + .fields + .iter() + .position(|field| field.name == relation.field) + .unwrap(); + let field = model.fields.len() + relation_index; + let parents = self.temp(); + self.line(format!("{parents} = load ptr, ptr {list_slot}")); + let (cursor, cursor_owner) = self.database_list_cursor(&parents, span); + let output = self.database_empty_list(model.record, span); + let output_ty = Type::List(Box::new(Type::Named(model.record))); + let output_owner = self.own_value(&output_ty, &output, false); + let output_slot = self.guards[output_owner].pointer.clone(); + let maximum = self.model_maximum(index, span); + let limit = self.database_call( + "db_relation_limit", + vec![format!("i64 {maximum}")], + "i64", + span, + ); + let ids = self.entry_slot_ir("[998 x i64]"); + let count_slot = self.entry_slot(&Type::Int); + let done_slot = self.entry_slot_ir("i1"); + self.line(format!("store i1 0, ptr {done_slot}")); + let batch_start = self.label("relation_batch_start"); + let read = self.label("relation_batch_read"); + let add = self.label("relation_batch_add"); + let exhausted = self.label("relation_batch_exhausted"); + let query_block = self.label("relation_batch_query"); + let queried = self.label("relation_batch_queried"); + let finish = self.label("relation_batches_done"); + self.line(format!("br label %{batch_start}")); + self.start(&batch_start); + let batch_first = self.guards.len(); + let batch = self.database_empty_list(model.record, span); + let batch_owner = self.own_value(&output_ty, &batch, false); + let batch_slot = self.guards[batch_owner].pointer.clone(); + self.line(format!("store i64 0, ptr {count_slot}")); + self.line(format!("br label %{read}")); + self.start(&read); + let next_ty = Type::Nullable(Box::new(Type::Named(model.record))); + let parent = self.runtime_call( + "dever_rt_v1_list_cursor_next", + vec![format!("ptr {cursor}")], + &next_ty, + true, + None, + span, + ); + let present = self.temp(); + self.line(format!( + "{present} = extractvalue {} {parent}, 0", + self.module.ty(&next_ty) + )); + self.line(format!("br i1 {present}, label %{add}, label %{exhausted}")); + self.start(&add); + let parent_row = self.temp(); + self.line(format!( + "{parent_row} = extractvalue {} {parent}, 1", + self.module.ty(&next_ty) + )); + let parent_owner = self.own_value(&Type::Named(model.record), &parent_row, false); + let id = self.temp(); + self.line(format!( + "{id} = extractvalue %T{} {parent_row}, 0", + model.record + )); + let count = self.temp(); + self.line(format!("{count} = load i64, ptr {count_slot}")); + let id_slot = self.temp(); + self.line(format!( + "{id_slot} = getelementptr [998 x i64], ptr {ids}, i32 0, i64 {count}" + )); + self.line(format!("store i64 {id}, ptr {id_slot}")); + self.database_append(&batch_slot, model.record, &parent_row, span); + self.release_guard(parent_owner); + let next = self.temp(); + self.line(format!("{next} = add i64 {count}, 1")); + self.line(format!("store i64 {next}, ptr {count_slot}")); + let full = self.temp(); + self.line(format!("{full} = icmp eq i64 {next}, 998")); + self.line(format!("br i1 {full}, label %{query_block}, label %{read}")); + self.start(&exhausted); + self.line(format!("store i1 1, ptr {done_slot}")); + let count = self.temp(); + self.line(format!("{count} = load i64, ptr {count_slot}")); + let empty = self.temp(); + self.line(format!("{empty} = icmp eq i64 {count}, 0")); + self.line(format!( + "br i1 {empty}, label %{queried}, label %{query_block}" + )); + self.start(&query_block); + let count = self.temp(); + self.line(format!("{count} = load i64, ptr {count_slot}")); + let batch = RelationBatch { + ids: &ids, + count: &count, + limit: &limit, + }; + let rows = self.relation_query(index, &target, &relation.field, batch, span); + let groups = self.relation_group_rows(&rows, target_index, foreign_key, span); + let batch = self.temp(); + self.line(format!("{batch} = load ptr, ptr {batch_slot}")); + self.database_for_each(&batch, model.record, span, |emitter, parent| { + let id = emitter.temp(); + emitter.line(format!( + "{id} = extractvalue %T{} {parent}, 0", + model.record + )); + let children = emitter.relation_children(&groups, &id, target.record, span); + let child_ty = Type::List(Box::new(Type::Named(target.record))); + let child_owner = emitter.own_value(&child_ty, &children, false); + let pointer = emitter.row_pointer(&child_ty, &children); + let related = emitter.protocol_call( + "dever_rt_v1_db_related_new", + vec![ + format!("ptr @dever_type_{}", emitter.module.type_index(&child_ty)), + format!("ptr {pointer}"), + ], + "ptr", + span, + ); + emitter.protocol_owner("db_related", &related); + let updated = emitter.temp(); + emitter.line(format!( + "{updated} = insertvalue %T{} {parent}, ptr {related}, {field}", + model.record + )); + emitter.database_append(&output_slot, model.record, &updated, span); + emitter.release_guard(child_owner); + }); + self.line(format!("br label %{queried}")); + self.start(&queried); + self.release_from(batch_first); + let exhausted = self.temp(); + self.line(format!("{exhausted} = load i1, ptr {done_slot}")); + self.line(format!( + "br i1 {exhausted}, label %{finish}, label %{batch_start}" + )); + self.start(&finish); + self.release_guard(cursor_owner); + let rebuilt = self.temp(); + self.line(format!("{rebuilt} = load ptr, ptr {output_slot}")); + self.line(format!( + "call void @dever_rt_v1_list_release(ptr {parents})" + )); + self.line(format!("store ptr {rebuilt}, ptr {list_slot}")); + self.disarm(output_owner); + } + } + + fn relation_query( + &mut self, + model: usize, + target: &ModelSchema, + field: &str, + batch: RelationBatch<'_>, + span: Span, + ) -> String { + let RelationBatch { ids, count, limit } = batch; + let prefix = format!( + "SELECT {} FROM (SELECT {}, ROW_NUMBER() OVER (PARTITION BY {} ORDER BY {} ASC) AS {} FROM {} WHERE {} IN (", + sql::select_columns(target), + sql::select_columns(target), + sql::quoted(field), + sql::quoted("id"), + sql::quoted("_dever_rank"), + sql::quoted(&target.table), + sql::quoted(field) + ); + let before = format!( + ")) AS {} WHERE {} <= ", + sql::quoted("_dever_children"), + sql::quoted("_dever_rank") + ); + let suffix = format!( + " ORDER BY {} ASC, {} ASC", + sql::quoted(field), + sql::quoted("id") + ); + let names = [prefix, before, suffix].map(|name| self.module.wire_names(&[name])); + let database = self.model_database(model); + let operation = self.database_operation( + "db_relation_query", + vec![ + format!("ptr {database}"), + format!("ptr {}", self.database_context), + format!("ptr {}", names[0]), + format!("ptr {}", names[1]), + format!("ptr {}", names[2]), + format!("ptr {ids}"), + format!("i64 {count}"), + format!("i64 {limit}"), + ], + span, + ); + let rows = self.await_database_handle(&operation, span); + self.protocol_owner("db_rows", &rows); + rows + } + + fn relation_children(&mut self, groups: &str, id: &str, record: usize, span: Span) -> String { + let retained = self.temp(); + self.line(format!( + "{retained} = call ptr @dever_rt_v1_map_retain(ptr {groups})" + )); + let key = self.row_pointer(&Type::Int, id); + let list_ty = Type::List(Box::new(Type::Named(record))); + let optional_ty = Type::Nullable(Box::new(list_ty.clone())); + let found = self.runtime_call( + "dever_rt_v1_map_get_take", + vec![format!("ptr {retained}"), format!("ptr {key}")], + &optional_ty, + true, + None, + span, + ); + let present = self.temp(); + self.line(format!( + "{present} = extractvalue {} {found}, 0", + self.module.ty(&optional_ty) + )); + let slot = self.entry_slot(&list_ty); + let some = self.label("relation_group_found"); + let missing = self.label("relation_group_empty"); + let done = self.label("relation_group_ready"); + self.line(format!("br i1 {present}, label %{some}, label %{missing}")); + self.start(&some); + let list = self.temp(); + self.line(format!( + "{list} = extractvalue {} {found}, 1", + self.module.ty(&optional_ty) + )); + self.line(format!("store ptr {list}, ptr {slot}")); + self.line(format!("br label %{done}")); + self.start(&missing); + let empty = self.database_empty_list(record, span); + self.line(format!("store ptr {empty}, ptr {slot}")); + self.line(format!("br label %{done}")); + self.start(&done); + let list = self.temp(); + self.line(format!("{list} = load ptr, ptr {slot}")); + list + } + + fn relation_group_rows( + &mut self, + rows: &str, + model: usize, + foreign_key: usize, + span: Span, + ) -> String { + let target = self.module.program.models[model].clone(); + let list_ty = Type::List(Box::new(Type::Named(target.record))); + let map_ty = Type::Map(Box::new(Type::Int), Box::new(list_ty.clone())); + let map = self.runtime_call( + "dever_rt_v1_map_new", + vec![ + format!("ptr @dever_type_{}", self.module.type_index(&Type::Int)), + format!("ptr @dever_type_{}", self.module.type_index(&list_ty)), + "ptr null".into(), + "ptr null".into(), + "i64 0".into(), + ], + &map_ty, + false, + None, + span, + ); + let owner = self.own_value(&map_ty, &map, false); + let slot = self.guards[owner].pointer.clone(); + let list = self.model_rows_list(model, rows, target.record, None, None, span); + let list_owner = self.own_value(&list_ty, &list, false); + self.database_for_each(&list, target.record, span, |emitter, row| { + let id = emitter.temp(); + emitter.line(format!( + "{id} = extractvalue %T{} {row}, {foreign_key}", + target.record + )); + let done = emitter.label("relation_child_grouped"); + let id = if target.fields[foreign_key].nullable { + let present = emitter.temp(); + emitter.line(format!("{present} = extractvalue {{ i1, i64 }} {id}, 0")); + let group = emitter.label("relation_child_parent"); + emitter.line(format!("br i1 {present}, label %{group}, label %{done}")); + emitter.start(&group); + let value = emitter.temp(); + emitter.line(format!("{value} = extractvalue {{ i1, i64 }} {id}, 1")); + value + } else { + id + }; + let map = emitter.temp(); + emitter.line(format!("{map} = load ptr, ptr {slot}")); + let children = emitter.relation_children(&map, &id, target.record, span); + let children_owner = emitter.own_value(&list_ty, &children, false); + let child_slot = emitter.guards[children_owner].pointer.clone(); + let key = emitter.row_pointer(&Type::Int, &id); + emitter.disarm(owner); + let map = emitter.runtime_call( + "dever_rt_v1_map_remove_take", + vec![format!("ptr {map}"), format!("ptr {key}")], + &map_ty, + false, + None, + span, + ); + emitter.line(format!("store ptr {map}, ptr {slot}")); + emitter.mark_live(owner); + emitter.database_append(&child_slot, target.record, row, span); + emitter.disarm(owner); + let changed = emitter.runtime_call( + "dever_rt_v1_map_put_take", + vec![ + format!("ptr {map}"), + format!("ptr {key}"), + format!("ptr {child_slot}"), + ], + &map_ty, + false, + None, + span, + ); + emitter.line(format!("store ptr {changed}, ptr {slot}")); + emitter.mark_live(owner); + emitter.release_guard(children_owner); + emitter.line(format!("br label %{done}")); + emitter.start(&done); + }); + self.release_guard(list_owner); + let groups = self.temp(); + self.line(format!("{groups} = load ptr, ptr {slot}")); + groups + } +} diff --git a/crates/dever-core/src/llvm/orm_stream.rs b/crates/dever-core/src/llvm/orm_stream.rs new file mode 100644 index 0000000..f02215a --- /dev/null +++ b/crates/dever-core/src/llvm/orm_stream.rs @@ -0,0 +1,89 @@ +//! RowStream pulls decode at the consumer's error location and own no borrowed row. +use super::*; +use crate::hir::QueryPlan; +use crate::model::ModelSchema; +use crate::native::orm as sql; + +impl Module<'_> { + pub(super) fn database_row_decoder( + &mut self, + record: usize, + instance: &Specialization, + ) -> String { + let symbol = format!("@dever_db_decoder_{record}"); + if !self.database_decoders.insert(record) { + return symbol; + } + let index = self.type_index(&Type::Named(record)); + writeln!(self.declarations, "{symbol} = private constant {{ ptr, ptr }} {{ ptr @dever_owned_{index}, ptr @dever_db_decode_{record} }}").unwrap(); + let span = self.program.types[record].span; + let mut emitter = FunctionEmitter::new(self, instance); + emitter.asynchronous = false; + emitter.database_errors_override = Some("%errors".into()); + emitter.start("entry"); + emitter + .prologue + .push_str(" %exit_status = alloca i32\n store i32 0, ptr %exit_status\n"); + let value = emitter.database_record("%row", record, 0, true, span); + emitter.line(format!("store %T{record} {value}, ptr %out")); + emitter.exit("0"); + emitter.start("cleanup"); + emitter.release_from(0); + let status = emitter.temp(); + emitter.line(format!("{status} = load i32, ptr %exit_status")); + emitter.line(format!("ret i32 {status}")); + let body = format!( + "define internal i32 @dever_db_decode_{record}(ptr %row, ptr %out, ptr %errors, ptr %fault, ptr %error) {{\nentry:\n{}{} }}\n", + emitter.prologue, emitter.body + ); + self.declarations.push_str(&body); + symbol + } +} + +impl FunctionEmitter<'_, '_> { + pub(super) fn model_stream( + &mut self, + index: usize, + model: &ModelSchema, + query: &QueryPlan, + result: &Expression, + ) -> Result<String, String> { + let mut expressions = Vec::new(); + let condition = sql::condition_sql(model, query.condition.as_ref(), &mut expressions); + let values = self.model_bind(&expressions)?; + let capacity = self.expression(&query.size)?; + let maximum = self.model_maximum(index, result.span); + let capacity = self.database_call( + "db_stream_capacity", + vec![format!("i64 {capacity}"), format!("i64 {maximum}")], + "i64", + result.span, + ); + let statement = self.module.planned_sql(&format!( + "SELECT {} FROM {}{condition} ORDER BY {}", + sql::select_columns(model), + sql::quoted(&model.table), + sql::order_sql(model, &query.order) + )); + let parameters = self.database_parameters(&values, result.span); + let database = self.model_database(index); + let decoder = self + .module + .database_row_decoder(model.record, self.instance); + let operation = self.database_operation( + "db_stream", + vec![ + format!("ptr {database}"), + format!("ptr {}", self.database_context), + format!("ptr {statement}"), + format!("ptr {parameters}"), + format!("i64 {}", values.len()), + format!("i64 {capacity}"), + format!("ptr {decoder}"), + ], + result.span, + ); + Ok(self.await_database_handle(&operation, result.span)) + } +} diff --git a/crates/dever-core/src/llvm/ownership.rs b/crates/dever-core/src/llvm/ownership.rs new file mode 100644 index 0000000..bea9134 --- /dev/null +++ b/crates/dever-core/src/llvm/ownership.rs @@ -0,0 +1,388 @@ +use super::*; + +pub(super) fn contains_owned(program: &Program, ty: &Type) -> bool { + match ty { + Type::Text + | Type::Json + | Type::Id + | Type::Secret + | Type::Uuid + | Type::ClientTls + | Type::ServerTls + | Type::HttpClient + | Type::HttpReply + | Type::WebSocket + | Type::Upload + | Type::Bytes + | Type::List(_) + | Type::Map(_, _) + | Type::File + | Type::Stream(_) + | Type::AsyncStream(_) + | Type::RowStream(_) + | Type::Related(_) + | Type::Socket + | Type::Listener + | Type::Task(_) + | Type::Group + | Type::Channel(_) => true, + Type::Nullable(inner) => contains_owned(program, inner), + Type::MapEntry(key, value) => { + contains_owned(program, key) || contains_owned(program, value) + } + Type::Outputs(fields) => fields + .iter() + .any(|field| contains_owned(program, &field.ty)), + Type::Named(id) => match &program.types[*id].shape { + Shape::Record(fields) => fields + .iter() + .any(|field| contains_owned(program, &field.ty)), + Shape::Choice(variants) => variants + .iter() + .flat_map(|variant| &variant.fields) + .any(|field| contains_owned(program, &field.ty)), + }, + _ => false, + } +} + +pub(super) fn hashable_key(program: &Program, ty: &Type) -> bool { + match ty { + Type::Bool | Type::Int | Type::Text | Type::Id => true, + Type::Named(id) if program.types[*id].kind == DefinitionKind::ModelId => true, + Type::Named(id) => matches!( + &program.types[*id].shape, + Shape::Choice(variants) if variants.iter().all(|variant| variant.fields.is_empty()) + ), + _ => false, + } +} + +impl Module<'_> { + pub(super) fn emit_type_equal(&mut self, index: usize, ty: &Type) -> Result<String, String> { + let instance = self + .instances + .iter() + .next() + .expect("reachable entry") + .clone(); + let llvm_type = self.ty(ty); + let mut emitter = FunctionEmitter::new(self, &instance); + emitter.start("entry"); + let left = emitter.temp(); + emitter.line(format!("{left} = load {llvm_type}, ptr %left")); + let right = emitter.temp(); + emitter.line(format!("{right} = load {llvm_type}, ptr %right")); + let equal = emitter.equals(ty, &left, &right)?; + let value = emitter.temp(); + emitter.line(format!("{value} = zext i1 {equal} to i8")); + emitter.line(format!("ret i8 {value}")); + Ok(format!( + "define internal i8 @dever_equal_{index}(ptr %left, ptr %right) {{\nentry:\n{} }}\n", + emitter.body + )) + } + + pub(super) fn emit_type_hash(&self, index: usize, ty: &Type) -> String { + if !hashable_key(self.program, ty) { + return String::new(); + } + let value = match ty { + Type::Bool => " %value = load i1, ptr %source\n %hash = zext i1 %value to i64\n".into(), + Type::Int => " %hash = load i64, ptr %source\n".into(), + Type::Named(id) if self.program.types[*id].kind == DefinitionKind::ModelId => " %hash = load i64, ptr %source\n".into(), + Type::Text | Type::Id => " %value = load ptr, ptr %source\n %hash = call i64 @dever_rt_v1_text_hash(ptr %value)\n".into(), + Type::Named(id) => format!( + " %tag = getelementptr %T{id}, ptr %source, i32 0, i32 0\n %value = load i32, ptr %tag\n %hash = zext i32 %value to i64\n" + ), + _ => unreachable!("checked Map key"), + }; + format!( + "define internal i64 @dever_hash_{index}(ptr %source) {{\nentry:\n{value} ret i64 %hash\n}}\n" + ) + } +} + +pub(super) fn collect_owned_types(program: &Program, ty: &Type, types: &mut Vec<Type>) { + if types.contains(ty) { + return; + } + types.push(ty.clone()); + match ty { + Type::Uuid => collect_owned_types(program, &Type::Text, types), + Type::Nullable(inner) + | Type::List(inner) + | Type::Stream(inner) + | Type::AsyncStream(inner) + | Type::RowStream(inner) + | Type::Related(inner) + | Type::Channel(inner) => collect_owned_types(program, inner, types), + Type::Map(key, value) | Type::MapEntry(key, value) => { + collect_owned_types(program, key, types); + collect_owned_types(program, value, types); + } + Type::Outputs(fields) | Type::Task(fields) => { + for field in fields { + collect_owned_types(program, &field.ty, types); + } + if matches!(ty, Type::Task(_)) { + collect_owned_types(program, &output_type(fields), types); + } + } + Type::Named(id) => match &program.types[*id].shape { + Shape::Record(fields) => { + for field in fields { + collect_owned_types(program, &field.ty, types); + } + } + Shape::Choice(variants) => { + for field in variants.iter().flat_map(|variant| &variant.fields) { + collect_owned_types(program, &field.ty, types); + } + } + }, + _ => {} + } +} + +pub(super) struct TypeOwner<'a> { + module: &'a Module<'a>, + ir: String, + next: usize, +} + +impl<'a> TypeOwner<'a> { + pub(super) fn new(module: &'a Module<'a>) -> Self { + Self { + module, + ir: String::new(), + next: 0, + } + } + + fn temp(&mut self) -> String { + let next = self.next; + self.next += 1; + format!("%o{next}") + } + + fn label(&mut self, name: &str) -> String { + let next = self.next; + self.next += 1; + format!("{name}{next}") + } + + fn line(&mut self, line: impl AsRef<str>) { + writeln!(self.ir, " {}", line.as_ref()).unwrap(); + } + + fn start(&mut self, label: &str) { + writeln!(self.ir, "{label}:").unwrap(); + } + + pub(super) fn emit(mut self, index: usize, ty: &Type) -> String { + let llvm_ty = self.module.ty(ty); + writeln!( + self.ir, + "define internal void @dever_clone_{index}(ptr %src, ptr %dst) {{\nentry:" + ) + .unwrap(); + let value = self.temp(); + self.line(format!("{value} = load {llvm_ty}, ptr %src")); + self.line(format!("store {llvm_ty} {value}, ptr %dst")); + self.adjust(ty, "%dst", true); + self.line("ret void"); + self.ir.push_str("}\n"); + + writeln!( + self.ir, + "define internal void @dever_drop_{index}(ptr %value) {{\nentry:" + ) + .unwrap(); + self.adjust(ty, "%value", false); + self.line("ret void"); + self.ir.push_str("}\n"); + self.ir + } + + fn adjust(&mut self, ty: &Type, address: &str, retain: bool) { + match ty { + Type::Text + | Type::Json + | Type::Id + | Type::Secret + | Type::Uuid + | Type::ClientTls + | Type::ServerTls + | Type::HttpClient + | Type::HttpReply + | Type::WebSocket + | Type::Upload + | Type::Bytes + | Type::List(_) + | Type::Map(_, _) + | Type::File + | Type::Stream(_) + | Type::AsyncStream(_) + | Type::RowStream(_) + | Type::Related(_) + | Type::Socket + | Type::Listener + | Type::Task(_) + | Type::Group + | Type::Channel(_) => { + let kind = match ty { + Type::Text | Type::Json | Type::Id => "text", + Type::Secret => "secret", + Type::Uuid => "uuid", + Type::ClientTls => "client_tls", + Type::ServerTls => "server_tls", + Type::HttpClient => "http_client", + Type::HttpReply => "http_reply", + Type::WebSocket => "websocket", + Type::Upload => "upload", + Type::Bytes => "bytes", + Type::List(_) => "list", + Type::Map(_, _) => "map", + Type::File => "file", + Type::Stream(_) => "stream", + Type::AsyncStream(_) => "async_stream", + Type::RowStream(_) => "db_stream", + Type::Related(_) => "db_related", + Type::Socket => "socket", + Type::Listener => "listener", + Type::Task(_) => "async_task", + Type::Group => "async_group", + Type::Channel(_) => "async_channel", + _ => unreachable!(), + }; + let handle = self.temp(); + self.line(format!("{handle} = load ptr, ptr {address}")); + if retain { + let owned = self.temp(); + self.line(format!( + "{owned} = call ptr @dever_rt_v1_{kind}_retain(ptr {handle})" + )); + self.line(format!("store ptr {owned}, ptr {address}")); + } else { + self.line(format!( + "call void @dever_rt_v1_{kind}_release(ptr {handle})" + )); + } + } + Type::Nullable(inner) => { + let present_ptr = self.temp(); + self.line(format!( + "{present_ptr} = getelementptr {}, ptr {address}, i32 0, i32 0", + self.module.ty(ty) + )); + let present = self.temp(); + self.line(format!("{present} = load i1, ptr {present_ptr}")); + let some = self.label("owned_some"); + let done = self.label("owned_done"); + self.line(format!("br i1 {present}, label %{some}, label %{done}")); + self.start(&some); + let payload = self.temp(); + self.line(format!( + "{payload} = getelementptr {}, ptr {address}, i32 0, i32 1", + self.module.ty(ty) + )); + self.adjust(inner, &payload, retain); + self.line(format!("br label %{done}")); + self.start(&done); + } + Type::Named(id) => match &self.module.program.types[*id].shape { + Shape::Record(fields) => { + let fields = fields + .iter() + .map(|field| field.ty.clone()) + .collect::<Vec<_>>(); + self.fields(ty, address, &fields, retain); + } + Shape::Choice(variants) => { + let variants = variants + .iter() + .map(|variant| { + variant + .fields + .iter() + .map(|field| field.ty.clone()) + .collect::<Vec<_>>() + }) + .collect::<Vec<Vec<Type>>>(); + self.choice(*id, address, &variants, retain); + } + }, + Type::MapEntry(key, value) => { + self.fields( + ty, + address, + &[key.as_ref().clone(), value.as_ref().clone()], + retain, + ); + } + Type::Outputs(fields) => { + let fields = fields + .iter() + .map(|field| field.ty.clone()) + .collect::<Vec<_>>(); + self.fields(ty, address, &fields, retain); + } + _ => {} + } + } + + fn fields(&mut self, ty: &Type, address: &str, fields: &[Type], retain: bool) { + for (index, field) in fields.iter().enumerate() { + if !contains_owned(self.module.program, field) { + continue; + } + let pointer = self.temp(); + self.line(format!( + "{pointer} = getelementptr {}, ptr {address}, i32 0, i32 {index}", + self.module.ty(ty) + )); + self.adjust(field, &pointer, retain); + } + } + + fn choice(&mut self, id: usize, address: &str, variants: &[Vec<Type>], retain: bool) { + let tag_ptr = self.temp(); + self.line(format!( + "{tag_ptr} = getelementptr %T{id}, ptr {address}, i32 0, i32 0" + )); + let tag = self.temp(); + self.line(format!("{tag} = load i32, ptr {tag_ptr}")); + let done = self.label("owned_choice_done"); + let arms = variants + .iter() + .map(|_| self.label("owned_variant")) + .collect::<Vec<_>>(); + let cases = arms + .iter() + .enumerate() + .map(|(index, arm)| format!("i32 {index}, label %{arm}")) + .collect::<Vec<_>>() + .join(" "); + self.line(format!("switch i32 {tag}, label %{done} [{cases}]")); + for (variant, fields) in variants.iter().enumerate() { + self.start(&arms[variant]); + let payload = self.temp(); + self.line(format!( + "{payload} = getelementptr %T{id}, ptr {address}, i32 0, i32 1" + )); + for (index, field) in fields.iter().enumerate() { + if !contains_owned(self.module.program, field) { + continue; + } + let pointer = self.temp(); + self.line(format!( + "{pointer} = getelementptr %T{id}V{variant}, ptr {payload}, i32 0, i32 {index}" + )); + self.adjust(field, &pointer, retain); + } + self.line(format!("br label %{done}")); + } + self.start(&done); + } +} diff --git a/crates/dever-core/src/llvm/parallel.rs b/crates/dever-core/src/llvm/parallel.rs new file mode 100644 index 0000000..48aa4c5 --- /dev/null +++ b/crates/dever-core/src/llvm/parallel.rs @@ -0,0 +1,99 @@ +//! Parallel dispatch preserves the runtime's bounded producer and task owners. +use super::*; + +pub(super) const DECLARATIONS: &str = "\ +%dever.parallel_function = type { ptr, ptr, ptr, ptr, ptr } +declare ptr @dever_rt_v1_async_parallel_each(ptr, i32, i64, ptr, ptr, ptr) +declare i32 @dever_rt_v1_parallel_each(ptr, i32, i64, ptr, ptr, ptr, ptr) +"; + +impl Module<'_> { + fn parallel_descriptor(&mut self, instance: &Specialization) -> String { + let name = self.names[instance].clone(); + let descriptor = format!("@dever_parallel_{name}"); + if !self.parallel_functions.insert(instance.clone()) { + return descriptor; + } + let function = &self.program.functions[instance.function]; + let input = asynchronous::input_type(function); + let Type::Outputs(fields) = &input else { + unreachable!() + }; + let context = fields.get(1).map_or("null".into(), |field| { + format!("@dever_type_{}", self.type_index(&field.ty)) + }); + let mut callback = format!( + "define internal void @dever_parallel_pack_{name}(ptr %element, ptr %context, ptr %out) {{\nentry:\n" + ); + for (index, field) in fields.iter().enumerate() { + let source = if index == 0 { "%element" } else { "%context" }; + writeln!(callback, " %p{index} = getelementptr {}, ptr %out, i32 0, i32 {index}\n call void @dever_clone_{}(ptr {source}, ptr %p{index})", self.ty(&input), self.type_index(&field.ty)).unwrap(); + } + callback.push_str(" ret void\n}\n"); + self.declarations.push_str(&callback); + let (asynchronous, synchronous) = if specialize::suspends(self.program, instance) { + (format!("@dever_async_{name}"), "null".into()) + } else { + ("null".into(), format!("@dever_sync_{name}")) + }; + writeln!(self.declarations, "{descriptor} = private constant %dever.parallel_function {{ ptr {asynchronous}, ptr {synchronous}, ptr @dever_owned_{}, ptr {context}, ptr @dever_parallel_pack_{name} }}", self.type_index(&input)).unwrap(); + descriptor + } +} + +impl FunctionEmitter<'_, '_> { + pub(super) fn parallel_each( + &mut self, + handler: crate::hir::HandlerTarget, + sequence: SequenceKind, + arguments: &[Expression], + result: &Expression, + ) -> String { + let source = self + .expression(&arguments[0]) + .expect("checked parallel source"); + let limit = self + .expression(&arguments[1]) + .expect("checked parallel limit"); + let context = arguments.get(2).map_or_else( + || "null".into(), + |argument| { + let value = self.expression(argument).expect("checked parallel context"); + self.row_pointer(&argument.ty, &value) + }, + ); + let instance = Specialization { + function: specialize::resolve_handler(handler, &self.bindings), + handlers: Vec::new(), + }; + let descriptor = self.module.parallel_descriptor(&instance); + let kind = match sequence { + SequenceKind::List => 0, + SequenceKind::Bytes => 1, + SequenceKind::Stream => 2, + SequenceKind::AsyncStream => 3, + _ => unreachable!("checked parallel sequence"), + }; + let arguments = vec![ + format!("ptr {source}"), + format!("i32 {kind}"), + format!("i64 {limit}"), + format!("ptr {descriptor}"), + format!("ptr {context}"), + ]; + if self.asynchronous && sequence != SequenceKind::Stream { + let operation = self.async_operation("async_parallel_each", arguments, result.span); + self.await_operation(&operation, &Type::Unit, result.span, true) + } else { + let error = self.entry_slot_ir("{ ptr, i64 }"); + let status = self.temp(); + self.line(format!( + "{status} = call i32 @dever_rt_v1_parallel_each({}, ptr %fault, ptr {error})", + arguments.join(", ") + )); + let legacy = self.poll_status(&status, &error, result.span); + self.propagate_status(&legacy, result.span, true); + "zeroinitializer".into() + } + } +} diff --git a/crates/dever-core/src/llvm/ports.rs b/crates/dever-core/src/llvm/ports.rs new file mode 100644 index 0000000..932f08f --- /dev/null +++ b/crates/dever-core/src/llvm/ports.rs @@ -0,0 +1,266 @@ +//! Closed Port dispatch and entry-owned, immutable selected Adapter settings. +use super::*; + +pub(super) const DECLARATIONS: &str = "\ +declare i32 @dever_rt_v1_adapter_settings_load(ptr, ptr) +declare void @dever_rt_v1_adapter_settings_release(ptr) +declare i32 @dever_rt_v1_adapter_settings_validate_names(ptr, ptr, i64, ptr) +declare i32 @dever_rt_v1_adapter_settings_select(ptr, ptr, ptr, i64, ptr, ptr, ptr, ptr) +"; + +impl Module<'_> { + pub(super) fn emit_port_slots(&mut self) { + let app = self.application.as_ref().unwrap(); + for port in &app.ports { + writeln!( + self.declarations, + "@dever_port_{port} = private global i64 0" + ) + .unwrap(); + } + for adapter in &app.adapters { + let adapter = &self.program.adapters[*adapter]; + if let Some(ty) = adapter.setting { + writeln!(self.declarations, "@dever_setting_{} = private global %T{ty} zeroinitializer\n@dever_setting_live_{} = private global i1 false", adapter.owner, adapter.owner).unwrap(); + } + } + } +} + +impl FunctionEmitter<'_, '_> { + pub(super) fn emit_port_body( + &mut self, + function: &crate::hir::Function, + ) -> Result<bool, String> { + if function.setting.is_some() { + let ty = output_type(&function.outputs); + self.line(format!( + "call void @dever_clone_{}(ptr @dever_setting_{}, ptr %out)", + self.module.type_index(&ty), + function.owner + )); + self.exit("0"); + return Ok(true); + } + let Some(port) = &function.port else { + return Ok(false); + }; + let selected = self.temp(); + self.line(format!( + "{selected} = load i64, ptr @dever_port_{}", + self.instance.function + )); + let cases = port + .implementations + .iter() + .map(|id| format!("i64 {id}, label %port_{id}")) + .collect::<Vec<_>>() + .join(" "); + self.line(format!( + "switch i64 {selected}, label %port_invalid [{cases}]" + )); + self.start("port_invalid"); + self.line("unreachable"); + let arguments = function + .parameters + .iter() + .enumerate() + .filter_map(|(index, parameter)| { + parameter + .value_type() + .map(|ty| (ty.clone(), self.parameter(index))) + }) + .collect::<Vec<_>>(); + let output = output_type(&function.outputs); + for implementation in &port.implementations { + self.start(&format!("port_{implementation}")); + let target = Specialization { + function: *implementation, + handlers: Vec::new(), + }; + let (status, row) = self.invoke_values(&target, &arguments, &output, function.span); + self.propagate_status(&status, function.span, false); + let value = self.temp(); + self.line(format!( + "{value} = load {}, ptr {row}", + self.module.ty(&output) + )); + self.line(format!( + "store {} {value}, ptr %out", + self.module.ty(&output) + )); + self.exit("0"); + } + Ok(true) + } + + pub(super) fn initialize_ports(&mut self, span: Span) { + let app = self.module.application.as_ref().unwrap().clone(); + if let Some(index) = app.test { + for port in app.ports { + let implementation = self.module.program.tests[index].port_bindings[&port]; + self.line(format!( + "store i64 {implementation}, ptr @dever_port_{port}" + )); + } + return; + } + if app.ports.is_empty() { + return; + } + let settings = self.protocol_call("dever_rt_v1_adapter_settings_load", vec![], "ptr", span); + let settings_guard = self.protocol_owner("adapter_settings", &settings); + let identities = self + .module + .program + .functions + .iter() + .filter_map(|function| function.port.as_ref().map(|port| port.identity.clone())) + .collect::<BTreeSet<_>>() + .into_iter() + .collect::<Vec<_>>(); + let names = self.module.wire_names(&identities); + self.wire_write( + "dever_rt_v1_adapter_settings_validate_names", + vec![ + format!("ptr {settings}"), + format!("ptr {names}"), + format!("i64 {}", identities.len()), + ], + ); + let owners = app + .ports + .iter() + .map(|port| self.module.program.functions[*port].owner) + .collect::<BTreeSet<_>>(); + for owner in owners { + let adapters = app + .adapters + .iter() + .filter_map(|id| { + let adapter = &self.module.program.adapters[*id]; + (adapter.port_owner == owner).then_some(adapter.clone()) + }) + .collect::<Vec<_>>(); + let names = self.module.wire_names( + &adapters + .iter() + .map(|adapter| adapter.name.clone()) + .collect::<Vec<_>>(), + ); + let identity = self.module.wire_names(&[adapters[0].identity.clone()]); + let index = self.entry_slot_ir("i64"); + let setting = self.entry_slot_ir("ptr"); + let present = self.entry_slot_ir("i8"); + self.wire_write( + "dever_rt_v1_adapter_settings_select", + vec![ + format!("ptr {settings}"), + format!("ptr {identity}"), + format!("ptr {names}"), + format!("i64 {}", adapters.len()), + format!("ptr {index}"), + format!("ptr {setting}"), + format!("ptr {present}"), + ], + ); + let optional = self.optional_row( + &Type::Nullable(Box::new(Type::Text)), + &Type::Text, + &setting, + &present, + ); + let setting_guard = + self.own_value(&Type::Nullable(Box::new(Type::Text)), &optional, false); + let has_setting = self.temp(); + self.line(format!( + "{has_setting} = extractvalue {{ i1, ptr }} {optional}, 0" + )); + let selected = self.temp(); + self.line(format!("{selected} = load i64, ptr {index}")); + let done = self.label("adapter_ready"); + let invalid = self.label("adapter_invalid"); + let labels = adapters + .iter() + .map(|_| self.label("adapter_selected")) + .collect::<Vec<_>>(); + let arms = labels + .iter() + .enumerate() + .map(|(index, label)| format!("i64 {index}, label %{label}")) + .collect::<Vec<_>>() + .join(" "); + self.line(format!("switch i64 {selected}, label %{invalid} [{arms}]")); + self.start(&invalid); + self.line("unreachable"); + for (adapter, label) in adapters.iter().zip(labels) { + self.start(&label); + let valid = self.label("setting_valid"); + let failed = self.label("setting_invalid"); + self.line(if adapter.setting.is_some() { + format!("br i1 {has_setting}, label %{valid}, label %{failed}") + } else { + format!("br i1 {has_setting}, label %{failed}, label %{valid}") + }); + self.start(&failed); + self.application_fault( + if adapter.setting.is_some() { + "selected Adapter requires setting" + } else { + "selected Adapter does not declare setting" + }, + span, + ); + self.start(&valid); + if let Some(id) = adapter.setting { + let ty = Type::Named(id); + let callback = + self.module + .wire_callback(&ty, crate::wire::Policy::SettingInput, false); + let text = self.temp(); + self.line(format!("{text} = extractvalue {{ i1, ptr }} {optional}, 1")); + let value = self.runtime_call( + "dever_rt_v1_wire_decode", + vec![ + format!("ptr {text}"), + format!("ptr @dever_type_{}", self.module.type_index(&ty)), + format!("ptr @{callback}"), + ], + &ty, + false, + None, + span, + ); + self.line(format!( + "store {} {value}, ptr @dever_setting_{}", + self.module.ty(&ty), + adapter.owner + )); + self.line(format!( + "store i1 true, ptr @dever_setting_live_{}", + adapter.owner + )); + } + let external_setting = if adapter.setting.is_some() { + let text = self.temp(); + self.line(format!("{text} = extractvalue {{ i1, ptr }} {optional}, 1")); + text + } else { + "null".into() + }; + self.start_external_adapter(adapter, &external_setting, span); + for (port, implementation) in &adapter.operations { + if app.ports.contains(port) { + self.line(format!( + "store i64 {implementation}, ptr @dever_port_{port}" + )); + } + } + self.line(format!("br label %{done}")); + } + self.start(&done); + self.release_guard(setting_guard); + } + self.release_guard(settings_guard); + } +} diff --git a/crates/dever-core/src/llvm/process_entry.rs b/crates/dever-core/src/llvm/process_entry.rs new file mode 100644 index 0000000..be274e5 --- /dev/null +++ b/crates/dever-core/src/llvm/process_entry.rs @@ -0,0 +1,176 @@ +//! Process glue reuses callable roots, typed fault rendering and their owners. +use super::*; + +const DECLARATIONS: &str = "\ +declare i32 @dever_rt_v1_process_init() +declare i32 @dever_rt_v1_process_stderr(ptr, i64) +declare i32 @dever_rt_v1_process_error(ptr) +"; + +impl Module<'_> { + pub(super) fn emit_process_entry(&mut self, root: &Specialization) { + let render = self.http_fault_message(root); + self.declarations.push_str(DECLARATIONS); + let calls = Calls { + run: "@dever_application_entry(ptr %out, ptr %fault)".into(), + render: format!("{render}(ptr %fault, ptr %message, ptr %error)"), + release_outputs: "@dever_outputs_release(ptr %out)".into(), + release_fault: "@dever_fault_release(ptr %fault)".into(), + }; + emit_start(&mut self.functions); + self.functions.push_str( + " %fault = alloca %dever.fault, align 8\n store %dever.fault zeroinitializer, ptr %fault\n", + ); + emit_body(&mut self.functions, &calls); + emit_diagnostic_literals(&mut self.declarations); + } +} + +pub(super) fn emit_test_main(suite: &mut String) { + suite.push_str(DECLARATIONS); + suite.push_str("declare i32 @dever_rt_v1_process_test_index(i32, ptr, i64, ptr)\n"); + emit_diagnostic_literals(suite); + let invalid = literal( + suite, + "dever_process_invalid_index", + "test program requires a valid case index\n", + ); + emit_start(suite); + writeln!( + suite, + " %index_slot = alloca i64, align 8 + %count = call i64 @dever_test_count() + %parsed = call i32 @dever_rt_v1_process_test_index(i32 %argc, ptr %argv, i64 %count, ptr %index_slot) + %valid = icmp eq i32 %parsed, 0 + br i1 %valid, label %selected, label %invalid +invalid: + %written = call i32 @dever_rt_v1_process_stderr({invalid}) + call void @dever_rt_v1_log_flush() + ret i32 1 +selected: + %index = load i64, ptr %index_slot + %size = call i64 @dever_test_fault_size(i64 %index) + %fault = alloca i8, i64 %size, align 8" + ) + .unwrap(); + // The checked case entry initializes its complete concrete fault layout. + // Size, render and release must all use this same validated stable index. + let calls = Calls { + run: "@dever_test_entry(i64 %index, ptr %out, ptr %fault)".into(), + render: "@dever_test_fault_render(i64 %index, ptr %fault, ptr %message, ptr %error)".into(), + release_outputs: "@dever_test_outputs_release(i64 %index, ptr %out)".into(), + release_fault: "@dever_test_fault_release(i64 %index, ptr %fault)".into(), + }; + emit_body(suite, &calls); +} + +struct Calls { + run: String, + render: String, + release_outputs: String, + release_fault: String, +} + +fn emit_start(output: &mut String) { + writeln!( + output, + "define i32 @main(i32 %argc, ptr %argv) {{ +entry: + %initialized = call i32 @dever_rt_v1_process_init() + %init_ok = icmp eq i32 %initialized, 0 + br i1 %init_ok, label %ready, label %init_failed +init_failed: + %init_written = call i32 @dever_rt_v1_process_stderr(ptr @dever_process_init_failed, i64 {}) + ret i32 1 +ready:", + INIT_FAILED.len() + ) + .unwrap(); +} + +/// Both roots return Unit; diagnostics are the only additional process owners. +/// Every exit after invocation releases the original fault even if rendering +/// fails, so a secondary diagnostic error cannot leak its business payload. +fn emit_body(output: &mut String, calls: &Calls) { + writeln!( + output, + " %out = alloca i8 + store i8 0, ptr %out + %message = alloca ptr + store ptr null, ptr %message + %error = alloca {{ ptr, i64 }} + store {{ ptr, i64 }} zeroinitializer, ptr %error + %status = call i32 {} + %failed = icmp ne i32 %status, 0 + br i1 %failed, label %report, label %cleanup +report: + %rendered = call i32 {} + %render_ok = icmp eq i32 %rendered, 0 + br i1 %render_ok, label %log, label %render_failed +log: + %text = load ptr, ptr %message + %logged = call i32 @dever_rt_v1_process_error(ptr %text) + %log_ok = icmp eq i32 %logged, 0 + br i1 %log_ok, label %cleanup, label %log_failed +log_failed: + %log_fallback = call i32 @dever_rt_v1_process_stderr(ptr @dever_process_log_failed, i64 {}) + br label %cleanup +render_failed: + %render_fallback = call i32 @dever_rt_v1_process_stderr(ptr @dever_process_render_failed, i64 {}) + %error_value = load {{ ptr, i64 }}, ptr %error + %error_bytes = extractvalue {{ ptr, i64 }} %error_value, 0 + %error_length = extractvalue {{ ptr, i64 }} %error_value, 1 + %error_written = call i32 @dever_rt_v1_process_stderr(ptr %error_bytes, i64 %error_length) + %newline_written = call i32 @dever_rt_v1_process_stderr(ptr @dever_process_newline, i64 1) + br label %cleanup +cleanup: + %owned_message = load ptr, ptr %message + call void @dever_rt_v1_text_release(ptr %owned_message) + %owned_error = load {{ ptr, i64 }}, ptr %error + %owned_bytes = extractvalue {{ ptr, i64 }} %owned_error, 0 + %owned_length = extractvalue {{ ptr, i64 }} %owned_error, 1 + call void @dever_rt_v1_buffer_free(ptr %owned_bytes, i64 %owned_length) + call void {} + call void {} + call void @dever_rt_v1_log_flush() + %exit = zext i1 %failed to i32 + ret i32 %exit +}}", + calls.run, + calls.render, + LOG_FAILED.len(), + RENDER_FAILED.len(), + calls.release_outputs, + calls.release_fault, + ) + .unwrap(); +} + +const LOG_FAILED: &str = "cannot log application failure\n"; +const INIT_FAILED: &str = "cannot initialize application process\n"; +const RENDER_FAILED: &str = "cannot render application failure: "; + +fn emit_diagnostic_literals(output: &mut String) { + for (name, value) in [ + ("dever_process_init_failed", INIT_FAILED), + ("dever_process_log_failed", LOG_FAILED), + ("dever_process_render_failed", RENDER_FAILED), + ("dever_process_newline", "\n"), + ] { + literal(output, name, value); + } +} + +fn literal(output: &mut String, name: &str, value: &str) -> String { + let bytes = value + .bytes() + .map(|byte| format!("\\{byte:02X}")) + .collect::<String>(); + writeln!( + output, + "@{name} = private constant [{} x i8] c\"{bytes}\"", + value.len() + ) + .unwrap(); + format!("ptr @{name}, i64 {}", value.len()) +} diff --git a/crates/dever-core/src/llvm/render.rs b/crates/dever-core/src/llvm/render.rs new file mode 100644 index 0000000..34f1a36 --- /dev/null +++ b/crates/dever-core/src/llvm/render.rs @@ -0,0 +1,644 @@ +//! Typed display only at terminal request diagnostics; faults stay owned and typed. +use super::*; + +impl Module<'_> { + fn related_renderer(&mut self, inner: &Type, instance: &Specialization, span: Span) -> String { + let index = self.type_index(inner); + let name = format!("dever_render_related_{index}"); + if !self.related_renderers.insert(index) { + return name; + } + let mut emitter = FunctionEmitter::new(self, instance); + emitter.asynchronous = false; + emitter.start("entry"); + emitter.display_literal("Related.Loaded(", span); + let owner = emitter.guards.len() - 1; + let output = emitter.guards[owner].pointer.clone(); + let value = emitter.runtime_call( + "dever_rt_v1_db_related_get", + vec!["ptr %input".into()], + inner, + false, + None, + span, + ); + emitter.own_value(inner, &value, false); + emitter.display_value(&output, inner, &value, span); + emitter.display_append_literal(&output, ")", span); + let result = emitter.temp(); + emitter.line(format!("{result} = load ptr, ptr {output}")); + emitter.line(format!("store ptr {result}, ptr %out")); + emitter.disarm(owner); + let body = emitter.finish_protocol_callback(&name, "ptr %input, ptr %out, ptr %error"); + self.declarations.push_str(&body); + name + } + pub(super) fn emit_location_renderer(&mut self) { + let mut entries = Vec::new(); + for span in self.locations.clone() { + let source = self.sources.get(span.source); + let (line, column) = source.position(span.start); + let (literal, length) = + self.text_literal(&format!("{}:{line}:{column}: ", source.path().display())); + entries.push(format!("{{ ptr, i64 }} {{ ptr {literal}, i64 {length} }}")); + } + let length = entries.len(); + writeln!( + self.declarations, + "@dever_location_strings = private constant [{length} x {{ ptr, i64 }}] [{}]", + entries.join(", ") + ) + .unwrap(); + writeln!(self.declarations, "define internal i32 @dever_location_text(i32 %index, ptr %out, ptr %error) {{\nentry:\n %slot = getelementptr [{length} x {{ ptr, i64 }}], ptr @dever_location_strings, i32 0, i32 %index\n %value = load {{ ptr, i64 }}, ptr %slot\n %bytes = extractvalue {{ ptr, i64 }} %value, 0\n %length = extractvalue {{ ptr, i64 }} %value, 1\n %status = call i32 @dever_rt_v1_text_new(ptr %bytes, i64 %length, ptr %out, ptr %error)\n ret i32 %status\n}}").unwrap(); + } + + pub(super) fn http_fault_message(&mut self, instance: &Specialization) -> String { + let name = format!("dever_http_fault_{}", self.names[instance]); + let span = self.program.functions[instance.function].span; + let failures = specialize::failures(self.program, instance); + let mut emitter = FunctionEmitter::new(self, instance); + emitter.asynchronous = false; + emitter.start("entry"); + emitter.display_literal("", span); + let guard = emitter.guards.len() - 1; + let output = emitter.guards[guard].pointer.clone(); + let depth_ptr = emitter.temp(); + emitter.line(format!( + "{depth_ptr} = getelementptr %dever.fault, ptr %input, i32 0, i32 2" + )); + let depth = emitter.temp(); + emitter.line(format!("{depth} = load i32, ptr {depth_ptr}")); + let index = emitter.entry_slot_ir("i32"); + emitter.line(format!("store i32 {depth}, ptr {index}")); + let next = emitter.label("trace_next"); + let frame = emitter.label("trace_frame"); + let origin = emitter.label("trace_origin"); + emitter.line(format!("br label %{next}")); + emitter.start(&next); + let position = emitter.temp(); + emitter.line(format!("{position} = load i32, ptr {index}")); + let has_frame = emitter.temp(); + emitter.line(format!("{has_frame} = icmp ne i32 {position}, 0")); + emitter.line(format!( + "br i1 {has_frame}, label %{frame}, label %{origin}" + )); + emitter.start(&frame); + let previous = emitter.temp(); + emitter.line(format!("{previous} = sub i32 {position}, 1")); + emitter.line(format!("store i32 {previous}, ptr {index}")); + let pointer = emitter.temp(); + emitter.line(format!( + "{pointer} = getelementptr %dever.fault, ptr %input, i32 0, i32 3, i32 {previous}" + )); + emitter.display_location(&output, &pointer, span); + emitter.line(format!("br label %{next}")); + emitter.start(&origin); + let pointer = emitter.temp(); + emitter.line(format!( + "{pointer} = getelementptr %dever.fault, ptr %input, i32 0, i32 1" + )); + emitter.display_location(&output, &pointer, span); + let kind_ptr = emitter.temp(); + emitter.line(format!( + "{kind_ptr} = getelementptr %dever.fault, ptr %input, i32 0, i32 0" + )); + let kind = emitter.temp(); + emitter.line(format!("{kind} = load i32, ptr {kind_ptr}")); + let done = emitter.label("message_ready"); + emitter.line("switch i32 ".to_owned() + &kind + ", label %invalid [i32 1, label %overflow i32 2, label %division i32 3, label %runtime i32 4, label %business]"); + emitter.start("invalid"); + emitter.line("unreachable"); + for (label, text) in [ + ("overflow", "Int overflow"), + ("division", "Int division by zero"), + ] { + emitter.start(label); + emitter.display_append_literal(&output, text, span); + emitter.line(format!("br label %{done}")); + } + emitter.start("runtime"); + let message_ptr = emitter.temp(); + emitter.line(format!( + "{message_ptr} = getelementptr %dever.fault, ptr %input, i32 0, i32 4" + )); + let message = emitter.temp(); + emitter.line(format!( + "{message} = load {{ ptr, i64 }}, ptr {message_ptr}" + )); + let bytes = emitter.temp(); + emitter.line(format!( + "{bytes} = extractvalue {{ ptr, i64 }} {message}, 0" + )); + let length = emitter.temp(); + emitter.line(format!( + "{length} = extractvalue {{ ptr, i64 }} {message}, 1" + )); + let first = emitter.guards.len(); + let text = emitter.runtime_call( + "dever_rt_v1_text_new", + vec![format!("ptr {bytes}"), format!("i64 {length}")], + &Type::Text, + false, + None, + span, + ); + emitter.own_value(&Type::Text, &text, false); + emitter.display_append(&output, &text, span); + emitter.release_from(first); + emitter.line(format!("br label %{done}")); + emitter.start("business"); + let type_ptr = emitter.temp(); + emitter.line(format!( + "{type_ptr} = getelementptr %dever.fault, ptr %input, i32 0, i32 5" + )); + let type_id = emitter.temp(); + emitter.line(format!("{type_id} = load i32, ptr {type_ptr}")); + let types = failures + .iter() + .map(|failure| failure.ty) + .collect::<BTreeSet<_>>(); + emitter.line(format!( + "switch i32 {type_id}, label %invalid [{}]", + types + .iter() + .map(|id| format!("i32 {}, label %business{id}", id + 1)) + .collect::<Vec<_>>() + .join(" ") + )); + for id in types { + emitter.start(&format!("business{id}")); + let payload_ptr = emitter.temp(); + emitter.line(format!( + "{payload_ptr} = getelementptr %dever.fault, ptr %input, i32 0, i32 7" + )); + let payload = emitter.temp(); + emitter.line(format!("{payload} = load %T{id}, ptr {payload_ptr}")); + let variants = failures + .iter() + .filter(|failure| failure.ty == id) + .map(|failure| failure.variant) + .collect::<BTreeSet<_>>(); + emitter.display_named(&output, id, &payload, span, Some(&variants)); + emitter.line(format!("br label %{done}")); + } + emitter.start(&done); + let cause_ptr = emitter.temp(); + emitter.line(format!( + "{cause_ptr} = getelementptr %dever.fault, ptr %input, i32 0, i32 8" + )); + let cause = emitter.temp(); + emitter.line(format!("{cause} = load ptr, ptr {cause_ptr}")); + let present = emitter.temp(); + emitter.line(format!("{present} = icmp ne ptr {cause}, null")); + let append = emitter.label("message_cause"); + let complete = emitter.label("message_complete"); + emitter.line(format!( + "br i1 {present}, label %{append}, label %{complete}" + )); + emitter.start(&append); + emitter.display_append_literal(&output, "; ", span); + emitter.display_append(&output, &cause, span); + emitter.line(format!("br label %{complete}")); + emitter.start(&complete); + let text = emitter.temp(); + emitter.line(format!("{text} = load ptr, ptr {output}")); + emitter.line(format!("store ptr {text}, ptr %out")); + emitter.disarm(guard); + let callback = emitter.finish_protocol_callback(&name, "ptr %input, ptr %out, ptr %error"); + self.declarations.push_str(&callback); + format!("@{name}") + } +} + +impl FunctionEmitter<'_, '_> { + pub(super) fn display_literal(&mut self, text: &str, span: Span) -> String { + let (bytes, length) = self.module.text_literal(text); + let value = self.runtime_call( + "dever_rt_v1_text_new", + vec![format!("ptr {bytes}"), format!("i64 {length}")], + &Type::Text, + false, + None, + span, + ); + self.own_value(&Type::Text, &value, false); + value + } + + fn display_append(&mut self, output: &str, text: &str, span: Span) { + let previous = self.temp(); + self.line(format!("{previous} = load ptr, ptr {output}")); + let joined = self.runtime_call( + "dever_rt_v1_text_concat", + vec![format!("ptr {previous}"), format!("ptr {text}")], + &Type::Text, + false, + None, + span, + ); + self.line(format!( + "call void @dever_rt_v1_text_release(ptr {previous})" + )); + self.line(format!("store ptr {joined}, ptr {output}")); + } + + pub(super) fn display_append_literal(&mut self, output: &str, text: &str, span: Span) { + let first = self.guards.len(); + let text = self.display_literal(text, span); + self.display_append(output, &text, span); + self.release_from(first); + } + + fn display_location(&mut self, output: &str, pointer: &str, span: Span) { + let first = self.guards.len(); + let location = self.temp(); + self.line(format!("{location} = load i32, ptr {pointer}")); + let text = self.runtime_call( + "dever_location_text", + vec![format!("i32 {location}")], + &Type::Text, + false, + None, + span, + ); + self.own_value(&Type::Text, &text, false); + self.display_append(output, &text, span); + self.release_from(first); + } + + pub(super) fn display_value(&mut self, output: &str, ty: &Type, value: &str, span: Span) { + assert!( + ty.observable(&self.module.program.types), + "checked diagnostic payload must be observable" + ); + let first = self.guards.len(); + match ty { + Type::Text | Type::Json | Type::Id => self.display_append(output, value, span), + Type::Int + | Type::DateTime + | Type::Date + | Type::Time + | Type::Duration + | Type::Float + | Type::Decimal + | Type::Uuid => { + let text = match ty { + Type::Decimal => { + let parts = self.decimal_parts(value); + self.render_text("dever_rt_v1_decimal_to_text", parts, span) + } + Type::Float => self.render_text( + "dever_rt_v1_float_to_text", + vec![format!("double {value}")], + span, + ), + Type::Uuid => self.runtime_call( + "dever_rt_v1_uuid_to_text", + vec![format!("ptr {value}")], + &Type::Text, + false, + None, + span, + ), + _ => self.render_text( + "dever_rt_v1_int_to_text", + vec![format!("i64 {value}")], + span, + ), + }; + self.own_value(&Type::Text, &text, false); + self.display_append(output, &text, span); + } + Type::Bool => { + let yes = self.label("display_true"); + let no = self.label("display_false"); + let done = self.label("display_bool"); + self.line(format!("br i1 {value}, label %{yes}, label %{no}")); + self.start(&yes); + self.display_append_literal(output, "true", span); + self.line(format!("br label %{done}")); + self.start(&no); + self.display_append_literal(output, "false", span); + self.line(format!("br label %{done}")); + self.start(&done); + } + Type::Nullable(inner) => { + let present = self.temp(); + self.line(format!( + "{present} = extractvalue {} {value}, 0", + self.module.ty(ty) + )); + let some = self.label("display_some"); + let null = self.label("display_null"); + let done = self.label("display_optional"); + self.line(format!("br i1 {present}, label %{some}, label %{null}")); + self.start(&null); + self.display_append_literal(output, "null", span); + self.line(format!("br label %{done}")); + self.start(&some); + let payload = self.temp(); + self.line(format!( + "{payload} = extractvalue {} {value}, 1", + self.module.ty(ty) + )); + self.display_value(output, inner, &payload, span); + self.line(format!("br label %{done}")); + self.start(&done); + } + Type::Named(id) if self.module.program.types[*id].kind == DefinitionKind::ModelId => { + self.display_value(output, &Type::Int, value, span); + } + Type::Named(id) => self.display_named(output, *id, value, span, None), + Type::List(_) | Type::Bytes | Type::Map(_, _) => { + self.display_collection(output, ty, value, span) + } + Type::MapEntry(key, mapped) => { + self.display_append_literal(output, "{key = ", span); + for (index, field) in [key.as_ref(), mapped.as_ref()].iter().enumerate() { + if index == 1 { + self.display_append_literal(output, ", value = ", span); + } + let part = self.temp(); + self.line(format!( + "{part} = extractvalue {} {value}, {index}", + self.module.ty(ty) + )); + self.display_value(output, field, &part, span); + } + self.display_append_literal(output, "}", span); + } + Type::File | Type::Socket | Type::Listener => { + self.display_append_literal(output, "<resource>", span) + } + Type::Stream(_) => self.display_append_literal(output, "<stream>", span), + Type::AsyncStream(_) | Type::RowStream(_) => { + self.display_append_literal(output, "<async-stream>", span) + } + Type::Related(inner) => { + let loaded = self.temp(); + self.line(format!("{loaded} = icmp ne ptr {value}, null")); + let some = self.label("display_related_loaded"); + let none = self.label("display_related_unloaded"); + let done = self.label("display_related_done"); + self.line(format!("br i1 {loaded}, label %{some}, label %{none}")); + self.start(&none); + self.display_append_literal(output, "Related.Unloaded", span); + self.line(format!("br label %{done}")); + self.start(&some); + let renderer = self.module.related_renderer(inner, self.instance, span); + let text = self.runtime_call( + &renderer, + vec![format!("ptr {value}")], + &Type::Text, + false, + None, + span, + ); + self.own_value(&Type::Text, &text, false); + self.display_append(output, &text, span); + self.line(format!("br label %{done}")); + self.start(&done); + } + Type::HttpReply => self.display_append_literal(output, "HttpReply(<resource>)", span), + Type::WebSocket => self.display_append_literal(output, "WebSocket(<resource>)", span), + Type::ClientTls => self.display_append_literal(output, "ClientTls(<resource>)", span), + Type::ServerTls => self.display_append_literal(output, "ServerTls(<resource>)", span), + Type::HttpClient => self.display_append_literal(output, "HttpClient(<resource>)", span), + Type::Channel(_) => self.display_append_literal(output, "Channel(<resource>)", span), + _ => unreachable!("checked observable error payload"), + } + self.release_from(first); + } + + fn display_named( + &mut self, + output: &str, + id: usize, + value: &str, + span: Span, + selected: Option<&BTreeSet<usize>>, + ) { + let definition = self.module.program.types[id].clone(); + match definition.shape { + Shape::Record(fields) => { + self.display_append_literal(output, &format!("{} {{ ", definition.name), span); + for (index, field) in fields.iter().enumerate() { + self.display_append_literal( + output, + &format!("{}{} = ", if index == 0 { "" } else { ", " }, field.name), + span, + ); + let part = self.temp(); + self.line(format!("{part} = extractvalue %T{id} {value}, {index}")); + self.display_value(output, &field.ty, &part, span); + } + self.display_append_literal(output, " }", span); + } + Shape::Choice(variants) => { + let row = self.row_pointer(&Type::Named(id), value); + let tag = self.temp(); + self.line(format!("{tag} = extractvalue %T{id} {value}, 0")); + let labels = variants + .iter() + .map(|_| self.label("display_variant")) + .collect::<Vec<_>>(); + let invalid = self.label("display_invalid"); + let done = self.label("display_choice"); + self.line(format!( + "switch i32 {tag}, label %{invalid} [{}]", + labels + .iter() + .enumerate() + .filter( + |(index, _)| selected.is_none_or(|selected| selected.contains(index)) + ) + .map(|(index, label)| format!("i32 {index}, label %{label}")) + .collect::<Vec<_>>() + .join(" ") + )); + self.start(&invalid); + self.line("unreachable"); + for (index, variant) in variants.iter().enumerate() { + if selected.is_some_and(|selected| !selected.contains(&index)) { + continue; + } + self.start(&labels[index]); + self.display_append_literal( + output, + &format!("{}.{}", definition.name, variant.name), + span, + ); + if !variant.fields.is_empty() { + self.display_append_literal(output, "(", span); + let payload = self.temp(); + self.line(format!( + "{payload} = getelementptr %T{id}, ptr {row}, i32 0, i32 1" + )); + for (field_index, field) in variant.fields.iter().enumerate() { + if field_index != 0 { + self.display_append_literal(output, ", ", span); + } + let pointer = self.temp(); + self.line(format!("{pointer} = getelementptr %T{id}V{index}, ptr {payload}, i32 0, i32 {field_index}")); + let part = self.temp(); + self.line(format!( + "{part} = load {}, ptr {pointer}", + self.module.ty(&field.ty) + )); + self.display_value(output, &field.ty, &part, span); + } + self.display_append_literal(output, ")", span); + } + self.line(format!("br label %{done}")); + } + self.start(&done); + } + } + } + + fn display_collection(&mut self, output: &str, ty: &Type, value: &str, span: Span) { + let map = matches!(ty, Type::Map(_, _)); + let bytes = *ty == Type::Bytes; + self.display_append_literal(output, if map { "{" } else { "[" }, span); + let types = match ty { + Type::Bytes => vec![Type::Int], + Type::List(element) => vec![element.as_ref().clone()], + Type::Map(key, value) => vec![key.as_ref().clone(), value.as_ref().clone()], + _ => unreachable!(), + }; + let kind = if map { "map" } else { "list" }; + let cursor = if bytes { + None + } else { + let retained = self.temp(); + self.line(format!( + "{retained} = call ptr @dever_rt_v1_{kind}_retain(ptr {value})" + )); + let cursor = self.runtime_call( + &format!("dever_rt_v1_{kind}_cursor_take"), + vec![format!("ptr {retained}")], + &Type::Unit, + false, + Some("ptr"), + span, + ); + let pointer = self.entry_slot_ir("ptr"); + self.line(format!("store ptr {cursor}, ptr {pointer}")); + let guard = self.register_owned( + if map { + OwnedKind::MapCursor + } else { + OwnedKind::ListCursor + }, + pointer, + ); + self.mark_live(guard); + Some((cursor, guard)) + }; + let length = bytes.then(|| { + self.runtime_call( + "dever_rt_v1_bytes_length", + vec![format!("ptr {value}")], + &Type::Int, + false, + None, + span, + ) + }); + let index = self.entry_slot_ir("i64"); + self.line(format!("store i64 0, ptr {index}")); + let rows = types + .iter() + .map(|ty| { + let row = self.entry_slot(ty); + let guard = contains_owned(self.module.program, ty) + .then(|| self.register_guard(ty, row.clone())); + (row, guard) + }) + .collect::<Vec<_>>(); + let present = self.entry_slot_ir("i8"); + let error = self.entry_slot_ir("{ ptr, i64 }"); + let next = self.label("display_next"); + let body = self.label("display_element"); + let separator = self.label("display_separator"); + let content = self.label("display_content"); + let done = self.label("display_collection_done"); + self.line(format!("br label %{next}")); + self.start(&next); + let position = self.temp(); + self.line(format!("{position} = load i64, ptr {index}")); + let exists = self.temp(); + if let Some(length) = length { + self.line(format!("{exists} = icmp slt i64 {position}, {length}")); + } else { + let mut args = vec![format!("ptr {}", cursor.as_ref().unwrap().0)]; + args.extend(rows.iter().map(|(row, _)| format!("ptr {row}"))); + args.extend([format!("ptr {present}"), format!("ptr {error}")]); + let status = self.temp(); + self.line(format!( + "{status} = call i32 @dever_rt_v1_{kind}_cursor_next({})", + args.join(", ") + )); + self.check_runtime_status(&status, &error, span); + let flag = self.temp(); + self.line(format!("{flag} = load i8, ptr {present}")); + self.line(format!("{exists} = icmp ne i8 {flag}, 0")); + } + self.line(format!("br i1 {exists}, label %{body}, label %{done}")); + self.start(&body); + for (_, guard) in &rows { + if let Some(guard) = guard { + self.mark_live(*guard); + } + } + let first = self.temp(); + self.line(format!("{first} = icmp eq i64 {position}, 0")); + self.line(format!( + "br i1 {first}, label %{content}, label %{separator}" + )); + self.start(&separator); + self.display_append_literal(output, ", ", span); + self.line(format!("br label %{content}")); + self.start(&content); + for (column, (ty, (row, _))) in types.iter().zip(&rows).enumerate() { + if column != 0 { + self.display_append_literal(output, " = ", span); + } + let element = if bytes { + let nullable = self.runtime_call( + "dever_rt_v1_bytes_at", + vec![format!("ptr {value}"), format!("i64 {position}")], + &Type::Nullable(Box::new(Type::Int)), + true, + None, + span, + ); + let byte = self.temp(); + self.line(format!("{byte} = extractvalue {{ i1, i64 }} {nullable}, 1")); + byte + } else { + let element = self.temp(); + self.line(format!( + "{element} = load {}, ptr {row}", + self.module.ty(ty) + )); + element + }; + self.display_value(output, ty, &element, span); + } + for (_, guard) in &rows { + if let Some(guard) = guard { + self.release_guard(*guard); + } + } + let following = self.temp(); + self.line(format!("{following} = add i64 {position}, 1")); + self.line(format!("store i64 {following}, ptr {index}")); + self.line(format!("br label %{next}")); + self.start(&done); + if let Some((_, guard)) = cursor { + self.release_guard(guard); + } + self.display_append_literal(output, if map { "}" } else { "]" }, span); + } +} diff --git a/crates/dever-core/src/llvm/resources.rs b/crates/dever-core/src/llvm/resources.rs new file mode 100644 index 0000000..a5f3669 --- /dev/null +++ b/crates/dever-core/src/llvm/resources.rs @@ -0,0 +1,296 @@ +//! Resource choices share the existing recoverable Bytes failure channel. +//! File I/O and shared Stream cursor ownership remain in the runtime. + +use super::*; + +pub(super) struct ResourceRow { + pub status: String, + pub output: String, + pub present: Option<String>, + pub error: String, + pub protocol_output: bool, +} + +impl Module<'_> { + pub(super) fn read_event(&mut self, id: usize) -> String { + let name = format!("@dever_read_event_{id}"); + if !self.read_events.insert(id) { + return name; + } + let Shape::Choice(variants) = &self.program.types[id].shape else { + unreachable!("checked read event choice") + }; + let connected = variants.iter().any(|variant| variant.name == "Connected"); + let definitions = [ + if connected { + ("Connected", Type::Socket, "socket") + } else { + ("Chunk", Type::Bytes, "bytes") + }, + ("Failed", Type::Text, "text"), + ] + .map(|(label, ty, kind)| { + let variant = variants + .iter() + .position(|value| value.name == label) + .expect("checked read event variant"); + (label, ty, kind, variant) + }); + for (label, ty, kind, variant) in definitions { + let instance = self + .instances + .iter() + .next() + .expect("reachable entry") + .clone(); + let mut emitter = FunctionEmitter::new(self, &instance); + emitter.start("entry"); + let retained = emitter.temp(); + emitter.line(format!( + "{retained} = call ptr @dever_rt_v1_{kind}_retain(ptr %handle)" + )); + emitter.write_variant(id, variant, &[(ty, retained)], "%out"); + emitter.line("ret void"); + let callback = format!( + "define internal void @dever_read_{label}_{id}(ptr %handle, ptr %out) {{\nentry:\n{} }}\n", + emitter.body + ); + self.declarations.push_str(&callback); + } + let success = if connected { "Connected" } else { "Chunk" }; + writeln!(self.declarations, + "{name} = private constant {{ ptr, ptr, ptr }} {{ ptr @dever_type_{}, ptr @dever_read_{success}_{id}, ptr @dever_read_Failed_{id} }}", + self.type_index(&Type::Named(id))).unwrap(); + name + } +} + +impl FunctionEmitter<'_, '_> { + pub(super) fn file_result( + &mut self, + operation: Intrinsic, + mut arguments: Vec<String>, + result: &Expression, + ) -> String { + use Intrinsic::*; + let (symbol, success, end) = match operation { + FileOpen => ("file_open", "Opened", None), + FileCreate => ("file_create", "Opened", None), + FileRead => ("file_read", "Read", Some("End")), + FileWrite => ("file_write", "Done", None), + FileClose => ("file_close", "Closed", None), + FileChunks => { + let Type::Named(id) = result.ty else { + unreachable!("checked chunks choice") + }; + let Shape::Choice(variants) = &self.module.program.types[id].shape else { + unreachable!("checked chunks choice") + }; + let variant = variants + .iter() + .find(|value| value.name == "Streaming") + .expect("checked Streaming variant"); + let Type::Stream(element) = &variant.fields[0].ty else { + unreachable!("checked chunks stream") + }; + let Type::Named(event) = element.as_ref() else { + unreachable!("checked read event identity") + }; + let descriptor = self.module.read_event(*event); + arguments.push(format!("ptr {descriptor}")); + ("file_chunks", "Streaming", None) + } + _ => unreachable!("checked File intrinsic"), + }; + self.recoverable_call( + &format!("dever_rt_v1_{symbol}"), + arguments, + result, + success, + end, + ) + } + + /// Only runtime status 1 is recoverable. Invalid ABI input never becomes a + /// user business error; an absent row never reads the uninitialized out slot. + pub(super) fn recoverable_call( + &mut self, + symbol: &str, + mut arguments: Vec<String>, + result: &Expression, + success_name: &str, + end_name: Option<&str>, + ) -> String { + let Type::Named(id) = result.ty else { + unreachable!("checked resource choice") + }; + let Shape::Choice(variants) = &self.module.program.types[id].shape else { + unreachable!("checked resource choice") + }; + let success = variants + .iter() + .position(|value| value.name == success_name) + .expect("checked resource success"); + let value_type = variants[success] + .fields + .first() + .map(|field| field.ty.clone()); + let output = match &value_type { + Some(ty) => self.entry_slot(ty), + None => self.entry_slot_ir("i8"), + }; + arguments.push(format!("ptr {output}")); + let present = end_name.map(|_| { + let slot = self.entry_slot_ir("i8"); + arguments.push(format!("ptr {slot}")); + slot + }); + let error = self.entry_slot_ir("{ ptr, i64 }"); + arguments.push(format!("ptr {error}")); + let status = self.temp(); + self.line(format!( + "{status} = call i32 @{symbol}({})", + arguments.join(", ") + )); + self.resource_row( + result, + success_name, + end_name, + ResourceRow { + status, + output, + present, + error, + protocol_output: false, + }, + ) + } + + pub(super) fn resource_row( + &mut self, + result: &Expression, + success_name: &str, + end_name: Option<&str>, + row: ResourceRow, + ) -> String { + let ResourceRow { + status, + output, + present, + error, + protocol_output, + } = row; + let Type::Named(id) = result.ty else { + unreachable!("checked resource choice") + }; + let Shape::Choice(variants) = &self.module.program.types[id].shape else { + unreachable!() + }; + let success = variants + .iter() + .position(|variant| variant.name == success_name) + .expect("checked resource success"); + let failed = variants + .iter() + .position(|variant| variant.name == "Failed") + .expect("checked resource failure"); + let end = end_name.map(|name| { + variants + .iter() + .position(|variant| variant.name == name) + .expect("checked resource end") + }); + let value_type = variants[success] + .fields + .first() + .map(|field| field.ty.clone()); + let choice = self.entry_slot(&result.ty); + let success_block = self.label("resource_success"); + let failure_block = self.label("resource_failed"); + let done = self.label("resource_done"); + let ok = self.temp(); + self.line(format!("{ok} = icmp eq i32 {status}, 0")); + let not_ok = self.label("resource_status"); + self.line(format!( + "br i1 {ok}, label %{success_block}, label %{not_ok}" + )); + self.start(¬_ok); + let business = self.temp(); + self.line(format!("{business} = icmp eq i32 {status}, 1")); + let abi_fault = self.label("resource_abi_fault"); + self.line(format!( + "br i1 {business}, label %{failure_block}, label %{abi_fault}" + )); + self.start(&abi_fault); + self.check_runtime_status(&status, &error, result.span); + self.line("unreachable"); + self.start(&success_block); + if let (Some(end), Some(present)) = (end, present) { + let flag = self.temp(); + self.line(format!("{flag} = load i8, ptr {present}")); + let has_value = self.temp(); + self.line(format!("{has_value} = icmp ne i8 {flag}, 0")); + let available = self.label("resource_present"); + let absent = self.label("resource_end"); + self.line(format!( + "br i1 {has_value}, label %{available}, label %{absent}" + )); + self.start(&absent); + self.write_variant(id, end, &[], &choice); + self.line(format!("br label %{done}")); + self.start(&available); + } + let fields = value_type + .map(|ty| { + let value = self.temp(); + self.line(format!( + "{value} = load {}, ptr {output}", + if protocol_output { + "ptr".into() + } else { + self.module.ty(&ty) + } + )); + let value = if protocol_output { + self.protocol_to_language(&ty, &value, result.span) + } else { + value + }; + (ty, value) + }) + .into_iter() + .collect::<Vec<_>>(); + self.write_variant(id, success, &fields, &choice); + self.line(format!("br label %{done}")); + self.start(&failure_block); + let text = self.error_text(&error, result.span); + self.write_variant(id, failed, &[(Type::Text, text)], &choice); + self.line(format!("br label %{done}")); + self.start(&done); + let value = self.temp(); + self.line(format!("{value} = load %T{id}, ptr {choice}")); + value + } + + pub(super) fn error_text(&mut self, error: &str, span: Span) -> String { + let buffer = self.temp(); + self.line(format!("{buffer} = load {{ ptr, i64 }}, ptr {error}")); + let bytes = self.temp(); + self.line(format!("{bytes} = extractvalue {{ ptr, i64 }} {buffer}, 0")); + let length = self.temp(); + self.line(format!( + "{length} = extractvalue {{ ptr, i64 }} {buffer}, 1" + )); + let message = self.entry_slot(&Type::Text); + let conversion_error = self.entry_slot_ir("{ ptr, i64 }"); + let converted = self.temp(); + self.line(format!("{converted} = call i32 @dever_rt_v1_text_new(ptr {bytes}, i64 {length}, ptr {message}, ptr {conversion_error})")); + self.line(format!( + "call void @dever_rt_v1_buffer_free(ptr {bytes}, i64 {length})" + )); + self.check_runtime_status(&converted, &conversion_error, span); + let text = self.temp(); + self.line(format!("{text} = load ptr, ptr {message}")); + text + } +} diff --git a/crates/dever-core/src/llvm/rest.rs b/crates/dever-core/src/llvm/rest.rs new file mode 100644 index 0000000..1818ea2 --- /dev/null +++ b/crates/dever-core/src/llvm/rest.rs @@ -0,0 +1,931 @@ +//! REST adapters reuse typed Model query, bind, row, and transaction lowering. +use super::*; +use crate::hir::{ApiRest, ModelCondition, ModelOperation, ModelOrder, QueryPlan}; +use crate::model::{DatabaseOwner, ModelFieldType, RestOwnerSource, RestValueSource}; +use crate::native::orm as sql; + +#[derive(Clone)] +pub(super) struct Handler { + pub function: usize, + pub route: usize, + pub method: &'static str, + pub detail: bool, + pub inputs: Vec<Input>, +} + +#[derive(Clone)] +pub(super) struct Input { + pub name: String, + pub ty: Type, + pub field: Option<usize>, +} + +pub(super) fn prepare(program: &mut Program) -> (Vec<Handler>, Option<usize>) { + if program.api_rest.is_empty() { + return (Vec::new(), None); + } + let span = program.api_rest[0].span; + let properties = program + .types + .iter() + .find(|definition| definition.kind == DefinitionKind::ModelId) + .expect("REST Model id properties") + .properties + .clone(); + let error = program.types.len(); + program.types.push(crate::types::Definition { + name: "<generated REST Error>".into(), + owner: 0, + public: false, + span, + shape: Shape::Choice( + ["Invalid", "Forbidden", "NotFound"] + .iter() + .map(|name| crate::types::Variant { + name: (*name).into(), + fields: if *name == "Invalid" { + vec![crate::types::Field { + name: "message".into(), + ty: Type::Text, + private: false, + bounds: Vec::new(), + }] + } else { + Vec::new() + }, + error: true, + label: None, + }) + .collect(), + ), + properties, + kind: DefinitionKind::Regular, + }); + let mut handlers = Vec::new(); + for (route_index, route) in program.api_rest.clone().iter().enumerate() { + let model = &program.models[route.model]; + let Shape::Record(fields) = program.types[model.record].shape.clone() else { + unreachable!(); + }; + for (method, detail) in [ + ("GET", false), + ("POST", false), + ("GET", true), + ("PUT", true), + ("DELETE", true), + ] { + let mut inputs = Vec::new(); + if detail { + inputs.push(Input { + name: "id".into(), + ty: Type::Int, + field: None, + }); + } + if method == "GET" && !detail { + for name in ["page", "size"] { + inputs.push(Input { + name: name.into(), + ty: Type::Nullable(Box::new(Type::Int)), + field: None, + }); + } + if let Some(search) = &route.contract.search { + inputs.push(Input { + name: fields[search.field].name.clone(), + ty: fields[search.field].ty.clone().nullable(), + field: Some(search.field), + }); + } + } + let writes = match method { + "POST" => &route.contract.create[..], + "PUT" => &route.contract.replace[..], + _ => &[], + }; + for write in writes { + if matches!( + write.source, + RestValueSource::Input + | RestValueSource::Binding { + uses_input: true, + .. + } + ) { + inputs.push(Input { + name: fields[write.field].name.clone(), + ty: fields[write.field].ty.clone(), + field: Some(write.field), + }); + } + } + let mut function = application::generated_function( + format!("<generated REST {route_index} {method} {detail}>"), + route.span, + ); + function.parameters = inputs + .iter() + .map(|input| Parameter::Value(input.ty.clone())) + .collect(); + function.outputs = vec![crate::types::Field { + name: "encoded".into(), + ty: Type::Text, + private: false, + bounds: Vec::new(), + }]; + let bindings = writes + .iter() + .filter_map(|write| match &write.source { + RestValueSource::Binding { expression, .. } => Some(expression.clone()), + _ => None, + }) + .chain( + route + .contract + .search + .iter() + .map(|search| search.expression.clone()), + ) + .collect::<Vec<_>>(); + // Metadata-only body preserves every actual expression dependency; + // emit_rest_body owns the compiler-generated orchestration. + function.clauses = vec![crate::hir::Clause { + patterns: Vec::new(), + bindings: Vec::new(), + locals: fields.iter().map(|field| field.ty.clone()).collect(), + outputs: Vec::new(), + body: bindings.into_iter().map(Statement::Call).collect(), + terminates: false, + }]; + let id = application::append_generated(program, function); + program.database_effects[id].insert(DatabaseOwner::Model(route.model)); + program.failures[id].extend(crate::capture::database_errors(&program.types)); + program.failures[id] + .extend((0..3).map(|variant| crate::hir::Failure { ty: error, variant })); + let instance = Specialization { + function: id, + handlers: Vec::new(), + }; + let reachable = specialize::execution_reachable(&program.functions, [instance]) + .expect("checked REST binding call graph"); + for root in reachable { + let failures = specialize::failures(program, &root); + program.failures[id].extend(failures); + } + let mut direct = BTreeSet::new(); + crate::check::visit(&program.functions[id], |expression| { + if let ExpressionKind::Fail(error) = &expression.kind + && let Type::Named(ty) = error.ty + && let ExpressionKind::Variant { variant, .. } = error.kind + { + direct.insert(crate::hir::Failure { ty, variant }); + } + }); + program.failures[id].extend(direct); + handlers.push(Handler { + function: id, + route: route_index, + method, + detail, + inputs, + }); + } + } + (handlers, Some(error)) +} + +impl Module<'_> { + pub(super) fn rest_fault_message(&mut self) { + let Some(id) = self + .application + .as_ref() + .and_then(|application| application.rest_error) + else { + return; + }; + writeln!(self.declarations, "define internal ptr @dever_rest_fault_message(ptr %fault) {{\nentry:\n %code_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 0\n %code = load i32, ptr %code_ptr\n %business = icmp eq i32 %code, 4\n %type_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 5\n %type = load i32, ptr %type_ptr\n %matches = icmp eq i32 %type, {}\n %variant_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 6\n %variant = load i32, ptr %variant_ptr\n %invalid = icmp eq i32 %variant, 0\n %nominal = and i1 %business, %matches\n %selected = and i1 %nominal, %invalid\n br i1 %selected, label %message, label %absent\nmessage:\n %payload = getelementptr %dever.fault, ptr %fault, i32 0, i32 7\n %field = getelementptr %T{id}, ptr %payload, i32 0, i32 1\n %text = load ptr, ptr %field\n ret ptr %text\nabsent:\n ret ptr null\n}}", id + 1).unwrap(); + } + pub(super) fn emit_rest_body( + &mut self, + instance: &Specialization, + handler: &Handler, + ) -> Result<(), String> { + let route = self.program.api_rest[handler.route].clone(); + let function = self.program.functions[instance.function].clone(); + let mut emitter = FunctionEmitter::new(self, instance); + let parameters = emitter.begin_generated_function(&function); + let encoded = emitter.rest_response(handler, &route)?; + let owner = emitter.own_value(&Type::Text, &encoded, false); + if handler.method != "GET" { + emitter.commit_command_transaction(); + } + emitter.line(format!("store ptr {encoded}, ptr %out")); + emitter.disarm(owner); + emitter.exit("0"); + let name = emitter.module.names[instance].clone(); + let body = emitter.finish_function(&name, ¶meters); + self.functions.push_str(&body); + Ok(()) + } + + pub(super) fn rest_handler(&mut self, index: usize, handler: &Handler) -> String { + let instance = Specialization { + function: handler.function, + handlers: Vec::new(), + }; + let input = asynchronous::input_type(&self.program.functions[handler.function]); + let decoded_inputs = &handler.inputs[usize::from(handler.detail)..]; + let decoded_type = Type::Outputs( + decoded_inputs + .iter() + .enumerate() + .map(|(index, input)| crate::types::Field { + name: format!("p{index}"), + ty: input.ty.clone(), + private: false, + bounds: Vec::new(), + }) + .collect(), + ); + let list = handler.method == "GET" && !handler.detail; + let fields = decoded_inputs + .iter() + .enumerate() + .map(|(index, input)| { + let decoded = if list && index == 2 { + let Type::Nullable(inner) = &input.ty else { + unreachable!() + }; + inner.as_ref() + } else { + &input.ty + }; + ( + input.name.clone(), + input.ty.clone(), + self.wire_callback(decoded, crate::wire::Policy::ApiInput, false), + ) + }) + .collect::<Vec<_>>(); + let mut emitter = FunctionEmitter::new(self, &instance); + emitter.asynchronous = false; + emitter.protocol_callback = true; + emitter.start("entry"); + let value = emitter.decode_fields( + &decoded_type, + &fields, + "%inputs", + if list { + wire::FieldInput::RestList + } else { + wire::FieldInput::Rest + }, + ); + let owner = emitter.own_value(&decoded_type, &value, false); + let mut row = "zeroinitializer".to_owned(); + if handler.detail { + let next = emitter.temp(); + emitter.line(format!( + "{next} = insertvalue {} {row}, i64 %detail_id, 0", + emitter.module.ty(&input) + )); + row = next; + } + for (index, field) in decoded_inputs.iter().enumerate() { + let child = emitter.temp(); + emitter.line(format!( + "{child} = extractvalue {} {value}, {index}", + emitter.module.ty(&decoded_type) + )); + if let Some(field_index) = field.field { + let model = &emitter.module.program.models + [emitter.module.program.api_rest[handler.route].model]; + let storage = model.fields[field_index].ty.clone(); + emitter.rest_text_bounds(&field.name, &field.ty, &child, &storage, false); + } + let next = emitter.temp(); + emitter.line(format!( + "{next} = insertvalue {} {row}, {} {child}, {}", + emitter.module.ty(&input), + emitter.module.ty(&field.ty), + index + usize::from(handler.detail) + )); + row = next; + } + emitter.line(format!( + "store {} {row}, ptr %out", + emitter.module.ty(&input) + )); + emitter.disarm(owner); + let callback = format!("dever_rest_decode_{index}"); + let body = emitter.finish_protocol_callback( + &callback, + "ptr %inputs, i64 %detail_id, ptr %out, ptr %error", + ); + self.declarations.push_str(&body); + let name = &self.names[&instance]; + let descriptor = format!("@dever_rest_handler_{index}"); + writeln!(self.declarations, "{descriptor} = private constant %dever.api_handler {{ ptr @dever_async_{name}, ptr null, ptr @dever_owned_{}, ptr @{callback}, ptr @dever_rest_fault_status, ptr @dever_rest_fault_message }}", self.type_index(&input)).unwrap(); + descriptor + } + + pub(super) fn rest_encoder(&mut self, route_index: usize, page: bool) -> String { + let name = format!("dever_rest_encode_{route_index}_{}", u8::from(page)); + if !self.rest_encoders.insert((route_index, page)) { + return name; + } + let route = self.program.api_rest[route_index].clone(); + let model = self.program.models[route.model].clone(); + let id = if page { model.page } else { model.record }; + let Shape::Record(fields) = self.program.types[id].shape.clone() else { + unreachable!(); + }; + let selected = if page { + (0..4).collect::<Vec<_>>() + } else { + route.contract.read.clone() + }; + let mut codecs = Vec::new(); + for field in &selected { + codecs.push(if page && *field == 0 { + self.rest_encoder(route_index, false) + } else { + self.wire_callback(&fields[*field].ty, crate::wire::Policy::Output, true) + }); + } + let instance = self.instances.first().unwrap().clone(); + let mut emitter = FunctionEmitter::new(self, &instance); + emitter.asynchronous = false; + emitter.protocol_callback = true; + emitter.start("entry"); + let value = emitter.temp(); + emitter.line(format!("{value} = load %T{id}, ptr %input")); + emitter.wire_write( + "dever_rt_v1_wire_encoder_begin_object", + vec!["ptr %encoder".into()], + ); + for (index, codec) in selected.iter().zip(codecs) { + let key = emitter.module.wire_names(&[fields[*index].name.clone()]); + emitter.wire_write( + "dever_rt_v1_wire_encoder_key", + vec!["ptr %encoder".into(), format!("ptr {key}")], + ); + let child = emitter.temp(); + emitter.line(format!("{child} = extractvalue %T{id} {value}, {index}")); + if page && *index == 0 { + emitter.encode_list_callback(&Type::Named(model.record), &codec, &child); + } else { + let row = emitter.row_pointer(&fields[*index].ty, &child); + emitter.wire_write(&codec, vec![format!("ptr {row}"), "ptr %encoder".into()]); + } + } + emitter.wire_write("dever_rt_v1_wire_encoder_end", vec!["ptr %encoder".into()]); + let body = emitter.finish_protocol_callback(&name, "ptr %input, ptr %encoder, ptr %error"); + self.declarations.push_str(&body); + name + } +} + +impl FunctionEmitter<'_, '_> { + fn rest_response(&mut self, handler: &Handler, route: &ApiRest) -> Result<String, String> { + let span = route.span; + let model = self.module.program.models[route.model].clone(); + let Shape::Record(fields) = self.module.program.types[model.record].shape.clone() else { + unreachable!(); + }; + // Bindings own one compiler-assigned input local. Other decoded values + // live in separate slots so evaluating a binding cannot alias them. + let empty = self.entry_slot(&Type::Unit); + self.locals.push((Type::Unit, empty)); + let mut inputs = Vec::new(); + for (index, input) in handler.inputs.iter().enumerate() { + inputs.push(self.rest_local(&input.ty, &self.parameter(index), span)); + } + let mut writes = self.rest_write_values(handler, route, &inputs)?; + if handler.method == "GET" && !handler.detail && route.contract.search.is_some() { + inputs[2] = self.rest_search_value(route, &inputs[2])?; + } + let owner = if let Some((field, source)) = route.contract.owner { + let symbol = match source { + RestOwnerSource::User => "auth_user_id", + RestOwnerSource::Tenant => "auth_tenant_id", + }; + let optional = self.runtime_call( + &format!("dever_rt_v1_{symbol}"), + Vec::new(), + &Type::Nullable(Box::new(Type::Int)), + true, + None, + span, + ); + let present = self.temp(); + let id = self.temp(); + self.line(format!( + "{present} = extractvalue {{ i1, i64 }} {optional}, 0" + )); + let found = self.label("rest_owner"); + let missing = self.label("rest_owner_missing"); + self.line(format!("br i1 {present}, label %{found}, label %{missing}")); + self.start(&missing); + self.rest_fail(1, span); + self.start(&found); + self.line(format!("{id} = extractvalue {{ i1, i64 }} {optional}, 1")); + Some((field, self.rest_local(&fields[field].ty, &id, span))) + } else { + None + }; + let mut condition = owner + .as_ref() + .map(|(field, value)| ModelCondition::Compare { + field: *field, + operator: BinaryOperator::Equal, + value: value.clone(), + }); + if handler.detail { + let id = ModelCondition::Compare { + field: 0, + operator: BinaryOperator::Equal, + value: inputs[0].clone(), + }; + condition = Some(match condition { + Some(owner) => ModelCondition::And(Box::new(id), Box::new(owner)), + None => id, + }); + } + let constant = |value| Expression { + kind: ExpressionKind::Constant(Constant::Int(value)), + ty: Type::Int, + span, + }; + let query = QueryPlan { + condition, + order: vec![ModelOrder { + field: 0, + descending: true, + }], + relations: Vec::new(), + after: None, + page: constant(1), + size: constant(20), + }; + if handler.method == "POST" + && let Some(owner) = owner + { + writes.push(owner); + } + if handler.method != "GET" { + self.begin_transaction_scope(self.instance, &Type::Unit, span); + } + let model_value = if handler.method == "GET" && !handler.detail { + ( + Type::Named(model.page), + self.rest_list(route, &inputs, query)?, + ) + } else if handler.method == "DELETE" { + let result = Expression { + ty: Type::Int, + span, + kind: ExpressionKind::Constant(Constant::Null), + }; + let affected = + self.model_operation(route.model, &ModelOperation::Delete { query }, &result)?; + self.rest_require_affected(&affected, span); + // The wrapper owns the encoded response across commit and transfers + // it to its caller. Do not register a second owner for this Text. + let (pointer, length) = self.module.text_literal("null"); + return Ok(self.runtime_call( + "dever_rt_v1_text_new", + vec![format!("ptr {pointer}"), format!("i64 {length}")], + &Type::Text, + false, + None, + span, + )); + } else if handler.method == "POST" { + let result = Expression { + ty: Type::Named(model.record), + span, + kind: ExpressionKind::Constant(Constant::Null), + }; + ( + result.ty.clone(), + self.model_operation( + route.model, + &ModelOperation::Create { values: writes }, + &result, + )?, + ) + } else { + let (statement, values) = if handler.method == "PUT" { + let mut expressions = writes + .iter() + .map(|(field, expression)| (expression, &model.fields[*field].ty)) + .collect::<Vec<_>>(); + let assignments = + sql::update_assignments(&model, writes.iter().map(|(field, _)| *field), 1); + let condition = + sql::condition_sql(&model, query.condition.as_ref(), &mut expressions); + let values = self.model_bind(&expressions)?; + ( + self.module.planned_sql(&format!( + "UPDATE {} SET {assignments}{condition} RETURNING {}", + sql::quoted(&model.table), + sql::select_columns(&model) + )), + values, + ) + } else { + self.model_select(&model, &query, " LIMIT 1")? + }; + let rows = self.model_query(route.model, &statement, &values, "0", span); + let value = self.model_first( + &rows, + &model, + None, + Some((7, "REST record was not found".into())), + span, + ); + (Type::Named(model.record), value) + }; + self.own_value(&model_value.0, &model_value.1, false); + let row = self.row_pointer(&model_value.0, &model_value.1); + let encoder = self + .module + .rest_encoder(handler.route, handler.method == "GET" && !handler.detail); + Ok(self.runtime_call( + "dever_rt_v1_wire_encode", + vec![ + format!("ptr {row}"), + format!("ptr @dever_type_{}", self.module.type_index(&model_value.0)), + format!("ptr @{encoder}"), + ], + &Type::Text, + false, + None, + span, + )) + } + + fn rest_write_values( + &mut self, + handler: &Handler, + route: &ApiRest, + inputs: &[Expression], + ) -> Result<Vec<(usize, Expression)>, String> { + let model = self.module.program.models[route.model].clone(); + let Shape::Record(fields) = self.module.program.types[model.record].shape.clone() else { + unreachable!(); + }; + let write_fields = match handler.method { + "POST" => &route.contract.create[..], + "PUT" => &route.contract.replace[..], + _ => &[], + }; + let mut writes = Vec::new(); + for write in write_fields { + let input = handler + .inputs + .iter() + .position(|input| input.field == Some(write.field)) + .map(|index| &inputs[index]); + let value = match &write.source { + RestValueSource::Input => input.unwrap().clone(), + RestValueSource::Binding { expression, .. } => { + self.rest_binding(expression, input)? + } + }; + if let RestValueSource::Binding { uses_input, .. } = write.source { + let rendered = self.expression(&value)?; + self.rest_text_bounds( + &fields[write.field].name, + &value.ty, + &rendered, + &model.fields[write.field].ty, + uses_input, + ); + } + writes.push((write.field, value)); + } + Ok(writes) + } + + fn rest_search_value( + &mut self, + route: &ApiRest, + input: &Expression, + ) -> Result<Expression, String> { + let search = route.contract.search.as_ref().unwrap(); + let model = self.module.program.models[route.model].clone(); + let Shape::Record(fields) = self.module.program.types[model.record].shape.clone() else { + unreachable!(); + }; + let field = &fields[search.field]; + let value = self.expression(input)?; + let slot = self.entry_slot(&input.ty); + self.line(format!( + "store {} zeroinitializer, ptr {slot}", + self.module.ty(&input.ty) + )); + let present = self.temp(); + self.line(format!( + "{present} = extractvalue {} {value}, 0", + self.module.ty(&input.ty) + )); + let bind = self.label("rest_search_binding"); + let done = self.label("rest_search_bound"); + self.line(format!("br i1 {present}, label %{bind}, label %{done}")); + self.start(&bind); + let payload = self.temp(); + self.line(format!( + "{payload} = extractvalue {} {value}, 1", + self.module.ty(&input.ty) + )); + let local = self.rest_local(&field.ty, &payload, route.span); + let value = self.rest_binding(&search.expression, Some(&local))?; + let rendered = self.expression(&value)?; + self.rest_text_bounds( + &field.name, + &field.ty, + &rendered, + &model.fields[search.field].ty, + true, + ); + let flag = self.temp(); + let row = self.temp(); + self.line(format!( + "{flag} = insertvalue {} zeroinitializer, i1 true, 0", + self.module.ty(&input.ty) + )); + self.line(format!( + "{row} = insertvalue {} {flag}, {} {rendered}, 1", + self.module.ty(&input.ty), + self.module.ty(&field.ty) + )); + self.line(format!( + "store {} {row}, ptr {slot}", + self.module.ty(&input.ty) + )); + self.line(format!("br label %{done}")); + self.start(&done); + let value = self.temp(); + self.line(format!( + "{value} = load {}, ptr {slot}", + self.module.ty(&input.ty) + )); + let owner = self.own_value(&input.ty, &value, true); + let value = self.load_owned_value(&input.ty, owner); + Ok(self.rest_local(&input.ty, &value, route.span)) + } + + fn rest_list( + &mut self, + route: &ApiRest, + inputs: &[Expression], + mut query: QueryPlan, + ) -> Result<String, String> { + let span = route.span; + let model = self.module.program.models[route.model].clone(); + let Shape::Record(fields) = self.module.program.types[model.record].shape.clone() else { + unreachable!(); + }; + query.page = self.rest_default_int(&inputs[0], 1)?; + query.size = self.rest_default_int(&inputs[1], 20)?; + let maximum = self.model_maximum(route.model, span); + let page = self.expression(&query.page)?; + let size = self.expression(&query.size)?; + let p_slot = self.entry_slot(&Type::Int); + let s_slot = self.entry_slot(&Type::Int); + let offset = self.entry_slot(&Type::Int); + let error = self.entry_slot_ir("{ ptr, i64 }"); + let status = self.temp(); + self.line(format!("{status} = call i32 @dever_rt_v1_api_pagination(i64 {page}, i64 {size}, i64 {maximum}, ptr {p_slot}, ptr {s_slot}, ptr {offset}, ptr {error})")); + self.rest_input_status(&status, &error); + let result = Expression { + ty: Type::Named(model.page), + span, + kind: ExpressionKind::Constant(Constant::Null), + }; + let Some(search) = &route.contract.search else { + return self.model_page(route.model, &model, &query, &result); + }; + let input = &inputs[2]; + let value = self.expression(input)?; + let present = self.temp(); + self.line(format!( + "{present} = extractvalue {} {value}, 0", + self.module.ty(&input.ty) + )); + let filtered = self.label("rest_search"); + let all = self.label("rest_unfiltered"); + let done = self.label("rest_list_done"); + let slot = self.entry_slot(&result.ty); + self.line(format!("br i1 {present}, label %{filtered}, label %{all}")); + self.start(&filtered); + let input_value = self.temp(); + self.line(format!( + "{input_value} = extractvalue {} {value}, 1", + self.module.ty(&input.ty) + )); + let value = self.rest_local(&fields[search.field].ty, &input_value, span); + let filter = ModelCondition::Compare { + field: search.field, + operator: BinaryOperator::Equal, + value, + }; + let mut filtered_query = query.clone(); + filtered_query.condition = Some(match query.condition.clone() { + Some(owner) => ModelCondition::And(Box::new(owner), Box::new(filter)), + None => filter, + }); + let value = self.model_page(route.model, &model, &filtered_query, &result)?; + self.line(format!( + "store {} {value}, ptr {slot}", + self.module.ty(&result.ty) + )); + self.line(format!("br label %{done}")); + self.start(&all); + let value = self.model_page(route.model, &model, &query, &result)?; + self.line(format!( + "store {} {value}, ptr {slot}", + self.module.ty(&result.ty) + )); + self.line(format!("br label %{done}")); + self.start(&done); + let value = self.temp(); + self.line(format!( + "{value} = load {}, ptr {slot}", + self.module.ty(&result.ty) + )); + Ok(value) + } + + fn rest_default_int( + &mut self, + expression: &Expression, + default: i64, + ) -> Result<Expression, String> { + let value = self.expression(expression)?; + let present = self.temp(); + let number = self.temp(); + let selected = self.temp(); + self.line(format!("{present} = extractvalue {{ i1, i64 }} {value}, 0")); + self.line(format!("{number} = extractvalue {{ i1, i64 }} {value}, 1")); + self.line(format!( + "{selected} = select i1 {present}, i64 {number}, i64 {default}" + )); + Ok(self.rest_local(&Type::Int, &selected, expression.span)) + } + + fn rest_require_affected(&mut self, affected: &str, span: Span) { + let found = self.temp(); + let success = self.label("rest_deleted"); + let missing = self.label("rest_not_found"); + self.line(format!("{found} = icmp sgt i64 {affected}, 0")); + self.line(format!("br i1 {found}, label %{success}, label %{missing}")); + self.start(&missing); + self.rest_fail(2, span); + self.start(&success); + } + + fn rest_fail(&mut self, variant: usize, span: Span) { + let id = self + .module + .application + .as_ref() + .unwrap() + .rest_error + .unwrap(); + self.fail( + &Expression { + ty: Type::Named(id), + span, + kind: ExpressionKind::Variant { + variant, + arguments: Vec::new(), + }, + }, + span, + ) + .unwrap(); + self.line("unreachable"); + } + + fn rest_local(&mut self, ty: &Type, value: &str, span: Span) -> Expression { + let slot = self.row_pointer(ty, value); + let index = self.locals.len(); + self.locals.push((ty.clone(), slot)); + Expression { + ty: ty.clone(), + kind: ExpressionKind::Local(index), + span, + } + } + + fn rest_binding( + &mut self, + expression: &Expression, + input: Option<&Expression>, + ) -> Result<Expression, String> { + let previous = self.locals.first().cloned(); + if let Some(input) = input { + let value = self.expression(input)?; + let row = self.row_pointer(&input.ty, &value); + self.locals[0] = (input.ty.clone(), row); + } + let value = self.expression(expression)?; + if let Some(previous) = previous { + self.locals[0] = previous; + } + Ok(self.rest_local(&expression.ty, &value, expression.span)) + } + + fn rest_text_bounds( + &mut self, + name: &str, + ty: &Type, + value: &str, + storage: &ModelFieldType, + source_fault: bool, + ) { + let ModelFieldType::Text { minimum, maximum } = storage else { + return; + }; + let name = self.module.wire_names(&[name.into()]); + let text = if matches!(ty, Type::Nullable(_)) { + self.nullable_handle(ty, value) + } else { + value.into() + }; + let error = self.entry_slot_ir("{ ptr, i64 }"); + let status = self.temp(); + self.line(format!("{status} = call i32 @dever_rt_v1_api_text_bounds(ptr {name}, ptr {text}, i64 {minimum}, i64 {}, i8 {}, ptr {error})", maximum.unwrap_or(0), u8::from(maximum.is_some()))); + if source_fault { + self.rest_input_status(&status, &error); + } else { + self.check_runtime_status( + &status, + &error, + self.module.program.functions[self.instance.function].span, + ); + } + } + + fn rest_input_status(&mut self, status: &str, error: &str) { + let failed = self.temp(); + let invalid = self.label("rest_invalid"); + let done = self.label("rest_valid"); + self.line(format!("{failed} = icmp eq i32 {status}, 1")); + self.line(format!("br i1 {failed}, label %{invalid}, label %{done}")); + self.start(&invalid); + let pointer = self.temp(); + let length = self.temp(); + let buffer = self.temp(); + self.line(format!("{buffer} = load {{ ptr, i64 }}, ptr {error}")); + self.line(format!( + "{pointer} = extractvalue {{ ptr, i64 }} {buffer}, 0" + )); + self.line(format!( + "{length} = extractvalue {{ ptr, i64 }} {buffer}, 1" + )); + let span = self.module.program.functions[self.instance.function].span; + let text = self.runtime_call( + "dever_rt_v1_text_new", + vec![format!("ptr {pointer}"), format!("i64 {length}")], + &Type::Text, + false, + None, + span, + ); + self.own_value(&Type::Text, &text, false); + self.line(format!( + "call void @dever_rt_v1_buffer_free(ptr {pointer}, i64 {length})" + )); + let message = self.rest_local(&Type::Text, &text, span); + let id = self + .module + .application + .as_ref() + .unwrap() + .rest_error + .unwrap(); + self.fail( + &Expression { + ty: Type::Named(id), + span, + kind: ExpressionKind::Variant { + variant: 0, + arguments: vec![message], + }, + }, + span, + ) + .unwrap(); + self.line("unreachable"); + self.start(&done); + self.check_runtime_status( + status, + error, + self.module.program.functions[self.instance.function].span, + ); + } +} diff --git a/crates/dever-core/src/llvm/system.rs b/crates/dever-core/src/llvm/system.rs new file mode 100644 index 0000000..c806e29 --- /dev/null +++ b/crates/dever-core/src/llvm/system.rs @@ -0,0 +1,154 @@ +//! Non-application system capabilities keep their native runtime contracts. +use super::*; + +pub(super) const DECLARATIONS: &str = "\ +declare i32 @dever_rt_v1_process_arguments(ptr, ptr, ptr) +declare i32 @dever_rt_v1_time_unix_millis(ptr, ptr) +declare i32 @dever_rt_v1_time_now(ptr, ptr) +declare i32 @dever_rt_v1_time_monotonic_nanos(ptr, ptr) +declare i32 @dever_rt_v1_time_parse_datetime(ptr, ptr, ptr) +declare i32 @dever_rt_v1_time_parse_date(ptr, ptr, ptr) +declare i32 @dever_rt_v1_time_parse_time(ptr, ptr, ptr) +declare i32 @dever_rt_v1_time_format_datetime(i64, ptr, ptr) +declare i32 @dever_rt_v1_time_format_date(i64, ptr, ptr) +declare i32 @dever_rt_v1_time_format_time(i64, ptr, ptr) +declare i32 @dever_rt_v1_time_add(i64, i64, ptr, ptr) +declare i32 @dever_rt_v1_time_subtract(i64, i64, ptr, ptr) +declare i32 @dever_rt_v1_time_difference(i64, i64, ptr, ptr) +declare i32 @dever_rt_v1_stdout_write(ptr, ptr, ptr) +declare i32 @dever_rt_v1_log_write(i32, ptr, ptr, ptr, ptr) +declare void @dever_rt_v1_log_flush() +declare i32 @dever_rt_v1_uuid_parse(ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_uuid_to_text(ptr, ptr, ptr) +declare i8 @dever_rt_v1_uuid_equal(ptr, ptr) +declare i32 @dever_rt_v1_secret_from_text(ptr, ptr, ptr) +declare i32 @dever_rt_v1_crypto_token(i64, ptr, ptr) +declare i32 @dever_rt_v1_crypto_password_hash(ptr, ptr, ptr) +declare i32 @dever_rt_v1_crypto_password_verify(ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_crypto_sha256(ptr, ptr, ptr) +declare i32 @dever_rt_v1_crypto_hmac_sha256(ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_crypto_constant_time_eq(ptr, ptr, ptr, ptr) +"; + +pub(super) fn supported(operation: Intrinsic) -> bool { + use Intrinsic::*; + matches!( + operation, + ProcessArguments + | TimeUnixMillis + | TimeNow + | TimeMonotonicNanos + | TimeParseDateTime + | TimeParseDate + | TimeParseTime + | TimeFormatDateTime + | TimeFormatDate + | TimeFormatTime + | TimeDuration + | TimeAdd + | TimeSubtract + | TimeDifference + | StdoutWrite + | LogDebug + | LogInfo + | LogWarn + | LogError + | UuidParse + | UuidToText + | TestSecret + | CryptoToken + | CryptoPasswordHash + | CryptoPasswordVerify + | CryptoSha256 + | CryptoHmacSha256 + | CryptoConstantTimeEq + ) +} + +impl FunctionEmitter<'_, '_> { + pub(super) fn uuid_from_literal_text(&mut self, text: &str, span: Span) -> String { + let parsed = self.runtime_call( + "dever_rt_v1_uuid_parse", + vec![format!("ptr {text}")], + &Type::Nullable(Box::new(Type::Uuid)), + true, + None, + span, + ); + // Literal validation already proved this exact spelling parses. + let uuid = self.temp(); + self.line(format!("{uuid} = extractvalue {{ i1, ptr }} {parsed}, 1")); + uuid + } + pub(super) fn system_intrinsic( + &mut self, + operation: Intrinsic, + mut args: Vec<String>, + values: &[String], + result: &Expression, + ) -> String { + use Intrinsic::*; + let (symbol, success) = match operation { + TimeDuration => return values[0].clone(), + ProcessArguments => { + args.push(format!( + "ptr @dever_type_{}", + self.module.type_index(&Type::Text) + )); + ("process_arguments", Some("Read")) + } + TimeUnixMillis => ("time_unix_millis", Some("Read")), + TimeNow => ("time_now", Some("Read")), + TimeMonotonicNanos => ("time_monotonic_nanos", Some("Read")), + TimeParseDateTime => ("time_parse_datetime", Some("Read")), + TimeParseDate => ("time_parse_date", Some("Read")), + TimeParseTime => ("time_parse_time", Some("Read")), + TimeFormatDateTime => ("time_format_datetime", Some("Read")), + TimeFormatDate => ("time_format_date", Some("Read")), + TimeFormatTime => ("time_format_time", Some("Read")), + TimeAdd => ("time_add", Some("Read")), + TimeSubtract => ("time_subtract", Some("Read")), + TimeDifference => ("time_difference", Some("Read")), + CryptoToken => ("crypto_token", Some("Ready")), + CryptoPasswordHash => ("crypto_password_hash", Some("Ready")), + CryptoPasswordVerify => ("crypto_password_verify", Some("Ready")), + CryptoSha256 => ("crypto_sha256", None), + CryptoHmacSha256 => ("crypto_hmac_sha256", None), + CryptoConstantTimeEq => ("crypto_constant_time_eq", None), + TestSecret => ("secret_from_text", None), + UuidParse => ("uuid_parse", None), + UuidToText => ("uuid_to_text", None), + StdoutWrite => ("stdout_write", None), + LogDebug | LogInfo | LogWarn | LogError => { + let level = match operation { + LogDebug => 0, + LogInfo => 1, + LogWarn => 2, + LogError => 3, + _ => unreachable!(), + }; + args.insert(0, format!("i32 {level}")); + ("log_write", None) + } + _ => unreachable!("checked system intrinsic"), + }; + let symbol = format!("dever_rt_v1_{symbol}"); + if let Some(success) = success { + return self.recoverable_call(&symbol, args, result, success, None); + } + let physical = matches!(result.ty, Type::Bool | Type::Unit).then_some("i8"); + let value = self.runtime_call( + &symbol, + args, + &result.ty, + matches!(result.ty, Type::Nullable(_)), + physical, + result.span, + ); + if result.ty == Type::Unit { + "zeroinitializer".into() + } else { + value + } + } +} diff --git a/crates/dever-core/src/llvm/testing.rs b/crates/dever-core/src/llvm/testing.rs new file mode 100644 index 0000000..dbe17f8 --- /dev/null +++ b/crates/dever-core/src/llvm/testing.rs @@ -0,0 +1,276 @@ +//! One compiled suite with independently checked, namespaced case modules. +use super::*; + +pub(super) fn supported(operation: Intrinsic) -> bool { + matches!( + operation, + Intrinsic::TestAssert + | Intrinsic::TestAssertEq + | Intrinsic::TestJobDrain + | Intrinsic::TestClockAdvance + ) +} + +/// Emit the same checked suite with one C ABI main selecting exactly one case. +/// The CLI handles zero discovered cases without compiling a suite. +pub fn emit_test_executable(program: &Program, sources: &SourceMap) -> Result<String, String> { + let mut suite = emit_test_suite(program, sources)?; + process_entry::emit_test_main(&mut suite); + Ok(suite) +} + +/// Emit one suite; the runner starts its executable once per stable case index. +/// Fault storage is opaque, eight-byte aligned, and sized/released with the same +/// index. The rendered diagnostic is an owned runtime Text (text_release). +pub fn emit_test_suite(program: &Program, sources: &SourceMap) -> Result<String, String> { + if program.tests.is_empty() { + return Err("cannot compile an empty test suite".into()); + } + let mut declarations = BTreeSet::new(); + let mut suite = String::new(); + for index in 0..program.tests.len() { + let mut case = crate::check::test_program(program, index).map_err(|errors| { + errors + .iter() + .map(|error| error.render(sources)) + .collect::<Vec<_>>() + .join("\n") + })?; + case.api_routes.clear(); + case.api_rest.clear(); + case.api_commands.clear(); + case.auth.clear(); + case.permissions.clear(); + let ir = application::emit_program(&case, sources, Some(index))?; + for line in ir.lines() { + if line.starts_with("declare ") && !declarations.insert(line.to_owned()) { + continue; + } + writeln!(suite, "{line}").unwrap(); + } + } + writeln!( + suite, + "define i64 @dever_test_count() {{\nentry:\n ret i64 {}\n}}", + program.tests.len() + ) + .unwrap(); + for (name, result, parameters, arguments, invalid) in [ + ( + "entry", + "i32", + "ptr %out, ptr %fault", + "ptr %out, ptr %fault", + "ret i32 2", + ), + ("fault_size", "i64", "", "", "ret i64 0"), + ( + "outputs_release", + "void", + "ptr %out", + "ptr %out", + "ret void", + ), + ( + "fault_release", + "void", + "ptr %fault", + "ptr %fault", + "ret void", + ), + ( + "fault_render", + "i32", + "ptr %fault, ptr %out, ptr %error", + "ptr %fault, ptr %out, ptr %error", + "ret i32 1", + ), + ] { + let extra = if parameters.is_empty() { + String::new() + } else { + format!(", {parameters}") + }; + writeln!(suite, "define {result} @dever_test_{name}(i64 %index{extra}) {{\nentry:\n switch i64 %index, label %invalid [{}]\ninvalid:\n {invalid}", (0..program.tests.len()).map(|index| format!("i64 {index}, label %case{index}")).collect::<Vec<_>>().join(" ")).unwrap(); + for index in 0..program.tests.len() { + let target = match name { + "entry" => "dever_application_entry".to_owned(), + "outputs_release" => "dever_outputs_release".to_owned(), + _ => format!("dever_test_{name}"), + }; + writeln!(suite, "case{index}:").unwrap(); + if result == "void" { + writeln!( + suite, + " call void @case{index}_{target}({arguments})\n ret void" + ) + .unwrap(); + } else { + writeln!(suite, " %result{index} = call {result} @case{index}_{target}({arguments})\n ret {result} %result{index}").unwrap(); + } + } + suite.push_str("}\n"); + } + suite.push_str("define void @dever_test_text_release(ptr %text) {\nentry:\n call void @dever_rt_v1_text_release(ptr %text)\n ret void\n}\n"); + Ok(suite) +} + +impl Module<'_> { + pub(super) fn emit_test_body(&mut self, instance: &Specialization) -> Result<(), String> { + let index = self.application.as_ref().unwrap().test.unwrap(); + let function = self.program.functions[instance.function].clone(); + let target = Specialization { + function: self.program.tests[index].function, + handlers: Vec::new(), + }; + let mut emitter = FunctionEmitter::new(self, instance); + let parameters = emitter.begin_generated_function(&function); + emitter.initialize_clock(function.span); + emitter.initialize_database(function.span, "false"); + emitter.initialize_ports(function.span); + if emitter.module.jobs_enabled() { + let input = emitter.entry_slot(&Type::Unit); + emitter.invoke_system(&target, &input, "0", function.span); + } else { + let (status, _) = emitter.invoke_values(&target, &[], &Type::Unit, function.span); + emitter.propagate_status(&status, function.span, false); + } + emitter.exit("0"); + let name = emitter.module.names[instance].clone(); + let body = emitter.finish_function(&name, ¶meters); + self.functions.push_str(&body); + Ok(()) + } + + pub(super) fn emit_test_exports(&mut self, root: &Specialization) { + let render = self.http_fault_message(root); + writeln!(self.functions, "define i64 @dever_test_fault_size() {{\nentry:\n ret i64 ptrtoint (ptr getelementptr (%dever.fault, ptr null, i32 1) to i64)\n}}\ndefine void @dever_test_fault_release(ptr %fault) {{\nentry:\n call void @dever_fault_release(ptr %fault)\n ret void\n}}\ndefine i32 @dever_test_fault_render(ptr %fault, ptr %out, ptr %error) {{\nentry:\n %status = call i32 {render}(ptr %fault, ptr %out, ptr %error)\n ret i32 %status\n}}").unwrap(); + } +} + +impl FunctionEmitter<'_, '_> { + pub(super) fn test_intrinsic( + &mut self, + operation: Intrinsic, + arguments: &[Expression], + values: &[String], + span: Span, + ) -> String { + if operation == Intrinsic::TestClockAdvance { + let clock = self.temp(); + self.line(format!("{clock} = load ptr, ptr @dever_job_clock")); + self.wire_write_at( + "dever_rt_v1_job_clock_advance", + vec![format!("ptr {clock}"), format!("i64 {}", values[0])], + span, + ); + return "zeroinitializer".into(); + } + if operation == Intrinsic::TestJobDrain { + if self.module.jobs_enabled() { + let session = self.temp(); + self.line(format!("{session} = load ptr, ptr @dever_job_session")); + let operation = self.async_operation( + "job_drain", + vec![format!("ptr {session}"), format!("i64 {}", values[0])], + span, + ); + return self.await_operation(&operation, &Type::Int, span, true); + } + let low = self.compare("icmp sge", &Type::Int, &values[0], "1"); + let high = self.compare("icmp sle", &Type::Int, &values[0], "10000"); + let valid = self.reduce_bool("and", vec![low, high], "1"); + self.application_require( + &valid, + "test Job drain limit must be between 1 and 10000", + span, + ); + return "0".into(); + } + let valid = if operation == Intrinsic::TestAssert { + values[0].clone() + } else { + self.equals(&arguments[0].ty, &values[0], &values[1]) + .expect("checked comparable assertion") + }; + let success = self.label("assertion_passed"); + let failure = self.label("assertion_failed"); + self.line(format!("br i1 {valid}, label %{success}, label %{failure}")); + self.start(&failure); + if operation == Intrinsic::TestAssert { + self.application_fault("assertion failed", span); + } else { + self.display_literal("assert_eq failed: actual = ", span); + let owner = self.guards.len() - 1; + let output = self.guards[owner].pointer.clone(); + self.display_value(&output, &arguments[0].ty, &values[0], span); + self.display_append_literal(&output, ", expected = ", span); + self.display_value(&output, &arguments[1].ty, &values[1], span); + let text = self.temp(); + self.line(format!("{text} = load ptr, ptr {output}")); + self.application_fault_text(&text, span); + } + self.start(&success); + "zeroinitializer".into() + } +} + +/// Qualify only module-defined symbols. Runtime/LLVM declarations and string +/// constants are not names: treating them as text replacement corrupts payloads. +pub(super) fn namespace_module(ir: &str, index: usize) -> String { + let mut globals = BTreeSet::new(); + let mut types = BTreeSet::new(); + for line in ir.lines() { + let line = line.trim_start(); + let (prefix, table) = if line.starts_with('@') || line.starts_with("define ") { + ('@', &mut globals) + } else if line.starts_with('%') && line.contains(" = type ") { + ('%', &mut types) + } else { + continue; + }; + if let Some(start) = line.find(prefix) { + let name = line[start + 1..] + .chars() + .take_while(|character| identifier(*character)) + .collect::<String>(); + table.insert(name); + } + } + let mut output = String::with_capacity(ir.len()); + let mut characters = ir.chars().peekable(); + let mut quoted = false; + let mut comment = false; + while let Some(character) = characters.next() { + if character == '\n' { + comment = false; + } + if !quoted && character == ';' { + comment = true; + } + if !comment && character == '"' { + quoted = !quoted; + } + output.push(character); + if quoted || comment || !matches!(character, '@' | '%') { + continue; + } + let mut name = String::new(); + while characters + .peek() + .is_some_and(|character| identifier(*character)) + { + name.push(characters.next().unwrap()); + } + let names = if character == '@' { &globals } else { &types }; + if names.contains(&name) { + write!(output, "case{index}_").unwrap(); + } + output.push_str(&name); + } + output +} + +fn identifier(character: char) -> bool { + character.is_ascii_alphanumeric() || matches!(character, '_' | '.' | '$' | '-') +} diff --git a/crates/dever-core/src/llvm/transaction.rs b/crates/dever-core/src/llvm/transaction.rs new file mode 100644 index 0000000..50249c6 --- /dev/null +++ b/crates/dever-core/src/llvm/transaction.rs @@ -0,0 +1,190 @@ +//! Sequential calls borrow the current transaction; detached descriptors pass NULL. +use super::*; + +#[derive(Clone)] +pub(super) struct TransactionScope { + outer: String, + finishing: String, + transaction: String, + output_guard: Option<usize>, + span: Span, +} + +impl FunctionEmitter<'_, '_> { + pub(super) fn begin_transaction(&mut self, function: &crate::hir::Function) { + if function.kind != FunctionKind::Transaction + || !self.module.database_function(self.instance) + { + return; + } + self.begin_transaction_scope( + self.instance, + &output_type(&function.outputs), + function.span, + ); + } + + pub(super) fn begin_command_transaction( + &mut self, + instance: &Specialization, + span: Span, + ) -> bool { + if !specialize::writes_database(self.module.program, instance) { + return false; + } + self.begin_transaction_scope(instance, &Type::Unit, span); + true + } + + pub(super) fn begin_transaction_scope( + &mut self, + instance: &Specialization, + output: &Type, + span: Span, + ) { + let owners = specialize::database_effects(self.module.program, instance); + let owner_database = *owners.first().expect("checked database transaction"); + let outer = self.entry_slot_ir("i1"); + let finishing = self.entry_slot_ir("i1"); + writeln!( + self.prologue, + " store i1 0, ptr {outer}\n store i1 0, ptr {finishing}" + ) + .unwrap(); + let transaction = self.entry_slot_ir("ptr"); + let owner = self.register_owned(OwnedKind::Protocol("db_transaction"), transaction.clone()); + let context = self.entry_slot_ir("ptr"); + self.line(format!( + "store ptr {}, ptr {context}", + self.database_context + )); + let output_guard = (*output != Type::Unit) + .then(|| self.register_owned(OwnedKind::Output(output.clone()), "%out".into())); + self.transactions.push(TransactionScope { + outer: outer.clone(), + finishing, + transaction: transaction.clone(), + output_guard, + span, + }); + let fresh = self.temp(); + self.line(format!( + "{fresh} = icmp eq ptr {}, null", + self.database_context + )); + let begin = self.label("transaction_begin"); + let ready = self.label("transaction_ready"); + self.line(format!("br i1 {fresh}, label %{begin}, label %{ready}")); + self.start(&begin); + let database = self.owner_database(owner_database, span); + let operation = self.database_operation("db_begin", vec![format!("ptr {database}")], span); + let handle = self.await_database_handle(&operation, span); + self.line(format!("store ptr {handle}, ptr {transaction}")); + self.mark_live(owner); + self.line(format!("store i1 1, ptr {outer}")); + self.line(format!("store ptr {handle}, ptr {context}")); + self.line(format!("br label %{ready}")); + self.start(&ready); + let handle = self.temp(); + self.line(format!("{handle} = load ptr, ptr {context}")); + self.database_context = handle; + } + + pub(super) fn finish_transaction(&mut self) { + for scope in self.transactions.clone().into_iter().rev() { + self.finish_transaction_scope(&scope); + } + } + + pub(super) fn commit_command_transaction(&mut self) { + let scope = self + .transactions + .last() + .expect("writing CMD transaction") + .clone(); + self.line(format!("store i1 1, ptr {}", scope.finishing)); + let transaction = self.temp(); + self.line(format!( + "{transaction} = load ptr, ptr {}", + scope.transaction + )); + let operation = self.database_operation( + "db_finish", + vec![format!("ptr {transaction}"), "i8 1".into()], + scope.span, + ); + self.await_operation(&operation, &Type::Unit, scope.span, false); + self.database_context = "null".into(); + } + + fn finish_transaction_scope(&mut self, scope: &TransactionScope) { + let outer = self.temp(); + let finishing = self.temp(); + let cancelling = self.temp(); + self.line(format!("{outer} = load i1, ptr {}", scope.outer)); + self.line(format!("{finishing} = load i1, ptr {}", scope.finishing)); + self.line(format!("{cancelling} = load i1, ptr %cancelling")); + let skip = self.temp(); + let fresh = self.temp(); + let needed = self.temp(); + self.line(format!("{skip} = or i1 {finishing}, {cancelling}")); + self.line(format!("{fresh} = xor i1 {skip}, 1")); + self.line(format!("{needed} = and i1 {outer}, {fresh}")); + let finish = self.label("transaction_finish"); + let done = self.label("transaction_cleaned"); + self.line(format!("br i1 {needed}, label %{finish}, label %{done}")); + self.start(&finish); + self.line(format!("store i1 1, ptr {}", scope.finishing)); + let transaction = self.temp(); + self.line(format!( + "{transaction} = load ptr, ptr {}", + scope.transaction + )); + let status = self.temp(); + self.line(format!("{status} = load i32, ptr %exit_status")); + let success = self.temp(); + self.line(format!("{success} = icmp eq i32 {status}, 0")); + let commit = self.label("transaction_commit"); + let rollback = self.label("transaction_rollback"); + self.line(format!( + "br i1 {success}, label %{commit}, label %{rollback}" + )); + self.start(&commit); + if let Some(guard) = scope.output_guard { + self.mark_live(guard); + } + let operation = self.database_operation( + "db_finish", + vec![format!("ptr {transaction}"), "i8 1".into()], + scope.span, + ); + self.await_operation(&operation, &Type::Unit, scope.span, false); + if let Some(guard) = scope.output_guard { + self.disarm(guard); + } + self.line(format!("br label %{done}")); + self.start(&rollback); + let descriptor = format!("@dever_fault_owned_{}", self.module.names[self.instance]); + let primary = self.entry_slot_ir("%dever.fault"); + self.line(format!( + "call void @dever_fault_move(ptr %fault, ptr {primary})" + )); + let primary_owner = self.register_owned(OwnedKind::Fault, primary.clone()); + self.mark_live(primary_owner); + let operation = self.database_operation( + "db_rollback_take", + vec![ + format!("ptr {transaction}"), + format!("ptr {primary}"), + format!("ptr {descriptor}"), + ], + scope.span, + ); + self.disarm(primary_owner); + let (status, _, _, error) = self.poll_raw(&operation, &Type::Unit); + let status = self.poll_status(&status, &error, scope.span); + self.line(format!("store i32 {status}, ptr %exit_status")); + self.line(format!("br label %{done}")); + self.start(&done); + } +} diff --git a/crates/dever-core/src/llvm/traversal.rs b/crates/dever-core/src/llvm/traversal.rs new file mode 100644 index 0000000..ad93121 --- /dev/null +++ b/crates/dever-core/src/llvm/traversal.rs @@ -0,0 +1,526 @@ +use super::*; +use crate::hir::HandlerTarget; + +impl FunctionEmitter<'_, '_> { + pub(super) fn invoke_handler( + &mut self, + handler: HandlerTarget, + arguments: &[(Type, String)], + span: Span, + ) -> (String, Type) { + let target = Specialization { + function: specialize::resolve_handler(handler, &self.bindings), + handlers: Vec::new(), + }; + let output_type = output_type(&self.module.program.functions[target.function].outputs); + let (status, output) = self.invoke_values(&target, arguments, &output_type, span); + let failed = self.temp(); + self.line(format!("{failed} = icmp ne i32 {status}, 0")); + let fault = self.label("handler_fault"); + let okay = self.label("handler_ok"); + self.line(format!("br i1 {failed}, label %{fault}, label %{okay}")); + self.start(&fault); + self.append_frame(span); + self.exit(&status); + self.start(&okay); + if output_type == Type::Unit { + ("zeroinitializer".into(), output_type) + } else { + let value = self.temp(); + self.line(format!( + "{value} = load {}, ptr {output}", + self.module.ty(&output_type) + )); + (value, output_type) + } + } + + fn append_output( + &mut self, + list_pointer: &str, + list_guard: usize, + list_ty: &Type, + element_ty: &Type, + element: &str, + span: Span, + ) { + let row = self.row_pointer(element_ty, element); + let current = self.temp(); + self.line(format!("{current} = load ptr, ptr {list_pointer}")); + self.disarm(list_guard); + let appended = self.runtime_call( + "dever_rt_v1_list_append_take", + vec![format!("ptr {current}"), format!("ptr {row}")], + list_ty, + false, + None, + span, + ); + self.line(format!("store ptr {appended}, ptr {list_pointer}")); + self.mark_live(list_guard); + } + + fn add_numeric(&mut self, ty: &Type, left: &str, right: &str, span: Span) -> String { + match ty { + Type::Int => self.checked_integer("sadd", left, right, span), + Type::Float => { + let value = self.temp(); + self.line(format!("{value} = fadd double {left}, {right}")); + value + } + Type::Decimal => { + let mut args = self.decimal_parts(left); + args.extend(self.decimal_parts(right)); + self.runtime_call("dever_rt_v1_decimal_add", args, ty, false, None, span) + } + _ => unreachable!("checked numeric Sum"), + } + } + + pub(super) fn traverse( + &mut self, + operation: CollectionOp, + handler: Option<HandlerTarget>, + sequence: SequenceKind, + arguments: &[Expression], + result: &Expression, + ) -> String { + let source = self.expression(&arguments[0]).expect("validated sequence"); + let source_guard = self.guards.len() - 1; + let element_ty = match &arguments[0].ty { + Type::List(element) + | Type::Stream(element) + | Type::AsyncStream(element) + | Type::RowStream(element) => element.as_ref().clone(), + Type::Bytes => Type::Int, + _ => unreachable!("checked sequence"), + }; + let additional = arguments.get(1).map(|value| { + self.expression(value) + .expect("validated traversal argument") + }); + let state_ty = matches!(operation, CollectionOp::Reduce | CollectionOp::ReduceUntil) + .then(|| arguments[1].ty.clone()); + let (state_pointer, state_guard) = + if let (Some(ty), Some(initial)) = (&state_ty, &additional) { + let guard = contains_owned(self.module.program, ty).then(|| self.guards.len() - 1); + let pointer = if let Some(guard) = guard { + self.guards[guard].pointer.clone() + } else { + let pointer = self.entry_slot(ty); + self.line(format!( + "store {} {initial}, ptr {pointer}", + self.module.ty(ty) + )); + pointer + }; + (Some(pointer), guard) + } else { + (None, None) + }; + // Argument evaluation precedes traversal ownership, as for native loops. + let stream_scope = if matches!( + sequence, + SequenceKind::AsyncStream | SequenceKind::RowStream + ) { + let pointer = self.row_pointer(&arguments[0].ty, &source); + let kind = if sequence == SequenceKind::RowStream { + OwnedKind::DatabaseStreamScope + } else { + OwnedKind::AsyncStreamScope + }; + let guard = self.register_owned(kind, pointer); + self.mark_live(guard); + Some(guard) + } else { + None + }; + + let (cursor, cursor_guard, length, index_pointer) = match sequence { + SequenceKind::List => { + self.disarm(source_guard); + let cursor = self.runtime_call( + "dever_rt_v1_list_cursor_take", + vec![format!("ptr {source}")], + &Type::Unit, + false, + Some("ptr"), + result.span, + ); + let pointer = self.entry_slot_ir("ptr"); + self.line(format!("store ptr {cursor}, ptr {pointer}")); + let guard = self.register_owned(OwnedKind::ListCursor, pointer); + self.mark_live(guard); + (Some(cursor), Some(guard), None, None) + } + SequenceKind::Bytes => { + let length = self.runtime_call( + "dever_rt_v1_bytes_length", + vec![format!("ptr {source}")], + &Type::Int, + false, + None, + result.span, + ); + let index = self.entry_slot(&Type::Int); + self.line(format!("store i64 0, ptr {index}")); + (None, None, Some(length), Some(index)) + } + SequenceKind::Stream | SequenceKind::AsyncStream | SequenceKind::RowStream => { + (Some(source.clone()), None, None, None) + } + }; + + let builder = matches!(operation, CollectionOp::Filter) + || operation == CollectionOp::Each && result.ty != Type::Unit; + let (list_pointer, list_guard) = if builder { + let Type::List(element) = &result.ty else { + unreachable!("checked List result") + }; + let list = self.runtime_call( + "dever_rt_v1_list_new", + vec![ + format!("ptr @dever_type_{}", self.module.type_index(element)), + "ptr null".into(), + "i64 0".into(), + ], + &result.ty, + false, + None, + result.span, + ); + let pointer = self.entry_slot(&result.ty); + self.line(format!("store ptr {list}, ptr {pointer}")); + let guard = self.register_guard(&result.ty, pointer.clone()); + self.mark_live(guard); + (Some(pointer), Some(guard)) + } else { + (None, None) + }; + let result_pointer = match operation { + CollectionOp::Find => { + let pointer = self.entry_slot(&result.ty); + self.line(format!( + "store {} zeroinitializer, ptr {pointer}", + self.module.ty(&result.ty) + )); + Some(pointer) + } + CollectionOp::Sum => { + let pointer = self.entry_slot(&result.ty); + let zero = match result.ty { + Type::Int => "0".into(), + Type::Float => "0.0".into(), + Type::Decimal => self.runtime_call( + "dever_rt_v1_decimal_from_int", + vec!["i64 0".into()], + &Type::Decimal, + false, + None, + result.span, + ), + _ => unreachable!("checked Sum type"), + }; + self.line(format!( + "store {} {zero}, ptr {pointer}", + self.module.ty(&result.ty) + )); + Some(pointer) + } + _ => None, + }; + + let element_pointer = matches!( + sequence, + SequenceKind::List + | SequenceKind::Stream + | SequenceKind::AsyncStream + | SequenceKind::RowStream + ) + .then(|| self.entry_slot(&element_ty)); + let element_guard = element_pointer.as_ref().and_then(|pointer| { + contains_owned(self.module.program, &element_ty) + .then(|| self.register_guard(&element_ty, pointer.clone())) + }); + let loop_label = self.label("sequence_next"); + let body = self.label("sequence_body"); + let done = self.label("sequence_done"); + self.line(format!("br label %{loop_label}")); + self.start(&loop_label); + let element = if let Some(cursor) = &cursor { + let output = element_pointer.as_ref().unwrap(); + let present = if matches!( + sequence, + SequenceKind::AsyncStream | SequenceKind::RowStream + ) { + let operation = if sequence == SequenceKind::RowStream { + self.database_operation( + "db_stream_pull", + vec![format!("ptr {cursor}")], + result.span, + ) + } else { + let operation = self.temp(); + self.line(format!( + "{operation} = call ptr @dever_rt_v1_async_stream_pull(ptr {cursor})" + )); + operation + }; + let (status, row, present, error) = self.poll_raw(&operation, &element_ty); + let legacy = self.poll_status(&status, &error, result.span); + self.propagate_status(&legacy, result.span, false); + // Only the presence branch may load the physical element row. + let flag = self.temp(); + self.line(format!("{flag} = load i8, ptr {present}")); + let available = self.temp(); + self.line(format!("{available} = icmp ne i8 {flag}, 0")); + let copy = self.label("stream_copy_row"); + let copied = self.label("stream_row_ready"); + self.line(format!("br i1 {available}, label %{copy}, label %{copied}")); + self.start(©); + let value = self.temp(); + self.line(format!( + "{value} = load {}, ptr {row}", + self.module.ty(&element_ty) + )); + self.line(format!( + "store {} {value}, ptr {output}", + self.module.ty(&element_ty) + )); + self.line(format!("br label %{copied}")); + self.start(&copied); + present + } else { + let present = self.entry_slot_ir("i8"); + let error = self.entry_slot_ir("{ ptr, i64 }"); + let status = self.temp(); + let symbol = if sequence == SequenceKind::Stream { + "stream_pull" + } else { + "list_cursor_next" + }; + self.line(format!("{status} = call i32 @dever_rt_v1_{symbol}(ptr {cursor}, ptr {output}, ptr {present}, ptr {error})")); + self.check_runtime_status(&status, &error, result.span); + present + }; + let flag = self.temp(); + self.line(format!("{flag} = load i8, ptr {present}")); + let available = self.temp(); + self.line(format!("{available} = icmp ne i8 {flag}, 0")); + self.line(format!("br i1 {available}, label %{body}, label %{done}")); + self.start(&body); + if let Some(guard) = element_guard { + self.mark_live(guard); + } + let value = self.temp(); + self.line(format!( + "{value} = load {}, ptr {output}", + self.module.ty(&element_ty) + )); + value + } else { + let index_pointer = index_pointer.as_ref().unwrap(); + let index = self.temp(); + self.line(format!("{index} = load i64, ptr {index_pointer}")); + let available = self.temp(); + self.line(format!( + "{available} = icmp slt i64 {index}, {}", + length.as_ref().unwrap() + )); + self.line(format!("br i1 {available}, label %{body}, label %{done}")); + self.start(&body); + let optional = self.runtime_call( + "dever_rt_v1_bytes_at", + vec![format!("ptr {source}"), format!("i64 {index}")], + &Type::Nullable(Box::new(Type::Int)), + true, + None, + result.span, + ); + let next = self.temp(); + self.line(format!("{next} = add i64 {index}, 1")); + self.line(format!("store i64 {next}, ptr {index_pointer}")); + let value = self.temp(); + self.line(format!( + "{value} = extractvalue {{ i1, i64 }} {optional}, 1" + )); + value + }; + + let projected = if let Some(handler) = handler { + let mut parameters = vec![(element_ty.clone(), element.clone())]; + if let Some(state_ty) = &state_ty { + let value = self.temp(); + self.line(format!( + "{value} = load {}, ptr {}", + self.module.ty(state_ty), + state_pointer.as_ref().unwrap() + )); + parameters.push((state_ty.clone(), value)); + } else if let Some(context) = &additional { + parameters.push((arguments[1].ty.clone(), context.clone())); + } + Some(self.invoke_handler(handler, ¶meters, result.span)) + } else { + None + }; + match operation { + CollectionOp::Each => { + if result.ty != Type::Unit { + let (value, ty) = projected.as_ref().unwrap(); + let guard = contains_owned(self.module.program, ty) + .then(|| self.own_value(ty, value, false)); + self.append_output( + list_pointer.as_ref().unwrap(), + list_guard.unwrap(), + &result.ty, + ty, + value, + result.span, + ); + if let Some(guard) = guard { + self.release_guard(guard); + } + } + } + CollectionOp::Filter | CollectionOp::Find => { + let condition = &projected.as_ref().unwrap().0; + let selected = self.label("sequence_selected"); + let next = self.label("sequence_continue"); + self.line(format!( + "br i1 {condition}, label %{selected}, label %{next}" + )); + self.start(&selected); + if operation == CollectionOp::Filter { + self.append_output( + list_pointer.as_ref().unwrap(), + list_guard.unwrap(), + &result.ty, + &element_ty, + &element, + result.span, + ); + self.line(format!("br label %{next}")); + } else { + let found = if element_ty == result.ty { + element.clone() + } else { + let first = self.temp(); + self.line(format!( + "{first} = insertvalue {} undef, i1 1, 0", + self.module.ty(&result.ty) + )); + let wrapped = self.temp(); + self.line(format!( + "{wrapped} = insertvalue {} {first}, {} {element}, 1", + self.module.ty(&result.ty), + self.module.ty(&element_ty) + )); + wrapped + }; + self.line(format!( + "store {} {found}, ptr {}", + self.module.ty(&result.ty), + result_pointer.as_ref().unwrap() + )); + if let Some(guard) = element_guard { + self.disarm(guard); + } + self.line(format!("br label %{done}")); + } + self.start(&next); + } + CollectionOp::Reduce | CollectionOp::ReduceUntil => { + let (step, step_ty) = projected.as_ref().unwrap(); + let next_state = if operation == CollectionOp::ReduceUntil { + let state = self.temp(); + self.line(format!( + "{state} = extractvalue {} {step}, 0", + self.module.ty(step_ty) + )); + state + } else { + step.clone() + }; + if let Some(guard) = state_guard { + self.release_guard(guard); + } + self.line(format!( + "store {} {next_state}, ptr {}", + self.module.ty(state_ty.as_ref().unwrap()), + state_pointer.as_ref().unwrap() + )); + if let Some(guard) = state_guard { + self.mark_live(guard); + } + if operation == CollectionOp::ReduceUntil { + let stop = self.temp(); + self.line(format!( + "{stop} = extractvalue {} {step}, 1", + self.module.ty(step_ty) + )); + if let Some(guard) = element_guard { + self.release_guard(guard); + } + self.line(format!("br i1 {stop}, label %{done}, label %{loop_label}")); + } + } + CollectionOp::Sum => { + let value = projected + .as_ref() + .map_or(&element, |projected| &projected.0); + let pointer = result_pointer.as_ref().unwrap(); + let current = self.temp(); + self.line(format!( + "{current} = load {}, ptr {pointer}", + self.module.ty(&result.ty) + )); + let sum = self.add_numeric(&result.ty, ¤t, value, result.span); + self.line(format!( + "store {} {sum}, ptr {pointer}", + self.module.ty(&result.ty) + )); + } + _ => unreachable!("preflighted traversal"), + } + if operation != CollectionOp::ReduceUntil { + if let Some(guard) = element_guard { + self.release_guard(guard); + } + self.line(format!("br label %{loop_label}")); + } + self.start(&done); + if let Some(guard) = stream_scope { + self.release_guard(guard); + } + if let Some(guard) = cursor_guard { + self.release_guard(guard); + } + if let (Some(pointer), Some(guard)) = (&list_pointer, list_guard) { + let output = self.temp(); + self.line(format!("{output} = load ptr, ptr {pointer}")); + self.disarm(guard); + return output; + } + if let Some(pointer) = &state_pointer { + let value = self.temp(); + self.line(format!( + "{value} = load {}, ptr {pointer}", + self.module.ty(state_ty.as_ref().unwrap()) + )); + if let Some(guard) = state_guard { + self.disarm(guard); + } + return value; + } + if let Some(pointer) = &result_pointer { + let value = self.temp(); + self.line(format!( + "{value} = load {}, ptr {pointer}", + self.module.ty(&result.ty) + )); + return value; + } + "zeroinitializer".into() + } +} diff --git a/crates/dever-core/src/llvm/wire.rs b/crates/dever-core/src/llvm/wire.rs new file mode 100644 index 0000000..76081fe --- /dev/null +++ b/crates/dever-core/src/llvm/wire.rs @@ -0,0 +1,703 @@ +//! Concrete codecs consume the checked wire DAG; runtime nodes never enter HIR values. +use super::*; +use crate::wire::{Node, Policy, Schema}; + +#[derive(Clone, Copy)] +pub(super) enum FieldInput { + Record, + External, + Command, + Api, + Rest, + RestList, +} + +pub(super) const DECLARATIONS: &str = "\ +declare i32 @dever_rt_v1_wire_decode(ptr, ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_cmd_decode(ptr, ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_cmd_input_field(ptr, ptr, i8, ptr, ptr, ptr) +declare i32 @dever_rt_v1_wire_encode(ptr, ptr, ptr, ptr, ptr) +declare i32 @dever_rt_v1_wire_node_is_null(ptr, ptr, ptr) +declare i32 @dever_rt_v1_wire_node_bool(ptr, ptr, ptr) +declare i32 @dever_rt_v1_wire_node_int(ptr, ptr, ptr) +declare i32 @dever_rt_v1_wire_node_float(ptr, ptr, ptr) +declare i32 @dever_rt_v1_wire_node_text(ptr, ptr, ptr) +declare i32 @dever_rt_v1_wire_node_bytes(ptr, ptr, ptr) +declare i32 @dever_rt_v1_wire_node_raw(ptr, ptr, ptr) +declare i32 @dever_rt_v1_wire_node_decimal(ptr, ptr, ptr) +declare i32 @dever_rt_v1_wire_node_uuid(ptr, ptr, ptr) +declare i32 @dever_rt_v1_wire_node_secret(ptr, ptr, ptr) +declare i32 @dever_rt_v1_wire_node_datetime(ptr, ptr, ptr) +declare i32 @dever_rt_v1_wire_node_date(ptr, ptr, ptr) +declare i32 @dever_rt_v1_wire_node_time(ptr, ptr, ptr) +declare i32 @dever_rt_v1_wire_node_fields(ptr, ptr, i64, ptr) +declare i32 @dever_rt_v1_wire_node_field(ptr, ptr, i8, ptr, ptr, ptr) +declare i32 @dever_rt_v1_wire_node_list_len(ptr, ptr, ptr) +declare i32 @dever_rt_v1_wire_node_list_at(ptr, i64, ptr, ptr) +declare i32 @dever_rt_v1_wire_encoder_null(ptr, ptr) +declare i32 @dever_rt_v1_wire_encoder_bool(ptr, i8, ptr) +declare i32 @dever_rt_v1_wire_encoder_int(ptr, i64, ptr) +declare i32 @dever_rt_v1_wire_encoder_float(ptr, double, ptr) +declare i32 @dever_rt_v1_wire_encoder_text(ptr, ptr, ptr) +declare i32 @dever_rt_v1_wire_encoder_bytes(ptr, ptr, ptr) +declare i32 @dever_rt_v1_wire_encoder_json(ptr, ptr, ptr) +declare i32 @dever_rt_v1_wire_encoder_decimal(ptr, i64, i64, ptr) +declare i32 @dever_rt_v1_wire_encoder_uuid(ptr, ptr, ptr) +declare i32 @dever_rt_v1_wire_encoder_datetime(ptr, i64, ptr) +declare i32 @dever_rt_v1_wire_encoder_date(ptr, i64, ptr) +declare i32 @dever_rt_v1_wire_encoder_time(ptr, i64, ptr) +declare i32 @dever_rt_v1_wire_encoder_begin_array(ptr, ptr) +declare i32 @dever_rt_v1_wire_encoder_begin_object(ptr, ptr) +declare i32 @dever_rt_v1_wire_encoder_key(ptr, ptr, ptr) +declare i32 @dever_rt_v1_wire_encoder_end(ptr, ptr) +"; + +impl Module<'_> { + pub(super) fn wire_names(&mut self, names: &[String]) -> String { + let entries = names + .iter() + .map(|name| { + let (pointer, length) = self.text_literal(name); + format!("{{ ptr, i64 }} {{ ptr {pointer}, i64 {length} }}") + }) + .collect::<Vec<_>>() + .join(", "); + let name = format!("@dever_wire_names_{}", self.next_literal); + self.next_literal += 1; + writeln!( + self.declarations, + "{name} = private constant [{} x {{ ptr, i64 }}] [{entries}]", + names.len() + ) + .unwrap(); + name + } + + pub(super) fn wire_codec(&mut self, ty: &Type, policy: Policy) -> String { + let schema = Schema::build(ty, &self.program.types, policy).expect("checked wire schema"); + if let Some(name) = self.wire_codecs.get(schema.fingerprint()) { + return name.clone(); + } + let prefix = format!("dever_wire_{}", self.wire_codecs.len()); + self.wire_codecs + .insert(schema.fingerprint().to_owned(), prefix.clone()); + let instance = self.instances.first().unwrap().clone(); + for (id, node) in schema.nodes.iter().enumerate() { + let ty = node_type(&schema, id); + let mut emitter = FunctionEmitter::new(self, &instance); + emitter.asynchronous = false; + emitter.protocol_callback = true; + emitter.start("entry"); + let value = emitter.decode_wire_node(&schema, node, &prefix, "%node"); + emitter.line(format!( + "store {} {value}, ptr %out", + emitter.module.ty(&ty) + )); + let function = emitter.finish_protocol_callback( + &format!("{prefix}_d{id}"), + "ptr %node, ptr %out, ptr %error", + ); + self.declarations.push_str(&function); + if !policy.permits_secret() { + let mut emitter = FunctionEmitter::new(self, &instance); + emitter.asynchronous = false; + emitter.protocol_callback = true; + emitter.start("entry"); + let value = emitter.temp(); + emitter.line(format!( + "{value} = load {}, ptr %input", + emitter.module.ty(&ty) + )); + emitter.encode_wire_node(&schema, node, &prefix, &value); + let function = emitter.finish_protocol_callback( + &format!("{prefix}_e{id}"), + "ptr %input, ptr %encoder, ptr %error", + ); + self.declarations.push_str(&function); + } + } + prefix + } + + pub(super) fn wire_callback(&mut self, ty: &Type, policy: Policy, encode: bool) -> String { + let schema = Schema::build(ty, &self.program.types, policy).expect("checked schema"); + let prefix = self.wire_codec(ty, policy); + format!("{prefix}_{}{}", if encode { "e" } else { "d" }, schema.root) + } + + /// External operation envelopes always retain field names, including a + /// single output or an explicitly null field. Their values use the same DAG. + pub(super) fn wire_fields_callback( + &mut self, + fields: &[crate::types::Field], + name: &str, + policy: Policy, + encode: bool, + ) -> String { + let ty = Type::Outputs(fields.to_vec()); + let callbacks = fields + .iter() + .map(|field| { + let callback = self.wire_callback(&field.ty, policy, encode); + (field.name.clone(), field.ty.clone(), callback) + }) + .collect::<Vec<_>>(); + let instance = self.instances.first().unwrap().clone(); + let mut emitter = FunctionEmitter::new(self, &instance); + emitter.asynchronous = false; + emitter.protocol_callback = true; + emitter.start("entry"); + let parameters = if encode { + emitter.wire_write( + "dever_rt_v1_wire_encoder_begin_object", + vec!["ptr %encoder".into()], + ); + for (index, (field, _, callback)) in callbacks.iter().enumerate() { + let key = emitter.module.wire_names(std::slice::from_ref(field)); + emitter.wire_write( + "dever_rt_v1_wire_encoder_key", + vec!["ptr %encoder".into(), format!("ptr {key}")], + ); + let pointer = emitter.temp(); + emitter.line(format!( + "{pointer} = getelementptr {}, ptr %input, i32 0, i32 {index}", + emitter.module.ty(&ty) + )); + emitter.wire_write( + callback, + vec![format!("ptr {pointer}"), "ptr %encoder".into()], + ); + } + emitter.wire_write("dever_rt_v1_wire_encoder_end", vec!["ptr %encoder".into()]); + "ptr %input, ptr %encoder, ptr %error" + } else { + let value = emitter.decode_fields(&ty, &callbacks, "%node", FieldInput::External); + emitter.line(format!( + "store {} {value}, ptr %out", + emitter.module.ty(&ty) + )); + "ptr %node, ptr %out, ptr %error" + }; + let body = emitter.finish_protocol_callback(name, parameters); + self.declarations.push_str(&body); + name.to_owned() + } +} + +fn node_type(schema: &Schema, id: usize) -> Type { + match &schema.nodes[id] { + Node::Bytes => Type::Bytes, + Node::Bool => Type::Bool, + Node::Int => Type::Int, + Node::Float => Type::Float, + Node::Decimal => Type::Decimal, + Node::Text => Type::Text, + Node::Id => Type::Id, + Node::Uuid => Type::Uuid, + Node::DateTime => Type::DateTime, + Node::Date => Type::Date, + Node::Time => Type::Time, + Node::Duration => Type::Duration, + Node::Json => Type::Json, + Node::Secret => Type::Secret, + Node::Nullable(child) => Type::Nullable(Box::new(node_type(schema, *child))), + Node::List(child) => Type::List(Box::new(node_type(schema, *child))), + Node::Record { id, .. } | Node::ModelId { id, .. } => Type::Named(*id), + } +} + +impl FunctionEmitter<'_, '_> { + fn wire_span(&self) -> Span { + self.module.program.functions[self.instance.function].span + } + + pub(super) fn wire_write(&mut self, symbol: &str, arguments: Vec<String>) { + self.wire_write_at(symbol, arguments, self.wire_span()); + } + + pub(super) fn wire_write_at(&mut self, symbol: &str, mut arguments: Vec<String>, span: Span) { + let error = self.entry_slot_ir("{ ptr, i64 }"); + arguments.push(format!("ptr {error}")); + let status = self.temp(); + self.line(format!( + "{status} = call i32 @{symbol}({})", + arguments.join(", ") + )); + self.check_runtime_status(&status, &error, span); + } + + fn decode_wire_child(&mut self, prefix: &str, child: usize, ty: &Type, node: &str) -> String { + self.runtime_call( + &format!("{prefix}_d{child}"), + vec![format!("ptr {node}")], + ty, + false, + None, + self.wire_span(), + ) + } + + pub(super) fn decode_fields( + &mut self, + ty: &Type, + fields: &[(String, Type, String)], + node: &str, + input: FieldInput, + ) -> String { + let names = self.module.wire_names( + &fields + .iter() + .map(|(name, _, _)| name.clone()) + .collect::<Vec<_>>(), + ); + if matches!(input, FieldInput::Record | FieldInput::External) { + self.wire_write( + "dever_rt_v1_wire_node_fields", + vec![ + format!("ptr {node}"), + format!("ptr {names}"), + format!("i64 {}", fields.len()), + ], + ); + } + let row = self.entry_slot(ty); + self.line(format!( + "store {} zeroinitializer, ptr {row}", + self.module.ty(ty) + )); + let guard = self.register_guard(ty, row.clone()); + self.mark_live(guard); + for (index, (_, field, callback)) in fields.iter().enumerate() { + let name = self.temp(); + self.line(format!( + "{name} = getelementptr {{ ptr, i64 }}, ptr {names}, i64 {index}" + )); + if *field == Type::Upload { + let upload = self.runtime_call( + "dever_rt_v1_api_input_upload", + vec![format!("ptr {node}"), format!("ptr {name}")], + field, + false, + None, + self.wire_span(), + ); + let target = self.temp(); + self.line(format!( + "{target} = getelementptr {}, ptr {row}, i32 0, i32 {index}", + self.module.ty(ty) + )); + self.line(format!("store ptr {upload}, ptr {target}")); + continue; + } + let optional = + matches!(field, Type::Nullable(_)) && !matches!(input, FieldInput::External); + let mut arguments = vec![ + format!("ptr {node}"), + format!("ptr {name}"), + format!("i8 {}", u8::from(!optional)), + ]; + let converted_query = matches!(input, FieldInput::Api) + || matches!(input, FieldInput::RestList) && index < 2; + if converted_query { + arguments.push(format!( + "i32 {}", + api_query_kind(self.module.program, field) + )); + } + let found = self.runtime_call( + match input { + FieldInput::Record | FieldInput::External => "dever_rt_v1_wire_node_field", + FieldInput::Command => "dever_rt_v1_cmd_input_field", + FieldInput::Api => "dever_rt_v1_api_input_field", + FieldInput::Rest => "dever_rt_v1_api_input_raw_field", + FieldInput::RestList if index < 2 => "dever_rt_v1_api_input_field", + FieldInput::RestList => "dever_rt_v1_api_input_raw_field", + }, + arguments, + &Type::Nullable(Box::new(Type::Text)), + true, + None, + self.wire_span(), + ); + // CMD Inputs transfers an owned raw Text. Record Nodes remain + // borrowed from their enclosing synchronous wire decode callback. + let raw_guard = (!matches!(input, FieldInput::Record | FieldInput::External)) + .then(|| self.own_value(&Type::Nullable(Box::new(Type::Text)), &found, false)); + let present = self.temp(); + self.line(format!("{present} = extractvalue {{ i1, ptr }} {found}, 0")); + let read = self.label("wire_field"); + let done = self.label("wire_field_done"); + self.line(format!("br i1 {present}, label %{read}, label %{done}")); + self.start(&read); + let child = self.temp(); + self.line(format!("{child} = extractvalue {{ i1, ptr }} {found}, 1")); + let search = matches!(input, FieldInput::RestList) && index == 2; + let decoded = if search { + let Type::Nullable(inner) = field else { + unreachable!() + }; + inner.as_ref() + } else { + field + }; + let (decode, arguments) = match input { + FieldInput::Record | FieldInput::External => { + (callback.as_str(), vec![format!("ptr {child}")]) + } + FieldInput::Command | FieldInput::Api | FieldInput::Rest | FieldInput::RestList => { + ( + "dever_rt_v1_wire_decode", + vec![ + format!("ptr {child}"), + format!("ptr @dever_type_{}", self.module.type_index(decoded)), + format!("ptr @{callback}"), + ], + ) + } + }; + let mut value = + self.runtime_call(decode, arguments, decoded, false, None, self.wire_span()); + if search { + let flag = self.temp(); + let payload = self.temp(); + self.line(format!( + "{flag} = insertvalue {} zeroinitializer, i1 true, 0", + self.module.ty(field) + )); + self.line(format!( + "{payload} = insertvalue {} {flag}, {} {value}, 1", + self.module.ty(field), + self.module.ty(decoded) + )); + value = payload; + } + let target = self.temp(); + self.line(format!( + "{target} = getelementptr {}, ptr {row}, i32 0, i32 {index}", + self.module.ty(ty) + )); + self.line(format!( + "store {} {value}, ptr {target}", + self.module.ty(field) + )); + self.line(format!("br label %{done}")); + self.start(&done); + if let Some(guard) = raw_guard { + self.release_guard(guard); + } + } + let value = self.temp(); + self.line(format!("{value} = load {}, ptr {row}", self.module.ty(ty))); + self.disarm(guard); + value + } + + fn decode_wire_node( + &mut self, + schema: &Schema, + node: &Node, + prefix: &str, + input: &str, + ) -> String { + if let Node::Record { id, fields, .. } = node { + let fields = fields + .iter() + .map(|(name, child)| { + ( + name.clone(), + node_type(schema, *child), + format!("{prefix}_d{child}"), + ) + }) + .collect::<Vec<_>>(); + return self.decode_fields(&Type::Named(*id), &fields, input, FieldInput::Record); + } + if let Node::Nullable(child) = node { + let ty = Type::Nullable(Box::new(node_type(schema, *child))); + let row = self.entry_slot(&ty); + self.line(format!( + "store {} zeroinitializer, ptr {row}", + self.module.ty(&ty) + )); + let null = self.runtime_call( + "dever_rt_v1_wire_node_is_null", + vec![format!("ptr {input}")], + &Type::Bool, + false, + Some("i8"), + self.wire_span(), + ); + let some = self.label("wire_some"); + let done = self.label("wire_optional_done"); + self.line(format!("br i1 {null}, label %{done}, label %{some}")); + self.start(&some); + let child_ty = node_type(schema, *child); + let value = self.decode_wire_child(prefix, *child, &child_ty, input); + let result = self.temp(); + self.line(format!( + "{result} = insertvalue {} {{ i1 true, {} zeroinitializer }}, {} {value}, 1", + self.module.ty(&ty), + self.module.ty(&child_ty), + self.module.ty(&child_ty) + )); + self.line(format!("store {} {result}, ptr {row}", self.module.ty(&ty))); + self.line(format!("br label %{done}")); + self.start(&done); + let value = self.temp(); + self.line(format!("{value} = load {}, ptr {row}", self.module.ty(&ty))); + return value; + } + if let Node::List(child) = node { + return self.decode_wire_list(schema, *child, prefix, input); + } + let (kind, ty, physical) = match node { + Node::Bytes => ("bytes", Type::Bytes, None), + Node::Bool => ("bool", Type::Bool, Some("i8")), + Node::Int | Node::ModelId { .. } | Node::Duration => ("int", Type::Int, None), + Node::Float => ("float", Type::Float, None), + Node::Decimal => ("decimal", Type::Decimal, None), + Node::Text | Node::Id => ("text", Type::Text, None), + Node::Json => ("raw", Type::Json, None), + Node::Uuid => ("uuid", Type::Uuid, None), + Node::Secret => ("secret", Type::Secret, None), + Node::DateTime => ("datetime", Type::DateTime, None), + Node::Date => ("date", Type::Date, None), + Node::Time => ("time", Type::Time, None), + _ => unreachable!(), + }; + self.runtime_call( + &format!("dever_rt_v1_wire_node_{kind}"), + vec![format!("ptr {input}")], + &ty, + false, + physical, + self.wire_span(), + ) + } + + fn decode_wire_list( + &mut self, + schema: &Schema, + child: usize, + prefix: &str, + input: &str, + ) -> String { + let element = node_type(schema, child); + let ty = Type::List(Box::new(element.clone())); + let list = self.runtime_call( + "dever_rt_v1_list_new", + vec![ + format!("ptr @dever_type_{}", self.module.type_index(&element)), + "ptr null".into(), + "i64 0".into(), + ], + &ty, + false, + None, + self.wire_span(), + ); + let guard = self.own_value(&ty, &list, false); + let slot = self.guards[guard].pointer.clone(); + let length = self.runtime_call( + "dever_rt_v1_wire_node_list_len", + vec![format!("ptr {input}")], + &Type::Int, + false, + None, + self.wire_span(), + ); + let index = self.entry_slot_ir("i64"); + self.line(format!("store i64 0, ptr {index}")); + let next = self.label("wire_list_next"); + let body = self.label("wire_list_item"); + let done = self.label("wire_list_done"); + self.line(format!("br label %{next}")); + self.start(&next); + let position = self.temp(); + self.line(format!("{position} = load i64, ptr {index}")); + let more = self.temp(); + self.line(format!("{more} = icmp ult i64 {position}, {length}")); + self.line(format!("br i1 {more}, label %{body}, label %{done}")); + self.start(&body); + let first = self.guards.len(); + let node = self.protocol_call( + "dever_rt_v1_wire_node_list_at", + vec![format!("ptr {input}"), format!("i64 {position}")], + "ptr", + self.wire_span(), + ); + let value = self.decode_wire_child(prefix, child, &element, &node); + self.own_value(&element, &value, false); + let item = self.row_pointer(&element, &value); + let old = self.temp(); + self.line(format!("{old} = load ptr, ptr {slot}")); + self.disarm(guard); + let appended = self.runtime_call( + "dever_rt_v1_list_append_take", + vec![format!("ptr {old}"), format!("ptr {item}")], + &ty, + false, + None, + self.wire_span(), + ); + self.line(format!("store ptr {appended}, ptr {slot}")); + self.mark_live(guard); + self.release_from(first); + let increment = self.temp(); + self.line(format!("{increment} = add i64 {position}, 1")); + self.line(format!("store i64 {increment}, ptr {index}")); + self.line(format!("br label %{next}")); + self.start(&done); + let value = self.temp(); + self.line(format!("{value} = load ptr, ptr {slot}")); + self.disarm(guard); + value + } + + fn encode_wire_node(&mut self, schema: &Schema, node: &Node, prefix: &str, value: &str) { + if let Node::Nullable(child) = node { + let ty = Type::Nullable(Box::new(node_type(schema, *child))); + let present = self.temp(); + self.line(format!( + "{present} = extractvalue {} {value}, 0", + self.module.ty(&ty) + )); + let some = self.label("wire_encode_some"); + let none = self.label("wire_encode_none"); + let done = self.label("wire_encode_done"); + self.line(format!("br i1 {present}, label %{some}, label %{none}")); + self.start(&none); + self.wire_write("dever_rt_v1_wire_encoder_null", vec!["ptr %encoder".into()]); + self.line(format!("br label %{done}")); + self.start(&some); + let payload = self.temp(); + self.line(format!( + "{payload} = extractvalue {} {value}, 1", + self.module.ty(&ty) + )); + self.encode_wire_child(schema, *child, prefix, &payload); + self.line(format!("br label %{done}")); + self.start(&done); + return; + } + if let Node::Record { id, fields, .. } = node { + self.wire_write( + "dever_rt_v1_wire_encoder_begin_object", + vec!["ptr %encoder".into()], + ); + for (index, (name, child)) in fields.iter().enumerate() { + let key = self.module.wire_names(std::slice::from_ref(name)); + self.wire_write( + "dever_rt_v1_wire_encoder_key", + vec!["ptr %encoder".into(), format!("ptr {key}")], + ); + let field = self.temp(); + self.line(format!("{field} = extractvalue %T{id} {value}, {index}")); + self.encode_wire_child(schema, *child, prefix, &field); + } + self.wire_write("dever_rt_v1_wire_encoder_end", vec!["ptr %encoder".into()]); + return; + } + if let Node::List(child) = node { + self.encode_wire_list(schema, *child, prefix, value); + return; + } + let mut args = vec!["ptr %encoder".into()]; + let (kind, llvm) = match node { + Node::Bytes => ("bytes", "ptr"), + Node::Bool => ("bool", "i8"), + Node::Int | Node::Duration | Node::ModelId { .. } => ("int", "i64"), + Node::Float => ("float", "double"), + Node::Text | Node::Id => ("text", "ptr"), + Node::Json => ("json", "ptr"), + Node::Decimal => ("decimal", ""), + Node::Uuid => ("uuid", "ptr"), + Node::DateTime => ("datetime", "i64"), + Node::Date => ("date", "i64"), + Node::Time => ("time", "i64"), + _ => unreachable!("checked output wire node"), + }; + if matches!(node, Node::Decimal) { + args.extend(self.decimal_parts(value)); + } else if matches!(node, Node::Bool) { + let byte = self.temp(); + self.line(format!("{byte} = zext i1 {value} to i8")); + args.push(format!("i8 {byte}")); + } else { + args.push(format!("{llvm} {value}")); + } + self.wire_write(&format!("dever_rt_v1_wire_encoder_{kind}"), args); + } + + fn encode_wire_child(&mut self, schema: &Schema, child: usize, prefix: &str, value: &str) { + let row = self.row_pointer(&node_type(schema, child), value); + self.wire_write( + &format!("{prefix}_e{child}"), + vec![format!("ptr {row}"), "ptr %encoder".into()], + ); + } + + fn encode_wire_list(&mut self, schema: &Schema, child: usize, prefix: &str, value: &str) { + self.encode_list_callback( + &node_type(schema, child), + &format!("{prefix}_e{child}"), + value, + ); + } + + pub(super) fn encode_list_callback(&mut self, element: &Type, callback: &str, value: &str) { + self.wire_write( + "dever_rt_v1_wire_encoder_begin_array", + vec!["ptr %encoder".into()], + ); + let retained = self.temp(); + self.line(format!( + "{retained} = call ptr @dever_rt_v1_list_retain(ptr {value})" + )); + let cursor = self.protocol_call( + "dever_rt_v1_list_cursor_take", + vec![format!("ptr {retained}")], + "ptr", + self.wire_span(), + ); + let slot = self.entry_slot_ir("ptr"); + self.line(format!("store ptr {cursor}, ptr {slot}")); + let cursor_guard = self.register_owned(OwnedKind::ListCursor, slot); + self.mark_live(cursor_guard); + let row = self.entry_slot(element); + let guard = self.register_guard(element, row.clone()); + let present = self.entry_slot_ir("i8"); + let error = self.entry_slot_ir("{ ptr, i64 }"); + let next = self.label("wire_encode_next"); + let body = self.label("wire_encode_item"); + let done = self.label("wire_encode_list_done"); + self.line(format!("br label %{next}")); + self.start(&next); + let status = self.temp(); + self.line(format!("{status} = call i32 @dever_rt_v1_list_cursor_next(ptr {cursor}, ptr {row}, ptr {present}, ptr {error})")); + self.check_runtime_status(&status, &error, self.wire_span()); + let flag = self.temp(); + self.line(format!("{flag} = load i8, ptr {present}")); + let more = self.temp(); + self.line(format!("{more} = icmp ne i8 {flag}, 0")); + self.line(format!("br i1 {more}, label %{body}, label %{done}")); + self.start(&body); + self.mark_live(guard); + self.wire_write(callback, vec![format!("ptr {row}"), "ptr %encoder".into()]); + self.release_guard(guard); + self.line(format!("br label %{next}")); + self.start(&done); + self.release_guard(cursor_guard); + self.wire_write("dever_rt_v1_wire_encoder_end", vec!["ptr %encoder".into()]); + } +} + +fn api_query_kind(program: &Program, ty: &Type) -> u32 { + let schema = Schema::build(ty, &program.types, Policy::ApiInput).expect("checked API schema"); + match schema.query_kind() { + "Int" => 1, + "Bool" => 2, + "Float" => 3, + _ => 0, + } +} diff --git a/crates/dever-core/src/markdown.rs b/crates/dever-core/src/markdown.rs new file mode 100644 index 0000000..6b37c05 --- /dev/null +++ b/crates/dever-core/src/markdown.rs @@ -0,0 +1,187 @@ +//! Markdown documents declarations; only explicit top-level program fences produce Dever tokens. + +use std::borrow::Cow; +use std::ops::Range; + +use pulldown_cmark::{CodeBlockKind, Event, Parser, Tag}; + +use crate::diagnostic::Diagnostic; +use crate::source::{SourceFile, Span}; +use crate::syntax::{Declaration, Package}; + +mod contract; +pub(crate) use contract::{validate_contract, validate_structure}; + +pub(crate) struct ProgramBlock { + pub outer: Range<usize>, + pub content: Range<usize>, + pub indentation: String, + pub newline: &'static str, +} + +pub(crate) fn blocks(source: &SourceFile) -> Result<Vec<ProgramBlock>, Vec<Diagnostic>> { + let mut blocks = Vec::new(); + let mut errors = Vec::new(); + let mut depth = 0; + let markdown = normalize_lone_carriage_returns(source.text()); + for (event, range) in Parser::new(&markdown).into_offset_iter() { + match event { + Event::Start(tag) => { + if depth == 0 + && matches!(tag, Tag::CodeBlock(CodeBlockKind::Fenced(ref info)) if is_program_info(info)) + { + match program_block(source, range) { + Ok(Some(block)) => blocks.push(block), + Ok(None) => {} + Err(error) => errors.push(error), + } + } + depth += 1; + } + Event::End(_) => depth -= 1, + _ => {} + } + } + if errors.is_empty() && blocks.is_empty() { + errors.push(Diagnostic::error( + "M001", + "Markdown source requires at least one top-level fenced 'dever' or 'typescript dever' code block", + source.span(0, 0), + )); + } + if errors.is_empty() { + Ok(blocks) + } else { + Err(errors) + } +} + +fn normalize_lone_carriage_returns(text: &str) -> Cow<'_, str> { + let is_lone_cr = + |offset, byte| byte == b'\r' && text.as_bytes().get(offset + 1) != Some(&b'\n'); + if !text + .bytes() + .enumerate() + .any(|(offset, byte)| is_lone_cr(offset, byte)) + { + return Cow::Borrowed(text); + } + // The Markdown parser requires LF. Replacing only bare CR preserves every byte offset. + Cow::Owned( + text.char_indices() + .map(|(offset, ch)| { + if ch == '\r' && is_lone_cr(offset, b'\r') { + '\n' + } else { + ch + } + }) + .collect(), + ) +} + +fn program_block( + source: &SourceFile, + range: Range<usize>, +) -> Result<Option<ProgramBlock>, Diagnostic> { + let outer = range.clone(); + let text = source.text(); + let opening_end = text[range.start..] + .find(['\r', '\n']) + .map_or(text.len(), |offset| range.start + offset); + let opening = &text[range.start..opening_end]; + let fence = opening.as_bytes()[0]; + let fence_length = opening.bytes().take_while(|byte| *byte == fence).count(); + // CommonMark unescapes info strings. Only a literal language marker opts into execution. + if !is_program_info(opening[fence_length..].trim()) { + return Ok(None); + } + let newline = if text[opening_end..].starts_with("\r\n") { + "\r\n" + } else if text[opening_end..].starts_with('\r') { + "\r" + } else { + "\n" + }; + let content_start = (opening_end + newline.len()).min(text.len()); + let closing_start = line_start(text, range.end); + let closing = &text[closing_start..range.end]; + let closing_fence = closing.trim_start_matches(' '); + let closing_length = closing_fence + .bytes() + .take_while(|byte| *byte == fence) + .count(); + // CommonMark permits an EOF-terminated fence; executable source deliberately requires closure. + if closing_start < content_start + || closing.len() - closing_fence.len() > 3 + || closing_length < fence_length + || !closing_fence[closing_length..].trim_matches(' ').is_empty() + { + return Err(Diagnostic::error( + "M002", + "Dever program fence must have a matching closing fence", + source.span(range.start, opening_end), + )); + } + Ok(Some(ProgramBlock { + outer, + content: content_start..closing_start, + indentation: text[line_start(text, range.start)..range.start].to_owned(), + newline, + })) +} + +fn is_program_info(info: &str) -> bool { + matches!(info.trim(), "dever" | "typescript dever") +} + +fn line_start(text: &str, offset: usize) -> usize { + text[..offset] + .rfind(['\r', '\n']) + .map_or(0, |index| index + 1) +} + +pub(crate) fn validate_declarations( + package: &Package, + blocks: &[ProgramBlock], +) -> Result<(), Vec<Diagnostic>> { + let declarations = package.declarations.iter().map(declaration_span); + let errors: Vec<_> = declarations + .filter_map(|span| { + let index = blocks.partition_point(|block| block.content.end <= span.start); + let complete = blocks.get(index).is_some_and(|block| { + block.content.start <= span.start && span.end <= block.content.end + }); + if complete { + return None; + } + Some(Diagnostic::error( + "M003", + "each declaration must be complete within one Dever code block", + span, + )) + }) + .collect(); + if errors.is_empty() { + Ok(()) + } else { + Err(errors) + } +} + +pub(crate) fn declaration_span(declaration: &Declaration) -> Span { + match declaration { + Declaration::Api(binding) => binding.span, + Declaration::Rest(rest) => rest.span, + Declaration::External(external) => external.span, + Declaration::Type(declaration) => declaration.span, + Declaration::Function(function) => function.span, + Declaration::Database(binding) => binding.span, + Declaration::Schedule(schedule) => schedule.span, + Declaration::ModelIndex(index) => index.span, + Declaration::Relation(relation) => relation.span, + Declaration::Seed(seed) => seed.span, + Declaration::Migration(migration) => migration.span, + Declaration::ModelSql(sql) => sql.span, + } +} diff --git a/crates/dever-core/src/markdown/contract.rs b/crates/dever-core/src/markdown/contract.rs new file mode 100644 index 0000000..c00b2d0 --- /dev/null +++ b/crates/dever-core/src/markdown/contract.rs @@ -0,0 +1,36 @@ +use crate::diagnostic::Diagnostic; +use crate::hir::Function; +use crate::source::SourceFile; +use crate::syntax::Package; +use crate::types::Definition; + +use super::ProgramBlock; + +mod metadata; +mod model; +mod parse; +mod validate; + +pub(crate) fn validate_structure( + source: &SourceFile, + package: &Package, + blocks: &[ProgramBlock], +) -> Result<(), Vec<Diagnostic>> { + parse::contract(source, package, blocks).map(|_| ()) +} + +pub(crate) fn validate_contract( + source: &SourceFile, + package: &Package, + owner: usize, + definitions: &[Definition], + functions: &[Function], +) -> Vec<Diagnostic> { + let Ok(blocks) = super::blocks(source) else { + return Vec::new(); + }; + let Ok(contract) = parse::contract(source, package, &blocks) else { + return Vec::new(); + }; + validate::contract(&contract, package, owner, definitions, functions) +} diff --git a/crates/dever-core/src/markdown/contract/metadata.rs b/crates/dever-core/src/markdown/contract/metadata.rs new file mode 100644 index 0000000..ba8218b --- /dev/null +++ b/crates/dever-core/src/markdown/contract/metadata.rs @@ -0,0 +1,339 @@ +use std::ops::Range; + +use pulldown_cmark::{Event, Parser, Tag, TagEnd}; + +use crate::diagnostic::Diagnostic; +use crate::markdown::normalize_lone_carriage_returns; +use crate::source::{SourceFile, Span}; + +use super::model::{ + DocumentValue, DocumentedList, FunctionDocumentation, ModelDocumentation, PackageDocumentation, + TypeDocumentation, TypeEntries, +}; + +pub(super) fn package( + source: &SourceFile, + range: Range<usize>, + heading: Span, +) -> Result<PackageDocumentation, Vec<Diagnostic>> { + let mut parser = MetadataParser::new(source, range, heading, "M004", "- 包：")?; + let name = parser.scalar("- 包：")?; + let public_types = parser.list("- 公开类型：", false)?; + let public_functions = parser.list("- 公开方法：", false)?; + let usages = parser.list("- 使用：", false)?; + parser.finish()?; + Ok(PackageDocumentation { + name, + public_types, + public_functions, + usages, + }) +} + +pub(super) fn type_declaration( + source: &SourceFile, + range: Range<usize>, + heading: Span, + entries: TypeEntries, +) -> Result<TypeDocumentation, Vec<Diagnostic>> { + let mut parser = MetadataParser::new(source, range, heading, "M005", "- 类型：")?; + let name = parser.scalar("- 类型：")?; + let entries = parser.list( + match entries { + TypeEntries::Fields => "- 字段：", + TypeEntries::Variants => "- 分支：", + }, + true, + )?; + parser.finish()?; + Ok(TypeDocumentation { name, entries }) +} + +pub(super) fn function( + source: &SourceFile, + range: Range<usize>, + heading: Span, + has_failures: bool, +) -> Result<FunctionDocumentation, Vec<Diagnostic>> { + let mut parser = MetadataParser::new(source, range, heading, "M005", "- 函数：")?; + let name = parser.scalar("- 函数：")?; + let inputs = parser.list("- 输入：", true)?; + let outputs = parser.list("- 输出：", true)?; + let fails = if has_failures { + Some(parser.scalar("- 允许失败：")?) + } else { + None + }; + parser.finish()?; + Ok(FunctionDocumentation { + name, + inputs, + outputs, + fails, + }) +} + +pub(super) fn model_declaration( + source: &SourceFile, + range: Range<usize>, + heading: Span, + sql: bool, +) -> Result<ModelDocumentation, Vec<Diagnostic>> { + let mut parser = MetadataParser::new(source, range, heading, "M005", "- 声明：")?; + let declaration = parser.scalar("- 声明：")?; + let signature = if sql { + Some(( + parser.list("- 输入：", true)?, + parser.list("- 输出：", true)?, + )) + } else { + None + }; + parser.finish()?; + Ok(ModelDocumentation { + declaration, + signature, + }) +} + +struct MetadataParser<'a> { + source: &'a SourceFile, + lines: Vec<SourceLine<'a>>, + cursor: usize, + fallback: Span, + code: &'static str, +} + +impl<'a> MetadataParser<'a> { + fn new( + source: &'a SourceFile, + range: Range<usize>, + fallback: Span, + code: &'static str, + marker: &str, + ) -> Result<Self, Vec<Diagnostic>> { + let description_start = range.start; + let lines = source_lines(source, range); + let Some(cursor) = lines + .iter() + .position(|line| line.text.trim_start().starts_with(marker)) + else { + return Err(vec![Diagnostic::error( + code, + format!("section requires metadata beginning with '{marker}'"), + fallback, + )]); + }; + if !has_natural_description(source, description_start..lines[cursor].start) { + return Err(vec![Diagnostic::error( + code, + "section requires a natural-language description before its metadata", + fallback, + )]); + } + Ok(Self { + source, + lines, + cursor, + fallback, + code, + }) + } + + fn scalar(&mut self, prefix: &str) -> Result<DocumentValue, Vec<Diagnostic>> { + self.skip_blank(); + let line = self.current()?; + let value = parse_inline_value(self.source, line, prefix, false).map_err(|message| { + vec![Diagnostic::error( + self.code, + message, + line.span(self.source), + )] + })?; + self.cursor += 1; + Ok(value) + } + + fn list(&mut self, header: &str, described: bool) -> Result<DocumentedList, Vec<Diagnostic>> { + self.skip_blank(); + let line = self.current()?; + let text = line.text.trim_end(); + if text == format!("{header}无") { + self.cursor += 1; + return Ok(DocumentedList { + values: Vec::new(), + span: line.span(self.source), + }); + } + if text != header { + return Err(vec![Diagnostic::error( + self.code, + format!("expected '{header}' or '{header}无'"), + line.span(self.source), + )]); + } + let span = line.span(self.source); + self.cursor += 1; + let mut values = Vec::new(); + loop { + self.skip_blank(); + let Some(line) = self.lines.get(self.cursor).copied() else { + break; + }; + if !line.text.starts_with(" - ") { + break; + } + let value = + parse_inline_value(self.source, line, " - ", described).map_err(|message| { + vec![Diagnostic::error( + self.code, + message, + line.span(self.source), + )] + })?; + values.push(value); + self.cursor += 1; + } + if values.is_empty() { + return Err(vec![Diagnostic::error( + self.code, + format!("'{header}' requires at least one item; use '{header}无' when empty"), + span, + )]); + } + Ok(DocumentedList { values, span }) + } + + fn finish(&mut self) -> Result<(), Vec<Diagnostic>> { + self.skip_blank(); + if let Some(line) = self.lines.get(self.cursor) { + return Err(vec![Diagnostic::error( + self.code, + "unexpected content between documentation metadata and its Dever code block", + line.span(self.source), + )]); + } + Ok(()) + } + + fn current(&self) -> Result<SourceLine<'a>, Vec<Diagnostic>> { + self.lines.get(self.cursor).copied().ok_or_else(|| { + vec![Diagnostic::error( + self.code, + "incomplete documentation metadata", + self.fallback, + )] + }) + } + + fn skip_blank(&mut self) { + while self + .lines + .get(self.cursor) + .is_some_and(|line| line.text.trim().is_empty()) + { + self.cursor += 1; + } + } +} + +fn has_natural_description(source: &SourceFile, range: Range<usize>) -> bool { + let markdown = normalize_lone_carriage_returns(source.text()); + let mut depth = 0usize; + let mut in_top_level_paragraph = false; + for event in Parser::new(&markdown[range]) { + match event { + Event::Start(Tag::Paragraph) if depth == 0 => { + in_top_level_paragraph = true; + depth += 1; + } + Event::Start(_) => depth += 1, + Event::End(TagEnd::Paragraph) => { + depth = depth.saturating_sub(1); + in_top_level_paragraph = false; + } + Event::End(_) => depth = depth.saturating_sub(1), + Event::Text(text) if in_top_level_paragraph && !text.trim().is_empty() => return true, + _ => {} + } + } + false +} + +fn parse_inline_value( + source: &SourceFile, + line: SourceLine<'_>, + prefix: &str, + described: bool, +) -> Result<DocumentValue, String> { + let text = line.text.trim_end(); + let rest = text + .strip_prefix(prefix) + .ok_or_else(|| format!("expected '{prefix}`...`'"))?; + let Some(rest) = rest.strip_prefix('`') else { + return Err(format!("expected '{prefix}`...`'")); + }; + let Some(closing) = rest.find('`') else { + return Err("documentation metadata has an unclosed inline code value".into()); + }; + let value = &rest[..closing]; + if value.is_empty() || value.trim() != value { + return Err( + "documentation metadata value must be nonempty without surrounding spaces".into(), + ); + } + let suffix = &rest[closing + 1..]; + if described { + let Some(description) = suffix.strip_prefix('：') else { + return Err( + "each input, output, field or variant requires '：' and a description".into(), + ); + }; + if description.trim().is_empty() { + return Err("input, output, field or variant description must not be empty".into()); + } + } else if !suffix.is_empty() { + return Err("unexpected text after documentation metadata value".into()); + } + let value_start = line.start + prefix.len() + 1; + Ok(DocumentValue { + text: value.to_owned(), + span: source.span(value_start, value_start + value.len()), + }) +} + +fn source_lines(source: &SourceFile, range: Range<usize>) -> Vec<SourceLine<'_>> { + let text = source.text(); + let bytes = text.as_bytes(); + let mut lines = Vec::new(); + let mut start = range.start; + while start < range.end { + let mut end = start; + while end < range.end && !matches!(bytes[end], b'\r' | b'\n') { + end += 1; + } + lines.push(SourceLine { + text: &text[start..end], + start, + end, + }); + if end == range.end { + break; + } + start = end + usize::from(bytes[end] == b'\r' && bytes.get(end + 1) == Some(&b'\n')) + 1; + } + lines +} + +#[derive(Clone, Copy)] +struct SourceLine<'a> { + text: &'a str, + start: usize, + end: usize, +} + +impl SourceLine<'_> { + fn span(self, source: &SourceFile) -> Span { + source.span(self.start, self.end) + } +} diff --git a/crates/dever-core/src/markdown/contract/model.rs b/crates/dever-core/src/markdown/contract/model.rs new file mode 100644 index 0000000..5d38898 --- /dev/null +++ b/crates/dever-core/src/markdown/contract/model.rs @@ -0,0 +1,84 @@ +use crate::source::Span; + +pub(super) struct DocumentContract { + pub package: PackageDocumentation, + pub sections: Vec<DeclarationSection>, +} + +pub(super) struct PackageDocumentation { + pub name: DocumentValue, + pub public_types: DocumentedList, + pub public_functions: DocumentedList, + pub usages: DocumentedList, +} + +pub(super) struct DeclarationSection { + pub documentation: SectionDocumentation, + pub code: SectionCode, +} + +pub(super) enum SectionDocumentation { + Type(TypeDocumentation), + Function(FunctionDocumentation), + Model(ModelDocumentation), +} + +pub(super) struct ModelDocumentation { + pub declaration: DocumentValue, + pub signature: Option<(DocumentedList, DocumentedList)>, +} + +pub(super) struct TypeDocumentation { + pub name: DocumentValue, + pub entries: DocumentedList, +} + +pub(super) struct FunctionDocumentation { + pub name: DocumentValue, + pub inputs: DocumentedList, + pub outputs: DocumentedList, + pub fails: Option<DocumentValue>, +} + +pub(super) enum SectionCode { + Type(TypeCode), + Function(FunctionCode), + Model(ModelCode), +} + +pub(super) struct ModelCode { + pub declaration: String, + pub span: Span, + pub sql: bool, +} + +pub(super) struct TypeCode { + pub name: String, + pub entries: TypeEntries, + pub span: Span, +} + +pub(super) struct FunctionCode { + pub name: String, + pub arity: usize, + pub span: Span, + pub input_names: Vec<DocumentValue>, + pub clause_inputs: Vec<Vec<DocumentValue>>, + pub fails: Option<String>, +} + +#[derive(Clone, Copy)] +pub(super) enum TypeEntries { + Fields, + Variants, +} + +pub(super) struct DocumentValue { + pub text: String, + pub span: Span, +} + +pub(super) struct DocumentedList { + pub values: Vec<DocumentValue>, + pub span: Span, +} diff --git a/crates/dever-core/src/markdown/contract/parse.rs b/crates/dever-core/src/markdown/contract/parse.rs new file mode 100644 index 0000000..aa3b403 --- /dev/null +++ b/crates/dever-core/src/markdown/contract/parse.rs @@ -0,0 +1,417 @@ +use std::collections::BTreeMap; +use std::ops::Range; + +use pulldown_cmark::{Event, HeadingLevel, Parser, Tag, TagEnd}; + +use crate::diagnostic::{Diagnostic, Label}; +use crate::source::{SourceFile, Span}; +use crate::syntax::{Declaration, Package, TypeShape}; + +use super::metadata; +use super::model::{ + DeclarationSection, DocumentContract, DocumentValue, FunctionCode, ModelCode, SectionCode, + SectionDocumentation, TypeCode, TypeEntries, +}; +use crate::markdown::{ProgramBlock, declaration_span, normalize_lone_carriage_returns}; + +pub(super) fn contract( + source: &SourceFile, + package: &Package, + blocks: &[ProgramBlock], +) -> Result<DocumentContract, Vec<Diagnostic>> { + let headings = headings(source); + let h1 = headings + .iter() + .enumerate() + .filter(|(_, heading)| heading.level == HeadingLevel::H1) + .collect::<Vec<_>>(); + if h1.len() != 1 { + let primary = h1 + .get(1) + .or_else(|| h1.first()) + .map_or_else(|| source.span(0, 0), |(_, heading)| heading.span); + return Err(vec![Diagnostic::error( + "M004", + "a .dever.md source must contain exactly one top-level package heading ('#')", + primary, + )]); + } + if h1[0].0 != 0 { + return Err(vec![Diagnostic::error( + "M004", + "the package heading ('#') must precede every declaration heading ('##')", + headings[0].span, + )]); + } + if h1[0].1.title.trim().is_empty() { + return Err(vec![Diagnostic::error( + "M004", + "the package heading must have a natural-language name", + h1[0].1.span, + )]); + } + + let section_blocks = match_blocks(source, &headings, blocks)?; + let package_section_end = headings + .get(1) + .map_or(source.text().len(), |heading| heading.range.start); + let package_documentation = metadata::package( + source, + headings[0].range.end..package_section_end, + headings[0].span, + )?; + + let mut sections = Vec::new(); + let mut documented_declarations = BTreeMap::new(); + for (heading_index, heading) in headings.iter().enumerate().skip(1) { + if heading.title.trim().is_empty() { + return Err(vec![Diagnostic::error( + "M005", + "each declaration heading must have a natural-language name", + heading.span, + )]); + } + let block = &blocks[section_blocks[heading_index - 1]]; + let declarations = declarations_in(package, block); + let code = section_code(&declarations, heading.span)?; + reject_duplicate_section(&code, heading.span, &mut documented_declarations)?; + let documentation = match &code { + SectionCode::Type(code) => SectionDocumentation::Type(metadata::type_declaration( + source, + heading.range.end..block.outer.start, + heading.span, + code.entries, + )?), + SectionCode::Function(code) => SectionDocumentation::Function(metadata::function( + source, + heading.range.end..block.outer.start, + heading.span, + code.fails.is_some(), + )?), + SectionCode::Model(code) => SectionDocumentation::Model(metadata::model_declaration( + source, + heading.range.end..block.outer.start, + heading.span, + code.sql, + )?), + }; + sections.push(DeclarationSection { + documentation, + code, + }); + } + + Ok(DocumentContract { + package: package_documentation, + sections, + }) +} + +fn match_blocks( + source: &SourceFile, + headings: &[Heading], + blocks: &[ProgramBlock], +) -> Result<Vec<usize>, Vec<Diagnostic>> { + let mut section_blocks = Vec::new(); + let mut assigned = vec![false; blocks.len()]; + for (index, heading) in headings.iter().enumerate() { + let end = headings + .get(index + 1) + .map_or(source.text().len(), |next| next.range.start); + let matched = blocks + .iter() + .enumerate() + .filter_map(|(block_index, block)| { + (heading.range.end <= block.outer.start && block.outer.start < end) + .then_some(block_index) + }) + .collect::<Vec<_>>(); + let expected = usize::from(heading.level != HeadingLevel::H1); + if matched.len() != expected { + return Err(vec![Diagnostic::error( + if heading.level == HeadingLevel::H1 { + "M004" + } else { + "M005" + }, + format!( + "each '{}' section must contain {expected} top-level Dever code blocks; found {}", + heading_marker(heading.level), + matched.len() + ), + heading.span, + )]); + } + if let Some(&block) = matched.first() { + assigned[block] = true; + section_blocks.push(block); + } + } + if let Some((index, _)) = assigned.iter().enumerate().find(|(_, used)| !**used) { + return Err(vec![Diagnostic::error( + "M005", + "every top-level Dever code block must belong to one '##' declaration section", + source.span(blocks[index].outer.start, blocks[index].outer.end), + )]); + } + Ok(section_blocks) +} + +fn reject_duplicate_section( + code: &SectionCode, + heading: Span, + documented: &mut BTreeMap<String, Span>, +) -> Result<(), Vec<Diagnostic>> { + let (key, code_span) = match code { + SectionCode::Type(code) => (format!("type:{}", code.name), code.span), + SectionCode::Function(code) => { + (format!("function:{}/{}", code.name, code.arity), code.span) + } + SectionCode::Model(code) => (format!("model:{}", code.declaration), code.span), + }; + let Some(first) = documented.insert(key, code_span) else { + return Ok(()); + }; + let mut error = Diagnostic::error( + "M005", + "one declaration or all clauses of one logical function must share exactly one '##' section", + heading, + ); + error.related.extend([ + Label { + span: first, + message: "the declaration was already documented here".into(), + }, + Label { + span: code_span, + message: "duplicate declaration section".into(), + }, + ]); + Err(vec![error]) +} + +fn section_code( + declarations: &[&Declaration], + heading: Span, +) -> Result<SectionCode, Vec<Diagnostic>> { + match declarations { + [Declaration::Type(declaration)] => Ok(SectionCode::Type(TypeCode { + name: declaration.name.text.clone(), + entries: match declaration.shape { + TypeShape::Record(_) => TypeEntries::Fields, + TypeShape::Choice(_) => TypeEntries::Variants, + }, + span: declaration.span, + })), + [Declaration::Function(_), ..] + if declarations + .iter() + .all(|declaration| matches!(declaration, Declaration::Function(_))) => + { + function_code(declarations, heading) + } + [declaration] => Ok(SectionCode::Model(ModelCode { + declaration: model_declaration_label(declaration), + span: declaration_span(declaration), + sql: matches!(declaration, Declaration::ModelSql(_)), + })), + _ => { + let mut error = Diagnostic::error( + "M005", + "each '##' section must contain one type, one Model declaration or all clauses of one function", + heading, + ); + error + .related + .extend(declarations.iter().map(|declaration| Label { + span: declaration_span(declaration), + message: "declaration found in this section".into(), + })); + Err(vec![error]) + } + } +} + +fn model_declaration_label(declaration: &Declaration) -> String { + match declaration { + Declaration::Api(binding) => format!("{} {}", binding.kind.keyword(), binding.action.text), + Declaration::Rest(rest) => match &rest.model { + None => "rest".into(), + Some(model) => format!( + "rest {}", + model + .iter() + .map(|part| part.text.as_str()) + .collect::<Vec<_>>() + .join(".") + ), + }, + Declaration::External(external) => format!( + "external {} {}", + external.ecosystem.keyword(), + external.entry, + ), + Declaration::Database(binding) => format!("database {}", binding.name.text), + Declaration::Schedule(schedule) => format!("schedule {}", schedule.target.text), + Declaration::ModelIndex(index) => format!( + "{}({})", + if index.unique { "unique" } else { "index" }, + index + .fields + .iter() + .map(|field| field.text.as_str()) + .collect::<Vec<_>>() + .join(", ") + ), + Declaration::Relation(relation) => format!( + "relation {} = {}", + relation.name.text, + relation + .field + .iter() + .map(|name| name.text.as_str()) + .collect::<Vec<_>>() + .join(".") + ), + Declaration::Seed(_) => "seed".into(), + Declaration::Migration(migration) => format!("migrate {}", migration.name.text), + Declaration::ModelSql(sql) => format!("sql {}", sql.name.text), + Declaration::Type(_) | Declaration::Function(_) => unreachable!("handled declaration kind"), + } +} + +fn function_code( + declarations: &[&Declaration], + heading: Span, +) -> Result<SectionCode, Vec<Diagnostic>> { + let clauses = declarations + .iter() + .map(|declaration| match declaration { + Declaration::Function(function) => function, + _ => unreachable!(), + }) + .collect::<Vec<_>>(); + let first = clauses[0]; + if clauses.iter().any(|clause| { + clause.name.text != first.name.text || clause.inputs.len() != first.inputs.len() + }) { + let mut error = Diagnostic::error( + "M005", + "one '##' section may contain clauses of only one function and arity", + heading, + ); + error.related.extend(clauses.iter().map(|clause| Label { + span: clause.span, + message: format!( + "function '{}'/{} is in this section", + clause.name.text, + clause.inputs.len() + ), + })); + return Err(vec![error]); + } + let last = clauses.last().expect("nonempty function section"); + Ok(SectionCode::Function(FunctionCode { + name: first.name.text.clone(), + arity: first.inputs.len(), + span: Span { + end: last.span.end, + ..first.span + }, + input_names: input_names(first), + clause_inputs: clauses.iter().map(|clause| input_names(clause)).collect(), + fails: first.fails.as_ref().map(|ty| { + ty.name + .iter() + .map(|name| name.text.as_str()) + .collect::<Vec<_>>() + .join(".") + }), + })) +} + +fn input_names(function: &crate::syntax::FunctionClause) -> Vec<DocumentValue> { + function + .inputs + .iter() + .map(|input| DocumentValue { + text: input.name.text.clone(), + span: input.name.span, + }) + .collect() +} + +fn declarations_in<'a>(package: &'a Package, block: &ProgramBlock) -> Vec<&'a Declaration> { + package + .declarations + .iter() + .filter(|declaration| { + let span = declaration_span(declaration); + block.content.start <= span.start && span.end <= block.content.end + }) + .collect() +} + +fn headings(source: &SourceFile) -> Vec<Heading> { + let markdown = normalize_lone_carriage_returns(source.text()); + let mut depth = 0usize; + let mut current: Option<Heading> = None; + let mut headings = Vec::new(); + for (event, range) in Parser::new(&markdown).into_offset_iter() { + match event { + Event::Start(Tag::Heading { level, .. }) if depth == 0 => { + if matches!(level, HeadingLevel::H1 | HeadingLevel::H2) { + current = Some(Heading { + level, + title: String::new(), + span: source.span(range.start, range.end), + range, + }); + } + depth += 1; + } + Event::Start(_) => depth += 1, + Event::End(TagEnd::Heading(level)) => { + depth = depth.saturating_sub(1); + if current + .as_ref() + .is_some_and(|heading| heading.level == level) + { + headings.push(current.take().expect("matched active heading")); + } + } + Event::End(_) => depth = depth.saturating_sub(1), + Event::Text(text) | Event::Code(text) if current.is_some() => { + current + .as_mut() + .expect("checked active heading") + .title + .push_str(&text); + } + Event::SoftBreak | Event::HardBreak if current.is_some() => { + current + .as_mut() + .expect("checked active heading") + .title + .push(' '); + } + _ => {} + } + } + headings +} + +fn heading_marker(level: HeadingLevel) -> &'static str { + match level { + HeadingLevel::H1 => "#", + HeadingLevel::H2 => "##", + _ => unreachable!("only package and declaration headings are collected"), + } +} + +struct Heading { + level: HeadingLevel, + title: String, + span: Span, + range: Range<usize>, +} diff --git a/crates/dever-core/src/markdown/contract/validate.rs b/crates/dever-core/src/markdown/contract/validate.rs new file mode 100644 index 0000000..175838c --- /dev/null +++ b/crates/dever-core/src/markdown/contract/validate.rs @@ -0,0 +1,593 @@ +use crate::diagnostic::{Diagnostic, Label}; +use crate::hir::Function; +use crate::source::Span; +use crate::syntax::{ + BinaryOperator, Declaration, Field, Input, InputKind, Package, TypeRef, TypeShape, Variant, +}; +use crate::types::{Definition, Parameter, Type}; + +use super::model::{ + DocumentContract, DocumentValue, DocumentedList, FunctionCode, FunctionDocumentation, + ModelCode, ModelDocumentation, SectionCode, SectionDocumentation, TypeCode, TypeDocumentation, + TypeEntries, +}; + +pub(super) fn contract( + contract: &DocumentContract, + package: &Package, + owner: usize, + definitions: &[Definition], + functions: &[Function], +) -> Vec<Diagnostic> { + let package_name = path_name(&package.name); + let mut errors = Vec::new(); + compare_value( + "M006", + "documented package name", + &contract.package.name, + &package_name, + package.name[0].span, + &mut errors, + ); + + let public_types = public_names(package, |declaration| match declaration { + Declaration::Type(ty) + if definitions.iter().any(|definition| { + definition.owner == owner + && definition.public + && definition.name.rsplit('.').next() == Some(ty.name.text.as_str()) + }) => + { + Some(&ty.name.text) + } + _ => None, + }); + let public_functions = public_names(package, |declaration| match declaration { + Declaration::Function(function) + if functions.iter().any(|checked| { + checked.owner == owner + && checked.public + && checked.name.rsplit('.').next() == Some(function.name.text.as_str()) + }) => + { + Some(&function.name.text) + } + _ => None, + }); + compare_list( + "M006", + "documented public types", + &contract.package.public_types, + &public_types, + package.span, + &mut errors, + ); + compare_list( + "M006", + "documented public methods", + &contract.package.public_functions, + &public_functions, + package.span, + &mut errors, + ); + + let mut usages = Vec::new(); + for exposed in &public_functions { + for section in &contract.sections { + let SectionCode::Function(code) = §ion.code else { + continue; + }; + if &code.name == exposed { + let prefix = package + .layout + .app_prefix() + .unwrap_or_else(|| package_name.clone()); + usages.push(format!( + "{prefix}.{}({})", + code.name, + code.input_names + .iter() + .map(|input| input.text.as_str()) + .collect::<Vec<_>>() + .join(", ") + )); + } + } + } + compare_list( + "M006", + "documented public method usage", + &contract.package.usages, + &usages, + package.span, + &mut errors, + ); + + for section in &contract.sections { + match (§ion.documentation, §ion.code) { + (SectionDocumentation::Type(documentation), SectionCode::Type(code)) => { + validate_type_section(documentation, code, package, &mut errors); + } + (SectionDocumentation::Function(documentation), SectionCode::Function(code)) => { + validate_function_section( + documentation, + code, + package, + owner, + definitions, + functions, + &mut errors, + ); + } + (SectionDocumentation::Model(documentation), SectionCode::Model(code)) => { + validate_model_section(documentation, code, package, &mut errors); + } + _ => unreachable!("section documentation follows its declaration kind"), + } + } + errors +} + +fn validate_model_section( + documentation: &ModelDocumentation, + code: &ModelCode, + package: &Package, + errors: &mut Vec<Diagnostic>, +) { + compare_value( + "M006", + "documented Model declaration", + &documentation.declaration, + &code.declaration, + code.span, + errors, + ); + let Some((inputs, outputs)) = &documentation.signature else { + return; + }; + let sql = package + .declarations + .iter() + .find_map(|declaration| match declaration { + Declaration::ModelSql(sql) if sql.span == code.span => Some(sql), + _ => None, + }) + .expect("SQL section maps to its parsed declaration"); + compare_list( + "M007", + "documented SQL inputs", + inputs, + &sql.inputs.iter().map(field_label).collect::<Vec<_>>(), + code.span, + errors, + ); + compare_list( + "M008", + "documented SQL outputs", + outputs, + &sql.outputs.iter().map(field_label).collect::<Vec<_>>(), + code.span, + errors, + ); +} + +fn public_names(package: &Package, name: impl Fn(&Declaration) -> Option<&String>) -> Vec<String> { + let mut names = Vec::new(); + for name in package.declarations.iter().filter_map(name) { + if !names.contains(name) { + names.push(name.clone()); + } + } + names +} + +fn validate_type_section( + documentation: &TypeDocumentation, + code: &TypeCode, + package: &Package, + errors: &mut Vec<Diagnostic>, +) { + compare_value( + "M006", + "documented type name", + &documentation.name, + &code.name, + code.span, + errors, + ); + let declaration = package + .declarations + .iter() + .find_map(|declaration| match declaration { + Declaration::Type(ty) if ty.span.start == code.span.start => Some(ty), + _ => None, + }) + .expect("document type section maps to a parsed type"); + let expected = match &declaration.shape { + TypeShape::Record(fields) => fields.iter().map(field_label).collect::<Vec<_>>(), + TypeShape::Choice(variants) => variants.iter().map(variant_label).collect::<Vec<_>>(), + }; + compare_list( + "M008", + match code.entries { + TypeEntries::Fields => "documented type fields", + TypeEntries::Variants => "documented choice variants", + }, + &documentation.entries, + &expected, + code.span, + errors, + ); +} + +fn validate_function_section( + documentation: &FunctionDocumentation, + code: &FunctionCode, + package: &Package, + owner: usize, + definitions: &[Definition], + functions: &[Function], + errors: &mut Vec<Diagnostic>, +) { + compare_value( + "M006", + "documented function name", + &documentation.name, + &code.name, + code.span, + errors, + ); + validate_clause_input_names(code, errors); + if let (Some(documented), Some(expected)) = (&documentation.fails, &code.fails) { + compare_value( + "M008", + "documented Port failure choice", + documented, + expected, + code.span, + errors, + ); + } + + let function = functions + .iter() + .find(|function| { + function.owner == owner + && function.span.start == code.span.start + && function.parameters.len() == code.arity + }) + .expect("document function section maps to a checked function"); + let clause = package + .declarations + .iter() + .find_map(|declaration| match declaration { + Declaration::Function(function) if function.span.start == code.span.start => { + Some(function) + } + _ => None, + }) + .expect("document function section maps to a parsed clause"); + let inputs = clause + .inputs + .iter() + .zip(&function.parameters) + .map(|(source, parameter)| parameter_label(source, parameter, definitions, owner)) + .collect::<Vec<_>>(); + let outputs = function + .outputs + .iter() + .map(|field| { + format!( + "{}: {}", + field.name, + type_label(&field.ty, definitions, owner) + ) + }) + .collect::<Vec<_>>(); + compare_list( + "M007", + "documented function inputs", + &documentation.inputs, + &inputs, + code.span, + errors, + ); + compare_list( + "M008", + "documented function outputs", + &documentation.outputs, + &outputs, + code.span, + errors, + ); +} + +fn validate_clause_input_names(code: &FunctionCode, errors: &mut Vec<Diagnostic>) { + for clause_inputs in &code.clause_inputs[1..] { + for (first, candidate) in code.input_names.iter().zip(clause_inputs) { + if first.text != candidate.text { + errors.push(mismatch( + "M007", + format!( + "all clauses documented by one section must use input name '{}'; found '{}'", + first.text, candidate.text + ), + candidate.span, + first.span, + )); + } + } + } +} + +fn parameter_label( + source: &Input, + parameter: &Parameter, + definitions: &[Definition], + owner: usize, +) -> String { + match (parameter, &source.kind) { + (Parameter::Value(ty), InputKind::Value(_)) => { + format!( + "{}: {}", + source.name.text, + type_label(ty, definitions, owner) + ) + } + (Parameter::Handler(signature), InputKind::Handler(source_signature)) => { + let inputs = source_signature + .inputs + .iter() + .zip(&signature.parameters) + .map(|(field, ty)| { + format!( + "{}: {}", + field.name.text, + type_label(ty, definitions, owner) + ) + }) + .collect::<Vec<_>>() + .join(", "); + let outputs = signature + .outputs + .iter() + .map(|field| { + format!( + "{}: {}", + field.name, + type_label(&field.ty, definitions, owner) + ) + }) + .collect::<Vec<_>>() + .join(", "); + format!("{}: handler({inputs}) ({outputs})", source.name.text) + } + _ => unreachable!("checked parameter preserves its source input kind"), + } +} + +fn type_label(ty: &Type, definitions: &[Definition], owner: usize) -> String { + match ty { + Type::Named(id) => { + let definition = &definitions[*id]; + if definition.owner == owner { + definition + .name + .rsplit_once('.') + .map_or_else(|| definition.name.clone(), |(_, name)| name.to_owned()) + } else { + definition.name.clone() + } + } + Type::Nullable(base) => format!("{}?", type_label(base, definitions, owner)), + Type::List(element) => format!("List<{}>", type_label(element, definitions, owner)), + Type::Map(key, value) => format!( + "Map<{}, {}>", + type_label(key, definitions, owner), + type_label(value, definitions, owner) + ), + Type::MapEntry(key, value) => format!( + "MapEntry<{}, {}>", + type_label(key, definitions, owner), + type_label(value, definitions, owner) + ), + Type::Stream(element) => format!("Stream<{}>", type_label(element, definitions, owner)), + Type::AsyncStream(element) => { + format!("AsyncStream<{}>", type_label(element, definitions, owner)) + } + Type::RowStream(element) => { + format!("AsyncStream<{}>", type_label(element, definitions, owner)) + } + Type::Related(element) => { + format!("Related<{}>", type_label(element, definitions, owner)) + } + Type::Channel(element) => { + format!("Channel<{}>", type_label(element, definitions, owner)) + } + Type::Task(_) | Type::Group => unreachable!("affine types cannot enter Markdown APIs"), + Type::File => "dever.system.File".into(), + Type::Socket => "dever.system.Socket".into(), + Type::Listener => "dever.system.Listener".into(), + Type::HttpReply => "dever.system.HttpReply".into(), + Type::WebSocket => "dever.system.WebSocket".into(), + Type::ClientTls => "dever.system.ClientTls".into(), + Type::ServerTls => "dever.system.ServerTls".into(), + Type::HttpClient => "dever.system.HttpClient".into(), + Type::Outputs(_) => "named outputs".into(), + _ => format!("{ty:?}"), + } +} + +fn field_label(field: &Field) -> String { + format!( + "{}{}: {}{}", + if field.private { "private " } else { "" }, + field.name.text, + type_ref_label(&field.ty), + bounds_label(&field.bounds) + ) +} + +fn variant_label(variant: &Variant) -> String { + let payload = if variant.payload.is_empty() { + String::new() + } else { + format!( + "({})", + variant + .payload + .iter() + .map(field_label) + .collect::<Vec<_>>() + .join(", ") + ) + }; + format!( + "{}{}{}", + if variant.error { "error " } else { "" }, + variant.name.text, + payload + ) +} + +fn type_ref_label(ty: &TypeRef) -> String { + let mut label = path_name(&ty.name); + if !ty.parameters.is_empty() { + label.push('('); + label.push_str( + &ty.parameters + .iter() + .map(|parameter| parameter.value.as_str()) + .collect::<Vec<_>>() + .join(", "), + ); + label.push(')'); + } + if !ty.arguments.is_empty() { + label.push('<'); + label.push_str( + &ty.arguments + .iter() + .map(type_ref_label) + .collect::<Vec<_>>() + .join(", "), + ); + label.push('>'); + } + if ty.nullable { + label.push('?'); + } + label +} + +fn bounds_label(bounds: &[crate::syntax::Bound]) -> String { + bounds + .iter() + .enumerate() + .map(|(index, bound)| { + format!( + "{}{} {}", + if index == 0 { " " } else { " and " }, + operator_label(bound.comparison), + bound.number + ) + }) + .collect() +} + +fn operator_label(operator: BinaryOperator) -> &'static str { + match operator { + BinaryOperator::Equal => "==", + BinaryOperator::NotEqual => "!=", + BinaryOperator::Less => "<", + BinaryOperator::LessEqual => "<=", + BinaryOperator::Greater => ">", + BinaryOperator::GreaterEqual => ">=", + _ => unreachable!("parser permits only comparisons in bounds"), + } +} + +fn path_name(path: &crate::syntax::Path) -> String { + path.iter() + .map(|name| name.text.as_str()) + .collect::<Vec<_>>() + .join(".") +} + +fn compare_value( + code: &'static str, + subject: &str, + actual: &DocumentValue, + expected: &str, + related: Span, + errors: &mut Vec<Diagnostic>, +) { + if actual.text != expected { + errors.push(mismatch( + code, + format!("{subject} must be `{expected}`; found `{}`", actual.text), + actual.span, + related, + )); + } +} + +fn compare_list( + code: &'static str, + subject: &str, + actual: &DocumentedList, + expected: &[String], + related: Span, + errors: &mut Vec<Diagnostic>, +) { + let actual_values = actual + .values + .iter() + .map(|value| value.text.as_str()) + .collect::<Vec<_>>(); + let expected_values = expected.iter().map(String::as_str).collect::<Vec<_>>(); + if actual_values == expected_values { + return; + } + let expected = if expected.is_empty() { + "无".into() + } else { + expected + .iter() + .map(|value| format!("`{value}`")) + .collect::<Vec<_>>() + .join(", ") + }; + let primary = actual + .values + .iter() + .zip(expected_values.iter()) + .find_map(|(actual, expected)| (actual.text != *expected).then_some(actual.span)) + .or_else(|| { + actual + .values + .get(expected_values.len()) + .map(|value| value.span) + }) + .unwrap_or(actual.span); + errors.push(mismatch( + code, + format!("{subject} must exactly match the code in order; expected {expected}"), + primary, + related, + )); +} + +fn mismatch( + code: &'static str, + message: impl Into<String>, + primary: Span, + related: Span, +) -> Diagnostic { + let mut diagnostic = Diagnostic::error(code, message, primary); + diagnostic.related.push(Label { + span: related, + message: "corresponding Dever declaration".into(), + }); + diagnostic +} diff --git a/crates/dever-core/src/model.rs b/crates/dever-core/src/model.rs new file mode 100644 index 0000000..954bc75 --- /dev/null +++ b/crates/dever-core/src/model.rs @@ -0,0 +1,641 @@ +use std::collections::BTreeMap; + +use crate::hir::Program; +use crate::types::{Definition, Shape}; +use crate::wire::{Policy, Schema}; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ModelSchema { + pub record: usize, + pub page: usize, + pub cursor: usize, + pub package: String, + pub name: String, + pub scope: ModelScope, + pub table: String, + pub connection: ConnectionSelector, + pub fields: Vec<ModelField>, + pub choices: Vec<ModelChoice>, + pub indexes: Vec<ModelIndex>, + pub relations: Vec<ModelRelation>, + pub sql: Vec<ModelSql>, + pub seed: Vec<SeedRow>, + pub migrations: Vec<ModelMigration>, + pub revision: String, + pub seed_revision: String, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ModelScope { + Global, + Tenant, +} + +impl ModelScope { + pub(crate) fn name(self) -> &'static str { + match self { + Self::Global => "global", + Self::Tenant => "tenant", + } + } +} + +#[derive(Clone, Debug)] +pub(crate) struct RestContract { + pub read: Vec<usize>, + pub create: Vec<RestWriteField>, + pub replace: Vec<RestWriteField>, + pub owner: Option<(usize, RestOwnerSource)>, + pub search: Option<RestSearch>, +} + +#[derive(Clone, Debug)] +pub(crate) struct RestWriteField { + pub field: usize, + pub source: RestValueSource, +} + +#[derive(Clone, Debug)] +pub(crate) enum RestValueSource { + Input, + Binding { + expression: crate::hir::Expression, + uses_input: bool, + }, +} + +#[derive(Clone, Debug)] +pub(crate) struct RestSearch { + pub field: usize, + pub expression: crate::hir::Expression, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum RestOwnerSource { + User, + Tenant, +} + +impl ModelSchema { + pub(crate) fn rest_contract(&self, types: &[Definition]) -> Result<RestContract, String> { + let Shape::Record(fields) = &types[self.record].shape else { + unreachable!("checked Model record") + }; + let mut contract = RestContract { + read: Vec::new(), + create: Vec::new(), + replace: Vec::new(), + owner: None, + search: None, + }; + for (index, (storage, field)) in self.fields.iter().zip(fields).enumerate() { + if let Some(owner) = storage.owner { + contract.owner = Some((index, owner)); + } + if field.private { + continue; + } + Schema::build(&field.ty, types, Policy::Output).map_err(|error| { + format!( + "REST field '{}.{}' cannot be serialized: {error}", + self.package, storage.name + ) + })?; + contract.read.push(index); + if storage.owner.is_some() { + continue; + } + if storage.generated { + continue; + } + Schema::build(&field.ty, types, Policy::ApiInput).map_err(|error| { + format!( + "REST field '{}.{}' cannot be read from a request: {error}", + self.package, storage.name + ) + })?; + contract.replace.push(RestWriteField { + field: index, + source: RestValueSource::Input, + }); + if storage.default.is_none() { + contract.create.push(RestWriteField { + field: index, + source: RestValueSource::Input, + }); + } + } + if contract.replace.is_empty() { + return Err(format!( + "REST Model '{}' has no writable public fields", + self.package + )); + } + Ok(contract) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum ConnectionSelector { + Explicit(String), + PackageRoot(String), +} + +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub(crate) enum DatabaseOwner { + Model(usize), + Job(usize), +} + +impl Program { + pub(crate) fn rest_binding_roots(&self) -> std::collections::BTreeSet<usize> { + let mut roots = std::collections::BTreeSet::new(); + for route in &self.api_rest { + for expression in route + .contract + .create + .iter() + .chain(&route.contract.replace) + .filter_map(|write| match &write.source { + RestValueSource::Binding { expression, .. } => Some(expression), + RestValueSource::Input => None, + }) + .chain( + route + .contract + .search + .iter() + .map(|search| &search.expression), + ) + { + crate::check::visit_expression(expression, |value| { + if let Some((target, arguments)) = value.kind.static_call() { + if let crate::hir::CallTarget::Function(target) = target { + roots.insert(target); + } + for argument in arguments { + if let crate::hir::CallArgument::Handler( + crate::hir::HandlerTarget::Function(target), + ) = argument + { + roots.insert(*target); + } + } + } + if let crate::hir::ExpressionKind::Collection { + handler: Some(crate::hir::HandlerTarget::Function(target)), + .. + } = &value.kind + { + roots.insert(*target); + } + }); + } + } + roots + } + + pub(crate) fn database_binding(&self, owner: DatabaseOwner) -> (&ConnectionSelector, &str) { + match owner { + DatabaseOwner::Model(id) => (&self.models[id].connection, &self.models[id].package), + DatabaseOwner::Job(function) => { + let job = self + .jobs + .iter() + .find(|job| job.function == function) + .expect("checked Job target"); + (&job.connection, &self.functions[function].name) + } + } + } + + pub(crate) fn database_selector(&self, owner: DatabaseOwner) -> (Option<&str>, &str) { + let (connection, identity) = self.database_binding(owner); + match connection { + ConnectionSelector::Explicit(name) => (Some(name), identity.split('.').next().unwrap()), + ConnectionSelector::PackageRoot(root) => (None, root), + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ModelField { + pub name: String, + pub ty: ModelFieldType, + pub nullable: bool, + pub generated: bool, + pub default: Option<ModelValue>, + pub rename_from: Option<String>, + pub logical_reference: bool, + pub owner: Option<RestOwnerSource>, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum ModelFieldType { + Bool, + Int, + Float, + Decimal { precision: u8, scale: u8 }, + Text { minimum: u32, maximum: Option<u32> }, + Bytes { minimum: u32, maximum: Option<u32> }, + Uuid, + DateTime, + Date, + Time, + Duration, + Json, + Choice(String), + ModelId(String), +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum ModelValue { + Null, + Bool(bool), + Number(String), + Text(String), + Choice(String), +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ModelChoice { + pub name: String, + pub options: Vec<(String, Option<String>)>, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ModelIndex { + pub fields: Vec<String>, + pub unique: bool, + pub implicit: bool, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ModelRelation { + pub name: String, + pub model: String, + pub field: String, + pub kind: ModelRelationKind, + pub logical: bool, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ModelRelationKind { + ToOne, + ToMany, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ModelSql { + pub name: String, + pub parameters: Vec<ModelSqlParameter>, + pub result: ModelSqlResult, + pub sqlite: String, + pub postgres: String, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ModelSqlParameter { + pub name: String, + pub ty: ModelFieldType, + pub nullable: bool, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct ModelSqlResult { + pub record: usize, + pub cardinality: ModelSqlCardinality, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ModelSqlCardinality { + One, + Optional, + Many, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct SeedRow { + pub fields: Vec<(String, ModelValue)>, + pub identity: Vec<String>, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ModelMigration { + pub name: String, + pub drops: Vec<String>, + pub statements: Vec<ModelMigrationSql>, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ModelMigrationSql { + pub phase: crate::syntax::MigrationPhase, + pub sqlite: String, + pub postgres: String, + pub parameters: Vec<(ModelFieldType, ModelValue)>, +} + +impl ModelMigration { + pub fn revision(&self) -> String { + let identity = if self.statements.is_empty() { + format!("{}:{}", self.name, self.drops.join(",")) + } else { + format!("{:?}:{:?}:{:?}", self.name, self.drops, self.statements) + }; + checksum(&identity) + } +} + +impl Program { + /// Stable logical schema output. SQL dialect details and source locations are excluded. + pub fn model_snapshot(&self) -> String { + let mut models = self.models.iter().collect::<Vec<_>>(); + models.sort_by(|left, right| left.package.cmp(&right.package)); + let mut output = String::from("dever-model 1\n"); + for model in models { + output.push_str(&format!( + "model {} name={} table={} connection={} scope={} revision={} seed={}\n", + model.package, + model.name, + model.table, + connection(&model.connection), + model.scope.name(), + model.revision, + model.seed_revision, + )); + for choice in &model.choices { + let options = choice + .options + .iter() + .map(|(name, label)| match label { + Some(label) => format!("{name}={label:?}"), + None => name.clone(), + }) + .collect::<Vec<_>>() + .join(","); + output.push_str(&format!("choice {} [{options}]\n", choice.name)); + } + for field in &model.fields { + let reference = if field.logical_reference { + " reference=logical" + } else { + "" + }; + let owner = match field.owner { + Some(RestOwnerSource::User) => " owner=auth.user_id", + Some(RestOwnerSource::Tenant) => " owner=auth.tenant_id", + None => "", + }; + output.push_str(&format!( + "field {} {} nullable={} generated={} default={} from={}{reference}{owner}\n", + field.name, + field_type(&field.ty), + field.nullable, + field.generated, + field.default.as_ref().map(value).unwrap_or("-".into()), + field.rename_from.as_deref().unwrap_or("-"), + )); + } + for index in &model.indexes { + output.push_str(&format!( + "index {} ({}) implicit={}\n", + if index.unique { "unique" } else { "plain" }, + index.fields.join(","), + index.implicit, + )); + } + for relation in &model.relations { + let kind = match relation.kind { + ModelRelationKind::ToOne => "to-one", + ModelRelationKind::ToMany => "to-many", + }; + output.push_str(&format!( + "relation {}{} {} = {}.{}\n", + if relation.logical { "logical " } else { "" }, + kind, + relation.name, + relation.model, + relation.field + )); + } + for sql in &model.sql { + output.push_str(&format!( + "sql {} ({}) -> {:?}:{}\n", + sql.name, + sql.parameters + .iter() + .map(|parameter| format!( + "{}:{}{}", + parameter.name, + field_type(¶meter.ty), + if parameter.nullable { "?" } else { "" } + )) + .collect::<Vec<_>>() + .join(","), + sql.result.cardinality, + self.types[sql.result.record].name, + )); + } + for row in &model.seed { + output.push_str(&format!( + "seed key({}) {{{}}}\n", + row.identity.join(","), + row.fields + .iter() + .map(|(name, item)| format!("{name}={}", value(item))) + .collect::<Vec<_>>() + .join(",") + )); + } + for migration in &model.migrations { + output.push_str(&format!( + "migrate {} drop({})\n", + migration.name, + migration.drops.join(",") + )); + if !migration.statements.is_empty() { + output.push_str(&format!( + "migration-data {} revision={}\n", + migration.name, + migration.revision() + )); + } + } + } + output + } + + pub fn validate_database_settings( + &self, + settings: &dever_runtime::config::Settings, + ) -> Result<dever_runtime::config::RuntimeProfile, String> { + self.validate_database_bindings(&settings.compilation_bindings()) + } + + pub fn validate_database_bindings( + &self, + settings: &dever_runtime::config::CompilationBindings, + ) -> Result<dever_runtime::config::RuntimeProfile, String> { + for job in &self.jobs { + let (explicit, root) = self.database_selector(DatabaseOwner::Job(job.function)); + settings.resolve_database(explicit, root)?; + } + let connections = self + .models + .iter() + .map(|model| resolve_connection(settings, model)) + .collect::<Result<Vec<_>, _>>()?; + let model_ids = self + .models + .iter() + .enumerate() + .map(|(index, model)| (model.package.as_str(), index)) + .collect::<BTreeMap<_, _>>(); + let mut tables = BTreeMap::new(); + for (index, model) in self.models.iter().enumerate() { + if matches!( + connections[index].1, + dever_runtime::config::DatabaseDriver::Postgres + ) { + validate_postgres_identifiers(model)?; + } + if let Some(existing) = tables.insert( + (connections[index].0, model.table.as_str()), + model.package.as_str(), + ) { + return Err(format!( + "Models '{}' and '{}' resolve to database '{}' and share table '{}'", + existing, model.package, connections[index].0, model.table + )); + } + for field in &model.fields { + let ModelFieldType::ModelId(target) = &field.ty else { + continue; + }; + let target = *model_ids + .get(target.as_str()) + .expect("validated Model foreign key"); + if connections[index].0 != connections[target].0 { + return Err(format!( + "foreign key '{}.{}' crosses database connections '{}' and '{}'", + model.package, field.name, connections[index].0, connections[target].0 + )); + } + } + } + let instances = crate::specialize::concrete(self) + .map_err(|_| "recursive specialized handler calls are not allowed".to_owned())?; + for instance in instances { + let function = &self.functions[instance.function]; + if function.kind != crate::syntax::FunctionKind::Transaction { + continue; + } + let mut connection = None; + for owner in crate::specialize::database_effects(self, &instance) { + let (explicit, root) = self.database_selector(owner); + let name = settings.resolve_database(explicit, root)?.0; + if connection + .replace(name) + .is_some_and(|current| current != name) + { + return Err(format!( + "transaction '{}' spans multiple database connections", + function.name + )); + } + } + } + Ok(settings.profile()) + } +} + +fn resolve_connection<'a>( + settings: &'a dever_runtime::config::CompilationBindings, + model: &ModelSchema, +) -> Result<(&'a str, &'a dever_runtime::config::DatabaseDriver), String> { + match &model.connection { + ConnectionSelector::Explicit(name) => settings.resolve_database(Some(name), &model.package), + ConnectionSelector::PackageRoot(root) => settings.resolve_database(None, root), + } +} + +fn validate_postgres_identifiers(model: &ModelSchema) -> Result<(), String> { + let identifiers = std::iter::once(("table", model.table.as_str())) + .chain(model.fields.iter().flat_map(|field| { + std::iter::once(("field", field.name.as_str())).chain( + field + .rename_from + .iter() + .map(|name| ("renamed field", name.as_str())), + ) + })) + .chain(model.migrations.iter().flat_map(|migration| { + migration + .drops + .iter() + .map(|name| ("dropped field", name.as_str())) + })); + for (kind, identifier) in identifiers { + if identifier.len() > 63 { + return Err(format!( + "PostgreSQL {kind} identifier '{}.{}' exceeds 63 bytes", + model.package, identifier + )); + } + } + Ok(()) +} + +fn connection(selector: &ConnectionSelector) -> String { + match selector { + ConnectionSelector::Explicit(name) => format!("explicit:{name}"), + ConnectionSelector::PackageRoot(name) => format!("package:{name}"), + } +} + +pub(crate) fn field_type(ty: &ModelFieldType) -> String { + match ty { + ModelFieldType::Bool => "Bool".into(), + ModelFieldType::Int => "Int".into(), + ModelFieldType::Float => "Float".into(), + ModelFieldType::Decimal { precision, scale } => format!("Decimal({precision},{scale})"), + ModelFieldType::Text { minimum, maximum } => bounded("Text", *minimum, *maximum), + ModelFieldType::Bytes { minimum, maximum } => bounded("Bytes", *minimum, *maximum), + ModelFieldType::Uuid => "Uuid".into(), + ModelFieldType::DateTime => "DateTime".into(), + ModelFieldType::Date => "Date".into(), + ModelFieldType::Time => "Time".into(), + ModelFieldType::Duration => "Duration".into(), + ModelFieldType::Json => "Json".into(), + ModelFieldType::Choice(name) => format!("choice:{name}"), + ModelFieldType::ModelId(model) => format!("id:{model}"), + } +} + +fn bounded(name: &str, minimum: u32, maximum: Option<u32>) -> String { + match (minimum, maximum) { + (0, None) => name.into(), + (0, Some(maximum)) => format!("{name}({maximum})"), + (minimum, Some(maximum)) => format!("{name}({minimum},{maximum})"), + (_, None) => unreachable!("minimum requires a maximum"), + } +} + +pub(crate) fn value(value: &ModelValue) -> String { + match value { + ModelValue::Null => "null".into(), + ModelValue::Bool(value) => value.to_string(), + ModelValue::Number(value) => value.clone(), + ModelValue::Text(value) => format!("{value:?}"), + ModelValue::Choice(value) => value.clone(), + } +} + +pub(crate) fn checksum(text: &str) -> String { + let mut hash = 0xcbf29ce484222325u64; + for byte in text.bytes() { + hash ^= u64::from(byte); + hash = hash.wrapping_mul(0x100000001b3); + } + format!("{hash:016x}") +} diff --git a/crates/dever-core/src/native.rs b/crates/dever-core/src/native.rs new file mode 100644 index 0000000..0d6ed06 --- /dev/null +++ b/crates/dever-core/src/native.rs @@ -0,0 +1,1375 @@ +use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::fmt::Write; + +use crate::hir::{self, Atom, Constant, Program, Projection, Statement}; +use crate::source::{SourceMap, Span}; +use crate::specialize::{self, Specialization}; +use crate::types::{Parameter, Shape, Type, output_type}; + +mod api; +mod build; +mod collections; +mod expressions; +mod http; +mod intrinsics; +pub(crate) mod jobs; +pub(crate) mod liveness; +pub(crate) mod orm; +mod ports; +mod records; +mod rest; +mod sequences; +pub(crate) mod updates; +pub mod wire; +pub use build::{ + CleanupSummary, NativeProgram, OPTIMIZATION_ARGS, clean_project_artifacts, compile, + compile_application, compile_artifact, compile_project, compile_project_with_resources, + compile_test_suite, compile_with_cache, +}; + +struct Emitter<'a> { + program: &'a Program, + sources: &'a SourceMap, + names: &'a BTreeMap<Specialization, String>, + bindings: BTreeMap<usize, usize>, + moves: HashSet<*const hir::Expression>, + async_context: bool, +} + +type EmittedDefinitions = ( + String, + BTreeSet<Specialization>, + BTreeMap<Specialization, String>, +); + +/// Verified external Worker bytes supplied by the offline packager. The core +/// compiler only serializes them into the generated native program; it does +/// not resolve registries or inspect a machine cache. +#[derive(Clone, Debug)] +pub struct EmbeddedResource { + pub path: String, + pub bytes: Vec<u8>, + pub sha256: String, + pub executable: bool, +} + +#[derive(Clone, Copy)] +enum DatabaseContext { + Inherit, + Detached, +} + +/// Generated Rust is a disposable backend artifact, not a public source format. +pub fn emit(program: &Program, sources: &SourceMap, entry: &str) -> Result<String, String> { + emit_with_profile(program, sources, entry, None) +} + +fn emit_with_profile( + program: &Program, + sources: &SourceMap, + entry: &str, + profile: Option<dever_runtime::config::RuntimeProfile>, +) -> Result<String, String> { + let entry = program.entry(entry)?; + emit_entry(program, sources, entry, profile) +} + +pub fn emit_project( + program: &Program, + sources: &SourceMap, + profile: dever_runtime::config::RuntimeProfile, +) -> Result<String, String> { + emit_project_with_resources(program, sources, profile, &[]) +} + +pub fn emit_project_with_resources( + program: &Program, + sources: &SourceMap, + profile: dever_runtime::config::RuntimeProfile, + resources: &[EmbeddedResource], +) -> Result<String, String> { + if !program.permissions.is_empty() && !profile.sqlite && !profile.postgres { + return Err("protected API permissions require a configured database".into()); + } + let serve_api = !program.api_routes.is_empty() || !program.api_rest.is_empty(); + let serve_jobs = !program.jobs.is_empty(); + if !serve_api && !serve_jobs && program.api_commands.is_empty() { + return Err("application has no API, Job, or CMD entry".into()); + } + let span = program + .api_rest + .first() + .map(|entry| entry.span) + .or_else(|| { + program + .api_routes + .first() + .map(|entry| program.functions[entry.function].span) + }) + .or_else(|| { + program + .api_commands + .first() + .map(|entry| program.functions[entry.function].span) + }) + .or_else(|| { + program + .jobs + .first() + .map(|entry| program.functions[entry.function].span) + }) + .expect("application entry exists"); + let roots = program + .api_routes + .iter() + .map(|route| route.function) + .chain(program.api_commands.iter().map(|command| command.function)) + .chain(program.jobs.iter().map(|job| job.function)) + .chain(program.auth.iter().map(|hook| hook.function)) + .chain(program.rest_binding_roots()) + .map(|function| Specialization { + function, + handlers: Vec::new(), + }) + .collect(); + let (mut output, instances, names) = + emit_definitions(program, sources, roots, Some(profile), serve_api)?; + let emitter = Emitter { + program, + sources, + names: &names, + bindings: BTreeMap::new(), + moves: HashSet::new(), + async_context: false, + }; + api::emit_commands(program, &mut output); + emit_command_tenant_scopes(program, &mut output); + output.push_str("async fn entry() -> Result<(), AppError> {\nlet args = std::env::args().skip(1).collect::<Vec<_>>();\n"); + let cleanup_database = profile.sqlite || profile.postgres; + let cleanup_components = ports::has_external(program, &instances); + if cleanup_database || cleanup_components { + output.push_str("let application: Result<(), AppError> = async {\n"); + } + output.push_str("let service_start = args.is_empty();\n"); + if !resources.is_empty() { + output.push_str("dever_runtime::external::prepare(&dever_runtime::config::executable_directory()?, DEVER_EXTERNAL_BUNDLE, DEVER_EXTERNAL_RESOURCES).map_err(|error| AppError::fault(\"external resources\", error))?;\n"); + } + emit_database_setup( + &emitter, + &mut output, + &instances, + profile, + span, + Some((serve_api, serve_jobs)), + ); + if !program.permissions.is_empty() { + output.push_str("if service_start { authorization_initialize().await?; }\n"); + } + output.push_str("if args.first().is_some_and(|argument| argument == \"--dever-tenant-migrate\") {\nif args.len() != 2 { return Err(AppError::fault(\"tenant migration\", \"expected --dever-tenant-migrate and one positive tenant id\")); }\nlet tenant_id = args[1].parse::<i64>().map_err(|_| AppError::fault(\"tenant migration\", \"tenant id must be a positive integer\"))?;\nif tenant_id <= 0 { return Err(AppError::fault(\"tenant migration\", \"tenant id must be positive\")); }\n"); + if profile.sqlite || profile.postgres { + output.push_str("tenant_migrate(tenant_id).await?;\nreturn Ok(());\n"); + } else { + output.push_str("return Err(AppError::fault(\"tenant migration\", \"application has no database runtime\"));\n"); + } + output.push_str("}\n"); + if !program.permissions.is_empty() { + output.push_str("if args.first().is_some_and(|argument| argument == \"--dever-tenant-owner\") {\nif args.len() != 4 { return Err(AppError::fault(\"tenant owner\", \"expected --dever-tenant-owner, one positive tenant id, one site, and one positive user id\")); }\nlet tenant_id = args[1].parse::<i64>().map_err(|_| AppError::fault(\"tenant owner\", \"tenant id must be a positive integer\"))?;\nlet user_id = args[3].parse::<i64>().map_err(|_| AppError::fault(\"tenant owner\", \"user id must be a positive integer\"))?;\nif tenant_id <= 0 || user_id <= 0 { return Err(AppError::fault(\"tenant owner\", \"tenant id and user id must be positive\")); }\ntenant_owner(tenant_id, &args[2], user_id).await?;\nreturn Ok(());\n}\n"); + } + output.push_str("if args.first().is_some_and(|argument| argument == \"--dever-tenant-component\") {\nif args.len() != 4 { return Err(AppError::fault(\"tenant component\", \"expected --dever-tenant-component, one positive tenant id, enable or disable, and one component\")); }\nlet tenant_id = args[1].parse::<i64>().map_err(|_| AppError::fault(\"tenant component\", \"tenant id must be a positive integer\"))?;\nif tenant_id <= 0 { return Err(AppError::fault(\"tenant component\", \"tenant id must be positive\")); }\n"); + if program.tenant_components.is_empty() { + output.push_str("return Err(AppError::fault(\"tenant component\", \"application has no tenant components\"));\n"); + } else { + output.push_str("match args[2].as_str() {\n\"enable\" => dever_runtime::tenant::enable_component(tenant_id, &args[3], TENANT_COMPONENTS).await.map_err(|error| AppError::database(\"tenant component\", error))?,\n\"disable\" => dever_runtime::tenant::disable_component(tenant_id, &args[3], TENANT_COMPONENTS).await.map_err(|error| AppError::database(\"tenant component\", error))?,\n_ => return Err(AppError::fault(\"tenant component\", \"operation must be enable or disable\")),\n}\nreturn Ok(());\n"); + } + output.push_str("}\n"); + ports::initialize(&emitter, &mut output, &instances, span)?; + output.push_str("if service_start {\n"); + if serve_api || serve_jobs { + let service = match (serve_api, serve_jobs) { + (true, true) => { + "dever_runtime::lifecycle::serve_both(dever_runtime::api::serve(api_route), job_serve()).await" + } + (true, false) => "dever_runtime::api::serve(api_route).await", + (false, true) => "job_serve().await", + (false, false) => unreachable!(), + }; + writeln!(output, "dever_runtime::lifecycle::run(async move {{ {service}.map_err(AppError::from) }}).await?;") + .expect("string formatting"); + } else { + output.push_str("return Err(AppError::fault(\"command\", \"application has no service entry; select a CMD and JSON input\"));\n"); + } + output.push_str("} else {\n"); + if !program.api_commands.is_empty() { + output.push_str("let data = if args.first().is_some_and(|argument| argument == \"--dever-tenant-cmd\") {\n"); + output.push_str("if args.len() != 4 { return Err(AppError::fault(\"command\", \"expected --dever-tenant-cmd, one positive tenant id, CMD name, and one JSON object\")); }\n"); + output.push_str("let tenant_id = args[1].parse::<i64>().map_err(|_| AppError::fault(\"command\", \"tenant id must be a positive integer\"))?;\nif tenant_id <= 0 { return Err(AppError::fault(\"command\", \"tenant id must be a positive integer\")); }\n"); + output.push_str("match cmd_requires_tenant(&args[2]) {\nSome(true) => {},\nSome(false) => return Err(AppError::fault(\"command\", \"selected CMD does not use tenant storage; remove --tenant\")),\nNone => return Err(AppError::fault(\"command\", format!(\"unknown CMD '{}'\", args[2]))),\n}\n"); + output.push_str("if dever_runtime::config::settings().tenant().is_none() { return Err(AppError::fault(\"command\", \"tenant storage is not configured\")); }\n"); + if !cleanup_database { + output.push_str("return Err(AppError::fault(\"command\", \"application has no tenant database runtime\"))\n"); + } else if program.jobs.is_empty() { + output.push_str("dever_runtime::tenant::scope(tenant_id, cmd_dispatch(&args[2], &args[3])).await?\n"); + } else { + output.push_str("dever_runtime::job::scope_system(Some(tenant_id), dever_runtime::tenant::scope(tenant_id, cmd_dispatch(&args[2], &args[3]))).await?\n"); + } + output.push_str("} else {\n"); + output.push_str("if args.len() != 2 { return Err(AppError::fault(\"command\", \"expected CMD name and one JSON object\")); }\n"); + output.push_str("match cmd_requires_tenant(&args[0]) {\nSome(true) => return Err(AppError::fault(\"command\", \"selected CMD requires an explicit tenant; use dever run --tenant\")),\nSome(false) => {},\nNone => return Err(AppError::fault(\"command\", format!(\"unknown CMD '{}'\", args[0]))),\n}\n"); + if program.jobs.is_empty() { + output.push_str("cmd_dispatch(&args[0], &args[1]).await?\n};\n"); + } else { + output.push_str("dever_runtime::job::scope_system(None, cmd_dispatch(&args[0], &args[1])).await?\n};\n"); + } + output.push_str("dever_runtime::println(&format!(\"{{\\\"code\\\":0,\\\"message\\\":\\\"ok\\\",\\\"data\\\":{}}}\", data.as_str())).map_err(|error| AppError::fault(\"command\", error))?;\n"); + } else { + output.push_str( + "return Err(AppError::fault(\"command\", \"application has no CMD entry\"));\n", + ); + } + output.push_str("}\nOk(())\n"); + if cleanup_database || cleanup_components { + emit_cleanup( + &emitter, + &mut output, + span, + cleanup_database, + cleanup_components, + ); + } + output.push_str("}\n"); + if !resources.is_empty() { + output.push_str(&embedded_resource_definitions(resources)?); + } + output.push_str("fn main() -> std::process::ExitCode {\nmatch task::run_entry_typed(entry())"); + emit_main_result(&mut output); + Ok(output) +} + +fn embedded_resource_definitions(resources: &[EmbeddedResource]) -> Result<String, String> { + validate_embedded_resources(resources)?; + let digest = embedded_bundle_digest(resources); + let inputs = resource_inputs(resources); + let definitions = inputs + .unique + .iter() + .enumerate() + .map(|(index, _)| { + format!( + "static DEVER_EXTERNAL_BYTES_{index}: &[u8] = include_bytes!({});\n", + rust_string(&resource_source_filename(index)) + ) + }) + .collect::<String>(); + let entries = resources.iter().zip(&inputs.indices).map(|(resource, index)| { + format!( + "dever_runtime::external::Resource {{ path: {}, bytes: DEVER_EXTERNAL_BYTES_{index}, sha256: {}, executable: {} }}", + rust_string(&resource.path), rust_string(&resource.sha256), resource.executable, + ) + }).collect::<Vec<_>>().join(", "); + Ok(format!( + "\n{definitions}static DEVER_EXTERNAL_RESOURCES: &[dever_runtime::external::Resource<'static>] = &[{entries}];\nconst DEVER_EXTERNAL_BUNDLE: &str = {};\n", + rust_string(&digest) + )) +} + +pub(crate) fn validate_embedded_resources(resources: &[EmbeddedResource]) -> Result<(), String> { + use sha2::{Digest, Sha256}; + for resource in resources { + let actual = Sha256::digest(&resource.bytes) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::<String>(); + if actual != resource.sha256 { + return Err(format!( + "external resource '{}' has an invalid digest", + resource.path + )); + } + } + Ok(()) +} + +pub(crate) struct ResourceInputs<'a> { + pub unique: Vec<&'a EmbeddedResource>, + pub indices: Vec<usize>, +} + +pub(crate) fn resource_inputs(resources: &[EmbeddedResource]) -> ResourceInputs<'_> { + let mut by_digest = BTreeMap::new(); + let mut unique = Vec::new(); + let mut indices = Vec::with_capacity(resources.len()); + for resource in resources { + let index = *by_digest + .entry(resource.sha256.as_str()) + .or_insert_with(|| { + let index = unique.len(); + unique.push(resource); + index + }); + indices.push(index); + } + ResourceInputs { unique, indices } +} + +fn resource_source_filename(index: usize) -> String { + format!("external-resource-{index}.bin") +} + +pub(crate) fn embedded_bundle_digest(resources: &[EmbeddedResource]) -> String { + use sha2::{Digest, Sha256}; + let mut digest = Sha256::new(); + for resource in resources { + digest.update(resource.path.as_bytes()); + digest.update([0]); + digest.update(resource.sha256.as_bytes()); + digest.update([0]); + digest.update((resource.bytes.len() as u64).to_le_bytes()); + digest.update([u8::from(resource.executable)]); + } + digest + .finalize() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +fn emit_entry( + program: &Program, + sources: &SourceMap, + entry: usize, + profile: Option<dever_runtime::config::RuntimeProfile>, +) -> Result<String, String> { + let root = Specialization { + function: entry, + handlers: Vec::new(), + }; + let serve_api = api_enabled(program, entry)?; + let serve_jobs = job_enabled(program, entry)?; + if serve_jobs && program.jobs.is_empty() { + return Err("dever.job.serve() requires at least one Job declaration".into()); + } + let roots = std::iter::once(root.clone()) + .chain( + program + .api_routes + .iter() + .filter(|_| serve_api) + .map(|route| Specialization { + function: route.function, + handlers: Vec::new(), + }), + ) + .chain( + program + .jobs + .iter() + .filter(|_| serve_jobs) + .map(|job| Specialization { + function: job.function, + handlers: Vec::new(), + }), + ) + .collect::<Vec<_>>(); + let (mut output, instances, names) = + emit_definitions(program, sources, roots, profile, serve_api)?; + let emitter = Emitter { + program, + sources, + names: &names, + bindings: BTreeMap::new(), + moves: HashSet::new(), + async_context: false, + }; + let root_suspends = specialize::suspends(program, &root); + let schema_suspends = profile + .is_some_and(|profile| profile.postgres || (profile.sqlite && !program.models.is_empty())); + let async_entry = root_suspends || schema_suspends; + output.push_str(if async_entry { + "async fn entry() -> Result<(), AppError> {\n" + } else { + "fn entry() -> Result<(), AppError> {\n" + }); + let cleanup_database = + async_entry && profile.is_some_and(|profile| profile.sqlite || profile.postgres); + let cleanup_components = ports::has_external(program, &instances); + if cleanup_database || cleanup_components { + output.push_str("let application: Result<(), AppError> = async {\n"); + } + ports::initialize( + &emitter, + &mut output, + &instances, + program.functions[entry].span, + )?; + if let Some(profile) = profile { + emit_database_setup( + &emitter, + &mut output, + &instances, + profile, + program.functions[entry].span, + None, + ); + } + let invocation = entry_invocation(program, &names, &root); + let invocation = if serve_api || serve_jobs { + format!("dever_runtime::lifecycle::run(async move {{ {invocation} }}).await") + } else { + invocation + }; + writeln!(output, "let result = {invocation}?;").expect("string formatting"); + for (index, field) in program.functions[entry].outputs.iter().enumerate() { + let value = if program.functions[entry].outputs.len() == 1 { + "result".into() + } else { + format!("result.{index}") + }; + writeln!( + output, + "let mut rendered{index} = String::from({}); rendered{index}.push_str(\" = \"); {value}.render_to(&mut rendered{index}); rendered{index}.push('\\n'); resource::stdout_write(&rendered{index})?;", + rust_string(&field.name), + ) + .expect("string formatting"); + } + output.push_str("Ok(())\n"); + if cleanup_database || cleanup_components { + emit_cleanup( + &emitter, + &mut output, + program.functions[entry].span, + cleanup_database, + cleanup_components, + ); + } + output.push_str("}\nfn main() -> std::process::ExitCode {\nmatch "); + output.push_str(if async_entry { + "task::run_entry_typed(entry())" + } else { + "entry()" + }); + emit_main_result(&mut output); + Ok(output) +} + +fn emit_test_suite( + program: &Program, + sources: &SourceMap, + profile: Option<dever_runtime::config::RuntimeProfile>, +) -> Result<String, String> { + if program.tests.is_empty() { + return Err("cannot compile an empty test suite".into()); + } + let mut output = String::from("#![forbid(unsafe_code)]\n"); + for index in 0..program.tests.len() { + let case = crate::check::test_program(program, index).map_err(|errors| { + errors + .iter() + .map(|error| error.render(sources)) + .collect::<Vec<_>>() + .join("\n") + })?; + writeln!(output, "mod case_{index} {{").expect("string formatting"); + output.push_str(&emit_test_case(&case, sources, index, profile)?); + output.push_str("}\n"); + } + writeln!(output, "fn main() -> std::process::ExitCode {{\nmatch std::env::args().nth(1).and_then(|value| value.parse::<usize>().ok()) {{").expect("string formatting"); + for index in 0..program.tests.len() { + writeln!(output, "Some({index}) => case_{index}::run(),").expect("string formatting"); + } + output.push_str("_ => { eprintln!(\"test program requires a valid case index\"); std::process::ExitCode::FAILURE },\n}\n}\n"); + Ok(output) +} + +fn emit_test_case( + program: &Program, + sources: &SourceMap, + index: usize, + profile: Option<dever_runtime::config::RuntimeProfile>, +) -> Result<String, String> { + let test = &program.tests[index]; + let root = Specialization { + function: test.function, + handlers: Vec::new(), + }; + let (mut output, instances, names) = + emit_definitions(program, sources, vec![root.clone()], profile, false)?; + let emitter = Emitter { + program, + sources, + names: &names, + bindings: BTreeMap::new(), + moves: HashSet::new(), + async_context: false, + }; + let async_entry = specialize::suspends(program, &root) + || profile.is_some_and(|profile| profile.sqlite && test.uses_database()); + output.push_str(if async_entry { + "async fn entry() -> Result<(), AppError> {\n" + } else { + "fn entry() -> Result<(), AppError> {\n" + }); + let cleanup_database = async_entry && test.uses_database(); + if cleanup_database { + output.push_str("let application: Result<(), AppError> = async {\n"); + } + jobs::initialize_clock(&emitter, &mut output, &instances, test.span); + if let Some(profile) = profile.filter(|_| test.uses_database()) { + emit_database_setup(&emitter, &mut output, &instances, profile, test.span, None); + } + let invocation = entry_invocation(program, &names, &root); + if jobs::targets(program, &instances).is_empty() { + writeln!(output, "let _ = {invocation}?;\nOk(())").expect("string formatting"); + } else { + writeln!(output, "dever_runtime::job::scope_system(None, async {{ let _ = {invocation}?; Ok(()) }}).await") + .expect("string formatting"); + } + if cleanup_database { + emit_cleanup(&emitter, &mut output, test.span, true, false); + } + output.push_str("}\n"); + writeln!( + output, + "pub fn run() -> std::process::ExitCode {{\nmatch {}", + if async_entry { + "task::run_entry_typed(entry())" + } else { + "entry()" + }, + ) + .expect("string formatting"); + emit_main_result(&mut output); + Ok(output) +} + +fn emit_cleanup( + emitter: &Emitter<'_>, + output: &mut String, + span: Span, + database: bool, + component: bool, +) { + output.push_str("}.await;\nlet mut cleanup: Option<AppError> = None;\n"); + if component { + writeln!(output, "if let Err(error) = dever_runtime::component::shutdown().await {{ cleanup = Some(AppError::fault({}, error)); }}", emitter.location(span)).expect("string formatting"); + } + if database { + writeln!(output, "if let Err(error) = dever_runtime::database::shutdown().await {{ cleanup = Some(match cleanup {{ Some(previous) => previous.cause(error), None => AppError::database({}, error) }}); }}", emitter.location(span)).expect("string formatting"); + } + output.push_str("match (application, cleanup) {\n(Ok(()), None) => Ok(()),\n(Err(error), None) => Err(error),\n(Ok(()), Some(cleanup)) => Err(cleanup),\n(Err(error), Some(cleanup)) => Err(error.cause(cleanup)),\n}\n"); +} + +fn emit_definitions( + program: &Program, + sources: &SourceMap, + roots: Vec<Specialization>, + profile: Option<dever_runtime::config::RuntimeProfile>, + serve_api: bool, +) -> Result<EmittedDefinitions, String> { + let instances = specialize::execution_reachable(&program.functions, roots) + .map_err(|_| "recursive specialized handler calls are not allowed".to_owned())?; + let names: BTreeMap<_, _> = instances + .iter() + .cloned() + .map(|instance| { + let name = if instance.handlers.is_empty() { + format!("f{}", instance.function) + } else { + format!( + "f{}h{}", + instance.function, + instance + .handlers + .iter() + .map(usize::to_string) + .collect::<Vec<_>>() + .join("h") + ) + }; + (instance, name) + }) + .collect(); + let emitter = Emitter { + program, + sources, + names: &names, + bindings: BTreeMap::new(), + moves: HashSet::new(), + async_context: false, + }; + let mut output = String::from( + "#![forbid(unsafe_code)]\n#![allow(dead_code, unused_mut, unused_variables, unused_parens)]\nuse dever_runtime::{number::{self, DecimalValue}, collections::{List, Map, MapEntry}, bytes::Bytes, channel, resource, render::Render, task};\n", + ); + writeln!( + output, + "static TENANT_COMPONENTS: &[&str] = &[{}];", + program + .tenant_components + .iter() + .map(|component| rust_string(component)) + .collect::<Vec<_>>() + .join(", ") + ) + .expect("string formatting"); + emitter.types(&mut output); + if profile.is_some() { + emitter.model_decoders(&mut output); + } + emitter.errors(&mut output); + ports::definitions(&emitter, &mut output, &instances); + jobs::definitions(&emitter, &mut output, &instances); + jobs::tenant_migration( + &emitter, + &mut output, + &instances, + profile.is_some_and(|profile| profile.sqlite || profile.postgres), + ); + for instance in &instances { + let emitter = Emitter { + program, + sources, + names: &names, + bindings: specialize::handler_bindings(&program.functions[instance.function], instance), + moves: HashSet::new(), + async_context: specialize::suspends(program, instance), + }; + emitter.function(&mut output, instance); + } + if serve_api { + api::emit_routes(&emitter, &mut output); + } + Ok((output, instances, names)) +} + +fn emit_database_setup( + emitter: &Emitter<'_>, + output: &mut String, + instances: &BTreeSet<Specialization>, + profile: dever_runtime::config::RuntimeProfile, + entry_span: Span, + application_services: Option<(bool, bool)>, +) { + let program = emitter.program; + let mut bindings = program + .models + .iter() + .map(|model| match &model.connection { + crate::model::ConnectionSelector::Explicit(name) => ( + Some(name.as_str()), + model.package.split('.').next().unwrap(), + model.scope == crate::model::ModelScope::Tenant, + ), + crate::model::ConnectionSelector::PackageRoot(root) => ( + None, + root.as_str(), + model.scope == crate::model::ModelScope::Tenant, + ), + }) + .collect::<Vec<_>>(); + bindings.extend(jobs::targets(program, instances).into_iter().map(|target| { + let (explicit, root) = program.database_selector(crate::model::DatabaseOwner::Job(target)); + ( + explicit, + root, + jobs::scope(program, target) == crate::model::ModelScope::Tenant, + ) + })); + bindings.sort_unstable(); + bindings.dedup(); + let bindings = bindings + .iter() + .map(|(explicit, root, tenant)| { + format!( + "({}, {}, {tenant})", + explicit + .map(|name| format!("Some({})", rust_string(name))) + .unwrap_or_else(|| "None".into()), + rust_string(root) + ) + }) + .collect::<Vec<_>>() + .join(", "); + let mut transaction_roots = instances + .iter() + .filter(|instance| { + program.functions[instance.function].kind == crate::syntax::FunctionKind::Transaction + }) + .cloned() + .collect::<BTreeSet<_>>(); + if application_services.is_some() { + for function in program + .api_routes + .iter() + .map(|route| route.function) + .chain(program.api_commands.iter().map(|command| command.function)) + .chain(program.jobs.iter().map(|job| job.function)) + { + if function_writes_database(program, function) { + transaction_roots.insert(Specialization { + function, + handlers: Vec::new(), + }); + } + } + } + let transactions = transaction_roots + .iter() + .map(|instance| { + let bindings = specialize::database_effects(program, instance) + .iter() + .map(|owner| { + let (explicit, root) = program.database_selector(*owner); + let tenant = match owner { + crate::model::DatabaseOwner::Model(index) => { + program.models[*index].scope == crate::model::ModelScope::Tenant + } + crate::model::DatabaseOwner::Job(target) => { + jobs::scope(program, *target) == crate::model::ModelScope::Tenant + } + }; + (explicit, root, tenant) + }) + .map(|(explicit, root, tenant)| { + format!( + "({}, {}, {tenant})", + explicit + .map(|name| format!("Some({})", rust_string(name))) + .unwrap_or_else(|| "None".into()), + rust_string(root) + ) + }) + .collect::<Vec<_>>() + .join(", "); + format!("&[{bindings}]") + }) + .collect::<Vec<_>>() + .join(", "); + writeln!( + output, + "dever_runtime::config::bootstrap_scoped(dever_runtime::config::RuntimeProfile {{ sqlite: {}, postgres: {} }}, &[{bindings}], &[{transactions}])?;", + profile.sqlite, + profile.postgres, + ) + .expect("string formatting"); + let serve_api = application_services.map_or_else( + || { + instances.iter().any(|instance| { + function_starts_service(&program.functions[instance.function], false) + }) + }, + |services| services.0, + ); + let serve_jobs = application_services.map_or_else( + || { + instances.iter().any(|instance| { + function_starts_service(&program.functions[instance.function], true) + }) + }, + |services| services.1, + ); + if serve_api || serve_jobs { + if application_services.is_some() { + output.push_str("if service_start {\n"); + } + writeln!( + output, + "dever_runtime::lifecycle::configure({serve_api}, {serve_jobs})?;" + ) + .expect("string formatting"); + if application_services.is_some() { + output.push_str("}\n"); + } + } + jobs::preflight(emitter, output, instances, entry_span); + if profile.postgres { + let prepare = + emitter.database_failure("dever_runtime::database::prepare().await", entry_span); + writeln!(output, "{prepare};").expect("string formatting"); + } + if profile.sqlite || profile.postgres { + emitter.initialize_models(output); + jobs::initialize(emitter, output, instances, entry_span); + } +} + +fn entry_invocation( + program: &Program, + names: &BTreeMap<Specialization, String>, + root: &Specialization, +) -> String { + let root_database = if specialize::database_effects(program, root).is_empty() { + "" + } else { + "None" + }; + if specialize::suspends(program, root) { + format!("{}({root_database}).await", names[root]) + } else { + format!("{}({root_database})", names[root]) + } +} + +fn emit_main_result(output: &mut String) { + output.push_str(" {\nOk(()) => { dever_runtime::log::flush(); std::process::ExitCode::SUCCESS },\nErr(error) => { dever_runtime::log::error(error.to_string(), vec![]); dever_runtime::log::flush(); std::process::ExitCode::FAILURE }\n}\n}\n"); +} + +fn api_enabled(program: &Program, entry: usize) -> Result<bool, String> { + service_enabled(program, entry, false) +} + +fn service_enabled(program: &Program, entry: usize, worker: bool) -> Result<bool, String> { + let instances = specialize::reachable( + &program.functions, + [Specialization { + function: entry, + handlers: Vec::new(), + }], + ) + .map_err(|_| "recursive specialized handler calls are not allowed".to_owned())?; + Ok(instances + .iter() + .any(|instance| function_starts_service(&program.functions[instance.function], worker))) +} + +fn job_enabled(program: &Program, entry: usize) -> Result<bool, String> { + service_enabled(program, entry, true) +} + +fn function_starts_service(function: &crate::hir::Function, worker: bool) -> bool { + let mut starts = false; + crate::check::visit(function, |expression| { + if let crate::hir::ExpressionKind::Intrinsic { operation, .. } = expression.kind { + starts |= if worker { + matches!(operation, crate::intrinsic::Intrinsic::JobServe) + } else { + matches!(operation, crate::intrinsic::Intrinsic::ApiServe) + }; + } + }); + starts +} + +fn function_writes_database(program: &Program, function: usize) -> bool { + let root = Specialization { + function, + handlers: Vec::new(), + }; + specialize::writes_database(program, &root) +} + +fn emit_command_tenant_scopes(program: &Program, output: &mut String) { + if program.api_commands.is_empty() { + return; + } + output.push_str("fn cmd_requires_tenant(name: &str) -> Option<bool> {\nmatch name {\n"); + for command in &program.api_commands { + let tenant = !command.components.is_empty(); + writeln!(output, "{} => Some({tenant}),", rust_string(&command.name)) + .expect("string formatting"); + } + output.push_str("_ => None,\n}\n}\n"); +} + +fn transaction_database(program: &Program, function: usize) -> Option<String> { + if !function_writes_database(program, function) { + return None; + } + let root = Specialization { + function, + handlers: Vec::new(), + }; + let owner = specialize::database_effects(program, &root) + .into_iter() + .next()?; + Some(match owner { + crate::model::DatabaseOwner::Model(index) => { + format!("model_database_{}().await", program.models[index].record,) + } + crate::model::DatabaseOwner::Job(index) => format!("job_database_{index}().await",), + }) +} + +impl Emitter<'_> { + fn function(&self, output: &mut String, instance: &Specialization) { + let id = instance.function; + let function = &self.program.functions[id]; + let suspends = specialize::suspends(self.program, instance); + let mut parameters = function + .parameters + .iter() + .enumerate() + .filter_map(|(index, parameter)| { + parameter + .value_type() + .map(|ty| format!("_p{index}: {}", rust_type(ty))) + }) + .collect::<Vec<_>>() + .join(", "); + let database_effects = specialize::database_effects(self.program, instance); + if !database_effects.is_empty() { + if !parameters.is_empty() { + parameters.push_str(", "); + } + parameters.push_str("_database: Option<&dever_runtime::database::Transaction>"); + } + writeln!( + output, + "{}fn {}({parameters}) -> Result<{}, AppError> {{", + if suspends { "async " } else { "" }, + self.names[instance], + rust_type(&output_type(&function.outputs)) + ) + .expect("string formatting"); + if ports::function(self, output, instance) { + return; + } + for owner in &database_effects { + match owner { + crate::model::DatabaseOwner::Model(index) => { + let model = &self.program.models[*index]; + let database = self.database_failure( + &format!("model_database_{}().await", model.record), + function.span, + ); + writeln!(output, "let _model_database_{} = {database};", model.record) + .expect("string formatting"); + } + crate::model::DatabaseOwner::Job(target) => { + let database = self + .database_failure(&format!("job_database_{target}().await"), function.span); + writeln!(output, "let _job_database_{target} = {database};") + .expect("string formatting"); + } + } + } + let transaction = function.kind == crate::syntax::FunctionKind::Transaction; + if transaction { + let owner = *database_effects + .iter() + .next() + .expect("validated transaction database effect"); + let database = self.database_owner_at(owner); + let begin = self.database_failure("transaction_database.begin().await", function.span); + writeln!( + output, + "let transaction_owner = if _database.is_none() {{ let transaction_database = {database}; Some({begin}) }} else {{ None }};\nlet _database = _database.or(transaction_owner.as_ref());\nlet transaction_result: Result<{}, AppError> = async {{", + rust_type(&output_type(&function.outputs)) + ) + .expect("string formatting"); + } + for clause in &function.clauses { + let mut optimized = clause.clone(); + self.inline_sequences(&mut optimized); + updates::coalesce(&mut optimized, &self.program.types); + updates::snapshot_scalars(&mut optimized, &self.program.types); + let clause = &optimized; + let clause_emitter = Emitter { + program: self.program, + sources: self.sources, + names: self.names, + bindings: self.bindings.clone(), + moves: liveness::last_uses(clause), + async_context: suspends, + }; + let mut domains = clause.patterns.iter(); + let guard = function + .parameters + .iter() + .enumerate() + .filter_map(|(index, parameter)| { + let Parameter::Value(ty) = parameter else { + return None; + }; + let domain = domains.next().expect("checked value parameter domain"); + Some(format!( + "({})", + domain + .iter() + .map(|atom| self.atom(atom, ty, &format!("_p{index}"))) + .collect::<Vec<_>>() + .join(" || ") + )) + }) + .collect::<Vec<_>>() + .join(" && "); + debug_assert!(domains.next().is_none()); + writeln!( + output, + "if {} {{", + if guard.is_empty() { "true" } else { &guard } + ) + .expect("string formatting"); + for (slot, ty) in clause.locals.iter().enumerate() { + writeln!(output, "let mut _v{slot}: {};", rust_type(ty)) + .expect("string formatting"); + } + for binding in &clause.bindings { + self.binding( + output, + binding, + function.parameters[binding.input] + .value_type() + .expect("value binding"), + clause + .bindings + .iter() + .filter(|other| other.input == binding.input) + .count() + == 1, + ); + } + for statement in &clause.body { + match statement { + Statement::Assign { + slot, + fields, + value, + } => { + let mut target = format!("_v{slot}"); + let mut ty = clause.locals[*slot].clone(); + for index in fields { + target.push_str(&field_suffix(&ty, *index)); + ty = self.field_type(&ty, *index); + } + writeln!(output, "{target} = {};", clause_emitter.expression(value)) + .expect("string formatting"); + } + Statement::Call(value) => { + writeln!(output, "{};", clause_emitter.expression(value)) + .expect("string formatting"); + } + } + } + if clause.terminates { + writeln!(output, "}}\n").expect("string formatting"); + continue; + } + let result = match clause.outputs.as_slice() { + [] => "()".into(), + [slot] => format!("_v{slot}"), + slots => format!( + "({})", + slots + .iter() + .map(|slot| format!("_v{slot}")) + .collect::<Vec<_>>() + .join(", ") + ), + }; + writeln!(output, "return Ok({result});\n}}").expect("string formatting"); + } + output.push_str("unreachable!(\"checked exhaustive clauses\")\n"); + if transaction { + writeln!( + output, + "}}.await;\nif let Some(transaction) = transaction_owner {{\nreturn match transaction_result {{\nOk(value) => {{ transaction.commit().await.map_err(|error| AppError::database({}, error))?; Ok(value) }},\nErr(error) => match transaction.rollback().await {{ Ok(()) => Err(error), Err(rollback) => Err(error.cause(rollback)) }}\n}};\n}}\ntransaction_result", + self.location(function.span), + ) + .expect("string formatting"); + } + output.push_str("}\n"); + } + + fn invoke( + &self, + function: &Specialization, + mut arguments: String, + database: DatabaseContext, + ) -> String { + if !specialize::database_effects(self.program, function).is_empty() { + if !arguments.is_empty() { + arguments.push_str(", "); + } + arguments.push_str(match database { + DatabaseContext::Inherit => "_database", + DatabaseContext::Detached => "None", + }); + } + format!("{}({arguments})", self.names[function]) + } + + fn binding(&self, output: &mut String, binding: &hir::Binding, ty: &Type, unique: bool) { + let input = format!("_p{}", binding.input); + let value = match binding.projection { + Projection::Whole if unique && ty.movable(&self.program.types) => input.clone(), + Projection::Whole => format!("{input}.clone()"), + Projection::NonNull if unique && ty.movable(&self.program.types) => { + format!("{input}.expect(\"checked non-null clause\")") + } + Projection::NonNull => { + format!("{input}.as_ref().expect(\"checked non-null clause\").clone()") + } + Projection::Payload { variant, field } => { + if let Type::Related(_) = ty.base() { + debug_assert_eq!(variant, 1); + debug_assert_eq!(field, 0); + let input = if matches!(ty, Type::Nullable(_)) { + format!("{input}.as_ref().expect(\"checked non-null variant\")") + } else { + format!("&{input}") + }; + writeln!( + output, + "_v{} = match {input} {{ dever_runtime::orm::Related::Loaded(payload) => payload.as_ref().clone(), _ => unreachable!(\"checked Related variant\") }};", + binding.slot, + ) + .expect("string formatting"); + return; + } + let Type::Named(id) = ty.base() else { + unreachable!() + }; + let Shape::Choice(variants) = &self.program.types[*id].shape else { + unreachable!() + }; + let fields = (0..variants[variant].fields.len()) + .map(|index| if index == field { "payload" } else { "_" }) + .collect::<Vec<_>>() + .join(", "); + let input = if matches!(ty, Type::Nullable(_)) { + format!("{input}.as_ref().expect(\"checked non-null variant\")") + } else { + format!("&{input}") + }; + format!( + "match {input} {{ T{id}::V{variant}({fields}) => payload.clone(), _ => unreachable!(\"checked variant\") }}" + ) + } + }; + writeln!(output, "_v{} = {value};", binding.slot).expect("string formatting"); + } + + fn atom(&self, atom: &Atom, ty: &Type, value: &str) -> String { + if matches!(atom, Atom::Null) { + return format!("{value}.is_none()"); + } + if let Type::Nullable(base) = ty { + return format!( + "{value}.as_ref().is_some_and(|_x| {})", + self.atom(atom, base, "(*_x)") + ); + } + match atom { + Atom::Any => "true".into(), + Atom::Bool(boolean) => format!("{value} == {boolean}"), + Atom::Variant(variant) => { + if matches!(ty, Type::Related(_)) { + return format!( + "matches!({value}, dever_runtime::orm::Related::{})", + if *variant == 0 { + "Unloaded" + } else { + "Loaded(..)" + } + ); + } + let Type::Named(id) = ty else { unreachable!() }; + let Shape::Choice(variants) = &self.program.types[*id].shape else { + unreachable!() + }; + format!( + "matches!({value}, T{id}::V{variant}{})", + if variants[*variant].fields.is_empty() { + "" + } else { + "(..)" + } + ) + } + Atom::Int(lo, hi) => range_guard( + value, + (*lo != i64::MIN).then(|| format!("{lo}i64")), + (*hi != i64::MAX).then(|| format!("{hi}i64")), + ), + Atom::Decimal(lo, hi) => range_guard( + value, + (*lo != dever_runtime::number::DecimalValue::minimum()) + .then(|| constant(&Constant::Decimal(*lo))), + (*hi != dever_runtime::number::DecimalValue::maximum()) + .then(|| constant(&Constant::Decimal(*hi))), + ), + Atom::Literal(literal) => format!("{value} == {}", typed_constant(literal, ty)), + Atom::Remainder(literals) => { + if literals.is_empty() { + "true".into() + } else { + literals + .iter() + .map(|literal| format!("{value} != {}", typed_constant(literal, ty))) + .collect::<Vec<_>>() + .join(" && ") + } + } + Atom::Null => unreachable!(), + } + } + + fn field_type(&self, ty: &Type, index: usize) -> Type { + match ty { + Type::Named(id) => match &self.program.types[*id].shape { + Shape::Record(fields) => fields[index].ty.clone(), + _ => unreachable!(), + }, + Type::Outputs(fields) => fields[index].ty.clone(), + Type::MapEntry(key, value) => { + if index == 0 { + *key.clone() + } else { + *value.clone() + } + } + _ => unreachable!("checked field access"), + } + } + + fn failure(&self, expression: &str, span: Span) -> String { + format!( + "({expression}).map_err(|error| AppError::fault({}, error))?", + self.location(span) + ) + } + + fn database_failure(&self, expression: &str, span: Span) -> String { + format!( + "({expression}).map_err(|error| AppError::database({}, error))?", + self.location(span) + ) + } + + fn call_failure(&self, expression: &str, span: Span) -> String { + format!( + "({expression}).map_err(|error| error.at({}))?", + self.location(span) + ) + } + + fn location(&self, span: Span) -> String { + let source = self.sources.get(span.source); + let (line, column) = source.position(span.start); + let location = format!("{}:{line}:{column}", source.path().display()); + rust_string(&location) + } +} + +// 类型已保证整个 Int/有限 Decimal 域，只有分句实际缩小的边界才需运行时比较。 +fn range_guard(value: &str, low: Option<String>, high: Option<String>) -> String { + let checks: Vec<_> = low + .into_iter() + .map(|low| format!("{value} >= {low}")) + .chain(high.into_iter().map(|high| format!("{value} <= {high}"))) + .collect(); + if checks.is_empty() { + "true".into() + } else { + checks.join(" && ") + } +} + +fn optional_result(expression: &str, value: &Type) -> String { + if matches!(value, Type::Nullable(_)) { + format!("({expression}).flatten()") + } else { + expression.into() + } +} + +fn rust_type(ty: &Type) -> String { + match ty { + Type::Unit => "()".into(), + Type::Bool => "bool".into(), + Type::Int => "i64".into(), + Type::Decimal => "DecimalValue".into(), + Type::Float => "f64".into(), + Type::Text => "String".into(), + Type::Id => "dever_runtime::Id".into(), + Type::Bytes => "Bytes".into(), + Type::Secret => "dever_runtime::secret::Secret".into(), + Type::Upload => "dever_runtime::api::upload::Upload".into(), + Type::Uuid => "dever_runtime::orm::Uuid".into(), + Type::Json => "String".into(), + Type::DateTime | Type::Date | Type::Time | Type::Duration => "i64".into(), + Type::File => "resource::FileHandle".into(), + Type::Socket => "dever_runtime::net::Socket".into(), + Type::Listener => "dever_runtime::net::Listener".into(), + Type::HttpReply => "dever_runtime::http::HttpReply".into(), + Type::WebSocket => "dever_runtime::websocket::WebSocket".into(), + Type::ClientTls => "dever_runtime::tls::ClientTls".into(), + Type::ServerTls => "dever_runtime::tls::ServerTls".into(), + Type::HttpClient => "dever_runtime::http::HttpClient".into(), + Type::Stream(element) => format!("resource::Stream<{}>", rust_type(element)), + Type::AsyncStream(element) => format!( + "dever_runtime::async_stream::AsyncStream<{}>", + rust_type(element) + ), + Type::RowStream(element) => { + format!("dever_runtime::orm::RowStream<{}>", rust_type(element)) + } + Type::Related(element) => format!("dever_runtime::orm::Related<{}>", rust_type(element)), + Type::Channel(element) => format!("channel::Channel<{}>", rust_type(element)), + Type::Task(outputs) => { + format!("task::Task<{}, AppError>", rust_type(&output_type(outputs))) + } + Type::Group => "task::Group<AppError>".into(), + Type::Named(id) => format!("T{id}"), + Type::Nullable(base) => format!("Option<{}>", rust_type(base)), + Type::List(base) => format!("List<{}>", rust_type(base)), + Type::Map(key, value) => format!("Map<{}, {}>", rust_type(key), rust_type(value)), + Type::MapEntry(key, value) => format!("MapEntry<{}, {}>", rust_type(key), rust_type(value)), + Type::Outputs(fields) => format!( + "({})", + fields + .iter() + .map(|field| rust_type(&field.ty)) + .collect::<Vec<_>>() + .join(", ") + ), + } +} + +fn field_suffix(ty: &Type, index: usize) -> String { + match ty { + Type::Outputs(_) => format!(".{index}"), + Type::MapEntry(_, _) => { + if index == 0 { + ".key".into() + } else { + ".value".into() + } + } + _ => format!(".f{index}"), + } +} + +fn constant(value: &Constant) -> String { + match value { + Constant::Null => "None".into(), + Constant::Bool(value) => value.to_string(), + Constant::Int(value) => format!("{value}i64"), + Constant::Decimal(value) => format!( + "DecimalValue::from_bytes({:?}).expect(\"checked Decimal literal\")", + value.to_bytes() + ), + Constant::Float(value) => format!("f64::from_bits({}u64)", value.to_bits()), + Constant::Text(value) => format!("String::from({})", rust_string(value)), + } +} + +fn typed_constant(value: &Constant, ty: &Type) -> String { + if let (Constant::Text(value), Type::Uuid) = (value, ty) { + let bytes = dever_runtime::orm::Uuid::parse(value) + .expect("checked Uuid literal") + .into_bytes(); + format!("dever_runtime::orm::Uuid::from_bytes({bytes:?})") + } else { + constant(value) + } +} + +fn rust_string(value: &str) -> String { + format!( + "\"{}\"", + value + .chars() + .flat_map(char::escape_default) + .collect::<String>() + ) +} diff --git a/crates/dever-core/src/native/api.rs b/crates/dever-core/src/native/api.rs new file mode 100644 index 0000000..5a4c83b --- /dev/null +++ b/crates/dever-core/src/native/api.rs @@ -0,0 +1,532 @@ +use std::fmt::Write; + +use crate::hir::{ApiRoute, Program}; +use crate::specialize::{self, Specialization}; +use crate::wire::{Node, Policy, Schema}; + +use super::rust_string; + +pub(super) fn emit_routes(emitter: &super::Emitter<'_>, output: &mut String) { + let program = emitter.program; + emit_authorization(program, output); + let mut codecs = std::collections::BTreeMap::new(); + let mut route_codecs = Vec::new(); + for (route_index, route) in program.api_routes.iter().enumerate() { + let schema = Schema::from_field( + &program.functions[route.function].outputs[0], + &program.types, + Policy::Output, + ) + .expect("checked API wire schema"); + let next = codecs.len(); + let index = *codecs + .entry(schema.fingerprint().to_owned()) + .or_insert_with(|| { + output.push_str(&super::wire::emit(&schema, next)); + next + }); + route_codecs.push(index); + for (input_index, parameter) in program.functions[route.function] + .parameters + .iter() + .enumerate() + { + if parameter.value_type() == Some(&crate::types::Type::Upload) { + continue; + } + let schema = Schema::build( + parameter.value_type().expect("checked API input"), + &program.types, + Policy::ApiInput, + ) + .expect("checked API input schema"); + output.push_str(&super::wire::emit_named( + &schema, + &format!("api_input_{route_index}_{input_index}"), + )); + } + } + if !program.api_rest.is_empty() { + super::rest::emit_helpers(program, output); + super::rest::emit_database_error(output); + } + emit_failure_mapper(program, output); + output.push_str("async fn api_route(mut request: dever_runtime::http::Request) -> Result<dever_runtime::http::Response, String> {\n"); + output.push_str("let path = request.target.split('?').next().unwrap_or(\"\").to_owned();\nmatch (request.method.as_str(), path.as_str()) {\n"); + for (route_index, (route, codec)) in program.api_routes.iter().zip(route_codecs).enumerate() { + writeln!( + output, + "({}, {}) => {{", + rust_string(route.method), + rust_string(&route.path) + ) + .expect("string formatting"); + emit_auth_begin( + program, + route.auth, + &route.directory, + route.anonymous, + output, + ); + emit_components(&route.components, output); + emit_permission(program, route.permission, output); + emit_handler(program, route, route_index, codec, output); + emit_auth_end(route.auth, output); + output.push_str("},\n"); + } + for rest in &program.api_rest { + for method in ["GET", "POST"] { + writeln!( + output, + "({}, {}) => {{", + rust_string(method), + rust_string(&rest.path) + ) + .expect("string formatting"); + emit_auth_begin(program, rest.auth, &rest.directory, false, output); + emit_components(&rest.components, output); + emit_permission(program, rest_permission(rest, method), output); + emit_read_body(output); + super::rest::emit_handler(emitter, rest, method, false, output); + emit_auth_end(rest.auth, output); + output.push_str("},\n"); + } + } + output.push_str("(_, path) => {\n"); + let mut paths = std::collections::BTreeMap::<&str, Vec<&str>>::new(); + for route in &program.api_routes { + paths.entry(&route.path).or_default().push(route.method); + } + for rest in &program.api_rest { + paths.entry(&rest.path).or_default().extend(["GET", "POST"]); + } + for rest in &program.api_rest { + writeln!( + output, + "if path.starts_with({}) {{ return match request.method.as_str() {{", + rust_string(&format!("{}/", rest.path)) + ) + .expect("string formatting"); + for method in ["GET", "PUT", "DELETE"] { + writeln!(output, "{} => {{", rust_string(method)).expect("string formatting"); + emit_auth_begin(program, rest.auth, &rest.directory, false, output); + emit_components(&rest.components, output); + emit_permission(program, rest_permission(rest, method), output); + emit_read_body(output); + super::rest::emit_handler(emitter, rest, method, true, output); + emit_auth_end(rest.auth, output); + output.push_str("},\n"); + } + output.push_str("_ => Ok(dever_runtime::api::method_not_allowed(&[\"GET\", \"PUT\", \"DELETE\"])),\n}; }\n"); + } + output.push_str("match path {\n"); + for (path, methods) in paths { + writeln!( + output, + "{} => Ok(dever_runtime::api::method_not_allowed(&{:?})),", + rust_string(path), + methods + ) + .expect("string formatting"); + } + output.push_str("_ => Ok(dever_runtime::api::not_found()),\n}\n}\n}\n}\n"); +} + +fn emit_authorization(program: &Program, output: &mut String) { + if program.permissions.is_empty() { + return; + } + output.push_str( + "static AUTHORIZATION_PERMISSIONS: &[dever_runtime::auth::store::Permission] = &[\n", + ); + for permission in &program.permissions { + writeln!(output, + "dever_runtime::auth::store::Permission {{ key: {}, component: {}, domain: {}, site: {}, action: {}, method: {} }},", + rust_string(&permission.key), + rust_string(&permission.component), + rust_string(&permission.domain), + rust_string(&permission.site), + rust_string(&permission.action), + rust_string(permission.method), + ).expect("string formatting"); + } + output.push_str(r#"]; +fn authorization_control_database() -> Result<dever_runtime::database::Database, dever_runtime::orm::Error> { +let settings = dever_runtime::config::settings(); +let connection = settings.tenant().map(|tenant| tenant.database()).unwrap_or("default"); +dever_runtime::database::database_for(Some(connection), "") +} +async fn authorization_initialize() -> Result<(), AppError> { +let database = authorization_control_database().map_err(|error| AppError::database("authorization initialization", error))?; +dever_runtime::auth::store::sync_catalog(database.clone(), AUTHORIZATION_PERMISSIONS).await.map_err(|error| AppError::database("permission catalog", error))?; +dever_runtime::auth::store::initialize_roles(database).await.map_err(|error| AppError::database("authorization roles", error)) +} +async fn tenant_owner(tenant_id: i64, site: &str, user_id: i64) -> Result<(), AppError> { +let settings = dever_runtime::config::settings(); +if !settings.sites().contains_key(site) { return Err(AppError::fault("tenant owner", format!("unknown site '{site}'"))); } +let tenant = settings.tenant().ok_or_else(|| AppError::fault("tenant owner", "tenant storage is not configured"))?; +let connection = tenant.database().to_owned(); +dever_runtime::tenant::scope(tenant_id, async move { +let database = dever_runtime::tenant::database(&connection).await.map_err(|error| AppError::database("tenant owner", error))?; +dever_runtime::auth::store::initialize_roles(database.clone()).await.map_err(|error| AppError::database("tenant owner", error))?; +dever_runtime::auth::store::provision_owner(database, user_id, site).await.map_err(|error| AppError::database("tenant owner", error))?; +Ok(()) +}).await +} +async fn api_require_components(components: &[&str]) -> Result<(), dever_runtime::auth::Error> { +if components.is_empty() { return Ok(()); } +let tenant_id = dever_runtime::auth::tenant_id()?.ok_or(dever_runtime::auth::Error::Forbidden)?; +match dever_runtime::tenant::require_components(tenant_id, components, TENANT_COMPONENTS).await { +Ok(()) => Ok(()), +Err(error) if error.kind() == dever_runtime::orm::ErrorKind::InvalidData => Err(dever_runtime::auth::Error::Forbidden), +Err(error) => { dever_runtime::log::error(error.to_string(), Vec::new()); Err(dever_runtime::auth::Error::Internal) } +} +} +async fn api_authorize(permission: &'static str, site: &'static str) -> Result<(), dever_runtime::auth::Error> { +dever_runtime::auth::authorize_permission(permission, site).await +} +"#); +} + +fn emit_permission(program: &Program, permission: Option<usize>, output: &mut String) { + let Some(permission) = permission.map(|id| &program.permissions[id]) else { + return; + }; + writeln!(output, + "if let Err(error) = api_authorize({}, {}).await {{ return Ok(dever_runtime::api::auth_error(error)); }};", + rust_string(&permission.key), + rust_string(&permission.site), + ).expect("string formatting"); +} + +fn emit_components(components: &[String], output: &mut String) { + if components.is_empty() { + return; + } + writeln!(output, + "if let Err(error) = api_require_components(&[{}]).await {{ return Ok(dever_runtime::api::auth_error(error)); }};", + components.iter().map(|component| rust_string(component)).collect::<Vec<_>>().join(", "), + ).expect("string formatting"); +} + +fn rest_permission(route: &crate::hir::ApiRest, method: &str) -> Option<usize> { + match method { + "GET" => route.permissions.read, + "POST" => route.permissions.create, + "PUT" => route.permissions.replace, + "DELETE" => route.permissions.delete, + _ => unreachable!("checked REST method"), + } +} + +fn emit_failure_mapper(program: &Program, output: &mut String) { + output.push_str("fn api_failure_response(error: AppError) -> dever_runtime::http::Response {\nmatch &error.kind {\n"); + if let Some((error_id, definition)) = program + .types + .iter() + .enumerate() + .find(|(_, definition)| definition.name == "dever.api.Error") + { + let crate::types::Shape::Choice(variants) = &definition.shape else { + unreachable!("official dever.api.Error is a choice") + }; + let failures = program + .failures + .iter() + .flat_map(|failures| failures.iter()) + .copied() + .collect::<std::collections::BTreeSet<_>>(); + for (variant, status) in [ + ("Invalid", 400), + ("Unauthorized", 401), + ("Forbidden", 403), + ("NotFound", 404), + ("Conflict", 409), + ("TooManyRequests", 429), + ] { + let index = variants + .iter() + .position(|candidate| candidate.name == variant) + .expect("official dever.api.Error variant"); + if failures.contains(&crate::hir::Failure { + ty: error_id, + variant: index, + }) { + writeln!(output, "AppErrorKind::E{error_id}V{index} => dever_runtime::api::standard_error({status}),") + .expect("string formatting"); + } + } + } + output.push_str("_ => { dever_runtime::log::error(\"API handler failed\", Vec::new()); dever_runtime::api::internal_error() }\n}\n}\n"); +} + +fn emit_auth_begin( + program: &Program, + auth: Option<usize>, + directory: &[String], + anonymous: bool, + output: &mut String, +) { + let Some(auth) = auth else { return }; + let hook = &program.auth[auth]; + let directory = directory + .iter() + .map(|part| rust_string(part)) + .collect::<Vec<_>>() + .join(", "); + writeln!(output, + "let auth_prepared = match dever_runtime::auth::prepare(&[{directory}], &request, {anonymous}) {{ Ok(prepared) => prepared, Err(error) => return Ok(dever_runtime::api::auth_error(error)), }};" + ).expect("string formatting"); + writeln!(output, + "let auth_identity = match auth_prepared.claims() {{\nSome(claims) => {{\nlet auth_claims = T{} {{ f0: claims.subject.clone(), f1: claims.session.clone(), f2: claims.tenant.clone(), f3: claims.site.clone() }};", + hook.claims, + ).expect("string formatting"); + let target = Specialization { + function: hook.function, + handlers: Vec::new(), + }; + let mut arguments = vec!["auth_claims".to_owned()]; + if !specialize::database_effects(program, &target).is_empty() { + arguments.push("None".to_owned()); + } + let call = format!( + "f{}({}){}", + hook.function, + arguments.join(", "), + if specialize::suspends(program, &target) { + ".await" + } else { + "" + }, + ); + writeln!(output, + "let identity: T{} = match {call} {{ Ok(identity) => identity, Err(error) => return Ok(api_failure_response(error)), }};\nSome(dever_runtime::auth::Identity {{ id: identity.f0, user_id: identity.f1.map(|value| value.0), tenant_id: identity.f2.map(|value| value.0) }})\n}},\nNone => None,\n}};\nlet auth_response = dever_runtime::auth::scope(auth_prepared, auth_identity, async move {{", + hook.identity, + ).expect("string formatting"); +} + +fn emit_auth_end(auth: Option<usize>, output: &mut String) { + if auth.is_some() { + output.push_str("}).await;\nmatch auth_response { Ok(response) => response, Err(error) => Ok(dever_runtime::api::auth_error(error)), }\n"); + } +} + +fn emit_handler( + program: &Program, + route: &ApiRoute, + route_index: usize, + codec: usize, + output: &mut String, +) { + let function = &program.functions[route.function]; + if route.multipart { + output.push_str("let mut inputs = match dever_runtime::api::upload::read_multipart(&request).await { Ok(inputs) => inputs, Err(error) => return Ok(dever_runtime::api::invalid_input(error)), };\n"); + } else { + emit_read_body(output); + output.push_str("let mut inputs = match dever_runtime::api::Inputs::from_request(&request) {\nOk(inputs) => inputs, Err(error) => return Ok(dever_runtime::api::invalid_input(error)),\n};\n"); + } + for (index, (name, parameter)) in route.inputs.iter().zip(&function.parameters).enumerate() { + if parameter.value_type() == Some(&crate::types::Type::Upload) { + writeln!(output, "let p{index} = match inputs.take({}) {{ Ok(upload) => upload, Err(error) => return Ok(dever_runtime::api::invalid_input(error)), }};", rust_string(name)).expect("string formatting"); + continue; + } + let schema = Schema::build( + parameter.value_type().expect("checked API value input"), + &program.types, + Policy::ApiInput, + ) + .expect("checked API input schema"); + let kind = schema.query_kind(); + let read = if matches!(schema.nodes[schema.root], Node::Nullable(_)) { + "optional_json" + } else { + "json" + }; + writeln!(output, + "let raw_{index} = match inputs.{read}({}, dever_runtime::api::QueryValue::{kind}) {{ Ok(value) => value, Err(error) => return Ok(dever_runtime::api::invalid_input(error)), }};\nlet p{index} = match api_input_{route_index}_{index}_decode(&raw_{index}) {{ Ok(value) => value, Err(error) => return Ok(dever_runtime::api::invalid_input(dever_runtime::api::InputError(error))), }};", + rust_string(name), + ).expect("string formatting"); + } + output.push_str("if let Err(error) = inputs.finish() { return Ok(dever_runtime::api::invalid_input(error)); }\n"); + let target = Specialization { + function: route.function, + handlers: Vec::new(), + }; + let password_crypto = specialize::uses_intrinsic( + program, + &target, + crate::intrinsic::Intrinsic::CryptoPasswordVerify, + ) || specialize::uses_intrinsic( + program, + &target, + crate::intrinsic::Intrinsic::CryptoPasswordHash, + ); + let transaction_database = (!password_crypto) + .then(|| super::transaction_database(program, route.function)) + .flatten(); + if let Some(database) = &transaction_database { + writeln!(output, "let transaction_database = match {database} {{ Ok(database) => database, Err(error) => {{ dever_runtime::log::error(error.to_string(), Vec::new()); return Ok(dever_runtime::api::internal_error()); }} }};\nlet transaction = match transaction_database.begin().await {{ Ok(value) => value, Err(error) => {{ dever_runtime::log::error(error.to_string(), Vec::new()); return Ok(dever_runtime::api::internal_error()); }} }};") + .expect("string formatting"); + } + let mut arguments = (0..route.inputs.len()) + .map(|index| format!("p{index}")) + .collect::<Vec<_>>(); + if !specialize::database_effects(program, &target).is_empty() { + arguments.push( + if transaction_database.is_some() { + "Some(&transaction)" + } else { + "None" + } + .to_owned(), + ); + } + let call = format!( + "f{}({}){}", + route.function, + arguments.join(", "), + if specialize::suspends(program, &target) { + ".await" + } else { + "" + } + ); + let failure_mapper = "api_failure_response"; + if transaction_database.is_some() { + writeln!(output, "let response = match {call} {{ Ok(value) => value, Err(error) => {{ if transaction.rollback().await.is_err() {{ dever_runtime::log::error(\"API rollback failed\", Vec::new()); }} return Ok({failure_mapper}(error)); }} }};") + .expect("string formatting"); + } else { + writeln!(output, "let response = match {call} {{ Ok(value) => value, Err(error) => return Ok({failure_mapper}(error)), }};") + .expect("string formatting"); + } + let conversion = format!("wire_{codec}_encode(&response)"); + writeln!(output, + "let value = match {conversion} {{ Ok(value) => value, Err(_error) => {{ {} dever_runtime::log::error(\"API response serialization failed\", Vec::new()); return Ok(dever_runtime::api::internal_error()); }} }};", + if transaction_database.is_some() { "let _ = transaction.rollback().await;" } else { "" }, + ).expect("string formatting"); + if transaction_database.is_some() { + output.push_str("if let Err(error) = transaction.commit().await { dever_runtime::log::error(error.to_string(), Vec::new()); return Ok(dever_runtime::api::internal_error()); }\n"); + } + output.push_str("if let Err(error) = dever_runtime::api::commit_response_metadata() { dever_runtime::log::error(error, Vec::new()); return Ok(dever_runtime::api::internal_error()); }\nOk(dever_runtime::api::success(value))"); +} + +fn emit_read_body(output: &mut String) { + output.push_str("if let Err(error) = dever_runtime::api::upload::read_body(&mut request).await { return Ok(dever_runtime::api::invalid_input(error)); }\n"); +} + +pub(super) fn emit_commands(program: &Program, output: &mut String) { + if program.api_commands.is_empty() { + return; + } + for (command_index, command) in program.api_commands.iter().enumerate() { + let function = &program.functions[command.function]; + let schema = Schema::from_field(&function.outputs[0], &program.types, Policy::Output) + .expect("checked CMD output schema"); + output.push_str(&super::wire::emit_named( + &schema, + &format!("cmd_output_{command_index}"), + )); + for (input_index, parameter) in function.parameters.iter().enumerate() { + let schema = Schema::build( + parameter.value_type().expect("checked CMD input"), + &program.types, + Policy::ApiInput, + ) + .expect("checked CMD input schema"); + output.push_str(&super::wire::emit_named( + &schema, + &format!("cmd_input_{command_index}_{input_index}"), + )); + } + } + output.push_str("async fn cmd_dispatch(name: &str, body: &str) -> Result<dever_runtime::wire::Encoded, AppError> {\nmatch name {\n"); + for (command_index, command) in program.api_commands.iter().enumerate() { + let function = &program.functions[command.function]; + writeln!(output, "{} => {{", rust_string(&command.name)).expect("string formatting"); + if !command.components.is_empty() { + writeln!(output, + "let tenant_id = dever_runtime::tenant::current_id().map_err(|error| AppError::database(\"command component\", error))?;\ndever_runtime::tenant::require_components(tenant_id, &[{}], TENANT_COMPONENTS).await.map_err(|error| AppError::database(\"command component\", error))?;", + command.components.iter().map(|component| rust_string(component)).collect::<Vec<_>>().join(", "), + ).expect("string formatting"); + } + output.push_str("let mut inputs = dever_runtime::api::Inputs::from_json_object(body).map_err(|error| AppError::fault(\"command\", error))?;\n"); + for (input_index, name) in command.inputs.iter().enumerate() { + let parameter = &function.parameters[input_index]; + let schema = Schema::build( + parameter.value_type().expect("checked CMD input"), + &program.types, + Policy::ApiInput, + ) + .expect("checked CMD input schema"); + let read = if matches!(schema.nodes[schema.root], Node::Nullable(_)) { + "optional_raw_json" + } else { + "raw_json" + }; + let absent = if read == "optional_raw_json" { + ".unwrap_or_else(|| \"null\".into())" + } else { + "" + }; + writeln!(output, + "let raw_{input_index} = inputs.{read}({}).map_err(|error| AppError::fault(\"command\", error))?{absent};\nlet p{input_index} = cmd_input_{command_index}_{input_index}_decode(&raw_{input_index}).map_err(|error| AppError::fault(\"command\", error))?;", + rust_string(name), + ).expect("string formatting"); + } + output.push_str("inputs.finish().map_err(|error| AppError::fault(\"command\", error))?;\n"); + let target = Specialization { + function: command.function, + handlers: Vec::new(), + }; + let transaction_database = super::transaction_database(program, command.function); + if let Some(database) = &transaction_database { + writeln!(output, "let transaction_database = ({database}).map_err(|error| AppError::database(\"command\", error))?;\nlet transaction = transaction_database.begin().await.map_err(|error| AppError::database(\"command\", error))?;") + .expect("string formatting"); + } + let mut arguments = (0..command.inputs.len()) + .map(|index| format!("p{index}")) + .collect::<Vec<_>>(); + if !specialize::database_effects(program, &target).is_empty() { + arguments.push( + if transaction_database.is_some() { + "Some(&transaction)" + } else { + "None" + } + .into(), + ); + } + writeln!( + output, + "let result = f{}({}){};", + command.function, + arguments.join(", "), + if specialize::suspends(program, &target) { + ".await" + } else { + "" + }, + ) + .expect("string formatting"); + if transaction_database.is_some() { + output.push_str("let value = match result { Ok(value) => value, Err(error) => { return Err(match transaction.rollback().await { Ok(()) => error, Err(rollback) => error.cause(rollback) }); } };\n"); + } else { + output.push_str("let value = result?;\n"); + } + if transaction_database.is_some() { + writeln!(output, "let encoded = match cmd_output_{command_index}_encode(&value) {{ Ok(encoded) => encoded, Err(error) => {{ let rollback = transaction.rollback().await; return Err(match rollback {{ Ok(()) => AppError::fault(\"command\", error), Err(rollback) => AppError::fault(\"command\", error).cause(rollback) }}); }} }};") + .expect("string formatting"); + } else { + writeln!(output, "let encoded = cmd_output_{command_index}_encode(&value).map_err(|error| AppError::fault(\"command\", error))?;") + .expect("string formatting"); + } + if transaction_database.is_some() { + output.push_str("transaction.commit().await.map_err(|error| AppError::database(\"command\", error))?;\n"); + } + output.push_str("Ok(encoded)\n"); + output.push_str("},\n"); + } + output.push_str( + "_ => Err(AppError::fault(\"command\", format!(\"unknown CMD '{name}'\"))),\n}\n}\n", + ); +} diff --git a/crates/dever-core/src/native/build.rs b/crates/dever-core/src/native/build.rs new file mode 100644 index 0000000..279eba0 --- /dev/null +++ b/crates/dever-core/src/native/build.rs @@ -0,0 +1,607 @@ +use std::collections::BTreeSet; +use std::ffi::OsStr; +use std::fs::{self, File, OpenOptions}; +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::SystemTime; + +use crate::hir::Program; +use crate::source::SourceMap; + +mod cache; +mod cleanup; +use cleanup::{BuildDirectory, build_directory, build_directory_at, clean_stale_build_directories}; +pub use cleanup::{CleanupSummary, clean_project_artifacts}; + +/// 单个生成源码统一优化，避免分区使标准集合的泛型内联取决于程序大小。 +pub const OPTIMIZATION_ARGS: [&str; 8] = [ + "-C", + "opt-level=3", + "-C", + "codegen-units=1", + "-C", + "lto=fat", + "-C", + "strip=symbols", +]; + +const TEST_COMPILATION_ARGS: [&str; 6] = [ + "-C", + "opt-level=1", + "-C", + "codegen-units=16", + "-C", + "strip=symbols", +]; + +/// Owns the temporary build directory for as long as the executable is needed. +pub struct NativeProgram { + directory: BuildDirectory, + executable: PathBuf, + cache_hit: bool, +} + +impl NativeProgram { + /// The authenticated compile client verifies the service response before + /// transferring these bytes into a caller-owned executable lifetime. + pub fn from_verified_bytes(bytes: &[u8]) -> Result<Self, String> { + let artifact = Self::new()?; + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o700); + } + let mut output = options + .open(&artifact.executable) + .map_err(|error| format!("cannot create compiled executable: {error}"))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + output + .set_permissions(fs::Permissions::from_mode(0o700)) + .map_err(|error| format!("cannot set compiled executable permissions: {error}"))?; + } + output + .write_all(bytes) + .and_then(|()| output.sync_all()) + .map_err(|error| format!("cannot write compiled executable: {error}"))?; + Ok(artifact) + } + + fn new() -> Result<Self, String> { + let directory = build_directory() + .map_err(|error| format!("cannot create native build directory: {error}"))?; + Ok(Self { + executable: directory.join(format!("program{}", std::env::consts::EXE_SUFFIX)), + directory: BuildDirectory(directory), + cache_hit: false, + }) + } + + pub fn executable(&self) -> &Path { + &self.executable + } + + pub fn cache_hit(&self) -> bool { + self.cache_hit + } + + /// Creates a new output file; existing files (including sources) are never replaced. + pub fn save(&self, path: &Path) -> io::Result<()> { + let mut input = File::open(&self.executable)?; + let mut output = OpenOptions::new().write(true).create_new(true).open(path)?; + let result = (|| { + io::copy(&mut input, &mut output)?; + output.set_permissions(input.metadata()?.permissions())?; + output.sync_all() + })(); + if result.is_err() { + drop(output); + let _ = fs::remove_file(path); + } + result + } +} + +/// Compile already identified inputs using the shared private artifact owner. +/// The callback receives the owned build directory and a new executable path. +/// It must link only the exact inputs represented by `identity`. +pub fn compile_artifact( + cache_directory: &Path, + identity: &str, + build: impl FnOnce(&Path, &Path) -> Result<(), String>, +) -> Result<NativeProgram, String> { + let _ = clean_stale_build_directories(SystemTime::now()); + let cache = cache::Cache::for_artifact(cache_directory, identity); + compile_cached(&cache, cache_directory, |artifact| { + build(&artifact.directory.0, &artifact.executable) + }) +} + +fn compile_cached( + cache: &cache::Cache, + cache_directory: &Path, + build: impl FnOnce(&NativeProgram) -> Result<(), String>, +) -> Result<NativeProgram, String> { + let mut artifact = NativeProgram::new()?; + if cache + .restore(&artifact.executable) + .map_err(|error| format!("cannot restore native cache: {error}"))? + { + artifact.cache_hit = true; + return Ok(artifact); + } + build(&artifact)?; + publish_artifact(cache, cache_directory, &artifact)?; + Ok(artifact) +} + +fn publish_artifact( + cache: &cache::Cache, + cache_directory: &Path, + artifact: &NativeProgram, +) -> Result<(), String> { + fs::create_dir_all(cache_directory) + .map_err(|error| format!("cannot create native cache: {error}"))?; + let staging = BuildDirectory( + build_directory_at(cache_directory) + .map_err(|error| format!("cannot stage native cache: {error}"))?, + ); + cache + .publish(&artifact.executable, &staging.0) + .map_err(|error| format!("cannot publish native cache: {error}")) +} + +pub fn compile( + program: &Program, + sources: &SourceMap, + entry: &str, + rustc: &OsStr, +) -> Result<NativeProgram, String> { + let workspace = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + compile_with_cache( + program, + sources, + entry, + rustc, + &workspace.join("target/native-artifacts"), + ) +} + +pub fn compile_application( + program: &Program, + sources: &SourceMap, + entry: &str, + rustc: &OsStr, + profile: dever_runtime::config::RuntimeProfile, +) -> Result<NativeProgram, String> { + let workspace = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let entry = program.entry(entry)?; + compile_entry( + program, + sources, + entry, + rustc, + &workspace.join("target/native-artifacts"), + Some(profile), + &OPTIMIZATION_ARGS, + ) +} + +pub fn compile_project( + program: &Program, + sources: &SourceMap, + rustc: &OsStr, + profile: dever_runtime::config::RuntimeProfile, +) -> Result<NativeProgram, String> { + compile_project_with_resources(program, sources, rustc, profile, &[]) +} + +pub fn compile_project_with_resources( + program: &Program, + sources: &SourceMap, + rustc: &OsStr, + profile: dever_runtime::config::RuntimeProfile, + resources: &[crate::native::EmbeddedResource], +) -> Result<NativeProgram, String> { + let workspace = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let generated = super::emit_project_with_resources(program, sources, profile, resources)?; + let api = !program.api_routes.is_empty() || !program.api_rest.is_empty(); + compile_generated( + GeneratedProgram { + source: generated, + resources, + }, + rustc, + &workspace.join("target/native-artifacts"), + Some(profile), + api, + &OPTIMIZATION_ARGS, + ) +} + +pub fn compile_test_suite( + program: &Program, + sources: &SourceMap, + rustc: &OsStr, +) -> Result<NativeProgram, String> { + let workspace = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let profile = program + .tests() + .iter() + .any(crate::hir::TestCase::uses_database) + .then_some(dever_runtime::config::RuntimeProfile { + sqlite: true, + postgres: false, + }); + let generated = super::emit_test_suite(program, sources, profile)?; + compile_generated( + GeneratedProgram { + source: generated, + resources: &[], + }, + rustc, + &workspace.join("target/native-artifacts"), + profile, + false, + &TEST_COMPILATION_ARGS, + ) +} + +/// The explicit cache directory lets embedding callers and tests own their artifacts. +pub fn compile_with_cache( + program: &Program, + sources: &SourceMap, + entry: &str, + rustc: &OsStr, + cache_directory: &Path, +) -> Result<NativeProgram, String> { + let entry = program.entry(entry)?; + compile_entry( + program, + sources, + entry, + rustc, + cache_directory, + None, + &OPTIMIZATION_ARGS, + ) +} + +fn compile_entry( + program: &Program, + sources: &SourceMap, + entry: usize, + rustc: &OsStr, + cache_directory: &Path, + profile: Option<dever_runtime::config::RuntimeProfile>, + compilation_args: &[&str], +) -> Result<NativeProgram, String> { + let generated = super::emit_entry(program, sources, entry, profile)?; + let api = super::api_enabled(program, entry)?; + compile_generated( + GeneratedProgram { + source: generated, + resources: &[], + }, + rustc, + cache_directory, + profile, + api, + compilation_args, + ) +} + +struct GeneratedProgram<'a> { + source: String, + resources: &'a [crate::native::EmbeddedResource], +} + +fn compile_generated( + generated: GeneratedProgram<'_>, + rustc: &OsStr, + cache_directory: &Path, + profile: Option<dever_runtime::config::RuntimeProfile>, + api: bool, + compilation_args: &[&str], +) -> Result<NativeProgram, String> { + // Stale cleanup is best effort and must never replace the current compile result. + let _ = clean_stale_build_directories(SystemTime::now()); + let version = compiler_version(rustc)?; + // Emitted code has already pruned unreachable functions and includes direct + // system intrinsics as well as official source wrappers. + let crypto = generated.source.contains("dever_runtime::crypto::"); + let wire = generated.source.contains("dever_runtime::wire::") + || generated.source.contains("initialize_ports"); + let external = generated.source.contains("dever_runtime::external::"); + let runtime = runtime_library(rustc, profile, api, crypto, wire, external)?; + let cache = cache::Cache::new( + cache_directory, + &generated.source, + rustc, + &version, + &runtime, + compilation_args, + ) + .map_err(|error| format!("cannot identify native build inputs: {error}"))?; + compile_cached(&cache, cache_directory, |artifact| { + compile_executable(artifact, &generated, &runtime, rustc, compilation_args) + }) +} + +fn compile_executable( + artifact: &NativeProgram, + generated: &GeneratedProgram<'_>, + runtime: &cache::RuntimeInputs, + rustc: &OsStr, + compilation_args: &[&str], +) -> Result<(), String> { + let runtime_directory = artifact.directory.0.join("runtime"); + runtime + .write(&runtime_directory) + .map_err(|error| format!("cannot stage native runtime: {error}"))?; + let source = artifact.directory.0.join("program.rs"); + // Raw resources are separate compiler inputs. Expanding an interpreter + // into decimal Rust literals multiplies source size and allocator use. + for (index, resource) in super::resource_inputs(generated.resources) + .unique + .iter() + .enumerate() + { + let path = artifact + .directory + .0 + .join(super::resource_source_filename(index)); + let mut options = fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options + .open(path) + .map_err(|error| format!("cannot stage external resource: {error}"))?; + file.write_all(&resource.bytes) + .map_err(|error| format!("cannot stage external resource: {error}"))?; + } + fs::write(&source, &generated.source) + .map_err(|error| format!("cannot write generated program: {error}"))?; + let result = Command::new(rustc) + .arg("--edition=2024") + .arg("--crate-name=dever_program") + .args(compilation_args) + .arg("--extern") + .arg(format!( + "dever_runtime={}", + runtime_directory.join("libdever_runtime.rlib").display() + )) + .arg("-L") + .arg(format!( + "dependency={}", + runtime_directory.join("deps").display() + )) + .arg(&source) + .arg("-o") + .arg(&artifact.executable) + .output() + .map_err(|error| { + format!( + "cannot invoke rustc '{}': {error}; set RUSTC or add rustc to PATH", + rustc.to_string_lossy() + ) + })?; + if !result.status.success() { + return Err(format!( + "native compiler failed ({}):\n{}", + result.status, + String::from_utf8_lossy(&result.stderr) + )); + } + Ok(()) +} + +fn compiler_version(rustc: &OsStr) -> Result<Vec<u8>, String> { + let result = Command::new(rustc) + .arg("--version") + .arg("--verbose") + .output() + .map_err(|error| { + format!( + "cannot invoke rustc '{}': {error}; set RUSTC or add rustc to PATH", + rustc.to_string_lossy() + ) + })?; + if !result.status.success() { + return Err(format!( + "cannot query rustc version ({}): {}", + result.status, + String::from_utf8_lossy(&result.stderr) + )); + } + Ok(result.stdout) +} + +fn runtime_library( + rustc: &OsStr, + profile: Option<dever_runtime::config::RuntimeProfile>, + api: bool, + crypto: bool, + wire: bool, + external: bool, +) -> Result<cache::RuntimeInputs, String> { + let workspace = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let runtime_root = workspace.join("target/native-runtime"); + let target = runtime_root.join("cargo"); + let cargo = std::env::var_os("CARGO").unwrap_or_else(|| { + Path::new(rustc) + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .map(|parent| { + parent + .join(format!("cargo{}", std::env::consts::EXE_SUFFIX)) + .into_os_string() + }) + .unwrap_or_else(|| "cargo".into()) + }); + let mut command = Command::new(cargo); + command.args([ + "build", + "--offline", + "--locked", + "--release", + "--message-format=json-render-diagnostics", + "-p", + "dever-runtime", + ]); + let mut features = Vec::new(); + if let Some(profile) = profile { + features.push(match (profile.sqlite, profile.postgres) { + (true, true) => "sqlite,postgres", + (true, false) => "sqlite", + (false, true) => "postgres", + (false, false) => "database", + }); + } + if api { + features.push("api"); + } + if crypto { + features.push("crypto"); + } + if wire { + features.push("wire"); + } + if external { + features.push("external"); + } + if !features.is_empty() { + command.args(["--features", &features.join(",")]); + } + let result = command + .arg("--manifest-path") + .arg(workspace.join("Cargo.toml")) + .arg("--target-dir") + .arg(&target) + .env("RUSTC", rustc) + .output() + .map_err(|error| format!("cannot prepare native runtime: {error}"))?; + if !result.status.success() { + return Err(format!( + "native runtime compilation failed ({}):\n{}", + result.status, + String::from_utf8_lossy(&result.stderr) + )); + } + let artifacts = cargo_runtime_artifacts( + &result.stdout, + &target, + &workspace.join("crates/dever-runtime/Cargo.toml"), + )?; + cache::RuntimeInputs::publish( + &runtime_root.join("inputs"), + &artifacts.runtime, + &artifacts.dependencies, + ) + .map_err(|error| format!("cannot publish native runtime inputs: {error}")) +} + +struct CargoRuntimeArtifacts { + runtime: PathBuf, + dependencies: Vec<PathBuf>, +} + +fn cargo_runtime_artifacts( + bytes: &[u8], + target: &Path, + runtime_manifest: &Path, +) -> Result<CargoRuntimeArtifacts, String> { + let dependency_root = fs::canonicalize(target.join("release/deps")) + .map_err(|error| format!("cannot locate Cargo runtime dependencies: {error}"))?; + let runtime_manifest = fs::canonicalize(runtime_manifest) + .map_err(|error| format!("cannot locate dever-runtime manifest: {error}"))?; + let mut runtime_candidates = BTreeSet::new(); + let mut dependencies = BTreeSet::new(); + for line in bytes + .split(|byte| *byte == b'\n') + .filter(|line| !line.is_empty()) + { + let message: serde_json::Value = serde_json::from_slice(line) + .map_err(|error| format!("invalid Cargo artifact message: {error}"))?; + if message.get("reason").and_then(serde_json::Value::as_str) != Some("compiler-artifact") { + continue; + } + let target_name = message + .pointer("/target/name") + .and_then(serde_json::Value::as_str); + let is_runtime = target_name == Some("dever_runtime") + && message + .get("manifest_path") + .and_then(serde_json::Value::as_str) + .and_then(|path| fs::canonicalize(path).ok()) + .is_some_and(|path| path == runtime_manifest); + let Some(filenames) = message + .get("filenames") + .and_then(serde_json::Value::as_array) + else { + continue; + }; + for filename in filenames.iter().filter_map(serde_json::Value::as_str) { + let path = fs::canonicalize(filename).map_err(|error| { + format!("cannot inspect Cargo runtime artifact '{filename}': {error}") + })?; + let extension = path.extension().and_then(OsStr::to_str); + if is_runtime + && extension == Some("rmeta") + && path.parent() == Some(dependency_root.as_path()) + { + let hashed_runtime = path.with_extension("rlib"); + if hashed_runtime.is_file() { + runtime_candidates.insert(fs::canonicalize(&hashed_runtime).map_err( + |error| { + format!( + "cannot inspect Cargo runtime artifact '{}': {error}", + hashed_runtime.display(), + ) + }, + )?); + } + } + if !matches!(extension, Some("rlib" | "so" | "dylib" | "dll")) { + continue; + } + if is_runtime && extension == Some("rlib") { + runtime_candidates.insert(path.clone()); + } + if path.parent() == Some(dependency_root.as_path()) { + dependencies.insert(path); + } + } + } + let hashed_runtime = runtime_candidates + .iter() + .filter(|path| path.parent() == Some(dependency_root.as_path())) + .cloned() + .collect::<Vec<_>>(); + let runtime = match hashed_runtime.as_slice() { + [runtime] => runtime.clone(), + [] => match runtime_candidates + .into_iter() + .collect::<Vec<_>>() + .as_slice() + { + [runtime] => runtime.clone(), + [] => return Err("Cargo did not report the dever-runtime library artifact".into()), + _ => return Err("Cargo reported multiple dever-runtime library artifacts".into()), + }, + _ => return Err("Cargo reported multiple hashed dever-runtime library artifacts".into()), + }; + dependencies.remove(&runtime); + Ok(CargoRuntimeArtifacts { + runtime, + dependencies: dependencies.into_iter().collect(), + }) +} diff --git a/crates/dever-core/src/native/build/cache.rs b/crates/dever-core/src/native/build/cache.rs new file mode 100644 index 0000000..a4de171 --- /dev/null +++ b/crates/dever-core/src/native/build/cache.rs @@ -0,0 +1,478 @@ +use std::ffi::OsStr; +use std::fs::{self, File}; +use std::hash::{Hash, Hasher}; +use std::io::{self, Read}; +use std::path::{Path, PathBuf}; +use std::sync::{ + OnceLock, + atomic::{AtomicU64, Ordering}, +}; + +use sha2::{Digest, Sha256}; + +static NEXT_RUNTIME_IDENTITY: AtomicU64 = AtomicU64::new(0); + +/// Cargo may rebuild its shared target after returning. Compile against the exact +/// bytes used for the cache identity rather than mutable files in that target. +pub(super) struct RuntimeInputs { + files: RuntimeFiles, + identity: String, +} + +enum RuntimeFiles { + Snapshot(Vec<(PathBuf, Vec<u8>)>), + Unchanged { root: PathBuf, paths: Vec<PathBuf> }, +} + +impl RuntimeInputs { + pub(super) fn publish( + root: &Path, + runtime: &Path, + dependencies: &[PathBuf], + ) -> io::Result<Self> { + fs::create_dir_all(root)?; + let staging = unique_directory(root, ".dever-runtime-input")?; + let result = (|| { + fs::create_dir(staging.join("deps"))?; + link_or_copy(runtime, &staging.join("libdever_runtime.rlib"))?; + for dependency in dependencies { + let name = dependency.file_name().ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "runtime dependency has no filename", + ) + })?; + link_or_copy(dependency, &staging.join("deps").join(name))?; + } + let staged = Self::read(&staging)?; + let destination = root.join(&staged.identity); + match fs::rename(&staging, &destination) { + Ok(()) => Ok(Self::read(&destination)?), + Err(_error) if is_real_directory(&destination) => { + let existing = Self::read(&destination)?; + if existing.identity != staged.identity { + return Err(io::Error::other("native runtime input identity collision")); + } + Ok(existing) + } + Err(error) => Err(error), + } + })(); + if staging.exists() { + let _ = fs::remove_dir_all(&staging); + } + result + } + + pub(super) fn read(root: &Path) -> io::Result<Self> { + validate_runtime_root(root)?; + let mut paths = vec![PathBuf::from("libdever_runtime.rlib")]; + for entry in fs::read_dir(root.join("deps"))? { + let path = entry?.path(); + paths.push(Path::new("deps").join(path.file_name().expect("dependency filename"))); + } + paths.sort(); + let metadata = paths + .iter() + .map(|path| { + let metadata = fs::metadata(root.join(path))?; + Ok((path.clone(), metadata.len(), metadata.modified()?)) + }) + .collect::<io::Result<Vec<_>>>()?; + let metadata_identity = fingerprint(&metadata); + if let Some(identity) = read_runtime_identity(root, &metadata_identity) { + cleanup_runtime_identities(root, &metadata_identity); + return Ok(Self { + files: RuntimeFiles::Unchanged { + root: root.to_path_buf(), + paths, + }, + identity, + }); + } + let files = read_files(root, &paths)?; + let identity = fingerprint(&files); + write_runtime_identity(root, &metadata_identity, &identity); + Ok(Self { + files: RuntimeFiles::Snapshot(files), + identity, + }) + } + + pub(super) fn write(&self, root: &Path) -> io::Result<()> { + fs::create_dir(root)?; + fs::create_dir(root.join("deps"))?; + match &self.files { + RuntimeFiles::Snapshot(files) => write_files(root, files), + RuntimeFiles::Unchanged { + root: source_root, + paths, + } => { + let files = read_files(source_root, paths)?; + if fingerprint(&files) != self.identity { + return Err(io::Error::other( + "native runtime changed after its cache identity was read", + )); + } + write_files(root, &files) + } + } + } +} + +fn validate_runtime_root(root: &Path) -> io::Result<()> { + let metadata = fs::symlink_metadata(root)?; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "native runtime input '{}' is not a real directory", + root.display() + ), + )); + } + for entry in fs::read_dir(root)? { + let entry = entry?; + let name = entry.file_name(); + let name = name.to_string_lossy(); + let metadata = fs::symlink_metadata(entry.path())?; + let valid = match name.as_ref() { + "libdever_runtime.rlib" => metadata.is_file() && !metadata.file_type().is_symlink(), + "deps" => metadata.is_dir() && !metadata.file_type().is_symlink(), + name if name.starts_with(".dever-runtime-identity-") => { + metadata.is_file() && !metadata.file_type().is_symlink() + } + _ => false, + }; + if !valid { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "unexpected native runtime input '{}'", + entry.path().display() + ), + )); + } + } + let runtime = root.join("libdever_runtime.rlib"); + let metadata = fs::symlink_metadata(&runtime)?; + if metadata.file_type().is_symlink() || !metadata.is_file() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "native runtime input '{}' is not a regular file", + runtime.display() + ), + )); + } + for entry in fs::read_dir(root.join("deps"))? { + let entry = entry?; + let path = entry.path(); + let metadata = fs::symlink_metadata(&path)?; + let supported = path.extension().is_some_and(|extension| { + matches!(extension.to_str(), Some("rlib" | "so" | "dylib" | "dll")) + }); + if metadata.file_type().is_symlink() || !metadata.is_file() || !supported { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("unexpected native runtime dependency '{}'", path.display()), + )); + } + } + Ok(()) +} + +fn is_real_directory(path: &Path) -> bool { + fs::symlink_metadata(path) + .is_ok_and(|metadata| metadata.is_dir() && !metadata.file_type().is_symlink()) +} + +fn unique_directory(root: &Path, prefix: &str) -> io::Result<PathBuf> { + loop { + let path = root.join(format!( + "{prefix}-{}-{}", + std::process::id(), + NEXT_RUNTIME_IDENTITY.fetch_add(1, Ordering::Relaxed), + )); + match fs::create_dir(&path) { + Ok(()) => return Ok(path), + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue, + Err(error) => return Err(error), + } + } +} + +fn link_or_copy(source: &Path, destination: &Path) -> io::Result<()> { + let metadata = fs::symlink_metadata(source)?; + if metadata.file_type().is_symlink() || !metadata.is_file() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "native runtime input '{}' is not a regular file", + source.display() + ), + )); + } + match fs::hard_link(source, destination) { + Ok(()) => Ok(()), + Err(_) => { + fs::copy(source, destination)?; + Ok(()) + } + } +} + +fn read_files(root: &Path, paths: &[PathBuf]) -> io::Result<Vec<(PathBuf, Vec<u8>)>> { + paths + .iter() + .map(|path| Ok((path.clone(), fs::read(root.join(path))?))) + .collect() +} + +fn write_files(root: &Path, files: &[(PathBuf, Vec<u8>)]) -> io::Result<()> { + for (path, bytes) in files { + fs::write(root.join(path), bytes)?; + } + Ok(()) +} + +fn read_runtime_identity(root: &Path, metadata_identity: &str) -> Option<String> { + let text = + fs::read_to_string(root.join(format!(".dever-runtime-identity-{metadata_identity}"))) + .ok()?; + let mut lines = text.lines(); + if lines.next()? != "dever-runtime-identity-v1" || lines.next()? != metadata_identity { + return None; + } + let identity = lines.next()?; + if lines.next().is_some() + || identity.len() != 64 + || !identity.bytes().all(|byte| byte.is_ascii_hexdigit()) + { + return None; + } + Some(identity.to_owned()) +} + +fn write_runtime_identity(root: &Path, metadata_identity: &str, identity: &str) { + let path = root.join(format!(".dever-runtime-identity-{metadata_identity}")); + let staging = root.join(format!( + ".dever-runtime-identity-{}-{}", + std::process::id(), + NEXT_RUNTIME_IDENTITY.fetch_add(1, Ordering::Relaxed), + )); + if fs::write( + &staging, + format!("dever-runtime-identity-v1\n{metadata_identity}\n{identity}\n"), + ) + .is_ok() + { + let _ = fs::rename(&staging, &path); + } + let _ = fs::remove_file(staging); + cleanup_runtime_identities(root, metadata_identity); +} + +fn cleanup_runtime_identities(root: &Path, metadata_identity: &str) { + let current = format!(".dever-runtime-identity-{metadata_identity}"); + if let Ok(entries) = fs::read_dir(root) { + for entry in entries.filter_map(Result::ok) { + let name = entry.file_name(); + let name = name.to_string_lossy(); + let stored_identity = name.strip_prefix(".dever-runtime-identity-"); + if name != current + && stored_identity.is_some_and(|identity| { + matches!(identity.len(), 32 | 64) + && identity.bytes().all(|byte| byte.is_ascii_hexdigit()) + }) + { + let _ = fs::remove_file(entry.path()); + } + } + } +} + +pub(super) struct Cache { + directory: PathBuf, + identity: String, +} + +impl Cache { + pub(super) fn for_artifact(root: &Path, identity: &str) -> Self { + Self { + directory: root.join(fingerprint(&identity)), + identity: identity.to_owned(), + } + } + + pub(super) fn new( + root: &Path, + generated: &str, + rustc: &OsStr, + version: &[u8], + runtime: &RuntimeInputs, + options: &[&str], + ) -> io::Result<Self> { + static COMPILER: OnceLock<String> = OnceLock::new(); + let compiler = match COMPILER.get() { + Some(identity) => identity.clone(), + None => { + // Generated source already captures every backend input. Partition + // compiler builds by their file identity without rehashing a large + // debug compiler binary on every separate CLI invocation. + let path = std::env::current_exe()?; + let metadata = fs::metadata(&path)?; + let identity = fingerprint(&( + env!("CARGO_PKG_VERSION"), + path, + metadata.len(), + metadata.modified()?, + )); + COMPILER.get_or_init(|| identity).clone() + } + }; + let tool = executable_path(rustc)?; + let environment: Vec<_> = [ + "PATH", + "RUSTUP_TOOLCHAIN", + "RUSTC_BOOTSTRAP", + "RUSTFLAGS", + "CARGO_ENCODED_RUSTFLAGS", + "CARGO_BUILD_TARGET", + "RUSTC_WRAPPER", + "RUSTC_WORKSPACE_WRAPPER", + "CC", + "CFLAGS", + "LDFLAGS", + "LIBRARY_PATH", + "LD_LIBRARY_PATH", + "SDKROOT", + "MACOSX_DEPLOYMENT_TARGET", + "SOURCE_DATE_EPOCH", + ] + .into_iter() + .map(|name| (name, std::env::var_os(name))) + .collect(); + let identity = format!( + "dever-native-cache-v1\ncompiler={compiler}\ngenerated={}\ntool={}\nversion={}\nruntime={}\noptions={}\nplatform={}-{}\nenvironment={}\n", + fingerprint(&generated), + fingerprint(&(rustc, tool.as_os_str(), file_fingerprint(&tool)?)), + fingerprint(&version), + runtime.identity, + fingerprint(&options), + std::env::consts::OS, + std::env::consts::ARCH, + fingerprint(&environment), + ); + Ok(Self { + directory: root.join(fingerprint(&identity)), + identity, + }) + } + + pub(super) fn restore(&self, executable: &Path) -> io::Result<bool> { + let stored = match fs::read_to_string(self.directory.join("identity")) { + Ok(identity) => identity, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error), + }; + if stored != self.identity { + return Err(io::Error::other("native cache identity mismatch")); + } + let program = self.directory.join("program"); + let expected = fs::read_to_string(self.directory.join("executable-fingerprint"))?; + if file_fingerprint(&program)? != expected { + return Err(io::Error::other( + "native cache executable is incomplete or modified", + )); + } + fs::copy(program, executable)?; + Ok(true) + } + + pub(super) fn publish(&self, executable: &Path, staging: &Path) -> io::Result<()> { + fs::copy(executable, staging.join("program"))?; + fs::write(staging.join("identity"), &self.identity)?; + fs::write( + staging.join("executable-fingerprint"), + file_fingerprint(executable)?, + )?; + match fs::rename(staging, &self.directory) { + Ok(()) => Ok(()), + // Another complete build won the same key. Incomplete staging directories + // never have a cache key and are never considered by restore. + Err(error) => match fs::read_to_string(self.directory.join("identity")) { + Ok(identity) if identity == self.identity => Ok(()), + Ok(_) => Err(io::Error::other("native cache identity collision")), + Err(_) => Err(error), + }, + } + } +} + +fn executable_path(command: &OsStr) -> io::Result<PathBuf> { + let path = Path::new(command); + if path.components().count() > 1 { + return fs::canonicalize(path); + } + for directory in std::env::split_paths(&std::env::var_os("PATH").unwrap_or_default()) { + let path = directory.join(command); + if path.is_file() { + return fs::canonicalize(path); + } + #[cfg(windows)] + { + let path = path.with_extension("exe"); + if path.is_file() { + return fs::canonicalize(path); + } + } + } + Err(io::Error::new( + io::ErrorKind::NotFound, + "compiler executable not found", + )) +} + +fn fingerprint(value: &impl Hash) -> String { + let mut hasher = CryptographicHasher(Sha256::new()); + value.hash(&mut hasher); + hex(&hasher.0.finalize()) +} + +fn file_fingerprint(path: &Path) -> io::Result<String> { + let mut file = File::open(path)?; + let mut hasher = Sha256::new(); + let mut bytes = [0_u8; 65536]; + loop { + let length = file.read(&mut bytes)?; + if length == 0 { + break; + } + hasher.update(&bytes[..length]); + } + Ok(hex(&hasher.finalize())) +} + +struct CryptographicHasher(Sha256); + +impl Hasher for CryptographicHasher { + fn finish(&self) -> u64 { + let digest = self.0.clone().finalize(); + u64::from_le_bytes(digest[..8].try_into().expect("SHA-256 prefix")) + } + + fn write(&mut self, bytes: &[u8]) { + self.0.update(bytes); + } +} + +fn hex(bytes: &[u8]) -> String { + const DIGITS: &[u8; 16] = b"0123456789abcdef"; + let mut output = String::with_capacity(bytes.len() * 2); + for byte in bytes { + output.push(DIGITS[usize::from(byte >> 4)] as char); + output.push(DIGITS[usize::from(byte & 0x0f)] as char); + } + output +} diff --git a/crates/dever-core/src/native/build/cleanup.rs b/crates/dever-core/src/native/build/cleanup.rs new file mode 100644 index 0000000..a74aeeb --- /dev/null +++ b/crates/dever-core/src/native/build/cleanup.rs @@ -0,0 +1,343 @@ +use std::fs::{self, OpenOptions}; +use std::io; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +static NEXT_BUILD: AtomicU64 = AtomicU64::new(0); +const BUILD_DIRECTORY_PREFIX: &str = "dever-build-"; +const BUILD_MARKER: &str = ".dever-owner"; +const BUILD_MARKER_VERSION: &str = "dever-build-owner-v1"; +const STALE_BUILD_AGE: Duration = Duration::from_secs(24 * 60 * 60); + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct CleanupSummary { + pub directories: u64, + pub files: u64, + pub bytes: u64, +} + +pub(super) struct BuildDirectory(pub(super) PathBuf); + +impl Drop for BuildDirectory { + fn drop(&mut self) { + // Cleanup cannot replace a compilation/runtime result with a second error. + let _ = fs::remove_dir_all(&self.0); + } +} + +pub(super) fn build_directory() -> io::Result<PathBuf> { + let path = build_directory_at(&std::env::temp_dir())?; + if let Err(error) = write_build_marker(&path, SystemTime::now()) { + let _ = fs::remove_dir_all(&path); + return Err(error); + } + Ok(path) +} + +pub(super) fn build_directory_at(parent: &Path) -> io::Result<PathBuf> { + loop { + let id = NEXT_BUILD.fetch_add(1, Ordering::Relaxed); + let path = parent.join(format!( + "{BUILD_DIRECTORY_PREFIX}{}-{id}", + std::process::id() + )); + let mut builder = fs::DirBuilder::new(); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + match builder.create(&path) { + Ok(()) => return Ok(path), + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue, + Err(error) => return Err(error), + } + } +} + +pub fn clean_project_artifacts(project_root: &Path) -> io::Result<CleanupSummary> { + let now = SystemTime::now(); + let mut summary = clean_stale_build_directories(now)?; + let owner = current_owner_id(&std::env::temp_dir())?; + let entries = match fs::read_dir(project_root) { + Ok(entries) => entries, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(summary), + Err(error) => return Err(error), + }; + for entry in entries { + let Ok(entry) = entry else { + continue; + }; + let name = entry.file_name(); + let Some(pid) = name.to_str().and_then(legacy_run_pid) else { + continue; + }; + if pid == u64::from(std::process::id()) { + continue; + } + let Ok(metadata) = fs::symlink_metadata(entry.path()) else { + continue; + }; + if metadata.file_type().is_symlink() + || !metadata.is_file() + || !owner_matches(&metadata, owner) + { + continue; + } + let Ok(modified) = metadata.modified() else { + continue; + }; + if !is_stale(modified, now) { + continue; + } + if process_is_active(pid) { + continue; + } + let bytes = metadata.len(); + fs::remove_file(entry.path())?; + summary.files += 1; + summary.bytes += bytes; + } + Ok(summary) +} + +pub(super) fn clean_stale_build_directories(now: SystemTime) -> io::Result<CleanupSummary> { + let root = std::env::temp_dir(); + let owner = current_owner_id(&root)?; + let mut summary = CleanupSummary::default(); + for entry in fs::read_dir(&root)? { + let Ok(entry) = entry else { + continue; + }; + let name = entry.file_name(); + let Some(pid) = name.to_str().and_then(build_directory_pid) else { + continue; + }; + if pid == u64::from(std::process::id()) { + continue; + } + let path = entry.path(); + let Ok(metadata) = fs::symlink_metadata(&path) else { + continue; + }; + if metadata.file_type().is_symlink() + || !metadata.is_dir() + || !owner_matches(&metadata, owner) + { + continue; + } + let Ok(Some(marker)) = read_build_marker(&path) else { + continue; + }; + if marker.pid != pid + || marker.owner != owner + || !is_stale(marker.created, now) + || process_is_active(pid) + { + continue; + } + let Ok(Some((files, bytes))) = directory_usage(&path) else { + continue; + }; + fs::remove_dir_all(&path)?; + summary.directories += 1; + summary.files += files; + summary.bytes += bytes; + } + Ok(summary) +} + +struct BuildMarker { + created: SystemTime, + pid: u64, + owner: u64, +} + +fn write_build_marker(path: &Path, created: SystemTime) -> io::Result<()> { + let seconds = created + .duration_since(UNIX_EPOCH) + .map_err(|_| io::Error::other("system clock is before the Unix epoch"))? + .as_secs(); + let owner = metadata_owner(&fs::metadata(path)?); + fs::write( + path.join(BUILD_MARKER), + format!( + "{BUILD_MARKER_VERSION}\ncreated={seconds}\npid={}\nowner={owner}\ncompiler={}\n", + std::process::id(), + env!("CARGO_PKG_VERSION"), + ), + ) +} + +fn read_build_marker(path: &Path) -> io::Result<Option<BuildMarker>> { + let marker_path = path.join(BUILD_MARKER); + let metadata = match fs::symlink_metadata(&marker_path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; + if metadata.file_type().is_symlink() || !metadata.is_file() { + return Ok(None); + } + let text = fs::read_to_string(marker_path)?; + let mut lines = text.lines(); + if lines.next() != Some(BUILD_MARKER_VERSION) { + return Ok(None); + } + let Some(created) = parse_marker_number(lines.next(), "created") else { + return Ok(None); + }; + let Some(pid) = parse_marker_number(lines.next(), "pid") else { + return Ok(None); + }; + let Some(owner) = parse_marker_number(lines.next(), "owner") else { + return Ok(None); + }; + let compiler = lines.next().and_then(|line| line.strip_prefix("compiler=")); + if compiler != Some(env!("CARGO_PKG_VERSION")) || lines.next().is_some() { + return Ok(None); + } + Ok(Some(BuildMarker { + created: UNIX_EPOCH + Duration::from_secs(created), + pid, + owner, + })) +} + +fn parse_marker_number(line: Option<&str>, key: &str) -> Option<u64> { + line.and_then(|line| line.strip_prefix(key)) + .and_then(|line| line.strip_prefix('=')) + .and_then(|value| value.parse().ok()) +} + +fn build_directory_pid(name: &str) -> Option<u64> { + let suffix = name.strip_prefix(BUILD_DIRECTORY_PREFIX)?; + let (pid, id) = suffix.split_once('-')?; + if id.is_empty() || !id.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + pid.parse().ok() +} + +fn legacy_run_pid(name: &str) -> Option<u64> { + let pid = name.strip_prefix(".dever-run-")?; + if pid.is_empty() || !pid.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + pid.parse().ok() +} + +fn is_stale(created: SystemTime, now: SystemTime) -> bool { + now.duration_since(created) + .is_ok_and(|age| age >= STALE_BUILD_AGE) +} + +#[cfg(target_os = "linux")] +fn process_is_active(pid: u64) -> bool { + match fs::symlink_metadata(Path::new("/proc").join(pid.to_string())) { + Ok(metadata) => metadata.is_dir() && !metadata.file_type().is_symlink(), + Err(error) if error.kind() == io::ErrorKind::NotFound => false, + Err(_) => true, + } +} + +#[cfg(all(unix, not(target_os = "linux")))] +fn process_is_active(pid: u64) -> bool { + std::process::Command::new("/bin/kill") + .args(["-0", &pid.to_string()]) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .map_or(true, |status| status.success()) +} + +#[cfg(windows)] +fn process_is_active(pid: u64) -> bool { + let filter = format!("PID eq {pid}"); + let Ok(output) = std::process::Command::new("tasklist.exe") + .args(["/FI", &filter, "/FO", "CSV", "/NH"]) + .output() + else { + return true; + }; + if !output.status.success() { + return true; + } + let expected = format!("\"{pid}\""); + String::from_utf8_lossy(&output.stdout) + .lines() + .any(|line| line.split(',').nth(1) == Some(expected.as_str())) +} + +#[cfg(not(any(unix, windows)))] +fn process_is_active(_pid: u64) -> bool { + true +} + +fn directory_usage(path: &Path) -> io::Result<Option<(u64, u64)>> { + let mut files = 0_u64; + let mut bytes = 0_u64; + for entry in fs::read_dir(path)? { + let entry = entry?; + let metadata = fs::symlink_metadata(entry.path())?; + if metadata.file_type().is_symlink() { + return Ok(None); + } + if metadata.is_dir() { + let Some((child_files, child_bytes)) = directory_usage(&entry.path())? else { + return Ok(None); + }; + files += child_files; + bytes += child_bytes; + } else if metadata.is_file() { + files += 1; + bytes += metadata.len(); + } else { + return Ok(None); + } + } + Ok(Some((files, bytes))) +} + +#[cfg(unix)] +fn metadata_owner(metadata: &fs::Metadata) -> u64 { + use std::os::unix::fs::MetadataExt; + u64::from(metadata.uid()) +} + +#[cfg(not(unix))] +fn metadata_owner(_metadata: &fs::Metadata) -> u64 { + 0 +} + +fn current_owner_id(parent: &Path) -> io::Result<u64> { + loop { + let path = parent.join(format!( + ".dever-owner-probe-{}-{}", + std::process::id(), + NEXT_BUILD.fetch_add(1, Ordering::Relaxed), + )); + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + match options.open(&path) { + Ok(file) => { + let result = file.metadata().map(|metadata| metadata_owner(&metadata)); + drop(file); + let _ = fs::remove_file(path); + return result; + } + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue, + Err(error) => return Err(error), + } + } +} + +fn owner_matches(metadata: &fs::Metadata, owner: u64) -> bool { + metadata_owner(metadata) == owner +} diff --git a/crates/dever-core/src/native/collections.rs b/crates/dever-core/src/native/collections.rs new file mode 100644 index 0000000..b010e89 --- /dev/null +++ b/crates/dever-core/src/native/collections.rs @@ -0,0 +1,330 @@ +use crate::hir::{CollectionOp as Op, Expression, HandlerTarget, SequenceKind}; +use crate::specialize::{self, Specialization}; +use crate::types::Type; + +use super::{DatabaseContext, Emitter, optional_result, rust_type}; + +impl Emitter<'_> { + pub(super) fn collection( + &self, + operation: Op, + handler: Option<HandlerTarget>, + sequence: Option<SequenceKind>, + arguments: &[Expression], + result: &Expression, + ) -> String { + let traverses = matches!( + operation, + Op::Each | Op::Reduce | Op::ReduceUntil | Op::Filter | Op::Find | Op::Sum + ); + let split = if traverses { + self.split_arguments(&arguments[0]) + } else { + None + }; + let pipeline = if traverses { + self.fused_iterator(&arguments[0]) + } else { + None + }; + let mut bindings = if let Some((bindings, _)) = &pipeline { + bindings.clone() + } else if let Some([text, separator]) = split { + format!( + "let a0_text = {}; let a0_separator = {};", + self.expression(text), + self.expression(separator) + ) + } else { + self.bind_arguments(arguments) + }; + if pipeline.is_some() || split.is_some() { + for (index, argument) in arguments.iter().enumerate().skip(1) { + bindings.push_str(&format!( + "let a{index}: {} = {};", + rust_type(&argument.ty), + self.expression(argument) + )); + } + } + if operation == Op::ParallelEach { + let target = Specialization { + function: specialize::resolve_handler( + handler.expect("checked parallel handler"), + &self.bindings, + ), + handlers: Vec::new(), + }; + let async_handler = specialize::suspends(self.program, &target); + let sequence = sequence.expect("checked parallel sequence"); + let (dispatch, values, element) = match sequence { + SequenceKind::AsyncStream => ("parallel_each_stream_typed", "a0", "element"), + SequenceKind::List if async_handler => ( + "parallel_each_stream_typed", + "dever_runtime::async_stream::AsyncStream::from_values(a0.into_values())", + "element", + ), + SequenceKind::Bytes if async_handler => ( + "parallel_each_stream_typed", + "dever_runtime::async_stream::AsyncStream::from_values(a0.into_values().map(i64::from))", + "element", + ), + SequenceKind::List if self.async_context => { + ("parallel_each_typed", "a0.into_values()", "element") + } + SequenceKind::Bytes if self.async_context => ( + "parallel_each_typed", + "a0.into_values()", + "i64::from(element)", + ), + SequenceKind::List => ("each_slice_typed", "a0.values()", "element.clone()"), + SequenceKind::Bytes => ("each_slice_typed", "a0.values()", "i64::from(*element)"), + SequenceKind::Stream => { + ("each_typed", "std::iter::from_fn(|| a0.pull())", "element") + } + SequenceKind::RowStream => unreachable!("checked sequential database stream"), + }; + let context = if arguments.len() == 3 { + ", a2.clone()" + } else { + "" + }; + let call = if async_handler { + let capture = if arguments.len() == 3 { + "let context = a2.clone();" + } else { + "" + }; + let argument = if arguments.len() == 3 { + ", context" + } else { + "" + }; + let invoke = self.invoke( + &target, + format!("{element}{argument}"), + DatabaseContext::Detached, + ); + format!( + "task::{dispatch}({values}, a1, move |element| {{ {capture} async move {{ {invoke}.await }} }}).await" + ) + } else if sequence == SequenceKind::AsyncStream { + let capture = if arguments.len() == 3 { + "let context = a2.clone();" + } else { + "" + }; + let argument = if arguments.len() == 3 { + ", context" + } else { + "" + }; + let invoke = self.invoke( + &target, + format!("{element}{argument}"), + DatabaseContext::Detached, + ); + format!( + "task::{dispatch}({values}, a1, move |element| {{ {capture} async move {{ {invoke} }} }}).await" + ) + } else if self.async_context { + let invoke = self.invoke( + &target, + format!("{element}{context}"), + DatabaseContext::Detached, + ); + format!("task::{dispatch}({values}, a1, move |element| {invoke}).await") + } else { + let invoke = self.invoke( + &target, + format!("{element}{context}"), + DatabaseContext::Detached, + ); + format!("dever_runtime::concurrent::{dispatch}({values}, a1, |element| {invoke})") + }; + return format!( + "{{ {bindings}\n{} }}", + self.call_failure(&call, result.span) + ); + } + let body = match operation { + Op::Append => "a0.append(a1)".into(), + Op::First => optional_result("a0.into_first()", list_element(&arguments[0].ty)), + Op::Get => { + let Type::Map(_, value) = &arguments[0].ty else { + unreachable!() + }; + optional_result("a0.into_get(&a1)", value) + } + Op::Put => "a0.put(a1, a2)".into(), + Op::Remove => "a0.remove(&a1)".into(), + Op::Entries => "a0.into_entries()".into(), + Op::Length => match &arguments[0].ty { + Type::List(_) => "a0.values().len() as i64".into(), + Type::Map(_, _) => "a0.pairs().count() as i64".into(), + Type::Text => "a0.chars().count() as i64".into(), + Type::Bytes => "a0.length()".into(), + _ => unreachable!(), + }, + Op::Close => "a0.close(); ()".into(), + _ => { + let sequence = sequence.expect("checked sequence operation"); + let consume = + sequence == SequenceKind::List && arguments[0].ty.movable(&self.program.types); + let list_values = if let Some((_, iterator)) = &pipeline { + iterator.as_str() + } else if split.is_some() { + "dever_runtime::text::pieces(&a0_text, &a0_separator)" + } else if consume { + "a0.into_values()" + } else { + "a0.values()" + }; + let element = match sequence { + SequenceKind::List + if consume && !matches!(operation, Op::Filter | Op::Find) => + { + "element" + } + SequenceKind::List => "element.clone()", + SequenceKind::Bytes => "i64::from(*element)", + SequenceKind::Stream | SequenceKind::AsyncStream | SequenceKind::RowStream => { + "element" + } + }; + let call = match handler { + Some(handler) => { + let target = Specialization { + function: specialize::resolve_handler(handler, &self.bindings), + handlers: Vec::new(), + }; + let state = if matches!(operation, Op::Reduce | Op::ReduceUntil) { + ", state" + } else if arguments.len() == 2 { + ", a1.clone()" + } else { + "" + }; + let wait = if specialize::suspends(self.program, &target) { + ".await" + } else { + "" + }; + let invoke = self.invoke( + &target, + format!("{element}{state}"), + DatabaseContext::Inherit, + ); + self.call_failure(&format!("{invoke}{wait}"), result.span) + } + None => element.into(), + }; + let loop_over = |body: &str| match sequence { + SequenceKind::List if consume => { + format!("for element in {list_values} {{ {body} }}") + } + SequenceKind::List | SequenceKind::Bytes => { + format!("for element in a0.values() {{ {body} }}") + } + SequenceKind::Stream => { + format!("while let Some(element) = a0.pull() {{ {body} }}") + } + SequenceKind::AsyncStream => { + format!( + "let _stream_owner = a0.close_on_drop(); while let Some(element) = a0.pull().await {{ {body} }}" + ) + } + SequenceKind::RowStream => { + let pull = self.database_failure("a0.pull().await", result.span); + format!( + "let _stream_owner = a0.close_on_drop(); while let Some(element) = {pull} {{ {body} }}" + ) + } + }; + match operation { + Op::Each if result.ty == Type::Unit => loop_over(&format!("{call};")), + Op::Each => { + let capacity = match sequence { + SequenceKind::List if split.is_some() || pipeline.is_some() => "0", + SequenceKind::List => "a0.values().len()", + SequenceKind::Bytes => "a0.values().len()", + SequenceKind::Stream + | SequenceKind::AsyncStream + | SequenceKind::RowStream => unreachable!("checked stream each output"), + }; + format!( + "let mut values = Vec::with_capacity({capacity}); {} List::new(values)", + loop_over(&format!("values.push({call});")) + ) + } + Op::Filter => { + let selected = if consume { + "element" + } else { + "element.clone()" + }; + format!( + "let mut values = Vec::new(); for element in {list_values} {{ if {call} {{ values.push({selected}); }} }} List::new(values)" + ) + } + Op::Find => { + let selected = if consume { + "element" + } else { + "element.clone()" + }; + let found = if matches!(list_element(&arguments[0].ty), Type::Nullable(_)) { + selected.to_owned() + } else { + format!("Some({selected})") + }; + format!( + "let mut found = None; for element in {list_values} {{ if {call} {{ found = {found}; break; }} }} found" + ) + } + Op::Sum => { + let zero = match result.ty { + Type::Int => "0i64", + Type::Decimal => "DecimalValue::ZERO", + Type::Float => "0.0f64", + _ => unreachable!(), + }; + let add = match result.ty { + Type::Int => { + self.failure("number::int_add(total, projected)", result.span) + } + Type::Decimal => { + self.failure("total.checked_add(projected)", result.span) + } + Type::Float => "total + projected".into(), + _ => unreachable!(), + }; + format!( + "let mut total: {} = {zero}; for element in {list_values} {{ let projected = {call}; total = {add}; }} total", + rust_type(&result.ty) + ) + } + Op::Reduce => format!( + "let mut state = a1; {} state", + loop_over(&format!("state = {call};")) + ), + Op::ReduceUntil => format!( + "let mut state = a1; {} state", + loop_over(&format!( + "let step = {call}; state = step.0; if step.1 {{ break; }}" + )) + ), + _ => unreachable!(), + } + } + }; + format!("{{ {bindings}\n{body} }}") + } +} + +fn list_element(ty: &Type) -> &Type { + let Type::List(element) = ty else { + unreachable!() + }; + element +} diff --git a/crates/dever-core/src/native/expressions.rs b/crates/dever-core/src/native/expressions.rs new file mode 100644 index 0000000..5c11bf4 --- /dev/null +++ b/crates/dever-core/src/native/expressions.rs @@ -0,0 +1,494 @@ +use crate::hir::{CallArgument, CallTarget, Expression, ExpressionKind as Expr}; +use crate::specialize::{self, Specialization}; +use crate::syntax::{BinaryOperator as Op, UnaryOperator}; +use crate::types::Type; + +use super::{DatabaseContext, Emitter, field_suffix, optional_result, rust_type, typed_constant}; + +impl Emitter<'_> { + pub(super) fn expression(&self, value: &Expression) -> String { + match &value.kind { + Expr::Constant(constant) => typed_constant(constant, &value.ty), + Expr::Local(slot) if self.can_move(value) => format!("_v{slot}"), + Expr::Local(slot) => format!("_v{slot}.clone()"), + Expr::Field { .. } => self.field_expression(value), + Expr::ChoiceOptions { choice } => { + let crate::types::Shape::Choice(variants) = &self.program.types[*choice].shape + else { + unreachable!("checked Model choice options") + }; + let entries = variants + .iter() + .enumerate() + .map(|(variant, definition)| { + format!( + "(T{choice}::V{variant}, String::from({}))", + super::rust_string( + definition.label.as_deref().unwrap_or(&definition.name) + ) + ) + }) + .collect::<Vec<_>>() + .join(", "); + format!("Map::<T{choice}, String>::from_unique(vec![{entries}])") + } + Expr::ModelOperation { model, operation } => { + self.model_operation(*model, operation, value) + } + Expr::JobEnqueue { + target, + arguments, + scheduled, + } => self.enqueue_job(*target, arguments, *scheduled, value.span), + Expr::Call { target, arguments } => { + let instance = self.call_instance(*target, arguments); + let call = self.call(&instance, arguments, DatabaseContext::Inherit); + let call = if specialize::suspends(self.program, &instance) { + format!("{call}.await") + } else { + call + }; + self.call_failure(&call, value.span) + } + Expr::Fail(error) => self.fail_expression(error, value), + Expr::CaptureResult { + target, + arguments, + choice, + success, + } => self.capture_result(*target, arguments, *choice, *success, value), + Expr::RunCall { + group, + target, + arguments, + } => { + let instance = self.call_instance(*target, arguments); + let call = self.call(&instance, arguments, DatabaseContext::Detached); + let call = if specialize::suspends(self.program, &instance) { + call + } else { + format!("task::blocking_typed(move || {call})") + }; + let operation = match group { + Some(group) => format!("{}.run({call}).await", self.group_local(group)), + None => format!("task::run_typed({call}).await"), + }; + if group.is_some() { + self.call_failure(&operation, value.span) + } else { + self.failure(&operation, value.span) + } + } + Expr::AwaitTask(task) => self.call_failure( + &format!("task::wait({}).await", self.expression(task)), + value.span, + ), + Expr::StopTask(task) => self.call_failure( + &format!("task::stop({}).await", self.expression(task)), + value.span, + ), + Expr::Group(limit) => self.failure( + &format!( + "task::Group::<AppError>::new_typed({})", + self.expression(limit) + ), + value.span, + ), + Expr::AwaitGroup(group) => self.call_failure( + &format!("task::Group::wait({}).await", self.expression(group)), + value.span, + ), + Expr::StopGroup(group) => self.call_failure( + &format!("task::Group::stop({}).await", self.expression(group)), + value.span, + ), + Expr::ParallelCall { target, arguments } => { + self.thread_call("parallel", *target, arguments, value) + } + Expr::BlockingCall { target, arguments } => { + self.thread_call("blocking", *target, arguments, value) + } + Expr::Channel { element, capacity } => self.failure( + &format!( + "channel::Channel::<{}>::new({})", + rust_type(element), + self.expression(capacity) + ), + value.span, + ), + Expr::ChannelSend { + channel, + value: item, + } => self.failure( + &format!( + "({}).send({}).await", + self.expression(channel), + self.expression(item) + ), + value.span, + ), + Expr::ChannelReceive(channel) => { + let Type::Channel(element) = &channel.ty else { + unreachable!("checked channel receive") + }; + let received = self.failure( + &format!("({}).receive().await", self.expression(channel)), + value.span, + ); + optional_result(&received, element) + } + Expr::ChannelClose(channel) => self.failure( + &format!("({}).close().await", self.expression(channel)), + value.span, + ), + Expr::Intrinsic { + operation, + handler, + arguments, + } => self.intrinsic(*operation, *handler, arguments, value), + Expr::Collection { + operation, + handler, + sequence, + arguments, + } => self.collection(*operation, *handler, *sequence, arguments, value), + Expr::Record { fields } => { + let Type::Named(id) = value.ty else { + unreachable!() + }; + // Field expressions execute in source order, independent of declaration order. + let bindings = fields + .iter() + .map(|(index, value)| format!("let field{index} = {};", self.expression(value))) + .collect::<Vec<_>>() + .join("\n"); + let fields = fields + .iter() + .map(|(index, _)| format!("f{index}: field{index}")) + .collect::<Vec<_>>() + .join(", "); + format!("{{ {bindings} T{id} {{ {fields} }} }}") + } + Expr::Variant { variant, arguments } => { + let Type::Named(id) = value.ty else { + unreachable!() + }; + if arguments.is_empty() { + format!("T{id}::V{variant}") + } else { + format!("T{id}::V{variant}({})", self.arguments(arguments)) + } + } + Expr::List(elements) => format!( + "{}::new(vec![{}])", + rust_type(&value.ty).replacen("List<", "List::<", 1), + self.arguments(elements) + ), + Expr::Map(entries) => { + let entries = entries + .iter() + .map(|(key, value)| { + format!("({}, {})", self.expression(key), self.expression(value)) + }) + .collect::<Vec<_>>() + .join(", "); + self.failure( + &format!( + "{}::new(vec![{entries}])", + rust_type(&value.ty).replacen("Map<", "Map::<", 1) + ), + value.span, + ) + } + Expr::Promote(value) => format!("DecimalValue::from_int({})", self.expression(value)), + Expr::Some(value) => format!("Some({})", self.expression(value)), + Expr::Unary { + operator, + value: operand, + } => { + let operand_text = self.expression(operand); + match (operator, &operand.ty) { + (UnaryOperator::Not, _) => format!("!({operand_text})"), + (UnaryOperator::Negate, Type::Float) => format!("-({operand_text})"), + (UnaryOperator::Negate, Type::Int) => { + self.failure(&format!("number::int_neg({operand_text})"), value.span) + } + (UnaryOperator::Negate, Type::Decimal) => { + self.failure(&format!("({operand_text}).checked_neg()"), value.span) + } + _ => unreachable!(), + } + } + Expr::Binary { + left, + operator, + right, + } => self.binary(left, *operator, right, value), + } + } + + fn can_move(&self, value: &Expression) -> bool { + self.moves.contains(&(value as *const Expression)) && value.ty.movable(&self.program.types) + } + + fn fail_expression(&self, error: &Expression, result: &Expression) -> String { + let Type::Named(ty) = error.ty else { + unreachable!("checked failure choice") + }; + let Expr::Variant { variant, arguments } = &error.kind else { + unreachable!("checked direct failure variant") + }; + let payload = if arguments.is_empty() { + String::new() + } else { + format!("({})", self.arguments(arguments)) + }; + format!( + "return Err(AppError::business(AppErrorKind::E{ty}V{variant}{payload}, {}))", + self.location(result.span) + ) + } + + fn capture_result( + &self, + target: CallTarget, + arguments: &[CallArgument], + choice: usize, + success: usize, + result: &Expression, + ) -> String { + let instance = self.call_instance(target, arguments); + let mut call = self.call(&instance, arguments, DatabaseContext::Inherit); + if specialize::suspends(self.program, &instance) { + call.push_str(".await"); + } + let success_fields = match &self.program.types[choice].shape { + crate::types::Shape::Choice(variants) => variants[success].fields.len(), + _ => unreachable!("checked result capture choice"), + }; + let success_arm = match success_fields { + 0 => format!("Ok(()) => T{choice}::V{success}"), + 1 => format!("Ok(value) => T{choice}::V{success}(value)"), + count => { + let values = (0..count) + .map(|index| format!("value{index}")) + .collect::<Vec<_>>(); + format!( + "Ok(({})) => T{choice}::V{success}({})", + values.join(", "), + values.join(", ") + ) + } + }; + let failures = specialize::failures(self.program, &instance); + let error_arms = crate::capture::targets(&self.program.types, choice, &failures) + .expect("validated result capture errors") + .into_iter() + .map(|(failure, target)| { + let crate::types::Shape::Choice(variants) = &self.program.types[failure.ty].shape + else { + unreachable!("checked error choice") + }; + let definition = &variants[failure.variant]; + let fields = (0..definition.fields.len()) + .map(|index| format!("field{index}")) + .collect::<Vec<_>>(); + let payload = if fields.is_empty() { + String::new() + } else { + format!("({})", fields.join(", ")) + }; + let error = format!("T{}::V{}{payload}", failure.ty, failure.variant); + let value = if target.wrapped { + format!("T{choice}::V{}({error})", target.variant) + } else { + error + }; + format!( + "Err(AppError {{ kind: AppErrorKind::E{}V{}{payload}, .. }}) => {value}", + failure.ty, failure.variant + ) + }) + .collect::<Vec<_>>() + .join(", "); + format!( + "match {call} {{ {success_arm}, {error_arms}, Err(error) => return Err(error.at({})) }}", + self.location(result.span) + ) + } + + fn call_instance(&self, target: CallTarget, arguments: &[CallArgument]) -> Specialization { + match target { + CallTarget::Function(function) => { + specialize::call_specialization(function, arguments, &self.bindings) + } + CallTarget::Handler(target) => Specialization { + function: specialize::resolve_handler(target, &self.bindings), + handlers: Vec::new(), + }, + } + } + + fn call( + &self, + function: &Specialization, + arguments: &[CallArgument], + database: DatabaseContext, + ) -> String { + self.invoke(function, self.call_arguments(arguments), database) + } + + fn group_local(&self, group: &Expression) -> String { + let Expr::Local(slot) = group.kind else { + unreachable!("checked Group operand") + }; + format!("_v{slot}") + } + + fn thread_call( + &self, + operation: &str, + target: CallTarget, + arguments: &[CallArgument], + result: &Expression, + ) -> String { + let mut value_index = 0; + let mut bindings = Vec::new(); + let mut call_arguments = Vec::new(); + for argument in arguments { + match argument { + CallArgument::Value(value) => { + bindings.push(format!( + "let a{value_index}: {} = {};", + rust_type(&value.ty), + self.expression(value) + )); + call_arguments.push(format!("a{value_index}")); + value_index += 1; + } + CallArgument::Handler(_) => {} + } + } + let function = self.call_instance(target, arguments); + let call = self.invoke( + &function, + call_arguments.join(", "), + DatabaseContext::Detached, + ); + let operation = self.call_failure( + &format!("task::{operation}_typed(move || {call}).await"), + result.span, + ); + format!("{{ {} {operation} }}", bindings.join(" ")) + } + + fn field_expression(&self, expression: &Expression) -> String { + let mut base = expression; + let mut fields = Vec::new(); + while let Expr::Field { value, index } = &base.kind { + fields.push(field_suffix(&value.ty, *index)); + base = value; + } + let suffix = fields.into_iter().rev().collect::<String>(); + if let Expr::Local(slot) = base.kind { + let projected = format!("_v{slot}{suffix}"); + if self.can_move(expression) || self.can_move(base) { + projected + } else { + format!("{projected}.clone()") + } + } else { + format!("({}){suffix}", self.expression(base)) + } + } + + pub(super) fn arguments(&self, arguments: &[Expression]) -> String { + arguments + .iter() + .map(|value| self.expression(value)) + .collect::<Vec<_>>() + .join(", ") + } + + fn call_arguments(&self, arguments: &[CallArgument]) -> String { + arguments + .iter() + .filter_map(|argument| match argument { + CallArgument::Value(value) => Some(self.expression(value)), + CallArgument::Handler(_) => None, + }) + .collect::<Vec<_>>() + .join(", ") + } + + pub(super) fn bind_arguments(&self, arguments: &[Expression]) -> String { + arguments + .iter() + .enumerate() + .map(|(index, value)| { + format!( + "let a{index}: {} = {};", + rust_type(&value.ty), + self.expression(value) + ) + }) + .collect::<Vec<_>>() + .join("\n") + } + + fn binary( + &self, + left: &Expression, + operator: Op, + right: &Expression, + result: &Expression, + ) -> String { + let left_text = self.expression(left); + let right_text = self.expression(right); + let symbol = match operator { + Op::Add => "+", + Op::Subtract => "-", + Op::Multiply => "*", + Op::Divide => "/", + Op::IntegerDivide => "/", + Op::Remainder => "%", + Op::Equal => "==", + Op::NotEqual => "!=", + Op::Less => "<", + Op::LessEqual => "<=", + Op::Greater => ">", + Op::GreaterEqual => ">=", + Op::And => "&&", + Op::Or => "||", + }; + if matches!(operator, Op::And | Op::Or) { + return format!("(({left_text}) {symbol} ({right_text}))"); + } + let operation = if matches!( + operator, + Op::Equal | Op::NotEqual | Op::Less | Op::LessEqual | Op::Greater | Op::GreaterEqual + ) { + format!("a0 {symbol} a1") + } else if left.ty == Type::Text { + "a0 + &a1".into() + } else if left.ty == Type::Float { + format!("a0 {symbol} a1") + } else { + let method = match operator { + Op::Add => "add", + Op::Subtract => "sub", + Op::Multiply => "mul", + Op::Divide | Op::IntegerDivide => "div", + Op::Remainder => "rem", + _ => unreachable!(), + }; + let operation = if left.ty == Type::Int { + format!("number::int_{method}(a0, a1)") + } else { + format!("a0.checked_{method}(a1)") + }; + self.failure(&operation, result.span) + }; + format!("{{ let a0 = {left_text}; let a1 = {right_text}; {operation} }}") + } +} diff --git a/crates/dever-core/src/native/http.rs b/crates/dever-core/src/native/http.rs new file mode 100644 index 0000000..1823c5b --- /dev/null +++ b/crates/dever-core/src/native/http.rs @@ -0,0 +1,314 @@ +use crate::hir::{Expression, HandlerTarget}; +use crate::specialize::{self, Specialization}; +use crate::types::{Parameter, Shape, Type}; + +use super::{DatabaseContext, Emitter}; + +#[derive(Clone, Copy)] +enum Direction { + ToRuntime, + ToLanguage, +} + +impl Emitter<'_> { + pub(super) fn live_network( + &self, + operation: crate::intrinsic::Intrinsic, + arguments: &[Expression], + result: &Expression, + ) -> String { + use crate::intrinsic::Intrinsic::*; + let input = |index: usize| { + self.network_value( + &format!("a{index}"), + &arguments[index].ty, + Direction::ToRuntime, + ) + }; + let call = match operation { + TlsClient => "dever_runtime::tls::client(&a0)".into(), + TlsServer => "dever_runtime::tls::server(&a0, &a1)".into(), + HttpClientNew => format!( + "dever_runtime::http::client(&a0, a1, {}, {}).await", + input(2), + input(3) + ), + HttpCloseClient => "dever_runtime::http::close_client(&a0).await".into(), + HttpRequest | HttpOpen | HttpOpenStream => { + let method = match operation { + HttpRequest => "request", + HttpOpen => "open", + _ => "open_stream", + }; + return self.http_result( + &format!("dever_runtime::http::{method}(&a0, {}).await", input(1)), + result, + ); + } + HttpRespond => format!("dever_runtime::http::respond(&a0, {}).await", input(1)), + HttpStart => format!("dever_runtime::http::start(&a0, a1, {}).await", input(2)), + HttpWrite => "dever_runtime::http::write(&a0, a1).await".into(), + HttpFinish => "dever_runtime::http::finish(&a0).await".into(), + SseStart => format!("dever_runtime::sse::start(&a0, {}).await", input(1)), + SseSend => format!("dever_runtime::sse::send(&a0, {}).await", input(1)), + WsAccept => format!("dever_runtime::websocket::accept(&a0, {}).await", input(1)), + WsConnect => format!( + "dever_runtime::websocket::connect(&a0, a1, &a2, {}).await", + input(3) + ), + WsOpen => format!( + "dever_runtime::websocket::open(&a0, a1, {}).await", + input(2) + ), + WsSend => format!("dever_runtime::websocket::send(&a0, {}).await", input(1)), + WsClose => "dever_runtime::websocket::close(&a0, a1, &a2).await".into(), + WsReceive | WsMessages => { + let event = match &result.ty { + Type::AsyncStream(event) => event.as_ref(), + event => event, + }; + let Type::Named(id) = event else { + unreachable!() + }; + let Shape::Choice(variants) = &self.program.types[*id].shape else { + unreachable!() + }; + let read = variants + .iter() + .find(|variant| variant.name == "Read") + .unwrap(); + let message = + self.network_value("message", &read.fields[0].ty, Direction::ToLanguage); + if matches!(operation, WsMessages) { + let event = self.resource_result( + &format!("event.map(|message| {message})"), + event, + "Read", + false, + ); + return format!("dever_runtime::websocket::messages(a0).map(|event| {event})"); + } + return self.resource_result(&format!( + "dever_runtime::websocket::receive(&a0).await.map(|message| message.map(|message| {message}))" + ), event, "Read", true); + } + _ => unreachable!("live network intrinsic"), + }; + let success = match operation { + TlsClient | TlsServer | HttpClientNew => "Ready", + WsAccept | WsConnect | WsOpen => "Connected", + HttpFinish | WsClose | HttpCloseClient => "Closed", + _ => "Done", + }; + self.resource_result(&call, &result.ty, success, false) + } + + pub(super) fn http_serve( + &self, + handler: HandlerTarget, + arguments: &[Expression], + result: &Expression, + live: bool, + tls: bool, + ) -> String { + let target = Specialization { + function: specialize::resolve_handler(handler, &self.bindings), + handlers: Vec::new(), + }; + let wait = if specialize::suspends(self.program, &target) { + ".await" + } else { + "" + }; + let function = &self.program.functions[target.function]; + let Parameter::Value(request) = &function.parameters[0] else { + unreachable!("checked HTTP request") + }; + let request = self.network_value("request", request, Direction::ToLanguage); + let tls_index = if live { 3 } else { 2 }; + let context_index = tls_index + usize::from(tls); + let tls_argument = if tls { + format!(", a{tls_index}") + } else { + String::new() + }; + let suffix = if tls { "_tls" } else { "" }; + let (capture, context) = if arguments.len() > context_index { + ( + format!("let context = a{context_index}.clone();"), + ", context", + ) + } else { + (String::new(), "") + }; + if live { + let limits = self.network_value("a1", &arguments[1].ty, Direction::ToRuntime); + let live_limits = self.network_value("a2", &arguments[2].ty, Direction::ToRuntime); + let invoke = self.invoke( + &target, + format!("{request}, reply{context}"), + DatabaseContext::Detached, + ); + return self.failure(&format!( + "dever_runtime::http::serve_live{suffix}(move |request, reply| {{ {capture} async move {{ {invoke}{wait}.map_err(|error| error.to_string()) }} }}, a0, {limits}, {live_limits}{tls_argument}).await", + ), result.span); + } + let response = + self.network_value("response", &function.outputs[0].ty, Direction::ToRuntime); + let limits = self.network_value("a1", &arguments[1].ty, Direction::ToRuntime); + let invoke = self.invoke( + &target, + format!("{request}{context}"), + DatabaseContext::Detached, + ); + self.failure(&format!( + "dever_runtime::http::serve{suffix}(move |request| {{ {capture} async move {{ let response = {invoke}{wait}.map_err(|error| error.to_string())?; Ok({response}) }} }}, a0, {limits}{tls_argument}).await", + ), result.span) + } + + pub(super) fn http_send(&self, arguments: &[Expression], result: &Expression) -> String { + let request = self.network_value("a2", &arguments[2].ty, Direction::ToRuntime); + let limits = self.network_value("a3", &arguments[3].ty, Direction::ToRuntime); + self.http_result( + &format!("dever_runtime::http::send(&a0, a1, {request}, {limits}).await"), + result, + ) + } + + fn http_result(&self, call: &str, result: &Expression) -> String { + let Type::Named(id) = &result.ty else { + unreachable!("checked HTTP result") + }; + let Shape::Choice(variants) = &self.program.types[*id].shape else { + unreachable!() + }; + let read = variants + .iter() + .find(|variant| variant.name == "Read") + .expect("checked Read variant"); + let response = self.network_value("response", &read.fields[0].ty, Direction::ToLanguage); + self.resource_result( + &format!("{call}.map(|response| {response})"), + &result.ty, + "Read", + false, + ) + } + + // 校验器固定报文记录的字段契约；桥接按字段名生成，源码重排字段不会错位。 + fn network_value(&self, value: &str, ty: &Type, direction: Direction) -> String { + match ty { + Type::Named(id) => { + let definition = &self.program.types[*id]; + let Shape::Record(fields) = &definition.shape else { + let Shape::Choice(variants) = &definition.shape else { + unreachable!() + }; + let arms = variants + .iter() + .enumerate() + .map(|(index, variant)| { + let runtime = format!( + "dever_runtime::{}::{}", + definition + .name + .strip_prefix("dever.") + .unwrap() + .replace('.', "::"), + variant.name + ); + let language = format!("T{id}::V{index}"); + let payload = + self.network_value("payload", &variant.fields[0].ty, direction); + let (source, target) = match direction { + Direction::ToRuntime => (language, runtime), + Direction::ToLanguage => (runtime, language), + }; + format!("{source}(payload) => {target}({payload})") + }) + .collect::<Vec<_>>() + .join(", "); + return format!("match {value} {{ {arms} }}"); + }; + let name = match direction { + Direction::ToRuntime => format!( + "dever_runtime::{}", + definition + .name + .strip_prefix("dever.") + .expect("official record") + .replace('.', "::"), + ), + Direction::ToLanguage => format!("T{id}"), + }; + let fields = fields + .iter() + .enumerate() + .map(|(index, field)| { + let (source, target) = match direction { + Direction::ToRuntime => { + (format!("record.f{index}"), field.name.clone()) + } + Direction::ToLanguage => { + (format!("record.{}", field.name), format!("f{index}")) + } + }; + format!( + "{target}: {}", + self.network_value(&source, &field.ty, direction) + ) + }) + .collect::<Vec<_>>() + .join(", "); + format!("{{ let record = {value}; {name} {{ {fields} }} }}") + } + Type::List(element) => { + let element = self.network_value("header", element, direction); + match direction { + Direction::ToRuntime => { + format!("{value}.into_values().map(|header| {element}).collect()") + } + Direction::ToLanguage => { + format!("List::new({value}.into_iter().map(|header| {element}).collect())") + } + } + } + Type::Nullable(element) => { + let element = self.network_value("element", element, direction); + format!("{value}.map(|element| {element})") + } + Type::AsyncStream(event) => { + let conversion = match direction { + Direction::ToLanguage => self.resource_result("event", event, "Chunk", false), + Direction::ToRuntime => { + let Type::Named(id) = event.as_ref() else { + unreachable!() + }; + let Shape::Choice(variants) = &self.program.types[*id].shape else { + unreachable!() + }; + let arms = variants + .iter() + .enumerate() + .map(|(index, variant)| { + let constructor = + if variant.name == "Chunk" { "Ok" } else { "Err" }; + format!("T{id}::V{index}(payload) => {constructor}(payload)") + }) + .collect::<Vec<_>>() + .join(", "); + format!("match event {{ {arms} }}") + } + }; + format!("{value}.map(|event| {conversion})") + } + Type::Text + | Type::Bytes + | Type::Int + | Type::ClientTls + | Type::ServerTls + | Type::HttpClient => value.into(), + _ => unreachable!("checked HTTP field type"), + } + } +} diff --git a/crates/dever-core/src/native/intrinsics.rs b/crates/dever-core/src/native/intrinsics.rs new file mode 100644 index 0000000..ecc495f --- /dev/null +++ b/crates/dever-core/src/native/intrinsics.rs @@ -0,0 +1,409 @@ +use crate::hir::{Expression, HandlerTarget}; +use crate::intrinsic::Intrinsic; +use crate::types::{Shape, Type}; + +use super::{DatabaseContext, Emitter}; + +impl Emitter<'_> { + pub(super) fn intrinsic( + &self, + intrinsic: Intrinsic, + handler: Option<HandlerTarget>, + arguments: &[Expression], + result: &Expression, + ) -> String { + use Intrinsic::*; + if matches!( + intrinsic, + UploadFilename | UploadContentType | UploadSize | UploadClose | UploadStore + ) { + let crate::hir::ExpressionKind::Local(slot) = arguments[0].kind else { + unreachable!("checked upload local") + }; + if matches!(intrinsic, UploadStore) { + return self.resource_result( + &format!("dever_runtime::api::upload::store(_v{slot}).await"), + &result.ty, + "Stored", + false, + ); + } + let method = match intrinsic { + UploadFilename => "filename", + UploadContentType => "content_type", + UploadSize => "size", + _ => "close", + }; + return format!("_v{slot}.{method}()"); + } + let operation = match intrinsic { + UploadFilename | UploadContentType | UploadSize | UploadClose | UploadStore => unreachable!("upload handled above"), + TestSecret => "dever_runtime::secret::Secret::from_input(a0.into_bytes())".into(), + CryptoToken => self.resource_result("dever_runtime::crypto::token(a0)", &result.ty, "Ready", false), + CryptoPasswordHash => self.resource_result("dever_runtime::crypto::password_hash(&a0)", &result.ty, "Ready", false), + CryptoPasswordVerify => self.resource_result("dever_runtime::crypto::password_verify(&a0, &a1)", &result.ty, "Ready", false), + CryptoSha256 => "dever_runtime::crypto::sha256(&a0)".into(), + CryptoHmacSha256 => "dever_runtime::crypto::hmac_sha256(&a0, &a1)".into(), + CryptoConstantTimeEq => "dever_runtime::crypto::constant_time_eq(&a0, &a1)".into(), + TestAssert => format!( + "if !a0 {{ return Err(AppError::fault({}, \"assertion failed\")); }}", + self.location(result.span) + ), + TestAssertEq => format!( + "if a0 != a1 {{ let mut actual = String::new(); a0.render_to(&mut actual); let mut expected = String::new(); a1.render_to(&mut expected); return Err(AppError::fault({}, format!(\"assert_eq failed: actual = {{actual}}, expected = {{expected}}\"))); }}", + self.location(result.span) + ), + TlsSystem => "dever_runtime::tls::system()".into(), + TaskTicks => self.failure("task::ticks(a0)", result.span), + TaskTimeout => { + let target = crate::specialize::Specialization { + function: crate::specialize::resolve_handler(handler.expect("timeout handler"), &self.bindings), + handlers: vec![], + }; + let wait = if crate::specialize::suspends(self.program, &target) { ".await" } else { "" }; + let invoke = self.invoke(&target, String::new(), DatabaseContext::Inherit); + format!("match {} {{ Some(value) => value, None => {} }}", + self.call_failure("task::wait_timeout(a0, a1).await", result.span), + self.call_failure(&format!("{invoke}{wait}"), result.span)) + } + TaskRace => { + let tasks = (0..arguments.len()).map(|index| format!("a{index}")).collect::<Vec<_>>().join(", "); + self.call_failure(&format!("task::race(vec![{tasks}]).await"), result.span) + } + StreamOf => match &arguments[0].ty { + Type::Channel(_) => "a0.stream()".into(), + Type::List(_) => "dever_runtime::async_stream::AsyncStream::from_values(a0.into_values())".into(), + Type::Bytes => "dever_runtime::async_stream::AsyncStream::from_values(a0.into_values().map(i64::from))".into(), + _ => unreachable!(), + }, + TextTrim => "a0.trim().to_owned()".into(), + TextLower => "a0.to_lowercase()".into(), + TextUpper => "a0.to_uppercase()".into(), + TextCodepoint => "dever_runtime::text::codepoint(&a0)".into(), + TextFromCodepoint => "dever_runtime::text::from_codepoint(a0)".into(), + TextAt => "dever_runtime::text::at(&a0, a1)".into(), + TextSlice => "dever_runtime::text::slice(&a0, a1, a2)".into(), + TextIndexOf => "dever_runtime::text::index_of(&a0, &a1)".into(), + TextContains => "a0.contains(&a1)".into(), + TextStartsWith => "a0.starts_with(&a1)".into(), + TextEndsWith => "a0.ends_with(&a1)".into(), + TextSplit => "dever_runtime::text::split(&a0, &a1)".into(), + TextReplace => "a0.replace(&a1, &a2)".into(), + IntParse => "number::parse_int(&a0)".into(), + IntToText | DecimalToText | FloatToText => "a0.to_string()".into(), + DecimalParse => "number::parse_decimal(&a0)".into(), + DecimalFromInt => "DecimalValue::from_int(a0)".into(), + FloatParse => "number::parse_float(&a0)".into(), + ProcessArguments => self.resource_result( + "dever_runtime::process::arguments()", + &result.ty, + "Read", + false, + ), + TimeUnixMillis => self.resource_result( + "dever_runtime::time::unix_millis()", + &result.ty, + "Read", + false, + ), + TimeNow => self.resource_result("dever_runtime::time::now()", &result.ty, "Read", false), + TimeParseDateTime => self.resource_result("dever_runtime::time::parse_datetime(&a0)", &result.ty, "Read", false), + TimeFormatDateTime => self.resource_result("dever_runtime::time::format_datetime(a0)", &result.ty, "Read", false), + TimeParseDate => self.resource_result("dever_runtime::time::parse_date(&a0)", &result.ty, "Read", false), + TimeFormatDate => self.resource_result("dever_runtime::time::format_date(a0)", &result.ty, "Read", false), + TimeParseTime => self.resource_result("dever_runtime::time::parse_time(&a0)", &result.ty, "Read", false), + TimeFormatTime => self.resource_result("dever_runtime::time::format_time(a0)", &result.ty, "Read", false), + TimeDuration => "dever_runtime::time::duration(a0)".into(), + TimeAdd => self.resource_result("dever_runtime::time::add(a0, a1)", &result.ty, "Read", false), + TimeSubtract => self.resource_result("dever_runtime::time::subtract(a0, a1)", &result.ty, "Read", false), + TimeDifference => self.resource_result("dever_runtime::time::difference(a0, a1)", &result.ty, "Read", false), + TimeMonotonicNanos => self.resource_result( + "dever_runtime::time::monotonic_nanos()", + &result.ty, + "Read", + false, + ), + TimeSleep => { + self.resource_result("dever_runtime::time::sleep(a0)", &result.ty, "Done", false) + } + TaskSleep => self.resource_result( + "task::sleep(a0).await", + &result.ty, + "Done", + false, + ), + DecimalRound => self.failure("a0.round(a1)", result.span), + FloatFromInt => "a0 as f64".into(), + FloatSqrt => "a0.sqrt()".into(), + FloatSin => "a0.sin()".into(), + FloatCos => "a0.cos()".into(), + FloatLog => "a0.ln()".into(), + FloatPow => "a0.powf(a1)".into(), + FloatIsNan => "a0.is_nan()".into(), + FloatIsFinite => "a0.is_finite()".into(), + FloatIsInfinite => "a0.is_infinite()".into(), + StdoutWrite => self.failure("resource::stdout_write(&a0)", result.span), + LogDebug | LogInfo | LogWarn | LogError => { + let level = match intrinsic { + LogDebug => "debug", + LogInfo => "info", + LogWarn => "warn", + LogError => "error", + _ => unreachable!(), + }; + format!( + "dever_runtime::log::{level}(a0, a1.pairs().map(|(name, value)| dever_runtime::log::Field {{ name: name.clone(), value: value.clone() }}).collect())" + ) + } + IdFromText => "dever_runtime::Id(a0)".into(), + IdToText => "a0.0".into(), + UuidParse => "dever_runtime::orm::Uuid::parse(&a0).ok()".into(), + UuidToText => "a0.to_string()".into(), + BytesFromText => "Bytes::from_string(a0)".into(), + BytesLength => "a0.length()".into(), + BytesAt => "a0.at(a1)".into(), + BytesConcat => "a0.concat(&a1)".into(), + BytesToText => self.resource_result("a0.into_text()", &result.ty, "Decoded", false), + BytesFromInts => { + self.resource_result("Bytes::from_ints(a0.values())", &result.ty, "Ready", false) + } + BytesSlice => self.resource_result("a0.slice(a1, a2)", &result.ty, "Ready", false), + FileOpen => { + self.resource_result("resource::file_open(&a0)", &result.ty, "Opened", false) + } + FileCreate => { + self.resource_result("resource::file_create(&a0)", &result.ty, "Opened", false) + } + FileRead => { + self.resource_result("resource::read(&a0, a1)", &result.ty, "Read", true) + } + FileWrite => { + self.resource_result("resource::write(&a0, &a1)", &result.ty, "Done", false) + } + FileClose | TcpClose | TcpCloseListener => { + self.resource_result("a0.close()", &result.ty, "Closed", false) + } + FileChunks => self.read_stream_result("resource::read_chunks(a0, a1)", &result.ty), + TcpChunks => self.read_stream_result("dever_runtime::net::chunks(a0, a1)", &result.ty), + TcpRead => self.resource_result( + "dever_runtime::net::read(&a0, a1).await", &result.ty, "Read", true, + ), + TcpWrite => self.resource_result( + "dever_runtime::net::write(&a0, &a1).await", &result.ty, "Done", false, + ), + TcpConnect => self.resource_result( + "dever_runtime::net::connect(&a0, a1).await", + &result.ty, + "Connected", + false, + ), + TcpConnectTimeout => self.resource_result( + "dever_runtime::net::connect_timeout(&a0, a1, a2).await", + &result.ty, + "Connected", + false, + ), + TcpListen => self.resource_result( + "dever_runtime::net::listen(&a0, a1).await", + &result.ty, + "Listening", + false, + ), + TcpAccept => { + self.resource_result("dever_runtime::net::accept(&a0).await", &result.ty, "Connected", false) + } + TcpPort => self.resource_result("dever_runtime::net::port(&a0)", &result.ty, "Bound", false), + TcpTimeout => { + self.resource_result("dever_runtime::net::timeout(&a0, a1)", &result.ty, "Done", false) + } + TcpConnections => self.connection_stream(&result.ty), + ApiServe => self.failure("dever_runtime::api::serve(api_route).await", result.span), + ApiRequestId => self.failure("dever_runtime::api::request_id()", result.span), + ApiMethod => self.failure("dever_runtime::api::method()", result.span), + ApiPath => self.failure("dever_runtime::api::path()", result.span), + ApiClientAddress => self.failure("dever_runtime::api::client_address()", result.span), + ApiHeader => self.failure("dever_runtime::api::header(&a0)", result.span), + ApiCookie => self.failure("dever_runtime::api::cookie(&a0)", result.span), + ApiSecretCookie => self.failure("dever_runtime::api::secret_cookie(&a0)", result.span), + ApiResponseHeader => self.failure("dever_runtime::api::response_header(&a0, &a1)", result.span), + ApiResponseCookie => self.failure(&format!( + "dever_runtime::api::response_cookie(&a0, &a1, {})", + self.api_cookie_options(), + ), result.span), + ApiResponseSecretCookie => self.failure(&format!( + "dever_runtime::api::response_secret_cookie(&a0, a1, {})", + self.api_cookie_options(), + ), result.span), + AuthIssue => self.failure("dever_runtime::auth::issue(&a0, &a1, a2.as_deref())", result.span), + AuthIssueCookie => self.failure("dever_runtime::auth::issue_cookie(&a0, &a1, a2.as_deref())", result.span), + AuthClearCookie => self.failure("dever_runtime::auth::clear_cookie()", result.span), + AuthOwnsUser => self.failure("dever_runtime::auth::owns_user(a0.0)", result.span), + AuthId => self.failure("dever_runtime::auth::id()", result.span), + AuthSession => self.failure("dever_runtime::auth::session()", result.span), + AuthUserId | AuthTenantId => { + let method = if intrinsic == AuthUserId { "user_id" } else { "tenant_id" }; + let call = format!("dever_runtime::auth::{method}()"); + let call = match &result.ty { + Type::Nullable(inner) => match inner.as_ref() { + Type::Named(id) => format!("{call}.map(|value| value.map(T{id}))"), + _ => call, + }, + _ => unreachable!("authentication ID getter returns a nullable value"), + }; + self.failure(&call, result.span) + } + AuthPermissions => { + let Type::List(permission) = &result.ty else { unreachable!("checked permission list") }; + let Type::Named(permission) = permission.as_ref() else { unreachable!("checked Permission type") }; + self.failure(&format!( + "dever_runtime::auth::permissions().await.map(|values| List::new(values.into_iter().map(|value| T{permission} {{ f0: value.key, f1: value.component, f2: value.domain, f3: value.site, f4: value.action, f5: value.method }}).collect()))" + ), result.span) + } + AuthSaveRole => self.failure( + "dever_runtime::auth::save_role(&a0, &a1, a2, a3.into_values().collect()).await", + result.span, + ), + AuthGrantRole => self.failure( + "dever_runtime::auth::grant_role(a0, &a1).await", + result.span, + ), + AuthRevokeRole => self.failure( + "dever_runtime::auth::revoke_role(a0, &a1).await", + result.span, + ), + AuthDisableRole => self.failure( + "dever_runtime::auth::disable_role(&a0).await", + result.span, + ), + SiteKey => self.failure("dever_runtime::auth::site_key()", result.span), + JobServe => self.failure("job_serve().await", result.span), + TestJobDrain => self.failure("job_drain(a0).await", result.span), + TestClockAdvance => self.failure("JOB_CLOCK.get().expect(\"test clock initialized\").advance(a0)", result.span), + HttpServe | HttpServeTls => self.http_serve(handler.expect("checked HTTP handler"), arguments, result, false, matches!(intrinsic, HttpServeTls)), + HttpServeLive | HttpServeLiveTls => self.http_serve(handler.expect("checked HTTP handler"), arguments, result, true, matches!(intrinsic, HttpServeLiveTls)), + HttpSend => self.http_send(arguments, result), + operation @ (HttpRespond | HttpStart | HttpWrite | HttpFinish | SseStart | SseSend + | WsAccept | WsConnect | WsSend | WsReceive | WsMessages | WsClose | WsOpen + | TlsClient | TlsServer | HttpClientNew | HttpCloseClient | HttpRequest | HttpOpen | HttpOpenStream) => self.live_network(operation, arguments, result), + }; + format!("{{ {}\n{operation} }}", self.bind_arguments(arguments)) + } + + fn api_cookie_options(&self) -> String { + let (id, definition) = self + .program + .types + .iter() + .enumerate() + .find(|(_, definition)| definition.name == "dever.api.SameSite") + .expect("official SameSite type"); + let Shape::Choice(variants) = &definition.shape else { + unreachable!("official SameSite choice") + }; + let arms = variants + .iter() + .enumerate() + .map(|(index, variant)| { + let name = match variant.name.as_str() { + "Strict" => "Strict", + "Lax" => "Lax", + "None" => "None", + _ => unreachable!("official SameSite variant"), + }; + format!("T{id}::V{index} => dever_runtime::api::SameSite::{name}") + }) + .collect::<Vec<_>>() + .join(", "); + format!( + "dever_runtime::api::CookieOptions {{ path: a2.f0, domain: a2.f1, same_site: match a2.f2 {{ {arms} }}, secure: a2.f3, http_only: a2.f4, max_age: a2.f5 }}" + ) + } + + fn read_stream_result(&self, call: &str, output: &Type) -> String { + let Type::Named(result_id) = output else { + unreachable!("checked read stream result") + }; + let Shape::Choice(result_variants) = &self.program.types[*result_id].shape else { + unreachable!() + }; + let streaming = variant(result_variants, "Streaming"); + let failed = variant(result_variants, "Failed"); + let (Type::Stream(event) | Type::AsyncStream(event)) = + &result_variants[streaming].fields[0].ty + else { + unreachable!() + }; + let Type::Named(event_id) = event.as_ref() else { + unreachable!() + }; + let Shape::Choice(event_variants) = &self.program.types[*event_id].shape else { + unreachable!() + }; + let chunk = variant(event_variants, "Chunk"); + let event_failed = variant(event_variants, "Failed"); + format!( + "match {call} {{ Ok(stream) => T{result_id}::V{streaming}(stream.map(|event| match event {{ Ok(bytes) => T{event_id}::V{chunk}(bytes), Err(message) => T{event_id}::V{event_failed}(message) }})), Err(message) => T{result_id}::V{failed}(message) }}" + ) + } + + fn connection_stream(&self, output: &Type) -> String { + let Type::AsyncStream(connection) = output else { + unreachable!("checked connection Stream") + }; + let Type::Named(id) = connection.as_ref() else { + unreachable!() + }; + let Shape::Choice(variants) = &self.program.types[*id].shape else { + unreachable!() + }; + let connected = variant(variants, "Connected"); + let failed = variant(variants, "Failed"); + format!( + "dever_runtime::net::connections(a0).map(|event| match event {{ Ok(socket) => T{id}::V{connected}(socket), Err(message) => T{id}::V{failed}(message) }})" + ) + } + + pub(super) fn resource_result( + &self, + call: &str, + output: &Type, + success_name: &str, + eof: bool, + ) -> String { + let Type::Named(id) = output else { + unreachable!("official result choice") + }; + let Shape::Choice(variants) = &self.program.types[*id].shape else { + unreachable!() + }; + let variant = |name| { + variants + .iter() + .position(|variant| variant.name == name) + .expect("checked system result contract") + }; + let success = variant(success_name); + let failed = variant("Failed"); + if eof { + let end = variant("End"); + format!( + "match {call} {{ Ok(Some(bytes)) => T{id}::V{success}(bytes), Ok(None) => T{id}::V{end}, Err(message) => T{id}::V{failed}(message.into()) }}" + ) + } else if variants[success].fields.is_empty() { + format!( + "match {call} {{ Ok(()) => T{id}::V{success}, Err(message) => T{id}::V{failed}(message.into()) }}" + ) + } else { + format!( + "match {call} {{ Ok(value) => T{id}::V{success}(value), Err(message) => T{id}::V{failed}(message.into()) }}" + ) + } + } +} + +fn variant(variants: &[crate::types::Variant], name: &str) -> usize { + variants + .iter() + .position(|variant| variant.name == name) + .expect("checked system result contract") +} diff --git a/crates/dever-core/src/native/jobs.rs b/crates/dever-core/src/native/jobs.rs new file mode 100644 index 0000000..2894869 --- /dev/null +++ b/crates/dever-core/src/native/jobs.rs @@ -0,0 +1,457 @@ +use super::{Emitter, rust_string}; +use crate::hir::Expression; +use crate::model::{DatabaseOwner, ModelScope}; +use crate::source::Span; +use std::fmt::Write; + +pub(super) fn targets( + program: &crate::hir::Program, + instances: &std::collections::BTreeSet<crate::specialize::Specialization>, +) -> std::collections::BTreeSet<usize> { + let mut targets = std::collections::BTreeSet::new(); + for instance in instances { + if matches!( + program.functions[instance.function].kind, + crate::syntax::FunctionKind::Job { .. } + ) { + targets.insert(instance.function); + } + crate::check::visit(&program.functions[instance.function], |expression| { + if let crate::hir::ExpressionKind::JobEnqueue { target, .. } = expression.kind { + targets.insert(target); + } + }); + } + targets +} + +pub(crate) fn scope(program: &crate::hir::Program, target: usize) -> ModelScope { + let instance = crate::specialize::Specialization { + function: target, + handlers: Vec::new(), + }; + let component_is_tenant = program.packages[program.functions[target].owner] + .domain + .as_ref() + .is_some_and(|(component, _)| program.tenant_components.contains(component)); + if component_is_tenant || crate::specialize::database_effects(program, &instance).iter().any(|owner| { + matches!(owner, DatabaseOwner::Model(model) if program.models[*model].scope == ModelScope::Tenant) + }) { + ModelScope::Tenant + } else { + ModelScope::Global + } +} + +fn scope_name(scope: ModelScope) -> &'static str { + match scope { + ModelScope::Global => "Global", + ModelScope::Tenant => "Tenant", + } +} + +pub(super) fn definitions( + emitter: &Emitter<'_>, + output: &mut String, + instances: &std::collections::BTreeSet<crate::specialize::Specialization>, +) { + let targets = targets(emitter.program, instances); + if targets.is_empty() && !uses_test_controls(emitter.program, instances) { + return; + } + output.push_str("static JOB_CLOCK: std::sync::OnceLock<dever_runtime::time::Clock> = std::sync::OnceLock::new();\n"); + if targets.is_empty() { + output.push_str("async fn job_drain(limit: i64) -> Result<i64, String> { if !(1..=10000).contains(&limit) { return Err(\"test Job drain limit must be between 1 and 10000\".into()); } Ok(0) }\n"); + return; + } + for target in &targets { + let target = *target; + let job = emitter + .program + .jobs + .iter() + .find(|job| job.function == target) + .expect("checked Job target"); + if let Some(schema) = &job.payload { + output.push_str(&super::wire::emit_named(schema, &format!("job_{target}"))); + } + writeln!(output, "static JOB_STORAGE_{target}: std::sync::OnceLock<dever_runtime::database::StorageBinding> = std::sync::OnceLock::new();\nasync fn job_database_{target}() -> Result<dever_runtime::database::Database, dever_runtime::orm::Error> {{ JOB_STORAGE_{target}.get().expect(\"Job storage initialized before entry\").database().await }}\nstatic JOB_SPEC_{target}: dever_runtime::job::Spec = dever_runtime::job::Spec {{ target: {}, schema: {}, attempts: {}, timeout_ms: {} }};", + rust_string(&emitter.program.functions[target].name), rust_string(job.payload.as_ref().map(|schema| schema.fingerprint()).unwrap_or("wire-v1:Job:Unit")), job.attempts, job.timeout_ms).expect("string formatting"); + } + output.push_str("fn job_bindings() -> Vec<dever_runtime::job::worker::Binding> { vec![\n"); + for target in &targets { + let job = emitter + .program + .jobs + .iter() + .find(|job| job.function == *target) + .expect("checked Job"); + let schedule = job + .schedule + .as_ref() + .map(|cron| { + format!( + "Some(dever_runtime::cron::Cron::parse({}).expect(\"checked cron\"))", + rust_string(cron) + ) + }) + .unwrap_or_else(|| "None".into()); + let scope = scope_name(scope(emitter.program, *target)); + writeln!(output, "dever_runtime::job::worker::Binding::scoped(JOB_STORAGE_{target}.get().expect(\"Job storage initialized before entry\"), dever_runtime::database::ModelScope::{scope}, &JOB_SPEC_{target}, {schedule}),").expect("string formatting"); + } + output.push_str("] }\n"); + emit_identity_failure(emitter.program, output); + output.push_str("async fn job_dispatch_verified(claim: dever_runtime::job::store::Claim) -> dever_runtime::job::store::Outcome {\nuse dever_runtime::job::store::Outcome;\nmatch claim.target.as_str() {\n"); + for target in &targets { + let job = emitter + .program + .jobs + .iter() + .find(|job| job.function == *target) + .expect("checked Job"); + writeln!(output, "{} => {{ if claim.schema != JOB_SPEC_{target}.schema {{ return Outcome::SchemaMismatch; }}", rust_string(&emitter.program.functions[*target].name)).expect("string formatting"); + let mut arguments = Vec::new(); + if job.payload.is_some() { + writeln!(output, "let payload = match job_{target}_decode(&claim.payload) {{ Ok(payload) => payload, Err(_) => return Outcome::InvalidPayload }};").expect("string formatting"); + arguments.push("payload"); + } else { + output.push_str("if !dever_runtime::wire::parse(&claim.payload).is_ok_and(|value| value.is_null()) { return Outcome::InvalidPayload; }\n"); + } + let instance = crate::specialize::Specialization { + function: *target, + handlers: Vec::new(), + }; + let transaction_database = super::transaction_database(emitter.program, *target); + if let Some(database) = &transaction_database { + writeln!(output, "let transaction_database = match {database} {{ Ok(database) => database, Err(_) => return Outcome::Retry }};\nlet transaction = match transaction_database.begin().await {{ Ok(value) => value, Err(_) => return Outcome::Retry }};") + .expect("string formatting"); + } + if !crate::specialize::database_effects(emitter.program, &instance).is_empty() { + arguments.push(if transaction_database.is_some() { + "Some(&transaction)" + } else { + "None" + }); + } + let call = format!("{}({})", emitter.names[&instance], arguments.join(", ")); + let call = if crate::specialize::suspends(emitter.program, &instance) { + format!("{call}.await") + } else { + format!("task::blocking_typed(move || {call}).await") + }; + if transaction_database.is_some() { + writeln!(output, "match {call} {{ Ok(()) => match transaction.commit().await {{ Ok(()) => Outcome::Success, Err(_) => Outcome::Retry }}, Err(_) => {{ let _ = transaction.rollback().await; Outcome::Retry }} }} }},").expect("string formatting"); + } else { + writeln!( + output, + "match {call} {{ Ok(()) => Outcome::Success, Err(_) => Outcome::Retry }} }}," + ) + .expect("string formatting"); + } + } + output.push_str("_ => Outcome::UnknownTarget,\n}\n}\n"); + emit_component_check(emitter.program, output); + emit_dispatch(emitter.program, output); + output.push_str("async fn job_drain(limit: i64) -> Result<i64, String> { dever_runtime::job::worker::drain(job_bindings(), JOB_CLOCK.get().expect(\"clock initialized\").clone(), limit, job_dispatch).await }\nasync fn job_serve() -> Result<(), String> { if !dever_runtime::lifecycle::worker_enabled() { return Ok(()); } dever_runtime::job::worker::serve(job_bindings(), JOB_CLOCK.get().expect(\"clock initialized\").clone(), dever_runtime::config::settings().jobs()?, job_dispatch).await }\n"); +} + +fn emit_identity_failure(program: &crate::hir::Program, output: &mut String) { + let failures = program + .failures + .iter() + .flat_map(|failures| failures.iter()) + .copied() + .collect::<std::collections::BTreeSet<_>>(); + let unauthorized = program + .types + .iter() + .enumerate() + .find(|(_, definition)| definition.name == "dever.api.Error") + .and_then(|(id, definition)| match &definition.shape { + crate::types::Shape::Choice(variants) => Some((id, variants)), + _ => None, + }); + output.push_str("fn job_identity_failure(error: AppError) -> dever_runtime::job::store::Outcome {\nuse dever_runtime::job::store::Outcome;\nmatch &error.kind {\n"); + if let Some((id, variants)) = unauthorized { + for name in ["Unauthorized", "Forbidden"] { + if let Some(index) = variants.iter().position(|variant| variant.name == name) + && failures.contains(&crate::hir::Failure { + ty: id, + variant: index, + }) + { + writeln!( + output, + "AppErrorKind::E{id}V{index} => Outcome::IdentityRejected," + ) + .expect("string formatting"); + } + } + } + output.push_str("_ => { dever_runtime::log::error(\"Job identity verification failed\", Vec::new()); Outcome::Retry }\n}\n}\n"); +} + +fn emit_dispatch(program: &crate::hir::Program, output: &mut String) { + output.push_str("async fn job_dispatch(claim: dever_runtime::job::store::Claim) -> dever_runtime::job::store::Outcome {\nuse dever_runtime::job::store::Outcome;\nmatch claim.execution.clone() {\ndever_runtime::job::ExecutionIdentity::System { .. } => { if let Err(outcome) = job_require_components(&claim).await { return outcome; } job_dispatch_verified(claim).await },\ndever_runtime::job::ExecutionIdentity::User { tenant_id, provider, site, subject, session, tenant_claim, permission_key } => {\n"); + if program.auth.is_empty() || program.permissions.is_empty() { + output.push_str("let _ = (tenant_id, provider, site, subject, session, tenant_claim, permission_key);\nOutcome::InvalidIdentity\n"); + } else { + output.push_str("if dever_runtime::auth::validate_job_source(&provider, &site).is_err() { return Outcome::InvalidIdentity; }\nmatch provider.as_str() {\n"); + for hook in &program.auth { + let instance = crate::specialize::Specialization { + function: hook.function, + handlers: Vec::new(), + }; + writeln!(output, "{} => {{", rust_string(&hook.provider)).expect("string formatting"); + writeln!(output, + "let auth_claims = T{} {{ f0: subject.clone(), f1: session.clone(), f2: tenant_claim.clone(), f3: site.clone() }};", + hook.claims, + ).expect("string formatting"); + let mut arguments = vec!["auth_claims".to_owned()]; + if !crate::specialize::database_effects(program, &instance).is_empty() { + arguments.push("None".into()); + } + let call = format!( + "f{}({}){}", + hook.function, + arguments.join(", "), + if crate::specialize::suspends(program, &instance) { + ".await" + } else { + "" + } + ); + writeln!(output, + "let identity: T{} = match {call} {{ Ok(identity) => identity, Err(error) => return job_identity_failure(error), }};\nlet verified_identity = dever_runtime::auth::Identity {{ id: identity.f0, user_id: identity.f1.map(|value| value.0), tenant_id: identity.f2.map(|value| value.0) }};\nlet verified_claims = dever_runtime::auth::Claims {{ subject, session, tenant: tenant_claim, site: site.clone() }};\nif dever_runtime::auth::validate_job_identity(&verified_claims, &verified_identity).is_err() || verified_identity.tenant_id != tenant_id {{ return Outcome::IdentityRejected; }}\nif !AUTHORIZATION_PERMISSIONS.iter().any(|permission| permission.key == permission_key && permission.site == site) {{ return Outcome::IdentityRejected; }}\nif let Err(outcome) = job_require_components(&claim).await {{ return outcome; }}\nlet Some(user_id) = verified_identity.user_id else {{ return Outcome::IdentityRejected; }};\nlet authorization_database = if tenant_id.is_some() {{ let Some(tenant) = dever_runtime::config::settings().tenant() else {{ return Outcome::InvalidIdentity; }}; match dever_runtime::tenant::database(tenant.database()).await {{ Ok(database) => database, Err(_) => return Outcome::Retry }} }} else {{ match authorization_control_database() {{ Ok(database) => database, Err(_) => return Outcome::Retry }} }};\nmatch dever_runtime::auth::store::authorize(authorization_database, user_id, &site, &permission_key).await {{ Ok(true) => job_dispatch_verified(claim).await, Ok(false) => Outcome::IdentityRejected, Err(_) => Outcome::Retry }}\n}},", + hook.identity, + ).expect("string formatting"); + } + output.push_str("_ => Outcome::InvalidIdentity,\n}\n"); + } + output.push_str("}\n}\n}\n"); +} + +fn emit_component_check(program: &crate::hir::Program, output: &mut String) { + output.push_str("async fn job_require_components(claim: &dever_runtime::job::store::Claim) -> Result<(), dever_runtime::job::store::Outcome> {\nmatch claim.target.as_str() {\n"); + for job in &program.jobs { + if job.components.is_empty() { + continue; + } + writeln!(output, + "{} => dever_runtime::job::require_components(&claim.execution, &[{}], TENANT_COMPONENTS).await?,", + rust_string(&program.functions[job.function].name), + job.components.iter().map(|component| rust_string(component)).collect::<Vec<_>>().join(", "), + ).expect("string formatting"); + } + output.push_str("_ => {},\n}\nOk(())\n}\n"); +} + +pub(super) fn initialize( + emitter: &Emitter<'_>, + output: &mut String, + instances: &std::collections::BTreeSet<crate::specialize::Specialization>, + span: Span, +) { + if targets(emitter.program, instances).is_empty() { + return; + } + if !emitter.sources.get(span.source).is_test() { + initialize_clock(emitter, output, instances, span); + } + let mut connections = std::collections::BTreeSet::new(); + for target in targets(emitter.program, instances) { + let (explicit, root) = emitter + .program + .database_selector(DatabaseOwner::Job(target)); + let explicit = explicit + .map(|name| format!("Some({})", rust_string(name))) + .unwrap_or_else(|| "None".into()); + let job_scope = scope(emitter.program, target); + let scope = scope_name(job_scope); + writeln!(output, "JOB_STORAGE_{target}.set(dever_runtime::database::StorageBinding::new({explicit}, {}, dever_runtime::database::ModelScope::{scope})).map_err(|_| AppError::fault({}, \"Job storage initialized twice\"))?;\nlet job_database_{target} = if dever_runtime::config::settings().tenant().is_some() && matches!(dever_runtime::database::ModelScope::{scope}, dever_runtime::database::ModelScope::Tenant) {{ None }} else {{ Some({}) }};", rust_string(root), emitter.location(span), emitter.database_failure(&format!("job_database_{target}().await"), span)).expect("string formatting"); + if connections.insert((explicit, root, job_scope == ModelScope::Tenant)) { + let initialize = emitter.database_failure(&format!("dever_runtime::job::store::initialize(job_database_{target}.as_ref().expect(\"selected Job database\").clone()).await"), span); + writeln!( + output, + "if job_database_{target}.is_some() {{ {initialize}; }}" + ) + .expect("string formatting"); + } + let job = emitter + .program + .jobs + .iter() + .find(|job| job.function == target) + .expect("checked Job"); + if let Some(cron) = &job.schedule { + let validate = emitter.database_failure(&format!("dever_runtime::job::store::validate_schedule(job_database_{target}.as_ref().expect(\"selected Job database\").clone(), &JOB_SPEC_{target}, &dever_runtime::cron::Cron::parse({}).expect(\"checked cron\")).await", rust_string(cron)), span); + writeln!( + output, + "if job_database_{target}.is_some() {{ {validate}; }}" + ) + .expect("string formatting"); + } + } +} + +pub(super) fn tenant_migration( + emitter: &Emitter<'_>, + output: &mut String, + instances: &std::collections::BTreeSet<crate::specialize::Specialization>, + database_enabled: bool, +) { + if !database_enabled { + return; + } + let targets = targets(emitter.program, instances); + if !targets + .iter() + .any(|target| scope(emitter.program, *target) == ModelScope::Tenant) + { + // ORM tenant migration calls the Job migration hook unconditionally. + // Keep that generated contract total for applications without tenant + // Jobs instead of leaving an unresolved function reference. + output.push_str("async fn job_migrate_tenant(_tenant_id: i64, _schema_fingerprint: &str, _databases: &mut std::collections::BTreeMap<String, dever_runtime::database::Database>) -> Result<(), AppError> { Ok(()) }\n"); + return; + } + output.push_str("async fn job_migrate_tenant(tenant_id: i64, schema_fingerprint: &str, databases: &mut std::collections::BTreeMap<String, dever_runtime::database::Database>) -> Result<(), AppError> {\nlet settings = dever_runtime::config::settings();\n"); + let span = targets + .iter() + .next() + .map(|target| emitter.program.functions[*target].span); + let mut connections = std::collections::BTreeMap::new(); + for target in targets + .into_iter() + .filter(|target| scope(emitter.program, *target) == ModelScope::Tenant) + { + let (explicit, root) = emitter + .program + .database_selector(DatabaseOwner::Job(target)); + let explicit_code = explicit + .map(|name| format!("Some({})", rust_string(name))) + .unwrap_or_else(|| "None".into()); + let identity = (explicit.map(str::to_owned), root.to_owned()); + let connection_target = *connections.entry(identity).or_insert(target); + if connection_target == target { + writeln!(output, "let job_connection_{target} = settings.resolve_database({explicit_code}, {}).map_err(|error| AppError::fault(\"tenant Job migration\", error))?.0.to_owned();\nif !databases.contains_key(&job_connection_{target}) {{ let database = dever_runtime::tenant::begin_migration(&job_connection_{target}, tenant_id, schema_fingerprint).await.map_err(|error| AppError::database(\"tenant Job migration\", error))?; databases.insert(job_connection_{target}.clone(), database); }}\n{};", rust_string(root), emitter.database_failure(&format!("dever_runtime::job::store::initialize(databases.get(&job_connection_{target}).expect(\"tenant Job database initialized\").clone()).await"), span.expect("tenant Job has source span"))).expect("string formatting"); + } + let job = emitter + .program + .jobs + .iter() + .find(|job| job.function == target) + .expect("checked Job"); + if let Some(cron) = &job.schedule { + let validate = emitter.database_failure(&format!("dever_runtime::job::store::validate_schedule(databases.get(&job_connection_{connection_target}).expect(\"tenant Job database initialized\").clone(), &JOB_SPEC_{target}, &dever_runtime::cron::Cron::parse({}).expect(\"checked cron\")).await", rust_string(cron)), span.expect("tenant Job has source span")); + writeln!(output, "{validate};").expect("string formatting"); + } + } + output.push_str("Ok(())\n}\n"); +} + +fn uses_test_controls( + program: &crate::hir::Program, + instances: &std::collections::BTreeSet<crate::specialize::Specialization>, +) -> bool { + instances.iter().any(|instance| { + let mut found = false; + crate::check::visit(&program.functions[instance.function], |expression| { + if let crate::hir::ExpressionKind::Intrinsic { + operation: + crate::intrinsic::Intrinsic::TestJobDrain + | crate::intrinsic::Intrinsic::TestClockAdvance, + .. + } = expression.kind + { + found = true; + } + }); + found + }) +} + +pub(super) fn initialize_clock( + emitter: &Emitter<'_>, + output: &mut String, + instances: &std::collections::BTreeSet<crate::specialize::Specialization>, + span: Span, +) { + if targets(emitter.program, instances).is_empty() + && !uses_test_controls(emitter.program, instances) + { + return; + } + let clock = if emitter.sources.get(span.source).is_test() { + "for_test" + } else { + "default" + }; + writeln!(output, "JOB_CLOCK.set(dever_runtime::time::Clock::{clock}()).map_err(|_| AppError::fault({}, \"Job clock initialized twice\"))?;", emitter.location(span)).expect("string formatting"); +} + +pub(super) fn preflight( + emitter: &Emitter<'_>, + output: &mut String, + instances: &std::collections::BTreeSet<crate::specialize::Specialization>, + span: Span, +) { + for target in targets(emitter.program, instances) { + writeln!( + output, + "{};", + emitter.database_failure(&format!("JOB_SPEC_{target}.validate()"), span) + ) + .expect("string formatting"); + let (explicit, root) = emitter + .program + .database_selector(DatabaseOwner::Job(target)); + writeln!(output, "if dever_runtime::lifecycle::worker_enabled() {{ dever_runtime::config::settings().validate_job_connection({}, {}, JOB_SPEC_{target}.timeout_ms)?; }}", explicit.map(|name| format!("Some({})", rust_string(name))).unwrap_or_else(|| "None".into()), rust_string(root)).expect("string formatting"); + } +} + +impl Emitter<'_> { + pub(super) fn database_owner_at(&self, owner: DatabaseOwner) -> String { + match owner { + DatabaseOwner::Model(id) => self.database_at(&self.program.models[id]), + DatabaseOwner::Job(function) => format!("_job_database_{function}.clone()"), + } + } + + pub(super) fn enqueue_job( + &self, + target: usize, + arguments: &[Expression], + scheduled: bool, + span: Span, + ) -> String { + let job = self + .program + .jobs + .iter() + .find(|job| job.function == target) + .expect("checked Job"); + let mut output = String::from("{"); + for (index, argument) in arguments.iter().enumerate() { + write!(output, "let j{index} = {};", self.expression(argument)) + .expect("string formatting"); + } + let payload = if job.payload.is_some() { + self.failure(&format!("job_{target}_encode(&j0)"), span) + } else { + "dever_runtime::wire::Encoded::null()".into() + }; + let key = usize::from(job.payload.is_some()); + let run_at = if scheduled { + format!("j{}", key + 1) + } else { + self.database_failure("JOB_CLOCK.get().expect(\"clock initialized\").now()", span) + }; + let database = self.database_failure(&format!("job_database_{target}().await"), span); + let call = format!( + "dever_runtime::job::enqueue({database}, _database, &JOB_SPEC_{target}, {payload}, &j{key}, {run_at}).await" + ); + output.push_str(&self.database_failure(&call, span)); + output.push('}'); + output + } +} diff --git a/crates/dever-core/src/native/liveness.rs b/crates/dever-core/src/native/liveness.rs new file mode 100644 index 0000000..6ab1591 --- /dev/null +++ b/crates/dever-core/src/native/liveness.rs @@ -0,0 +1,140 @@ +use std::collections::HashSet; + +use crate::hir::{CallArgument, Clause, Expression, ExpressionKind, Statement}; + +/// HIR 在发射期间不可变；节点地址只标识同一子句中的具体读取，不进入生成程序。 +pub(crate) fn last_uses(clause: &Clause) -> HashSet<*const Expression> { + let mut live: HashSet<usize> = clause.outputs.iter().copied().collect(); + let mut moves = HashSet::new(); + for statement in clause.body.iter().rev() { + let mut overwritten = None; + let value = match statement { + Statement::Assign { + slot, + fields, + value, + } => { + if fields.is_empty() { + live.remove(slot); + } else { + // 字段赋值必须保留宿主的其余字段，不能在右侧读值时移动宿主。 + live.insert(*slot); + overwritten = Some((*slot, fields.clone())); + } + value + } + Statement::Call(value) => value, + }; + visit(value, &mut live, &mut moves, &mut overwritten); + } + moves +} + +fn visit( + expression: &Expression, + live: &mut HashSet<usize>, + moves: &mut HashSet<*const Expression>, + overwritten: &mut Option<(usize, Vec<usize>)>, +) { + use ExpressionKind::*; + if let Option::Some((slot, path)) = super::updates::place(expression) { + if let Option::Some((target, fields)) = overwritten + && *target == slot + && super::updates::overlaps(fields, &path) + { + // 右侧最后一次重叠读取必须恰好是被覆盖字段，不能提前移动整个宿主。 + if *fields == path { + moves.insert(expression as *const Expression); + } + *overwritten = None; + } + let mut base = expression; + while let Field { value, .. } = &base.kind { + base = value; + } + if live.insert(slot) { + moves.insert(base as *const Expression); + } + return; + } + match &expression.kind { + Local(slot) => { + if live.insert(*slot) { + moves.insert(expression as *const Expression); + } + } + Constant(_) | ChoiceOptions { .. } => {} + ModelOperation { operation, .. } => { + for value in operation.expressions().into_iter().rev() { + visit(value, live, moves, overwritten); + } + } + JobEnqueue { arguments, .. } => { + for value in arguments.iter().rev() { + visit(value, live, moves, overwritten); + } + } + Field { value, .. } + | Fail(value) + | AwaitTask(value) + | StopTask(value) + | Group(value) + | AwaitGroup(value) + | StopGroup(value) + | ChannelReceive(value) + | ChannelClose(value) + | Promote(value) + | Some(value) + | Unary { value, .. } => visit(value, live, moves, overwritten), + Call { arguments, .. } + | CaptureResult { arguments, .. } + | ParallelCall { arguments, .. } + | BlockingCall { arguments, .. } => { + for argument in arguments.iter().rev() { + if let CallArgument::Value(value) = argument { + visit(value, live, moves, overwritten); + } + } + } + RunCall { + group, arguments, .. + } => { + for argument in arguments.iter().rev() { + if let CallArgument::Value(value) = argument { + visit(value, live, moves, overwritten); + } + } + if let Option::Some(group) = group { + visit(group, live, moves, overwritten); + } + } + Channel { capacity, .. } => visit(capacity, live, moves, overwritten), + ChannelSend { channel, value } => { + visit(value, live, moves, overwritten); + visit(channel, live, moves, overwritten); + } + Intrinsic { arguments, .. } + | Collection { arguments, .. } + | Variant { arguments, .. } + | List(arguments) => { + for argument in arguments.iter().rev() { + visit(argument, live, moves, overwritten); + } + } + Record { fields } => { + for (_, value) in fields.iter().rev() { + visit(value, live, moves, overwritten); + } + } + Map(entries) => { + for (key, value) in entries.iter().rev() { + visit(value, live, moves, overwritten); + visit(key, live, moves, overwritten); + } + } + Binary { left, right, .. } => { + visit(right, live, moves, overwritten); + visit(left, live, moves, overwritten); + } + } +} diff --git a/crates/dever-core/src/native/orm.rs b/crates/dever-core/src/native/orm.rs new file mode 100644 index 0000000..d9533d5 --- /dev/null +++ b/crates/dever-core/src/native/orm.rs @@ -0,0 +1,2349 @@ +use std::collections::BTreeSet; +use std::fmt::Write; + +use crate::hir::{Expression, ModelCondition, ModelOperation, QueryPlan}; +use crate::model::{ + ConnectionSelector, ModelFieldType, ModelRelationKind, ModelSchema, ModelScope, + ModelSqlCardinality, +}; +use crate::source::Span; +use crate::syntax::BinaryOperator; +use crate::types::{DefinitionKind, Shape, Type}; + +use super::{Emitter, rust_string, rust_type}; + +impl Emitter<'_> { + pub(super) fn initialize_models(&self, output: &mut String) { + let mut model_migrations = Vec::new(); + let mut foreign_key_migrations = Vec::new(); + for (index, model) in self.program.models.iter().enumerate() { + let span = self.program.types[model.record].span; + let scope = match model.scope { + ModelScope::Global => "Global", + ModelScope::Tenant => "Tenant", + }; + let (explicit, root) = model_storage_selector(model); + writeln!( + output, + "MODEL_STORAGE_{}.set(dever_runtime::database::StorageBinding::new({explicit}, {}, dever_runtime::database::ModelScope::{scope})).map_err(|_| AppError::fault({}, \"Model storage was initialized more than once\"))?;", + model.record, + rust_string(root), + self.location(span), + ) + .expect("string formatting"); + let database = + self.database_failure(&format!("model_database_{}().await", model.record), span); + writeln!( + output, + "let schema_database_{index} = if dever_runtime::config::settings().tenant().is_some() && matches!(dever_runtime::database::ModelScope::{scope}, dever_runtime::database::ModelScope::Tenant) {{ None }} else {{ Some({database}) }};" + ) + .expect("string formatting"); + let fields = model + .fields + .iter() + .map(|field| { + format!( + "dever_runtime::database::Field {{ name: String::from({}), ty: String::from({}), nullable: {}, generated: {}, default: {}, rename_from: {} }}", + rust_string(&field.name), + rust_string(&crate::model::field_type(&field.ty)), + field.nullable, + field.generated, + field.default.as_ref().map(|value| format!("Some(String::from({}))", rust_string(&crate::model::value(value)))).unwrap_or_else(|| "None".into()), + field.rename_from.as_ref().map(|name| format!("Some(String::from({}))", rust_string(name))).unwrap_or_else(|| "None".into()), + ) + }) + .collect::<Vec<_>>() + .join(", "); + let indexes = model + .indexes + .iter() + .filter(|candidate| candidate.fields.as_slice() != ["id"]) + .map(|candidate| { + let name = index_name(model, candidate); + let fields = candidate + .fields + .iter() + .map(|field| format!("String::from({})", rust_string(field))) + .collect::<Vec<_>>() + .join(", "); + format!( + "dever_runtime::database::Index {{ name: String::from({}), fields: vec![{fields}], unique: {} }}", + rust_string(&name), + candidate.unique, + ) + }) + .collect::<Vec<_>>() + .join(", "); + let migrations = model + .migrations + .iter() + .map(|migration| { + let drops = migration + .drops + .iter() + .map(|field| format!("String::from({})", rust_string(field))) + .collect::<Vec<_>>() + .join(", "); + let revision = migration.revision(); + format!( + "dever_runtime::database::Migration {{ name: String::from({}), revision: String::from({}), drops: vec![{drops}] }}", + rust_string(&migration.name), + rust_string(&revision), + ) + }) + .collect::<Vec<_>>() + .join(", "); + let data_migrations = model.migrations.iter().flat_map(|migration| { + migration.statements.iter().map(move |statement| { + let phase = match statement.phase { + crate::syntax::MigrationPhase::Before => "Before", + crate::syntax::MigrationPhase::After => "After", + }; + let parameters = statement.parameters.iter() + .map(|(ty, value)| seed_value(value, ty)) + .collect::<Vec<_>>().join(", "); + format!( + "dever_runtime::database::DataMigration {{ name: String::from({}), phase: dever_runtime::database::MigrationPhase::{phase}, sql: dever_runtime::database::Sql {{ sqlite: {}, postgres: {} }}, parameters: vec![{parameters}] }}", + rust_string(&migration.name), rust_string(&statement.sqlite), rust_string(&statement.postgres), + ) + }) + }).collect::<Vec<_>>().join(", "); + let seeds = model + .seed + .iter() + .map(|seed| { + let generated = model + .fields + .iter() + .filter(|field| field.generated && matches!(field.ty, ModelFieldType::Uuid)) + .collect::<Vec<_>>(); + let columns = seed + .fields + .iter() + .map(|(field, _)| quoted(field)) + .chain(generated.iter().map(|field| quoted(&field.name))) + .collect::<Vec<_>>(); + let parameter_types = seed + .fields + .iter() + .map(|(name, _)| { + &model + .fields + .iter() + .find(|field| field.name == *name) + .expect("validated Seed field") + .ty + }) + .chain(generated.iter().map(|field| &field.ty)) + .collect::<Vec<_>>(); + let placeholders = parameter_types + .iter() + .enumerate() + .map(|(index, ty)| parameter_sql(index + 1, ty)) + .collect::<Vec<_>>(); + let sql = format!( + "INSERT INTO {} ({}) VALUES ({}) ON CONFLICT ({}) DO NOTHING", + quoted(&model.table), + columns.join(", "), + placeholders.join(", "), + seed.identity + .iter() + .map(|field| quoted(field)) + .collect::<Vec<_>>() + .join(", ") + ); + let mut values = seed + .fields + .iter() + .map(|(name, value)| { + let field = model + .fields + .iter() + .find(|field| field.name == *name) + .expect("validated Seed field"); + seed_value(value, &field.ty) + }) + .collect::<Vec<_>>(); + for _ in generated { + let uuid = + self.database_failure("dever_runtime::orm::Uuid::new_v7()", span); + values.push(format!("dever_runtime::orm::Value::Uuid({uuid})")); + } + format!( + "dever_runtime::database::Seed {{ sql: {}, parameters: vec![{values}] }}", + database_sql(&sql), + values = values.join(", "), + ) + }) + .collect::<Vec<_>>() + .join(", "); + let temporary = format!("_dever_{}_{}", model.table, &model.revision[..8]); + let table = sqlite_table(model, &self.program.models, &model.table); + let postgres_constraints = postgres_constraints(model, &self.program.models); + let postgres_table = postgres_table(model, &postgres_constraints, &model.table); + let postgres_columns = model + .fields + .iter() + .map(|field| { + format!( + "dever_runtime::database::PostgresColumn {{ name: String::from({}), definition: String::from({}), sql_type: String::from({}), default: {} }}", + rust_string(&field.name), + rust_string(&postgres_column(field)), + rust_string(&postgres_type(&field.ty)), + postgres_field_default(field) + .map(|value| format!("Some(String::from({}))", rust_string(&value))) + .unwrap_or_else(|| "None".into()), + ) + }) + .collect::<Vec<_>>() + .join(", "); + let postgres_constraints_code = postgres_constraints + .iter() + .map(|(name, definition, foreign_key)| { + format!( + "dever_runtime::database::PostgresConstraint {{ name: String::from({}), definition: String::from({}), foreign_key: {foreign_key} }}", + rust_string(name), + rust_string(definition), + ) + }) + .collect::<Vec<_>>() + .join(", "); + let foreign_keys = postgres_constraints + .iter() + .filter(|(_, _, foreign_key)| *foreign_key) + .map(|(name, definition, foreign_key)| { + format!( + "dever_runtime::database::PostgresConstraint {{ name: String::from({}), definition: String::from({}), foreign_key: {foreign_key} }}", + rust_string(name), + rust_string(definition), + ) + }) + .collect::<Vec<_>>() + .join(", "); + if !foreign_keys.is_empty() { + foreign_key_migrations.push(( + index, + span, + model.package.clone(), + model.table.clone(), + foreign_keys.clone(), + )); + } + let temporary_table = sqlite_table(model, &self.program.models, &temporary); + let create_indexes = sqlite_indexes(model) + .iter() + .map(|sql| { + format!( + "dever_runtime::database::SchemaSql {{ sqlite: String::from({}), postgres: String::from({}) }}", + rust_string(sql), + rust_string(sql), + ) + }) + .collect::<Vec<_>>() + .join(", "); + let request = format!( + "dever_runtime::database::Model {{ schema: dever_runtime::database::Schema {{ model: String::from({}), table: String::from({}), revision: String::from({}), seed_revision: String::from({}), fields: vec![{fields}], indexes: vec![{indexes}], migrations: vec![{migrations}] }}, create_table: dever_runtime::database::SchemaSql {{ sqlite: String::from({}), postgres: String::from({}) }}, create_temporary_table: String::from({}), create_indexes: vec![{create_indexes}], postgres_columns: vec![{postgres_columns}], postgres_constraints: vec![{postgres_constraints_code}], seeds: vec![{seeds}], data_migrations: vec![{data_migrations}] }}", + rust_string(&model.package), + rust_string(&model.table), + rust_string(&model.revision), + rust_string(&model.seed_revision), + rust_string(&table), + rust_string(&postgres_table), + rust_string(&temporary_table), + ); + writeln!( + output, + "fn model_definition_{index}() -> Result<dever_runtime::database::Model, AppError> {{ Ok({request}) }}" + ) + .expect("string formatting"); + writeln!( + output, + "fn model_foreign_keys_{index}() -> Vec<dever_runtime::database::PostgresConstraint> {{ vec![{foreign_keys}] }}" + ) + .expect("string formatting"); + let migration = self.database_failure( + &format!("schema_database_{index}.as_ref().expect(\"selected Model database\").migrate(model_definition_{index}()?).await"), + span, + ); + model_migrations.push((index, migration)); + } + self.emit_tenant_migrate(output); + if model_migrations.is_empty() { + return; + } + output.push_str("let mut schema_migration_order = Vec::new();\n"); + for (index, _) in &model_migrations { + writeln!(output, "if let Some(database) = schema_database_{index}.as_ref() {{ schema_migration_order.push((database.name(), {index}usize)); }}").expect("string formatting"); + } + output.push_str("schema_migration_order.sort_unstable();\nfor (_, schema_model) in schema_migration_order.iter().copied() {\nmatch schema_model {\n"); + for (index, migration) in &model_migrations { + writeln!(output, "{index} => {{ {migration}; }},").expect("string formatting"); + } + output.push_str( + "_ => unreachable!(\"schema migration references an unknown Model\"),\n}\n}\n", + ); + + if foreign_key_migrations.is_empty() { + return; + } + output.push_str("for (_, schema_model) in schema_migration_order.iter().copied() {\nmatch schema_model {\n"); + for (index, span, model, table, constraints) in foreign_key_migrations { + let migration = self.database_failure( + &format!( + "schema_database_{index}.as_ref().expect(\"selected Model database\").migrate_foreign_keys(String::from({}), String::from({}), vec![{constraints}]).await", + rust_string(&model), + rust_string(&table), + ), + span, + ); + writeln!(output, "{index} => {{ {migration}; }},").expect("string formatting"); + } + output.push_str("_ => {},\n}\n}\n"); + } + + fn emit_tenant_migrate(&self, output: &mut String) { + let tenant_models = self + .program + .models + .iter() + .enumerate() + .filter(|(_, model)| model.scope == ModelScope::Tenant) + .collect::<Vec<_>>(); + let fingerprint = tenant_fingerprint(&tenant_models, self.program); + output.push_str("async fn tenant_migrate(tenant_id: i64) -> Result<(), AppError> {\n"); + if tenant_models.is_empty() { + output.push_str("let _ = tenant_id;\nreturn Err(AppError::fault(\"tenant migration\", \"application has no tenant Model\"));\n}\n"); + return; + } + output.push_str("if tenant_id <= 0 { return Err(AppError::fault(\"tenant migration\", \"tenant id must be positive\")); }\n"); + output.push_str("let settings = dever_runtime::config::settings();\nif settings.tenant().is_none() { return Err(AppError::fault(\"tenant migration\", \"tenant storage is not configured\")); }\nlet mut tenant_databases = std::collections::BTreeMap::<String, dever_runtime::database::Database>::new();\n"); + if !self.program.permissions.is_empty() { + writeln!(output, + "let authorization_connection = settings.tenant().expect(\"checked tenant settings\").database().to_owned();\nif !tenant_databases.contains_key(&authorization_connection) {{ let database = dever_runtime::tenant::begin_migration(&authorization_connection, tenant_id, {}).await.map_err(|error| AppError::database(\"tenant authorization migration\", error))?; tenant_databases.insert(authorization_connection.clone(), database); }}", + rust_string(&fingerprint), + ).expect("string formatting"); + } + for (index, model) in &tenant_models { + let (explicit, root) = model_storage_selector(model); + writeln!( + output, + "let tenant_connection_{index} = settings.resolve_database({explicit}, {}).map_err(|error| AppError::fault(\"tenant migration\", error))?.0.to_owned();\nif !tenant_databases.contains_key(&tenant_connection_{index}) {{ let database = dever_runtime::tenant::begin_migration(&tenant_connection_{index}, tenant_id, {}).await.map_err(|error| AppError::database(\"tenant migration\", error))?; tenant_databases.insert(tenant_connection_{index}.clone(), database); }}", + rust_string(root), + rust_string(&fingerprint), + ) + .expect("string formatting"); + } + writeln!( + output, + "let mut tenant_migration_order = [{}];", + tenant_models + .iter() + .map(|(index, _)| format!("(tenant_connection_{index}.as_str(), {index}usize)")) + .collect::<Vec<_>>() + .join(", ") + ) + .expect("string formatting"); + output.push_str("tenant_migration_order.sort_unstable();\nfor (connection, schema_model) in tenant_migration_order.iter().copied() { let database = tenant_databases.get(connection).expect(\"tenant migration database initialized\"); match schema_model {\n"); + for (index, model) in &tenant_models { + let span = self.program.types[model.record].span; + let migration = self.database_failure( + &format!("database.migrate(model_definition_{index}()?).await"), + span, + ); + writeln!(output, "{index} => {{ {migration}; }},").expect("string formatting"); + } + output.push_str( + "_ => unreachable!(\"tenant migration references an unknown Model\"),\n} }\n", + ); + output.push_str("for (connection, schema_model) in tenant_migration_order.iter().copied() { let database = tenant_databases.get(connection).expect(\"tenant migration database initialized\"); match schema_model {\n"); + for (index, model) in &tenant_models { + let constraints = postgres_constraints(model, &self.program.models); + if constraints.iter().any(|(_, _, foreign_key)| *foreign_key) { + let span = self.program.types[model.record].span; + let migration = self.database_failure( + &format!( + "database.migrate_foreign_keys(String::from({}), String::from({}), model_foreign_keys_{index}()).await", + rust_string(&model.package), + rust_string(&model.table), + ), + span, + ); + writeln!(output, "{index} => {{ {migration}; }},").expect("string formatting"); + } + } + output.push_str("_ => {},\n} }\n"); + writeln!( + output, + "job_migrate_tenant(tenant_id, {}, &mut tenant_databases).await?;", + rust_string(&fingerprint) + ) + .expect("string formatting"); + if !self.program.permissions.is_empty() { + output.push_str("dever_runtime::auth::store::initialize_roles(tenant_databases.get(&authorization_connection).expect(\"tenant authorization database initialized\").clone()).await.map_err(|error| AppError::database(\"tenant authorization migration\", error))?;\n"); + } + writeln!(output, "for database in tenant_databases.values() {{ dever_runtime::tenant::mark_database_ready(database, tenant_id, {}).await.map_err(|error| AppError::database(\"tenant migration\", error))?; }}\ndever_runtime::tenant::finish_migration(tenant_id, {}).await.map_err(|error| AppError::database(\"tenant migration\", error))?;\nOk(())\n}}", rust_string(&fingerprint), rust_string(&fingerprint)).expect("string formatting"); + writeln!(output, "if dever_runtime::config::settings().tenant().is_some() {{ dever_runtime::tenant::initialize({}).await.map_err(|error| AppError::database(\"tenant initialization\", error))?; }}", rust_string(&fingerprint)).expect("string formatting"); + } + + pub(super) fn model_decoders(&self, output: &mut String) { + for model in &self.program.models { + writeln!( + output, + "static MODEL_STORAGE_{}: std::sync::OnceLock<dever_runtime::database::StorageBinding> = std::sync::OnceLock::new();\nasync fn model_database_{}() -> Result<dever_runtime::database::Database, dever_runtime::orm::Error> {{ MODEL_STORAGE_{}.get().expect(\"Model storage is initialized before entry\").database().await }}", + model.record, + model.record, + model.record + ) + .expect("string formatting"); + writeln!( + output, + "fn decode_model_fields_{}(values: &mut std::vec::IntoIter<dever_runtime::orm::Value>) -> Result<T{}, dever_runtime::orm::Error> {{", + model.record, model.record + ) + .expect("string formatting"); + let Shape::Record(fields) = &self.program.types[model.record].shape else { + unreachable!("Model is a record") + }; + for (index, field) in fields.iter().take(model.fields.len()).enumerate() { + let column = rust_string(&format!("{}.{}", model.package, field.name)); + let value = format!("dever_runtime::orm::column(values, {column})?"); + writeln!( + output, + "let f{index} = {};", + self.decode_value(&field.ty, &value) + ) + .expect("string formatting"); + } + for index in model.fields.len()..fields.len() { + writeln!( + output, + "let f{index} = dever_runtime::orm::Related::Unloaded;" + ) + .expect("string formatting"); + } + writeln!( + output, + "Ok(T{} {{ {} }})\n}}\nfn decode_model_{}(row: dever_runtime::orm::Row) -> Result<T{}, dever_runtime::orm::Error> {{\nlet mut values = row.into_values();\nlet model = decode_model_fields_{}(&mut values)?;\nif !values.as_slice().is_empty() {{ return Err(dever_runtime::orm::Error::invalid_data({})); }}\nOk(model)\n}}", + model.record, + (0..fields.len()) + .map(|index| format!("f{index}")) + .collect::<Vec<_>>() + .join(", "), + model.record, + model.record, + model.record, + rust_string(&format!("database row for {} has extra columns", model.package)), + ) + .expect("string formatting"); + } + let records = self + .program + .models + .iter() + .flat_map(|model| model.sql.iter().map(|sql| sql.result.record)) + .filter(|record| self.program.types[*record].kind == DefinitionKind::Regular) + .collect::<BTreeSet<_>>(); + for record in records { + let Shape::Record(fields) = &self.program.types[record].shape else { + unreachable!("checked SQL result is a record") + }; + writeln!( + output, + "fn decode_sql_record_{record}(row: dever_runtime::orm::Row) -> Result<T{record}, dever_runtime::orm::Error> {{\nlet mut values = row.into_values();" + ) + .expect("string formatting"); + for (index, field) in fields.iter().enumerate() { + let column = rust_string(&format!( + "{}.{}", + self.program.types[record].name, field.name + )); + let value = format!("dever_runtime::orm::column(&mut values, {column})?"); + writeln!( + output, + "let f{index} = {};", + self.decode_value(&field.ty, &value) + ) + .expect("string formatting"); + } + writeln!( + output, + "if !values.as_slice().is_empty() {{ return Err(dever_runtime::orm::Error::invalid_data({})); }}\nOk(T{record} {{ {} }})\n}}", + rust_string(&format!( + "database row for {} has extra columns", + self.program.types[record].name + )), + (0..fields.len()) + .map(|index| format!("f{index}")) + .collect::<Vec<_>>() + .join(", "), + ) + .expect("string formatting"); + } + } + + pub(super) fn model_operation( + &self, + model_index: usize, + operation: &ModelOperation, + result: &Expression, + ) -> String { + let model = &self.program.models[model_index]; + match operation { + ModelOperation::Create { values } => self.create(model, values, result), + ModelOperation::CreateMany { rows } => self.create_many(model, rows, result), + ModelOperation::Get { id } => self.get(model, id, result), + ModelOperation::First { query } => self.first(model, query, result), + ModelOperation::List { query } => self.list(model, query, result), + ModelOperation::Cursor { query } => self.cursor(model, query, result), + ModelOperation::Count { query } => self.count(model, query, result), + ModelOperation::Exists { query } => self.exists(model, query, result), + ModelOperation::Stream { query } => self.stream(model, query, result), + ModelOperation::Update { query, values } => self.update(model, query, values, result), + ModelOperation::Delete { query } => self.delete(model, query, result), + ModelOperation::Upsert { + key, + create, + update, + } => self.upsert(model, key, create, update, result), + ModelOperation::Sql { + operation, + arguments, + .. + } => self.sql(model, *operation, arguments, result), + } + } + + fn sql( + &self, + model: &ModelSchema, + operation: usize, + arguments: &[Expression], + result: &Expression, + ) -> String { + let operation = &model.sql[operation]; + let expressions = arguments + .iter() + .zip(&operation.parameters) + .map(|(argument, parameter)| (argument, ¶meter.ty)) + .collect::<Vec<_>>(); + let (bindings, parameters) = self.bind_values(&expressions); + let database = self.database(model, result); + let maximum = match operation.result.cardinality { + ModelSqlCardinality::Many => "database.max_page_size()", + ModelSqlCardinality::One | ModelSqlCardinality::Optional => "1usize", + }; + let rows = self.database_failure( + &format!( + "database.query_bounded(dever_runtime::database::Sql {{ sqlite: {}, postgres: {} }}, vec![{}], {maximum}).await", + rust_string(&operation.sqlite), + rust_string(&operation.postgres), + parameters.join(", ") + ), + result.span, + ); + let decoder = if self.program.types[operation.result.record].kind == DefinitionKind::Model { + format!("decode_model_{}", operation.result.record) + } else { + format!("decode_sql_record_{}", operation.result.record) + }; + let name = rust_string(&format!("{}.{}", model.package, operation.name)); + let decoded = match operation.result.cardinality { + ModelSqlCardinality::Many => { + let rows = self.database_failure( + &format!( + "rows.into_iter().map({decoder}).collect::<Result<Vec<_>, dever_runtime::orm::Error>>()" + ), + result.span, + ); + format!("List::new({rows})") + } + ModelSqlCardinality::One => { + let row = self.database_failure( + &format!("dever_runtime::orm::required_row(rows, {name})"), + result.span, + ); + self.database_failure(&format!("{decoder}({row})"), result.span) + } + ModelSqlCardinality::Optional => { + let row = self.database_failure( + &format!("dever_runtime::orm::optional_row(rows, {name})"), + result.span, + ); + let decoded = self.database_failure(&format!("{decoder}(row)"), result.span); + format!("match {row} {{ Some(row) => Some({decoded}), None => None }}") + } + }; + format!("{{ {bindings} let database = {database}; let rows = {rows}; {decoded} }}") + } + + fn create( + &self, + model: &ModelSchema, + values: &[(usize, Expression)], + result: &Expression, + ) -> String { + let generated = model + .fields + .iter() + .filter(|field| field.generated && matches!(field.ty, ModelFieldType::Uuid)) + .collect::<Vec<_>>(); + let columns = values + .iter() + .map(|(field, _)| quoted(&model.fields[*field].name)) + .chain(generated.iter().map(|field| quoted(&field.name))) + .collect::<Vec<_>>(); + let parameter_types = values + .iter() + .map(|(field, _)| &model.fields[*field].ty) + .chain(generated.iter().map(|field| &field.ty)) + .collect::<Vec<_>>(); + let placeholders = parameter_types + .iter() + .enumerate() + .map(|(index, ty)| parameter_sql(index + 1, ty)) + .collect::<Vec<_>>(); + let insertion = if columns.is_empty() { + "DEFAULT VALUES".into() + } else { + format!( + "({}) VALUES ({})", + columns.join(", "), + placeholders.join(", ") + ) + }; + let sql = format!( + "INSERT INTO {} {insertion} RETURNING {}", + quoted(&model.table), + select_columns(model) + ); + let expressions = values + .iter() + .map(|(field, value)| (value, &model.fields[*field].ty)) + .collect::<Vec<_>>(); + let (mut bindings, mut parameters) = self.bind_values(&expressions); + for index in 0..generated.len() { + let uuid = self.database_failure("dever_runtime::orm::Uuid::new_v7()", result.span); + writeln!( + bindings, + "let generated_uuid_{index} = dever_runtime::orm::Value::Uuid({uuid});" + ) + .expect("string formatting"); + parameters.push(format!("generated_uuid_{index}")); + } + let database = self.database(model, result); + let rows = self.database_failure( + &format!( + "database.query({}, vec![{}]).await", + database_sql(&sql), + parameters.join(", ") + ), + result.span, + ); + let row = self.database_failure( + &format!( + "rows.into_iter().next().ok_or_else(|| dever_runtime::orm::Error::invalid_data({}))", + rust_string("database INSERT RETURNING produced no row") + ), + result.span, + ); + let decoded = self.database_failure( + &format!("decode_model_{}({row})", model.record), + result.span, + ); + format!("{{ {bindings} let database = {database}; let rows = {rows}; {decoded} }}") + } + + fn create_many( + &self, + model: &ModelSchema, + rows: &[Vec<(usize, Expression)>], + result: &Expression, + ) -> String { + let generated = model + .fields + .iter() + .filter(|field| field.generated && matches!(field.ty, ModelFieldType::Uuid)) + .collect::<Vec<_>>(); + let columns = rows[0] + .iter() + .map(|(field, _)| quoted(&model.fields[*field].name)) + .chain(generated.iter().map(|field| quoted(&field.name))) + .collect::<Vec<_>>(); + let explicit_per_row = rows[0].len(); + let explicit_total = explicit_per_row * rows.len(); + let values = if columns.is_empty() { + (0..rows.len()) + .map(|_| "(NULL)".to_owned()) + .collect::<Vec<_>>() + .join(", ") + } else { + rows.iter() + .enumerate() + .map(|(row, values)| { + let placeholders = values + .iter() + .enumerate() + .map(|(column, (field, _))| { + parameter_sql( + row * explicit_per_row + column + 1, + &model.fields[*field].ty, + ) + }) + .chain(generated.iter().enumerate().map(|(column, field)| { + parameter_sql( + explicit_total + row * generated.len() + column + 1, + &field.ty, + ) + })) + .collect::<Vec<_>>(); + format!("({})", placeholders.join(", ")) + }) + .collect::<Vec<_>>() + .join(", ") + }; + let (sqlite_sql, postgres_sql) = if columns.is_empty() { + let sqlite = format!( + "INSERT INTO {} ({}) VALUES {values}", + quoted(&model.table), + quoted("id") + ); + let defaults = (0..rows.len()) + .map(|_| "(DEFAULT)") + .collect::<Vec<_>>() + .join(", "); + let postgres = format!( + "INSERT INTO {} ({}) VALUES {defaults}", + quoted(&model.table), + quoted("created_at") + ); + (sqlite, postgres) + } else { + let sql = format!( + "INSERT INTO {} ({}) VALUES {values}", + quoted(&model.table), + columns.join(", ") + ); + (sql.clone(), sql) + }; + let expressions = rows + .iter() + .flat_map(|row| row.iter()) + .map(|(field, value)| (value, &model.fields[*field].ty)) + .collect::<Vec<_>>(); + let (mut bindings, mut parameters) = self.bind_values(&expressions); + for index in 0..rows.len() * generated.len() { + let uuid = self.database_failure("dever_runtime::orm::Uuid::new_v7()", result.span); + writeln!( + bindings, + "let generated_uuid_{index} = dever_runtime::orm::Value::Uuid({uuid});" + ) + .expect("string formatting"); + parameters.push(format!("generated_uuid_{index}")); + } + let database = self.database(model, result); + let affected = self.database_failure( + &format!( + "database.execute({}, vec![{}]).await", + database_sql_pair(&sqlite_sql, &postgres_sql), + parameters.join(", ") + ), + result.span, + ); + let affected = self.database_failure( + &format!("dever_runtime::orm::affected({affected})"), + result.span, + ); + format!("{{ {bindings} let database = {database}; {affected} }}") + } + + fn get(&self, model: &ModelSchema, id: &Expression, result: &Expression) -> String { + let sql = format!( + "SELECT {} FROM {} WHERE {} = ?1 LIMIT 1", + select_columns(model), + quoted(&model.table), + quoted("id") + ); + let id_type = &model + .fields + .iter() + .find(|field| field.name == "id") + .expect("synthetic Model id") + .ty; + let (bindings, parameters) = self.bind_values(&[(id, id_type)]); + let database = self.database(model, result); + let rows = self.database_failure( + &format!( + "database.query({}, vec![{}]).await", + database_sql(&sql), + parameters.join(", ") + ), + result.span, + ); + let row = self.database_failure( + &format!( + "rows.into_iter().next().ok_or_else(|| dever_runtime::orm::Error::not_found({}))", + rust_string(&model.package) + ), + result.span, + ); + let decoded = self.database_failure( + &format!("decode_model_{}({row})", model.record), + result.span, + ); + format!("{{ {bindings} let database = {database}; let rows = {rows}; {decoded} }}") + } + + fn first(&self, model: &ModelSchema, query: &QueryPlan, result: &Expression) -> String { + let mut expressions = Vec::new(); + let joined = has_to_one(model, query); + let alias = joined.then_some("_dever_parent"); + let where_clause = + condition_sql_qualified(model, query.condition.as_ref(), &mut expressions, alias); + let (projection, source) = select_source(model, query, &self.program.models); + let sql = format!( + "SELECT {projection} FROM {source}{} ORDER BY {} LIMIT 1", + where_clause, + order_sql_qualified(model, &query.order, alias) + ); + let (bindings, parameters) = self.bind_values(&expressions); + let database = self.database(model, result); + let rows = self.database_failure( + &format!( + "database.query({}, vec![{}]).await", + database_sql(&sql), + parameters.join(", ") + ), + result.span, + ); + let decoded = + self.database_failure(&self.decode_included_row(model, query, "row"), result.span); + let load_relations = self.load_to_many(model, query, "items", result); + format!( + "{{ {bindings} let database = {database}; let rows = {rows}; let mut items = match rows.into_iter().next() {{ Some(row) => vec![{decoded}], None => Vec::new() }}; {load_relations} items.into_iter().next() }}" + ) + } + + fn list(&self, model: &ModelSchema, query: &QueryPlan, result: &Expression) -> String { + let mut expressions = Vec::new(); + let where_clause = condition_sql(model, query.condition.as_ref(), &mut expressions); + let count_sql = format!( + "SELECT COUNT(*) FROM {}{}", + quoted(&model.table), + where_clause + ); + let joined = has_to_one(model, query); + let alias = joined.then_some("_dever_parent"); + let (projection, source) = select_source(model, query, &self.program.models); + let mut data_expressions = Vec::new(); + let data_where = condition_sql_qualified( + model, + query.condition.as_ref(), + &mut data_expressions, + alias, + ); + debug_assert_eq!(data_expressions.len(), expressions.len()); + let data_sql = format!( + "SELECT {projection} FROM {source}{data_where} ORDER BY {} LIMIT ?{} OFFSET ?{}", + order_sql_qualified(model, &query.order, alias), + expressions.len() + 1, + expressions.len() + 2 + ); + let (bindings, parameters) = self.bind_values(&expressions); + let page = self.expression(&query.page); + let size = self.expression(&query.size); + let database = self.database(model, result); + let page_info = self.database_failure( + &format!("dever_runtime::orm::pagination({page}, {size}, database.max_page_size())"), + result.span, + ); + let count_rows = self.database_failure( + &format!( + "database.query({}, parameters.clone()).await", + database_sql(&count_sql) + ), + result.span, + ); + let count_row = self.database_failure( + &format!( + "count_rows.into_iter().next().ok_or_else(|| dever_runtime::orm::Error::invalid_data({}))", + rust_string("database COUNT returned no row") + ), + result.span, + ); + let count_value = self.database_failure( + &format!("dever_runtime::orm::scalar({count_row})"), + result.span, + ); + let total = self.database_failure( + &format!("dever_runtime::orm::int({count_value})"), + result.span, + ); + let rows = self.database_failure( + &format!( + "database.query({}, {{ parameters.push(dever_runtime::orm::Value::Int(size)); parameters.push(dever_runtime::orm::Value::Int(offset)); parameters }}).await", + database_sql(&data_sql) + ), + result.span, + ); + let decoded = self.database_failure( + &self.decode_included_rows(model, query, "rows"), + result.span, + ); + let load_relations = self.load_to_many(model, query, "items", result); + format!( + "{{ {bindings} let database = {database}; let (page, size, offset) = {page_info}; let mut parameters = vec![{}]; let count_rows = {count_rows}; let total = {total}; let rows = {rows}; let mut items = {decoded}; {load_relations} let items = List::new(items); let pages = total / size + i64::from(total % size != 0); T{} {{ f0: items, f1: page, f2: size, f3: total, f4: pages }} }}", + parameters.join(", "), + model.page + ) + } + + fn cursor(&self, model: &ModelSchema, query: &QueryPlan, result: &Expression) -> String { + let mut expressions = Vec::new(); + let joined = has_to_one(model, query); + let alias = joined.then_some("_dever_parent"); + let mut where_clause = + condition_sql_qualified(model, query.condition.as_ref(), &mut expressions, alias); + if query.after.is_some() { + let condition = cursor_condition(model, &query.order, expressions.len() + 1, alias); + if where_clause.is_empty() { + where_clause = format!(" WHERE {condition}"); + } else { + where_clause.push_str(" AND "); + where_clause.push_str(&condition); + } + } + let limit = expressions.len() + query.after.as_ref().map_or(0, |_| query.order.len()) + 1; + let (projection, source) = select_source(model, query, &self.program.models); + let sql = format!( + "SELECT {projection} FROM {source}{} ORDER BY {} LIMIT ?{limit}", + where_clause, + order_sql_qualified(model, &query.order, alias), + ); + let (mut bindings, mut parameters) = self.bind_values(&expressions); + if let Some(after) = &query.after { + writeln!( + bindings, + "let cursor_after: Option<T{}> = {};", + model.record, + self.expression(after) + ) + .expect("string formatting"); + let Shape::Record(fields) = &self.program.types[model.record].shape else { + unreachable!("Model is a record") + }; + for (index, order) in query.order.iter().enumerate() { + let field = &fields[order.field]; + let encoded = self.encode_value( + &field.ty, + &model.fields[order.field].ty, + &format!("item.f{}.clone()", order.field), + after.span, + ); + writeln!( + bindings, + "let cursor_p{index} = match cursor_after.as_ref() {{ Some(item) => {encoded}, None => dever_runtime::orm::Value::Null }};" + ) + .expect("string formatting"); + parameters.push(format!("cursor_p{index}")); + } + } + let size = self.expression(&query.size); + let database = self.database(model, result); + let size = self.database_failure( + &format!("dever_runtime::orm::cursor_size({size}, database.max_page_size())"), + result.span, + ); + let rows = self.database_failure( + &format!( + "database.query({}, {{ parameters.push(dever_runtime::orm::Value::Int(size + 1)); parameters }}).await", + database_sql(&sql) + ), + result.span, + ); + let decoded = self.database_failure( + &self.decode_included_rows(model, query, "rows"), + result.span, + ); + let load_relations = self.load_to_many(model, query, "items", result); + format!( + "{{ {bindings} let database = {database}; let size = {size}; let mut parameters = vec![{}]; let rows = {rows}; let mut items = {decoded}; let has_more = items.len() > size as usize; if has_more {{ items.truncate(size as usize); }} {load_relations} let next = if has_more {{ items.last().cloned() }} else {{ None }}; T{} {{ f0: List::new(items), f1: next, f2: has_more }} }}", + parameters.join(", "), + model.cursor, + ) + } + + fn stream(&self, model: &ModelSchema, query: &QueryPlan, result: &Expression) -> String { + let mut expressions = Vec::new(); + let where_clause = condition_sql(model, query.condition.as_ref(), &mut expressions); + let sql = format!( + "SELECT {} FROM {}{} ORDER BY {}", + select_columns(model), + quoted(&model.table), + where_clause, + order_sql(model, &query.order), + ); + let (bindings, parameters) = self.bind_values(&expressions); + let capacity = self.expression(&query.size); + let database = self.database(model, result); + let capacity = self.database_failure( + &format!("dever_runtime::orm::stream_capacity({capacity}, database.max_page_size())"), + result.span, + ); + let stream = self.database_failure( + &format!( + "database.stream({}, vec![{}], capacity).await", + database_sql(&sql), + parameters.join(", ") + ), + result.span, + ); + format!( + "{{ {bindings} let database = {database}; let capacity = {capacity}; let stream = {stream}; stream.map(decode_model_{}) }}", + model.record + ) + } + + fn decode_included_rows(&self, model: &ModelSchema, query: &QueryPlan, rows: &str) -> String { + format!( + "{rows}.into_iter().map(|row| {}).collect::<Result<Vec<_>, dever_runtime::orm::Error>>()", + self.decode_included_row(model, query, "row") + ) + } + + fn decode_included_row(&self, model: &ModelSchema, query: &QueryPlan, row: &str) -> String { + let mut code = format!( + "{{ let mut values = {row}.into_values(); let mut item = decode_model_fields_{}(&mut values)?;", + model.record + ); + for relation_index in &query.relations { + let relation = &model.relations[*relation_index]; + if relation.kind != ModelRelationKind::ToOne { + continue; + } + let target = self + .program + .models + .iter() + .find(|candidate| candidate.package == relation.model) + .expect("validated relation Model"); + let record_field = model.fields.len() + relation_index; + let nullable = model + .fields + .iter() + .find(|field| field.name == relation.field) + .expect("validated to-one foreign key") + .nullable; + if nullable { + write!( + code, + " let related = if matches!(values.as_slice().first(), Some(dever_runtime::orm::Value::Null)) {{ for _ in 0..{} {{ dever_runtime::orm::column(&mut values, {})?; }} None }} else {{ Some(decode_model_fields_{}(&mut values)?) }}; item.f{} = dever_runtime::orm::Related::Loaded(Box::new(related));", + target.fields.len(), + rust_string(&format!("{}.{}", target.package, relation.name)), + target.record, + record_field, + ) + .expect("string formatting"); + } else { + write!( + code, + " item.f{} = dever_runtime::orm::Related::Loaded(Box::new(decode_model_fields_{}(&mut values)?));", + record_field, target.record, + ) + .expect("string formatting"); + } + } + write!( + code, + " if !values.as_slice().is_empty() {{ return Err(dever_runtime::orm::Error::invalid_data({})); }} Ok(item) }}", + rust_string(&format!("database row for {} has extra columns", model.package)), + ) + .expect("string formatting"); + // Keep decoder ?/return inside the ORM Result boundary, including + // first(), whose caller is an AppError-returning function. + format!( + "(|| -> Result<T{}, dever_runtime::orm::Error> {code})()", + model.record + ) + } + + fn load_to_many( + &self, + model: &ModelSchema, + query: &QueryPlan, + items: &str, + result: &Expression, + ) -> String { + let mut code = String::new(); + for relation_index in &query.relations { + let relation = &model.relations[*relation_index]; + if relation.kind != ModelRelationKind::ToMany { + continue; + } + let target = self + .program + .models + .iter() + .find(|candidate| candidate.package == relation.model) + .expect("validated relation Model"); + let foreign_key = target + .fields + .iter() + .position(|field| field.name == relation.field) + .expect("validated reverse foreign key"); + let nullable = target.fields[foreign_key].nullable; + let record_field = model.fields.len() + relation_index; + let prefix = format!( + "SELECT {} FROM (SELECT {}, ROW_NUMBER() OVER (PARTITION BY {} ORDER BY {} ASC) AS {} FROM {} WHERE {} IN (", + select_columns(target), + select_columns(target), + quoted(&relation.field), + quoted("id"), + quoted("_dever_rank"), + quoted(&target.table), + quoted(&relation.field), + ); + let before_limit = format!( + ")) AS {} WHERE {} <= ", + quoted("_dever_children"), + quoted("_dever_rank"), + ); + let suffix = format!( + " ORDER BY {} ASC, {} ASC", + quoted(&relation.field), + quoted("id"), + ); + let query_rows = self.database_failure( + "database.query_owned(relation_sql, relation_parameters).await", + result.span, + ); + let decoded = self.database_failure( + &format!( + "relation_rows.into_iter().map(decode_model_{}).collect::<Result<Vec<_>, _>>()", + target.record + ), + result.span, + ); + let relation_limit = self.database_failure( + "dever_runtime::orm::relation_limit(database.max_page_size())", + result.span, + ); + let parent_key = if nullable { + format!( + "match child.f{foreign_key}.as_ref() {{ Some(parent) => parent.0, None => continue }}" + ) + } else { + format!("child.f{foreign_key}.0") + }; + write!( + code, + "let mut relation_groups_{relation_index}: std::collections::BTreeMap<i64, Vec<T{}>> = std::collections::BTreeMap::new(); let relation_parent_ids_{relation_index} = {items}.iter().map(|parent| parent.f0.0).collect::<Vec<_>>(); let relation_limit_{relation_index} = {relation_limit}; for relation_ids in relation_parent_ids_{relation_index}.chunks(998) {{ let mut relation_parameters = relation_ids.iter().copied().map(dever_runtime::orm::Value::Int).collect::<Vec<_>>(); relation_parameters.push(dever_runtime::orm::Value::Int(relation_limit_{relation_index})); let relation_sql = dever_runtime::database::relation_sql({}, {}, {}, relation_ids.len()); let relation_rows = {query_rows}; let relation_children = {decoded}; for child in relation_children {{ let parent_id = {parent_key}; relation_groups_{relation_index}.entry(parent_id).or_default().push(child); }} }} for parent in {items}.iter_mut() {{ let children = relation_groups_{relation_index}.remove(&parent.f0.0).unwrap_or_default(); parent.f{record_field} = dever_runtime::orm::Related::Loaded(Box::new(List::new(children))); }}", + target.record, + rust_string(&prefix), + rust_string(&before_limit), + rust_string(&suffix), + ) + .expect("string formatting"); + } + code + } + + fn count(&self, model: &ModelSchema, query: &QueryPlan, result: &Expression) -> String { + self.scalar_query(model, query, result, "COUNT(*)", "int") + } + + fn exists(&self, model: &ModelSchema, query: &QueryPlan, result: &Expression) -> String { + self.scalar_query( + model, + query, + result, + &format!("EXISTS(SELECT 1 FROM {}", quoted(&model.table)), + "boolean", + ) + } + + fn scalar_query( + &self, + model: &ModelSchema, + query: &QueryPlan, + result: &Expression, + projection: &str, + decoder: &str, + ) -> String { + let mut expressions = Vec::new(); + let where_clause = condition_sql(model, query.condition.as_ref(), &mut expressions); + let sql = if projection.starts_with("EXISTS(") { + format!("SELECT {projection}{where_clause})") + } else { + format!( + "SELECT {projection} FROM {}{where_clause}", + quoted(&model.table) + ) + }; + let (bindings, parameters) = self.bind_values(&expressions); + let database = self.database(model, result); + let rows = self.database_failure( + &format!( + "database.query({}, vec![{}]).await", + database_sql(&sql), + parameters.join(", ") + ), + result.span, + ); + let row = self.database_failure( + &format!( + "rows.into_iter().next().ok_or_else(|| dever_runtime::orm::Error::invalid_data({}))", + rust_string("database scalar query returned no row") + ), + result.span, + ); + let value = + self.database_failure(&format!("dever_runtime::orm::scalar({row})"), result.span); + let decoded = self.database_failure( + &format!("dever_runtime::orm::{decoder}({value})"), + result.span, + ); + format!("{{ {bindings} let database = {database}; let rows = {rows}; {decoded} }}") + } + + fn update( + &self, + model: &ModelSchema, + query: &QueryPlan, + values: &[(usize, Expression)], + result: &Expression, + ) -> String { + let mut expressions = values + .iter() + .map(|(field, value)| (value, &model.fields[*field].ty)) + .collect::<Vec<_>>(); + let assignments = update_assignments(model, values.iter().map(|(field, _)| *field), 1); + let where_clause = condition_sql(model, query.condition.as_ref(), &mut expressions); + let sql = format!( + "UPDATE {} SET {assignments}{where_clause}", + quoted(&model.table) + ); + self.write(model, &sql, &expressions, result) + } + + fn delete(&self, model: &ModelSchema, query: &QueryPlan, result: &Expression) -> String { + let mut expressions = Vec::new(); + let where_clause = condition_sql(model, query.condition.as_ref(), &mut expressions); + let sql = format!("DELETE FROM {}{where_clause}", quoted(&model.table)); + self.write(model, &sql, &expressions, result) + } + + fn upsert( + &self, + model: &ModelSchema, + key: &[(usize, Expression)], + create: &[(usize, Expression)], + update: &[(usize, Expression)], + result: &Expression, + ) -> String { + let generated = model + .fields + .iter() + .filter(|field| field.generated && matches!(field.ty, ModelFieldType::Uuid)) + .collect::<Vec<_>>(); + let inserted = key.iter().chain(create); + let columns = inserted + .clone() + .map(|(field, _)| quoted(&model.fields[*field].name)) + .chain(generated.iter().map(|field| quoted(&field.name))) + .collect::<Vec<_>>(); + let insert_types = key + .iter() + .chain(create) + .map(|(field, _)| &model.fields[*field].ty) + .chain(generated.iter().map(|field| &field.ty)) + .collect::<Vec<_>>(); + let insert_parameters = insert_types + .iter() + .enumerate() + .map(|(index, ty)| parameter_sql(index + 1, ty)) + .collect::<Vec<_>>(); + let update_offset = insert_parameters.len(); + let assignments = update_assignments( + model, + update.iter().map(|(field, _)| *field), + update_offset + 1, + ); + let sql = format!( + "INSERT INTO {} ({}) VALUES ({}) ON CONFLICT ({}) DO UPDATE SET {assignments} RETURNING {}", + quoted(&model.table), + columns.join(", "), + insert_parameters.join(", "), + key.iter() + .map(|(field, _)| quoted(&model.fields[*field].name)) + .collect::<Vec<_>>() + .join(", "), + select_columns(model), + ); + let expressions = key + .iter() + .chain(create) + .chain(update) + .map(|(field, value)| (value, &model.fields[*field].ty)) + .collect::<Vec<_>>(); + let (mut bindings, mut parameters) = self.bind_values(&expressions); + for index in 0..generated.len() { + let uuid = self.database_failure("dever_runtime::orm::Uuid::new_v7()", result.span); + writeln!( + bindings, + "let generated_uuid_{index} = dever_runtime::orm::Value::Uuid({uuid});" + ) + .expect("string formatting"); + parameters.insert( + key.len() + create.len() + index, + format!("generated_uuid_{index}"), + ); + } + let database = self.database(model, result); + let rows = self.database_failure( + &format!( + "database.query({}, vec![{}]).await", + database_sql(&sql), + parameters.join(", ") + ), + result.span, + ); + let row = self.database_failure( + &format!( + "rows.into_iter().next().ok_or_else(|| dever_runtime::orm::Error::invalid_data({}))", + rust_string("database UPSERT RETURNING produced no row") + ), + result.span, + ); + let decoded = self.database_failure( + &format!("decode_model_{}({row})", model.record), + result.span, + ); + format!("{{ {bindings} let database = {database}; let rows = {rows}; {decoded} }}") + } + + fn write( + &self, + model: &ModelSchema, + sql: &str, + expressions: &[(&Expression, &ModelFieldType)], + result: &Expression, + ) -> String { + let (bindings, parameters) = self.bind_values(expressions); + let database = self.database(model, result); + let affected = self.database_failure( + &format!( + "database.execute({}, vec![{}]).await", + database_sql(sql), + parameters.join(", ") + ), + result.span, + ); + let affected = self.database_failure( + &format!("dever_runtime::orm::affected({affected})"), + result.span, + ); + format!("{{ {bindings} let database = {database}; {affected} }}") + } + + fn database(&self, model: &ModelSchema, result: &Expression) -> String { + let pool = self.database_at(model); + self.database_failure( + &format!("dever_runtime::database::Executor::new({pool}, _database)"), + result.span, + ) + } + + pub(super) fn database_at(&self, model: &ModelSchema) -> String { + format!("_model_database_{}.clone()", model.record) + } + + fn bind_values(&self, values: &[(&Expression, &ModelFieldType)]) -> (String, Vec<String>) { + let mut bindings = String::new(); + let mut parameters = Vec::new(); + for (index, (value, storage)) in values.iter().enumerate() { + writeln!( + bindings, + "let p{index}: {} = {};", + rust_type(&value.ty), + self.expression(value) + ) + .expect("string formatting"); + parameters.push(self.encode_value( + &value.ty, + storage, + &format!("p{index}"), + value.span, + )); + } + (bindings, parameters) + } + + pub(super) fn encode_value( + &self, + ty: &Type, + storage: &ModelFieldType, + value: &str, + span: Span, + ) -> String { + match ty { + Type::Nullable(inner) => format!( + "match {value} {{ Some(value) => {}, None => dever_runtime::orm::Value::Null }}", + self.encode_value(inner, storage, "value", span) + ), + Type::Bool => format!("dever_runtime::orm::Value::Bool({value})"), + Type::Int | Type::DateTime | Type::Date | Type::Time | Type::Duration => { + format!("dever_runtime::orm::Value::Int({value})") + } + Type::Float => format!("dever_runtime::orm::Value::Float({value})"), + Type::Decimal => { + let ModelFieldType::Decimal { precision, scale } = storage else { + unreachable!("Decimal expression bound to Decimal Model field") + }; + self.database_failure( + &format!("dever_runtime::orm::decimal_value({value}, {precision}, {scale})"), + span, + ) + } + Type::Uuid => { + format!("dever_runtime::orm::Value::Uuid({value})") + } + Type::Text | Type::Json => { + format!("dever_runtime::orm::Value::Text({value})") + } + Type::Bytes => { + format!("dever_runtime::orm::Value::Bytes({value}.into_values().collect())") + } + Type::Named(id) if self.program.types[*id].kind == DefinitionKind::ModelId => { + format!("dever_runtime::orm::Value::Int({value}.0)") + } + Type::Named(id) if matches!(&self.program.types[*id].shape, Shape::Choice(variants) if variants.iter().all(|variant| !variant.error && variant.fields.is_empty())) => + { + let Shape::Choice(variants) = &self.program.types[*id].shape else { + unreachable!() + }; + let arms = variants + .iter() + .enumerate() + .map(|(variant, definition)| { + format!( + "T{id}::V{variant} => dever_runtime::orm::Value::Text(String::from({}))", + rust_string(&definition.name) + ) + }) + .collect::<Vec<_>>() + .join(", "); + format!("match {value} {{ {arms} }}") + } + _ => unreachable!("checked Model parameter type"), + } + } + + fn decode_value(&self, ty: &Type, value: &str) -> String { + match ty { + Type::Nullable(inner) => format!( + "dever_runtime::orm::nullable({value}, |value| {})?", + self.decode_result(inner, "value") + ), + _ => format!("{}?", self.decode_result(ty, value)), + } + } + + fn decode_result(&self, ty: &Type, value: &str) -> String { + match ty { + Type::Bool => format!("dever_runtime::orm::boolean({value})"), + Type::Int | Type::DateTime | Type::Date | Type::Time | Type::Duration => { + format!("dever_runtime::orm::int({value})") + } + Type::Float => format!("dever_runtime::orm::float({value})"), + Type::Decimal => format!("dever_runtime::orm::decimal({value})"), + Type::Text | Type::Json => { + format!("dever_runtime::orm::text({value})") + } + Type::Uuid => format!("dever_runtime::orm::uuid({value})"), + Type::Bytes => format!("dever_runtime::orm::bytes({value})"), + Type::Named(id) if self.program.types[*id].kind == DefinitionKind::ModelId => { + format!("dever_runtime::orm::int({value}).map(T{id})") + } + Type::Named(id) if matches!(&self.program.types[*id].shape, Shape::Choice(variants) if variants.iter().all(|variant| !variant.error && variant.fields.is_empty())) => + { + let Shape::Choice(variants) = &self.program.types[*id].shape else { + unreachable!() + }; + let arms = variants + .iter() + .enumerate() + .map(|(variant, definition)| { + format!("{} => Ok(T{id}::V{variant})", rust_string(&definition.name)) + }) + .collect::<Vec<_>>() + .join(", "); + format!( + "{{ let choice = dever_runtime::orm::text({value})?; match choice.as_str() {{ {arms}, _ => Err(dever_runtime::orm::Error::invalid_data(format!(\"invalid database choice '{{}}'\", choice))) }} }}" + ) + } + Type::Nullable(_) => unreachable!("nullable decoded by wrapper"), + _ => unreachable!("checked Model field type"), + } + } +} + +fn model_storage_selector(model: &ModelSchema) -> (String, &str) { + match &model.connection { + ConnectionSelector::Explicit(name) => ( + format!("Some({})", rust_string(name)), + model.package.split('.').next().expect("Model package root"), + ), + ConnectionSelector::PackageRoot(root) => ("None".into(), root.as_str()), + } +} + +pub(crate) fn tenant_fingerprint( + models: &[(usize, &ModelSchema)], + program: &crate::hir::Program, +) -> String { + let mut identities = models + .iter() + .map(|(_, model)| { + let connection = match &model.connection { + ConnectionSelector::Explicit(name) => format!("explicit:{name}"), + ConnectionSelector::PackageRoot(root) => format!("root:{root}"), + }; + format!("{}\0{}\0{connection}", model.package, model.revision) + }) + .collect::<Vec<_>>(); + identities.extend( + program + .jobs + .iter() + .filter(|job| super::jobs::scope(program, job.function) == ModelScope::Tenant) + .map(|job| { + let connection = match &job.connection { + ConnectionSelector::Explicit(name) => format!("explicit:{name}"), + ConnectionSelector::PackageRoot(root) => format!("root:{root}"), + }; + format!( + "job-v2\0{}\0{}\0{}\0{}\0{}\0{connection}", + program.functions[job.function].name, + job.payload + .as_ref() + .map(|schema| schema.fingerprint()) + .unwrap_or("wire-v1:Job:Unit"), + job.attempts, + job.timeout_ms, + job.schedule.as_deref().unwrap_or(""), + ) + }), + ); + crate::model::checksum(&identities.join("\n")) +} + +pub(crate) fn sqlite_table(model: &ModelSchema, models: &[ModelSchema], table: &str) -> String { + let fields = model + .fields + .iter() + .map(|field| sqlite_field(field, model, models)) + .collect::<Vec<_>>() + .join(",\n "); + format!("CREATE TABLE {} (\n {fields}\n)", quoted(table)) +} + +pub(crate) fn sqlite_indexes(model: &ModelSchema) -> Vec<String> { + let mut statements = Vec::new(); + for index in &model.indexes { + if index.fields.as_slice() == ["id"] { + continue; + } + let name = index_name(model, index); + statements.push(format!( + "CREATE {}INDEX IF NOT EXISTS {} ON {} ({})", + if index.unique { "UNIQUE " } else { "" }, + quoted(&name), + quoted(&model.table), + index + .fields + .iter() + .map(|field| quoted(field)) + .collect::<Vec<_>>() + .join(", ") + )); + } + statements +} + +pub(crate) fn index_name(model: &ModelSchema, index: &crate::model::ModelIndex) -> String { + let identity = format!( + "{}:{}:{}:{}", + model.package, + model.table, + index.unique, + index.fields.join("\0") + ); + format!("_dever_idx_{}", crate::model::checksum(&identity)) +} + +fn sqlite_field( + field: &crate::model::ModelField, + model: &ModelSchema, + models: &[ModelSchema], +) -> String { + use crate::model::ModelFieldType; + if field.name == "id" { + return format!("{} INTEGER PRIMARY KEY AUTOINCREMENT", quoted("id")); + } + let mut definition = format!( + "{} {}", + quoted(&field.name), + match field.ty { + ModelFieldType::Bool + | ModelFieldType::Int + | ModelFieldType::DateTime + | ModelFieldType::Date + | ModelFieldType::Time + | ModelFieldType::Duration + | ModelFieldType::ModelId(_) => "INTEGER", + ModelFieldType::Float => "REAL", + ModelFieldType::Bytes { .. } | ModelFieldType::Uuid => "BLOB", + ModelFieldType::Decimal { .. } + | ModelFieldType::Text { .. } + | ModelFieldType::Json + | ModelFieldType::Choice(_) => "TEXT", + } + ); + if !field.nullable { + definition.push_str(" NOT NULL"); + } + if field.name == "created_at" { + definition.push_str(" DEFAULT (unixepoch() * 1000)"); + } else if let Some(default) = &field.default { + write!( + definition, + " DEFAULT {}", + sqlite_default(default, &field.ty) + ) + .expect("string formatting"); + } + match &field.ty { + ModelFieldType::Bool => { + write!(definition, " CHECK ({} IN (0, 1))", quoted(&field.name)) + .expect("string formatting"); + } + ModelFieldType::Decimal { precision, scale } => { + let name = quoted(&field.name); + let length = usize::from(*precision) + 5; + let suffix = sql_text(&format!(":{scale:02}")); + write!( + definition, + " CHECK ({name} IS NULL OR (typeof({name}) = 'text' AND length({name}) = {length} AND substr({name}, 1, 2) IN ('D0', 'D1') AND substr({name}, 3, {precision}) NOT GLOB '*[^0-9]*' AND substr({name}, {}) = {suffix}))", + usize::from(*precision) + 3, + ) + .expect("string formatting"); + } + ModelFieldType::Uuid => { + let name = quoted(&field.name); + write!( + definition, + " CHECK ({name} IS NULL OR (typeof({name}) = 'blob' AND length({name}) = 16))" + ) + .expect("string formatting"); + } + ModelFieldType::Text { minimum, maximum } | ModelFieldType::Bytes { minimum, maximum } => { + if *minimum > 0 { + write!( + definition, + " CHECK (length({}) >= {minimum})", + quoted(&field.name) + ) + .expect("string formatting"); + } + if let Some(maximum) = maximum { + write!( + definition, + " CHECK (length({}) <= {maximum})", + quoted(&field.name) + ) + .expect("string formatting"); + } + } + ModelFieldType::Choice(choice) => { + let options = model + .choices + .iter() + .find(|candidate| candidate.name == *choice) + .expect("validated Model choice") + .options + .iter() + .map(|(name, _)| sql_text(name)) + .collect::<Vec<_>>() + .join(", "); + write!( + definition, + " CHECK ({} IN ({options}))", + quoted(&field.name) + ) + .expect("string formatting"); + } + ModelFieldType::ModelId(target) => { + if field.logical_reference { + return definition; + } + let table = &models + .iter() + .find(|candidate| candidate.package == *target) + .expect("validated Model foreign key") + .table; + write!( + definition, + " REFERENCES {} ({})", + quoted(table), + quoted("id") + ) + .expect("string formatting"); + } + _ => {} + } + definition +} + +pub(crate) fn postgres_table( + model: &ModelSchema, + constraints: &[(String, String, bool)], + table: &str, +) -> String { + let mut fields = model.fields.iter().map(postgres_column).collect::<Vec<_>>(); + fields.extend( + constraints + .iter() + .filter(|(_, _, foreign_key)| !foreign_key) + .map(|(name, definition, _)| format!("CONSTRAINT {} {definition}", quoted(name))), + ); + format!( + "CREATE TABLE {} (\n {}\n)", + quoted(table), + fields.join(",\n ") + ) +} + +pub(crate) fn postgres_column(field: &crate::model::ModelField) -> String { + if field.name == "id" { + return format!( + "{} BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY", + quoted("id") + ); + } + let mut definition = format!("{} {}", quoted(&field.name), postgres_type(&field.ty),); + if !field.nullable { + definition.push_str(" NOT NULL"); + } + if let Some(default) = postgres_field_default(field) { + write!(definition, " DEFAULT {default}").expect("string formatting"); + } + definition +} + +pub(crate) fn postgres_constraints( + model: &ModelSchema, + models: &[ModelSchema], +) -> Vec<(String, String, bool)> { + use crate::model::ModelFieldType; + let mut constraints = Vec::new(); + for field in &model.fields { + match &field.ty { + ModelFieldType::Text { minimum, maximum } => { + if *minimum > 0 { + constraints.push(postgres_length_constraint( + model, + field, + "min", + "char_length", + ">=", + *minimum, + )); + } + if let Some(maximum) = maximum { + constraints.push(postgres_length_constraint( + model, + field, + "max", + "char_length", + "<=", + *maximum, + )); + } + } + ModelFieldType::Bytes { minimum, maximum } => { + if *minimum > 0 { + constraints.push(postgres_length_constraint( + model, + field, + "min", + "octet_length", + ">=", + *minimum, + )); + } + if let Some(maximum) = maximum { + constraints.push(postgres_length_constraint( + model, + field, + "max", + "octet_length", + "<=", + *maximum, + )); + } + } + ModelFieldType::Choice(choice) => { + let options = model + .choices + .iter() + .find(|candidate| candidate.name == *choice) + .expect("validated Model choice") + .options + .iter() + .map(|(name, _)| sql_text(name)) + .collect::<Vec<_>>() + .join(", "); + constraints.push(( + postgres_constraint_name("ck", model, field, "choice"), + format!("CHECK ({} IN ({options}))", quoted(&field.name)), + false, + )); + } + ModelFieldType::ModelId(target) => { + if field.generated || field.logical_reference { + continue; + } + let table = &models + .iter() + .find(|candidate| candidate.package == *target) + .expect("validated Model foreign key") + .table; + constraints.push(( + postgres_constraint_name("fk", model, field, target), + format!( + "FOREIGN KEY ({}) REFERENCES {} ({})", + quoted(&field.name), + quoted(table), + quoted("id") + ), + true, + )); + } + _ => {} + } + } + constraints +} + +fn postgres_length_constraint( + model: &ModelSchema, + field: &crate::model::ModelField, + suffix: &str, + function: &str, + operator: &str, + bound: u32, +) -> (String, String, bool) { + ( + postgres_constraint_name("ck", model, field, suffix), + format!( + "CHECK ({function}({}) {operator} {bound})", + quoted(&field.name) + ), + false, + ) +} + +fn postgres_constraint_name( + kind: &str, + model: &ModelSchema, + field: &crate::model::ModelField, + purpose: &str, +) -> String { + format!( + "_dever_{kind}_{}", + crate::model::checksum(&format!( + "{}:{}:{}:{purpose}", + model.package, model.table, field.name + )) + ) +} + +pub(crate) fn postgres_type(ty: &crate::model::ModelFieldType) -> String { + use crate::model::ModelFieldType; + match ty { + ModelFieldType::Bool => "BOOLEAN".into(), + ModelFieldType::Int + | ModelFieldType::DateTime + | ModelFieldType::Date + | ModelFieldType::Time + | ModelFieldType::Duration + | ModelFieldType::ModelId(_) => "BIGINT".into(), + ModelFieldType::Float => "DOUBLE PRECISION".into(), + ModelFieldType::Bytes { .. } => "BYTEA".into(), + ModelFieldType::Decimal { precision, scale } => { + format!("NUMERIC({precision},{scale})") + } + ModelFieldType::Uuid => "UUID".into(), + ModelFieldType::Text { .. } | ModelFieldType::Json | ModelFieldType::Choice(_) => { + "TEXT".into() + } + } +} + +pub(crate) fn postgres_field_default(field: &crate::model::ModelField) -> Option<String> { + if field.name == "created_at" { + Some("((extract(epoch from clock_timestamp()) * 1000)::bigint)".into()) + } else { + field.default.as_ref().map(postgres_default) + } +} + +fn postgres_default(value: &crate::model::ModelValue) -> String { + match value { + crate::model::ModelValue::Bool(value) => { + if *value { + "TRUE".into() + } else { + "FALSE".into() + } + } + value => sql_default(value), + } +} + +fn sqlite_default(value: &crate::model::ModelValue, ty: &crate::model::ModelFieldType) -> String { + match (value, ty) { + ( + crate::model::ModelValue::Number(value), + crate::model::ModelFieldType::Decimal { precision, scale }, + ) => sql_text( + &dever_runtime::orm::decimal_storage(value, *precision, *scale) + .expect("checked Decimal default"), + ), + (crate::model::ModelValue::Text(value), crate::model::ModelFieldType::Uuid) => { + let bytes = dever_runtime::orm::Uuid::parse(value) + .expect("checked Uuid default") + .into_bytes() + .into_iter() + .map(|byte| format!("{byte:02x}")) + .collect::<String>(); + format!("X'{bytes}'") + } + _ => sql_default(value), + } +} + +fn sql_default(value: &crate::model::ModelValue) -> String { + use crate::model::ModelValue; + match value { + ModelValue::Null => "NULL".into(), + ModelValue::Bool(value) => i64::from(*value).to_string(), + ModelValue::Number(value) => value.clone(), + ModelValue::Text(value) => sql_text(value), + ModelValue::Choice(value) => sql_text(value.rsplit('.').next().unwrap()), + } +} + +fn seed_value(value: &crate::model::ModelValue, ty: &crate::model::ModelFieldType) -> String { + use crate::model::{ModelFieldType, ModelValue}; + match value { + ModelValue::Null => "dever_runtime::orm::Value::Null".into(), + ModelValue::Bool(value) => format!("dever_runtime::orm::Value::Bool({value})"), + ModelValue::Number(value) if matches!(ty, ModelFieldType::Float) => { + format!("dever_runtime::orm::Value::Float({value}f64)") + } + ModelValue::Number(value) if matches!(ty, ModelFieldType::Decimal { .. }) => { + let ModelFieldType::Decimal { precision, scale } = ty else { + unreachable!() + }; + let value = dever_runtime::orm::decimal_storage(value, *precision, *scale) + .expect("checked Decimal Seed value"); + format!( + "dever_runtime::orm::Value::Decimal(String::from({}))", + rust_string(&value) + ) + } + ModelValue::Number(value) => { + format!("dever_runtime::orm::Value::Int({value}i64)") + } + ModelValue::Text(value) if matches!(ty, ModelFieldType::Uuid) => { + let bytes = dever_runtime::orm::Uuid::parse(value) + .expect("checked Uuid Seed value") + .into_bytes() + .into_iter() + .map(|byte| format!("{byte}u8")) + .collect::<Vec<_>>() + .join(", "); + format!( + "dever_runtime::orm::Value::Uuid(dever_runtime::orm::Uuid::from_bytes([{bytes}]))" + ) + } + ModelValue::Text(value) => format!( + "dever_runtime::orm::Value::Text(String::from({}))", + rust_string(value) + ), + ModelValue::Choice(value) => format!( + "dever_runtime::orm::Value::Text(String::from({}))", + rust_string(value.rsplit('.').next().unwrap()) + ), + } +} + +fn sql_text(value: &str) -> String { + format!("'{}'", value.replace('\'', "''")) +} + +pub(super) fn database_sql(sqlite: &str) -> String { + database_sql_pair(sqlite, sqlite) +} + +fn database_sql_pair(sqlite: &str, postgres: &str) -> String { + let (sqlite, postgres) = sql_dialects(sqlite, postgres); + format!( + "dever_runtime::database::Sql {{ sqlite: {}, postgres: {} }}", + rust_string(&sqlite), + rust_string(&postgres) + ) +} + +/// Both native emitters consume the same checked dialect substitutions. +pub(crate) fn sql_dialects(sqlite: &str, postgres: &str) -> (String, String) { + let postgres = postgres + .replace("__DEVER_CONTAINS__", "strpos") + .replace("__DEVER_NOT_EQUAL__", "IS DISTINCT FROM") + .replace("__DEVER_EQUAL__", "IS NOT DISTINCT FROM"); + let sqlite = sqlite + .replace("__DEVER_CONTAINS__", "instr") + .replace("__DEVER_EQUAL__", "IS") + .replace("__DEVER_NOT_EQUAL__", "IS NOT"); + ( + typed_placeholders(&sqlite, false), + typed_placeholders(&postgres, true), + ) +} + +pub(crate) fn cursor_condition( + model: &ModelSchema, + order: &[crate::hir::ModelOrder], + first_parameter: usize, + table: Option<&str>, +) -> String { + let placeholders = order + .iter() + .enumerate() + .map(|(index, order)| parameter_sql(first_parameter + index, &model.fields[order.field].ty)) + .collect::<Vec<_>>(); + let branches = order + .iter() + .enumerate() + .map(|(index, current)| { + let mut terms = order[..index] + .iter() + .enumerate() + .map(|(previous, prior)| { + format!( + "{} __DEVER_EQUAL__ {}", + qualified(table, &model.fields[prior.field].name), + placeholders[previous] + ) + }) + .collect::<Vec<_>>(); + terms.push(format!( + "{} {} {}", + qualified(table, &model.fields[current.field].name), + if current.descending { "<" } else { ">" }, + placeholders[index] + )); + format!("({})", terms.join(" AND ")) + }) + .collect::<Vec<_>>() + .join(" OR "); + format!( + "(CAST({} AS {}) IS NULL OR ({branches}))", + placeholders[0], + postgres_type(&model.fields[order[0].field].ty), + ) +} + +fn typed_placeholders(sql: &str, postgres: bool) -> String { + let mut output = String::with_capacity(sql.len()); + let mut characters = sql.chars().peekable(); + while let Some(character) = characters.next() { + if character != '?' || !characters.peek().is_some_and(char::is_ascii_digit) { + output.push(character); + continue; + } + output.push(if postgres { '$' } else { '?' }); + while characters.peek().is_some_and(char::is_ascii_digit) { + output.push(characters.next().expect("peeked placeholder digit")); + } + let marker = characters.next_if(|marker| matches!(marker, 'D' | 'U')); + if postgres { + match marker { + Some('D') => output.push_str("::text::numeric"), + Some('U') => output.push_str("::uuid"), + _ => {} + } + } + } + output +} + +pub(crate) fn parameter_sql(index: usize, ty: &ModelFieldType) -> String { + let marker = match ty { + ModelFieldType::Decimal { .. } => "D", + ModelFieldType::Uuid => "U", + _ => "", + }; + format!("?{index}{marker}") +} + +pub(crate) fn update_assignments( + model: &ModelSchema, + fields: impl IntoIterator<Item = usize>, + first_parameter: usize, +) -> String { + fields + .into_iter() + .enumerate() + .map(|(offset, field)| { + format!( + "{} = {}", + quoted(&model.fields[field].name), + parameter_sql(first_parameter + offset, &model.fields[field].ty) + ) + }) + .collect::<Vec<_>>() + .join(", ") +} + +pub(crate) fn select_columns(model: &ModelSchema) -> String { + select_columns_qualified(model, None) +} + +fn select_columns_qualified(model: &ModelSchema, table: Option<&str>) -> String { + model + .fields + .iter() + .map(|field| qualified(table, &field.name)) + .collect::<Vec<_>>() + .join(", ") +} + +pub(crate) fn has_to_one(model: &ModelSchema, query: &QueryPlan) -> bool { + query + .relations + .iter() + .any(|index| model.relations[*index].kind == ModelRelationKind::ToOne) +} + +pub(crate) fn select_source( + model: &ModelSchema, + query: &QueryPlan, + models: &[ModelSchema], +) -> (String, String) { + if !has_to_one(model, query) { + return (select_columns(model), quoted(&model.table)); + } + let parent = "_dever_parent"; + let mut projection = vec![select_columns_qualified(model, Some(parent))]; + let mut source = format!("{} AS {}", quoted(&model.table), quoted(parent)); + let mut join_index = 0; + for relation_index in &query.relations { + let relation = &model.relations[*relation_index]; + if relation.kind != ModelRelationKind::ToOne { + continue; + } + let target = models + .iter() + .find(|candidate| candidate.package == relation.model) + .expect("validated relation Model"); + let alias = format!("_dever_relation_{join_index}"); + let nullable = model + .fields + .iter() + .find(|field| field.name == relation.field) + .expect("validated to-one foreign key") + .nullable; + write!( + source, + " {} JOIN {} AS {} ON {} = {}", + if nullable { "LEFT" } else { "INNER" }, + quoted(&target.table), + quoted(&alias), + qualified(Some(parent), &relation.field), + qualified(Some(&alias), "id"), + ) + .expect("string formatting"); + projection.push(select_columns_qualified(target, Some(&alias))); + join_index += 1; + } + (projection.join(", "), source) +} + +pub(crate) fn order_sql(model: &ModelSchema, order: &[crate::hir::ModelOrder]) -> String { + order_sql_qualified(model, order, None) +} + +pub(crate) fn order_sql_qualified( + model: &ModelSchema, + order: &[crate::hir::ModelOrder], + table: Option<&str>, +) -> String { + order + .iter() + .map(|order| { + format!( + "{} {}", + qualified(table, &model.fields[order.field].name), + if order.descending { "DESC" } else { "ASC" } + ) + }) + .collect::<Vec<_>>() + .join(", ") +} + +pub(crate) fn condition_sql<'a>( + model: &'a ModelSchema, + condition: Option<&'a ModelCondition>, + parameters: &mut Vec<(&'a Expression, &'a ModelFieldType)>, +) -> String { + condition_sql_qualified(model, condition, parameters, None) +} + +pub(crate) fn condition_sql_qualified<'a>( + model: &'a ModelSchema, + condition: Option<&'a ModelCondition>, + parameters: &mut Vec<(&'a Expression, &'a ModelFieldType)>, + table: Option<&str>, +) -> String { + condition + .map(|condition| { + format!( + " WHERE {}", + condition_expression_qualified(model, condition, parameters, table) + ) + }) + .unwrap_or_default() +} + +fn condition_expression_qualified<'a>( + model: &'a ModelSchema, + condition: &'a ModelCondition, + parameters: &mut Vec<(&'a Expression, &'a ModelFieldType)>, + table: Option<&str>, +) -> String { + match condition { + ModelCondition::Compare { + field, + operator, + value, + } => { + let field_index = *field; + let field = qualified(table, &model.fields[field_index].name); + if matches!( + value.kind, + crate::hir::ExpressionKind::Constant(crate::hir::Constant::Null) + ) { + return format!( + "{field} IS {}NULL", + if *operator == BinaryOperator::NotEqual { + "NOT " + } else { + "" + } + ); + } + let ty = &model.fields[field_index].ty; + parameters.push((value, ty)); + let operator = match operator { + BinaryOperator::Equal => "__DEVER_EQUAL__", + BinaryOperator::NotEqual => "__DEVER_NOT_EQUAL__", + BinaryOperator::Less => "<", + BinaryOperator::LessEqual => "<=", + BinaryOperator::Greater => ">", + BinaryOperator::GreaterEqual => ">=", + _ => unreachable!("checked Model comparison"), + }; + format!("{field} {operator} {}", parameter_sql(parameters.len(), ty)) + } + ModelCondition::In { field, values } => { + let ty = &model.fields[*field].ty; + let placeholders = values + .iter() + .map(|value| { + parameters.push((value, ty)); + parameter_sql(parameters.len(), ty) + }) + .collect::<Vec<_>>() + .join(", "); + format!( + "{} IN ({placeholders})", + qualified(table, &model.fields[*field].name) + ) + } + ModelCondition::Between { + field, + lower, + upper, + } => { + let ty = &model.fields[*field].ty; + parameters.push((lower, ty)); + let lower = parameter_sql(parameters.len(), ty); + parameters.push((upper, ty)); + let upper = parameter_sql(parameters.len(), ty); + format!( + "{} BETWEEN {lower} AND {upper}", + qualified(table, &model.fields[*field].name) + ) + } + ModelCondition::Text { + field, + operation, + value, + } => { + let ty = &model.fields[*field].ty; + parameters.push((value, ty)); + let parameter = parameter_sql(parameters.len(), ty); + let field = qualified(table, &model.fields[*field].name); + match operation { + crate::hir::TextCondition::Contains => { + format!("__DEVER_CONTAINS__({field}, {parameter}) > 0") + } + crate::hir::TextCondition::StartsWith => { + format!("substr({field}, 1, length({parameter})) = {parameter}") + } + crate::hir::TextCondition::EndsWith => format!( + "substr({field}, length({field}) - length({parameter}) + 1) = {parameter}" + ), + } + } + ModelCondition::Not(condition) => { + format!( + "NOT ({})", + condition_expression_qualified(model, condition, parameters, table) + ) + } + ModelCondition::And(left, right) | ModelCondition::Or(left, right) => { + let left = condition_expression_qualified(model, left, parameters, table); + let right = condition_expression_qualified(model, right, parameters, table); + let operator = if matches!(condition, ModelCondition::And(_, _)) { + "AND" + } else { + "OR" + }; + format!("({left} {operator} {right})") + } + } +} + +pub(crate) fn quoted(name: &str) -> String { + format!("\"{}\"", name.replace('"', "\"\"")) +} + +fn qualified(table: Option<&str>, field: &str) -> String { + match table { + Some(table) => format!("{}.{}", quoted(table), quoted(field)), + None => quoted(field), + } +} diff --git a/crates/dever-core/src/native/ports.rs b/crates/dever-core/src/native/ports.rs new file mode 100644 index 0000000..752e491 --- /dev/null +++ b/crates/dever-core/src/native/ports.rs @@ -0,0 +1,368 @@ +use super::*; + +pub(super) fn definitions( + emitter: &Emitter<'_>, + output: &mut String, + instances: &BTreeSet<Specialization>, +) { + for instance in instances { + let id = instance.function; + let function = &emitter.program.functions[id]; + if function.port.is_some() { + writeln!( + output, + "static PORT_{id}: std::sync::OnceLock<usize> = std::sync::OnceLock::new();" + ) + .expect("string formatting"); + } + } + for adapter in &emitter.program.adapters { + let Some(ty) = adapter.setting else { + continue; + }; + if !adapter + .operations + .values() + .any(|id| instances.iter().any(|instance| instance.function == *id)) + { + continue; + } + let schema = crate::wire::Schema::build( + &Type::Named(ty), + &emitter.program.types, + crate::wire::Policy::SettingInput, + ) + .expect("checked setting schema"); + // Type IDs occupy their own range after API route schema indexes. + let codec = emitter.program.api_routes.len() + ty; + output.push_str(&wire::emit(&schema, codec)); + writeln!( + output, + "static ADAPTER_SETTING_{}: std::sync::OnceLock<T{ty}> = std::sync::OnceLock::new();", + adapter.owner + ) + .expect("string formatting"); + } + let external = instances + .iter() + .filter_map(|instance| { + emitter.program.functions[instance.function] + .external + .map(|_| instance.function) + }) + .collect::<BTreeSet<_>>(); + for implementation in external { + external_codecs(emitter, output, implementation); + } +} + +pub(super) fn function( + emitter: &Emitter<'_>, + output: &mut String, + instance: &Specialization, +) -> bool { + let function = &emitter.program.functions[instance.function]; + if function.setting.is_some() { + writeln!(output, "Ok(ADAPTER_SETTING_{}.get().expect(\"Adapter setting initialized before entry\").clone())\n}}", function.owner).expect("string formatting"); + return true; + } + if let Some(external) = function.external { + let adapter = &emitter.program.adapters[external.adapter]; + let contract = emitter.program.functions[external.contract] + .port + .as_ref() + .expect("external Port contract"); + let arguments = (0..function.parameters.len()) + .map(|index| format!("&_p{index}")) + .collect::<Vec<_>>() + .join(", "); + let location = emitter.location(function.span); + writeln!(output, "let payload = external_{0}_inputs({arguments}).map_err(|error| AppError::fault({location}, error))?;", instance.function).expect("string formatting"); + writeln!(output, "match dever_runtime::component::call({}, {}, &payload).await.map_err(|error| AppError::fault({location}, error))? {{", rust_string(&external_key(adapter)), rust_string(&contract.operation)).expect("string formatting"); + writeln!(output, "dever_runtime::component::Reply::Result(payload) => external_{0}_outputs(&payload).map_err(|error| AppError::fault({location}, error)),", instance.function).expect("string formatting"); + writeln!(output, "dever_runtime::component::Reply::Error {{ identity, payload }} => Err(external_{0}_error(&identity, &payload, {location}).map_err(|error| AppError::fault({location}, error))?),\n}}\n}}", instance.function).expect("string formatting"); + return true; + } + let Some(port) = &function.port else { + return false; + }; + if port.implementations.is_empty() { + output.push_str("unreachable!(\"checked Port has an implementation\")\n}\n"); + return true; + } + let arguments = (0..function.parameters.len()) + .map(|index| format!("_p{index}")) + .collect::<Vec<_>>() + .join(", "); + let invoke = |id: usize| { + let target = Specialization { + function: id, + handlers: Vec::new(), + }; + format!( + "{}({arguments}){}", + emitter.names[&target], + if specialize::suspends(emitter.program, &target) { + ".await" + } else { + "" + } + ) + }; + if let [implementation] = port.implementations.as_slice() { + writeln!(output, "{}\n}}", invoke(*implementation)).expect("string formatting"); + } else { + writeln!( + output, + "match *PORT_{}.get().expect(\"Port initialized before entry\") {{", + instance.function + ) + .expect("string formatting"); + for implementation in &port.implementations { + writeln!(output, "{implementation} => {},", invoke(*implementation)) + .expect("string formatting"); + } + output.push_str("_ => unreachable!(\"closed Port binding\"),\n}\n}\n"); + } + true +} + +pub(super) fn initialize( + emitter: &Emitter<'_>, + output: &mut String, + instances: &BTreeSet<Specialization>, + span: Span, +) -> Result<(), String> { + let program = emitter.program; + let ports: BTreeSet<_> = instances + .iter() + .filter_map(|instance| { + program.functions[instance.function] + .port + .as_ref() + .map(|_| program.functions[instance.function].owner) + }) + .collect(); + if ports.is_empty() { + return Ok(()); + } + let identities: BTreeSet<_> = program + .functions + .iter() + .filter_map(|function| { + function + .port + .as_ref() + .map(|port| rust_string(&port.identity)) + }) + .collect(); + let location = emitter.location(span); + let asynchronous = has_external(program, instances); + let closure = if asynchronous { + "|| async {" + } else { + "|| -> Result<(), String> {" + }; + let settings = if asynchronous { + "dever_runtime::component::adapter_settings()" + } else { + "dever_runtime::config::Settings::load_adapter_settings()" + }; + writeln!(output, "let initialize_ports = {closure}\nlet settings = {settings}?;\nsettings.validate_adapter_names(&[{}])?;", identities.into_iter().collect::<Vec<_>>().join(", ")).expect("string formatting"); + for owner in ports { + let adapters: Vec<_> = program + .adapters + .iter() + .filter(|adapter| !adapter.fake && adapter.port_owner == owner) + .collect(); + if adapters.is_empty() { + return Err("reachable Port has no production Adapter".into()); + } + let identity = &adapters[0].identity; + let names = adapters + .iter() + .map(|adapter| rust_string(&adapter.name)) + .collect::<Vec<_>>() + .join(", "); + writeln!(output, "let (selected, setting) = settings.select_adapter({}, &[{names}])?;\nmatch selected {{", rust_string(identity)).expect("string formatting"); + for (index, adapter) in adapters.iter().enumerate() { + writeln!(output, "{index} => {{").expect("string formatting"); + if let Some(ty) = adapter.setting { + let codec = program.api_routes.len() + ty; + writeln!(output, "let value = wire_{codec}_decode(setting.as_deref().ok_or(\"selected Adapter requires setting\")?)?;\nADAPTER_SETTING_{}.set(value).map_err(|_| String::from(\"Adapter setting initialized twice\"))?;", adapter.owner).expect("string formatting"); + } else { + output.push_str("if setting.is_some() { return Err(String::from(\"selected Adapter does not declare setting\")); }\n"); + } + if let crate::hir::AdapterImplementation::External(external) = &adapter.implementation { + let capabilities = external + .capabilities + .iter() + .map(|capability| format!("{}.into()", rust_string(capability))) + .collect::<Vec<_>>() + .join(", "); + let operations = adapter + .operations + .keys() + .map(|port| { + format!( + "{}.into()", + rust_string( + &program.functions[*port] + .port + .as_ref() + .expect("Port operation") + .operation, + ) + ) + }) + .collect::<Vec<_>>() + .join(", "); + let setting = if adapter.setting.is_some() { + "setting" + } else { + "None" + }; + writeln!(output, "dever_runtime::component::start(dever_runtime::component::Definition {{ key: {}.into(), ecosystem: {}.into(), entry: {}.into(), port: {}.into(), adapter: {}.into(), schema: {}.into(), capabilities: vec![{capabilities}], operations: vec![{operations}], setting: {setting}, timeout_ms: 30_000 }}).await?;", rust_string(&external_key(adapter)), rust_string(external.ecosystem.keyword()), rust_string(&external.entry), rust_string(&adapter.identity), rust_string(&adapter.name), rust_string(&external.schema)).expect("string formatting"); + } + for (port, implementation) in &adapter.operations { + if instances.iter().any(|instance| instance.function == *port) { + writeln!(output, "PORT_{port}.set({implementation}).map_err(|_| String::from(\"Port initialized twice\"))?;").expect("string formatting"); + } + } + output.push_str("},\n"); + } + output.push_str("_ => unreachable!(\"validated Adapter selection\"),\n}\n"); + } + writeln!(output, "Ok::<(), String>(())\n}};\ninitialize_ports(){}.map_err(|error| AppError::fault({location}, error))?;", if asynchronous { ".await" } else { "" }).expect("string formatting"); + Ok(()) +} + +pub(super) fn has_external(program: &Program, instances: &BTreeSet<Specialization>) -> bool { + instances + .iter() + .any(|instance| program.functions[instance.function].external.is_some()) +} + +fn external_key(adapter: &crate::hir::Adapter) -> String { + format!("{}:{}", adapter.identity, adapter.name) +} + +fn external_codecs(emitter: &Emitter<'_>, output: &mut String, implementation: usize) { + let function = &emitter.program.functions[implementation]; + let external = function.external.expect("external implementation"); + let crate::hir::AdapterImplementation::External(adapter) = + &emitter.program.adapters[external.adapter].implementation + else { + unreachable!("external implementation Adapter") + }; + let policy = crate::wire::Policy::external(adapter.ecosystem); + let contract = emitter.program.functions[external.contract] + .port + .as_ref() + .expect("Port contract"); + let mut input_schemas = Vec::new(); + for (index, parameter) in function.parameters.iter().enumerate() { + let field = crate::types::Field { + name: contract.input_names[index].clone(), + ty: parameter + .value_type() + .expect("Port value parameter") + .clone(), + private: false, + bounds: Vec::new(), + }; + let schema = crate::wire::Schema::from_field(&field, &emitter.program.types, policy) + .expect("checked external input schema"); + let prefix = format!("external_{implementation}_input_{index}"); + output.push_str(&wire::emit_named(&schema, &prefix)); + input_schemas.push((field, schema, prefix)); + } + let input_parameters = input_schemas + .iter() + .enumerate() + .map(|(index, (field, _, _))| format!("p{index}: &{}", rust_type(&field.ty))) + .collect::<Vec<_>>() + .join(", "); + writeln!(output, "fn external_{implementation}_inputs({input_parameters}) -> Result<dever_runtime::wire::Encoded, String> {{\nlet mut writer = dever_runtime::wire::Encoder::default();\nwriter.begin_object()?;").expect("string formatting"); + for (index, (field, schema, prefix)) in input_schemas.iter().enumerate() { + writeln!( + output, + "writer.key({})?; {prefix}_e{}(p{index}, &mut writer)?;", + rust_string(&field.name), + schema.root + ) + .expect("string formatting"); + } + output.push_str("writer.end()?; writer.finish()\n}\n"); + + let mut output_schemas = Vec::new(); + for (index, field) in function.outputs.iter().enumerate() { + let schema = crate::wire::Schema::from_field(field, &emitter.program.types, policy) + .expect("checked external output schema"); + let prefix = format!("external_{implementation}_output_{index}"); + output.push_str(&wire::emit_named(&schema, &prefix)); + output_schemas.push((field, schema, prefix)); + } + let output_names = function + .outputs + .iter() + .map(|field| rust_string(&field.name)) + .collect::<Vec<_>>() + .join(", "); + writeln!(output, "fn external_{implementation}_outputs(text: &str) -> Result<{}, String> {{\nlet node = dever_runtime::wire::parse(text)?;\nlet fields = node.fields(&[{output_names}])?;\nif fields.len() != {} {{ return Err(\"external Adapter result has missing fields\".into()); }}", rust_type(&output_type(&function.outputs)), function.outputs.len()).expect("string formatting"); + for (index, (field, schema, prefix)) in output_schemas.iter().enumerate() { + writeln!(output, "let value_{index} = {prefix}_d{}(fields.get({}).ok_or(\"external Adapter result has missing fields\")?)?;", schema.root, rust_string(&field.name)).expect("string formatting"); + } + let value = match function.outputs.len() { + 0 => "()".into(), + 1 => "value_0".into(), + count => format!( + "({})", + (0..count) + .map(|index| format!("value_{index}")) + .collect::<Vec<_>>() + .join(", ") + ), + }; + writeln!(output, "Ok({value})\n}}").expect("string formatting"); + + writeln!(output, "fn external_{implementation}_error(identity: &str, text: &str, location: &'static str) -> Result<AppError, String> {{\nmatch identity {{").expect("string formatting"); + for failure in &contract.failures { + let crate::types::Shape::Choice(variants) = &emitter.program.types[failure.ty].shape else { + unreachable!() + }; + let variant = &variants[failure.variant]; + let identity = format!( + "{}.{}", + emitter.program.types[failure.ty].name, variant.name + ); + writeln!(output, "{} => {{\nlet node = dever_runtime::wire::parse(text)?;\nlet fields = node.fields(&[{}])?;\nif fields.len() != {} {{ return Err(\"external Adapter error has missing fields\".into()); }}", rust_string(&identity), variant.fields.iter().map(|field| rust_string(&field.name)).collect::<Vec<_>>().join(", "), variant.fields.len()).expect("string formatting"); + for (index, field) in variant.fields.iter().enumerate() { + let schema = crate::wire::Schema::from_field(field, &emitter.program.types, policy) + .expect("checked external error schema"); + let prefix = format!("external_{implementation}_error_{}_{}", failure.ty, index); + output.push_str(&wire::emit_named(&schema, &prefix)); + writeln!(output, "let field_{index} = {prefix}_d{}(fields.get({}).ok_or(\"external Adapter error has missing fields\")?)?;", schema.root, rust_string(&field.name)).expect("string formatting"); + } + let payload = if variant.fields.is_empty() { + String::new() + } else { + format!( + "({})", + (0..variant.fields.len()) + .map(|index| format!("field_{index}")) + .collect::<Vec<_>>() + .join(", ") + ) + }; + writeln!( + output, + "Ok(AppError::business(AppErrorKind::E{}V{}{payload}, location))\n}},", + failure.ty, failure.variant + ) + .expect("string formatting"); + } + output.push_str( + "_ => Err(\"external Adapter returned an undeclared business error\".into()),\n}\n}\n", + ); +} diff --git a/crates/dever-core/src/native/records.rs b/crates/dever-core/src/native/records.rs new file mode 100644 index 0000000..f1569a5 --- /dev/null +++ b/crates/dever-core/src/native/records.rs @@ -0,0 +1,214 @@ +use std::fmt::Write; + +use crate::types::{DefinitionKind, Shape, Type}; + +use super::{Emitter, rust_string, rust_type}; + +impl Emitter<'_> { + pub(super) fn types(&self, output: &mut String) { + for (id, definition) in self.program.types.iter().enumerate() { + if definition.kind == DefinitionKind::ModelId { + writeln!( + output, + "#[derive(Clone, Debug, Eq, Hash, PartialEq)]\nstruct T{id}(i64);\nimpl Render for T{id} {{ fn render_to(&self, output: &mut String) {{ self.0.render_to(output); }} }}" + ) + .expect("string formatting"); + continue; + } + let equality = if Type::Named(id).comparable(&self.program.types) { + ", PartialEq" + } else { + "" + }; + let key = matches!(&definition.shape, Shape::Choice(variants) if variants.iter().all(|variant| variant.fields.is_empty())); + writeln!( + output, + "#[derive(Clone, Debug{equality}{})]", + if key { ", Eq, Hash" } else { "" } + ) + .expect("string formatting"); + match &definition.shape { + Shape::Record(fields) => { + writeln!(output, "struct T{id} {{").expect("string formatting"); + for (index, field) in fields.iter().enumerate() { + writeln!(output, "f{index}: {},", rust_type(&field.ty)) + .expect("string formatting"); + } + output.push_str("}\n"); + if !Type::Named(id).observable(&self.program.types) { + continue; + } + writeln!( + output, + "impl Render for T{id} {{ fn render_to(&self, output: &mut String) {{" + ) + .expect("string formatting"); + writeln!( + output, + "output.push_str({}); output.push_str(\" {{ \");", + rust_string(&definition.name) + ) + .expect("string formatting"); + for (index, field) in fields.iter().enumerate() { + if index != 0 { + output.push_str("output.push_str(\", \");\n"); + } + writeln!( + output, + "output.push_str({}); output.push_str(\" = \"); self.f{index}.render_to(output);", + rust_string(&field.name) + ) + .expect("string formatting"); + } + output.push_str("output.push_str(\" }\");\n} }\n"); + } + Shape::Choice(variants) => { + writeln!(output, "enum T{id} {{").expect("string formatting"); + for (index, variant) in variants.iter().enumerate() { + let payload = if variant.fields.is_empty() { + String::new() + } else { + format!( + "({})", + variant + .fields + .iter() + .map(|field| rust_type(&field.ty)) + .collect::<Vec<_>>() + .join(", ") + ) + }; + writeln!(output, "V{index}{payload},").expect("string formatting"); + } + output.push_str("}\n"); + if !Type::Named(id).observable(&self.program.types) { + continue; + } + writeln!( + output, + "impl Render for T{id} {{ fn render_to(&self, output: &mut String) {{ match self {{" + ) + .expect("string formatting"); + for (index, variant) in variants.iter().enumerate() { + let name = rust_string(&format!("{}.{}", definition.name, variant.name)); + if variant.fields.is_empty() { + writeln!(output, "Self::V{index} => output.push_str({name}),") + .expect("string formatting"); + } else { + let bindings = (0..variant.fields.len()) + .map(|index| format!("field{index}")) + .collect::<Vec<_>>(); + write!( + output, + "Self::V{index}({}) => {{ output.push_str({name}); output.push('(');", + bindings.join(", ") + ) + .expect("string formatting"); + for (field, binding) in bindings.iter().enumerate() { + if field != 0 { + output.push_str("output.push_str(\", \");"); + } + write!(output, "{binding}.render_to(output);") + .expect("string formatting"); + } + output.push_str("output.push(')'); },\n"); + } + } + output.push_str("} } }\n"); + } + } + } + } + + pub(super) fn errors(&self, output: &mut String) { + let failures = self + .program + .failures + .iter() + .flat_map(|failures| failures.iter()) + .copied() + .chain(crate::capture::database_errors(&self.program.types)) + .collect::<std::collections::BTreeSet<_>>(); + output.push_str("#[derive(Debug)]\nenum AppErrorKind {\n"); + for failure in &failures { + let Shape::Choice(variants) = &self.program.types[failure.ty].shape else { + unreachable!("failure type is a choice") + }; + let variant = &variants[failure.variant]; + if variant.fields.is_empty() { + writeln!(output, "E{}V{},", failure.ty, failure.variant) + .expect("string formatting"); + } else { + writeln!( + output, + "E{}V{}({}),", + failure.ty, + failure.variant, + variant + .fields + .iter() + .map(|field| rust_type(&field.ty)) + .collect::<Vec<_>>() + .join(", ") + ) + .expect("string formatting"); + } + } + output.push_str("Fault(String),\n}\n#[derive(Debug)]\nstruct AppError { kind: AppErrorKind, locations: Vec<&'static str>, causes: Vec<String> }\n"); + output.push_str("impl AppError {\nfn business(kind: AppErrorKind, location: &'static str) -> Self { Self { kind, locations: vec![location], causes: Vec::new() } }\nfn fault(location: &'static str, error: impl std::fmt::Display) -> Self { Self { kind: AppErrorKind::Fault(error.to_string()), locations: vec![location], causes: Vec::new() } }\nfn at(mut self, location: &'static str) -> Self { self.locations.push(location); self }\nfn cause(mut self, error: impl std::fmt::Display) -> Self { self.causes.push(error.to_string()); self }\n}\n"); + output.push_str("impl From<String> for AppError { fn from(message: String) -> Self { Self { kind: AppErrorKind::Fault(message), locations: Vec::new(), causes: Vec::new() } } }\n"); + output.push_str("impl AppError { fn database(location: &'static str, error: dever_runtime::orm::Error) -> Self { let kind = match error.kind() {\n"); + for failure in crate::capture::database_errors(&self.program.types) { + let Shape::Choice(variants) = &self.program.types[failure.ty].shape else { + unreachable!("database error choice") + }; + writeln!( + output, + "dever_runtime::orm::ErrorKind::{} => AppErrorKind::E{}V{}(error.to_string()),", + variants[failure.variant].name, failure.ty, failure.variant + ) + .expect("string formatting"); + } + output.push_str("}; Self::business(kind, location) } }\n"); + output.push_str("impl std::fmt::Display for AppError { fn fmt(&self, output: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { for location in self.locations.iter().rev() { write!(output, \"{location}: \")?; } match &self.kind {\n"); + for failure in &failures { + let Shape::Choice(variants) = &self.program.types[failure.ty].shape else { + unreachable!("failure type is a choice") + }; + let variant = &variants[failure.variant]; + let name = rust_string(&format!( + "{}.{}", + self.program.types[failure.ty].name, variant.name + )); + if variant.fields.is_empty() { + writeln!( + output, + "AppErrorKind::E{}V{} => output.write_str({name}),", + failure.ty, failure.variant + ) + .expect("string formatting"); + } else { + let bindings = (0..variant.fields.len()) + .map(|index| format!("field{index}")) + .collect::<Vec<_>>(); + write!( + output, + "AppErrorKind::E{}V{}({}) => {{ let mut rendered = String::from({name}); rendered.push('(');", + failure.ty, + failure.variant, + bindings.join(", ") + ) + .expect("string formatting"); + for (index, binding) in bindings.iter().enumerate() { + if index != 0 { + output.push_str("rendered.push_str(\", \");"); + } + write!(output, "{binding}.render_to(&mut rendered);") + .expect("string formatting"); + } + output.push_str("rendered.push(')'); output.write_str(&rendered) },\n"); + } + } + output.push_str("AppErrorKind::Fault(message) => output.write_str(message),\n}?; for cause in &self.causes { write!(output, \"; caused by: {cause}\")?; } Ok(()) } }\nimpl std::error::Error for AppError {}\n"); + } +} diff --git a/crates/dever-core/src/native/rest.rs b/crates/dever-core/src/native/rest.rs new file mode 100644 index 0000000..32d6ae1 --- /dev/null +++ b/crates/dever-core/src/native/rest.rs @@ -0,0 +1,491 @@ +use std::fmt::Write; + +use crate::hir::{ApiRest, Program}; +use crate::model::{ + ModelFieldType, ModelSchema, RestContract, RestOwnerSource, RestValueSource, RestWriteField, +}; +use crate::types::{Shape, Type}; +use crate::wire::{Policy, Schema}; + +use super::orm::{database_sql, parameter_sql, quoted, select_columns}; +use super::rust_string; + +pub(super) fn emit_helpers(program: &Program, output: &mut String) { + let mut emitted = std::collections::BTreeSet::new(); + for route in &program.api_rest { + let model = &program.models[route.model]; + if !emitted.insert(model.record) { + continue; + } + let contract = &route.contract; + let Shape::Record(fields) = &program.types[model.record].shape else { + unreachable!() + }; + for &index in &contract.read { + let schema = Schema::build(&fields[index].ty, &program.types, Policy::Output) + .expect("checked REST output field"); + output.push_str(&super::wire::emit_named( + &schema, + &format!("rest_{}_f{index}_out", model.record), + )); + } + let inputs = contract + .create + .iter() + .chain(&contract.replace) + .filter_map(|write| match write.source { + RestValueSource::Input + | RestValueSource::Binding { + uses_input: true, .. + } => Some(write.field), + RestValueSource::Binding { + uses_input: false, .. + } => None, + }) + .chain(contract.search.iter().map(|search| search.field)) + .collect::<std::collections::BTreeSet<_>>(); + for index in inputs { + let schema = Schema::build(&fields[index].ty, &program.types, Policy::ApiInput) + .expect("checked REST input field"); + output.push_str(&super::wire::emit_named( + &schema, + &format!("rest_{}_f{index}_in", model.record), + )); + } + writeln!( + output, + "fn rest_{}_encode(value: &T{}) -> Result<dever_runtime::wire::Encoded, String> {{", + model.record, model.record + ) + .expect("format"); + output.push_str( + "let mut writer = dever_runtime::wire::Encoder::default(); writer.begin_object()?;\n", + ); + for &index in &contract.read { + writeln!(output, "writer.key({})?; writer.json(rest_{}_f{index}_out_encode(&value.f{index})?.as_str())?;", rust_string(&model.fields[index].name), model.record).expect("format"); + } + output.push_str("writer.end()?; writer.finish()\n}\n"); + } +} + +/// Emit a REST match arm body. The caller owns routing and supplies the checked method/detail pair. +pub(super) fn emit_handler( + emitter: &super::Emitter<'_>, + route: &ApiRest, + method: &str, + detail: bool, + output: &mut String, +) { + let program = emitter.program; + let model = &program.models[route.model]; + let contract = &route.contract; + let table = quoted(&model.table); + let id_condition = format!("{} = ?1", quoted("id")); + if detail { + writeln!(output, "let rest_id = match path.strip_prefix({}).and_then(|id| id.parse::<i64>().ok()).filter(|id| *id > 0) {{ Some(id) => id, None => return Ok(dever_runtime::api::not_found()) }};", rust_string(&format!("{}/", route.path))).expect("format"); + } + match (method, detail) { + ("GET", false) => { + output.push_str("let mut inputs = match dever_runtime::api::Inputs::from_request(&request) { Ok(inputs) => inputs, Err(error) => return Ok(dever_runtime::api::invalid_input(error)) };\n"); + output.push_str("let page = match inputs.optional_int(\"page\") { Ok(value) => value.unwrap_or(1), Err(error) => return Ok(dever_runtime::api::invalid_input(error)) };\n"); + output.push_str("let size = match inputs.optional_int(\"size\") { Ok(value) => value.unwrap_or(20), Err(error) => return Ok(dever_runtime::api::invalid_input(error)) };\n"); + parse_search(emitter, model, contract, output); + finish_inputs(output); + owner_value(output, contract); + database(output, model); + output.push_str("let (page, size, offset) = match dever_runtime::orm::pagination(page, size, database.max_page_size()) { Ok(value) => value, Err(error) => return Ok(dever_runtime::api::invalid_input(dever_runtime::api::InputError(error.to_string()))) };\n"); + let (owner_filter, count_parameters, page_parameters, limit_slot) = match contract.owner + { + Some((index, _)) => ( + format!(" WHERE {} = ?1", quoted(&model.fields[index].name)), + "vec![dever_runtime::orm::Value::Int(rest_owner_id)]", + "vec![dever_runtime::orm::Value::Int(rest_owner_id), dever_runtime::orm::Value::Int(size), dever_runtime::orm::Value::Int(offset)]", + 2, + ), + None => ( + String::new(), + "vec![]", + "vec![dever_runtime::orm::Value::Int(size), dever_runtime::orm::Value::Int(offset)]", + 1, + ), + }; + let count = format!("SELECT COUNT(*) FROM {table}{owner_filter}"); + let list = format!( + "SELECT {} FROM {table}{owner_filter} ORDER BY {} DESC LIMIT ?{limit_slot} OFFSET ?{}", + select_columns(model), + quoted("id"), + limit_slot + 1 + ); + if let Some(search) = &contract.search { + let column = quoted(&model.fields[search.field].name); + let search_slot = usize::from(contract.owner.is_some()) + 1; + let search_parameter = parameter_sql(search_slot, &model.fields[search.field].ty); + let search_filter = if contract.owner.is_some() { + format!("{owner_filter} AND {column} = {search_parameter}") + } else { + format!(" WHERE {column} = {search_parameter}") + }; + let search_count = format!("SELECT COUNT(*) FROM {table}{search_filter}"); + let search_limit = search_slot + 1; + let search_list = format!( + "SELECT {} FROM {table}{search_filter} ORDER BY {} DESC LIMIT ?{search_limit} OFFSET ?{}", + select_columns(model), + quoted("id"), + search_limit + 1 + ); + let search_count_parameters = if contract.owner.is_some() { + "vec![dever_runtime::orm::Value::Int(rest_owner_id), rest_search_parameter.clone()]" + } else { + "vec![rest_search_parameter.clone()]" + }; + let search_page_parameters = if contract.owner.is_some() { + "vec![dever_runtime::orm::Value::Int(rest_owner_id), rest_search_parameter, dever_runtime::orm::Value::Int(size), dever_runtime::orm::Value::Int(offset)]" + } else { + "vec![rest_search_parameter, dever_runtime::orm::Value::Int(size), dever_runtime::orm::Value::Int(offset)]" + }; + writeln!(output, "let (count_sql, list_sql, count_parameters, page_parameters) = if let Some(rest_search_parameter) = rest_search_parameter {{ ({}, {}, {search_count_parameters}, {search_page_parameters}) }} else {{ ({}, {}, {count_parameters}, {page_parameters}) }};", database_sql(&search_count), database_sql(&search_list), database_sql(&count), database_sql(&list)).expect("format"); + output.push_str("let count_rows = match database.query(count_sql, count_parameters).await { Ok(rows) => rows, Err(error) => return Ok(rest_database_error(error)) };\n"); + } else { + writeln!(output, "let count_rows = match database.query({}, {count_parameters}).await {{ Ok(rows) => rows, Err(error) => return Ok(rest_database_error(error)) }};", database_sql(&count)).expect("format"); + } + output.push_str("let total = match count_rows.into_iter().next().ok_or_else(|| dever_runtime::orm::Error::invalid_data(\"REST count returned no row\")).and_then(dever_runtime::orm::scalar).and_then(dever_runtime::orm::int) { Ok(value) => value, Err(error) => return Ok(rest_database_error(error)) };\n"); + if contract.search.is_some() { + output.push_str("let rows = match database.query(list_sql, page_parameters).await { Ok(rows) => rows, Err(error) => return Ok(rest_database_error(error)) };\n"); + } else { + writeln!(output, "let rows = match database.query({}, {page_parameters}).await {{ Ok(rows) => rows, Err(error) => return Ok(rest_database_error(error)) }};", database_sql(&list)).expect("format"); + } + writeln!(output, "let mut writer = dever_runtime::wire::Encoder::default(); let value: Result<_, String> = (|| {{ writer.begin_object()?; writer.key(\"items\")?; writer.begin_array()?; for row in rows {{ let model = decode_model_{}(row).map_err(|error| error.to_string())?; writer.json(rest_{}_encode(&model)?.as_str())?; }} writer.end()?; writer.key(\"page\")?; writer.int(page)?; writer.key(\"size\")?; writer.int(size)?; writer.key(\"total\")?; writer.int(total)?; writer.end()?; writer.finish() }})();", model.record, model.record).expect("format"); + success_value(output); + } + ("GET", true) => { + output.push_str("let inputs = match dever_runtime::api::Inputs::from_request(&request) { Ok(inputs) => inputs, Err(error) => return Ok(dever_runtime::api::invalid_input(error)) };\n"); + finish_inputs(output); + owner_value(output, contract); + database(output, model); + let (owner_filter, parameters) = owner_filter(model, contract, 2); + let select = format!( + "SELECT {} FROM {table} WHERE {id_condition}{owner_filter} LIMIT 1", + select_columns(model) + ); + writeln!(output, "let rows = match database.query({}, vec![dever_runtime::orm::Value::Int(rest_id){parameters}]).await {{ Ok(rows) => rows, Err(error) => return Ok(rest_database_error(error)) }};", database_sql(&select)).expect("format"); + output.push_str("let Some(row) = rows.into_iter().next() else { return Ok(dever_runtime::api::not_found()) };\n"); + encode_row(output, model, false); + } + ("POST", false) => { + parse_fields(emitter, model, &contract.create, output); + owner_value(output, contract); + database(output, model); + let generated = model + .fields + .iter() + .enumerate() + .filter(|(_, field)| field.generated && matches!(field.ty, ModelFieldType::Uuid)) + .map(|(index, _)| index) + .collect::<Vec<_>>(); + let mut write_fields = contract + .create + .iter() + .map(|write| write.field) + .collect::<Vec<_>>(); + if let Some((owner, _)) = contract.owner { + write_fields.push(owner); + } + write_fields.extend(generated.iter().copied()); + let columns = write_fields + .iter() + .map(|&index| quoted(&model.fields[index].name)) + .collect::<Vec<_>>(); + let sql = if columns.is_empty() { + format!( + "INSERT INTO {table} DEFAULT VALUES RETURNING {}", + select_columns(model) + ) + } else { + let values = write_fields + .iter() + .enumerate() + .map(|(slot, &index)| parameter_sql(slot + 1, &model.fields[index].ty)) + .collect::<Vec<_>>(); + format!( + "INSERT INTO {table} ({}) VALUES ({}) RETURNING {}", + columns.join(", "), + values.join(", "), + select_columns(model) + ) + }; + let mut parameters = values(program, model, &contract.create, output); + if contract.owner.is_some() { + parameters.push_str("dever_runtime::orm::Value::Int(rest_owner_id), "); + } + for index in generated { + writeln!(output, "let generated_{index} = match dever_runtime::orm::Uuid::new_v7() {{ Ok(value) => value, Err(error) => return Ok(rest_database_error(error)) }};").expect("format"); + write!( + parameters, + "dever_runtime::orm::Value::Uuid(generated_{index}), " + ) + .expect("format"); + } + begin_transaction(output); + writeln!(output, "let rows = match executor.query({}, vec![{parameters}]).await {{ Ok(rows) => rows, Err(error) => {{ drop(executor); rest_rollback(transaction).await; return Ok(rest_database_error(error)); }} }};", database_sql(&sql)).expect("format"); + output.push_str("let Some(row) = rows.into_iter().next() else { drop(executor); rest_rollback(transaction).await; return Ok(dever_runtime::api::internal_error()) };\n"); + encode_row(output, model, true); + } + ("PUT", true) => { + parse_fields(emitter, model, &contract.replace, output); + owner_value(output, contract); + database(output, model); + let assignments = super::orm::update_assignments( + model, + contract.replace.iter().map(|write| write.field), + 1, + ); + let id_slot = contract.replace.len() + 1; + let (owner_filter, owner_parameters) = owner_filter(model, contract, id_slot + 1); + let sql = format!( + "UPDATE {table} SET {assignments} WHERE {} = ?{id_slot}{owner_filter} RETURNING {}", + quoted("id"), + select_columns(model) + ); + let parameters = values(program, model, &contract.replace, output); + begin_transaction(output); + writeln!(output, "let rows = match executor.query({}, vec![{parameters}dever_runtime::orm::Value::Int(rest_id){owner_parameters}]).await {{ Ok(rows) => rows, Err(error) => {{ drop(executor); rest_rollback(transaction).await; return Ok(rest_database_error(error)); }} }};", database_sql(&sql)).expect("format"); + output.push_str("let Some(row) = rows.into_iter().next() else { drop(executor); rest_rollback(transaction).await; return Ok(dever_runtime::api::not_found()) };\n"); + encode_row(output, model, true); + } + ("DELETE", true) => { + output.push_str("let inputs = match dever_runtime::api::Inputs::from_request(&request) { Ok(inputs) => inputs, Err(error) => return Ok(dever_runtime::api::invalid_input(error)) };\n"); + finish_inputs(output); + owner_value(output, contract); + database(output, model); + let (owner_filter, owner_parameters) = owner_filter(model, contract, 2); + let sql = format!("DELETE FROM {table} WHERE {id_condition}{owner_filter}"); + begin_transaction(output); + writeln!(output, "let affected = match executor.execute({}, vec![dever_runtime::orm::Value::Int(rest_id){owner_parameters}]).await {{ Ok(affected) => affected, Err(error) => {{ drop(executor); rest_rollback(transaction).await; return Ok(rest_database_error(error)); }} }};", database_sql(&sql)).expect("format"); + output.push_str("drop(executor); if affected == 0 { rest_rollback(transaction).await; return Ok(dever_runtime::api::not_found()); } if let Err(error) = transaction.commit().await { return Ok(rest_database_error(error)); } Ok(dever_runtime::api::success(dever_runtime::wire::Encoded::null()))\n"); + } + _ => unreachable!("checked REST route"), + } +} + +pub(super) fn emit_database_error(output: &mut String) { + output.push_str("fn rest_database_error(error: dever_runtime::orm::Error) -> dever_runtime::http::Response { dever_runtime::log::error(\"REST database operation failed\", Vec::new()); if error.kind() == dever_runtime::orm::ErrorKind::NotFound { dever_runtime::api::not_found() } else { dever_runtime::api::internal_error() } }\n"); + output.push_str("async fn rest_rollback(transaction: dever_runtime::database::Transaction) { if transaction.rollback().await.is_err() { dever_runtime::log::error(\"REST rollback failed\", Vec::new()); } }\n"); +} + +fn parse_fields( + emitter: &super::Emitter<'_>, + model: &ModelSchema, + writes: &[RestWriteField], + output: &mut String, +) { + let program = emitter.program; + output.push_str("let mut inputs = match dever_runtime::api::Inputs::from_request(&request) { Ok(inputs) => inputs, Err(error) => return Ok(dever_runtime::api::invalid_input(error)) };\n"); + let Shape::Record(fields) = &program.types[model.record].shape else { + unreachable!() + }; + for write in writes { + let index = write.field; + let reads_input = matches!( + write.source, + RestValueSource::Input + | RestValueSource::Binding { + uses_input: true, + .. + } + ); + if !reads_input { + continue; + } + let name = rust_string(&model.fields[index].name); + let prefix = format!("rest_{}_f{index}_in", model.record); + let nullable = matches!(fields[index].ty, Type::Nullable(_)); + let read = if nullable { + "optional_raw_json" + } else { + "raw_json" + }; + let absent = if nullable { + ".unwrap_or_else(|| \"null\".into())" + } else { + "" + }; + writeln!(output, "let raw_{index} = match inputs.{read}({name}) {{ Ok(value) => value{absent}, Err(error) => return Ok(dever_runtime::api::invalid_input(error)) }};").expect("format"); + writeln!(output, "let value_{index} = match {prefix}_decode(&raw_{index}) {{ Ok(value) => value, Err(error) => return Ok(dever_runtime::api::invalid_input(dever_runtime::api::InputError(error))) }};").expect("format"); + if let ModelFieldType::Text { minimum, maximum } = model.fields[index].ty { + let value = if nullable { + format!("value_{index}.as_deref()") + } else { + format!("Some(value_{index}.as_str())") + }; + let maximum = + maximum.map_or_else(|| "None".to_owned(), |value| format!("Some({value})")); + writeln!(output, "if let Err(error) = dever_runtime::api::text_bounds({name}, {value}, {minimum}, {maximum}) {{ return Ok(dever_runtime::api::invalid_input(error)); }}").expect("format"); + } + } + finish_inputs(output); + for write in writes { + let RestValueSource::Binding { + expression, + uses_input, + } = &write.source + else { + continue; + }; + let index = write.field; + if *uses_input { + writeln!(output, "let _v0 = value_{index};").expect("format"); + } + writeln!(output, "let value_{index} = match (|| -> Result<_, AppError> {{ Ok({}) }})() {{ Ok(value) => value, Err(error) => return Ok(api_failure_response(error)) }};", emitter.expression(expression)).expect("format"); + if let ModelFieldType::Text { minimum, maximum } = model.fields[index].ty { + let nullable = matches!(fields[index].ty, Type::Nullable(_)); + let value = if nullable { + format!("value_{index}.as_deref()") + } else { + format!("Some(value_{index}.as_str())") + }; + let maximum = + maximum.map_or_else(|| "None".to_owned(), |value| format!("Some({value})")); + let failure = if *uses_input { + "return Ok(dever_runtime::api::invalid_input(error))" + } else { + "dever_runtime::log::error(\"REST server field binding violated Model bounds\", Vec::new()); return Ok(dever_runtime::api::internal_error())" + }; + writeln!(output, "if let Err(error) = dever_runtime::api::text_bounds({}, {value}, {minimum}, {maximum}) {{ {failure}; }}", rust_string(&model.fields[index].name)).expect("format"); + } + } +} + +fn parse_search( + emitter: &super::Emitter<'_>, + model: &ModelSchema, + contract: &RestContract, + output: &mut String, +) { + let Some(search) = &contract.search else { + return; + }; + let program = emitter.program; + let Shape::Record(fields) = &program.types[model.record].shape else { + unreachable!() + }; + let index = search.field; + let name = rust_string(&model.fields[index].name); + let prefix = format!("rest_{}_f{index}_in", model.record); + writeln!(output, "let raw_search = match inputs.optional_raw_json({name}) {{ Ok(value) => value, Err(error) => return Ok(dever_runtime::api::invalid_input(error)) }};").expect("format"); + writeln!(output, "let rest_search_value = match raw_search {{ Some(raw) => match {prefix}_decode(&raw) {{ Ok(value) => Some(value), Err(error) => return Ok(dever_runtime::api::invalid_input(dever_runtime::api::InputError(error))) }}, None => None }};").expect("format"); + if let ModelFieldType::Text { minimum, maximum } = model.fields[index].ty { + let maximum = maximum.map_or_else(|| "None".to_owned(), |value| format!("Some({value})")); + writeln!(output, "if let Err(error) = dever_runtime::api::text_bounds({name}, rest_search_value.as_deref(), {minimum}, {maximum}) {{ return Ok(dever_runtime::api::invalid_input(error)); }}").expect("format"); + } + writeln!(output, "let rest_search_value = match rest_search_value {{ Some(value) => {{ let _v0 = value; match (|| -> Result<_, AppError> {{ Ok({}) }})() {{ Ok(value) => Some(value), Err(error) => return Ok(api_failure_response(error)) }} }}, None => None }};", emitter.expression(&search.expression)).expect("format"); + if let ModelFieldType::Text { minimum, maximum } = model.fields[index].ty { + let maximum = maximum.map_or_else(|| "None".to_owned(), |value| format!("Some({value})")); + writeln!(output, "if let Err(error) = dever_runtime::api::text_bounds({name}, rest_search_value.as_deref(), {minimum}, {maximum}) {{ return Ok(dever_runtime::api::invalid_input(error)); }}").expect("format"); + } + let value = value_expression(&fields[index].ty, &model.fields[index].ty, "value"); + writeln!(output, "let rest_search_parameter = match rest_search_value {{ Some(value) => match {value} {{ Ok(value) => Some(value), Err(error) => return Ok(dever_runtime::api::invalid_input(dever_runtime::api::InputError(error.to_string()))) }}, None => None }};").expect("format"); +} + +fn values( + program: &Program, + model: &ModelSchema, + writes: &[RestWriteField], + output: &mut String, +) -> String { + let Shape::Record(fields) = &program.types[model.record].shape else { + unreachable!() + }; + writes.iter().map(|write| { + let index = write.field; + let value = value_expression(&fields[index].ty, &model.fields[index].ty, &format!("value_{index}")); + writeln!(output, "let parameter_{index} = match {value} {{ Ok(value) => value, Err(error) => return Ok(dever_runtime::api::invalid_input(dever_runtime::api::InputError(error.to_string()))) }};").expect("format"); + format!("parameter_{index}, ") + }).collect() +} + +fn value_expression(ty: &Type, storage: &ModelFieldType, name: &str) -> String { + let value = "dever_runtime::orm::Value"; + match ty { + Type::Nullable(inner) => format!( + "match {name} {{ Some(value) => {}, None => Ok::<_, dever_runtime::orm::Error>({value}::Null) }}", + value_expression(inner, storage, "value") + ), + Type::Bool => format!("Ok::<_, dever_runtime::orm::Error>({value}::Bool({name}))"), + Type::Int | Type::DateTime | Type::Date | Type::Time | Type::Duration => { + format!("Ok::<_, dever_runtime::orm::Error>({value}::Int({name}))") + } + Type::Float => format!("Ok::<_, dever_runtime::orm::Error>({value}::Float({name}))"), + Type::Text | Type::Json => { + format!("Ok::<_, dever_runtime::orm::Error>({value}::Text({name}))") + } + Type::Uuid => format!("Ok::<_, dever_runtime::orm::Error>({value}::Uuid({name}))"), + Type::Named(_) => format!("Ok::<_, dever_runtime::orm::Error>({value}::Int({name}.0))"), + Type::Decimal => { + let ModelFieldType::Decimal { precision, scale } = storage else { + unreachable!() + }; + format!("dever_runtime::orm::decimal_value({name}, {precision}, {scale})") + } + _ => unreachable!("checked REST wire field"), + } +} + +fn database(output: &mut String, model: &ModelSchema) { + writeln!(output, "let database = match model_database_{}().await {{ Ok(database) => database, Err(error) => return Ok(rest_database_error(error)) }};", model.record).expect("format"); +} + +fn begin_transaction(output: &mut String) { + output.push_str("let transaction = match database.begin().await { Ok(transaction) => transaction, Err(error) => return Ok(rest_database_error(error)) };\n"); + output.push_str("let executor = match dever_runtime::database::Executor::new(database, Some(&transaction)) { Ok(executor) => executor, Err(error) => return Ok(rest_database_error(error)) };\n"); +} + +fn finish_inputs(output: &mut String) { + output.push_str("if let Err(error) = inputs.finish() { return Ok(dever_runtime::api::invalid_input(error)); }\n"); +} + +fn owner_value(output: &mut String, contract: &RestContract) { + let Some((_, source)) = contract.owner else { + return; + }; + let read = match source { + RestOwnerSource::User => "dever_runtime::auth::user_id()", + RestOwnerSource::Tenant => "dever_runtime::auth::tenant_id()", + }; + writeln!( + output, + "let rest_owner_id = match {read} {{ Ok(Some(value)) => value, Ok(None) => return Ok(dever_runtime::api::standard_error(403)), Err(error) => return Ok(dever_runtime::api::auth_error(error)) }};" + ) + .expect("format"); +} + +fn owner_filter( + model: &ModelSchema, + contract: &RestContract, + parameter: usize, +) -> (String, &'static str) { + match contract.owner { + Some((index, _)) => ( + format!(" AND {} = ?{parameter}", quoted(&model.fields[index].name)), + ", dever_runtime::orm::Value::Int(rest_owner_id)", + ), + None => (String::new(), ""), + } +} + +fn encode_row(output: &mut String, model: &ModelSchema, transactional: bool) { + let rollback = if transactional { + "drop(executor); rest_rollback(transaction).await; " + } else { + "" + }; + writeln!(output, "let model = match decode_model_{}(row) {{ Ok(model) => model, Err(error) => {{ {rollback}return Ok(rest_database_error(error)); }} }};", model.record).expect("format"); + writeln!(output, "let value = match rest_{}_encode(&model) {{ Ok(value) => value, Err(_error) => {{ {rollback}return Ok(dever_runtime::api::internal_error()); }} }};", model.record).expect("format"); + if transactional { + output.push_str("drop(executor); if let Err(error) = transaction.commit().await { return Ok(rest_database_error(error)); }\n"); + } + output.push_str("Ok(dever_runtime::api::success(value))\n"); +} + +fn success_value(output: &mut String) { + output.push_str("match value { Ok(value) => Ok(dever_runtime::api::success(value)), Err(_error) => Ok(dever_runtime::api::internal_error()) }\n"); +} diff --git a/crates/dever-core/src/native/sequences.rs b/crates/dever-core/src/native/sequences.rs new file mode 100644 index 0000000..3e8cbdb --- /dev/null +++ b/crates/dever-core/src/native/sequences.rs @@ -0,0 +1,273 @@ +use std::collections::BTreeSet; + +use crate::hir::{ + CallArgument, CallTarget, Clause, CollectionOp, Expression, ExpressionKind as Expr, Projection, + SequenceKind, Statement, +}; +use crate::intrinsic::Intrinsic; +use crate::specialize::{self, Specialization}; +use crate::syntax::{BinaryOperator, UnaryOperator}; +use crate::types::{Parameter, Type}; + +use super::Emitter; + +impl Emitter<'_> { + pub(super) fn inline_sequences(&self, clause: &mut Clause) { + let mut index = 0; + while index + 1 < clause.body.len() { + let Statement::Assign { + slot, + fields, + value, + } = &clause.body[index] + else { + index += 1; + continue; + }; + if !fields.is_empty() || clause.outputs.contains(slot) || !self.fusible(value) { + index += 1; + continue; + } + let next = match &clause.body[index + 1] { + Statement::Assign { value, .. } | Statement::Call(value) => value, + }; + let Expr::Collection { arguments, .. } = &next.kind else { + index += 1; + continue; + }; + if !matches!(arguments[0].kind, Expr::Local(id) if id == *slot) + || !super::liveness::last_uses(clause) + .contains(&(&arguments[0] as *const Expression)) + { + index += 1; + continue; + } + let replacement = value.clone(); + let next = match &mut clause.body[index + 1] { + Statement::Assign { value, .. } | Statement::Call(value) => value, + }; + let Expr::Collection { arguments, .. } = &mut next.kind else { + unreachable!() + }; + arguments[0] = replacement; + clause.body.remove(index); + } + } + + fn fusible(&self, expression: &Expression) -> bool { + let Expr::Collection { + operation, + handler: Some(handler), + arguments, + sequence, + .. + } = &expression.kind + else { + return false; + }; + matches!(operation, CollectionOp::Each | CollectionOp::Filter) + && matches!(sequence, Some(SequenceKind::List | SequenceKind::Bytes)) + && arguments.len() == 1 + && matches!(expression.ty, Type::List(_)) + && expression.ty.movable(&self.program.types) + && arguments[0].ty.movable(&self.program.types) + && self.total_handler(specialize::resolve_handler(*handler, &self.bindings)) + } + + pub(super) fn fused_iterator(&self, expression: &Expression) -> Option<(String, String)> { + if !self.fusible(expression) { + return None; + } + let Expr::Collection { + operation, + handler: Some(handler), + arguments, + sequence, + .. + } = &expression.kind + else { + unreachable!() + }; + let (bindings, iterator) = self.fused_iterator(&arguments[0]).unwrap_or_else(|| { + if let Some([text, separator]) = self.split_arguments(&arguments[0]) { + return ( + format!( + "let pipeline_text = {}; let pipeline_separator = {};", + self.expression(text), + self.expression(separator) + ), + "dever_runtime::text::pieces(&pipeline_text, &pipeline_separator)".into(), + ); + } + ( + format!("let pipeline_source = {};", self.expression(&arguments[0])), + match sequence.expect("checked sequence") { + SequenceKind::List => "pipeline_source.into_values()", + SequenceKind::Bytes => { + "pipeline_source.values().iter().copied().map(i64::from)" + } + SequenceKind::Stream | SequenceKind::AsyncStream | SequenceKind::RowStream => { + unreachable!() + } + } + .into(), + ) + }); + let target = Specialization { + function: specialize::resolve_handler(*handler, &self.bindings), + handlers: Vec::new(), + }; + let name = &self.names[&target]; + let stage = if *operation == CollectionOp::Each { + format!("map(|element| {name}(element).expect(\"checked total handler\"))") + } else { + format!("filter(|element| {name}(element.clone()).expect(\"checked total handler\"))") + }; + Some((bindings, format!("{iterator}.{stage}"))) + } + + // pure 只证明无外部效果；融合还要求所有可达操作都不会产生程序故障。 + fn total_handler(&self, root: usize) -> bool { + let mut pending = vec![root]; + let mut visited = BTreeSet::new(); + while let Some(id) = pending.pop() { + if !visited.insert(id) { + continue; + } + let function = &self.program.functions[id]; + if function.suspends || !self.program.suspension_handlers[id].is_empty() || !self.program.effects[id].is_empty() || !self.program.failures[id].is_empty() || !self.program.failure_handlers[id].is_empty() || function.parameters.iter().any(|parameter| !matches!(parameter, Parameter::Value(ty) if ty.movable(&self.program.types))) { + return false; + } + let mut total = true; + crate::check::visit(function, |expression| { + total &= match &expression.kind { + Expr::Constant(_) + | Expr::Local(_) + | Expr::Field { .. } + | Expr::Record { .. } + | Expr::Variant { .. } + | Expr::List(_) + | Expr::Some(_) + | Expr::Promote(_) => true, + Expr::Unary { operator, .. } => { + *operator == UnaryOperator::Not || expression.ty == Type::Float + } + Expr::Binary { operator, .. } => { + matches!( + operator, + BinaryOperator::Equal + | BinaryOperator::NotEqual + | BinaryOperator::Less + | BinaryOperator::LessEqual + | BinaryOperator::Greater + | BinaryOperator::GreaterEqual + | BinaryOperator::And + | BinaryOperator::Or + ) || matches!(expression.ty, Type::Float | Type::Text) + } + Expr::Collection { operation, .. } => matches!( + operation, + CollectionOp::Length + | CollectionOp::First + | CollectionOp::Get + | CollectionOp::Entries + ), + Expr::Call { + target: CallTarget::Function(callee), + arguments, + } if arguments + .iter() + .all(|argument| matches!(argument, CallArgument::Value(_))) => + { + pending.push(*callee); + true + } + Expr::Intrinsic { operation, .. } => matches!( + operation, + Intrinsic::TextTrim + | Intrinsic::TextLower + | Intrinsic::TextUpper + | Intrinsic::TextCodepoint + | Intrinsic::TextFromCodepoint + | Intrinsic::TextAt + | Intrinsic::TextSlice + | Intrinsic::TextIndexOf + | Intrinsic::TextContains + | Intrinsic::TextStartsWith + | Intrinsic::TextEndsWith + | Intrinsic::TextSplit + | Intrinsic::TextReplace + | Intrinsic::IntToText + | Intrinsic::FloatToText + | Intrinsic::FloatFromInt + ), + _ => false, + }; + }); + if !total { + return false; + } + } + true + } + + /// 普通单表达式转发函数和直接原语共用识别，不依赖标准包或样例名称。 + pub(super) fn split_arguments<'a>( + &self, + expression: &'a Expression, + ) -> Option<[&'a Expression; 2]> { + match &expression.kind { + Expr::Intrinsic { + operation: Intrinsic::TextSplit, + arguments, + .. + } => Some([&arguments[0], &arguments[1]]), + Expr::Call { + target: CallTarget::Function(id), + arguments, + } => { + let [clause] = self.program.functions[*id].clauses.as_slice() else { + return None; + }; + let [ + Statement::Assign { + slot, + fields, + value, + }, + ] = clause.body.as_slice() + else { + return None; + }; + if !fields.is_empty() || clause.outputs != [*slot] { + return None; + } + let Expr::Intrinsic { + operation: Intrinsic::TextSplit, + arguments: forwarded, + .. + } = &value.kind + else { + return None; + }; + let input = |expression: &Expression| { + let Expr::Local(slot) = expression.kind else { + return None; + }; + let binding = clause.bindings.iter().find(|binding| { + binding.slot == slot && matches!(binding.projection, Projection::Whole) + })?; + let CallArgument::Value(value) = &arguments[binding.input] else { + return None; + }; + Some((binding.input, value)) + }; + let (left, text) = input(&forwarded[0])?; + let (right, separator) = input(&forwarded[1])?; + // 不能删除额外参数的副作用，或重排调用方参数求值。 + (arguments.len() == 2 && left == 0 && right == 1).then_some([text, separator]) + } + _ => None, + } + } +} diff --git a/crates/dever-core/src/native/updates.rs b/crates/dever-core/src/native/updates.rs new file mode 100644 index 0000000..669d9de --- /dev/null +++ b/crates/dever-core/src/native/updates.rs @@ -0,0 +1,250 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use crate::hir::{CallArgument, Clause, Expression, ExpressionKind as Expr, Statement}; +use crate::types::{Definition, Shape, Type}; + +/// 更新副本只读取旧值中尚未修改的字段时，旧局部可直接转交给副本。 +/// 在已完成契约检查的 HIR 副本上执行，不改变诊断使用的源码事实。 +pub(super) fn coalesce(clause: &mut Clause, types: &[Definition]) { + for index in 0..clause.body.len() { + let Statement::Assign { + slot: target, + fields, + value, + } = &clause.body[index] + else { + continue; + }; + let Expr::Local(source) = value.kind else { + continue; + }; + let (target, source) = (*target, source); + if target == source || !fields.is_empty() || clause.outputs.contains(&source) { + continue; + } + let Type::Named(id) = value.ty else { continue }; + if !matches!(types[id].shape, Shape::Record(_)) || !value.ty.movable(types) { + continue; + } + let mut changed = Vec::<Vec<usize>>::new(); + let mut safe = true; + for statement in &mut clause.body[index + 1..] { + visit_reads(statement_value(statement), &mut |read| { + if let Some((slot, path)) = place(read) + && slot == source + && (path.is_empty() || changed.iter().any(|write| overlaps(write, &path))) + { + safe = false; + } + }); + if let Statement::Assign { slot, fields, .. } = statement { + if *slot == source { + safe = false; + } + if *slot == target { + changed.push(fields.clone()); + } + } + if !safe { + break; + } + } + if safe { + for statement in &mut clause.body[index + 1..] { + visit_reads(statement_value(statement), &mut |read| { + rename_place(read, source, target); + }); + } + } + } +} + +/// 大记录传给下一步后若只剩标量读取，先保留这些标量，避免为它们复制整个记录。 +pub(super) fn snapshot_scalars(clause: &mut Clause, types: &[Definition]) { + let mut index = 0; + while index < clause.body.len() { + let mut sources = BTreeSet::new(); + visit_reads(statement_value(&mut clause.body[index]), &mut |read| { + if let Expr::Local(slot) = read.kind + && matches!(read.ty, Type::Named(_)) + && read.ty.movable(types) + { + sources.insert(slot); + } + }); + for source in sources { + if clause.outputs.contains(&source) + || matches!(&clause.body[index], Statement::Assign { slot, .. } if *slot == source) + { + continue; + } + let mut snapshots = BTreeMap::new(); + let mut safe = true; + for statement in &mut clause.body[index + 1..] { + if matches!(statement, Statement::Assign { slot, .. } if *slot == source) { + safe = false; + break; + } + visit_reads(statement_value(statement), &mut |read| { + if let Some((slot, path)) = place(read) + && slot == source + { + if !path.is_empty() + && matches!( + read.ty, + Type::Bool | Type::Int | Type::Float | Type::Decimal + ) + { + snapshots.entry(path).or_insert_with(|| read.clone()); + } else { + safe = false; + } + } + }); + if !safe { + break; + } + } + if !safe || snapshots.is_empty() { + continue; + } + let mut slots = BTreeMap::new(); + let mut assignments = Vec::new(); + for (path, value) in snapshots { + let slot = clause.locals.len(); + clause.locals.push(value.ty.clone()); + slots.insert(path, slot); + assignments.push(Statement::Assign { + slot, + fields: Vec::new(), + value, + }); + } + for statement in &mut clause.body[index + 1..] { + visit_reads(statement_value(statement), &mut |read| { + if let Some((slot, path)) = place(read) + && slot == source + { + read.kind = Expr::Local(slots[&path]); + } + }); + } + let added = assignments.len(); + clause.body.splice(index..index, assignments); + index += added; + } + index += 1; + } +} + +fn rename_place(expression: &mut Expression, source: usize, target: usize) { + match &mut expression.kind { + Expr::Local(slot) if *slot == source => *slot = target, + Expr::Field { value, .. } => rename_place(value, source, target), + _ => {} + } +} + +pub(super) fn overlaps(left: &[usize], right: &[usize]) -> bool { + left.starts_with(right) || right.starts_with(left) +} + +pub(crate) fn place(expression: &Expression) -> Option<(usize, Vec<usize>)> { + let mut base = expression; + let mut path = Vec::new(); + while let Expr::Field { value, index } = &base.kind { + path.push(*index); + base = value; + } + let Expr::Local(slot) = base.kind else { + return None; + }; + path.reverse(); + Some((slot, path)) +} + +fn statement_value(statement: &mut Statement) -> &mut Expression { + match statement { + Statement::Assign { value, .. } | Statement::Call(value) => value, + } +} + +fn visit_reads(expression: &mut Expression, apply: &mut impl FnMut(&mut Expression)) { + if place(expression).is_some() { + apply(expression); + return; + } + match &mut expression.kind { + Expr::Local(_) | Expr::Constant(_) | Expr::ChoiceOptions { .. } => {} + Expr::JobEnqueue { arguments, .. } => { + for value in arguments { + visit_reads(value, apply); + } + } + Expr::ModelOperation { operation, .. } => { + operation.visit_mut(&mut |value| visit_reads(value, apply)); + } + Expr::Field { value, .. } + | Expr::Fail(value) + | Expr::AwaitTask(value) + | Expr::StopTask(value) + | Expr::Group(value) + | Expr::AwaitGroup(value) + | Expr::StopGroup(value) + | Expr::ChannelReceive(value) + | Expr::ChannelClose(value) + | Expr::Promote(value) + | Expr::Some(value) + | Expr::Unary { value, .. } => visit_reads(value, apply), + Expr::Call { arguments, .. } + | Expr::CaptureResult { arguments, .. } + | Expr::ParallelCall { arguments, .. } + | Expr::BlockingCall { arguments, .. } => { + for argument in arguments { + if let CallArgument::Value(value) = argument { + visit_reads(value, apply); + } + } + } + Expr::RunCall { + group, arguments, .. + } => { + if let Some(group) = group { + visit_reads(group, apply); + } + for argument in arguments { + if let CallArgument::Value(value) = argument { + visit_reads(value, apply); + } + } + } + Expr::Channel { capacity, .. } => visit_reads(capacity, apply), + Expr::ChannelSend { channel, value } => { + visit_reads(channel, apply); + visit_reads(value, apply); + } + Expr::Intrinsic { arguments, .. } + | Expr::Collection { arguments, .. } + | Expr::Variant { arguments, .. } + | Expr::List(arguments) => { + for argument in arguments { + visit_reads(argument, apply); + } + } + Expr::Record { fields } => { + for (_, value) in fields { + visit_reads(value, apply); + } + } + Expr::Map(entries) => { + for (key, value) in entries { + visit_reads(key, apply); + visit_reads(value, apply); + } + } + Expr::Binary { left, right, .. } => { + visit_reads(left, apply); + visit_reads(right, apply); + } + } +} diff --git a/crates/dever-core/src/native/wire.rs b/crates/dever-core/src/native/wire.rs new file mode 100644 index 0000000..7339bb8 --- /dev/null +++ b/crates/dever-core/src/native/wire.rs @@ -0,0 +1,135 @@ +use std::fmt::Write; + +use super::rust_string; +use crate::wire::{Node, Schema}; + +/// Emit concrete helpers into the checked program's generated Rust module. +/// The index is local to that module; callers cache schemas by fingerprint. +pub fn emit(schema: &Schema, index: usize) -> String { + emit_named(schema, &format!("wire_{index}")) +} + +pub(super) fn emit_named(schema: &Schema, prefix: &str) -> String { + let mut output = String::new(); + let root = schema.root; + let ty = rust_type(schema, root); + writeln!(output, "fn {prefix}_decode(text: &str) -> Result<{ty}, String> {{ let node = dever_runtime::wire::parse(text)?; {prefix}_d{root}(&node) }}").expect("string formatting"); + if !schema.policy.permits_secret() { + writeln!(output, "fn {prefix}_encode(value: &{ty}) -> Result<dever_runtime::wire::Encoded, String> {{ let mut writer = dever_runtime::wire::Encoder::default(); {prefix}_e{root}(value, &mut writer)?; writer.finish() }}").expect("string formatting"); + } + for (id, node) in schema.nodes.iter().enumerate() { + let ty = rust_type(schema, id); + writeln!(output, "fn {prefix}_d{id}(node: &dever_runtime::wire::Node<'_>) -> Result<{ty}, String> {{ {} }}", decode(schema, node, prefix)).expect("string formatting"); + if !schema.policy.permits_secret() { + writeln!(output, "fn {prefix}_e{id}(value: &{ty}, writer: &mut dever_runtime::wire::Encoder) -> Result<(), String> {{ {} }}", encode(node, prefix)).expect("string formatting"); + } + } + output +} + +fn rust_type(schema: &Schema, id: usize) -> String { + match &schema.nodes[id] { + Node::Bytes => "dever_runtime::bytes::Bytes".into(), + Node::Bool => "bool".into(), + Node::Int | Node::DateTime | Node::Date | Node::Time | Node::Duration => "i64".into(), + Node::Float => "f64".into(), + Node::Decimal => "dever_runtime::number::DecimalValue".into(), + Node::Text | Node::Json => "String".into(), + Node::Id => "dever_runtime::Id".into(), + Node::Uuid => "dever_runtime::orm::Uuid".into(), + Node::Secret => "dever_runtime::secret::Secret".into(), + Node::Nullable(inner) => format!("Option<{}>", rust_type(schema, *inner)), + Node::List(inner) => format!( + "dever_runtime::collections::List<{}>", + rust_type(schema, *inner) + ), + Node::Record { id, .. } | Node::ModelId { id, .. } => format!("T{id}"), + } +} + +fn decode(schema: &Schema, node: &Node, prefix: &str) -> String { + match node { + Node::Bytes => "node.bytes()".into(), + Node::Bool => "node.boolean()".into(), + Node::Int | Node::Duration => "node.int()".into(), + Node::ModelId { id, .. } => format!("Ok(T{id}(node.int()?))"), + Node::Float => "node.float()".into(), + Node::Text => "Ok(node.text()?.to_owned())".into(), + Node::Json => "Ok(node.raw().to_owned())".into(), + Node::Secret => { + "Ok(dever_runtime::secret::Secret::from_input(node.text()?.as_bytes().to_vec()))".into() + } + Node::Id => "Ok(dever_runtime::Id(node.text()?.to_owned()))".into(), + Node::Uuid => { + "dever_runtime::orm::Uuid::parse(node.text()?).map_err(|error| error.to_string())" + .into() + } + Node::Decimal => { + "dever_runtime::number::DecimalValue::parse(node.text()?).map_err(str::to_owned)".into() + } + Node::DateTime => "dever_runtime::time::parse_datetime(node.text()?)".into(), + Node::Date => "dever_runtime::time::parse_date(node.text()?)".into(), + Node::Time => "dever_runtime::time::parse_time(node.text()?)".into(), + Node::Nullable(inner) => format!( + "if node.is_null() {{ Ok(None) }} else {{ Ok(Some({prefix}_d{inner}(node)?)) }}" + ), + Node::List(inner) => format!( + "Ok(dever_runtime::collections::List::new(node.list()?.iter().map({prefix}_d{inner}).collect::<Result<Vec<_>, String>>()?))" + ), + Node::Record { id, fields, .. } => { + let names = fields + .iter() + .map(|(name, _)| rust_string(name)) + .collect::<Vec<_>>() + .join(", "); + let mut body = format!("let fields = node.fields(&[{names}])?; Ok(T{id} {{"); + for (index, (name, child)) in fields.iter().enumerate() { + let name = rust_string(name); + if matches!(schema.nodes[*child], Node::Nullable(_)) { + write!(body, "f{index}: match fields.get({name}) {{ Some(value) => {prefix}_d{child}(value)?, None => None }},").expect("string formatting"); + } else { + write!(body, "f{index}: {prefix}_d{child}(fields.get({name}).ok_or(\"missing required wire field\")?)?,").expect("string formatting"); + } + } + body.push_str("})"); + body + } + } +} + +fn encode(node: &Node, prefix: &str) -> String { + match node { + Node::Bytes => "writer.bytes(value)".into(), + Node::Bool => "writer.boolean(*value)".into(), + Node::Int | Node::Duration => "writer.int(*value)".into(), + Node::ModelId { .. } => "writer.int(value.0)".into(), + Node::Float => "writer.float(*value)".into(), + Node::Text => "writer.text(value)".into(), + Node::Json => "writer.json(value)".into(), + Node::Id => "writer.text(&value.0)".into(), + Node::Uuid | Node::Decimal => "writer.text(&value.to_string())".into(), + Node::DateTime => "writer.text(&dever_runtime::time::format_datetime(*value)?)".into(), + Node::Date => "writer.text(&dever_runtime::time::format_date(*value)?)".into(), + Node::Time => "writer.text(&dever_runtime::time::format_time(*value)?)".into(), + Node::Nullable(inner) => format!( + "match value {{ Some(value) => {prefix}_e{inner}(value, writer), None => writer.null() }}" + ), + Node::List(inner) => format!( + "writer.begin_array()?; for value in value.values() {{ {prefix}_e{inner}(value, writer)?; }} writer.end()" + ), + Node::Record { fields, .. } => { + let mut body = "writer.begin_object()?;".to_owned(); + for (index, (name, child)) in fields.iter().enumerate() { + write!( + body, + "writer.key({})?; {prefix}_e{child}(&value.f{index}, writer)?;", + rust_string(name) + ) + .expect("string formatting"); + } + body.push_str("writer.end()"); + body + } + Node::Secret => unreachable!("input-only schema never emits an encoder"), + } +} diff --git a/crates/dever-core/src/parser.rs b/crates/dever-core/src/parser.rs new file mode 100644 index 0000000..6419b05 --- /dev/null +++ b/crates/dever-core/src/parser.rs @@ -0,0 +1,1185 @@ +use crate::diagnostic::Diagnostic; +use crate::source::{SourceFile, SourceMap, Span}; +use crate::syntax::*; +use crate::token::{Kind, Token}; + +mod expression; + +type ParseResult<T> = Result<T, Diagnostic>; + +pub(crate) fn parse(tokens: Vec<Token>, source: &SourceFile) -> Result<Package, Vec<Diagnostic>> { + let layout = source.layout().map_err(|error| vec![error])?; + let mut comments = Vec::new(); + let tokens = tokens + .into_iter() + .filter(|token| { + if let Kind::Comment(text) = &token.kind { + comments.push(Comment { + text: text.clone(), + span: token.span, + }); + false + } else { + true + } + }) + .collect(); + let mut parser = Parser { + tokens, + cursor: 0, + brace_depth: 0, + nesting: 0, + model: source.is_model(), + application: !SourceMap::is_standard(source.id()) + && match &layout { + crate::source::SourceLayout::Main => source.strict_layout(), + crate::source::SourceLayout::Test { .. } => true, + crate::source::SourceLayout::Role { .. } => true, + crate::source::SourceLayout::Loose => false, + }, + }; + let name = source.package_path().map_err(|error| vec![error])?; + parser.package(name, layout, comments) +} + +struct Parser { + tokens: Vec<Token>, + cursor: usize, + brace_depth: usize, + nesting: usize, + model: bool, + application: bool, +} + +impl Parser { + fn current(&self) -> &Token { + &self.tokens[self.cursor] + } + + fn at(&self, kind: &Kind) -> bool { + &self.current().kind == kind + } + + fn at_name(&self, name: &str) -> bool { + matches!(&self.current().kind, Kind::Name(text) if text == name) + } + + fn modifier(&mut self, name: &str) -> Option<Span> { + if self.at_name(name) + && self + .tokens + .get(self.cursor + 1) + .is_some_and(|token| matches!(token.kind, Kind::Name(_))) + { + Some(self.advance().span) + } else { + None + } + } + + fn advance(&mut self) -> Token { + let token = self.current().clone(); + match token.kind { + Kind::LeftBrace => self.brace_depth += 1, + Kind::RightBrace => self.brace_depth = self.brace_depth.saturating_sub(1), + _ => {} + } + if !self.at(&Kind::End) { + self.cursor += 1; + } + token + } + + fn take(&mut self, kind: &Kind) -> Option<Token> { + if self.at(kind) { + Some(self.advance()) + } else { + None + } + } + + fn expect(&mut self, kind: &Kind, description: &str) -> ParseResult<Token> { + self.take(kind) + .ok_or_else(|| self.error(format!("expected {description}"))) + } + + fn error(&self, message: impl Into<String>) -> Diagnostic { + Diagnostic::error("P001", message, self.current().span) + } + + fn skip_lines(&mut self) { + while self.take(&Kind::Newline).is_some() {} + } + + fn previous_span(&self) -> Span { + self.tokens[self.cursor - 1].span + } + + fn name(&mut self) -> ParseResult<Name> { + if let Kind::Name(text) = &self.current().kind { + let name = Name { + text: text.clone(), + span: self.current().span, + }; + self.advance(); + Ok(name) + } else { + Err(self.error("expected an identifier")) + } + } + + fn path(&mut self) -> ParseResult<Path> { + let mut path = vec![self.name()?]; + while self.take(&Kind::Dot).is_some() { + path.push(self.name()?); + } + Ok(path) + } + + // Cap structural recursion while parsing untrusted source, before using the host stack. + fn nested<T>(&mut self, parse: impl FnOnce(&mut Self) -> ParseResult<T>) -> ParseResult<T> { + if self.nesting == MAX_SYNTAX_DEPTH { + return Err(Diagnostic::error( + "P009", + "syntax nesting exceeds the compiler limit", + self.current().span, + )); + } + self.nesting += 1; + let result = parse(self); + self.nesting -= 1; + result + } + + fn line_end(&mut self, closing: &Kind) -> ParseResult<()> { + if self.take(&Kind::Newline).is_some() { + self.skip_lines(); + return Ok(()); + } + if self.at(closing) { + return Ok(()); + } + Err(self.error("expected a newline or the closing delimiter")) + } + + fn separated<T>( + &mut self, + closing: Kind, + parse: impl Fn(&mut Self) -> ParseResult<T>, + ) -> ParseResult<Vec<T>> { + let mut members = Vec::new(); + self.skip_lines(); + if self.take(&closing).is_some() { + return Ok(members); + } + loop { + members.push(parse(self)?); + self.skip_lines(); + if self.take(&closing).is_some() { + return Ok(members); + } + self.expect(&Kind::Comma, "',' between positional entries")?; + self.skip_lines(); + } + } + + fn package( + &mut self, + name: Path, + layout: crate::source::SourceLayout, + comments: Vec<Comment>, + ) -> Result<Package, Vec<Diagnostic>> { + self.skip_lines(); + let start = self.current().span; + let mut declarations = Vec::new(); + let mut errors = Vec::new(); + while !self.at(&Kind::End) { + let visibility = self.take(&Kind::Public).map(|token| token.span); + let anonymous_http = layout.role() == Some(crate::source::SourceRole::Api) + && ["get", "post", "put", "delete"] + .iter() + .any(|name| self.at_name(name)); + if self.application + && !anonymous_http + && let Some(span) = visibility + { + errors.push(Diagnostic::error( + "P001", + "application visibility is defined by the source role; remove 'public'", + span, + )); + } + let declaration = if self.at(&Kind::Package) || self.at(&Kind::Exposes) { + Err(self.error("package/exposes declarations were removed; the source path and role define identity and visibility")) + } else if layout.role() == Some(crate::source::SourceRole::Api) + && ["get", "post", "put", "delete", "cmd", "rest"] + .iter() + .any(|name| self.at_name(name)) + { + self.api_declaration() + } else if layout.role() == Some(crate::source::SourceRole::Adapter) + && self.at_name("external") + { + self.external_adapter().map(Declaration::External) + } else if self.at(&Kind::Type) { + self.type_declaration(None).map(Declaration::Type) + } else if self.at_name("global") + && self + .tokens + .get(self.cursor + 1) + .is_some_and(|token| token.kind == Kind::Type) + { + let start = self.advance().span; + self.type_declaration(Some(start)).map(Declaration::Type) + } else if self.at_name("setting") + && self + .tokens + .get(self.cursor + 1) + .is_some_and(|token| token.kind == Kind::LeftBrace) + { + self.setting_declaration().map(Declaration::Type) + } else if (self.model || layout.role() == Some(crate::source::SourceRole::Job)) + && self.at_name("database") + { + self.database_binding().map(Declaration::Database) + } else if self.at_name("schedule") + && layout.role() == Some(crate::source::SourceRole::Job) + { + self.schedule().map(Declaration::Schedule) + } else if self.model && self.at_name("index") { + self.model_index(false).map(Declaration::ModelIndex) + } else if self.model && self.at_name("unique") { + self.model_index(true).map(Declaration::ModelIndex) + } else if self.model && self.at_name("relation") { + self.relation().map(Declaration::Relation) + } else if self.model && self.at_name("seed") { + self.seed().map(Declaration::Seed) + } else if self.model && self.at_name("migrate") { + self.migration().map(Declaration::Migration) + } else if self.model && self.at_name("sql") { + self.model_sql().map(Declaration::ModelSql) + } else if self.at_name("job") { + let start = self.advance().span; + self.function( + FunctionKind::Job { + attempts: 0, + timeout_ms: 0, + }, + Some(start), + ) + .map(Declaration::Function) + } else if self.at_name("transaction") { + let start = self.advance().span; + self.function(FunctionKind::Transaction, Some(start)) + .map(Declaration::Function) + } else { + self.function(FunctionKind::Ordinary, None) + .map(Declaration::Function) + }; + match declaration { + Ok(mut declaration) => { + if let Declaration::Api(binding) = &mut declaration { + binding.anonymous = visibility.is_some() && anonymous_http; + if let Some(start) = visibility { + binding.span = start.through(binding.span); + } + } + if let Some(start) = visibility.filter(|_| !self.application) { + match &mut declaration { + Declaration::Type(ty) => { + ty.public = true; + ty.span = start.through(ty.span); + } + Declaration::Function(function) => { + function.public = true; + function.span = start.through(function.span); + } + _ => errors.push(Diagnostic::error( + "P001", + "'public' is allowed only on types and functions", + start, + )), + } + } + declarations.push(declaration); + } + Err(error) => { + errors.push(error); + self.recover_declaration(); + } + } + self.skip_lines(); + } + if errors.is_empty() { + Ok(Package { + name, + layout, + declarations, + comments, + span: start.through(self.current().span), + }) + } else { + Err(errors) + } + } + + fn external_adapter(&mut self) -> ParseResult<ExternalAdapter> { + let start = self.advance().span; + let ecosystem = self.name()?; + let ecosystem = match ecosystem.text.as_str() { + "exec" => ExternalEcosystem::Exec, + "command" => ExternalEcosystem::Command, + "pip" => ExternalEcosystem::Pip, + "npm" => ExternalEcosystem::Npm, + "go" => ExternalEcosystem::Go, + _ => { + return Err(Diagnostic::error( + "P001", + "external Adapter ecosystem must be exec, command, pip, npm, or go", + ecosystem.span, + )); + } + }; + let Kind::Text(entry) = &self.current().kind else { + return Err(self.error("external requires a literal relative entry path")); + }; + let entry = entry.clone(); + let entry_span = self.advance().span; + self.expect(&Kind::LeftBrace, "'{' after the external entry")?; + self.skip_lines(); + let mut libs = Vec::new(); + let mut capabilities = Vec::new(); + while !self.at(&Kind::RightBrace) { + if self.at_name("lib") { + self.advance(); + let Kind::Text(lib) = &self.current().kind else { + return Err(self.error("external lib requires a literal name@version or <ecosystem>:<name>@<version>")); + }; + libs.push(Name { + text: lib.clone(), + span: self.current().span, + }); + self.advance(); + self.line_end(&Kind::RightBrace)?; + continue; + } + if !self.at_name("allow") { + return Err(self + .error("external accepts only 'lib <spec>' or 'allow <capability>' entries")); + } + self.advance(); + capabilities.push(self.name()?); + self.line_end(&Kind::RightBrace)?; + } + let end = self.advance().span; + Ok(ExternalAdapter { + ecosystem, + entry, + entry_span, + libs, + capabilities, + span: start.through(end), + }) + } + + fn api_declaration(&mut self) -> ParseResult<Declaration> { + let keyword = self.name()?; + if keyword.text == "rest" { + let model = if self.at(&Kind::Newline) || self.at(&Kind::End) { + None + } else { + Some(self.path()?) + }; + let span = keyword.span.through(self.previous_span()); + self.line_end(&Kind::End)?; + return Ok(Declaration::Rest(RestDeclaration { model, span })); + } + let kind = match keyword.text.as_str() { + "get" => ApiKind::Get, + "post" => ApiKind::Post, + "put" => ApiKind::Put, + "delete" => ApiKind::Delete, + "cmd" => ApiKind::Command, + _ => unreachable!("checked API declaration keyword"), + }; + let action = self.name()?; + self.expect(&Kind::Assign, "'=' before the App binding")?; + let target = self.path()?; + let span = keyword.span.through(self.previous_span()); + self.line_end(&Kind::End)?; + Ok(Declaration::Api(ApiBinding { + anonymous: false, + kind, + action, + target, + span, + })) + } + + fn schedule(&mut self) -> ParseResult<Schedule> { + let start = self.advance().span; + let target = self.name()?; + self.expect(&Kind::Assign, "'=' before UTC cron expression")?; + let Kind::Text(cron) = &self.current().kind else { + return Err(self.error("schedule requires a literal five-field UTC cron expression")); + }; + let cron = cron.clone(); + let span = start.through(self.advance().span); + Ok(Schedule { target, cron, span }) + } + + fn database_binding(&mut self) -> ParseResult<DatabaseBinding> { + let start = self.advance().span; + let name = self.name()?; + Ok(DatabaseBinding { + span: start.through(name.span), + name, + }) + } + + fn model_index(&mut self, unique: bool) -> ParseResult<ModelIndex> { + let start = self.advance().span; + self.expect(&Kind::LeftParen, "'(' after the index kind")?; + let fields = self.separated(Kind::RightParen, Self::name)?; + if fields.is_empty() { + return Err(Diagnostic::error( + "P001", + "an index requires at least one field", + start, + )); + } + Ok(ModelIndex { + unique, + fields, + span: start.through(self.previous_span()), + }) + } + + fn relation(&mut self) -> ParseResult<Relation> { + let start = self.advance().span; + let name = self.name()?; + self.expect(&Kind::Assign, "'=' after the relation name")?; + let field = self.path()?; + Ok(Relation { + name, + field, + span: start.through(self.previous_span()), + }) + } + + fn seed(&mut self) -> ParseResult<Seed> { + let start = self.advance().span; + self.expect(&Kind::LeftBrace, "'{' after 'seed'")?; + self.skip_lines(); + let mut rows = Vec::new(); + while !self.at(&Kind::RightBrace) { + rows.push(self.seed_row()?); + self.line_end(&Kind::RightBrace)?; + } + let end = self.advance().span; + Ok(Seed { + rows, + span: start.through(end), + }) + } + + fn seed_row(&mut self) -> ParseResult<Vec<RecordField>> { + self.expect(&Kind::LeftBrace, "'{' around a seed row")?; + self.skip_lines(); + let mut fields = Vec::new(); + while !self.at(&Kind::RightBrace) { + let name = self.name()?; + let start = name.span; + self.expect(&Kind::Assign, "'=' after the seed field name")?; + let value = self.expression(0, false)?; + fields.push(RecordField { + name, + value, + span: start.through(self.previous_span()), + }); + self.line_end(&Kind::RightBrace)?; + } + self.advance(); + Ok(fields) + } + + fn migration(&mut self) -> ParseResult<Migration> { + let start = self.advance().span; + let name = self.name()?; + self.expect(&Kind::LeftBrace, "'{' after the migration name")?; + self.skip_lines(); + let mut operations = Vec::new(); + while !self.at(&Kind::RightBrace) { + if self.at_name("drop") { + self.advance(); + operations.push(MigrationOperation::Drop(self.name()?)); + } else if self.at_name("before") || self.at_name("after") { + operations.push(MigrationOperation::Sql(self.migration_sql()?)); + } else { + return Err(self.error("expected a migration operation")); + } + self.line_end(&Kind::RightBrace)?; + } + let end = self.advance().span; + Ok(Migration { + name, + operations, + span: start.through(end), + }) + } + + fn migration_sql(&mut self) -> ParseResult<MigrationSql> { + let phase = if self.at_name("before") { + MigrationPhase::Before + } else { + MigrationPhase::After + }; + let start = self.advance().span; + self.expect(&Kind::LeftBrace, "'{' after the migration phase")?; + self.skip_lines(); + let mut sqlite = None; + let mut postgres = None; + let mut parameters = None; + while !self.at(&Kind::RightBrace) { + let field = self.name()?; + self.expect(&Kind::Assign, "'=' after the migration SQL field")?; + if field.text == "parameters" { + self.expect(&Kind::LeftBracket, "'[' for migration parameters")?; + let values = self.separated(Kind::RightBracket, |this| this.expression(0, true))?; + if parameters.replace(values).is_some() { + return Err(Diagnostic::error( + "P001", + "duplicate migration parameters", + field.span, + )); + } + } else { + let slot = match field.text.as_str() { + "sqlite" => &mut sqlite, + "postgres" => &mut postgres, + _ => { + return Err(Diagnostic::error( + "P001", + "migration SQL requires sqlite, postgres and parameters", + field.span, + )); + } + }; + if slot.replace(self.sql_text()?).is_some() { + return Err(Diagnostic::error( + "P001", + "duplicate migration SQL dialect", + field.span, + )); + } + } + self.line_end(&Kind::RightBrace)?; + } + let end = self.advance().span; + Ok(MigrationSql { + phase, + sqlite: sqlite.ok_or_else(|| { + Diagnostic::error("P001", "migration SQL requires sqlite text", start) + })?, + postgres: postgres.ok_or_else(|| { + Diagnostic::error("P001", "migration SQL requires postgres text", start) + })?, + parameters: parameters.ok_or_else(|| { + Diagnostic::error( + "P001", + "migration SQL requires an explicit parameters list", + start, + ) + })?, + span: start.through(end), + }) + } + + fn sql_text(&mut self) -> ParseResult<SqlText> { + let Kind::Text(value) = &self.current().kind else { + return Err(self.error("SQL text must be a static Text literal")); + }; + let value = SqlText { + value: value.clone(), + span: self.current().span, + }; + self.advance(); + Ok(value) + } + + fn model_sql(&mut self) -> ParseResult<ModelSql> { + let start = self.advance().span; + let name = self.name()?; + self.expect(&Kind::LeftParen, "'(' after a SQL operation name")?; + let inputs = self.separated(Kind::RightParen, Self::field)?; + self.expect(&Kind::LeftParen, "the SQL result list '('")?; + let outputs = self.separated(Kind::RightParen, Self::field)?; + self.expect(&Kind::LeftBrace, "the SQL body '{'")?; + self.skip_lines(); + let mut sqlite = None; + let mut postgres = None; + while !self.at(&Kind::RightBrace) { + let dialect = self.name()?; + if !matches!(dialect.text.as_str(), "sqlite" | "postgres") { + return Err(Diagnostic::error( + "P001", + "a SQL body accepts only 'sqlite' and 'postgres'", + dialect.span, + )); + } + self.expect(&Kind::Assign, "'=' after the SQL dialect")?; + let value = self.sql_text()?; + let slot = if dialect.text == "sqlite" { + &mut sqlite + } else { + &mut postgres + }; + if slot.replace(value).is_some() { + return Err(Diagnostic::error( + "P001", + format!("duplicate '{}' SQL dialect", dialect.text), + dialect.span, + )); + } + self.line_end(&Kind::RightBrace)?; + } + let end = self.advance().span; + let sqlite = sqlite.ok_or_else(|| { + Diagnostic::error("P001", "a SQL declaration requires sqlite text", name.span) + })?; + let postgres = postgres.ok_or_else(|| { + Diagnostic::error( + "P001", + "a SQL declaration requires postgres text", + name.span, + ) + })?; + Ok(ModelSql { + name, + inputs, + outputs, + sqlite, + postgres, + span: start.through(end), + }) + } + + fn recover_declaration(&mut self) { + // Resume only after a balanced body or a top-level line, never inside a record. + while !self.at(&Kind::End) { + let token = self.advance(); + if self.brace_depth == 0 && matches!(token.kind, Kind::RightBrace | Kind::Newline) { + return; + } + } + } + + fn type_declaration(&mut self, global: Option<Span>) -> ParseResult<TypeDeclaration> { + let type_start = self.expect(&Kind::Type, "'type' after 'global'")?.span; + let start = global.unwrap_or(type_start); + let name = self.name()?; + self.expect(&Kind::LeftBrace, "'{' after the type name")?; + self.skip_lines(); + if let Some(end) = self.take(&Kind::RightBrace) { + return Ok(TypeDeclaration { + public: false, + setting: false, + global: global.is_some(), + name, + shape: TypeShape::Record(Vec::new()), + span: start.through(end.span), + }); + } + let private = self.modifier("private"); + let owner = self.modifier("owner"); + let first = self.name()?; + let shape = if self.at(&Kind::Colon) { + let mut fields = vec![self.field_after_name(first, private, owner)?]; + self.line_end(&Kind::RightBrace)?; + while !self.at(&Kind::RightBrace) { + let private = self.modifier("private"); + let owner = self.modifier("owner"); + let name = self.name()?; + fields.push(self.field_after_name(name, private, owner)?); + self.line_end(&Kind::RightBrace)?; + } + TypeShape::Record(fields) + } else { + if private.is_some() { + return Err(self.error("'private' is allowed only on record fields")); + } + let mut variants = vec![self.variant(first)?]; + self.line_end(&Kind::RightBrace)?; + while !self.at(&Kind::RightBrace) { + let name = self.name()?; + variants.push(self.variant(name)?); + self.line_end(&Kind::RightBrace)?; + } + TypeShape::Choice(variants) + }; + let end = self.expect(&Kind::RightBrace, "'}'")?.span; + Ok(TypeDeclaration { + public: false, + setting: false, + global: global.is_some(), + name, + shape, + span: start.through(end), + }) + } + + fn setting_declaration(&mut self) -> ParseResult<TypeDeclaration> { + let start = self.advance().span; + self.expect(&Kind::LeftBrace, "'{' after setting")?; + self.skip_lines(); + let mut fields = Vec::new(); + while !self.at(&Kind::RightBrace) { + fields.push(self.field()?); + self.line_end(&Kind::RightBrace)?; + } + let end = self.advance().span; + Ok(TypeDeclaration { + public: false, + setting: true, + global: false, + name: Name { + text: "Setting".into(), + span: start, + }, + shape: TypeShape::Record(fields), + span: start.through(end), + }) + } + + fn field(&mut self) -> ParseResult<Field> { + let name = self.name()?; + self.field_after_name(name, None, None) + } + + fn field_after_name( + &mut self, + name: Name, + private: Option<Span>, + owner: Option<Span>, + ) -> ParseResult<Field> { + self.expect(&Kind::Colon, "':' followed by a type")?; + let ty = self.type_ref()?; + let bounds = self.bounds()?; + let mut storage = self.field_storage()?; + if owner.is_some() { + self.expect( + &Kind::Assign, + "'=' and a trusted identity source after an owner field", + )?; + storage.owner = Some(self.expression(0, false)?); + } + let span = private + .or(owner) + .unwrap_or(name.span) + .through(self.previous_span()); + Ok(Field { + name, + ty, + private: private.is_some(), + bounds, + storage, + span, + }) + } + + fn field_storage(&mut self) -> ParseResult<FieldStorage> { + let mut storage = FieldStorage::default(); + while matches!( + &self.current().kind, + Kind::Name(name) + if matches!(name.as_str(), "generated" | "default" | "index" | "unique" | "from" | "create" | "replace" | "search") + ) { + let modifier = self.name()?; + match modifier.text.as_str() { + "generated" if !storage.generated => storage.generated = true, + "default" if storage.default.is_none() => { + storage.default = Some(self.expression(0, false)?); + } + "index" if !storage.index => storage.index = true, + "unique" if !storage.unique => storage.unique = true, + "from" if storage.from.is_none() => storage.from = Some(self.name()?), + "create" if storage.create.is_none() => { + self.expect(&Kind::Assign, "'=' and a REST create binding expression")?; + storage.create = Some(self.expression(0, false)?); + } + "replace" if storage.replace.is_none() => { + self.expect(&Kind::Assign, "'=' and a REST replace binding expression")?; + storage.replace = Some(self.expression(0, false)?); + } + "search" if storage.search.is_none() => { + self.expect(&Kind::Assign, "'=' and a REST search binding expression")?; + storage.search = Some(self.expression(0, false)?); + } + _ => { + return Err(Diagnostic::error( + "P001", + format!("duplicate '{}' field modifier", modifier.text), + modifier.span, + )); + } + } + } + Ok(storage) + } + + fn variant(&mut self, name: Name) -> ParseResult<Variant> { + let start = name.span; + let error = name.text == "error" && matches!(self.current().kind, Kind::Name(_)); + let name = if error { self.name()? } else { name }; + let payload = if self.take(&Kind::LeftParen).is_some() { + self.separated(Kind::RightParen, Self::field)? + } else { + Vec::new() + }; + let label = if self.take(&Kind::Assign).is_some() { + let Kind::Text(label) = &self.current().kind else { + return Err(self.error("expected a choice display label string")); + }; + let label = label.clone(); + self.advance(); + Some(label) + } else { + None + }; + let span = start.through(self.previous_span()); + Ok(Variant { + name, + error, + payload, + label, + span, + }) + } + + fn type_ref(&mut self) -> ParseResult<TypeRef> { + self.nested(Self::type_ref_inner) + } + + fn type_ref_inner(&mut self) -> ParseResult<TypeRef> { + let name = self.path()?; + let generic = name.len() == 1 + && matches!( + name[0].text.as_str(), + "List" | "Map" | "MapEntry" | "Stream" | "AsyncStream" | "Channel" | "Related" + ) + || name.len() == 2 + && name[0].text == "Related" + && matches!(name[1].text.as_str(), "Loaded" | "Unloaded"); + let arguments = if generic && self.take(&Kind::Less).is_some() { + self.separated(Kind::Greater, Self::type_ref)? + } else { + Vec::new() + }; + let storage_parameters = self.at(&Kind::LeftParen) + && self + .tokens + .get(self.cursor + 1) + .is_some_and(|token| matches!(token.kind, Kind::Number(_))); + let parameters = if storage_parameters { + self.advance(); + self.separated(Kind::RightParen, Self::number_parameter)? + } else { + Vec::new() + }; + let nullable = self.take(&Kind::Question).is_some(); + if self.at(&Kind::Question) { + return Err(self.error("a type may have only one '?' suffix")); + } + let span = name[0].span.through(self.previous_span()); + Ok(TypeRef { + name, + arguments, + parameters, + nullable, + span, + }) + } + + fn number_parameter(&mut self) -> ParseResult<NumberParameter> { + let Kind::Number(value) = &self.current().kind else { + return Err(self.error("expected a numeric type parameter")); + }; + let value = value.clone(); + let span = self.advance().span; + Ok(NumberParameter { value, span }) + } + + fn function( + &mut self, + mut kind: FunctionKind, + declaration_start: Option<Span>, + ) -> ParseResult<FunctionClause> { + let mut name = self.name()?; + while self.take(&Kind::Dot).is_some() { + let segment = self.name()?; + name.text.push('.'); + name.text.push_str(&segment.text); + name.span = name.span.through(segment.span); + } + self.expect(&Kind::LeftParen, "'(' after a function name")?; + let inputs = self.separated(Kind::RightParen, Self::input)?; + self.expect(&Kind::LeftParen, "the named output list '('")?; + let outputs = self.separated(Kind::RightParen, Self::field)?; + if matches!(kind, FunctionKind::Job { .. }) { + let attempts = self.job_policy("retry", 100)?; + let timeout_ms = + self.job_policy("timeout", dever_runtime::config::MAX_JOB_TIMEOUT_MS)?; + kind = FunctionKind::Job { + attempts, + timeout_ms, + }; + } + let (pure, recovery) = self.function_contracts()?; + let fails = if self.at_name("fails") { + self.advance(); + Some(self.type_ref()?) + } else { + None + }; + if let Some(failure) = &fails + && !self.at(&Kind::LeftBrace) + { + let span = declaration_start.unwrap_or(name.span).through(failure.span); + return Ok(FunctionClause { + public: false, + name, + kind, + inputs, + outputs, + pure, + recovery, + fails, + bodyless: true, + body: Vec::new(), + span, + }); + } + self.expect(&Kind::LeftBrace, "the function body '{'")?; + self.skip_lines(); + let mut body = Vec::new(); + while !self.at(&Kind::RightBrace) { + body.push(self.statement()?); + self.line_end(&Kind::RightBrace)?; + } + let end = self.advance().span; + let span = declaration_start.unwrap_or(name.span).through(end); + Ok(FunctionClause { + public: false, + name, + kind, + inputs, + outputs, + pure, + recovery, + fails, + bodyless: false, + body, + span, + }) + } + + fn job_policy(&mut self, name: &str, maximum: u32) -> ParseResult<u32> { + if !self.at_name(name) { + return Err(self.error(format!("job requires {name}(positive integer)"))); + } + self.advance(); + self.expect(&Kind::LeftParen, "'(' before job policy")?; + let number = self.signed_number()?; + let value = number + .parse::<u32>() + .ok() + .filter(|value| *value > 0 && *value <= maximum) + .ok_or_else(|| self.error(format!("{name} must be an integer in 1..={maximum}")))?; + self.expect(&Kind::RightParen, "')' after job policy")?; + Ok(value) + } + + fn function_contracts(&mut self) -> ParseResult<(bool, Option<String>)> { + let mut pure = false; + let mut recovery = None; + self.skip_lines(); + while self.at_name("pure") || self.at_name("recover") { + if self.at_name("pure") { + if pure { + return Err(self.error("duplicate 'pure' contract")); + } + self.advance(); + pure = true; + } else { + if recovery.is_some() { + return Err(self.error("duplicate 'recover' contract")); + } + self.advance(); + self.skip_lines(); + self.expect(&Kind::LeftParen, "'(' after 'recover'")?; + self.skip_lines(); + let Kind::Text(reason) = &self.current().kind else { + return Err(self.error("expected a nonempty recovery reason string")); + }; + if reason.trim().is_empty() { + return Err(self.error("recovery reason must not be empty")); + } + recovery = Some(reason.clone()); + self.advance(); + self.skip_lines(); + self.expect(&Kind::RightParen, "')' after the recovery reason")?; + } + self.skip_lines(); + } + Ok((pure, recovery)) + } + + fn input(&mut self) -> ParseResult<Input> { + let name = self.name()?; + self.expect(&Kind::Colon, "':' followed by an input type or pattern")?; + let kind = if matches!(&self.current().kind, Kind::Name(value) if value == "handler") { + let handler_span = self.advance().span; + self.expect(&Kind::LeftParen, "'(' after 'handler'")?; + let inputs = self.separated(Kind::RightParen, Self::field)?; + self.expect(&Kind::LeftParen, "the handler output list '('")?; + let outputs = self.separated(Kind::RightParen, Self::field)?; + InputKind::Handler(HandlerSignature { + inputs, + outputs, + span: handler_span.through(self.previous_span()), + }) + } else { + InputKind::Value(self.pattern()?) + }; + let span = name.span.through(self.previous_span()); + Ok(Input { name, kind, span }) + } + + fn pattern(&mut self) -> ParseResult<Pattern> { + let start = self.current().span; + if let Some(token) = self.take(&Kind::Other) { + return Ok(Pattern::Other(token.span)); + } + if let Some(value) = self.literal() { + return Ok(Pattern::Literal { + value, + span: start.through(self.previous_span()), + }); + } + if self.at(&Kind::Minus) { + let number = self.signed_number()?; + return Ok(Pattern::Literal { + value: Literal::Number(number), + span: start.through(self.previous_span()), + }); + } + let ty = self.type_ref()?; + if self.take(&Kind::LeftParen).is_some() { + if ty.nullable { + return Err(self.error("expected a qualified choice variant")); + } + let bindings = self.separated(Kind::RightParen, Self::name)?; + return Ok(Pattern::Variant { + name: ty.name, + arguments: ty.arguments, + bindings, + span: start.through(self.previous_span()), + }); + } + let bounds = self.bounds()?; + Ok(Pattern::Typed { ty, bounds }) + } + + fn bounds(&mut self) -> ParseResult<Vec<Bound>> { + let mut bounds = Vec::new(); + if self.range_operator().is_some() { + bounds.push(self.bound()?); + if self.take(&Kind::And).is_some() { + bounds.push(self.bound()?); + } + } + Ok(bounds) + } + + fn range_operator(&self) -> Option<BinaryOperator> { + match self.current().kind { + Kind::Less => Some(BinaryOperator::Less), + Kind::LessEqual => Some(BinaryOperator::LessEqual), + Kind::Greater => Some(BinaryOperator::Greater), + Kind::GreaterEqual => Some(BinaryOperator::GreaterEqual), + _ => None, + } + } + + fn bound(&mut self) -> ParseResult<Bound> { + let comparison = self + .range_operator() + .ok_or_else(|| self.error("expected '<', '<=', '>' or '>='"))?; + let start = self.advance().span; + let number = self.signed_number()?; + Ok(Bound { + comparison, + number, + span: start.through(self.previous_span()), + }) + } + + fn signed_number(&mut self) -> ParseResult<String> { + let negative = self.take(&Kind::Minus).is_some(); + if let Kind::Number(number) = &self.current().kind { + let spelling = if negative { + format!("-{number}") + } else { + number.clone() + }; + self.advance(); + Ok(spelling) + } else { + Err(self.error("expected a numeric literal")) + } + } + + fn literal(&mut self) -> Option<Literal> { + let value = match &self.current().kind { + Kind::True => Literal::Bool(true), + Kind::False => Literal::Bool(false), + Kind::Null => Literal::Null, + Kind::Text(text) => Literal::Text(text.clone()), + Kind::Number(number) => Literal::Number(number.clone()), + _ => return None, + }; + self.advance(); + Some(value) + } + + fn statement(&mut self) -> ParseResult<Statement> { + let expression = self.expression(0, false)?; + let start = expression.span; + let kind = if self.take(&Kind::Assign).is_some() { + let ExpressionKind::Name(target) = expression.kind else { + return Err(Diagnostic::error( + "P002", + "assignment requires a local name or field path", + start, + )); + }; + let value = self.expression(0, false)?; + StatementKind::Assign { target, value } + } else if matches!( + expression.kind, + ExpressionKind::Call { .. } | ExpressionKind::Fail(_) | ExpressionKind::Contextual(_) + ) { + StatementKind::Call(expression) + } else { + return Err(Diagnostic::error( + "P002", + "a body statement must be an assignment or function call", + start, + )); + }; + Ok(Statement { + kind, + span: start.through(self.previous_span()), + }) + } +} diff --git a/crates/dever-core/src/parser/expression.rs b/crates/dever-core/src/parser/expression.rs new file mode 100644 index 0000000..0ca6344 --- /dev/null +++ b/crates/dever-core/src/parser/expression.rs @@ -0,0 +1,253 @@ +use super::{ParseResult, Parser}; +use crate::diagnostic::Diagnostic; +use crate::source::Span; +use crate::syntax::*; +use crate::token::Kind; + +impl Parser { + pub(super) fn expression(&mut self, minimum: u8, multiline: bool) -> ParseResult<Expression> { + self.nested(|parser| parser.binary_expression(minimum, multiline)) + } + + fn binary_expression(&mut self, minimum: u8, multiline: bool) -> ParseResult<Expression> { + if multiline { + self.skip_lines(); + } + let mut left = self.prefix(multiline)?; + loop { + if multiline { + self.skip_lines(); + } + if self.at(&Kind::LeftParen) { + self.advance(); + let mut arguments = + self.separated(Kind::RightParen, |parser| parser.expression(0, true))?; + let span = left.span.through(self.previous_span()); + let core_name = match &left.kind { + ExpressionKind::Name(path) + if path.len() == 1 + && matches!(path[0].text.as_str(), "fail" | "result") => + { + Some(path[0].text.as_str()) + } + _ => None, + }; + let kind = if let Some(core_name) = core_name { + if arguments.len() != 1 { + return Err(Diagnostic::error( + "P001", + format!("'{core_name}' expects exactly one argument"), + span, + )); + } + let argument = Box::new(arguments.remove(0)); + if core_name == "fail" { + ExpressionKind::Fail(argument) + } else { + ExpressionKind::CaptureResult(argument) + } + } else { + ExpressionKind::Call { + function: Box::new(left), + arguments, + } + }; + left = self.expression_node(kind, span)?; + continue; + } + if self.take(&Kind::Dot).is_some() { + let name = self.name()?; + let span = left.span.through(name.span); + left = self.expression_node( + ExpressionKind::Field { + value: Box::new(left), + name, + }, + span, + )?; + continue; + } + let Some(operator) = self.binary_operator() else { + break; + }; + let precedence = operator.precedence(); + if precedence < minimum { + break; + } + self.advance(); + let right = self.expression(precedence + 1, multiline)?; + let span = left.span.through(right.span); + left = self.expression_node( + ExpressionKind::Binary { + left: Box::new(left), + operator, + right: Box::new(right), + }, + span, + )?; + } + Ok(left) + } + + fn binary_operator(&self) -> Option<BinaryOperator> { + let operator = match self.current().kind { + Kind::Plus => BinaryOperator::Add, + Kind::Minus => BinaryOperator::Subtract, + Kind::Star => BinaryOperator::Multiply, + Kind::Slash => BinaryOperator::Divide, + Kind::IntegerDivide => BinaryOperator::IntegerDivide, + Kind::Percent => BinaryOperator::Remainder, + Kind::Equal => BinaryOperator::Equal, + Kind::NotEqual => BinaryOperator::NotEqual, + Kind::Less => BinaryOperator::Less, + Kind::LessEqual => BinaryOperator::LessEqual, + Kind::Greater => BinaryOperator::Greater, + Kind::GreaterEqual => BinaryOperator::GreaterEqual, + Kind::And => BinaryOperator::And, + Kind::Or => BinaryOperator::Or, + _ => return None, + }; + Some(operator) + } + + fn prefix(&mut self, multiline: bool) -> ParseResult<Expression> { + let start = self.current().span; + if let Some(value) = self.literal() { + return self.expression_node(ExpressionKind::Literal(value), start); + } + let kind = match &self.current().kind { + Kind::Name(_) => { + let name = self.path()?; + if name.len() == 1 + && name[0].text == "channel" + && self.take(&Kind::LeftParen).is_some() + { + let element = self.type_ref()?; + self.expect(&Kind::Comma, "',' after the channel item type")?; + let capacity = self.expression(0, true)?; + self.expect(&Kind::RightParen, "')' after the channel capacity")?; + return self.expression_node( + ExpressionKind::Contextual(ContextualExpression::Channel { + element, + capacity: Box::new(capacity), + }), + start.through(self.previous_span()), + ); + } + if self.take(&Kind::LeftBrace).is_some() { + self.record(name)? + } else { + ExpressionKind::Name(name) + } + } + Kind::Minus | Kind::Not => { + let operator = if self.at(&Kind::Minus) { + UnaryOperator::Negate + } else { + UnaryOperator::Not + }; + self.advance(); + let value = self.expression(6, multiline)?; + ExpressionKind::Unary { + operator, + value: Box::new(value), + } + } + Kind::LeftParen => { + self.advance(); + let value = self.expression(0, true)?; + self.expect(&Kind::RightParen, "')'")?; + ExpressionKind::Group(Box::new(value)) + } + Kind::LeftBracket => { + self.advance(); + ExpressionKind::List( + self.separated(Kind::RightBracket, |parser| parser.expression(0, true))?, + ) + } + Kind::LeftBrace => { + self.advance(); + self.map()? + } + _ => return Err(self.error("expected a value, name, construction or function call")), + }; + self.expression_node(kind, start.through(self.previous_span())) + } + + fn expression_node(&self, kind: ExpressionKind, span: Span) -> ParseResult<Expression> { + // Iterative operator/postfix parsing also creates recursive trees consumed by Drop and later phases. + let child_depth = match &kind { + ExpressionKind::Literal(_) | ExpressionKind::Name(_) => 0, + ExpressionKind::Group(value) + | ExpressionKind::Field { value, .. } + | ExpressionKind::Fail(value) + | ExpressionKind::CaptureResult(value) + | ExpressionKind::Unary { value, .. } => value.depth, + ExpressionKind::Binary { left, right, .. } => left.depth.max(right.depth), + ExpressionKind::Call { + function, + arguments, + } => arguments + .iter() + .map(|argument| argument.depth) + .max() + .unwrap_or(0) + .max(function.depth), + ExpressionKind::Contextual(ContextualExpression::Channel { capacity, .. }) => { + capacity.depth + } + ExpressionKind::Record { fields, .. } => fields + .iter() + .map(|field| field.value.depth) + .max() + .unwrap_or(0), + ExpressionKind::List(values) => { + values.iter().map(|value| value.depth).max().unwrap_or(0) + } + ExpressionKind::Map(entries) => entries + .iter() + .map(|entry| entry.key.depth.max(entry.value.depth)) + .max() + .unwrap_or(0), + }; + let depth = child_depth + 1; + if depth > MAX_SYNTAX_DEPTH { + return Err(Diagnostic::error( + "P009", + "syntax nesting exceeds the compiler limit", + span, + )); + } + Ok(Expression { kind, span, depth }) + } + + fn record(&mut self, name: Path) -> ParseResult<ExpressionKind> { + let mut fields = Vec::new(); + self.skip_lines(); + while !self.at(&Kind::RightBrace) { + let name = self.name()?; + self.expect(&Kind::Assign, "'=' after a record field name")?; + let value = self.expression(0, false)?; + let span = name.span.through(value.span); + fields.push(RecordField { name, value, span }); + self.line_end(&Kind::RightBrace)?; + } + self.advance(); + Ok(ExpressionKind::Record { name, fields }) + } + + fn map(&mut self) -> ParseResult<ExpressionKind> { + let mut entries = Vec::new(); + self.skip_lines(); + while !self.at(&Kind::RightBrace) { + let key = self.expression(0, false)?; + self.expect(&Kind::Assign, "'=' after a Map key")?; + let value = self.expression(0, false)?; + let span = key.span.through(value.span); + entries.push(MapEntry { key, value, span }); + self.line_end(&Kind::RightBrace)?; + } + self.advance(); + Ok(ExpressionKind::Map(entries)) + } +} diff --git a/crates/dever-core/src/reference.rs b/crates/dever-core/src/reference.rs new file mode 100644 index 0000000..6ad7194 --- /dev/null +++ b/crates/dever-core/src/reference.rs @@ -0,0 +1,254 @@ +//! Opt-in executable HIR specification for conformance tests; never a production backend. +use std::collections::BTreeMap; + +use crate::hir::{Atom, Program, Projection, Statement}; +use crate::source::{SourceMap, Span}; +use crate::specialize::{self, Specialization}; + +mod collections; +mod expressions; +mod intrinsics; +mod value; +use value::Value; + +#[derive(Clone, Debug, PartialEq)] +pub struct RuntimeFault { + pub message: String, + /// Innermost operation first, followed by its Dever call sites. + pub spans: Vec<Span>, + business: Option<Value>, +} + +impl RuntimeFault { + fn new(message: impl Into<String>, span: Span) -> Self { + Self { + message: message.into(), + spans: vec![span], + business: None, + } + } + + fn business(value: Value, message: String, span: Span) -> Self { + Self { + message, + spans: vec![span], + business: Some(value), + } + } + + fn at(mut self, span: Span) -> Self { + self.spans.push(span); + self + } + + fn into_business( + mut self, + targets: &BTreeMap<crate::hir::Failure, crate::capture::Target>, + choice: usize, + ) -> Result<Value, Self> { + let Some(Value::Variant(ty, variant, _)) = &self.business else { + return Err(self); + }; + let Some(target) = targets.get(&crate::hir::Failure { + ty: *ty, + variant: *variant, + }) else { + return Err(self); + }; + let error = self.business.take().expect("matched business error"); + Ok(if target.wrapped { + Value::Variant(choice, target.variant, vec![error]) + } else { + error + }) + } + + pub fn render(&self, sources: &SourceMap) -> String { + let mut locations = self + .spans + .iter() + .rev() + .map(|span| { + let source = sources.get(span.source); + let (line, column) = source.position(span.start); + format!("{}:{line}:{column}", source.path().display()) + }) + .collect::<Vec<_>>(); + locations.push(self.message.clone()); + locations.join(": ") + } +} + +#[derive(Debug)] +pub struct Evaluation { + /// Includes observable writes and successful entry named outputs, like a native process. + pub stdout: String, + pub fault: Option<RuntimeFault>, +} + +pub fn evaluate(program: &Program, entry: &str) -> Result<Evaluation, String> { + let id = program.entry(entry)?; + let mut evaluator = Evaluator { + program, + stdout: String::new(), + }; + let result = evaluator.call( + Specialization { + function: id, + handlers: vec![], + }, + vec![], + ); + let fault = match result { + Ok(result) => { + let outputs = &program.functions[id].outputs; + let values = if outputs.len() > 1 { + result.fields().to_vec() + } else { + vec![result] + }; + for (field, value) in outputs.iter().zip(values) { + evaluator + .stdout + .push_str(&format!("{} = {}\n", field.name, value.render(program))); + } + None + } + Err(fault) => Some(fault), + }; + Ok(Evaluation { + stdout: evaluator.stdout, + fault, + }) +} + +type ResultValue = Result<Value, RuntimeFault>; + +struct Evaluator<'a> { + program: &'a Program, + stdout: String, +} + +struct Frame { + locals: Vec<Option<Value>>, + handlers: BTreeMap<usize, usize>, +} + +impl Evaluator<'_> { + fn call(&mut self, instance: Specialization, arguments: Vec<Value>) -> ResultValue { + let function = &self.program.functions[instance.function]; + if let Some(port) = &function.port { + if let [implementation] = port.implementations.as_slice() { + return self.call( + Specialization { + function: *implementation, + handlers: Vec::new(), + }, + arguments, + ); + } + return Err(RuntimeFault::new( + "configured Port dispatch requires native execution", + function.span, + )); + } + if function.setting.is_some() { + return Err(RuntimeFault::new( + "Adapter settings require native execution", + function.span, + )); + } + if function.external.is_some() { + return Err(RuntimeFault::new( + "external Adapter requires native execution", + function.span, + )); + } + // HIR domains are proven exhaustive and disjoint; source order cannot select a winner. + let mut matches = function.clauses.iter().filter(|clause| { + clause + .patterns + .iter() + .zip(&arguments) + .all(|(domain, value)| domain.iter().any(|atom| matches_atom(atom, value))) + }); + let clause = matches.next().expect("checked exhaustive clause domains"); + assert!(matches.next().is_none(), "checked disjoint clause domains"); + let mut frame = Frame { + locals: vec![None; clause.locals.len()], + handlers: specialize::handler_bindings(function, &instance), + }; + let inputs: BTreeMap<_, _> = function + .parameters + .iter() + .enumerate() + .filter_map(|(index, parameter)| parameter.value_type().map(|_| index)) + .zip(arguments) + .collect(); + for binding in &clause.bindings { + let input = &inputs[&binding.input]; + let value = match binding.projection { + Projection::Whole | Projection::NonNull => input.clone(), + Projection::Payload { variant: _, field } => input.fields()[field].clone(), + }; + frame.locals[binding.slot] = Some(value); + } + for statement in &clause.body { + match statement { + Statement::Assign { + slot, + fields, + value, + } => { + let value = self.expression(value, &frame)?; + if fields.is_empty() { + frame.locals[*slot] = Some(value); + } else { + let mut host = frame.locals[*slot] + .as_mut() + .expect("initialized assignment host"); + for index in fields { + host = &mut host.fields_mut()[*index]; + } + *host = value; + } + } + Statement::Call(value) => { + self.expression(value, &frame)?; + } + } + } + let mut outputs = clause + .outputs + .iter() + .map(|slot| frame.locals[*slot].take().expect("initialized output")) + .collect::<Vec<_>>(); + Ok(match outputs.len() { + 0 => Value::Unit, + 1 => outputs.pop().unwrap(), + _ => Value::Fields(outputs), + }) + } +} + +fn matches_atom(atom: &Atom, value: &Value) -> bool { + if matches!(value, Value::Null) { + return matches!(atom, Atom::Null); + } + match atom { + Atom::Null => false, + Atom::Any => true, + Atom::Bool(expected) => value == &Value::Bool(*expected), + Atom::Variant(expected) => { + matches!(value, Value::Variant(_, actual, _) if expected == actual) + } + Atom::Int(lo, hi) => matches!(value, Value::Int(actual) if lo <= actual && actual <= hi), + Atom::Decimal(lo, hi) => { + matches!(value, Value::Decimal(actual) if lo <= actual && actual <= hi) + } + Atom::Literal(literal) => value == &Value::constant(literal), + Atom::Remainder(excluded) => excluded + .iter() + .all(|literal| value != &Value::constant(literal)), + } +} diff --git a/crates/dever-core/src/reference/collections.rs b/crates/dever-core/src/reference/collections.rs new file mode 100644 index 0000000..0f95a4b --- /dev/null +++ b/crates/dever-core/src/reference/collections.rs @@ -0,0 +1,176 @@ +use super::{Evaluator, ResultValue, RuntimeFault, Value, expressions::binary, value::Key}; +use crate::{ + hir::{CollectionOp as Op, Expression}, + specialize::Specialization, + syntax::BinaryOperator, + types::Type, +}; +use dever_runtime::{collections::List, number::DecimalValue}; + +impl Evaluator<'_> { + pub(super) fn collection( + &mut self, + operation: Op, + handler: Option<usize>, + arguments: Vec<Value>, + result: &Expression, + ) -> ResultValue { + if operation == Op::ParallelEach { + return self.parallel_each(handler, arguments, result); + } + let mut arguments = arguments.into_iter(); + let sequence = arguments.next().expect("collection source"); + match operation { + Op::Append => { + let Value::List(list) = sequence else { + unreachable!() + }; + return Ok(Value::List(list.append(arguments.next().unwrap()))); + } + Op::First => { + let Value::List(list) = sequence else { + unreachable!() + }; + return Ok(list.values().first().cloned().unwrap_or(Value::Null)); + } + Op::Get | Op::Put | Op::Remove => { + let Value::Map(map) = sequence else { + unreachable!() + }; + let key = Key::from_value(&arguments.next().unwrap()); + return Ok(match operation { + Op::Get => map.get(&key).unwrap_or(Value::Null), + Op::Put => Value::Map(map.put(key, arguments.next().unwrap())), + Op::Remove => Value::Map(map.remove(&key)), + _ => unreachable!(), + }); + } + Op::Entries => { + let Value::Map(map) = sequence else { + unreachable!() + }; + return Ok(Value::List(List::new( + map.pairs() + .map(|(key, value)| Value::Fields(vec![key.value(), value.clone()])) + .collect(), + ))); + } + Op::Length => { + return Ok(Value::Int(match sequence { + Value::List(list) => list.values().len() as i64, + Value::Map(map) => map.pairs().count() as i64, + Value::Text(text) => text.chars().count() as i64, + Value::Bytes(bytes) => bytes.length(), + _ => unreachable!("checked length input"), + })); + } + Op::Close => { + let Value::Stream(stream) = sequence else { + unreachable!() + }; + stream.close(); + return Ok(Value::Unit); + } + Op::Each | Op::Reduce | Op::ReduceUntil | Op::Filter | Op::Find | Op::Sum => {} + Op::ParallelEach => unreachable!("parallel schedule handled before traversal"), + } + let context = arguments.next(); + let reducing = matches!(operation, Op::Reduce | Op::ReduceUntil); + let mut state = if reducing { + context.clone().unwrap() + } else { + match result.ty { + Type::Int => Value::Int(0), + Type::Decimal => Value::Decimal(DecimalValue::ZERO), + Type::Float => Value::Float(0.0), + _ => Value::Unit, + } + }; + let mut mapped = Vec::new(); + let mut cursor = 0; + loop { + let element = match &sequence { + Value::List(list) => list.values().get(cursor).cloned(), + Value::Bytes(bytes) => bytes + .values() + .get(cursor) + .map(|byte| Value::Int(i64::from(*byte))), + Value::Stream(stream) => stream.pull(), + _ => unreachable!("checked sequence"), + }; + let Some(element) = element else { + break; + }; + cursor += 1; + let projected = if let Some(handler) = handler { + let mut inputs = vec![element.clone()]; + if reducing { + inputs.push(state.clone()); + } else if let Some(context) = &context { + inputs.push(context.clone()); + } + self.call( + Specialization { + function: handler, + handlers: vec![], + }, + inputs, + ) + .map_err(|fault| fault.at(result.span))? + } else { + element.clone() + }; + match operation { + Op::Each => { + if result.ty != Type::Unit { + mapped.push(projected); + } + } + Op::Filter => { + if projected.boolean() { + mapped.push(element); + } + } + Op::Find => { + if projected.boolean() { + return Ok(element); + } + } + Op::Sum => { + state = binary(state, BinaryOperator::Add, projected) + .map_err(|message| RuntimeFault::new(message, result.span))?; + } + Op::Reduce => state = projected, + Op::ReduceUntil => { + let stop = projected.fields()[1].boolean(); + state = projected.fields()[0].clone(); + if stop { + break; + } + } + _ => unreachable!(), + } + } + Ok(match operation { + Op::Each if result.ty == Type::Unit => Value::Unit, + Op::Each | Op::Filter => Value::List(List::new(mapped)), + Op::Find => Value::Null, + Op::Sum | Op::Reduce | Op::ReduceUntil => state, + _ => unreachable!(), + }) + } + + /// The oracle chooses a serial schedule. Concurrent side-effect ordering is deliberately + /// unspecified, so native concurrency tests must assert invariants rather than stdout order. + fn parallel_each( + &mut self, + handler: Option<usize>, + mut arguments: Vec<Value>, + result: &Expression, + ) -> ResultValue { + let workers = arguments.remove(1).int(); + dever_runtime::concurrent::worker_count(workers) + .map_err(|message| RuntimeFault::new(message, result.span))?; + self.collection(Op::Each, handler, arguments, result) + } +} diff --git a/crates/dever-core/src/reference/expressions.rs b/crates/dever-core/src/reference/expressions.rs new file mode 100644 index 0000000..feaa127 --- /dev/null +++ b/crates/dever-core/src/reference/expressions.rs @@ -0,0 +1,281 @@ +use super::{Evaluator, Frame, ResultValue, RuntimeFault, Value, value::Key}; +use crate::{ + hir::{CallArgument, CallTarget, Expression, ExpressionKind as Expr}, + specialize::{self, Specialization}, + syntax::{BinaryOperator as Op, UnaryOperator}, + types::{Shape, Type}, +}; +use dever_runtime::{ + collections::{List, Map}, + number::{self, DecimalValue}, +}; +use std::cmp::Ordering; + +impl Evaluator<'_> { + pub(super) fn expression(&mut self, expression: &Expression, frame: &Frame) -> ResultValue { + Ok(match &expression.kind { + Expr::Constant(crate::hir::Constant::Text(value)) if expression.ty == Type::Uuid => { + Value::Uuid(dever_runtime::orm::Uuid::parse(value).expect("checked Uuid literal")) + } + Expr::Constant(value) => Value::constant(value), + Expr::Local(slot) => frame.locals[*slot] + .as_ref() + .expect("initialized local") + .clone(), + Expr::Field { value, index } => self.expression(value, frame)?.fields()[*index].clone(), + Expr::ChoiceOptions { choice } => { + let Shape::Choice(variants) = &self.program.types[*choice].shape else { + unreachable!("checked Model choice options") + }; + Value::Map(Map::from_unique( + variants + .iter() + .enumerate() + .map(|(variant, definition)| { + ( + super::value::Key::Variant(*choice, variant), + Value::Text( + definition + .label + .as_deref() + .unwrap_or(&definition.name) + .to_owned(), + ), + ) + }) + .collect(), + )) + } + Expr::Some(value) => self.expression(value, frame)?, + Expr::Promote(value) => { + Value::Decimal(DecimalValue::from_int(self.expression(value, frame)?.int())) + } + Expr::List(values) => Value::List(List::new(self.arguments(values, frame)?)), + Expr::Map(entries) => { + let mut pairs = Vec::with_capacity(entries.len()); + for (key, value) in entries { + let key = self.expression(key, frame)?; + let value = self.expression(value, frame)?; + pairs.push((Key::from_value(&key), value)); + } + Value::Map( + Map::new(pairs) + .map_err(|message| RuntimeFault::new(message, expression.span))?, + ) + } + Expr::Record { fields } => { + let Type::Named(id) = expression.ty else { + unreachable!() + }; + let Shape::Record(definition) = &self.program.types[id].shape else { + unreachable!() + }; + let mut values = vec![Value::Unit; definition.len()]; + // Storage order follows the declaration, execution follows the source initializer. + for (index, value) in fields { + values[*index] = self.expression(value, frame)?; + } + Value::Record(id, values) + } + Expr::Variant { variant, arguments } => { + let Type::Named(id) = expression.ty else { + unreachable!() + }; + Value::Variant(id, *variant, self.arguments(arguments, frame)?) + } + Expr::Fail(error) => { + let error = self.expression(error, frame)?; + return Err(RuntimeFault::business( + error.clone(), + error.render(self.program), + expression.span, + )); + } + Expr::Call { target, arguments } => { + let instance = match target { + CallTarget::Function(function) => { + specialize::call_specialization(*function, arguments, &frame.handlers) + } + CallTarget::Handler(target) => Specialization { + function: specialize::resolve_handler(*target, &frame.handlers), + handlers: vec![], + }, + }; + let mut values = Vec::new(); + for argument in arguments { + if let CallArgument::Value(value) = argument { + values.push(self.expression(value, frame)?); + } + } + self.call(instance, values) + .map_err(|fault| fault.at(expression.span))? + } + Expr::CaptureResult { + target, + arguments, + choice, + success, + } => { + let instance = match target { + CallTarget::Function(function) => { + specialize::call_specialization(*function, arguments, &frame.handlers) + } + CallTarget::Handler(target) => Specialization { + function: specialize::resolve_handler(*target, &frame.handlers), + handlers: vec![], + }, + }; + let mut values = Vec::new(); + for argument in arguments { + if let CallArgument::Value(value) = argument { + values.push(self.expression(value, frame)?); + } + } + let failures = specialize::failures(self.program, &instance); + let targets = crate::capture::targets(&self.program.types, *choice, &failures) + .expect("checked capture"); + match self.call(instance, values) { + Ok(Value::Unit) => Value::Variant(*choice, *success, Vec::new()), + Ok(Value::Fields(values)) => Value::Variant(*choice, *success, values), + Ok(value) => Value::Variant(*choice, *success, vec![value]), + Err(fault) => fault + .into_business(&targets, *choice) + .map_err(|fault| fault.at(expression.span))?, + } + } + Expr::RunCall { .. } + | Expr::ModelOperation { .. } + | Expr::JobEnqueue { .. } + | Expr::AwaitTask(_) + | Expr::StopTask(_) + | Expr::Group(_) + | Expr::AwaitGroup(_) + | Expr::StopGroup(_) + | Expr::ParallelCall { .. } + | Expr::BlockingCall { .. } + | Expr::Channel { .. } + | Expr::ChannelSend { .. } + | Expr::ChannelReceive(_) + | Expr::ChannelClose(_) => { + return Err(RuntimeFault::new( + "the reference evaluator does not execute async operations", + expression.span, + )); + } + Expr::Intrinsic { + operation, + arguments, + .. + } => { + let arguments = self.arguments(arguments, frame)?; + self.intrinsic(*operation, arguments, expression)? + } + Expr::Collection { + operation, + handler, + sequence: _, + arguments, + } => { + let arguments = self.arguments(arguments, frame)?; + let handler = + handler.map(|target| specialize::resolve_handler(target, &frame.handlers)); + self.collection(*operation, handler, arguments, expression)? + } + Expr::Unary { operator, value } => { + let value = self.expression(value, frame)?; + match (operator, value) { + (UnaryOperator::Not, Value::Bool(value)) => Value::Bool(!value), + (UnaryOperator::Negate, Value::Float(value)) => Value::Float(-value), + (UnaryOperator::Negate, Value::Int(value)) => Value::Int( + number::int_neg(value) + .map_err(|message| RuntimeFault::new(message, expression.span))?, + ), + (UnaryOperator::Negate, Value::Decimal(value)) => Value::Decimal( + value + .checked_neg() + .map_err(|message| RuntimeFault::new(message, expression.span))?, + ), + _ => unreachable!("checked unary operator"), + } + } + Expr::Binary { + left, + operator, + right, + } => { + let left = self.expression(left, frame)?; + if *operator == Op::And && !left.boolean() { + return Ok(Value::Bool(false)); + } + if *operator == Op::Or && left.boolean() { + return Ok(Value::Bool(true)); + } + let right = self.expression(right, frame)?; + binary(left, *operator, right) + .map_err(|message| RuntimeFault::new(message, expression.span))? + } + }) + } + + fn arguments( + &mut self, + expressions: &[Expression], + frame: &Frame, + ) -> Result<Vec<Value>, RuntimeFault> { + expressions + .iter() + .map(|expression| self.expression(expression, frame)) + .collect() + } +} + +pub(super) fn binary(left: Value, operator: Op, right: Value) -> Result<Value, &'static str> { + match operator { + Op::Equal => return Ok(Value::Bool(left == right)), + Op::NotEqual => return Ok(Value::Bool(left != right)), + Op::And | Op::Or => return Ok(right), + Op::Less | Op::LessEqual | Op::Greater | Op::GreaterEqual => { + let ordering = match (&left, &right) { + (Value::Int(a), Value::Int(b)) => a.partial_cmp(b), + (Value::Decimal(a), Value::Decimal(b)) => a.partial_cmp(b), + (Value::Float(a), Value::Float(b)) => a.partial_cmp(b), + (Value::Text(a), Value::Text(b)) => a.partial_cmp(b), + _ => unreachable!("checked ordered operands"), + }; + return Ok(Value::Bool(match operator { + Op::Less => ordering == Some(Ordering::Less), + Op::LessEqual => matches!(ordering, Some(Ordering::Less | Ordering::Equal)), + Op::Greater => ordering == Some(Ordering::Greater), + Op::GreaterEqual => matches!(ordering, Some(Ordering::Greater | Ordering::Equal)), + _ => unreachable!(), + })); + } + _ => {} + } + Ok(match (left, right) { + (Value::Int(a), Value::Int(b)) => Value::Int(match operator { + Op::Add => number::int_add(a, b)?, + Op::Subtract => number::int_sub(a, b)?, + Op::Multiply => number::int_mul(a, b)?, + Op::IntegerDivide => number::int_div(a, b)?, + Op::Remainder => number::int_rem(a, b)?, + _ => unreachable!("checked Int operator"), + }), + (Value::Decimal(a), Value::Decimal(b)) => Value::Decimal(match operator { + Op::Add => a.checked_add(b)?, + Op::Subtract => a.checked_sub(b)?, + Op::Multiply => a.checked_mul(b)?, + Op::Divide => a.checked_div(b)?, + _ => unreachable!("checked Decimal operator"), + }), + (Value::Float(a), Value::Float(b)) => Value::Float(match operator { + Op::Add => a + b, + Op::Subtract => a - b, + Op::Multiply => a * b, + Op::Divide => a / b, + _ => unreachable!("checked Float operator"), + }), + (Value::Text(a), Value::Text(b)) => Value::Text(a + &b), + _ => unreachable!("checked arithmetic operands"), + }) +} diff --git a/crates/dever-core/src/reference/intrinsics.rs b/crates/dever-core/src/reference/intrinsics.rs new file mode 100644 index 0000000..3f7d8c8 --- /dev/null +++ b/crates/dever-core/src/reference/intrinsics.rs @@ -0,0 +1,345 @@ +use super::{Evaluator, ResultValue, RuntimeFault, Value}; +use crate::{ + hir::Expression, + intrinsic::Intrinsic, + types::{Shape, Type, Variant}, +}; +use dever_runtime::{ + Id, + bytes::Bytes, + collections::List, + number::{self, DecimalValue}, + process, resource, text, time, +}; + +impl Evaluator<'_> { + pub(super) fn intrinsic( + &mut self, + operation: Intrinsic, + arguments: Vec<Value>, + result: &Expression, + ) -> ResultValue { + use Intrinsic::*; + let args = &arguments; + let recover = |outcome: Result<Value, String>, success| { + self.resource_result(outcome, &result.ty, success) + }; + Ok(match operation { + TestSecret => Value::Secret(dever_runtime::secret::Secret::from_input( + args[0].text().as_bytes().to_vec(), + )), + CryptoToken => recover( + dever_runtime::crypto::token(args[0].int()).map(Value::Secret), + "Ready", + ), + CryptoPasswordHash => recover( + dever_runtime::crypto::password_hash(args[0].secret()).map(Value::Text), + "Ready", + ), + CryptoPasswordVerify => recover( + dever_runtime::crypto::password_verify(args[0].secret(), args[1].text()) + .map(Value::Bool), + "Ready", + ), + CryptoSha256 => Value::Bytes(dever_runtime::crypto::sha256(args[0].bytes())), + CryptoHmacSha256 => Value::Bytes(dever_runtime::crypto::hmac_sha256( + args[0].secret(), + args[1].bytes(), + )), + CryptoConstantTimeEq => Value::Bool(dever_runtime::crypto::constant_time_eq( + args[0].bytes(), + args[1].bytes(), + )), + TestAssert => { + if !args[0].boolean() { + return Err(RuntimeFault::new("assertion failed", result.span)); + } + Value::Unit + } + TestAssertEq => { + if args[0] != args[1] { + return Err(RuntimeFault::new( + format!( + "assert_eq failed: actual = {}, expected = {}", + args[0].render(self.program), + args[1].render(self.program) + ), + result.span, + )); + } + Value::Unit + } + TextTrim => Value::Text(args[0].text().trim().to_owned()), + TextLower => Value::Text(args[0].text().to_lowercase()), + TextUpper => Value::Text(args[0].text().to_uppercase()), + TextCodepoint => optional(text::codepoint(args[0].text()), Value::Int), + TextFromCodepoint => optional(text::from_codepoint(args[0].int()), Value::Text), + TextAt => optional(text::at(args[0].text(), args[1].int()), Value::Text), + TextSlice => optional( + text::slice(args[0].text(), args[1].int(), args[2].int()), + Value::Text, + ), + TextIndexOf => optional(text::index_of(args[0].text(), args[1].text()), Value::Int), + TextContains => Value::Bool(args[0].text().contains(args[1].text())), + TextStartsWith => Value::Bool(args[0].text().starts_with(args[1].text())), + TextEndsWith => Value::Bool(args[0].text().ends_with(args[1].text())), + TextSplit => text_list(text::split(args[0].text(), args[1].text())), + TextReplace => Value::Text(args[0].text().replace(args[1].text(), args[2].text())), + IntParse => optional(number::parse_int(args[0].text()), Value::Int), + DecimalParse => optional(number::parse_decimal(args[0].text()), Value::Decimal), + FloatParse => optional(number::parse_float(args[0].text()), Value::Float), + IntToText | DecimalToText | FloatToText => Value::Text(args[0].render(self.program)), + DecimalFromInt => Value::Decimal(DecimalValue::from_int(args[0].int())), + ProcessArguments => recover(process::arguments().map(text_list), "Read"), + TimeUnixMillis => recover(time::unix_millis().map(Value::Int), "Read"), + TimeNow => recover(time::now().map(Value::Int), "Read"), + TimeParseDateTime => { + recover(time::parse_datetime(args[0].text()).map(Value::Int), "Read") + } + TimeFormatDateTime => recover( + time::format_datetime(args[0].int()).map(Value::Text), + "Read", + ), + TimeParseDate => recover(time::parse_date(args[0].text()).map(Value::Int), "Read"), + TimeFormatDate => recover(time::format_date(args[0].int()).map(Value::Text), "Read"), + TimeParseTime => recover(time::parse_time(args[0].text()).map(Value::Int), "Read"), + TimeFormatTime => recover(time::format_time(args[0].int()).map(Value::Text), "Read"), + TimeDuration => Value::Int(time::duration(args[0].int())), + TimeAdd => recover( + time::add(args[0].int(), args[1].int()).map(Value::Int), + "Read", + ), + TimeSubtract => recover( + time::subtract(args[0].int(), args[1].int()).map(Value::Int), + "Read", + ), + TimeDifference => recover( + time::difference(args[0].int(), args[1].int()).map(Value::Int), + "Read", + ), + TimeMonotonicNanos => recover(time::monotonic_nanos().map(Value::Int), "Read"), + TimeSleep => recover(time::sleep(args[0].int()).map(|()| Value::Unit), "Done"), + TaskTicks + | TaskTimeout + | TaskRace + | StreamOf + | TlsSystem + | TlsClient + | TlsServer + | HttpClientNew + | HttpRequest + | HttpOpen + | HttpOpenStream + | HttpCloseClient + | HttpServeTls + | HttpServeLiveTls + | WsOpen + | AuthIssue + | AuthIssueCookie + | AuthClearCookie + | AuthOwnsUser + | AuthId + | AuthSession + | AuthUserId + | AuthTenantId + | SiteKey + | AuthPermissions + | AuthSaveRole + | AuthGrantRole + | AuthRevokeRole + | AuthDisableRole + | ApiRequestId + | ApiMethod + | ApiPath + | ApiClientAddress + | ApiHeader + | ApiCookie + | ApiSecretCookie + | ApiResponseHeader + | ApiResponseCookie + | ApiResponseSecretCookie + | UploadFilename + | UploadContentType + | UploadSize + | UploadClose + | UploadStore + | HttpServe + | ApiServe + | JobServe + | TestJobDrain + | TestClockAdvance + | HttpSend + | HttpServeLive + | HttpRespond + | HttpStart + | HttpWrite + | HttpFinish + | SseStart + | SseSend + | WsAccept + | WsConnect + | WsSend + | WsReceive + | WsMessages + | WsClose + | TaskSleep + | TcpConnect + | TcpConnectTimeout + | TcpListen + | TcpAccept + | TcpPort + | TcpRead + | TcpWrite + | TcpTimeout + | TcpClose + | TcpCloseListener + | TcpChunks + | TcpConnections => { + return Err(RuntimeFault::new( + "the reference evaluator does not execute async operations", + result.span, + )); + } + DecimalRound => Value::Decimal( + args[0] + .decimal() + .round(args[1].int()) + .map_err(|message| RuntimeFault::new(message, result.span))?, + ), + FloatFromInt => Value::Float(args[0].int() as f64), + FloatSqrt => Value::Float(args[0].float().sqrt()), + FloatSin => Value::Float(args[0].float().sin()), + FloatCos => Value::Float(args[0].float().cos()), + FloatLog => Value::Float(args[0].float().ln()), + FloatPow => Value::Float(args[0].float().powf(args[1].float())), + FloatIsNan => Value::Bool(args[0].float().is_nan()), + FloatIsFinite => Value::Bool(args[0].float().is_finite()), + FloatIsInfinite => Value::Bool(args[0].float().is_infinite()), + StdoutWrite => { + self.stdout.push_str(args[0].text()); + Value::Unit + } + LogDebug | LogInfo | LogWarn | LogError => Value::Unit, + IdFromText => Value::Id(Id(args[0].text().to_owned())), + IdToText => { + let Value::Id(id) = &args[0] else { + unreachable!() + }; + Value::Text(id.0.clone()) + } + UuidParse => optional( + dever_runtime::orm::Uuid::parse(args[0].text()).ok(), + Value::Uuid, + ), + UuidToText => Value::Text(args[0].uuid().to_string()), + BytesFromText => Value::Bytes(Bytes::from_text(args[0].text())), + BytesLength => Value::Int(args[0].bytes().length()), + BytesAt => optional(args[0].bytes().at(args[1].int()), Value::Int), + BytesConcat => Value::Bytes(args[0].bytes().clone().concat(args[1].bytes())), + BytesToText => recover(args[0].bytes().to_text().map(Value::Text), "Decoded"), + BytesFromInts => { + let Value::List(values) = &args[0] else { + unreachable!() + }; + recover( + Bytes::from_ints(&values.values().iter().map(Value::int).collect::<Vec<_>>()) + .map(Value::Bytes) + .map_err(str::to_owned), + "Ready", + ) + } + BytesSlice => recover( + args[0] + .bytes() + .slice(args[1].int(), args[2].int()) + .map(Value::Bytes) + .map_err(str::to_owned), + "Ready", + ), + FileOpen => recover( + resource::file_open(args[0].text()).map(Value::File), + "Opened", + ), + FileCreate => recover( + resource::file_create(args[0].text()).map(Value::File), + "Opened", + ), + FileRead => match resource::read(args[0].file(), args[1].int()) { + Ok(Some(bytes)) => self.choice(&result.ty, "Read", vec![Value::Bytes(bytes)]), + Ok(None) => self.choice(&result.ty, "End", vec![]), + Err(message) => recover(Err(message), "Read"), + }, + FileWrite => recover( + resource::write(args[0].file(), args[1].bytes()).map(|()| Value::Unit), + "Done", + ), + FileClose => recover(args[0].file().close().map(|()| Value::Unit), "Closed"), + FileChunks => self.read_stream(args, &result.ty), + }) + } + + fn read_stream(&self, arguments: &[Value], output: &Type) -> Value { + let outcome = resource::read_chunks(arguments[0].file().clone(), arguments[1].int()); + let (_, variants) = self.variants(output); + let Type::Stream(element) = &variants[variant(variants, "Streaming")].fields[0].ty else { + unreachable!() + }; + let (id, events) = self.variants(element); + let chunk = variant(events, "Chunk"); + let failed = variant(events, "Failed"); + self.resource_result( + outcome.map(|stream| { + Value::Stream(stream.map(move |event| match event { + Ok(bytes) => Value::Variant(id, chunk, vec![Value::Bytes(bytes)]), + Err(message) => Value::Variant(id, failed, vec![Value::Text(message)]), + })) + }), + output, + "Streaming", + ) + } + + fn resource_result( + &self, + outcome: Result<Value, String>, + output: &Type, + success: &str, + ) -> Value { + match outcome { + Ok(Value::Unit) => self.choice(output, success, vec![]), + Ok(value) => self.choice(output, success, vec![value]), + Err(message) => self.choice(output, "Failed", vec![Value::Text(message)]), + } + } + + fn choice(&self, output: &Type, name: &str, payload: Vec<Value>) -> Value { + let (id, variants) = self.variants(output); + Value::Variant(id, variant(variants, name), payload) + } + + fn variants(&self, output: &Type) -> (usize, &[Variant]) { + let Type::Named(id) = output else { + unreachable!("checked result type") + }; + let Shape::Choice(variants) = &self.program.types[*id].shape else { + unreachable!() + }; + (*id, variants) + } +} + +fn variant(variants: &[Variant], name: &str) -> usize { + variants + .iter() + .position(|variant| variant.name == name) + .expect("checked result variant") +} + +fn optional<T>(value: Option<T>, wrap: impl FnOnce(T) -> Value) -> Value { + value.map(wrap).unwrap_or(Value::Null) +} +fn text_list(values: List<String>) -> Value { + Value::List(List::new( + values.values().iter().cloned().map(Value::Text).collect(), + )) +} diff --git a/crates/dever-core/src/reference/value.rs b/crates/dever-core/src/reference/value.rs new file mode 100644 index 0000000..77cfc0d --- /dev/null +++ b/crates/dever-core/src/reference/value.rs @@ -0,0 +1,237 @@ +use crate::{ + hir::{Constant, Program}, + types::Shape, +}; +use dever_runtime::{ + Id, + bytes::Bytes, + collections::{List, Map}, + number::DecimalValue, + orm::Uuid, + render::Render, + resource::{FileHandle, Stream}, +}; + +#[derive(Clone, Debug)] +pub(super) enum Value { + Unit, + Null, + Bool(bool), + Int(i64), + Decimal(DecimalValue), + Float(f64), + Text(String), + Id(Id), + Uuid(Uuid), + Bytes(Bytes), + Secret(dever_runtime::secret::Secret), + Record(usize, Vec<Value>), + Variant(usize, usize, Vec<Value>), + Fields(Vec<Value>), + List(List<Value>), + Map(Map<Key, Value>), + File(FileHandle), + Stream(Stream<Value>), +} + +/// Only statically eligible Map keys cross the runtime's Eq + Hash boundary. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub(super) enum Key { + Bool(bool), + Int(i64), + Text(String), + Id(Id), + Variant(usize, usize), +} + +impl Key { + pub fn from_value(value: &Value) -> Self { + match value { + Value::Bool(v) => Self::Bool(*v), + Value::Int(v) => Self::Int(*v), + Value::Text(v) => Self::Text(v.clone()), + Value::Id(v) => Self::Id(v.clone()), + Value::Variant(id, variant, fields) if fields.is_empty() => { + Self::Variant(*id, *variant) + } + _ => unreachable!("checked Map key"), + } + } + pub fn value(&self) -> Value { + match self { + Self::Bool(v) => Value::Bool(*v), + Self::Int(v) => Value::Int(*v), + Self::Text(v) => Value::Text(v.clone()), + Self::Id(v) => Value::Id(v.clone()), + Self::Variant(id, variant) => Value::Variant(*id, *variant, vec![]), + } + } +} + +impl Value { + pub fn secret(&self) -> &dever_runtime::secret::Secret { + match self { + Self::Secret(secret) => secret, + _ => unreachable!("checked Secret"), + } + } + pub fn file(&self) -> &FileHandle { + match self { + Self::File(file) => file, + _ => unreachable!("checked File"), + } + } + + pub fn constant(value: &Constant) -> Self { + match value { + Constant::Null => Self::Null, + Constant::Bool(v) => Self::Bool(*v), + Constant::Int(v) => Self::Int(*v), + Constant::Decimal(v) => Self::Decimal(*v), + Constant::Float(v) => Self::Float(*v), + Constant::Text(v) => Self::Text(v.clone()), + } + } + pub fn fields(&self) -> &[Self] { + match self { + Self::Record(_, fields) | Self::Variant(_, _, fields) | Self::Fields(fields) => fields, + _ => unreachable!("checked fields"), + } + } + pub fn fields_mut(&mut self) -> &mut Vec<Self> { + match self { + Self::Record(_, fields) | Self::Fields(fields) => fields, + _ => unreachable!("checked mutable fields"), + } + } + pub fn int(&self) -> i64 { + match self { + Self::Int(v) => *v, + _ => unreachable!("checked Int"), + } + } + pub fn float(&self) -> f64 { + match self { + Self::Float(v) => *v, + _ => unreachable!("checked Float"), + } + } + pub fn text(&self) -> &str { + match self { + Self::Text(v) => v, + _ => unreachable!("checked Text"), + } + } + pub fn bytes(&self) -> &Bytes { + match self { + Self::Bytes(v) => v, + _ => unreachable!("checked Bytes"), + } + } + pub fn uuid(&self) -> Uuid { + match self { + Self::Uuid(v) => *v, + _ => unreachable!("checked Uuid"), + } + } + pub fn boolean(&self) -> bool { + match self { + Self::Bool(v) => *v, + _ => unreachable!("checked Bool"), + } + } + pub fn decimal(&self) -> DecimalValue { + match self { + Self::Decimal(v) => *v, + _ => unreachable!("checked Decimal"), + } + } + pub fn render(&self, program: &Program) -> String { + let joined = |values: &[Value]| { + values + .iter() + .map(|v| v.render(program)) + .collect::<Vec<_>>() + .join(", ") + }; + match self { + Self::Unit => "()".into(), + Self::Null => "null".into(), + Self::Bool(v) => v.render(), + Self::Int(v) => v.render(), + Self::Decimal(v) => v.render(), + Self::Float(v) => v.render(), + Self::Text(v) => v.clone(), + Self::Id(v) => v.render(), + Self::Uuid(v) => v.to_string(), + Self::Bytes(v) => v.render(), + Self::Secret(_) => unreachable!("Secret cannot cross a checked render boundary"), + Self::List(v) => format!("[{}]", joined(v.values())), + Self::Map(v) => format!( + "{{{}}}", + v.pairs() + .map(|(key, value)| format!( + "{} = {}", + key.value().render(program), + value.render(program) + )) + .collect::<Vec<_>>() + .join(", ") + ), + Self::Fields(v) => format!( + "{{key = {}, value = {}}}", + v[0].render(program), + v[1].render(program) + ), + Self::Record(id, values) => { + let definition = &program.types[*id]; + let Shape::Record(fields) = &definition.shape else { + unreachable!() + }; + let fields = fields + .iter() + .zip(values) + .map(|(field, value)| format!("{} = {}", field.name, value.render(program))) + .collect::<Vec<_>>() + .join(", "); + format!("{} {{ {fields} }}", definition.name) + } + Self::Variant(id, variant, values) => { + let definition = &program.types[*id]; + let Shape::Choice(variants) = &definition.shape else { + unreachable!() + }; + let name = format!("{}.{}", definition.name, variants[*variant].name); + if values.is_empty() { + name + } else { + format!("{name}({})", joined(values)) + } + } + Self::File(_) => "<resource>".into(), + Self::Stream(_) => "<stream>".into(), + } + } +} + +impl PartialEq for Value { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Self::Unit, Self::Unit) | (Self::Null, Self::Null) => true, + (Self::Bool(a), Self::Bool(b)) => a == b, + (Self::Int(a), Self::Int(b)) => a == b, + (Self::Decimal(a), Self::Decimal(b)) => a == b, + (Self::Float(a), Self::Float(b)) => a == b, + (Self::Text(a), Self::Text(b)) => a == b, + (Self::Id(a), Self::Id(b)) => a == b, + (Self::Uuid(a), Self::Uuid(b)) => a == b, + (Self::Bytes(a), Self::Bytes(b)) => a == b, + (Self::List(a), Self::List(b)) => a == b, + (Self::Map(a), Self::Map(b)) => a == b, + (Self::Fields(a), Self::Fields(b)) => a == b, + (Self::Record(a, av), Self::Record(b, bv)) => a == b && av == bv, + (Self::Variant(a, ai, av), Self::Variant(b, bi, bv)) => a == b && ai == bi && av == bv, + _ => false, + } + } +} diff --git a/crates/dever-core/src/source.rs b/crates/dever-core/src/source.rs new file mode 100644 index 0000000..9ed417a --- /dev/null +++ b/crates/dever-core/src/source.rs @@ -0,0 +1,726 @@ +use std::fmt; +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::OnceLock; + +const STANDARD_SOURCE: usize = 1 << (usize::BITS - 1); + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum SourceOrigin { + Application, + Test, +} + +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub struct SourceId(pub(crate) usize); + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct Span { + pub source: SourceId, + pub start: usize, + pub end: usize, +} + +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub(crate) enum SourceRole { + App, + Domain, + Model, + Port, + Adapter, + Api, + Job, +} + +impl SourceRole { + pub(crate) fn name(self) -> &'static str { + match self { + Self::App => "app", + Self::Domain => "domain", + Self::Model => "model", + Self::Port => "port", + Self::Adapter => "adapter", + Self::Api => "api", + Self::Job => "job", + } + } + + fn from_name(name: &str) -> Option<Self> { + match name { + "app" => Some(Self::App), + "domain" => Some(Self::Domain), + "model" => Some(Self::Model), + "port" => Some(Self::Port), + "adapter" => Some(Self::Adapter), + "api" => Some(Self::Api), + "job" => Some(Self::Job), + _ => None, + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum SourceLayout { + Main, + Test { + component: String, + domain: String, + topic: String, + }, + Role { + component: String, + domain: String, + role: SourceRole, + topics: Vec<String>, + }, + Loose, +} + +impl SourceLayout { + pub(crate) fn role(&self) -> Option<SourceRole> { + match self { + Self::Role { role, .. } => Some(*role), + Self::Main | Self::Test { .. } | Self::Loose => None, + } + } + + pub(crate) fn domain(&self) -> Option<(&str, &str)> { + match self { + Self::Role { + component, domain, .. + } + | Self::Test { + component, domain, .. + } => Some((component, domain)), + Self::Main | Self::Loose => None, + } + } + + pub(crate) fn app_prefix(&self) -> Option<String> { + self.domain() + .map(|(component, domain)| format!("{component}.{domain}")) + } +} + +impl Span { + pub(crate) fn through(self, last: Span) -> Self { + debug_assert_eq!(self.source, last.source); + Self { + end: last.end, + ..self + } + } +} + +#[derive(Debug)] +pub struct SourceFile { + id: SourceId, + path: PathBuf, + logical_path: PathBuf, + text: String, + line_starts: Vec<usize>, + strict_layout: bool, + origin: SourceOrigin, +} + +impl SourceFile { + pub fn id(&self) -> SourceId { + self.id + } + + pub fn path(&self) -> &Path { + &self.path + } + + pub(crate) fn logical_path(&self) -> &Path { + &self.logical_path + } + + pub(crate) fn origin(&self) -> SourceOrigin { + self.origin + } + + pub fn text(&self) -> &str { + &self.text + } + + pub fn is_test(&self) -> bool { + self.origin == SourceOrigin::Test + } + + pub(crate) fn is_markdown(&self) -> bool { + is_markdown_path(&self.path) + } + + pub(crate) fn is_model(&self) -> bool { + self.layout().is_ok_and(|layout| { + layout.role() == Some(SourceRole::Model) + || matches!(layout, SourceLayout::Loose) + && self + .path + .parent() + .and_then(Path::file_name) + .is_some_and(|name| name == "model") + }) + } + + pub(crate) fn layout(&self) -> Result<SourceLayout, crate::diagnostic::Diagnostic> { + let segments = self.path_segments()?; + if self.origin == SourceOrigin::Test { + let [component, domain, topic] = segments.as_slice() else { + return Err(crate::diagnostic::Diagnostic::error( + "C003", + "test source must be <component>/<domain>/<topic>.dever or .dever.md", + self.span(0, 0), + )); + }; + return Ok(SourceLayout::Test { + component: (*component).to_owned(), + domain: (*domain).to_owned(), + topic: (*topic).to_owned(), + }); + } + if segments.as_slice() == ["main"] { + return Ok(SourceLayout::Main); + } + if segments.len() < 3 { + return Ok(SourceLayout::Loose); + } + let Some(role) = SourceRole::from_name(segments[2]) else { + return Ok(SourceLayout::Loose); + }; + Ok(SourceLayout::Role { + component: segments[0].to_owned(), + domain: segments[1].to_owned(), + role, + topics: segments[3..] + .iter() + .map(|segment| (*segment).to_owned()) + .collect(), + }) + } + + pub(crate) fn strict_layout(&self) -> bool { + self.strict_layout + } + + pub(crate) fn package_path( + &self, + ) -> Result<crate::syntax::Path, crate::diagnostic::Diagnostic> { + let segments = self.path_segments()?; + Ok(segments + .into_iter() + .map(|segment| crate::syntax::Name { + text: segment.to_owned(), + span: self.span(0, 0), + }) + .collect()) + } + + fn path_segments(&self) -> Result<Vec<&str>, crate::diagnostic::Diagnostic> { + let invalid_path = || { + crate::diagnostic::Diagnostic::error( + "C003", + "source must have a relative .dever or .dever.md path below the source root", + self.span(0, 0), + ) + }; + let mut segments = self + .logical_path + .components() + .map(|component| match component { + std::path::Component::Normal(segment) => segment.to_str().ok_or_else(invalid_path), + _ => Err(invalid_path()), + }) + .collect::<Result<Vec<_>, _>>()?; + let filename = segments.pop().ok_or_else(invalid_path)?; + let stem = filename + .strip_suffix(".dever.md") + .or_else(|| filename.strip_suffix(".dever")) + .filter(|stem| !stem.is_empty()) + .ok_or_else(invalid_path)?; + segments.push(stem); + Ok(segments) + } + + /// Returns a one-based line and Unicode scalar column for a byte boundary. + pub fn position(&self, offset: usize) -> (usize, usize) { + let line = self.line_starts.partition_point(|start| *start <= offset) - 1; + let column = self.text[self.line_starts[line]..offset].chars().count() + 1; + (line + 1, column) + } + + pub fn line(&self, one_based_line: usize) -> &str { + let start = self.line_starts[one_based_line - 1]; + let end = self + .line_starts + .get(one_based_line) + .copied() + .unwrap_or(self.text.len()); + self.text[start..end].trim_end_matches(['\r', '\n']) + } + + pub(crate) fn span(&self, start: usize, end: usize) -> Span { + Span { + source: self.id, + start, + end, + } + } +} + +#[derive(Debug, Default)] +pub struct SourceMap { + files: Vec<SourceFile>, + strict_layout: bool, +} + +/// Verified Package source. `path` has the same component-relative identity +/// as a file below the application's `module/` directory. +#[derive(Clone, Debug)] +pub struct PackageSource { + pub path: PathBuf, + pub display: PathBuf, + pub text: String, +} + +/// Portable application/test inputs. Standard sources are always compiler-owned. +#[derive(Clone, Debug)] +pub struct CompilationSource { + pub logical_path: String, + pub is_test: bool, + pub text: String, +} + +impl SourceMap { + pub fn compilation_sources(&self) -> Result<Vec<CompilationSource>, String> { + self.files + .iter() + .map(|source| { + let logical_path = source + .logical_path + .to_str() + .ok_or("compilation source paths must be UTF-8")? + .replace('\\', "/"); + Ok(CompilationSource { + logical_path, + is_test: source.is_test(), + text: source.text.clone(), + }) + }) + .collect() + } + + pub fn from_compilation_sources(inputs: &[CompilationSource]) -> Result<Self, String> { + let mut sources = Self { + strict_layout: true, + ..Self::default() + }; + let mut paths = std::collections::BTreeSet::new(); + for input in inputs { + let path = &input.logical_path; + if path.is_empty() + || path.contains(['\\', ':', '\0']) + || path + .split('/') + .any(|part| part.is_empty() || part == "." || part == "..") + || !is_source_path(Path::new(path)) + { + return Err("compilation source paths must be normalized relative .dever or .dever.md paths".into()); + } + if !paths.insert((input.is_test, path)) { + return Err(format!("duplicate compilation source '{path}'")); + } + let origin = if input.is_test { + SourceOrigin::Test + } else { + SourceOrigin::Application + }; + let display = if input.is_test { + Path::new("test").join(path) + } else { + PathBuf::from(path) + }; + sources.add_source(display, PathBuf::from(path), input.text.clone(), origin); + } + finish_load(sources, Vec::new()).map_err(|_| "invalid compilation sources".into()) + } + + pub fn add(&mut self, path: impl Into<PathBuf>, text: impl Into<String>) -> SourceId { + let path = path.into(); + self.add_source(path.clone(), path, text.into(), SourceOrigin::Application) + } + + fn add_source( + &mut self, + path: PathBuf, + logical_path: PathBuf, + text: String, + origin: SourceOrigin, + ) -> SourceId { + let mut line_starts = vec![0]; + let bytes = text.as_bytes(); + line_starts.extend(bytes.iter().enumerate().filter_map(|(offset, byte)| { + let newline = + *byte == b'\n' || (*byte == b'\r' && bytes.get(offset + 1) != Some(&b'\n')); + newline.then_some(offset + 1) + })); + let id = SourceId(self.files.len()); + self.files.push(SourceFile { + id, + path, + logical_path, + text, + line_starts, + strict_layout: self.strict_layout, + origin, + }); + id + } + + pub fn get(&self, id: SourceId) -> &SourceFile { + if Self::is_standard(id) { + &standard_sources()[id.0 & !STANDARD_SOURCE] + } else { + &self.files[id.0] + } + } + + pub fn is_standard(id: SourceId) -> bool { + id.0 & STANDARD_SOURCE != 0 + } + + pub(crate) fn all_files(&self) -> impl Iterator<Item = &SourceFile> { + self.files.iter().chain(standard_sources()) + } + + pub fn files(&self) -> &[SourceFile] { + &self.files + } + + pub(crate) fn strict_layout(&self) -> bool { + self.strict_layout + } + + pub fn load(root: &Path) -> Result<Self, Vec<LoadError>> { + Self::load_with_packages(root, None, &[]) + } + + pub fn load_with_packages( + module_root: &Path, + test_root: Option<&Path>, + packages: &[PackageSource], + ) -> Result<Self, Vec<LoadError>> { + let mut sources = Self { + strict_layout: true, + ..Self::default() + }; + let mut errors = Vec::new(); + let package_only = !packages.is_empty() + && fs::symlink_metadata(module_root) + .is_err_and(|error| error.kind() == std::io::ErrorKind::NotFound); + if !package_only { + sources.load_root(module_root, None, SourceOrigin::Application, &mut errors); + } + if let Some(test_root) = test_root.filter(|root| root.exists()) { + sources.load_root( + test_root, + Some(Path::new("test")), + SourceOrigin::Test, + &mut errors, + ); + } + for package in packages { + if package + .path + .components() + .any(|part| !matches!(part, std::path::Component::Normal(_))) + || !is_source_path(&package.path) + { + errors.push(LoadError { + path: package.display.clone(), + message: "invalid Package source path".into(), + }); + continue; + } + if sources.files.iter().any(|source| { + source.origin == SourceOrigin::Application && source.logical_path == package.path + }) { + errors.push(LoadError { + path: package.display.clone(), + message: "Package source collides with another application component".into(), + }); + continue; + } + sources.add_source( + package.path.clone(), + package.path.clone(), + package.text.clone(), + SourceOrigin::Application, + ); + } + finish_load(sources, errors) + } + + pub fn load_project(module_root: &Path, test_root: &Path) -> Result<Self, Vec<LoadError>> { + Self::load_with_packages(module_root, Some(test_root), &[]) + } + + fn load_root( + &mut self, + root: &Path, + display_prefix: Option<&Path>, + origin: SourceOrigin, + errors: &mut Vec<LoadError>, + ) { + let mut paths = Vec::new(); + collect_paths(root, &mut paths, errors); + paths.sort(); + for path in paths { + match read_utf8(&path) { + Ok(text) => { + let logical = path + .strip_prefix(root) + .expect("collected below source root") + .to_path_buf(); + let display = display_prefix + .map(|prefix| prefix.join(&logical)) + .unwrap_or_else(|| logical.clone()); + self.add_source(display, logical, text, origin); + } + Err(error) => errors.push(error), + } + } + } +} + +fn finish_load( + mut sources: SourceMap, + mut errors: Vec<LoadError>, +) -> Result<SourceMap, Vec<LoadError>> { + sources + .files + .sort_by(|left, right| left.path.cmp(&right.path)); + for (index, source) in sources.files.iter_mut().enumerate() { + source.id = SourceId(index); + } + errors.sort_by(|left, right| { + left.path + .cmp(&right.path) + .then(left.message.cmp(&right.message)) + }); + if errors.is_empty() { + Ok(sources) + } else { + Err(errors) + } +} + +// Origins are assigned by the compiler, never inferred from user-controlled paths. +fn standard_sources() -> &'static [SourceFile] { + static SOURCES: OnceLock<Vec<SourceFile>> = OnceLock::new(); + SOURCES.get_or_init(|| { + let mut sources = SourceMap::default(); + for (path, text) in [ + ("text.dever", include_str!("../../../library/text.dever")), + ( + "dever/storage.dever", + include_str!("../../../library/dever/storage.dever"), + ), + ( + "dever/auth.dever", + include_str!("../../../library/dever/auth.dever"), + ), + ( + "dever/site.dever", + include_str!("../../../library/dever/site.dever"), + ), + ("int.dever", include_str!("../../../library/int.dever")), + ("math.dever", include_str!("../../../library/math.dever")), + ( + "decimal.dever", + include_str!("../../../library/decimal.dever"), + ), + ("float.dever", include_str!("../../../library/float.dever")), + ("uuid.dever", include_str!("../../../library/uuid.dever")), + ( + "dever/api.dever", + include_str!("../../../library/dever/api.dever"), + ), + ( + "dever/process.dever", + include_str!("../../../library/dever/process.dever"), + ), + ( + "dever/crypto.dever", + include_str!("../../../library/dever/crypto.dever"), + ), + ( + "dever/time.dever", + include_str!("../../../library/dever/time.dever"), + ), + ( + "dever/task.dever", + include_str!("../../../library/dever/task.dever"), + ), + ( + "dever/json.dever", + include_str!("../../../library/dever/json.dever"), + ), + ( + "dever/json/value.dever", + include_str!("../../../library/dever/json/value.dever"), + ), + ( + "dever/json/number.dever", + include_str!("../../../library/dever/json/number.dever"), + ), + ( + "dever/json/scan.dever", + include_str!("../../../library/dever/json/scan.dever"), + ), + ( + "dever/json/parse.dever", + include_str!("../../../library/dever/json/parse.dever"), + ), + ( + "dever/json/write.dever", + include_str!("../../../library/dever/json/write.dever"), + ), + ( + "dever/http.dever", + include_str!("../../../library/dever/http.dever"), + ), + ( + "dever/sse.dever", + include_str!("../../../library/dever/sse.dever"), + ), + ( + "dever/websocket.dever", + include_str!("../../../library/dever/websocket.dever"), + ), + ( + "dever/job.dever", + include_str!("../../../library/dever/job.dever"), + ), + ( + "dever/tls.dever", + include_str!("../../../library/dever/tls.dever"), + ), + ( + "dever/io.dever", + include_str!("../../../library/dever/io.dever"), + ), + ( + "dever/log.dever", + include_str!("../../../library/dever/log.dever"), + ), + ( + "dever/bytes.dever", + include_str!("../../../library/dever/bytes.dever"), + ), + ( + "dever/net.dever", + include_str!("../../../library/dever/net.dever"), + ), + ( + "dever/id.dever", + include_str!("../../../library/dever/id.dever"), + ), + ( + "dever/database.dever", + include_str!("../../../library/dever/database.dever"), + ), + ] { + sources.add(path, text); + } + for source in &mut sources.files { + source.id.0 |= STANDARD_SOURCE; + } + sources.files + }) +} + +pub(crate) fn parsed_standard( + id: SourceId, +) -> Result<crate::syntax::Package, Vec<crate::diagnostic::Diagnostic>> { + type Parsed = Result<crate::syntax::Package, Vec<crate::diagnostic::Diagnostic>>; + static PACKAGES: OnceLock<Vec<Parsed>> = OnceLock::new(); + PACKAGES.get_or_init(|| { + standard_sources() + .iter() + .map(crate::parse_uncached) + .collect() + })[id.0 & !STANDARD_SOURCE] + .clone() +} + +#[derive(Debug)] +pub struct LoadError { + pub path: PathBuf, + pub message: String, +} + +impl fmt::Display for LoadError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "{}: {}", self.path.display(), self.message) + } +} + +impl std::error::Error for LoadError {} + +fn read_utf8(path: &Path) -> Result<String, LoadError> { + let bytes = fs::read(path).map_err(|error| LoadError { + path: path.into(), + message: error.to_string(), + })?; + String::from_utf8(bytes).map_err(|error| LoadError { + path: path.into(), + message: format!("invalid UTF-8 at byte {}", error.utf8_error().valid_up_to()), + }) +} + +fn collect_paths(root: &Path, paths: &mut Vec<PathBuf>, errors: &mut Vec<LoadError>) { + let mut pending = vec![root.to_path_buf()]; + while let Some(path) = pending.pop() { + let result = (|| -> std::io::Result<()> { + let metadata = fs::symlink_metadata(&path)?; + if metadata.file_type().is_symlink() { + errors.push(LoadError { + path: path.clone(), + message: "source symbolic links are not supported".into(), + }); + } else if metadata.is_dir() { + for entry in fs::read_dir(&path)? { + pending.push(entry?.path()); + } + } else if path == root { + errors.push(LoadError { + path: path.clone(), + message: "expected a source directory".into(), + }); + } else if metadata.is_file() && is_source_path(&path) { + paths.push(path.clone()); + } + Ok(()) + })(); + if let Err(error) = result { + errors.push(LoadError { + path, + message: error.to_string(), + }); + } + } +} + +fn is_source_path(path: &Path) -> bool { + path.extension() + .is_some_and(|extension| extension == "dever") + || is_markdown_path(path) +} + +fn is_markdown_path(path: &Path) -> bool { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.ends_with(".dever.md")) +} diff --git a/crates/dever-core/src/specialize.rs b/crates/dever-core/src/specialize.rs new file mode 100644 index 0000000..8a27de5 --- /dev/null +++ b/crates/dever-core/src/specialize.rs @@ -0,0 +1,376 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use crate::hir::{ + CallArgument, CallTarget, CollectionOp, ExpressionKind, Function, HandlerTarget, SequenceKind, +}; +use crate::source::Span; +use crate::types::Parameter; + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub(crate) struct Specialization { + pub function: usize, + pub handlers: Vec<usize>, +} + +pub(crate) fn reachable( + functions: &[Function], + roots: impl IntoIterator<Item = Specialization>, +) -> Result<BTreeSet<Specialization>, Span> { + let mut state = BTreeMap::<Specialization, u8>::new(); + for root in roots { + if state.contains_key(&root) { + continue; + } + state.insert(root.clone(), 1); + let mut pending = vec![Frame::new(root, functions)]; + while let Some(frame) = pending.last_mut() { + if frame.edge == frame.successors.len() { + state.insert(frame.instance.clone(), 2); + pending.pop(); + continue; + } + let (next, span) = frame.successors[frame.edge].clone(); + frame.edge += 1; + match state.get(&next).copied() { + Some(1) => return Err(span), + Some(2) => {} + None => { + state.insert(next.clone(), 1); + pending.push(Frame::new(next, functions)); + } + _ => unreachable!(), + } + } + } + Ok(state.into_keys().collect()) +} + +/// Job edges create new execution roots and never participate in synchronous cycle detection. +pub(crate) fn execution_reachable( + functions: &[Function], + roots: impl IntoIterator<Item = Specialization>, +) -> Result<BTreeSet<Specialization>, Span> { + let mut instances = reachable(functions, roots)?; + loop { + let mut jobs = BTreeSet::new(); + for instance in &instances { + crate::check::visit(&functions[instance.function], |expression| { + if let crate::hir::ExpressionKind::JobEnqueue { target, .. } = expression.kind { + jobs.insert(Specialization { + function: target, + handlers: Vec::new(), + }); + } + }); + } + jobs.retain(|job| !instances.contains(job)); + if jobs.is_empty() { + return Ok(instances); + } + instances.extend(reachable(functions, jobs)?); + } +} + +pub(crate) fn handler_bindings( + function: &Function, + instance: &Specialization, +) -> BTreeMap<usize, usize> { + function + .parameters + .iter() + .enumerate() + .filter_map(|(index, parameter)| { + matches!(parameter, Parameter::Handler(_)).then_some(index) + }) + .zip(&instance.handlers) + .map(|(parameter, function)| (parameter, *function)) + .collect() +} + +pub(crate) fn suspends(program: &crate::hir::Program, instance: &Specialization) -> bool { + let function = &program.functions[instance.function]; + if function.suspends { + return true; + } + let bindings = handler_bindings(function, instance); + program.suspension_handlers[instance.function] + .iter() + .any(|parameter| { + let target = bindings[parameter]; + program.functions[target].suspends || !program.suspension_handlers[target].is_empty() + }) +} + +pub(crate) fn database_effects( + program: &crate::hir::Program, + instance: &Specialization, +) -> BTreeSet<crate::model::DatabaseOwner> { + let function = &program.functions[instance.function]; + let mut databases = program.database_effects[instance.function].clone(); + let bindings = handler_bindings(function, instance); + for parameter in &program.suspension_handlers[instance.function] { + databases.extend(&program.database_effects[bindings[parameter]]); + } + databases +} + +pub(crate) fn failures( + program: &crate::hir::Program, + instance: &Specialization, +) -> BTreeSet<crate::hir::Failure> { + let mut failures = program.failures[instance.function].clone(); + let bindings = handler_bindings(&program.functions[instance.function], instance); + for parameter in &program.failure_handlers[instance.function] { + failures.extend(&program.failures[bindings[parameter]]); + } + failures +} + +pub(crate) fn has_effect( + program: &crate::hir::Program, + instance: &Specialization, + effect: &str, +) -> bool { + if program.effects[instance.function].contains(effect) { + return true; + } + let function = &program.functions[instance.function]; + let bindings = handler_bindings(function, instance); + program.suspension_handlers[instance.function] + .iter() + .any(|parameter| program.effects[bindings[parameter]].contains(effect)) +} + +pub(crate) fn writes_database(program: &crate::hir::Program, root: &Specialization) -> bool { + let reachable = reachable(&program.functions, [root.clone()]).expect("checked call graph"); + reachable.iter().any(|instance| { + let mut writes = false; + crate::check::visit(&program.functions[instance.function], |expression| { + writes |= match &expression.kind { + crate::hir::ExpressionKind::ModelOperation { operation, .. } => { + operation.writes_database() + } + crate::hir::ExpressionKind::JobEnqueue { .. } => true, + _ => false, + }; + }); + writes + }) +} + +pub(crate) fn uses_intrinsic( + program: &crate::hir::Program, + root: &Specialization, + expected: crate::intrinsic::Intrinsic, +) -> bool { + reachable(&program.functions, [root.clone()]).expect("checked call graph").iter().any(|instance| { + let mut found = false; + crate::check::visit(&program.functions[instance.function], |expression| { + found |= matches!(expression.kind, ExpressionKind::Intrinsic { operation, .. } if operation == expected); + }); + found + }) +} + +/// Finds a database write reachable without first crossing an explicit +/// transaction function. Password crypto APIs use this to keep slow work out +/// of SQLite transactions while preserving a short, explicit write boundary. +pub(crate) fn write_outside_transaction( + program: &crate::hir::Program, + root: &Specialization, +) -> Option<Span> { + let mut pending = vec![( + root.clone(), + program.functions[root.function].kind == crate::syntax::FunctionKind::Transaction, + )]; + let mut visited = BTreeSet::new(); + while let Some((instance, in_transaction)) = pending.pop() { + if !visited.insert((instance.clone(), in_transaction)) { + continue; + } + if !in_transaction { + let mut write = None; + crate::check::visit(&program.functions[instance.function], |expression| { + if write.is_none() + && match &expression.kind { + ExpressionKind::ModelOperation { operation, .. } => { + operation.writes_database() + } + ExpressionKind::JobEnqueue { .. } => true, + _ => false, + } + { + write = Some(expression.span); + } + }); + if write.is_some() { + return write; + } + } + for (successor, _) in Frame::new(instance, &program.functions).successors { + let successor_in_transaction = in_transaction + || program.functions[successor.function].kind + == crate::syntax::FunctionKind::Transaction; + pending.push((successor, successor_in_transaction)); + } + } + None +} + +/// Returns whether a transaction can outlive its sequential call chain. +/// +/// `blocking(call())` runs work on the bounded blocking pool, but it remains +/// structured: the caller waits for completion. Its target is already required +/// to be synchronous and cannot contain database or concurrent work. Everything +/// else classified here can overlap, escape, or stream across the transaction. +pub(crate) fn starts_transaction_unsafe_work( + program: &crate::hir::Program, + root: &Specialization, +) -> bool { + let reachable = reachable(&program.functions, [root.clone()]).expect("checked call graph"); + reachable + .iter() + .any(|instance| function_starts_transaction_unsafe_work(program, instance.function)) +} + +pub(crate) fn function_starts_transaction_unsafe_work( + program: &crate::hir::Program, + function: usize, +) -> bool { + let mut unsafe_work = false; + crate::check::visit(&program.functions[function], |expression| { + unsafe_work |= match &expression.kind { + ExpressionKind::RunCall { .. } + | ExpressionKind::AwaitTask(_) + | ExpressionKind::StopTask(_) + | ExpressionKind::Group(_) + | ExpressionKind::AwaitGroup(_) + | ExpressionKind::StopGroup(_) + | ExpressionKind::ParallelCall { .. } + | ExpressionKind::Channel { .. } + | ExpressionKind::ChannelSend { .. } + | ExpressionKind::ChannelReceive(_) + | ExpressionKind::ChannelClose(_) => true, + ExpressionKind::ModelOperation { operation, .. } => { + matches!( + operation.as_ref(), + crate::hir::ModelOperation::Stream { .. } + ) + } + ExpressionKind::Intrinsic { operation, .. } => { + operation.is_async() + && !matches!(operation, crate::intrinsic::Intrinsic::UploadStore) + } + ExpressionKind::Collection { + operation, + sequence, + .. + } => { + *operation == CollectionOp::ParallelEach + || (matches!( + sequence, + Some(SequenceKind::AsyncStream | SequenceKind::RowStream) + ) && *operation != CollectionOp::Close) + } + // BlockingCall is deliberately excluded: validate_blocking_calls + // enforces a synchronous, non-concurrent target. + _ => false, + }; + }); + unsafe_work +} + +pub(crate) fn concrete(program: &crate::hir::Program) -> Result<BTreeSet<Specialization>, Span> { + let roots = program + .functions + .iter() + .enumerate() + .filter(|(_, function)| { + function + .parameters + .iter() + .all(|parameter| matches!(parameter, Parameter::Value(_))) + }) + .map(|(function, _)| Specialization { + function, + handlers: Vec::new(), + }); + reachable(&program.functions, roots) +} + +pub(crate) fn resolve_handler(target: HandlerTarget, bindings: &BTreeMap<usize, usize>) -> usize { + match target { + HandlerTarget::Function(function) => function, + HandlerTarget::Parameter(parameter) => bindings[¶meter], + } +} + +pub(crate) fn call_specialization( + function: usize, + arguments: &[CallArgument], + bindings: &BTreeMap<usize, usize>, +) -> Specialization { + Specialization { + function, + handlers: arguments + .iter() + .filter_map(|argument| match argument { + CallArgument::Handler(target) => Some(resolve_handler(*target, bindings)), + CallArgument::Value(_) => None, + }) + .collect(), + } +} + +struct Frame { + instance: Specialization, + successors: Vec<(Specialization, Span)>, + edge: usize, +} + +impl Frame { + fn new(instance: Specialization, functions: &[Function]) -> Self { + let bindings = handler_bindings(&functions[instance.function], &instance); + let mut successors = Vec::new(); + if let Some(port) = &functions[instance.function].port { + successors.extend(port.implementations.iter().map(|function| { + ( + Specialization { + function: *function, + handlers: Vec::new(), + }, + functions[instance.function].span, + ) + })); + } + super::check::visit(&functions[instance.function], |expression| { + if let Some((target, arguments)) = expression.kind.static_call() { + let successor = match target { + CallTarget::Function(function) => { + call_specialization(function, arguments, &bindings) + } + CallTarget::Handler(target) => Specialization { + function: resolve_handler(target, &bindings), + handlers: Vec::new(), + }, + }; + successors.push((successor, expression.span)); + return; + } + if let Some(target) = expression.kind.callback() { + successors.push(( + Specialization { + function: resolve_handler(target, &bindings), + handlers: Vec::new(), + }, + expression.span, + )); + } + }); + Self { + instance, + successors, + edge: 0, + } + } +} diff --git a/crates/dever-core/src/syntax.rs b/crates/dever-core/src/syntax.rs new file mode 100644 index 0000000..b60e73c --- /dev/null +++ b/crates/dever-core/src/syntax.rs @@ -0,0 +1,450 @@ +use crate::source::Span; + +pub(crate) const MAX_SYNTAX_DEPTH: usize = 128; + +#[derive(Clone, Debug, PartialEq)] +pub struct Name { + pub text: String, + pub span: Span, +} + +pub type Path = Vec<Name>; + +#[derive(Clone, Debug, PartialEq)] +pub struct Package { + pub name: Path, + pub(crate) layout: crate::source::SourceLayout, + pub declarations: Vec<Declaration>, + pub comments: Vec<Comment>, + pub span: Span, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Comment { + pub text: String, + pub span: Span, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum Declaration { + Api(ApiBinding), + Rest(RestDeclaration), + External(ExternalAdapter), + Type(TypeDeclaration), + Function(FunctionClause), + Database(DatabaseBinding), + Schedule(Schedule), + ModelIndex(ModelIndex), + Relation(Relation), + Seed(Seed), + Migration(Migration), + ModelSql(ModelSql), +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ExternalEcosystem { + Exec, + Command, + Pip, + Npm, + Go, +} + +impl ExternalEcosystem { + pub fn keyword(self) -> &'static str { + match self { + Self::Exec => "exec", + Self::Command => "command", + Self::Pip => "pip", + Self::Npm => "npm", + Self::Go => "go", + } + } +} + +#[derive(Clone, Debug, PartialEq)] +pub struct ExternalAdapter { + pub ecosystem: ExternalEcosystem, + pub entry: String, + pub entry_span: Span, + pub libs: Vec<Name>, + pub capabilities: Vec<Name>, + pub span: Span, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum ApiKind { + Get, + Post, + Put, + Delete, + Command, +} + +impl ApiKind { + pub fn keyword(self) -> &'static str { + match self { + Self::Get => "get", + Self::Post => "post", + Self::Put => "put", + Self::Delete => "delete", + Self::Command => "cmd", + } + } + + pub(crate) fn method(self) -> Option<&'static str> { + match self { + Self::Get => Some("GET"), + Self::Post => Some("POST"), + Self::Put => Some("PUT"), + Self::Delete => Some("DELETE"), + Self::Command => None, + } + } +} + +#[derive(Clone, Debug, PartialEq)] +pub struct ApiBinding { + pub anonymous: bool, + pub kind: ApiKind, + pub action: Name, + pub target: Path, + pub span: Span, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct RestDeclaration { + pub model: Option<Path>, + pub span: Span, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct DatabaseBinding { + pub name: Name, + pub span: Span, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Schedule { + pub target: Name, + pub cron: String, + pub span: Span, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct ModelIndex { + pub unique: bool, + pub fields: Vec<Name>, + pub span: Span, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Relation { + pub name: Name, + pub field: Path, + pub span: Span, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Seed { + pub rows: Vec<Vec<RecordField>>, + pub span: Span, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Migration { + pub name: Name, + pub operations: Vec<MigrationOperation>, + pub span: Span, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum MigrationOperation { + Drop(Name), + Sql(MigrationSql), +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum MigrationPhase { + Before, + After, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct MigrationSql { + pub phase: MigrationPhase, + pub sqlite: SqlText, + pub postgres: SqlText, + pub parameters: Vec<Expression>, + pub span: Span, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct ModelSql { + pub name: Name, + pub inputs: Vec<Field>, + pub outputs: Vec<Field>, + pub sqlite: SqlText, + pub postgres: SqlText, + pub span: Span, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct SqlText { + pub value: String, + pub span: Span, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct TypeDeclaration { + pub public: bool, + pub setting: bool, + pub global: bool, + pub name: Name, + pub shape: TypeShape, + pub span: Span, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum TypeShape { + Record(Vec<Field>), + Choice(Vec<Variant>), +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Field { + pub name: Name, + pub ty: TypeRef, + pub private: bool, + pub bounds: Vec<Bound>, + pub storage: FieldStorage, + pub span: Span, +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct FieldStorage { + pub generated: bool, + pub default: Option<Expression>, + pub index: bool, + pub unique: bool, + pub from: Option<Name>, + pub owner: Option<Expression>, + pub create: Option<Expression>, + pub replace: Option<Expression>, + pub search: Option<Expression>, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Variant { + pub name: Name, + pub error: bool, + pub payload: Vec<Field>, + pub label: Option<String>, + pub span: Span, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct TypeRef { + pub name: Path, + pub arguments: Vec<TypeRef>, + pub parameters: Vec<NumberParameter>, + pub nullable: bool, + pub span: Span, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct NumberParameter { + pub value: String, + pub span: Span, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct FunctionClause { + pub public: bool, + pub name: Name, + pub kind: FunctionKind, + pub inputs: Vec<Input>, + pub outputs: Vec<Field>, + pub pure: bool, + pub recovery: Option<String>, + pub fails: Option<TypeRef>, + pub bodyless: bool, + pub body: Vec<Statement>, + pub span: Span, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum FunctionKind { + Ordinary, + Transaction, + Job { attempts: u32, timeout_ms: u32 }, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Input { + pub name: Name, + pub kind: InputKind, + pub span: Span, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum InputKind { + Value(Pattern), + Handler(HandlerSignature), +} + +#[derive(Clone, Debug, PartialEq)] +pub struct HandlerSignature { + pub inputs: Vec<Field>, + pub outputs: Vec<Field>, + pub span: Span, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum Pattern { + Typed { + ty: TypeRef, + bounds: Vec<Bound>, + }, + Variant { + name: Path, + arguments: Vec<TypeRef>, + bindings: Vec<Name>, + span: Span, + }, + Literal { + value: Literal, + span: Span, + }, + Other(Span), +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Bound { + pub comparison: BinaryOperator, + pub number: String, + pub span: Span, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Statement { + pub kind: StatementKind, + pub span: Span, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum StatementKind { + Assign { target: Path, value: Expression }, + Call(Expression), +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Expression { + pub kind: ExpressionKind, + pub span: Span, + pub(crate) depth: usize, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum ExpressionKind { + Literal(Literal), + Name(Path), + Group(Box<Expression>), + Field { + value: Box<Expression>, + name: Name, + }, + Call { + function: Box<Expression>, + arguments: Vec<Expression>, + }, + Fail(Box<Expression>), + CaptureResult(Box<Expression>), + Contextual(ContextualExpression), + Record { + name: Path, + fields: Vec<RecordField>, + }, + List(Vec<Expression>), + Map(Vec<MapEntry>), + Unary { + operator: UnaryOperator, + value: Box<Expression>, + }, + Binary { + left: Box<Expression>, + operator: BinaryOperator, + right: Box<Expression>, + }, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum ContextualExpression { + Channel { + element: TypeRef, + capacity: Box<Expression>, + }, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct RecordField { + pub name: Name, + pub value: Expression, + pub span: Span, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct MapEntry { + pub key: Expression, + pub value: Expression, + pub span: Span, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum Literal { + Bool(bool), + Null, + Number(String), + Text(String), +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum UnaryOperator { + Negate, + Not, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum BinaryOperator { + Add, + Subtract, + Multiply, + Divide, + IntegerDivide, + Remainder, + Equal, + NotEqual, + Less, + LessEqual, + Greater, + GreaterEqual, + And, + Or, +} + +impl BinaryOperator { + pub fn precedence(self) -> u8 { + match self { + Self::Or => 1, + Self::And => 2, + Self::Equal + | Self::NotEqual + | Self::Less + | Self::LessEqual + | Self::Greater + | Self::GreaterEqual => 3, + Self::Add | Self::Subtract => 4, + Self::Multiply | Self::Divide | Self::IntegerDivide | Self::Remainder => 5, + } + } +} diff --git a/crates/dever-core/src/token.rs b/crates/dever-core/src/token.rs new file mode 100644 index 0000000..2d1465c --- /dev/null +++ b/crates/dever-core/src/token.rs @@ -0,0 +1,51 @@ +use crate::source::Span; + +#[derive(Clone, Debug, PartialEq)] +pub(crate) enum Kind { + Name(String), + Number(String), + Text(String), + Comment(String), + Package, + Exposes, + Public, + Type, + True, + False, + Null, + Other, + And, + Or, + Not, + LeftParen, + RightParen, + LeftBrace, + RightBrace, + LeftBracket, + RightBracket, + Colon, + Comma, + Dot, + Question, + Assign, + Plus, + Minus, + Star, + Slash, + IntegerDivide, + Percent, + Equal, + NotEqual, + Less, + LessEqual, + Greater, + GreaterEqual, + Newline, + End, +} + +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct Token { + pub kind: Kind, + pub span: Span, +} diff --git a/crates/dever-core/src/types.rs b/crates/dever-core/src/types.rs new file mode 100644 index 0000000..3859f2a --- /dev/null +++ b/crates/dever-core/src/types.rs @@ -0,0 +1,417 @@ +use std::collections::VecDeque; +use std::sync::OnceLock; + +use crate::source::Span; + +#[derive(Clone, Debug, PartialEq)] +pub(crate) enum Type { + Unit, + Bool, + Int, + Decimal, + Float, + Text, + Id, + Bytes, + Secret, + Upload, + Uuid, + DateTime, + Date, + Time, + Duration, + Json, + File, + Socket, + Listener, + HttpReply, + WebSocket, + ClientTls, + ServerTls, + HttpClient, + Stream(Box<Type>), + AsyncStream(Box<Type>), + /// Database-backed stream. It is rendered as AsyncStream in the source contract. + RowStream(Box<Type>), + Related(Box<Type>), + Channel(Box<Type>), + Task(Vec<Field>), + Group, + Named(usize), + Nullable(Box<Type>), + List(Box<Type>), + Map(Box<Type>, Box<Type>), + MapEntry(Box<Type>, Box<Type>), + Outputs(Vec<Field>), +} + +impl Type { + pub fn nullable(self) -> Self { + if matches!(self, Self::Nullable(_)) { + self + } else { + Self::Nullable(Box::new(self)) + } + } + + pub fn base(&self) -> &Self { + match self { + Self::Nullable(base) => base, + _ => self, + } + } + + pub fn numeric(&self) -> bool { + matches!(self, Self::Int | Self::Decimal | Self::Float) + } + + pub fn value(&self) -> bool { + !matches!(self, Self::Unit | Self::Outputs(_)) + } + + pub fn comparable(&self, definitions: &[Definition]) -> bool { + self.properties(definitions).comparable + } + + pub fn transferable(&self, definitions: &[Definition]) -> bool { + self.properties(definitions).transferable + } + + pub fn movable(&self, definitions: &[Definition]) -> bool { + // 资源的最后别名必须保留到源码作用域结束，避免提前发出可观察的 EOF。 + self.properties(definitions).movable + } + + pub fn observable(&self, definitions: &[Definition]) -> bool { + let (secret, private_model) = self.sensitivity(definitions); + !secret && !private_model + } + + pub fn contains_private_model(&self, definitions: &[Definition]) -> bool { + self.sensitivity(definitions).1 + } + + fn sensitivity(&self, definitions: &[Definition]) -> (bool, bool) { + let mut sensitive = (false, false); + visit_type(self, &mut |ty| match ty { + Self::Secret | Self::Upload => sensitive.0 = true, + Self::Named(id) => { + let properties = definitions[*id] + .properties + .get() + .expect("cached type properties"); + sensitive.0 |= properties.secret; + sensitive.1 |= properties.private_model; + } + _ => {} + }); + sensitive + } + + fn properties(&self, definitions: &[Definition]) -> Properties { + match self { + Self::Named(id) => *definitions[*id] + .properties + .get() + .expect("validated nominal type properties"), + Self::Nullable(base) | Self::List(base) => base.properties(definitions), + // Related is an ORM-owned indirection. Its payload is always a Model, + // nullable Model, or bounded List<Model>, so recursive Model graphs do + // not participate in nominal layout/property cycles. + Self::Related(_) => Properties::ALL, + Self::Map(key, value) | Self::MapEntry(key, value) => key + .properties(definitions) + .intersection(value.properties(definitions)), + Self::Channel(element) | Self::AsyncStream(element) => Properties { + comparable: false, + transferable: element.transferable(definitions), + movable: false, + ..Properties::ALL + }, + Self::RowStream(element) => Properties { + comparable: false, + transferable: element.transferable(definitions), + movable: false, + ..Properties::ALL + }, + Self::Task(_) | Self::Group | Self::Upload => Properties { + comparable: false, + transferable: false, + movable: true, + ..Properties::ALL + }, + Self::File + | Self::Socket + | Self::Listener + | Self::HttpReply + | Self::WebSocket + | Self::ClientTls + | Self::ServerTls + | Self::HttpClient => Properties { + comparable: false, + transferable: true, + movable: false, + ..Properties::ALL + }, + Self::Stream(_) => Properties { + comparable: false, + transferable: false, + movable: false, + ..Properties::ALL + }, + Self::Unit | Self::Outputs(_) => Properties { + comparable: false, + transferable: false, + movable: match self { + Self::Outputs(fields) => { + fields.iter().all(|field| field.ty.movable(definitions)) + } + _ => true, + }, + ..Properties::ALL + }, + Self::Secret => Properties { + comparable: false, + secret: true, + ..Properties::ALL + }, + _ => Properties::ALL, + } + } + + pub fn label(&self, types: &[Definition]) -> String { + match self { + Self::Named(id) => types[*id].name.clone(), + Self::Nullable(base) => format!("{}?", base.label(types)), + Self::List(element) => format!("List<{}>", element.label(types)), + Self::Map(key, value) => format!("Map<{}, {}>", key.label(types), value.label(types)), + Self::MapEntry(key, value) => { + format!("MapEntry<{}, {}>", key.label(types), value.label(types)) + } + Self::Stream(element) => format!("Stream<{}>", element.label(types)), + Self::AsyncStream(element) => format!("AsyncStream<{}>", element.label(types)), + Self::RowStream(element) => format!("AsyncStream<{}>", element.label(types)), + Self::Related(element) => format!("Related<{}>", element.label(types)), + Self::Channel(element) => format!("Channel<{}>", element.label(types)), + Self::Task(fields) => format!( + "Task<({})>", + fields + .iter() + .map(|field| format!("{}: {}", field.name, field.ty.label(types))) + .collect::<Vec<_>>() + .join(", ") + ), + Self::Group => "Group".into(), + Self::Outputs(_) => "named outputs".into(), + _ => format!("{self:?}"), + } + } +} + +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct HandlerSignature { + pub parameters: Vec<Type>, + pub bounds: Vec<crate::hir::Domain>, + pub outputs: Vec<Field>, +} + +#[derive(Clone, Debug, PartialEq)] +pub(crate) enum Parameter { + Value(Type), + Handler(HandlerSignature), +} + +impl Parameter { + pub fn value_type(&self) -> Option<&Type> { + match self { + Self::Value(ty) => Some(ty), + Self::Handler(_) => None, + } + } +} + +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct Field { + pub name: String, + pub ty: Type, + pub private: bool, + pub bounds: crate::hir::Domain, +} + +#[derive(Clone, Debug)] +pub(crate) struct Variant { + pub name: String, + pub fields: Vec<Field>, + pub error: bool, + pub label: Option<String>, +} + +#[derive(Clone, Debug)] +pub(crate) enum Shape { + Record(Vec<Field>), + Choice(Vec<Variant>), +} + +#[derive(Clone, Debug)] +pub(crate) struct Definition { + pub name: String, + pub owner: usize, + pub public: bool, + pub shape: Shape, + pub span: Span, + pub properties: OnceLock<Properties>, + pub kind: DefinitionKind, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum DefinitionKind { + Regular, + Model, + ModelId, + ModelChoice, + ModelPage, + ModelCursor, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct Properties { + comparable: bool, + transferable: bool, + movable: bool, + secret: bool, + private_model: bool, +} + +impl Properties { + const ALL: Self = Self { + comparable: true, + transferable: true, + movable: true, + secret: false, + private_model: false, + }; + + fn intersection(self, other: Self) -> Self { + Self { + comparable: self.comparable && other.comparable, + transferable: self.transferable && other.transferable, + movable: self.movable && other.movable, + secret: self.secret || other.secret, + private_model: self.private_model || other.private_model, + } + } +} + +pub(crate) fn cache_properties(definitions: &[Definition], dependencies: &[Vec<(usize, Span)>]) { + let sensitivity = nominal_sensitivity(definitions); + // 类型环已经被拒绝。按依赖顺序计算一次，避免展开共享 DAG 或递归宿主调用栈。 + let mut callers = vec![Vec::new(); definitions.len()]; + let mut pending: Vec<_> = dependencies.iter().map(Vec::len).collect(); + for (caller, references) in dependencies.iter().enumerate() { + for (callee, _) in references { + callers[*callee].push(caller); + } + } + let mut ready: VecDeque<_> = pending + .iter() + .enumerate() + .filter_map(|(id, count)| (*count == 0).then_some(id)) + .collect(); + while let Some(id) = ready.pop_front() { + let mut properties = Properties::ALL; + let mut include = |field: &Field| { + properties = properties.intersection(field.ty.properties(definitions)); + }; + match &definitions[id].shape { + Shape::Record(fields) => fields.iter().for_each(&mut include), + Shape::Choice(variants) => variants + .iter() + .flat_map(|variant| &variant.fields) + .for_each(&mut include), + } + (properties.secret, properties.private_model) = sensitivity[id]; + definitions[id] + .properties + .set(properties) + .expect("type properties initialized once after validation"); + for caller in &callers[id] { + pending[*caller] -= 1; + if pending[*caller] == 0 { + ready.push_back(*caller); + } + } + } +} + +/// Walk container structure, stopping at nominal identities. Related edges participate +/// in sensitivity propagation without expanding cyclic ORM association graphs. +pub(crate) fn visit_type(ty: &Type, visit: &mut impl FnMut(&Type)) { + visit(ty); + match ty { + Type::Nullable(inner) + | Type::List(inner) + | Type::Related(inner) + | Type::Stream(inner) + | Type::AsyncStream(inner) + | Type::RowStream(inner) + | Type::Channel(inner) => visit_type(inner, visit), + Type::Map(key, value) | Type::MapEntry(key, value) => { + visit_type(key, visit); + visit_type(value, visit); + } + Type::Task(fields) | Type::Outputs(fields) => { + for field in fields { + visit_type(&field.ty, visit); + } + } + _ => {} + } +} + +fn nominal_sensitivity(definitions: &[Definition]) -> Vec<(bool, bool)> { + let mut sensitivity = vec![(false, false); definitions.len()]; + let mut callers = vec![Vec::new(); definitions.len()]; + for (id, definition) in definitions.iter().enumerate() { + let mut include = |field: &Field| { + sensitivity[id].1 |= definition.kind == DefinitionKind::Model && field.private; + visit_type(&field.ty, &mut |ty| match ty { + Type::Secret => sensitivity[id].0 = true, + Type::Named(target) => callers[*target].push(id), + _ => {} + }); + }; + match &definition.shape { + Shape::Record(fields) => fields.iter().for_each(&mut include), + Shape::Choice(variants) => variants + .iter() + .flat_map(|v| &v.fields) + .for_each(&mut include), + } + } + let mut ready: VecDeque<_> = sensitivity + .iter() + .enumerate() + .filter_map(|(id, (secret, private))| (*secret || *private).then_some(id)) + .collect(); + while let Some(id) = ready.pop_front() { + for caller in &callers[id] { + let combined = ( + sensitivity[*caller].0 || sensitivity[id].0, + sensitivity[*caller].1 || sensitivity[id].1, + ); + if combined != sensitivity[*caller] { + sensitivity[*caller] = combined; + ready.push_back(*caller); + } + } + } + sensitivity +} + +pub(crate) fn output_type(fields: &[Field]) -> Type { + match fields { + [] => Type::Unit, + [field] => field.ty.clone(), + _ => Type::Outputs(fields.to_vec()), + } +} diff --git a/crates/dever-core/src/wire.rs b/crates/dever-core/src/wire.rs new file mode 100644 index 0000000..fcd3ffe --- /dev/null +++ b/crates/dever-core/src/wire.rs @@ -0,0 +1,262 @@ +//! One checked schema owns eligibility and concrete codec generation for all wire boundaries. +use std::collections::{BTreeMap, BTreeSet}; + +use crate::hir::Program; +use crate::types::{Definition, DefinitionKind, Field, Shape, Type}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Policy { + Output, + ApiInput, + SettingInput, + Job, + Command, +} + +impl Policy { + pub(crate) fn external(ecosystem: crate::syntax::ExternalEcosystem) -> Self { + match ecosystem { + crate::syntax::ExternalEcosystem::Command => Self::Command, + _ => Self::Output, + } + } + + pub(crate) fn permits_secret(self) -> bool { + matches!(self, Self::ApiInput | Self::SettingInput) + } +} + +#[derive(Clone, Debug)] +pub struct Schema { + pub(crate) policy: Policy, + pub(crate) nodes: Vec<Node>, + pub(crate) root: usize, + identity: String, +} + +#[derive(Clone, Debug)] +pub(crate) enum Node { + Bytes, + Bool, + Int, + Float, + Decimal, + Text, + Id, + ModelId { + id: usize, + name: String, + }, + Uuid, + DateTime, + Date, + Time, + Duration, + Json, + Secret, + Nullable(usize), + List(usize), + Record { + id: usize, + name: String, + fields: Vec<(String, usize)>, + }, +} + +impl Schema { + /// HTTP query conversion precedes decoding the concrete wire value. + pub(crate) fn query_kind(&self) -> &'static str { + let node = match &self.nodes[self.root] { + Node::Nullable(inner) => &self.nodes[*inner], + node => node, + }; + match node { + Node::Int | Node::ModelId { .. } | Node::Duration => "Int", + Node::Bool => "Bool", + Node::Float => "Float", + _ => "Text", + } + } + + /// Deterministic protocol identity; separate from the persistent Model fingerprint. + pub fn fingerprint(&self) -> &str { + &self.identity + } + + pub(crate) fn worker_type(&self) -> serde_json::Value { + self.worker_node(self.root) + } + + fn worker_node(&self, id: usize) -> serde_json::Value { + use serde_json::json; + match &self.nodes[id] { + Node::Bytes => json!({"type": "bytes", "encoding": "base64"}), + Node::Bool => json!({"type": "bool"}), + Node::Int => json!({"type": "int64"}), + Node::Float => json!({"type": "float64"}), + Node::Decimal => json!({"type": "decimal"}), + Node::Text => json!({"type": "text"}), + Node::Id => json!({"type": "id"}), + Node::ModelId { name, .. } => json!({"type": "model_id", "name": name}), + Node::Uuid => json!({"type": "uuid"}), + Node::DateTime => json!({"type": "datetime"}), + Node::Date => json!({"type": "date"}), + Node::Time => json!({"type": "time"}), + Node::Duration => json!({"type": "duration"}), + Node::Json => json!({"type": "json"}), + Node::Secret => json!({"type": "secret"}), + Node::Nullable(child) => json!({"type": "nullable", "value": self.worker_node(*child)}), + Node::List(child) => json!({"type": "list", "value": self.worker_node(*child)}), + Node::Record { name, fields, .. } => json!({ + "type": "record", "name": name, + "fields": fields.iter().map(|(name, child)| (name.clone(), self.worker_node(*child))) + .collect::<BTreeMap<_, _>>() + }), + } + } + + pub(crate) fn from_field( + field: &Field, + types: &[Definition], + policy: Policy, + ) -> Result<Self, String> { + validate_field(field)?; + Self::build(&field.ty, types, policy) + } + + pub(crate) fn build(ty: &Type, types: &[Definition], policy: Policy) -> Result<Self, String> { + let mut builder = Builder { + types, + policy, + nodes: Vec::new(), + cached: BTreeMap::new(), + visiting: BTreeSet::new(), + }; + let root = builder.node(ty, 0)?; + // A memoized child can occur at a deeper path than its first occurrence. + // Compute the longest DAG path after interning, rather than trusting visit depth. + let mut heights = Vec::with_capacity(builder.nodes.len()); + for node in &builder.nodes { + let height = match node { + Node::Nullable(child) | Node::List(child) => heights[*child] + 1, + Node::Record { fields, .. } => fields + .iter() + .map(|(_, child)| heights[*child] + 1) + .max() + .unwrap_or(1), + _ => 0, + }; + heights.push(height); + } + if heights[root] > 64 { + return Err("wire schema exceeds depth limit".into()); + } + let mut identity = format!("wire-v1:{policy:?}:"); + for node in &builder.nodes { + match node { + Node::Record { name, fields, .. } => { + identity.push_str(&format!("Record({name:?},{fields:?});")) + } + Node::ModelId { name, .. } => identity.push_str(&format!("ModelId({name:?});")), + other => identity.push_str(&format!("{other:?};")), + } + } + Ok(Self { + policy, + nodes: builder.nodes, + root, + identity, + }) + } +} + +impl Program { + /// Inspect a checked nominal type without reinterpreting source declarations. + pub fn wire_schema(&self, name: &str, policy: Policy) -> Result<Schema, String> { + let id = self + .types + .iter() + .position(|ty| ty.name == name) + .ok_or_else(|| format!("unknown wire type {name}"))?; + Schema::build(&Type::Named(id), &self.types, policy) + } +} + +struct Builder<'a> { + types: &'a [Definition], + policy: Policy, + nodes: Vec<Node>, + cached: BTreeMap<String, usize>, + visiting: BTreeSet<usize>, +} + +impl Builder<'_> { + fn node(&mut self, ty: &Type, depth: usize) -> Result<usize, String> { + if depth > 64 { + return Err("wire schema exceeds depth limit".into()); + } + let key = ty.label(self.types); + if let Some(id) = self.cached.get(&key) { + return Ok(*id); + } + if !self.policy.permits_secret() && !ty.observable(self.types) { + return Err( + "wire output and Job payload cannot contain Secret or private Model data".into(), + ); + } + let node = match ty { + Type::Bytes if self.policy == Policy::Command => Node::Bytes, + Type::Bool => Node::Bool, + Type::Int => Node::Int, + Type::Float => Node::Float, + Type::Decimal => Node::Decimal, + Type::Text => Node::Text, + Type::Id => Node::Id, + Type::Uuid => Node::Uuid, + Type::DateTime => Node::DateTime, + Type::Date => Node::Date, + Type::Time => Node::Time, + Type::Duration => Node::Duration, + Type::Json => Node::Json, + Type::Secret if self.policy.permits_secret() => Node::Secret, + Type::Nullable(inner) => Node::Nullable(self.node(inner, depth + 1)?), + Type::List(inner) => Node::List(self.node(inner, depth + 1)?), + Type::Named(id) if self.types[*id].kind == DefinitionKind::ModelId => Node::ModelId { + id: *id, + name: self.types[*id].name.clone(), + }, + Type::Named(id) => { + let definition = &self.types[*id]; + if !self.visiting.insert(*id) { + return Err(format!("unsupported wire type {key}")); + } + let Shape::Record(fields) = &definition.shape else { + return Err("wire Choice is not supported".into()); + }; + let mut wire_fields = Vec::new(); + for field in fields { + validate_field(field)?; + wire_fields.push((field.name.clone(), self.node(&field.ty, depth + 1)?)); + } + self.visiting.remove(id); + Node::Record { + id: *id, + name: definition.name.clone(), + fields: wire_fields, + } + } + _ => return Err(format!("unsupported wire type {key}")), + }; + let id = self.nodes.len(); + self.nodes.push(node); + self.cached.insert(key, id); + Ok(id) + } +} + +fn validate_field(field: &Field) -> Result<(), String> { + if field.private || !field.bounds.is_empty() { + return Err("wire fields must be public and unconstrained".into()); + } + Ok(()) +} diff --git a/crates/dever-runtime/Cargo.toml b/crates/dever-runtime/Cargo.toml new file mode 100644 index 0000000..3d74b64 --- /dev/null +++ b/crates/dever-runtime/Cargo.toml @@ -0,0 +1,49 @@ +[package] +name = "dever-runtime" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +publish.workspace = true + +[lints] +workspace = true + +[features] +default = [] +auth-crypto = ["dep:sha2", "dep:hmac", "dep:subtle"] +crypto = ["auth-crypto", "dep:argon2", "dep:getrandom"] +wire = ["dep:serde", "dep:serde_json", "serde_json/raw_value", "dep:base64", "dep:getrandom"] +external = ["wire", "dep:sha2", "dep:dever-sandbox"] +api = ["wire", "auth-crypto"] +database = ["wire", "dep:getrandom"] +sqlite = ["database", "dep:deadpool-sqlite"] +postgres = ["database", "dep:deadpool-postgres", "dep:tokio-postgres", "dep:tokio-postgres-rustls"] + +[dependencies] +dever-sandbox = { path = "../dever-sandbox", optional = true } +dec.workspace = true +indexmap.workspace = true +tokio.workspace = true +futures-util.workspace = true +bytes.workspace = true +base64 = { workspace = true, optional = true } +hyper.workspace = true +hyper-util.workspace = true +http-body-util.workspace = true +tokio-tungstenite.workspace = true +tokio-rustls.workspace = true +rustls.workspace = true +webpki-roots.workspace = true +serde = { workspace = true, optional = true } +serde_json = { workspace = true, optional = true } +deadpool-sqlite = { workspace = true, optional = true } +deadpool-postgres = { workspace = true, optional = true } +tokio-postgres = { workspace = true, optional = true } +tokio-postgres-rustls = { workspace = true, optional = true } +getrandom = { workspace = true, optional = true } +argon2 = { workspace = true, optional = true } +sha2 = { workspace = true, optional = true } +hmac = { workspace = true, optional = true } +subtle = { workspace = true, optional = true } +zeroize.workspace = true +chrono.workspace = true diff --git a/crates/dever-runtime/src/abi.rs b/crates/dever-runtime/src/abi.rs new file mode 100644 index 0000000..2701df9 --- /dev/null +++ b/crates/dever-runtime/src/abi.rs @@ -0,0 +1,49 @@ +//! Safe value operations shared by the Rust backend and the native runtime ABI. + +use crate::bytes::Bytes; +use crate::number::DecimalValue; +use crate::render::Render; + +/// The finite Decimal128 interchange representation is little-endian bits, +/// independent of the private Rust DecimalValue layout. +#[repr(C)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct DecimalBits { + pub low: u64, + pub high: u64, +} + +impl DecimalBits { + pub fn decode(self) -> Result<DecimalValue, &'static str> { + let mut bytes = [0; 16]; + bytes[..8].copy_from_slice(&self.low.to_le_bytes()); + bytes[8..].copy_from_slice(&self.high.to_le_bytes()); + DecimalValue::from_bytes(bytes) + } + + pub fn encode(value: DecimalValue) -> Self { + let bytes = value.to_bytes(); + Self { + low: u64::from_le_bytes(bytes[..8].try_into().expect("Decimal low word")), + high: u64::from_le_bytes(bytes[8..].try_into().expect("Decimal high word")), + } + } +} + +pub fn text_from_utf8(bytes: &[u8]) -> Result<Vec<u8>, String> { + Bytes::new(bytes.to_vec()) + .into_text() + .map(String::into_bytes) +} + +pub fn int_to_text(value: i64) -> String { + value.render() +} + +pub fn float_to_text(value: f64) -> String { + value.render() +} + +pub fn decimal_to_text(value: DecimalBits) -> Result<String, &'static str> { + value.decode().map(|value| value.render()) +} diff --git a/crates/dever-runtime/src/api.rs b/crates/dever-runtime/src/api.rs new file mode 100644 index 0000000..212ff31 --- /dev/null +++ b/crates/dever-runtime/src/api.rs @@ -0,0 +1,581 @@ +use base64::Engine as _; +use std::cell::RefCell; +use std::future::Future; + +use std::collections::BTreeMap; + +use crate::bytes::Bytes; +use crate::http::{self, Header, Request, Response}; + +pub use serde_json::{Map as Object, Value}; +mod input; +pub mod upload; +pub use input::{InputError, Inputs, QueryValue, text_bounds}; + +tokio::task_local! { static REQUEST: RefCell<RequestState>; } + +struct RequestState { + id: u64, + method: String, + path: String, + client_address: Option<String>, + headers: Vec<Header>, + cookies: BTreeMap<String, crate::secret::Secret>, + pending_response_headers: Vec<Header>, + committed_response_headers: Vec<Header>, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum SameSite { + Strict, + Lax, + None, +} + +pub struct CookieOptions { + pub path: String, + pub domain: Option<String>, + pub same_site: SameSite, + pub secure: bool, + pub http_only: bool, + pub max_age: Option<i64>, +} + +/// 路由由编译器生成；运行时只负责既有 HTTP 引擎的配置与启动。 +pub async fn serve<F, Fut>(route: F) -> Result<(), String> +where + F: Fn(Request) -> Fut + Clone + Send + Sync + 'static, + Fut: Future<Output = Result<Response, String>> + Send + 'static, +{ + if !crate::lifecycle::api_enabled() { + return Ok(()); + } + let settings = crate::config::http_settings()?; + let listener = crate::net::listen(&settings.host, settings.port).await?; + let limits = settings.limits; + let components = crate::component::current(); + #[cfg(feature = "api")] + let session = crate::application::current(); + let server = http::serve_incoming( + move |incoming| { + let route = route.clone(); + let settings = settings.clone(); + let request = async move { + upload::scope( + incoming.body, + settings, + scoped_request(incoming.head, route), + ) + .await + }; + #[cfg(feature = "api")] + let request = crate::application::scope_resources(session.clone(), request); + crate::component::scope_resources(components.clone(), request) + }, + listener.clone(), + limits, + ); + tokio::pin!(server); + tokio::select! { + result = &mut server => result, + _ = crate::lifecycle::shutdown_requested() => { + listener.close()?; + server.await + } + } +} + +/// The scope owns response metadata; no source-level Context value exists. +pub async fn scoped_request<F, Fut>(request: Request, route: F) -> Result<Response, String> +where + F: FnOnce(Request) -> Fut, + Fut: Future<Output = Result<Response, String>>, +{ + let cookies = match read_cookies(&request.headers) { + Ok(cookies) => cookies, + Err(error) => return Ok(invalid_input(error)), + }; + let state = RequestState { + id: http::request_id(), + method: request.method.clone(), + path: request.target.split('?').next().unwrap_or("").to_owned(), + client_address: http::client_address(), + headers: request.headers.clone(), + cookies, + pending_response_headers: Vec::new(), + committed_response_headers: Vec::new(), + }; + REQUEST + .scope(RefCell::new(state), async move { + let mut response = route(request).await?; + REQUEST.with(|state| { + response + .headers + .append(&mut state.borrow_mut().committed_response_headers) + }); + Ok(response) + }) + .await +} + +/// Publish staged metadata only after the generated handler has serialized its +/// response and committed its database transaction. +pub fn commit_response_metadata() -> Result<(), String> { + with_request_mut(|request| { + request + .committed_response_headers + .append(&mut request.pending_response_headers); + Ok(()) + }) +} + +fn with_request<T>(read: impl FnOnce(&RequestState) -> Result<T, String>) -> Result<T, String> { + REQUEST + .try_with(|state| read(&state.borrow())) + .map_err(|_| "API request capability is available only within an HTTP request".to_owned())? +} + +fn with_request_mut<T>( + write: impl FnOnce(&mut RequestState) -> Result<T, String>, +) -> Result<T, String> { + REQUEST + .try_with(|state| write(&mut state.borrow_mut())) + .map_err(|_| "API request capability is available only within an HTTP request".to_owned())? +} + +pub fn request_id() -> Result<String, String> { + with_request(|request| Ok(request.id.to_string())) +} +pub fn method() -> Result<String, String> { + with_request(|request| Ok(request.method.clone())) +} +pub fn path() -> Result<String, String> { + with_request(|request| Ok(request.path.clone())) +} +pub fn client_address() -> Result<Option<String>, String> { + with_request(|request| Ok(request.client_address.clone())) +} + +pub fn header(name: &str) -> Result<Option<String>, String> { + validate_header_name(name)?; + if [ + "cookie", + "authorization", + "proxy-authorization", + "set-cookie", + "forwarded", + "x-forwarded-for", + "x-forwarded-host", + "x-forwarded-proto", + "x-real-ip", + ] + .iter() + .any(|blocked| name.eq_ignore_ascii_case(blocked)) + { + return Err("sensitive request headers require a typed Secret reader".into()); + } + with_request(|request| { + let mut values = request + .headers + .iter() + .filter(|header| header.name.eq_ignore_ascii_case(name)); + let Some(value) = values.next() else { + return Ok(None); + }; + if values.next().is_some() { + return Err("duplicate request header".into()); + } + let value = std::str::from_utf8(value.value.values()) + .map_err(|_| "request header is not UTF-8".to_owned())?; + Ok(Some(value.to_owned())) + }) +} + +pub fn cookie(name: &str) -> Result<Option<String>, String> { + validate_cookie_name(name)?; + with_request(|request| { + let Some(raw) = request.cookies.get(name) else { + return Ok(None); + }; + if raw.0.starts_with(b"v1.") { + return Err("Secret Cookie values require the typed Secret reader".into()); + } + let value = + std::str::from_utf8(&raw.0).map_err(|_| "Cookie value is not UTF-8".to_owned())?; + Ok(Some(value.to_owned())) + }) +} + +pub fn secret_cookie(name: &str) -> Result<Option<crate::secret::Secret>, String> { + validate_cookie_name(name)?; + with_request(|request| { + let Some(raw) = request.cookies.get(name) else { + return Ok(None); + }; + let Some(encoded) = raw.0.strip_prefix(b"v1.") else { + return Ok(None); + }; + Ok(base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(encoded) + .ok() + .map(crate::secret::Secret::from_input)) + }) +} + +fn read_cookies(headers: &[Header]) -> Result<BTreeMap<String, crate::secret::Secret>, InputError> { + let mut cookies = BTreeMap::new(); + for header in headers + .iter() + .filter(|header| header.name.eq_ignore_ascii_case("cookie")) + { + let raw = header.value.values(); + if !raw.is_ascii() { + return Err(InputError("Cookie header must be ASCII".into())); + } + for pair in raw.split(|byte| *byte == b';') { + let pair = pair.trim_ascii(); + if pair.is_empty() { + continue; + } + let Some(separator) = pair.iter().position(|byte| *byte == b'=') else { + return Err(InputError("invalid Cookie header".into())); + }; + let (key, encoded) = pair.split_at(separator); + let value = &encoded[1..]; + let value = if value.len() >= 2 + && value.first() == Some(&b'"') + && value.last() == Some(&b'"') + { + &value[1..value.len() - 1] + } else { + value + }; + let name = std::str::from_utf8(key).expect("ASCII Cookie header"); + validate_cookie_name(name).map_err(InputError)?; + validate_cookie_value(value).map_err(InputError)?; + if cookies + .insert( + name.to_owned(), + crate::secret::Secret::from_input(value.to_vec()), + ) + .is_some() + { + return Err(InputError("duplicate Cookie name".into())); + } + } + } + Ok(cookies) +} + +pub fn response_header(name: &str, value: &str) -> Result<(), String> { + validate_header_name(name)?; + if [ + "content-length", + "content-type", + "content-encoding", + "transfer-encoding", + "connection", + "set-cookie", + "host", + "upgrade", + "keep-alive", + "trailer", + "te", + ] + .iter() + .any(|blocked| name.eq_ignore_ascii_case(blocked)) + { + return Err("response header is controlled by the HTTP runtime".into()); + } + hyper::header::HeaderValue::from_str(value) + .map_err(|_| "invalid response header value".to_owned())?; + with_request_mut(|request| { + request.pending_response_headers.push(Header { + name: name.to_owned(), + value: Bytes::from_text(value), + }); + Ok(()) + }) +} + +pub fn response_cookie(name: &str, value: &str, options: CookieOptions) -> Result<(), String> { + write_cookie(name, value.as_bytes(), options, false) +} + +pub fn response_secret_cookie( + name: &str, + value: crate::secret::Secret, + options: CookieOptions, +) -> Result<(), String> { + let length = base64::encoded_len(value.0.len(), false) + .ok_or_else(|| "Secret Cookie value is too large".to_owned())?; + let mut encoded = vec![0; length + 3]; + encoded[..3].copy_from_slice(b"v1."); + let written = base64::engine::general_purpose::URL_SAFE_NO_PAD + .encode_slice(&value.0, &mut encoded[3..]) + .map_err(|_| "Secret Cookie encoding failed".to_owned())?; + encoded.truncate(written + 3); + write_cookie(name, &encoded, options, true) +} + +fn write_cookie( + name: &str, + value: &[u8], + options: CookieOptions, + secret: bool, +) -> Result<(), String> { + require_same_origin_for_cookie_write()?; + validate_cookie_name(name)?; + validate_cookie_value(value)?; + if secret && (!options.secure || !options.http_only) { + return Err("Secret cookies require Secure and HttpOnly".into()); + } + if options.same_site == SameSite::None && !options.secure { + return Err("SameSite=None requires Secure".into()); + } + if !options.path.starts_with('/') + || !options + .path + .bytes() + .all(|byte| (0x21..=0x7e).contains(&byte) && byte != b';') + { + return Err("invalid Cookie Path".into()); + } + if let Some(domain) = &options.domain + && (domain.len() > 253 + || domain.is_empty() + || domain.split('.').any(|label| { + label.is_empty() + || label.len() > 63 + || label.starts_with('-') + || label.ends_with('-') + || !label + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-') + })) + { + return Err("invalid Cookie Domain".into()); + } + if options.max_age.is_some_and(|age| age < 0) { + return Err("Cookie Max-Age must be nonnegative".into()); + } + let mut encoded = Vec::with_capacity(name.len() + value.len() + 96); + encoded.extend_from_slice(name.as_bytes()); + encoded.push(b'='); + encoded.extend_from_slice(value); + encoded.extend_from_slice(b"; Path="); + encoded.extend_from_slice(options.path.as_bytes()); + if let Some(domain) = options.domain { + encoded.extend_from_slice(b"; Domain="); + encoded.extend_from_slice(domain.as_bytes()); + } + encoded.extend_from_slice(match options.same_site { + SameSite::Strict => b"; SameSite=Strict", + SameSite::Lax => b"; SameSite=Lax", + SameSite::None => b"; SameSite=None", + }); + if options.secure { + encoded.extend_from_slice(b"; Secure"); + } + if options.http_only { + encoded.extend_from_slice(b"; HttpOnly"); + } + if let Some(age) = options.max_age { + encoded.extend_from_slice(format!("; Max-Age={age}").as_bytes()); + } + if encoded.len() > 4096 { + return Err("Set-Cookie header exceeds 4096 bytes".into()); + } + with_request_mut(|request| { + request.pending_response_headers.push(Header { + name: "set-cookie".into(), + value: Bytes::new(encoded), + }); + Ok(()) + }) +} + +fn require_same_origin_for_cookie_write() -> Result<(), String> { + with_request(|request| { + if matches!(request.method.as_str(), "GET" | "HEAD" | "OPTIONS") { + return Ok(()); + } + #[cfg(feature = "api")] + { + let origin = crate::auth::cookie_origin() + .map_err(|_| "site origin is required for unsafe Cookie writes".to_owned())?; + validate_same_origin(&request.headers, &origin) + } + #[cfg(not(feature = "api"))] + { + Err("site authentication is required for unsafe Cookie writes".into()) + } + }) +} + +#[cfg(feature = "api")] +pub(crate) fn validate_same_origin( + headers: &[Header], + expected: &crate::config::HttpOrigin, +) -> Result<(), String> { + let host = unique_header(headers, "host")? + .ok_or_else(|| "unsafe Cookie request requires Host".to_owned())?; + let origin = unique_header(headers, "origin")? + .ok_or_else(|| "unsafe Cookie request requires Origin".to_owned())?; + let origin = origin + .parse::<hyper::Uri>() + .map_err(|_| "invalid Origin".to_owned())?; + if origin.path() != "/" || origin.query().is_some() { + return Err("Origin must not contain a path or query".into()); + } + let origin_authority = origin + .authority() + .ok_or_else(|| "invalid Origin".to_owned())?; + let request_authority = host + .parse::<hyper::http::uri::Authority>() + .map_err(|_| "invalid Host".to_owned())?; + let default_port = if expected.scheme() == "https" { + 443 + } else { + 80 + }; + if origin.scheme_str() != Some(expected.scheme()) + || normalized_authority(origin_authority, default_port) + != (expected.host().to_owned(), expected.port()) + || normalized_authority(&request_authority, default_port) + != (expected.host().to_owned(), expected.port()) + { + return Err("Origin and Host must exactly match the configured site origin".into()); + } + Ok(()) +} + +#[cfg(feature = "api")] +fn unique_header(headers: &[Header], name: &str) -> Result<Option<String>, String> { + let mut values = headers + .iter() + .filter(|header| header.name.eq_ignore_ascii_case(name)); + let Some(value) = values.next() else { + return Ok(None); + }; + if values.next().is_some() { + return Err(format!("duplicate {name} header")); + } + let value = std::str::from_utf8(value.value.values()) + .map_err(|_| format!("{name} header is not UTF-8"))?; + Ok(Some(value.to_owned())) +} + +#[cfg(feature = "api")] +fn normalized_authority( + authority: &hyper::http::uri::Authority, + default_port: u16, +) -> (String, u16) { + ( + authority.host().to_ascii_lowercase(), + authority.port_u16().unwrap_or(default_port), + ) +} + +fn validate_header_name(name: &str) -> Result<(), String> { + hyper::header::HeaderName::from_bytes(name.as_bytes()) + .map(|_| ()) + .map_err(|_| "invalid HTTP header name".to_owned()) +} + +fn validate_cookie_name(name: &str) -> Result<(), String> { + if name.is_empty() + || !name.bytes().all(|byte| { + byte.is_ascii_alphanumeric() + || matches!( + byte, + b'!' | b'#' + ..=b'\'' | b'*' | b'+' | b'-' | b'.' | b'^' | b'_' | b'`' | b'|' | b'~' + ) + }) + { + return Err("invalid Cookie name".into()); + } + Ok(()) +} + +fn validate_cookie_value(value: &[u8]) -> Result<(), String> { + if value + .iter() + .all(|byte| matches!(byte, 0x21 | 0x23..=0x2b | 0x2d..=0x3a | 0x3c..=0x5b | 0x5d..=0x7e)) + { + Ok(()) + } else { + Err("invalid Cookie value".into()) + } +} + +pub fn success(data: crate::wire::Encoded) -> Response { + envelope(200, 0, "ok", data) +} + +pub fn invalid_input(error: InputError) -> Response { + envelope(400, 400, &error.0, crate::wire::Encoded::null()) +} + +pub fn standard_error(status: i64) -> Response { + let message = match status { + 400 => "invalid request", + 401 => "unauthorized", + 403 => "forbidden", + 404 => "not found", + 409 => "conflict", + 429 => "too many requests", + _ => return internal_error(), + }; + envelope(status, status, message, crate::wire::Encoded::null()) +} + +#[cfg(feature = "api")] +pub fn auth_error(error: crate::auth::Error) -> Response { + match error { + crate::auth::Error::Unauthorized => standard_error(401), + crate::auth::Error::Forbidden => standard_error(403), + crate::auth::Error::Internal => { + crate::log::error("authentication runtime failed", Vec::new()); + internal_error() + } + } +} + +pub fn not_found() -> Response { + envelope(404, 404, "not found", crate::wire::Encoded::null()) +} + +pub fn method_not_allowed(methods: &[&str]) -> Response { + let mut response = envelope(405, 405, "method not allowed", crate::wire::Encoded::null()); + response.headers.push(Header { + name: "allow".into(), + value: Bytes::from_string(methods.join(", ")), + }); + response +} + +pub fn internal_error() -> Response { + envelope( + 500, + 500, + "internal server error", + crate::wire::Encoded::null(), + ) +} + +fn envelope(status: i64, code: i64, message: &str, data: crate::wire::Encoded) -> Response { + Response { + status, + headers: vec![Header { + name: "content-type".into(), + value: Bytes::from_text("application/json; charset=utf-8"), + }], + body: Bytes::from_string(format!( + "{{\"code\":{code},\"message\":{},\"data\":{}}}", + serde_json::to_string(message).expect("Text JSON encoding"), + data.as_str() + )), + } +} diff --git a/crates/dever-runtime/src/api/input.rs b/crates/dever-runtime/src/api/input.rs new file mode 100644 index 0000000..67a7ce4 --- /dev/null +++ b/crates/dever-runtime/src/api/input.rs @@ -0,0 +1,264 @@ +use std::collections::BTreeMap; +use std::fmt; + +use crate::http::Request; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct InputError(pub String); + +impl fmt::Display for InputError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.0) + } +} + +impl std::error::Error for InputError {} + +pub struct Inputs { + values: BTreeMap<String, String>, + query: bool, +} + +#[derive(Clone, Copy)] +pub enum QueryValue { + Text, + Int, + Bool, + Float, +} + +impl Inputs { + pub(crate) fn from_fields(values: BTreeMap<String, String>) -> Self { + Self { + values, + query: true, + } + } + pub fn from_request(request: &Request) -> Result<Self, InputError> { + let query = request + .target + .split_once('?') + .map_or("", |(_, query)| query); + match request.method.as_str() { + "GET" | "DELETE" => { + if !request.body.values().is_empty() { + return Err(InputError( + "GET and DELETE inputs must use the query string".into(), + )); + } + let mut values = BTreeMap::new(); + for pair in query.split('&').filter(|pair| !pair.is_empty()) { + let (name, value) = pair.split_once('=').unwrap_or((pair, "")); + let name = decode_query(name)?; + let value = serde_json::to_string(&decode_query(value)?) + .map_err(|error| InputError(error.to_string()))?; + if values.insert(name.clone(), value).is_some() { + return Err(InputError(format!("duplicate input '{name}'"))); + } + } + Ok(Self { + values, + query: true, + }) + } + "POST" | "PUT" => { + if !query.is_empty() { + return Err(InputError( + "POST and PUT inputs must use a JSON object body".into(), + )); + } + let mut content_types = request + .headers + .iter() + .filter(|header| header.name.eq_ignore_ascii_case("content-type")); + let content_type = content_types + .next() + .and_then(|header| std::str::from_utf8(header.value.values()).ok()); + if content_types.next().is_some() + || !content_type.is_some_and(|value| { + value + .split(';') + .next() + .unwrap_or("") + .trim() + .eq_ignore_ascii_case("application/json") + }) + { + return Err(InputError( + "POST and PUT require Content-Type: application/json".into(), + )); + } + let body = std::str::from_utf8(request.body.values()) + .map_err(|_| InputError("JSON body must be UTF-8".into()))?; + Self::from_json_object(body) + } + _ => Err(InputError("unsupported API request method".into())), + } + } + + pub fn from_json_object(body: &str) -> Result<Self, InputError> { + let node = crate::wire::parse(body) + .map_err(|error| InputError(format!("invalid JSON object body: {error}")))?; + let values = node + .object() + .map_err(|error| InputError(format!("invalid JSON object body: {error}")))? + .iter() + .map(|(name, value)| (name.clone(), value.raw().to_owned())) + .collect(); + Ok(Self { + values, + query: false, + }) + } + + pub fn raw_json(&mut self, name: &str) -> Result<String, InputError> { + self.values.remove(name).ok_or_else(|| missing(name)) + } + + pub fn optional_raw_json(&mut self, name: &str) -> Result<Option<String>, InputError> { + Ok(self.values.remove(name)) + } + + pub fn json(&mut self, name: &str, kind: QueryValue) -> Result<String, InputError> { + let value = self.raw_json(name)?; + self.convert_query(name, kind, value) + } + + pub fn optional_json(&mut self, name: &str, kind: QueryValue) -> Result<String, InputError> { + let value = self + .optional_raw_json(name)? + .unwrap_or_else(|| "null".into()); + self.convert_query(name, kind, value) + } + + fn convert_query( + &self, + name: &str, + kind: QueryValue, + value: String, + ) -> Result<String, InputError> { + if !self.query || value == "null" { + return Ok(value); + } + let text: String = + serde_json::from_str(&value).map_err(|error| InputError(error.to_string()))?; + match kind { + QueryValue::Text => Ok(value), + QueryValue::Int => text + .parse::<i64>() + .map(|value| value.to_string()) + .map_err(|_| InputError(format!("input '{name}' must be Int"))), + QueryValue::Bool => text + .parse::<bool>() + .map(|value| value.to_string()) + .map_err(|_| InputError(format!("input '{name}' must be Bool"))), + QueryValue::Float => text + .parse::<f64>() + .ok() + .filter(|value| value.is_finite()) + .map(|value| value.to_string()) + .ok_or_else(|| InputError(format!("input '{name}' must be Float"))), + } + } + + pub fn text(&mut self, name: &str) -> Result<String, InputError> { + self.optional_text(name)?.ok_or_else(|| missing(name)) + } + + pub fn optional_text(&mut self, name: &str) -> Result<Option<String>, InputError> { + self.take(name, "Text", |node, _| node.text().ok().map(str::to_owned)) + } + + pub fn int(&mut self, name: &str) -> Result<i64, InputError> { + self.optional_int(name)?.ok_or_else(|| missing(name)) + } + + pub fn optional_int(&mut self, name: &str) -> Result<Option<i64>, InputError> { + self.take(name, "Int", |node, query| { + if query { + node.text().ok()?.parse().ok() + } else { + node.int().ok() + } + }) + } + + pub fn boolean(&mut self, name: &str) -> Result<bool, InputError> { + self.optional_boolean(name)?.ok_or_else(|| missing(name)) + } + + pub fn optional_boolean(&mut self, name: &str) -> Result<Option<bool>, InputError> { + self.take(name, "Bool", |node, query| { + if query { + node.text().ok()?.parse().ok() + } else { + node.boolean().ok() + } + }) + } + + fn take<T>( + &mut self, + name: &str, + expected: &str, + decode: impl FnOnce(&crate::wire::Node<'_>, bool) -> Option<T>, + ) -> Result<Option<T>, InputError> { + match self.values.remove(name) { + None => Ok(None), + Some(value) if value == "null" => Ok(None), + Some(value) => { + let node = crate::wire::parse(&value).map_err(InputError)?; + decode(&node, self.query) + .map(Some) + .ok_or_else(|| InputError(format!("input '{name}' must be {expected}"))) + } + } + } + + pub fn finish(self) -> Result<(), InputError> { + match self.values.keys().next() { + None => Ok(()), + Some(name) => Err(InputError(format!("unknown input '{name}'"))), + } + } +} + +pub fn text_bounds( + name: &str, + value: Option<&str>, + minimum: u32, + maximum: Option<u32>, +) -> Result<(), InputError> { + let Some(value) = value else { return Ok(()) }; + let length = value.chars().count(); + if length < minimum as usize || maximum.is_some_and(|limit| length > limit as usize) { + return Err(InputError(format!( + "input '{name}' is outside Model Text length bounds" + ))); + } + Ok(()) +} + +fn missing(name: &str) -> InputError { + InputError(format!("input '{name}' is required")) +} + +fn decode_query(value: &str) -> Result<String, InputError> { + let mut bytes = value.bytes(); + let mut decoded = Vec::with_capacity(value.len()); + while let Some(byte) = bytes.next() { + decoded.push(match byte { + b'+' => b' ', + b'%' => { + let high = bytes.next().and_then(|byte| (byte as char).to_digit(16)); + let low = bytes.next().and_then(|byte| (byte as char).to_digit(16)); + match (high, low) { + (Some(high), Some(low)) => ((high << 4) | low) as u8, + _ => return Err(InputError("query contains invalid percent encoding".into())), + } + } + byte => byte, + }); + } + String::from_utf8(decoded).map_err(|_| InputError("query must contain UTF-8 text".into())) +} diff --git a/crates/dever-runtime/src/api/upload.rs b/crates/dever-runtime/src/api/upload.rs new file mode 100644 index 0000000..d0034ce --- /dev/null +++ b/crates/dever-runtime/src/api/upload.rs @@ -0,0 +1,580 @@ +use std::cell::RefCell; +use std::collections::BTreeMap; +use std::future::Future; +use std::io::{Seek, Write}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; + +use http_body_util::BodyExt; +use hyper::body::{Body, Incoming}; + +use super::{InputError, Inputs, QueryValue}; +use crate::config::{HttpSettings, UploadLimits}; +use crate::http::Request; + +tokio::task_local! { static BODY: RefCell<(Option<Incoming>, HttpSettings)>; } + +pub(crate) async fn scope<T>( + body: Incoming, + settings: HttpSettings, + future: impl Future<Output = T>, +) -> T { + BODY.scope(RefCell::new((Some(body), settings)), future) + .await +} + +fn take_body() -> Result<Option<(Incoming, HttpSettings)>, InputError> { + BODY.try_with(|state| { + let mut state = state.borrow_mut(); + let body = state + .0 + .take() + .ok_or_else(|| invalid("request body was already consumed"))?; + Ok((body, state.1.clone())) + }) + .ok() + .transpose() +} + +/// Called only after route selection and authentication. Buffered unit-test requests +/// have no Incoming owner and already carry their bounded body. +pub async fn read_body(request: &mut Request) -> Result<(), InputError> { + let Some((mut body, settings)) = take_body()? else { + return Ok(()); + }; + let mut bytes = Vec::new(); + while let Some(frame) = body.frame().await { + let frame = frame.map_err(|_| invalid("incomplete request body"))?; + let chunk = frame + .into_data() + .map_err(|_| invalid("request trailers are not supported"))?; + if chunk.len() > settings.limits.body_bytes as usize - bytes.len() { + return Err(invalid("request body exceeds body_bytes")); + } + bytes.extend_from_slice(&chunk); + } + request.body = crate::bytes::Bytes::new(bytes); + Ok(()) +} + +struct Temporary { + path: PathBuf, + file: Mutex<Option<std::fs::File>>, +} + +impl Drop for Temporary { + fn drop(&mut self) { + // Close before unlinking, including on platforms that cannot unlink open files. + self.file + .get_mut() + .unwrap_or_else(|error| error.into_inner()) + .take(); + if let Err(error) = std::fs::remove_file(&self.path) + && error.kind() != std::io::ErrorKind::NotFound + { + crate::log::error("upload temporary cleanup failed", Vec::new()); + } + } +} + +/// Non-cloneable request resource. The path and file are never exposed to source code. +pub struct Upload { + temporary: Arc<Temporary>, + filename: String, + content_type: String, + size: i64, +} + +impl std::fmt::Debug for Upload { + fn fmt(&self, output: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + output.write_str("Upload(<request resource>)") + } +} + +impl Upload { + pub fn filename(&self) -> String { + self.filename.clone() + } + pub fn content_type(&self) -> String { + self.content_type.clone() + } + pub fn size(&self) -> i64 { + self.size + } + pub fn close(self) { + drop(self); + } +} + +struct PendingStore { + path: Option<PathBuf>, + file: Option<std::fs::File>, + key: crate::orm::Uuid, +} + +impl PendingStore { + fn commit(mut self) -> crate::orm::Uuid { + self.file.take(); + self.path.take(); + self.key + } +} + +impl Drop for PendingStore { + fn drop(&mut self) { + self.file.take(); + if let Some(path) = self.path.take() + && let Err(error) = std::fs::remove_file(path) + && error.kind() != std::io::ErrorKind::NotFound + { + crate::log::error("uncommitted upload storage cleanup failed", Vec::new()); + } + } +} + +/// Only generated storage Adapter calls reach this consuming sink. If cancellation +/// drops the blocking result before delivery, PendingStore removes its target too. +pub async fn store(upload: Upload) -> Result<crate::orm::Uuid, String> { + let directory = BODY + .try_with(|state| state.borrow().1.upload_directory.with_file_name("upload")) + .map_err(|_| "upload storage requires a live API request".to_owned())?; + let pending = tokio::task::spawn_blocking(move || { + std::fs::create_dir_all(&directory) + .map_err(|_| "cannot create upload storage directory")?; + let mut random = [0u8; 16]; + getrandom::getrandom(&mut random).map_err(|_| "upload storage randomness unavailable")?; + random[6] = (random[6] & 0x0f) | 0x40; + random[8] = (random[8] & 0x3f) | 0x80; + let key = crate::orm::Uuid::from_bytes(random); + let path = directory.join(key.to_string()); + let mut options = std::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let file = options + .open(&path) + .map_err(|_| "cannot create upload storage file")?; + let mut pending = PendingStore { + path: Some(path), + file: Some(file), + key, + }; + let mut source = upload + .temporary + .file + .lock() + .map_err(|_| "upload source lock failed")?; + let source = source.as_mut().ok_or("upload source is closed")?; + source.rewind().map_err(|_| "cannot rewind upload source")?; + let target = pending.file.as_mut().expect("pending target owns its file"); + std::io::copy(source, target).map_err(|_| "upload storage copy failed")?; + target + .sync_all() + .map_err(|_| "upload storage flush failed")?; + Ok::<_, String>(pending) + }) + .await + .map_err(|_| "upload storage worker failed".to_owned())??; + Ok(pending.commit()) +} + +pub struct Multipart { + inputs: Inputs, + uploads: BTreeMap<String, Upload>, +} + +impl Multipart { + pub fn raw_json(&mut self, name: &str) -> Result<String, InputError> { + self.inputs.raw_json(name) + } + pub fn optional_raw_json(&mut self, name: &str) -> Result<Option<String>, InputError> { + self.inputs.optional_raw_json(name) + } + pub fn take(&mut self, name: &str) -> Result<Upload, InputError> { + self.uploads + .remove(name) + .ok_or_else(|| invalid("missing upload field")) + } + pub fn json(&mut self, name: &str, kind: QueryValue) -> Result<String, InputError> { + self.inputs.json(name, kind) + } + pub fn optional_json(&mut self, name: &str, kind: QueryValue) -> Result<String, InputError> { + self.inputs.optional_json(name, kind) + } + pub fn finish(self) -> Result<(), InputError> { + if !self.uploads.is_empty() { + return Err(invalid("unknown upload field")); + } + self.inputs.finish() + } +} + +pub async fn read_multipart(request: &Request) -> Result<Multipart, InputError> { + let Some((body, settings)) = take_body()? else { + return Err(invalid("multipart requires an incoming HTTP body")); + }; + if request.method != "POST" || request.target.contains('?') { + return Err(invalid("multipart requires POST without query inputs")); + } + let mut types = request + .headers + .iter() + .filter(|header| header.name.eq_ignore_ascii_case("content-type")); + let content_type = types + .next() + .and_then(|header| std::str::from_utf8(header.value.values()).ok()) + .ok_or_else(|| invalid("missing multipart Content-Type"))?; + if types.next().is_some() { + return Err(invalid("duplicate Content-Type")); + } + decode( + body, + content_type, + &settings.upload_directory, + settings.upload, + ) + .await +} + +/// The same streaming decoder is used by the HTTP owner and bounded body tests. +pub async fn decode<B>( + body: B, + content_type: &str, + directory: &Path, + limits: UploadLimits, +) -> Result<Multipart, InputError> +where + B: Body<Data = bytes::Bytes> + Unpin, +{ + limits.validate().map_err(InputError)?; + let (media, mut parameters) = parameters(content_type)?; + let boundary = parameters + .remove("boundary") + .ok_or_else(|| invalid("missing multipart boundary"))?; + if !media.eq_ignore_ascii_case("multipart/form-data") + || !parameters.is_empty() + || boundary.is_empty() + || boundary.len() > 70 + || !boundary + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"'-_.,+/:=?()".contains(&byte)) + { + return Err(invalid("invalid multipart Content-Type")); + } + let mut reader = Reader { + body, + pending: bytes::Bytes::new(), + buffer: Vec::new(), + total: 0, + limit: limits.total_bytes, + ended: false, + }; + reader.exact(format!("--{boundary}\r\n").as_bytes()).await?; + let delimiter = format!("\r\n--{boundary}").into_bytes(); + let mut uploads = BTreeMap::new(); + let mut fields = BTreeMap::new(); + let mut names = std::collections::BTreeSet::new(); + for part in 0..limits.parts { + let headers = reader.until(b"\r\n\r\n", limits.header_bytes).await?; + let (name, filename, content_type) = part_headers(&headers, limits.filename_bytes)?; + if !names.insert(name.clone()) { + return Err(invalid("duplicate multipart field")); + } + let mut upload = match filename { + Some(filename) => Some(create(directory.to_path_buf(), filename, content_type).await?), + None => { + if fields.len() >= limits.fields { + return Err(invalid("multipart fields exceed limit")); + } + None + } + }; + let mut field = Vec::new(); + let mut size = 0usize; + loop { + let (chunk, end) = reader.part_chunk(&delimiter).await?; + let maximum = if upload.is_some() { + limits.file_bytes + } else { + limits.field_bytes + }; + if chunk.len() > maximum - size { + return Err(invalid("multipart part exceeds byte limit")); + } + size += chunk.len(); + if let Some(upload) = &mut upload { + let temporary = upload.temporary.clone(); + tokio::task::spawn_blocking(move || { + temporary + .file + .lock() + .map_err(|_| invalid("upload file lock failed"))? + .as_mut() + .ok_or_else(|| invalid("upload file closed"))? + .write_all(&chunk) + .map_err(|_| invalid("upload write failed")) + }) + .await + .map_err(|_| invalid("upload writer failed"))??; + upload.size = size as i64; + } else { + field.extend_from_slice(&chunk); + } + if end { + break; + } + } + if let Some(upload) = upload { + uploads.insert(name, upload); + } else { + let value = + String::from_utf8(field).map_err(|_| invalid("multipart scalar is not UTF-8"))?; + fields.insert( + name, + serde_json::to_string(&value) + .map_err(|_| invalid("multipart scalar encoding failed"))?, + ); + } + reader.fill(2).await?; + if reader.buffer.starts_with(b"--") { + reader.exact(b"--\r\n").await?; + if reader.fill(1).await? { + return Err(invalid("multipart epilogue is not supported")); + } + return Ok(Multipart { + inputs: Inputs::from_fields(fields), + uploads, + }); + } + reader.exact(b"\r\n").await?; + if part + 1 == limits.parts { + return Err(invalid("multipart parts exceed limit")); + } + } + Err(invalid("incomplete multipart body")) +} + +async fn create( + directory: PathBuf, + filename: String, + content_type: String, +) -> Result<Upload, InputError> { + tokio::task::spawn_blocking(move || { + std::fs::create_dir_all(&directory) + .map_err(|_| invalid("cannot create upload directory"))?; + let mut random = [0u8; 16]; + getrandom::getrandom(&mut random).map_err(|_| invalid("upload randomness unavailable"))?; + let name = random + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::<String>(); + let path = directory.join(format!("upload-{name}")); + let mut options = std::fs::OpenOptions::new(); + options.read(true).write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let file = options + .open(&path) + .map_err(|_| invalid("cannot create upload file"))?; + Ok(Upload { + temporary: Arc::new(Temporary { + path, + file: Mutex::new(Some(file)), + }), + filename, + content_type, + size: 0, + }) + }) + .await + .map_err(|_| invalid("upload owner failed"))? +} + +fn invalid(message: &str) -> InputError { + InputError(message.to_owned()) +} + +fn parameters(value: &str) -> Result<(&str, BTreeMap<String, String>), InputError> { + let mut parts = value.split(';'); + let name = parts.next().unwrap_or("").trim(); + let mut parameters = BTreeMap::new(); + for part in parts { + let (key, value) = part + .trim() + .split_once('=') + .ok_or_else(|| invalid("invalid multipart parameter"))?; + let value = value.trim(); + let value = value + .strip_prefix('"') + .and_then(|value| value.strip_suffix('"')) + .unwrap_or(value); + if value + .bytes() + .any(|byte| byte < 32 || byte == 127 || byte == b'"' || byte == b'\\') + || parameters + .insert(key.trim().to_ascii_lowercase(), value.to_owned()) + .is_some() + { + return Err(invalid("invalid or duplicate multipart parameter")); + } + } + Ok((name, parameters)) +} + +fn part_headers( + bytes: &[u8], + filename_limit: usize, +) -> Result<(String, Option<String>, String), InputError> { + let headers = std::str::from_utf8(bytes).map_err(|_| invalid("invalid multipart headers"))?; + let mut disposition = None; + let mut content_type = None; + for line in headers.split("\r\n") { + let (name, value) = line + .split_once(':') + .ok_or_else(|| invalid("invalid multipart header"))?; + if value.bytes().any(|byte| byte < 32 || byte == 127) { + return Err(invalid("invalid multipart header value")); + } + match name.to_ascii_lowercase().as_str() { + "content-disposition" if disposition.is_none() => disposition = Some(value.trim()), + "content-type" if content_type.is_none() => { + content_type = Some(value.trim().to_owned()) + } + _ => return Err(invalid("unsupported or duplicate multipart header")), + } + } + let (kind, mut parameters) = + parameters(disposition.ok_or_else(|| invalid("missing Content-Disposition"))?)?; + if !kind.eq_ignore_ascii_case("form-data") { + return Err(invalid("expected form-data disposition")); + } + let name = parameters + .remove("name") + .ok_or_else(|| invalid("missing multipart field name"))?; + if name.is_empty() + || name.len() > 128 + || !name + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') + { + return Err(invalid("invalid multipart field name")); + } + let filename = parameters.remove("filename"); + if !parameters.is_empty() { + return Err(invalid("unsupported multipart disposition parameter")); + } + if filename.as_ref().is_some_and(|name| { + name.is_empty() + || name.len() > filename_limit + || name == "." + || name == ".." + || name.contains(['/', '\\', ':']) + }) { + return Err(invalid("invalid upload filename")); + } + let content_type = content_type.unwrap_or_else(|| "application/octet-stream".into()); + Ok((name, filename, content_type)) +} + +struct Reader<B> { + body: B, + pending: bytes::Bytes, + buffer: Vec<u8>, + total: usize, + limit: usize, + ended: bool, +} + +impl<B: Body<Data = bytes::Bytes> + Unpin> Reader<B> { + async fn fill(&mut self, required: usize) -> Result<bool, InputError> { + while self.buffer.len() < required && !self.ended { + if self.pending.is_empty() { + match self.body.frame().await { + Some(Ok(frame)) => { + self.pending = frame + .into_data() + .map_err(|_| invalid("multipart trailers are not supported"))?; + if self.pending.len() > self.limit - self.total { + return Err(invalid("multipart total exceeds byte limit")); + } + self.total += self.pending.len(); + } + Some(Err(_)) => return Err(invalid("incomplete multipart body")), + None => { + self.ended = true; + continue; + } + } + } + let count = self.pending.len().min(16_384); + self.buffer.extend_from_slice(&self.pending.split_to(count)); + } + Ok(self.buffer.len() >= required) + } + + async fn exact(&mut self, expected: &[u8]) -> Result<(), InputError> { + if !self.fill(expected.len()).await? || !self.buffer.starts_with(expected) { + return Err(invalid("invalid multipart delimiter")); + } + self.buffer.drain(..expected.len()); + Ok(()) + } + + async fn until(&mut self, delimiter: &[u8], limit: usize) -> Result<Vec<u8>, InputError> { + loop { + if let Some(index) = self + .buffer + .windows(delimiter.len()) + .position(|part| part == delimiter) + { + if index > limit { + return Err(invalid("multipart headers exceed limit")); + } + let bytes = self.buffer.drain(..index).collect(); + self.buffer.drain(..delimiter.len()); + return Ok(bytes); + } + if self.buffer.len() > limit + delimiter.len() + || !self.fill(self.buffer.len() + 1).await? + { + return Err(invalid("incomplete or oversized multipart headers")); + } + } + } + + async fn part_chunk(&mut self, delimiter: &[u8]) -> Result<(Vec<u8>, bool), InputError> { + loop { + if self.buffer.len() <= delimiter.len() + 2 && !self.ended { + self.fill(16_384).await?; + } + let mut cursor = 0; + while let Some(offset) = self.buffer[cursor..] + .windows(delimiter.len()) + .position(|part| part == delimiter) + { + let index = cursor + offset; + self.fill(index + delimiter.len() + 4).await?; + let suffix = &self.buffer[index + delimiter.len()..]; + if suffix.starts_with(b"--\r\n") || suffix.starts_with(b"\r\n") { + let chunk = self.buffer.drain(..index).collect(); + self.buffer.drain(..delimiter.len()); + return Ok((chunk, true)); + } + cursor = index + 1; + } + if self.buffer.len() > delimiter.len() + 2 { + let count = self.buffer.len() - delimiter.len() - 2; + return Ok((self.buffer.drain(..count).collect(), false)); + } + if !self.fill(self.buffer.len() + 1).await? { + return Err(invalid("incomplete multipart file")); + } + } + } +} diff --git a/crates/dever-runtime/src/application.rs b/crates/dever-runtime/src/application.rs new file mode 100644 index 0000000..1a92e24 --- /dev/null +++ b/crates/dever-runtime/src/application.rs @@ -0,0 +1,212 @@ +//! Invocation-owned resources shared by generated application roots and requests. +use std::cell::RefCell; +use std::future::Future; +use std::sync::Arc; + +use crate::config::{RuntimeProfile, Settings}; + +tokio::task_local! { + static CURRENT: RefCell<Option<Arc<Session>>>; +} + +pub struct Session { + settings: Arc<Settings>, + lifecycle: std::sync::OnceLock<Arc<crate::lifecycle::Lifecycle>>, + authorization: std::sync::OnceLock<bool>, + #[cfg(any(feature = "sqlite", feature = "postgres"))] + databases: Arc<crate::database::Session>, + #[cfg(any(feature = "sqlite", feature = "postgres"))] + tenant: std::sync::OnceLock<Arc<crate::tenant::Manager>>, + #[cfg(any(feature = "sqlite", feature = "postgres"))] + migrations: std::sync::Mutex<Vec<crate::database::Database>>, + #[cfg(any(feature = "sqlite", feature = "postgres"))] + jobs: std::sync::OnceLock<Arc<crate::job::resources::Resources>>, +} + +impl Session { + pub fn load( + profile: RuntimeProfile, + bindings: &[(Option<&str>, &str, bool)], + transactions: &[&[(Option<&str>, &str, bool)]], + ) -> Result<Arc<Self>, String> { + let settings = Arc::new(Settings::load_for_executable()?); + #[cfg(any(feature = "sqlite", feature = "postgres"))] + let databases = Arc::new(crate::database::Session::from_settings( + settings.clone(), + profile, + bindings, + transactions, + )?); + #[cfg(not(any(feature = "sqlite", feature = "postgres")))] + settings.prepare_databases(profile, bindings, transactions)?; + Ok(Arc::new(Self { + settings, + lifecycle: std::sync::OnceLock::new(), + authorization: std::sync::OnceLock::new(), + #[cfg(any(feature = "sqlite", feature = "postgres"))] + databases, + #[cfg(any(feature = "sqlite", feature = "postgres"))] + tenant: std::sync::OnceLock::new(), + #[cfg(any(feature = "sqlite", feature = "postgres"))] + migrations: std::sync::Mutex::new(Vec::new()), + #[cfg(any(feature = "sqlite", feature = "postgres"))] + jobs: std::sync::OnceLock::new(), + })) + } + + pub fn settings(&self) -> &Arc<Settings> { + &self.settings + } + + pub fn initialize_authorization(&self, enabled: bool) -> Result<(), String> { + self.authorization + .set(enabled) + .map_err(|_| "application authorization initialized twice".into()) + } + + pub fn has_authorization(&self) -> bool { + self.authorization.get().copied().unwrap_or(false) + } + + pub(crate) fn lifecycle(&self) -> Option<Arc<crate::lifecycle::Lifecycle>> { + self.lifecycle.get().cloned() + } + + pub(crate) fn set_lifecycle( + &self, + state: Arc<crate::lifecycle::Lifecycle>, + ) -> Result<(), String> { + self.lifecycle + .set(state) + .map_err(|_| "application lifecycle initialized twice".into()) + } + + #[cfg(any(feature = "sqlite", feature = "postgres"))] + pub fn databases(&self) -> &Arc<crate::database::Session> { + &self.databases + } + + #[cfg(any(feature = "sqlite", feature = "postgres"))] + pub fn jobs(&self) -> Option<&Arc<crate::job::resources::Resources>> { + self.jobs.get() + } + + #[cfg(any(feature = "sqlite", feature = "postgres"))] + pub fn initialize_jobs( + &self, + jobs: Arc<crate::job::resources::Resources>, + ) -> Result<(), String> { + self.jobs + .set(jobs) + .map_err(|_| "application Jobs initialized twice".into()) + } + + #[cfg(any(feature = "sqlite", feature = "postgres"))] + pub fn own_migration_database(&self, database: crate::database::Database) { + self.migrations + .lock() + .unwrap_or_else(|error| error.into_inner()) + .push(database); + } + + #[cfg(any(feature = "sqlite", feature = "postgres"))] + pub(crate) fn tenant(&self) -> Option<Arc<crate::tenant::Manager>> { + self.tenant.get().cloned() + } + + #[cfg(any(feature = "sqlite", feature = "postgres"))] + pub(crate) fn set_tenant( + &self, + manager: Arc<crate::tenant::Manager>, + ) -> Result<(), crate::orm::Error> { + self.tenant.set(manager).map_err(|_| { + crate::orm::Error::migration("tenant storage was initialized more than once") + }) + } + + pub async fn scope<T>(self: Arc<Self>, future: impl Future<Output = T>) -> T { + CURRENT.scope(RefCell::new(Some(self)), future).await + } + + pub async fn close(&self) -> Result<(), String> { + #[cfg(any(feature = "sqlite", feature = "postgres"))] + { + self.databases.stop_accepting(); + let migrations = std::mem::take( + &mut *self + .migrations + .lock() + .unwrap_or_else(|error| error.into_inner()), + ); + for database in &migrations { + database.stop_accepting(); + } + let mut migration_error = None; + for database in migrations { + if let Err(error) = database.close_pool().await { + migration_error = Some(match migration_error { + Some(previous) => crate::orm::Error::cause(previous, error), + None => error, + }); + } + } + let tenant = match self.tenant.get() { + Some(manager) => manager.close().await, + None => Ok(()), + }; + let databases = self.databases.close().await; + for result in [tenant, databases] { + if let Err(error) = result { + migration_error = Some(match migration_error { + Some(previous) => previous.cause(error), + None => error, + }); + } + } + crate::log::flush(); + migration_error.map_or(Ok(()), |error| Err(error.to_string())) + } + #[cfg(not(any(feature = "sqlite", feature = "postgres")))] + { + crate::log::flush(); + Ok(()) + } + } +} + +pub fn current() -> Option<Arc<Session>> { + CURRENT + .try_with(|current| current.borrow().clone()) + .ok() + .flatten() +} + +/// Installation occurs inside the generated root, before any application work. +pub fn install(session: Arc<Session>) -> Result<(), String> { + CURRENT + .try_with(|current| { + let mut current = current.borrow_mut(); + if current.is_some() { + return Err("application session was initialized more than once".into()); + } + *current = Some(session); + Ok(()) + }) + .map_err(|_| "application session requires an application root".to_owned())? +} + +pub async fn scope_entry<T>(future: impl Future<Output = T>) -> T { + CURRENT.scope(RefCell::new(None), future).await +} + +/// Carry application resources across a task boundary, never request identity. +pub async fn scope_resources<T>( + session: Option<Arc<Session>>, + future: impl Future<Output = T>, +) -> T { + CURRENT.scope(RefCell::new(session), future).await +} + +pub fn with_resources<T>(session: Option<Arc<Session>>, apply: impl FnOnce() -> T) -> T { + CURRENT.sync_scope(RefCell::new(session), apply) +} diff --git a/crates/dever-runtime/src/async_stream.rs b/crates/dever-runtime/src/async_stream.rs new file mode 100644 index 0000000..b983fed --- /dev/null +++ b/crates/dever-runtime/src/async_stream.rs @@ -0,0 +1,104 @@ +use std::pin::Pin; +use std::sync::Arc; + +use futures_util::{Stream, StreamExt, stream}; +use tokio::sync::{Mutex, Semaphore}; + +use crate::resource::Pull; + +type Producer<T> = Pin<Box<dyn Stream<Item = Pull<T>> + Send>>; + +struct State<T> { + producer: Mutex<Option<Producer<T>>>, + closed: Semaphore, +} + +pub struct AsyncStream<T>(Arc<State<T>>); + +impl<T> Clone for AsyncStream<T> { + fn clone(&self) -> Self { + Self(Arc::clone(&self.0)) + } +} + +impl<T> std::fmt::Debug for AsyncStream<T> { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("AsyncStream(<resource>)") + } +} + +impl<T> AsyncStream<T> { + pub fn new(producer: impl Stream<Item = Pull<T>> + Send + 'static) -> Self { + Self(Arc::new(State { + producer: Mutex::new(Some(Box::pin(producer))), + closed: Semaphore::new(0), + })) + } + + async fn pull_next(&self) -> Pull<T> { + let mut producer = self.0.producer.lock().await; + let result = match producer.as_mut() { + Some(producer) => tokio::select! { + biased; + _ = self.0.closed.acquire() => Pull::End, + value = producer.next() => value.unwrap_or(Pull::End), + }, + None => Pull::End, + }; + if matches!(&result, Pull::Last(_) | Pull::End) { + producer.take(); + } + result + } + + pub async fn pull(&self) -> Option<T> { + match self.pull_next().await { + Pull::Item(value) | Pull::Last(value) => Some(value), + Pull::End => None, + } + } + + pub fn close(&self) { + self.0.closed.close(); + // 挂起的 pull 收到关闭信号后，会在其持有的锁内释放生产者。 + if let Ok(mut producer) = self.0.producer.try_lock() { + producer.take(); + } + } + + pub fn close_on_drop(&self) -> CloseOnDrop<T> { + CloseOnDrop(self.clone()) + } +} + +impl<T: Send + 'static> AsyncStream<T> { + pub fn from_values(values: impl IntoIterator<Item = T, IntoIter: Send + 'static>) -> Self { + Self::new(stream::iter(values.into_iter().map(Pull::Item))) + } + + pub fn map<U: Send + 'static>( + self, + transform: impl FnMut(T) -> U + Send + 'static, + ) -> AsyncStream<U> { + let owner = self.close_on_drop(); + AsyncStream::new(stream::unfold( + (owner, transform), + |(owner, mut transform)| async move { + let result = match owner.0.pull_next().await { + Pull::Item(value) => Pull::Item(transform(value)), + Pull::Last(value) => Pull::Last(transform(value)), + Pull::End => return None, + }; + Some((result, (owner, transform))) + }, + )) + } +} + +pub struct CloseOnDrop<T>(AsyncStream<T>); + +impl<T> Drop for CloseOnDrop<T> { + fn drop(&mut self) { + self.0.close(); + } +} diff --git a/crates/dever-runtime/src/auth.rs b/crates/dever-runtime/src/auth.rs new file mode 100644 index 0000000..7a11c5b --- /dev/null +++ b/crates/dever-runtime/src/auth.rs @@ -0,0 +1,738 @@ +use std::future::Future; +use std::time::{SystemTime, UNIX_EPOCH}; + +use base64::Engine as _; +use hmac::{Hmac, Mac}; +use serde::{Deserialize, Serialize}; +use sha2::Sha256; + +use crate::config::{AuthProvider, Settings}; +use crate::http::{Header, Request}; +use crate::secret::Secret; + +const CLOCK_SKEW_SECONDS: i64 = 30; +const MAX_TOKEN_BYTES: usize = 8 * 1024; +const MAX_JWT_PART_BYTES: usize = 6 * 1024; +const MAX_IDENTITY_TEXT_BYTES: usize = 256; + +#[cfg(any(feature = "sqlite", feature = "postgres"))] +pub mod store; + +tokio::task_local! { static INVOCATION: Invocation; } + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Claims { + pub subject: String, + pub session: String, + pub tenant: Option<String>, + pub site: String, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Identity { + pub id: String, + pub user_id: Option<i64>, + pub tenant_id: Option<i64>, +} + +pub struct Prepared { + site: String, + provider_key: String, + provider: AuthProvider, + origin: Option<crate::config::HttpOrigin>, + claims: Option<Claims>, +} + +impl Prepared { + pub fn claims(&self) -> Option<&Claims> { + self.claims.as_ref() + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Error { + Unauthorized, + Forbidden, + Internal, +} + +impl std::fmt::Display for Error { + fn fmt(&self, output: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + output.write_str(match self { + Self::Unauthorized => "unauthorized", + Self::Forbidden => "forbidden", + Self::Internal => "authentication configuration error", + }) + } +} + +struct Invocation { + site: String, + provider_key: String, + provider: AuthProvider, + origin: Option<crate::config::HttpOrigin>, + session: Option<String>, + identity: Option<Identity>, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct JwtHeader { + alg: String, + typ: String, +} + +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct JwtClaims { + iss: String, + sub: String, + aud: String, + sid: String, + tenant: Option<String>, + iat: i64, + #[serde(skip_serializing_if = "Option::is_none")] + nbf: Option<i64>, + exp: i64, +} + +#[derive(Serialize)] +struct IssuedHeader { + alg: &'static str, + typ: &'static str, +} + +#[derive(Clone, Copy, Eq, PartialEq)] +enum CredentialKind { + Authorization, + Cookie, +} + +pub fn prepare(directory: &[&str], request: &Request, anonymous: bool) -> Result<Prepared, Error> { + prepare_with_settings( + &crate::config::current_settings(), + directory, + request, + anonymous, + ) +} + +pub fn prepare_with_settings( + settings: &Settings, + directory: &[&str], + request: &Request, + anonymous: bool, +) -> Result<Prepared, Error> { + let (site_key, site) = settings + .site_for_directory(directory) + .map_err(|_| Error::Internal)? + .ok_or(Error::Internal)?; + let provider = settings + .auth() + .provider(site.auth()) + .ok_or(Error::Internal)?; + validate_request_host(request, site.hosts())?; + + let authorization = bearer_token(request)?; + // Public entries are anonymous for browser Cookie credentials. Expired or + // revoked sessions therefore cannot prevent a fresh login. An explicit + // Authorization credential remains strict. + let cookie = if anonymous { + None + } else { + cookie_token(request, provider.cookie())? + }; + let credential = match (authorization, cookie) { + (Some(_), Some(_)) => return Err(Error::Unauthorized), + (Some(token), None) => Some((CredentialKind::Authorization, token)), + (None, Some(token)) => Some((CredentialKind::Cookie, token)), + (None, None) if anonymous => None, + (None, None) => return Err(Error::Unauthorized), + }; + if credential + .as_ref() + .is_some_and(|(kind, _)| *kind == CredentialKind::Cookie) + { + validate_cookie_origin(request, site.origin())?; + } + let claims = credential + .map(|(_, token)| verify_jwt(&token, site_key, site.auth(), provider)) + .transpose()?; + Ok(Prepared { + site: site_key.to_owned(), + provider_key: site.auth().to_owned(), + provider: provider.clone(), + origin: site.origin().cloned(), + claims, + }) +} + +pub async fn scope<F>( + prepared: Prepared, + identity: Option<Identity>, + future: F, +) -> Result<F::Output, Error> +where + F: Future, +{ + validate_identity(prepared.claims.as_ref(), identity.as_ref())?; + #[cfg(any(feature = "sqlite", feature = "postgres"))] + let execution = prepared + .claims + .as_ref() + .zip(identity.as_ref()) + .map(|(claims, identity)| crate::job::ExecutionIdentity::User { + tenant_id: identity.tenant_id, + provider: prepared.provider_key.clone(), + site: prepared.site.clone(), + subject: claims.subject.clone(), + session: claims.session.clone(), + tenant_claim: claims.tenant.clone(), + permission_key: String::new(), + }); + let session = prepared + .claims + .as_ref() + .map(|claims| claims.session.clone()); + let invocation = Invocation { + site: prepared.site, + provider_key: prepared.provider_key, + provider: prepared.provider, + origin: prepared.origin, + session, + identity, + }; + #[cfg(any(feature = "sqlite", feature = "postgres"))] + if let Some(tenant_id) = invocation + .identity + .as_ref() + .and_then(|identity| identity.tenant_id) + { + let future = crate::tenant::scope(tenant_id, future); + return Ok(match execution { + Some(execution) => { + INVOCATION + .scope(invocation, crate::job::scope_execution(execution, future)) + .await + } + None => INVOCATION.scope(invocation, future).await, + }); + } + #[cfg(any(feature = "sqlite", feature = "postgres"))] + if let Some(execution) = execution { + return Ok(INVOCATION + .scope(invocation, crate::job::scope_execution(execution, future)) + .await); + } + Ok(INVOCATION.scope(invocation, future).await) +} + +pub fn validate_job_source(provider_key: &str, site_key: &str) -> Result<(), Error> { + let settings = crate::config::current_settings(); + let site = settings.sites().get(site_key).ok_or(Error::Internal)?; + if site.auth() != provider_key || settings.auth().provider(provider_key).is_none() { + return Err(Error::Internal); + } + Ok(()) +} + +pub fn validate_job_identity(claims: &Claims, identity: &Identity) -> Result<(), Error> { + validate_identity(Some(claims), Some(identity)) +} + +pub fn site_key() -> Result<String, Error> { + with_invocation(|invocation| Ok(invocation.site.clone())) +} + +pub fn id() -> Result<String, Error> { + with_identity(|identity| Ok(identity.id.clone())) +} + +pub fn session() -> Result<String, Error> { + with_invocation(|invocation| { + if invocation.identity.is_none() { + return Err(Error::Unauthorized); + } + invocation.session.clone().ok_or(Error::Internal) + }) +} + +pub fn user_id() -> Result<Option<i64>, Error> { + with_identity(|identity| Ok(identity.user_id)) +} + +pub fn tenant_id() -> Result<Option<i64>, Error> { + with_identity(|identity| Ok(identity.tenant_id)) +} + +pub fn owns_user(user_id: i64) -> Result<bool, Error> { + with_identity(|identity| Ok(identity.user_id == Some(user_id))) +} + +#[cfg(any(feature = "sqlite", feature = "postgres"))] +pub async fn require_components( + components: &[&str], + tenant_components: &[&str], +) -> Result<(), Error> { + if crate::config::current_settings().tenant().is_none() || components.is_empty() { + return Ok(()); + } + let tenant = tenant_id()?.ok_or(Error::Forbidden)?; + crate::tenant::require_components(tenant, components, tenant_components) + .await + .map_err(|error| { + if error.kind() == crate::orm::ErrorKind::InvalidData { + Error::Forbidden + } else { + crate::log::error(error.to_string(), Vec::new()); + Error::Internal + } + }) +} + +#[cfg(any(feature = "sqlite", feature = "postgres"))] +pub async fn authorize_permission(permission: &str, site: &str) -> Result<(), Error> { + let user = user_id()?.ok_or(Error::Forbidden)?; + let (_, database, actual_site) = authorization_context().await.map_err(|error| { + crate::log::error(error, Vec::new()); + Error::Internal + })?; + if actual_site != site { + return Err(Error::Forbidden); + } + match store::authorize(database, user, site, permission).await { + Ok(true) => crate::job::bind_permission(permission).map_err(|error| { + crate::log::error(error.to_string(), Vec::new()); + Error::Internal + }), + Ok(false) => Err(Error::Forbidden), + Err(error) => { + crate::log::error(error.to_string(), Vec::new()); + Err(Error::Internal) + } + } +} + +#[cfg(any(feature = "sqlite", feature = "postgres"))] +pub async fn permissions() -> Result<Vec<store::CatalogPermission>, String> { + let (catalog, _, site) = authorization_context().await?; + let mut permissions = store::list_permissions(catalog) + .await + .map_err(|error| error.to_string())?; + permissions.retain(|permission| permission.site == site); + Ok(permissions) +} + +#[cfg(any(feature = "sqlite", feature = "postgres"))] +pub async fn save_role( + id: &str, + name: &str, + all_permissions: bool, + permission_keys: Vec<String>, +) -> Result<(), String> { + let (catalog, authorization, site) = authorization_context().await?; + store::save_role( + catalog, + authorization, + &store::Role { + id: id.to_owned(), + site, + name: name.to_owned(), + all_permissions, + }, + &permission_keys, + ) + .await + .map_err(|error| error.to_string()) +} + +#[cfg(any(feature = "sqlite", feature = "postgres"))] +pub async fn grant_role(user_id: i64, role_id: &str) -> Result<(), String> { + let (_, authorization, site) = authorization_context().await?; + store::grant_role(authorization, user_id, &site, role_id) + .await + .map_err(|error| error.to_string()) +} + +#[cfg(any(feature = "sqlite", feature = "postgres"))] +pub async fn revoke_role(user_id: i64, role_id: &str) -> Result<(), String> { + let (_, authorization, site) = authorization_context().await?; + store::revoke_role(authorization, user_id, &site, role_id) + .await + .map_err(|error| error.to_string()) +} + +#[cfg(any(feature = "sqlite", feature = "postgres"))] +pub async fn disable_role(role_id: &str) -> Result<(), String> { + let (_, authorization, site) = authorization_context().await?; + store::disable_role(authorization, &site, role_id) + .await + .map_err(|error| error.to_string()) +} + +#[cfg(any(feature = "sqlite", feature = "postgres"))] +async fn authorization_context() +-> Result<(crate::database::Database, crate::database::Database, String), String> { + let settings = crate::config::current_settings(); + let connection = settings + .tenant() + .map(|tenant| tenant.database()) + .unwrap_or("default"); + let catalog = + crate::database::database_for(Some(connection), "").map_err(|error| error.to_string())?; + let authorization = match tenant_id().map_err(|error| error.to_string())? { + Some(_) => crate::tenant::database(connection) + .await + .map_err(|error| error.to_string())?, + None => catalog.clone(), + }; + let site = site_key().map_err(|error| error.to_string())?; + Ok((catalog, authorization, site)) +} + +pub fn issue(subject: &str, session: &str, tenant: Option<&str>) -> Result<Secret, String> { + validate_identity_text("subject", subject).map_err(|error| error.to_string())?; + validate_identity_text("session", session).map_err(|error| error.to_string())?; + if let Some(tenant) = tenant { + validate_identity_text("tenant", tenant).map_err(|error| error.to_string())?; + } + INVOCATION + .try_with(|invocation| { + let now = unix_seconds().map_err(|error| error.to_string())?; + let ttl = i64::try_from(invocation.provider.ttl_seconds()) + .map_err(|_| Error::Internal.to_string())?; + let claims = JwtClaims { + iss: invocation.provider_key.clone(), + sub: subject.to_owned(), + aud: invocation.site.clone(), + sid: session.to_owned(), + tenant: tenant.map(str::to_owned), + iat: now, + nbf: None, + exp: now + .checked_add(ttl) + .ok_or_else(|| Error::Internal.to_string())?, + }; + encode_jwt(&claims, &invocation.provider) + .map(Secret::from_input) + .map_err(|error| error.to_string()) + }) + .map_err(|_| { + "authentication capability is available only within a site API entry".to_owned() + })? +} + +pub fn issue_cookie(subject: &str, session: &str, tenant: Option<&str>) -> Result<(), String> { + let token = issue(subject, session, tenant)?; + INVOCATION + .try_with(|invocation| { + let max_age = i64::try_from(invocation.provider.ttl_seconds()) + .map_err(|_| "authentication configuration error".to_owned())?; + crate::api::response_secret_cookie( + invocation.provider.cookie(), + token, + session_cookie_options(Some(max_age)), + ) + }) + .map_err(|_| { + "authentication capability is available only within a site API entry".to_owned() + })? +} + +pub fn clear_cookie() -> Result<(), String> { + INVOCATION + .try_with(|invocation| { + crate::api::response_cookie( + invocation.provider.cookie(), + "", + session_cookie_options(Some(0)), + ) + }) + .map_err(|_| { + "authentication capability is available only within a site API entry".to_owned() + })? +} + +fn session_cookie_options(max_age: Option<i64>) -> crate::api::CookieOptions { + crate::api::CookieOptions { + path: "/".into(), + domain: None, + same_site: crate::api::SameSite::Lax, + secure: true, + http_only: true, + max_age, + } +} + +pub(crate) fn cookie_origin() -> Result<crate::config::HttpOrigin, Error> { + INVOCATION + .try_with(|invocation| invocation.origin.clone().ok_or(Error::Internal)) + .map_err(|_| Error::Internal)? +} + +pub fn log_fields() -> Vec<crate::log::Field> { + INVOCATION + .try_with(|invocation| { + let mut fields = vec![crate::log::Field { + name: "site".into(), + value: invocation.site.clone(), + }]; + if let Some(identity) = &invocation.identity { + fields.push(crate::log::Field { + name: "auth_id".into(), + value: identity.id.clone(), + }); + if let Some(user_id) = identity.user_id { + fields.push(crate::log::Field { + name: "user_id".into(), + value: user_id.to_string(), + }); + } + if let Some(tenant_id) = identity.tenant_id { + fields.push(crate::log::Field { + name: "tenant_id".into(), + value: tenant_id.to_string(), + }); + } + } + fields + }) + .unwrap_or_default() +} + +fn validate_identity(claims: Option<&Claims>, identity: Option<&Identity>) -> Result<(), Error> { + match (claims, identity) { + (None, None) => return Ok(()), + (Some(_), None) => return Err(Error::Unauthorized), + (None, Some(_)) => return Err(Error::Internal), + (Some(claims), Some(identity)) => { + if identity.id != claims.subject + || claims.tenant.is_some() != identity.tenant_id.is_some() + || identity.user_id.is_some_and(|id| id <= 0) + || identity.tenant_id.is_some_and(|id| id <= 0) + { + return Err(Error::Unauthorized); + } + } + } + validate_identity_text("identity", &identity.expect("checked identity").id) +} + +fn validate_request_host(request: &Request, allowed: &[String]) -> Result<(), Error> { + let host = unique_header(&request.headers, "host")?.ok_or(Error::Unauthorized)?; + let host = std::str::from_utf8(host).map_err(|_| Error::Unauthorized)?; + let host = crate::config::normalize_authority_host(host).map_err(|_| Error::Unauthorized)?; + if !allowed.is_empty() && !allowed.iter().any(|allowed| allowed == &host) { + return Err(Error::Unauthorized); + } + Ok(()) +} + +fn bearer_token(request: &Request) -> Result<Option<Vec<u8>>, Error> { + let Some(value) = unique_header(&request.headers, "authorization")? else { + return Ok(None); + }; + if !value.is_ascii() || value.len() > MAX_TOKEN_BYTES + 7 { + return Err(Error::Unauthorized); + } + let Some(separator) = value.iter().position(|byte| *byte == b' ') else { + return Err(Error::Unauthorized); + }; + let (scheme, token) = (&value[..separator], &value[separator + 1..]); + if !scheme.eq_ignore_ascii_case(b"bearer") + || token.is_empty() + || token.len() > MAX_TOKEN_BYTES + || token.iter().any(|byte| byte.is_ascii_whitespace()) + { + return Err(Error::Unauthorized); + } + Ok(Some(token.to_vec())) +} + +fn cookie_token(request: &Request, name: &str) -> Result<Option<Vec<u8>>, Error> { + let mut selected = None; + for header in request + .headers + .iter() + .filter(|header| header.name.eq_ignore_ascii_case("cookie")) + { + let raw = header.value.values(); + if !raw.is_ascii() { + return Err(Error::Unauthorized); + } + for pair in raw.split(|byte| *byte == b';') { + let pair = pair.trim_ascii(); + let Some(separator) = pair.iter().position(|byte| *byte == b'=') else { + return Err(Error::Unauthorized); + }; + if &pair[..separator] != name.as_bytes() { + continue; + } + if selected.is_some() { + return Err(Error::Unauthorized); + } + let encoded = &pair[separator + 1..]; + let encoded = if encoded.len() >= 2 + && encoded.first() == Some(&b'"') + && encoded.last() == Some(&b'"') + { + &encoded[1..encoded.len() - 1] + } else { + encoded + }; + let encoded = encoded.strip_prefix(b"v1.").ok_or(Error::Unauthorized)?; + if encoded.len() > MAX_TOKEN_BYTES.saturating_mul(2) { + return Err(Error::Unauthorized); + } + let token = base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(encoded) + .map_err(|_| Error::Unauthorized)?; + if token.is_empty() || token.len() > MAX_TOKEN_BYTES { + return Err(Error::Unauthorized); + } + selected = Some(token); + } + } + Ok(selected) +} + +fn validate_cookie_origin( + request: &Request, + origin: Option<&crate::config::HttpOrigin>, +) -> Result<(), Error> { + if matches!(request.method.as_str(), "GET" | "HEAD" | "OPTIONS") { + return Ok(()); + } + let origin = origin.ok_or(Error::Internal)?; + crate::api::validate_same_origin(&request.headers, origin).map_err(|_| Error::Forbidden) +} + +fn unique_header<'a>(headers: &'a [Header], name: &str) -> Result<Option<&'a [u8]>, Error> { + let mut values = headers + .iter() + .filter(|header| header.name.eq_ignore_ascii_case(name)); + let value = values.next().map(|header| header.value.values()); + if values.next().is_some() { + return Err(Error::Unauthorized); + } + Ok(value) +} + +fn verify_jwt( + token: &[u8], + site: &str, + provider_key: &str, + provider: &AuthProvider, +) -> Result<Claims, Error> { + let text = std::str::from_utf8(token).map_err(|_| Error::Unauthorized)?; + let mut parts = text.split('.'); + let header_part = parts.next().ok_or(Error::Unauthorized)?; + let claims_part = parts.next().ok_or(Error::Unauthorized)?; + let signature_part = parts.next().ok_or(Error::Unauthorized)?; + if parts.next().is_some() + || [header_part, claims_part, signature_part] + .iter() + .any(|part| part.is_empty()) + { + return Err(Error::Unauthorized); + } + let header = decode_part(header_part)?; + let claims = decode_part(claims_part)?; + let signature = decode_part(signature_part)?; + let header_text = std::str::from_utf8(&header).map_err(|_| Error::Unauthorized)?; + let claims_text = std::str::from_utf8(&claims).map_err(|_| Error::Unauthorized)?; + crate::wire::parse(header_text).map_err(|_| Error::Unauthorized)?; + crate::wire::parse(claims_text).map_err(|_| Error::Unauthorized)?; + let header: JwtHeader = serde_json::from_slice(&header).map_err(|_| Error::Unauthorized)?; + if header.alg != "HS256" || header.typ != "JWT" { + return Err(Error::Unauthorized); + } + let mut mac = + Hmac::<Sha256>::new_from_slice(&provider.jwt_secret().0).map_err(|_| Error::Internal)?; + mac.update(format!("{header_part}.{claims_part}").as_bytes()); + mac.verify_slice(&signature) + .map_err(|_| Error::Unauthorized)?; + + let claims: JwtClaims = serde_json::from_slice(&claims).map_err(|_| Error::Unauthorized)?; + let now = unix_seconds()?; + if claims.iss != provider_key + || claims.aud != site + || claims.iat <= 0 + || claims.exp <= now - CLOCK_SKEW_SECONDS + || claims.iat > now + CLOCK_SKEW_SECONDS + || claims.nbf.is_some_and(|nbf| nbf > now + CLOCK_SKEW_SECONDS) + || claims.exp <= claims.iat + || claims.exp - claims.iat > provider.ttl_seconds() as i64 + CLOCK_SKEW_SECONDS + { + return Err(Error::Unauthorized); + } + validate_identity_text("subject", &claims.sub)?; + validate_identity_text("session", &claims.sid)?; + if let Some(tenant) = &claims.tenant { + validate_identity_text("tenant", tenant)?; + } + Ok(Claims { + subject: claims.sub, + session: claims.sid, + tenant: claims.tenant, + site: claims.aud, + }) +} + +fn encode_jwt(claims: &JwtClaims, provider: &AuthProvider) -> Result<Vec<u8>, Error> { + let engine = base64::engine::general_purpose::URL_SAFE_NO_PAD; + let header = serde_json::to_vec(&IssuedHeader { + alg: "HS256", + typ: "JWT", + }) + .map_err(|_| Error::Internal)?; + let claims = serde_json::to_vec(claims).map_err(|_| Error::Internal)?; + let signed = format!("{}.{}", engine.encode(header), engine.encode(claims)); + let mut mac = + Hmac::<Sha256>::new_from_slice(&provider.jwt_secret().0).map_err(|_| Error::Internal)?; + mac.update(signed.as_bytes()); + let signature = mac.finalize().into_bytes(); + Ok(format!("{signed}.{}", engine.encode(signature)).into_bytes()) +} + +fn decode_part(part: &str) -> Result<Vec<u8>, Error> { + if part.len() > MAX_JWT_PART_BYTES { + return Err(Error::Unauthorized); + } + base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(part) + .map_err(|_| Error::Unauthorized) +} + +fn unix_seconds() -> Result<i64, Error> { + let seconds = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|_| Error::Internal)? + .as_secs(); + i64::try_from(seconds).map_err(|_| Error::Internal) +} + +fn validate_identity_text(_field: &str, value: &str) -> Result<(), Error> { + if value.is_empty() + || value.len() > MAX_IDENTITY_TEXT_BYTES + || value.chars().any(char::is_control) + { + Err(Error::Unauthorized) + } else { + Ok(()) + } +} + +fn with_invocation<T>(read: impl FnOnce(&Invocation) -> Result<T, Error>) -> Result<T, Error> { + INVOCATION.try_with(read).map_err(|_| Error::Unauthorized)? +} + +fn with_identity<T>(read: impl FnOnce(&Identity) -> Result<T, Error>) -> Result<T, Error> { + with_invocation(|invocation| { + invocation + .identity + .as_ref() + .ok_or(Error::Unauthorized) + .and_then(read) + }) +} diff --git a/crates/dever-runtime/src/auth/store.rs b/crates/dever-runtime/src/auth/store.rs new file mode 100644 index 0000000..844d966 --- /dev/null +++ b/crates/dever-runtime/src/auth/store.rs @@ -0,0 +1,646 @@ +use std::collections::BTreeSet; + +use crate::database::{Database, Executor, Sql, Transaction}; +use crate::orm::{self, Error, Row, Value}; + +const OWNER_ROLE_PREFIX: &str = "_dever_owner:"; +const ROLE_SCHEMA_VERSION: i64 = 2; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct Permission { + pub key: &'static str, + pub component: &'static str, + pub domain: &'static str, + pub site: &'static str, + pub action: &'static str, + pub method: &'static str, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CatalogPermission { + pub key: String, + pub component: String, + pub domain: String, + pub site: String, + pub action: String, + pub method: String, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Role { + pub id: String, + pub site: String, + pub name: String, + pub all_permissions: bool, +} + +pub async fn initialize_catalog(database: Database) -> Result<(), Error> { + require_control_database(&database)?; + let transaction = database.begin().await?; + let executor = Executor::new(database, Some(&transaction))?; + let result = initialize_catalog_in(&executor).await; + finish_transaction(transaction, result).await +} + +pub async fn sync_catalog(database: Database, permissions: &[Permission]) -> Result<(), Error> { + require_control_database(&database)?; + validate_catalog(permissions)?; + let transaction = database.begin().await?; + let executor = Executor::new(database, Some(&transaction))?; + let result = sync_catalog_in(&executor, permissions).await; + finish_transaction(transaction, result).await +} + +pub async fn list_permissions(database: Database) -> Result<Vec<CatalogPermission>, Error> { + require_control_database(&database)?; + let rows = database.query(Sql { + sqlite: "SELECT key,component,domain,site,action,method FROM _dever_permission WHERE active=1 ORDER BY component,domain,site,action", + postgres: "SELECT key,component,domain,site,action,method FROM _dever_permission WHERE active=1 ORDER BY component,domain,site,action", + }, vec![]).await?; + rows.into_iter().map(catalog_permission).collect() +} + +pub async fn initialize_roles(database: Database) -> Result<(), Error> { + let transaction = database.begin().await?; + let executor = Executor::new(database, Some(&transaction))?; + let result = initialize_roles_in(&executor).await; + finish_transaction(transaction, result).await +} + +pub async fn authorize( + database: Database, + user_id: i64, + site: &str, + permission_key: &str, +) -> Result<bool, Error> { + validate_user_id(user_id)?; + validate_name("site", site)?; + validate_permission_key(permission_key, Some(site))?; + let rows = database.query(Sql { + sqlite: "SELECT 1 FROM _dever_auth_user_role AS ur JOIN _dever_auth_role AS r ON r.site=ur.site AND r.id=ur.role_id LEFT JOIN _dever_auth_role_permission AS rp ON rp.site=r.site AND rp.role_id=r.id AND rp.permission_key=?3 WHERE ur.user_id=?1 AND ur.site=?2 AND r.enabled=1 AND (r.all_permissions=1 OR rp.permission_key=?3) LIMIT 1", + postgres: "SELECT 1 FROM _dever_auth_user_role AS ur JOIN _dever_auth_role AS r ON r.site=ur.site AND r.id=ur.role_id LEFT JOIN _dever_auth_role_permission AS rp ON rp.site=r.site AND rp.role_id=r.id AND rp.permission_key=$3 WHERE ur.user_id=$1 AND ur.site=$2 AND r.enabled=1 AND (r.all_permissions=1 OR rp.permission_key=$3) LIMIT 1", + }, vec![number(user_id), text(site), text(permission_key)]).await?; + Ok(!rows.is_empty()) +} + +pub async fn save_role( + catalog_database: Database, + authorization_database: Database, + role: &Role, + permission_keys: &[String], +) -> Result<(), Error> { + require_control_database(&catalog_database)?; + validate_role(role)?; + let permissions = validate_role_permissions(&catalog_database, role, permission_keys).await?; + let transaction = authorization_database.begin().await?; + let executor = Executor::new(authorization_database, Some(&transaction))?; + let result = save_role_in(&executor, role, &permissions).await; + finish_transaction(transaction, result).await +} + +pub async fn grant_role( + database: Database, + user_id: i64, + site: &str, + role_id: &str, +) -> Result<(), Error> { + validate_user_id(user_id)?; + validate_name("site", site)?; + validate_mutable_role_id(role_id)?; + let transaction = database.begin().await?; + let executor = Executor::new(database, Some(&transaction))?; + let result = grant_role_in(&executor, user_id, site, role_id).await; + finish_transaction(transaction, result).await +} + +pub async fn revoke_role( + database: Database, + user_id: i64, + site: &str, + role_id: &str, +) -> Result<(), Error> { + validate_user_id(user_id)?; + validate_name("site", site)?; + validate_mutable_role_id(role_id)?; + database.execute(Sql { + sqlite: "DELETE FROM _dever_auth_user_role WHERE user_id=?1 AND site=?2 AND role_id=?3", + postgres: "DELETE FROM _dever_auth_user_role WHERE user_id=$1 AND site=$2 AND role_id=$3", + }, vec![number(user_id), text(site), text(role_id)]).await?; + Ok(()) +} + +pub async fn disable_role(database: Database, site: &str, role_id: &str) -> Result<(), Error> { + validate_name("site", site)?; + validate_mutable_role_id(role_id)?; + let changed = database + .execute( + Sql { + sqlite: "UPDATE _dever_auth_role SET enabled=0 WHERE id=?1 AND site=?2", + postgres: "UPDATE _dever_auth_role SET enabled=0 WHERE id=$1 AND site=$2", + }, + vec![text(role_id), text(site)], + ) + .await?; + if changed == 0 { + return Err(Error::invalid_data("role does not exist")); + } + Ok(()) +} + +pub async fn provision_owner( + database: Database, + user_id: i64, + site: &str, +) -> Result<String, Error> { + validate_user_id(user_id)?; + validate_name("site", site)?; + let role_id = format!("{OWNER_ROLE_PREFIX}{site}"); + let transaction = database.begin().await?; + let executor = Executor::new(database, Some(&transaction))?; + let result = async { + executor.execute(Sql { + sqlite: "INSERT INTO _dever_auth_role(id,site,name,all_permissions,enabled) VALUES (?1,?2,'Owner',1,1) ON CONFLICT(site,id) DO UPDATE SET name=excluded.name,all_permissions=1,enabled=1", + postgres: "INSERT INTO _dever_auth_role(id,site,name,all_permissions,enabled) VALUES ($1,$2,'Owner',1,1) ON CONFLICT(site,id) DO UPDATE SET name=excluded.name,all_permissions=1,enabled=1", + }, vec![text(&role_id), text(site)]).await?; + grant_role_in(&executor, user_id, site, &role_id).await + }.await; + finish_transaction(transaction, result).await?; + Ok(role_id) +} + +async fn initialize_catalog_in(executor: &Executor<'_>) -> Result<(), Error> { + lock_private_schema(executor).await?; + executor.execute(Sql { + sqlite: "CREATE TABLE IF NOT EXISTS _dever_permission (key TEXT PRIMARY KEY, component TEXT NOT NULL, domain TEXT NOT NULL, site TEXT NOT NULL, action TEXT NOT NULL, method TEXT NOT NULL, active BIGINT NOT NULL CHECK(active IN (0,1)), updated_at BIGINT NOT NULL)", + postgres: "CREATE TABLE IF NOT EXISTS _dever_permission (key TEXT PRIMARY KEY, component TEXT NOT NULL, domain TEXT NOT NULL, site TEXT NOT NULL, action TEXT NOT NULL, method TEXT NOT NULL, active BIGINT NOT NULL CHECK(active IN (0,1)), updated_at BIGINT NOT NULL)", + }, vec![]).await?; + executor.execute(Sql { + sqlite: "CREATE INDEX IF NOT EXISTS _dever_permission_catalog ON _dever_permission(active,component,domain,site,action)", + postgres: "CREATE INDEX IF NOT EXISTS _dever_permission_catalog ON _dever_permission(active,component,domain,site,action)", + }, vec![]).await?; + Ok(()) +} + +async fn sync_catalog_in(executor: &Executor<'_>, permissions: &[Permission]) -> Result<(), Error> { + initialize_catalog_in(executor).await?; + let updated_at = crate::time::now().map_err(Error::migration)?; + executor + .execute( + Sql { + sqlite: "UPDATE _dever_permission SET active=0,updated_at=?1 WHERE active=1", + postgres: "UPDATE _dever_permission SET active=0,updated_at=$1 WHERE active=1", + }, + vec![number(updated_at)], + ) + .await?; + for permission in permissions { + executor.execute(Sql { + sqlite: "INSERT INTO _dever_permission(key,component,domain,site,action,method,active,updated_at) VALUES (?1,?2,?3,?4,?5,?6,1,?7) ON CONFLICT(key) DO UPDATE SET component=excluded.component,domain=excluded.domain,site=excluded.site,action=excluded.action,method=excluded.method,active=1,updated_at=excluded.updated_at", + postgres: "INSERT INTO _dever_permission(key,component,domain,site,action,method,active,updated_at) VALUES ($1,$2,$3,$4,$5,$6,1,$7) ON CONFLICT(key) DO UPDATE SET component=excluded.component,domain=excluded.domain,site=excluded.site,action=excluded.action,method=excluded.method,active=1,updated_at=excluded.updated_at", + }, vec![ + text(permission.key), + text(permission.component), + text(permission.domain), + text(permission.site), + text(permission.action), + text(permission.method), + number(updated_at), + ]).await?; + } + Ok(()) +} + +async fn initialize_roles_in(executor: &Executor<'_>) -> Result<(), Error> { + lock_private_schema(executor).await?; + executor.execute(Sql { + sqlite: "CREATE TABLE IF NOT EXISTS _dever_auth_version (singleton INTEGER PRIMARY KEY CHECK(singleton=1), version BIGINT NOT NULL)", + postgres: "CREATE TABLE IF NOT EXISTS _dever_auth_version (singleton BIGINT PRIMARY KEY CHECK(singleton=1), version BIGINT NOT NULL)", + }, vec![]).await?; + let versions = executor + .query( + Sql { + sqlite: "SELECT version FROM _dever_auth_version", + postgres: "SELECT version FROM _dever_auth_version", + }, + vec![], + ) + .await?; + if let Some(version) = versions.first() { + if versions.len() != 1 || integer(version, 0)? != ROLE_SCHEMA_VERSION { + return Err(Error::migration( + "unsupported private authorization schema version", + )); + } + validate_role_schema_v2(executor).await?; + create_role_indexes(executor).await?; + return Ok(()); + } + + let role = private_table_exists(executor, "_dever_auth_role").await?; + let permission = private_table_exists(executor, "_dever_auth_role_permission").await?; + let user = private_table_exists(executor, "_dever_auth_user_role").await?; + match (role, permission, user) { + (false, false, false) => create_role_schema_v2(executor).await?, + (true, true, true) => { + let permission_site = + private_column_exists(executor, "_dever_auth_role_permission", "site").await?; + let user_site = + private_column_exists(executor, "_dever_auth_user_role", "site").await?; + match (permission_site, user_site) { + (false, false) => migrate_role_schema_v1(executor).await?, + (true, true) => { + validate_role_schema_v2(executor).await?; + create_role_indexes(executor).await?; + } + _ => { + return Err(Error::migration( + "inconsistent private authorization schema", + )); + } + } + } + _ => return Err(Error::migration("incomplete private authorization schema")), + } + executor + .execute( + Sql { + sqlite: "INSERT INTO _dever_auth_version(singleton,version) VALUES (1,?1)", + postgres: "INSERT INTO _dever_auth_version(singleton,version) VALUES (1,$1)", + }, + vec![number(ROLE_SCHEMA_VERSION)], + ) + .await?; + Ok(()) +} + +async fn create_role_schema_v2(executor: &Executor<'_>) -> Result<(), Error> { + for sql in [ + "CREATE TABLE IF NOT EXISTS _dever_auth_role (id TEXT NOT NULL, site TEXT NOT NULL, name TEXT NOT NULL, all_permissions BIGINT NOT NULL CHECK(all_permissions IN (0,1)), enabled BIGINT NOT NULL CHECK(enabled IN (0,1)), PRIMARY KEY(site,id))", + "CREATE TABLE IF NOT EXISTS _dever_auth_role_permission (site TEXT NOT NULL, role_id TEXT NOT NULL, permission_key TEXT NOT NULL, PRIMARY KEY(site,role_id,permission_key), FOREIGN KEY(site,role_id) REFERENCES _dever_auth_role(site,id) ON DELETE CASCADE)", + "CREATE TABLE IF NOT EXISTS _dever_auth_user_role (user_id BIGINT NOT NULL, site TEXT NOT NULL, role_id TEXT NOT NULL, PRIMARY KEY(user_id,site,role_id), FOREIGN KEY(site,role_id) REFERENCES _dever_auth_role(site,id) ON DELETE CASCADE)", + ] { + executor + .execute( + Sql { + sqlite: sql, + postgres: sql, + }, + vec![], + ) + .await?; + } + create_role_indexes(executor).await +} + +async fn create_role_indexes(executor: &Executor<'_>) -> Result<(), Error> { + for sql in [ + "CREATE INDEX IF NOT EXISTS _dever_auth_role_site ON _dever_auth_role(site,enabled)", + "CREATE INDEX IF NOT EXISTS _dever_auth_user_role_user ON _dever_auth_user_role(user_id,site,role_id)", + ] { + executor + .execute( + Sql { + sqlite: sql, + postgres: sql, + }, + vec![], + ) + .await?; + } + Ok(()) +} + +async fn migrate_role_schema_v1(executor: &Executor<'_>) -> Result<(), Error> { + for sql in [ + "CREATE TABLE _dever_auth_role_v2 (id TEXT NOT NULL, site TEXT NOT NULL, name TEXT NOT NULL, all_permissions BIGINT NOT NULL CHECK(all_permissions IN (0,1)), enabled BIGINT NOT NULL CHECK(enabled IN (0,1)), PRIMARY KEY(site,id))", + "CREATE TABLE _dever_auth_role_permission_v2 (site TEXT NOT NULL, role_id TEXT NOT NULL, permission_key TEXT NOT NULL, PRIMARY KEY(site,role_id,permission_key), FOREIGN KEY(site,role_id) REFERENCES _dever_auth_role_v2(site,id) ON DELETE CASCADE)", + "CREATE TABLE _dever_auth_user_role_v2 (user_id BIGINT NOT NULL, site TEXT NOT NULL, role_id TEXT NOT NULL, PRIMARY KEY(user_id,site,role_id), FOREIGN KEY(site,role_id) REFERENCES _dever_auth_role_v2(site,id) ON DELETE CASCADE)", + "INSERT INTO _dever_auth_role_v2(id,site,name,all_permissions,enabled) SELECT id,site,name,all_permissions,enabled FROM _dever_auth_role", + "INSERT INTO _dever_auth_role_permission_v2(site,role_id,permission_key) SELECT role.site,permission.role_id,permission.permission_key FROM _dever_auth_role_permission AS permission JOIN _dever_auth_role AS role ON role.id=permission.role_id", + "INSERT INTO _dever_auth_user_role_v2(user_id,site,role_id) SELECT assignment.user_id,role.site,assignment.role_id FROM _dever_auth_user_role AS assignment JOIN _dever_auth_role AS role ON role.id=assignment.role_id", + "DROP INDEX IF EXISTS _dever_auth_role_site", + "DROP INDEX IF EXISTS _dever_auth_user_role_user", + "DROP TABLE _dever_auth_role_permission", + "DROP TABLE _dever_auth_user_role", + "DROP TABLE _dever_auth_role", + "ALTER TABLE _dever_auth_role_v2 RENAME TO _dever_auth_role", + "ALTER TABLE _dever_auth_role_permission_v2 RENAME TO _dever_auth_role_permission", + "ALTER TABLE _dever_auth_user_role_v2 RENAME TO _dever_auth_user_role", + ] { + executor + .execute( + Sql { + sqlite: sql, + postgres: sql, + }, + vec![], + ) + .await?; + } + create_role_indexes(executor).await +} + +async fn validate_role_schema_v2(executor: &Executor<'_>) -> Result<(), Error> { + for table in [ + "_dever_auth_role", + "_dever_auth_role_permission", + "_dever_auth_user_role", + ] { + if !private_table_exists(executor, table).await? { + return Err(Error::migration("incomplete private authorization schema")); + } + } + for table in ["_dever_auth_role_permission", "_dever_auth_user_role"] { + if !private_column_exists(executor, table, "site").await? { + return Err(Error::migration( + "invalid private authorization schema version", + )); + } + } + Ok(()) +} + +async fn private_table_exists(executor: &Executor<'_>, table: &str) -> Result<bool, Error> { + let rows = executor.query(Sql { + sqlite: "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?1", + postgres: "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=current_schema() AND table_name=$1", + }, vec![text(table)]).await?; + Ok(integer(&orm::required_row(rows, "authorization table lookup")?, 0)? == 1) +} + +async fn private_column_exists( + executor: &Executor<'_>, + table: &str, + column: &str, +) -> Result<bool, Error> { + let rows = executor.query(Sql { + sqlite: "SELECT COUNT(*) FROM pragma_table_info(?1) WHERE name=?2", + postgres: "SELECT COUNT(*) FROM information_schema.columns WHERE table_schema=current_schema() AND table_name=$1 AND column_name=$2", + }, vec![text(table), text(column)]).await?; + Ok(integer(&orm::required_row(rows, "authorization column lookup")?, 0)? == 1) +} + +async fn save_role_in( + executor: &Executor<'_>, + role: &Role, + permission_keys: &BTreeSet<String>, +) -> Result<(), Error> { + executor.execute(Sql { + sqlite: "INSERT INTO _dever_auth_role(id,site,name,all_permissions,enabled) VALUES (?1,?2,?3,?4,1) ON CONFLICT(site,id) DO UPDATE SET name=excluded.name,all_permissions=excluded.all_permissions", + postgres: "INSERT INTO _dever_auth_role(id,site,name,all_permissions,enabled) VALUES ($1,$2,$3,$4,1) ON CONFLICT(site,id) DO UPDATE SET name=excluded.name,all_permissions=excluded.all_permissions", + }, vec![ + text(&role.id), + text(&role.site), + text(&role.name), + number(i64::from(role.all_permissions)), + ]).await?; + executor + .execute( + Sql { + sqlite: "DELETE FROM _dever_auth_role_permission WHERE site=?1 AND role_id=?2", + postgres: "DELETE FROM _dever_auth_role_permission WHERE site=$1 AND role_id=$2", + }, + vec![text(&role.site), text(&role.id)], + ) + .await?; + for permission_key in permission_keys { + executor.execute(Sql { + sqlite: "INSERT INTO _dever_auth_role_permission(site,role_id,permission_key) VALUES (?1,?2,?3)", + postgres: "INSERT INTO _dever_auth_role_permission(site,role_id,permission_key) VALUES ($1,$2,$3)", + }, vec![text(&role.site), text(&role.id), text(permission_key)]).await?; + } + Ok(()) +} + +async fn grant_role_in( + executor: &Executor<'_>, + user_id: i64, + site: &str, + role_id: &str, +) -> Result<(), Error> { + let rows = executor + .query( + Sql { + sqlite: "SELECT enabled FROM _dever_auth_role WHERE id=?1 AND site=?2", + postgres: "SELECT enabled FROM _dever_auth_role WHERE id=$1 AND site=$2 FOR SHARE", + }, + vec![text(role_id), text(site)], + ) + .await?; + let role = orm::required_row(rows, "grant role")?; + if integer(&role, 0)? != 1 { + return Err(Error::invalid_data("disabled role cannot be granted")); + } + executor.execute(Sql { + sqlite: "INSERT INTO _dever_auth_user_role(user_id,site,role_id) VALUES (?1,?2,?3) ON CONFLICT DO NOTHING", + postgres: "INSERT INTO _dever_auth_user_role(user_id,site,role_id) VALUES ($1,$2,$3) ON CONFLICT DO NOTHING", + }, vec![number(user_id), text(site), text(role_id)]).await?; + Ok(()) +} + +async fn validate_role_permissions( + catalog_database: &Database, + role: &Role, + permission_keys: &[String], +) -> Result<BTreeSet<String>, Error> { + if role.all_permissions && !permission_keys.is_empty() { + return Err(Error::invalid_data( + "all-permissions role cannot store individual permissions", + )); + } + let requested = permission_keys.iter().cloned().collect::<BTreeSet<_>>(); + if requested.len() != permission_keys.len() { + return Err(Error::invalid_data("role permissions contain duplicates")); + } + for permission_key in &requested { + validate_permission_key(permission_key, Some(&role.site))?; + } + if requested.is_empty() { + return Ok(requested); + } + let available = catalog_database + .query( + Sql { + sqlite: "SELECT key FROM _dever_permission WHERE site=?1 AND active=1", + postgres: "SELECT key FROM _dever_permission WHERE site=$1 AND active=1", + }, + vec![text(&role.site)], + ) + .await? + .into_iter() + .map(|row| row_text(&row, 0)) + .collect::<Result<BTreeSet<_>, _>>()?; + if let Some(unknown) = requested.iter().find(|key| !available.contains(*key)) { + return Err(Error::invalid_data(format!( + "permission '{unknown}' is not active for site '{}'", + role.site + ))); + } + Ok(requested) +} + +fn validate_catalog(permissions: &[Permission]) -> Result<(), Error> { + let mut keys = BTreeSet::new(); + for permission in permissions { + validate_name("component", permission.component)?; + validate_name("domain", permission.domain)?; + validate_name("site", permission.site)?; + validate_name("action", permission.action)?; + if !matches!(permission.method, "GET" | "POST" | "PUT" | "DELETE") { + return Err(Error::invalid_data( + "permission method must be GET, POST, PUT, or DELETE", + )); + } + let expected = format!( + "{}.{}.{}.{}", + permission.component, permission.domain, permission.site, permission.action + ); + if permission.key != expected { + return Err(Error::invalid_data(format!( + "permission key '{}' must be '{expected}'", + permission.key + ))); + } + if !keys.insert(permission.key) { + return Err(Error::invalid_data(format!( + "duplicate permission key '{}'", + permission.key + ))); + } + } + Ok(()) +} + +fn validate_role(role: &Role) -> Result<(), Error> { + validate_mutable_role_id(&role.id)?; + validate_name("site", &role.site)?; + validate_text("role name", &role.name, 128) +} + +fn validate_mutable_role_id(role_id: &str) -> Result<(), Error> { + validate_text("role id", role_id, 128)?; + if role_id.starts_with(OWNER_ROLE_PREFIX) { + return Err(Error::invalid_data("reserved role id")); + } + Ok(()) +} + +fn validate_permission_key(key: &str, expected_site: Option<&str>) -> Result<(), Error> { + if key.len() > 512 { + return Err(Error::invalid_data("permission key exceeds 512 bytes")); + } + let parts = key.split('.').collect::<Vec<_>>(); + if parts.len() != 4 || parts.iter().any(|part| !valid_name(part)) { + return Err(Error::invalid_data( + "permission key must contain component.domain.site.action", + )); + } + if expected_site.is_some_and(|site| parts[2] != site) { + return Err(Error::invalid_data( + "permission key belongs to another site", + )); + } + Ok(()) +} + +fn validate_name(label: &str, value: &str) -> Result<(), Error> { + if !valid_name(value) { + return Err(Error::invalid_data(format!( + "{label} must use lower_snake_case" + ))); + } + Ok(()) +} + +fn valid_name(value: &str) -> bool { + !value.is_empty() + && value.len() <= 128 + && value.starts_with(|character: char| character.is_ascii_lowercase()) + && value.split('_').all(|word| { + !word.is_empty() + && word + .chars() + .all(|character| character.is_ascii_lowercase() || character.is_ascii_digit()) + }) +} + +fn validate_text(label: &str, value: &str, maximum: usize) -> Result<(), Error> { + if value.is_empty() || value.len() > maximum || value.chars().any(char::is_control) { + return Err(Error::invalid_data(format!( + "{label} must contain 1..={maximum} non-control bytes" + ))); + } + Ok(()) +} + +fn validate_user_id(user_id: i64) -> Result<(), Error> { + if user_id <= 0 { + return Err(Error::invalid_data("user id must be positive")); + } + Ok(()) +} + +fn require_control_database(database: &Database) -> Result<(), Error> { + if database.key().tenant_id().is_some() { + return Err(Error::invalid_data( + "permission catalog requires the global control database", + )); + } + Ok(()) +} + +async fn lock_private_schema(executor: &Executor<'_>) -> Result<(), Error> { + executor + .query( + Sql { + sqlite: "SELECT 1", + postgres: "SELECT pg_advisory_xact_lock(1684371045, 1635087476) IS NULL", + }, + vec![], + ) + .await?; + Ok(()) +} + +fn catalog_permission(row: Row) -> Result<CatalogPermission, Error> { + Ok(CatalogPermission { + key: row_text(&row, 0)?, + component: row_text(&row, 1)?, + domain: row_text(&row, 2)?, + site: row_text(&row, 3)?, + action: row_text(&row, 4)?, + method: row_text(&row, 5)?, + }) +} + +fn row_text(row: &Row, index: usize) -> Result<String, Error> { + orm::text(row.get(index)?.clone()) +} + +fn integer(row: &Row, index: usize) -> Result<i64, Error> { + orm::int(row.get(index)?.clone()) +} + +fn text(value: impl Into<String>) -> Value { + Value::Text(value.into()) +} + +fn number(value: i64) -> Value { + Value::Int(value) +} + +async fn finish_transaction<T>( + transaction: Transaction, + result: Result<T, Error>, +) -> Result<T, Error> { + match result { + Ok(value) => { + transaction.commit().await?; + Ok(value) + } + Err(error) => match transaction.rollback().await { + Ok(()) => Err(error), + Err(rollback) => Err(error.cause(rollback)), + }, + } +} diff --git a/crates/dever-runtime/src/bytes.rs b/crates/dever-runtime/src/bytes.rs new file mode 100644 index 0000000..e768960 --- /dev/null +++ b/crates/dever-runtime/src/bytes.rs @@ -0,0 +1,80 @@ +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Bytes(bytes::Bytes); + +impl Bytes { + pub fn new(bytes: Vec<u8>) -> Self { + Self(bytes.into()) + } + pub fn values(&self) -> &[u8] { + &self.0 + } + pub fn into_values(self) -> BytesIntoValues { + self.0.into_iter() + } + pub(crate) fn from_http(bytes: bytes::Bytes) -> Self { + Self(bytes) + } + pub(crate) fn into_http(self) -> bytes::Bytes { + self.0 + } + pub fn from_text(text: &str) -> Self { + Self::new(text.as_bytes().to_vec()) + } + pub fn from_string(text: String) -> Self { + Self::new(text.into_bytes()) + } + pub fn to_text(&self) -> Result<String, String> { + decode_text(self.0.to_vec()) + } + pub fn into_text(self) -> Result<String, String> { + decode_text(self.0.into()) + } + pub fn length(&self) -> i64 { + self.0.len() as i64 + } + pub fn at(&self, index: i64) -> Option<i64> { + usize::try_from(index) + .ok() + .and_then(|index| self.0.get(index)) + .map(|value| i64::from(*value)) + } + pub fn slice(&self, start: i64, end: i64) -> Result<Self, &'static str> { + let start = usize::try_from(start).map_err(|_| "byte range is out of bounds")?; + let end = usize::try_from(end).map_err(|_| "byte range is out of bounds")?; + if start > end || end > self.0.len() { + return Err("byte range is out of bounds"); + } + Ok(Self(self.0.slice(start..end))) + } + pub fn concat(self, other: &Self) -> Self { + if other.values().is_empty() { + return self; + } + if self.values().is_empty() { + return other.clone(); + } + let mut bytes = match self.0.try_into_mut() { + Ok(bytes) => bytes, + Err(bytes) => { + let mut result = bytes::BytesMut::with_capacity(bytes.len() + other.0.len()); + result.extend_from_slice(&bytes); + result + } + }; + bytes.extend_from_slice(other.values()); + Self(bytes.freeze()) + } + pub fn from_ints(values: &[i64]) -> Result<Self, &'static str> { + values + .iter() + .map(|value| u8::try_from(*value).map_err(|_| "byte value must be between 0 and 255")) + .collect::<Result<Vec<_>, _>>() + .map(Self::new) + } +} + +pub type BytesIntoValues = bytes::buf::IntoIter<bytes::Bytes>; + +fn decode_text(bytes: Vec<u8>) -> Result<String, String> { + String::from_utf8(bytes).map_err(|error| error.to_string()) +} diff --git a/crates/dever-runtime/src/channel.rs b/crates/dever-runtime/src/channel.rs new file mode 100644 index 0000000..fd6be74 --- /dev/null +++ b/crates/dever-runtime/src/channel.rs @@ -0,0 +1,117 @@ +use std::collections::VecDeque; +use std::sync::{Arc, Mutex}; + +use tokio::sync::Semaphore; + +pub const MAX_CHANNEL_CAPACITY: usize = 65_536; + +struct State<T> { + queue: Mutex<Queue<T>>, + available_items: Arc<Semaphore>, + available_slots: Arc<Semaphore>, +} + +struct Queue<T> { + values: VecDeque<T>, + closed: bool, +} + +pub struct Channel<T>(Arc<State<T>>); + +impl<T> std::fmt::Debug for Channel<T> { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("Channel(<resource>)") + } +} + +impl<T> Clone for Channel<T> { + fn clone(&self) -> Self { + Self(Arc::clone(&self.0)) + } +} + +impl<T: Send + 'static> Channel<T> { + pub fn stream(self) -> crate::async_stream::AsyncStream<T> { + crate::async_stream::AsyncStream::new(futures_util::stream::unfold( + self, + |channel| async move { + let value = channel + .receive() + .await + .expect("channel state lock poisoned")?; + Some((crate::resource::Pull::Item(value), channel)) + }, + )) + } +} + +impl<T> Channel<T> { + pub fn new(capacity: i64) -> Result<Self, &'static str> { + let capacity = usize::try_from(capacity) + .ok() + .filter(|capacity| (1..=MAX_CHANNEL_CAPACITY).contains(capacity)) + .ok_or("channel capacity must be between 1 and 65536")?; + Ok(Self(Arc::new(State { + queue: Mutex::new(Queue { + values: VecDeque::new(), + closed: false, + }), + available_items: Arc::new(Semaphore::new(0)), + available_slots: Arc::new(Semaphore::new(capacity)), + }))) + } + + pub async fn send(&self, value: T) -> Result<(), String> { + let slot = Arc::clone(&self.0.available_slots) + .acquire_owned() + .await + .map_err(|_| "channel is closed".to_owned())?; + let mut queue = self + .0 + .queue + .lock() + .map_err(|_| "channel state lock failed".to_owned())?; + if queue.closed { + return Err("channel is closed".to_owned()); + } + queue.values.push_back(value); + slot.forget(); + drop(queue); + self.0.available_items.add_permits(1); + Ok(()) + } + + pub async fn receive(&self) -> Result<Option<T>, String> { + let item = Arc::clone(&self.0.available_items).acquire_owned().await; + let value = self + .0 + .queue + .lock() + .map_err(|_| "channel state lock failed".to_owned())? + .values + .pop_front(); + if let Ok(item) = item { + item.forget(); + if value.is_some() { + self.0.available_slots.add_permits(1); + } + } + Ok(value) + } + + pub async fn close(&self) -> Result<(), String> { + let mut queue = self + .0 + .queue + .lock() + .map_err(|_| "channel state lock failed".to_owned())?; + if queue.closed { + return Ok(()); + } + queue.closed = true; + drop(queue); + self.0.available_slots.close(); + self.0.available_items.close(); + Ok(()) + } +} diff --git a/crates/dever-runtime/src/collections.rs b/crates/dever-runtime/src/collections.rs new file mode 100644 index 0000000..8338615 --- /dev/null +++ b/crates/dever-runtime/src/collections.rs @@ -0,0 +1,175 @@ +use std::hash::Hash; +use std::sync::Arc; + +use indexmap::IndexMap; + +#[derive(Clone, Debug)] +pub struct List<T>(Arc<Vec<T>>); + +impl<T> List<T> { + pub fn new(values: Vec<T>) -> Self { + Self(Arc::new(values)) + } + pub fn values(&self) -> &[T] { + &self.0 + } +} + +impl<T: Clone> List<T> { + pub fn into_first(self) -> Option<T> { + match Arc::try_unwrap(self.0) { + Ok(values) => values.into_iter().next(), + Err(values) => values.first().cloned(), + } + } + + pub fn into_values(self) -> IntoValues<T> { + match Arc::try_unwrap(self.0) { + Ok(values) => IntoValues::Owned(values.into_iter()), + Err(values) => IntoValues::Shared { values, index: 0 }, + } + } + + pub fn append(mut self, value: T) -> Self { + Arc::make_mut(&mut self.0).push(value); + self + } +} + +/// 独占列表转移元素；共享列表按需复制，提前停止不复制剩余元素。 +pub enum IntoValues<T> { + Owned(std::vec::IntoIter<T>), + Shared { values: Arc<Vec<T>>, index: usize }, +} + +impl<T: Clone> Iterator for IntoValues<T> { + type Item = T; + + fn next(&mut self) -> Option<T> { + match self { + Self::Owned(values) => values.next(), + Self::Shared { values, index } => { + let value = values.get(*index)?.clone(); + *index += 1; + Some(value) + } + } + } + + fn size_hint(&self) -> (usize, Option<usize>) { + let remaining = match self { + Self::Owned(values) => values.len(), + Self::Shared { values, index } => values.len() - index, + }; + (remaining, Some(remaining)) + } +} + +impl<T: Clone> ExactSizeIterator for IntoValues<T> {} + +impl<T: PartialEq> PartialEq for List<T> { + fn eq(&self, other: &Self) -> bool { + self.values().len() == other.values().len() + && self + .values() + .iter() + .zip(other.values()) + .all(|(left, right)| left == right) + } +} + +#[derive(Clone, Debug, PartialEq)] +pub struct MapEntry<K, V> { + pub key: K, + pub value: V, +} + +#[derive(Clone, Debug)] +pub struct Map<K, V>(Arc<IndexMap<K, V>>); + +impl<K: Clone + Eq + Hash, V: Clone> Map<K, V> { + pub fn new(entries: Vec<(K, V)>) -> Result<Self, &'static str> { + let mut map = IndexMap::with_capacity(entries.len()); + for (key, value) in entries { + if map.insert(key, value).is_some() { + return Err("duplicate Map key"); + } + } + Ok(Self(Arc::new(map))) + } + + pub fn from_unique(entries: Vec<(K, V)>) -> Self { + let mut map = IndexMap::with_capacity(entries.len()); + for (key, value) in entries { + debug_assert!(map.insert(key, value).is_none()); + } + Self(Arc::new(map)) + } + + pub fn get(&self, key: &K) -> Option<V> { + self.0.get(key).cloned() + } + + pub fn into_get(self, key: &K) -> Option<V> { + match Arc::try_unwrap(self.0) { + Ok(mut entries) => entries.swap_remove(key), + Err(entries) => entries.get(key).cloned(), + } + } + + // 将小型 COW 包装内联，使调用方的循环能一起优化所有权检查与插入。 + #[inline] + pub fn put(mut self, key: K, value: V) -> Self { + Arc::make_mut(&mut self.0).insert(key, value); + self + } + + pub fn remove(mut self, key: &K) -> Self { + Arc::make_mut(&mut self.0).shift_remove(key); + self + } + + pub fn entries(&self) -> List<MapEntry<K, V>> { + cloned_entries(&self.0) + } + + pub fn into_entries(self) -> List<MapEntry<K, V>> { + match Arc::try_unwrap(self.0) { + Ok(entries) => List::new( + entries + .into_iter() + .map(|(key, value)| MapEntry { key, value }) + .collect(), + ), + Err(entries) => cloned_entries(&entries), + } + } +} + +fn cloned_entries<K: Clone, V: Clone>(entries: &IndexMap<K, V>) -> List<MapEntry<K, V>> { + List::new( + entries + .iter() + .map(|(key, value)| MapEntry { + key: key.clone(), + value: value.clone(), + }) + .collect(), + ) +} + +impl<K, V> Map<K, V> { + pub fn pairs(&self) -> impl Iterator<Item = (&K, &V)> { + self.0.iter() + } +} + +impl<K: PartialEq, V: PartialEq> PartialEq for Map<K, V> { + fn eq(&self, other: &Self) -> bool { + self.0.len() == other.0.len() + && self + .pairs() + .zip(other.pairs()) + .all(|((ak, av), (bk, bv))| ak == bk && av == bv) + } +} diff --git a/crates/dever-runtime/src/component.rs b/crates/dever-runtime/src/component.rs new file mode 100644 index 0000000..b190697 --- /dev/null +++ b/crates/dever-runtime/src/component.rs @@ -0,0 +1,831 @@ +//! Supervised process boundary for external Adapter implementations. +use std::collections::{BTreeMap, BTreeSet, VecDeque}; +use std::future::Future; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; +use tokio::process::{Child, ChildStderr, ChildStdin, ChildStdout, Command}; +use tokio::sync::{Mutex, mpsc, oneshot}; + +mod command; + +const VERSION: &str = "dever-component-1"; +const COMMAND_CAPACITY: usize = 64; +const START_LIMIT: usize = 3; +const START_WINDOW: Duration = Duration::from_secs(10); +const SHUTDOWN_GRACE: Duration = Duration::from_secs(2); + +#[derive(Clone)] +pub struct Definition { + pub key: String, + pub ecosystem: String, + pub entry: String, + pub port: String, + pub adapter: String, + pub schema: String, + pub capabilities: Vec<String>, + pub operations: Vec<String>, + pub setting: Option<String>, + pub timeout_ms: u64, +} + +#[derive(Debug)] +pub enum Reply { + Result(String), + Error { identity: String, payload: String }, +} + +struct Managed { + definition: Definition, + state: Mutex<State>, +} + +#[derive(Default)] +struct State { + sender: Option<mpsc::Sender<Request>>, + stop: Option<oneshot::Sender<()>>, + task: Option<crate::task::Task<()>>, + starts: VecDeque<Instant>, + stopping: bool, +} + +struct Request { + operation: String, + payload: String, + response: oneshot::Sender<Result<Reply, String>>, + command_lease: Option<command::RequestLease>, +} + +tokio::task_local! { + static CURRENT: Option<Arc<Session>>; +} + +/// Worker handles belong to one runtime invocation, including partial startup. +pub(crate) struct Session { + scope: crate::task::Scope, + registry: Mutex<Registry>, + shutdown: Mutex<()>, + pub(crate) settings: std::sync::OnceLock<Result<Arc<crate::config::Settings>, String>>, + #[cfg(feature = "external")] + pub(crate) resources: std::sync::OnceLock<crate::external::PreparedBundle>, +} + +#[derive(Default)] +struct Registry { + workers: BTreeMap<String, Arc<Managed>>, + stopping: bool, + closed: bool, +} + +impl Session { + pub(crate) fn new(scope: crate::task::Scope) -> Arc<Self> { + Arc::new(Self { + scope, + registry: Mutex::new(Registry::default()), + shutdown: Mutex::new(()), + settings: std::sync::OnceLock::new(), + #[cfg(feature = "external")] + resources: std::sync::OnceLock::new(), + }) + } +} + +pub(crate) fn current() -> Option<Arc<Session>> { + CURRENT.try_with(Clone::clone).ok().flatten() +} + +fn session() -> Result<Arc<Session>, String> { + current().ok_or_else(|| "external Adapter requires a Dever invocation".into()) +} + +/// One settings snapshot owns both Adapter selection and deployment grants. +pub fn adapter_settings() -> Result<Arc<crate::config::Settings>, String> { + #[cfg(feature = "api")] + if let Some(application) = crate::application::current() { + return Ok(application.settings().clone()); + } + session()? + .settings + .get_or_init(|| crate::config::Settings::load_adapter_settings().map(Arc::new)) + .clone() +} + +pub(crate) fn scope_resources<T>( + session: Option<Arc<Session>>, + future: impl Future<Output = T>, +) -> impl Future<Output = T> { + CURRENT.scope(session, future) +} + +pub(crate) fn with_resources<T>(session: Option<Arc<Session>>, apply: impl FnOnce() -> T) -> T { + CURRENT.sync_scope(session, apply) +} + +pub async fn start(definition: Definition) -> Result<(), String> { + validate_definition(&definition)?; + let session = session()?; + let mut registered = session.registry.lock().await; + if registered.stopping || registered.closed { + return Err("external Adapter invocation is stopped".into()); + } + if registered.workers.contains_key(&definition.key) { + return Err("external Adapter initialized twice".into()); + } + let managed = Arc::new(Managed { + definition, + state: Mutex::new(State::default()), + }); + // Register before awaiting startup so cancellation cannot orphan the owner. + registered + .workers + .insert(managed.definition.key.clone(), managed.clone()); + drop(registered); + managed.ensure_started().await?; + Ok(()) +} + +pub async fn call( + key: &str, + operation: &str, + payload: &crate::wire::Encoded, +) -> Result<Reply, String> { + let session = session()?; + let registered = session.registry.lock().await; + if registered.stopping || registered.closed { + return Err("external Adapter invocation is stopped".into()); + } + let managed = registered + .workers + .get(key) + .cloned() + .ok_or_else(|| "external Adapter is not initialized".to_owned())?; + drop(registered); + if !managed + .definition + .operations + .iter() + .any(|name| name == operation) + { + return Err("external Adapter operation is not declared by its Port".into()); + } + for _ in 0..2 { + let sender = managed.ensure_started().await?; + let (response, result) = oneshot::channel(); + let lease = (managed.definition.ecosystem == "command") + .then(command::call_lease) + .transpose()?; + let request = Request { + operation: operation.to_owned(), + payload: payload.as_str().to_owned(), + response, + command_lease: lease.as_ref().map(|lease| lease.request()), + }; + if sender.send(request).await.is_ok() { + return result + .await + .unwrap_or_else(|_| Err("external Adapter worker stopped".into())); + } + } + Err("external Adapter worker is unavailable".into()) +} + +pub async fn shutdown() -> Result<(), String> { + shutdown_session(false).await +} + +pub(crate) async fn close() -> Result<(), String> { + shutdown_session(true).await +} + +async fn shutdown_session(terminal: bool) -> Result<(), String> { + let session = session()?; + let _shutdown = session.shutdown.lock().await; + let managed = { + let mut registered = session.registry.lock().await; + registered.stopping = true; + registered.closed |= terminal; + registered.workers.values().cloned().collect::<Vec<_>>() + }; + let mut first = None; + for component in managed { + if let Err(error) = component.stop().await { + first = Some(match first { + Some(previous) => format!("{previous}; worker cleanup: {error}"), + None => error, + }); + } + session + .registry + .lock() + .await + .workers + .remove(&component.definition.key); + } + if !terminal { + session.registry.lock().await.stopping = false; + } + first.map_or(Ok(()), Err) +} + +impl Managed { + async fn ensure_started(&self) -> Result<mpsc::Sender<Request>, String> { + let mut state = self.state.lock().await; + if state.stopping { + return Err("external Adapter worker is stopped".into()); + } + if let Some(sender) = &state.sender + && !sender.is_closed() + { + return Ok(sender.clone()); + } + if state.task.is_some() { + let _ = crate::task::wait_resource(&mut state.task).await; + } + state.sender = None; + state.stop = None; + if self.definition.ecosystem != "command" { + let now = Instant::now(); + while state + .starts + .front() + .is_some_and(|started| now.duration_since(*started) >= START_WINDOW) + { + state.starts.pop_front(); + } + if state.starts.len() == START_LIMIT { + return Err("external Adapter worker restart limit exceeded".into()); + } + state.starts.push_back(now); + } + let (sender, receiver) = mpsc::channel(COMMAND_CAPACITY); + let (stop, stopping) = oneshot::channel(); + let (ready, started) = oneshot::channel(); + let definition = self.definition.clone(); + let session = session()?; + let task = crate::task::run_in(&session.scope, async move { + supervise(definition, receiver, stopping, ready).await + })?; + state.stop = Some(stop); + state.task = Some(task); + match started.await { + Ok(Ok(())) => { + state.sender = Some(sender.clone()); + Ok(sender) + } + Ok(Err(error)) => { + let cleanup = crate::task::wait_resource(&mut state.task).await; + state.stop = None; + match cleanup { + Err(cleanup) if cleanup != error => { + Err(format!("{error}; worker cleanup: {cleanup}")) + } + _ => Err(error), + } + } + Err(_) => crate::task::wait_resource(&mut state.task) + .await + .and_then(|_| Err("external Adapter worker stopped during startup".into())), + } + } + + async fn stop(&self) -> Result<(), String> { + let mut state = self.state.lock().await; + state.stopping = true; + state.sender = None; + if let Some(stop) = state.stop.take() { + let _ = stop.send(()); + } + // Keep the task in its owner if an explicit shutdown caller is cancelled. + crate::task::wait_resource(&mut state.task).await + } +} + +fn validate_definition(definition: &Definition) -> Result<(), String> { + if !matches!( + definition.ecosystem.as_str(), + "exec" | "pip" | "npm" | "go" | "command" + ) { + return Err("invalid external Adapter ecosystem".into()); + } + if definition.key.is_empty() + || definition.port.is_empty() + || definition.adapter.is_empty() + || definition.schema.is_empty() + || definition.operations.is_empty() + || definition.timeout_ms == 0 + { + return Err("incomplete external Adapter definition".into()); + } + unique(&definition.capabilities, "capability")?; + if definition + .capabilities + .iter() + .any(|capability| !matches!(capability.as_str(), "network" | "file" | "process" | "gpu")) + { + return Err("external Adapter declares an unknown capability".into()); + } + unique(&definition.operations, "operation")?; + if definition.ecosystem == "command" + && (definition.operations.len() != 1 || definition.setting.is_some()) + { + return Err("command Adapter requires one operation and no setting".into()); + } + Ok(()) +} + +fn unique(values: &[String], label: &str) -> Result<(), String> { + let mut found = BTreeSet::new(); + if values + .iter() + .any(|value| value.is_empty() || !found.insert(value)) + { + return Err(format!( + "external Adapter has an invalid or duplicate {label}" + )); + } + Ok(()) +} + +async fn supervise( + definition: Definition, + receiver: mpsc::Receiver<Request>, + stopping: oneshot::Receiver<()>, + ready: oneshot::Sender<Result<(), String>>, +) -> Result<(), String> { + if definition.ecosystem == "command" { + return command::supervise(definition, receiver, stopping, ready).await; + } + let mut command = worker_command(&definition)?; + command + .arg("--dever-component") + .env_clear() + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .kill_on_drop(true); + let mut child = command + .spawn() + .map_err(|error| format!("cannot start external Adapter worker: {error}"))?; + let Some(stdin) = child.stdin.take() else { + terminate_child(&mut child).await?; + return Err("external Adapter worker has no stdin".into()); + }; + let Some(stdout) = child.stdout.take() else { + terminate_child(&mut child).await?; + return Err("external Adapter worker has no stdout".into()); + }; + let Some(stderr) = child.stderr.take() else { + terminate_child(&mut child).await?; + return Err("external Adapter worker has no stderr".into()); + }; + let protocol = protocol_loop( + definition, &mut child, stdin, stdout, receiver, stopping, ready, + ); + let logs = drain_stderr(stderr); + let (protocol, logs) = tokio::join!(protocol, logs); + protocol.and(logs) +} + +fn worker_command(definition: &Definition) -> Result<Command, String> { + #[cfg(feature = "external")] + { + crate::external::worker_command(definition).map(Command::from) + } + #[cfg(not(feature = "external"))] + { + let _ = definition; + Err("external Worker sandbox runtime feature is not enabled".into()) + } +} + +async fn protocol_loop( + definition: Definition, + child: &mut Child, + mut input: ChildStdin, + mut output: ChildStdout, + mut requests: mpsc::Receiver<Request>, + mut stopping: oneshot::Receiver<()>, + ready: oneshot::Sender<Result<(), String>>, +) -> Result<(), String> { + let protocol = protocol_session( + &definition, + &mut input, + &mut output, + &mut requests, + &mut stopping, + ready, + ) + .await; + drop(input); + drop(output); + match protocol { + Ok(()) => wait_for_exit(child).await, + Err(error) => match terminate_child(child).await { + Ok(()) => Err(error), + Err(cleanup) => Err(format!("{error}; worker cleanup failed: {cleanup}")), + }, + } +} + +async fn protocol_session( + definition: &Definition, + input: &mut ChildStdin, + output: &mut ChildStdout, + requests: &mut mpsc::Receiver<Request>, + stopping: &mut oneshot::Receiver<()>, + ready: oneshot::Sender<Result<(), String>>, +) -> Result<(), String> { + let startup = handshake(definition, input, output).await; + if let Err(error) = startup { + let _ = ready.send(Err(error.clone())); + return Err(error); + } + let _ = ready.send(Ok(())); + let mut next_id = 1_u64; + loop { + let request = tokio::select! { + biased; + _ = &mut *stopping => return shutdown_protocol(input, output).await, + request = requests.recv() => request, + }; + let Some(mut request) = request else { + return shutdown_protocol(input, output).await; + }; + let id = next_id; + next_id = next_id + .checked_add(1) + .ok_or_else(|| "external Adapter request id exhausted".to_owned())?; + if request.response.is_closed() { + continue; + } + let message = call_message(id, &request.operation, &request.payload)?; + tokio::select! { + biased; + _ = &mut *stopping => { + let _ = request.response.send(Err("external Adapter worker stopped".into())); + return Err("external Adapter worker stopped while writing a call".into()); + } + result = write_frame_timeout(input, &message, Duration::from_millis(definition.timeout_ms)) => result?, + } + let timeout = tokio::time::sleep(Duration::from_millis(definition.timeout_ms)); + tokio::pin!(timeout); + let result = tokio::select! { + biased; + _ = &mut *stopping => { + let _ = request.response.send(Err("external Adapter worker stopped".into())); + cancel_request(input, output, id).await?; + return shutdown_protocol(input, output).await; + } + frame = read_frame(output) => frame.and_then(|frame| parse_reply(frame, id)), + _ = request.response.closed() => { + cancel_request(input, output, id).await?; + continue; + } + _ = &mut timeout => { + let _ = request.response.send(Err("external Adapter call timed out".into())); + cancel_request(input, output, id).await?; + continue; + } + }; + match result { + Ok(reply) => { + let _ = request.response.send(Ok(reply)); + } + Err(error) => { + requests.close(); + let _ = request.response.send(Err(error.clone())); + return Err(error); + } + } + } +} + +async fn cancel_request( + input: &mut ChildStdin, + output: &mut ChildStdout, + id: u64, +) -> Result<(), String> { + write_frame_timeout(input, &id_message("cancel", id)?, SHUTDOWN_GRACE).await?; + let _ = parse_reply(read_frame_timeout(output, SHUTDOWN_GRACE).await?, id)?; + Ok(()) +} + +async fn shutdown_protocol(input: &mut ChildStdin, output: &mut ChildStdout) -> Result<(), String> { + write_frame_timeout(input, &kind_message("shutdown")?, SHUTDOWN_GRACE).await?; + let shutdown = read_frame_timeout(output, SHUTDOWN_GRACE).await?; + expect_kind(&shutdown, "shutdown")?; + Ok(()) +} + +async fn wait_for_exit(child: &mut Child) -> Result<(), String> { + match tokio::time::timeout(SHUTDOWN_GRACE, child.wait()).await { + Ok(Ok(status)) if status.success() => Ok(()), + Ok(Ok(status)) => Err(format!("external Adapter worker exited with {status}")), + Ok(Err(error)) => Err(format!("cannot wait for external Adapter worker: {error}")), + Err(_) => { + terminate_child(child).await?; + Err("external Adapter worker exceeded shutdown deadline".into()) + } + } +} + +async fn terminate_child(child: &mut Child) -> Result<(), String> { + match child.try_wait() { + Ok(Some(_)) => return Ok(()), + Ok(None) => {} + Err(error) => return Err(format!("cannot inspect external Adapter worker: {error}")), + } + if let Err(error) = child.start_kill() + && !matches!(child.try_wait(), Ok(Some(_))) + { + return Err(format!("cannot stop external Adapter worker: {error}")); + } + match tokio::time::timeout(SHUTDOWN_GRACE, child.wait()).await { + Ok(Ok(_)) => Ok(()), + Ok(Err(error)) => Err(format!("cannot wait for external Adapter worker: {error}")), + Err(_) => Err("external Adapter worker could not be reaped before its deadline".into()), + } +} + +async fn handshake( + definition: &Definition, + input: &mut ChildStdin, + output: &mut ChildStdout, +) -> Result<(), String> { + let timeout = Duration::from_millis(definition.timeout_ms); + write_frame_timeout(input, &hello_message(definition)?, timeout).await?; + let ready = read_frame_timeout(output, timeout).await?; + validate_ready(definition, &ready)?; + write_frame_timeout(input, &id_message("health", 0)?, timeout).await?; + let health = read_frame_timeout(output, timeout).await?; + let node = crate::wire::parse(&health)?; + let fields = node.fields(&["kind", "id"])?; + if text_field(fields, "kind")? != "health" || int_field(fields, "id")? != 0 { + return Err("external Adapter returned an invalid health response".into()); + } + require_fields(fields, &["kind", "id"]) +} + +async fn drain_stderr(mut stderr: ChildStderr) -> Result<(), String> { + let mut buffer = [0_u8; 8192]; + loop { + match stderr.read(&mut buffer).await { + Ok(0) => return Ok(()), + Ok(_) => {} + Err(error) => return Err(format!("cannot read external Adapter stderr: {error}")), + } + } +} + +async fn write_frame(output: &mut (impl AsyncWrite + Unpin), body: &str) -> Result<(), String> { + if body.is_empty() || body.len() > crate::wire::MAX_BYTES { + return Err("component protocol frame exceeds byte limit".into()); + } + let length = + u32::try_from(body.len()).map_err(|_| "component protocol frame exceeds byte limit")?; + output + .write_all(&length.to_be_bytes()) + .await + .map_err(|error| format!("cannot write component protocol frame header: {error}"))?; + output + .write_all(body.as_bytes()) + .await + .map_err(|error| format!("cannot write component protocol frame body: {error}"))?; + output + .flush() + .await + .map_err(|error| format!("cannot flush component protocol frame: {error}")) +} + +async fn write_frame_timeout( + output: &mut (impl AsyncWrite + Unpin), + body: &str, + timeout: Duration, +) -> Result<(), String> { + tokio::time::timeout(timeout, write_frame(output, body)) + .await + .map_err(|_| "component protocol write timed out".to_owned())? +} + +async fn read_frame(input: &mut (impl AsyncRead + Unpin)) -> Result<String, String> { + let mut length = [0_u8; 4]; + input + .read_exact(&mut length) + .await + .map_err(|error| format!("cannot read component protocol frame header: {error}"))?; + let length = usize::try_from(u32::from_be_bytes(length)).expect("u32 fits usize"); + if length == 0 || length > crate::wire::MAX_BYTES { + return Err("component protocol frame exceeds byte limit".into()); + } + let mut body = vec![0_u8; length]; + input + .read_exact(&mut body) + .await + .map_err(|error| format!("cannot read complete component protocol frame: {error}"))?; + String::from_utf8(body).map_err(|_| "component protocol frame is not UTF-8".into()) +} + +async fn read_frame_timeout( + input: &mut (impl AsyncRead + Unpin), + timeout: Duration, +) -> Result<String, String> { + tokio::time::timeout(timeout, read_frame(input)) + .await + .map_err(|_| "component protocol response timed out".to_owned())? +} + +fn hello_message(definition: &Definition) -> Result<String, String> { + let mut writer = crate::wire::Encoder::default(); + writer.begin_object()?; + field_text(&mut writer, "kind", "hello")?; + field_text(&mut writer, "version", VERSION)?; + field_text(&mut writer, "port", &definition.port)?; + field_text(&mut writer, "schema", &definition.schema)?; + field_text(&mut writer, "adapter", &definition.adapter)?; + field_text_list(&mut writer, "capabilities", &definition.capabilities)?; + field_text_list(&mut writer, "operations", &definition.operations)?; + writer.key("setting")?; + match &definition.setting { + Some(setting) => writer.json(setting)?, + None => writer.null()?, + } + writer.end()?; + Ok(writer.finish()?.as_str().to_owned()) +} + +fn call_message(id: u64, operation: &str, payload: &str) -> Result<String, String> { + let mut writer = crate::wire::Encoder::default(); + writer.begin_object()?; + field_text(&mut writer, "kind", "call")?; + field_int(&mut writer, "id", id)?; + field_text(&mut writer, "operation", operation)?; + writer.key("payload")?; + writer.json(payload)?; + writer.end()?; + Ok(writer.finish()?.as_str().to_owned()) +} + +fn id_message(kind: &str, id: u64) -> Result<String, String> { + let mut writer = crate::wire::Encoder::default(); + writer.begin_object()?; + field_text(&mut writer, "kind", kind)?; + field_int(&mut writer, "id", id)?; + writer.end()?; + Ok(writer.finish()?.as_str().to_owned()) +} + +fn kind_message(kind: &str) -> Result<String, String> { + let mut writer = crate::wire::Encoder::default(); + writer.begin_object()?; + field_text(&mut writer, "kind", kind)?; + writer.end()?; + Ok(writer.finish()?.as_str().to_owned()) +} + +fn field_text(writer: &mut crate::wire::Encoder, name: &str, value: &str) -> Result<(), String> { + writer.key(name)?; + writer.text(value) +} + +fn field_int(writer: &mut crate::wire::Encoder, name: &str, value: u64) -> Result<(), String> { + let value = i64::try_from(value).map_err(|_| "component protocol request id exhausted")?; + writer.key(name)?; + writer.int(value) +} + +fn field_text_list( + writer: &mut crate::wire::Encoder, + name: &str, + values: &[String], +) -> Result<(), String> { + writer.key(name)?; + writer.begin_array()?; + for value in values { + writer.text(value)?; + } + writer.end() +} + +fn validate_ready(definition: &Definition, body: &str) -> Result<(), String> { + let node = crate::wire::parse(body)?; + let names = [ + "kind", + "version", + "port", + "schema", + "adapter", + "capabilities", + "operations", + ]; + let fields = node.fields(&names)?; + require_fields(fields, &names)?; + if text_field(fields, "kind")? != "ready" + || text_field(fields, "version")? != VERSION + || text_field(fields, "port")? != definition.port + || text_field(fields, "schema")? != definition.schema + || text_field(fields, "adapter")? != definition.adapter + || text_list_field(fields, "capabilities")? != definition.capabilities + || text_list_field(fields, "operations")? != definition.operations + { + return Err("external Adapter handshake does not match its compiled Port schema".into()); + } + Ok(()) +} + +fn parse_reply(body: String, expected_id: u64) -> Result<Reply, String> { + let node = crate::wire::parse(&body)?; + let fields = node.object()?; + let kind = text_field(fields, "kind")?; + let id = int_field(fields, "id")?; + if id != expected_id { + return Err("component protocol response has an unknown or out-of-order request id".into()); + } + match kind { + "result" => { + require_fields(fields, &["kind", "id", "payload"])?; + Ok(Reply::Result(raw_field(fields, "payload")?.to_owned())) + } + "error" => { + require_fields(fields, &["kind", "id", "error", "payload"])?; + let identity = text_field(fields, "error")?; + if identity.is_empty() { + return Err("component protocol error identity is empty".into()); + } + Ok(Reply::Error { + identity: identity.to_owned(), + payload: raw_field(fields, "payload")?.to_owned(), + }) + } + _ => Err("component protocol returned an unknown message kind".into()), + } +} + +fn expect_kind(body: &str, expected: &str) -> Result<(), String> { + let node = crate::wire::parse(body)?; + let fields = node.fields(&["kind"])?; + require_fields(fields, &["kind"])?; + if text_field(fields, "kind")? == expected { + Ok(()) + } else { + Err("component protocol returned an unexpected message kind".into()) + } +} + +fn require_fields( + fields: &BTreeMap<String, crate::wire::Node<'_>>, + names: &[&str], +) -> Result<(), String> { + if fields.len() != names.len() || names.iter().any(|name| !fields.contains_key(*name)) { + return Err("component protocol message has missing or extra fields".into()); + } + Ok(()) +} + +fn text_field<'a>( + fields: &'a BTreeMap<String, crate::wire::Node<'a>>, + name: &str, +) -> Result<&'a str, String> { + fields + .get(name) + .ok_or_else(|| "component protocol message is missing a field".to_owned())? + .text() +} + +fn int_field(fields: &BTreeMap<String, crate::wire::Node<'_>>, name: &str) -> Result<u64, String> { + let value = fields + .get(name) + .ok_or_else(|| "component protocol message is missing a field".to_owned())? + .int()?; + u64::try_from(value).map_err(|_| "component protocol request id must be non-negative".into()) +} + +fn raw_field<'a>( + fields: &'a BTreeMap<String, crate::wire::Node<'a>>, + name: &str, +) -> Result<&'a str, String> { + fields + .get(name) + .map(crate::wire::Node::raw) + .ok_or_else(|| "component protocol message is missing a field".to_owned()) +} + +fn text_list_field<'a>( + fields: &'a BTreeMap<String, crate::wire::Node<'a>>, + name: &str, +) -> Result<Vec<&'a str>, String> { + let values = fields + .get(name) + .ok_or_else(|| "component protocol message is missing a field".to_owned())? + .list()?; + let mut result = Vec::with_capacity(values.len()); + let mut unique = BTreeSet::new(); + for value in values { + let value = value.text()?; + if !unique.insert(value) { + return Err("component protocol list contains a duplicate value".into()); + } + result.push(value); + } + Ok(result) +} diff --git a/crates/dever-runtime/src/component/command.rs b/crates/dever-runtime/src/component/command.rs new file mode 100644 index 0000000..30f090b --- /dev/null +++ b/crates/dever-runtime/src/component/command.rs @@ -0,0 +1,342 @@ +//! One ordinary child per request, owned by the existing Adapter supervisor. +use super::{Definition, Reply, Request, adapter_settings, terminate_child, worker_command}; +use crate::bytes::Bytes; +use crate::config::CommandLimits; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::process::{Child, ChildStderr, ChildStdin, ChildStdout}; +use tokio::sync::{mpsc, oneshot}; + +mod observation; +use observation::Observation; + +#[cfg(feature = "external")] +const ARGUMENT_BYTES: usize = 65_536; +#[cfg(feature = "external")] +const STDIN_BYTES: usize = 8 * 1024 * 1024; + +type CompletionSlot = std::sync::Arc<std::sync::Mutex<Option<crate::task::ScopedCompletion>>>; + +pub(super) struct CallLease(CompletionSlot); +pub(super) struct RequestLease(CompletionSlot); + +pub(super) fn call_lease() -> Result<CallLease, String> { + Ok(CallLease(std::sync::Arc::new(std::sync::Mutex::new(Some( + crate::task::scoped_completion()?, + ))))) +} + +impl CallLease { + pub(super) fn request(&self) -> RequestLease { + RequestLease(self.0.clone()) + } +} + +impl Drop for CallLease { + fn drop(&mut self) { + // Queued calls have no process to drain. An executing supervisor has + // already taken the guard and retains it until the child is reaped. + self.0 + .lock() + .unwrap_or_else(|error| error.into_inner()) + .take(); + } +} + +impl RequestLease { + fn begin(self) -> Option<crate::task::ScopedCompletion> { + self.0 + .lock() + .unwrap_or_else(|error| error.into_inner()) + .take() + } +} + +struct Input { + #[cfg(feature = "external")] + arguments: Vec<String>, + stdin: Option<Bytes>, +} + +struct Execution { + reply: Result<Reply, String>, + stopped: bool, + cleanup_error: Option<String>, +} + +pub(super) async fn supervise( + definition: Definition, + mut requests: mpsc::Receiver<Request>, + mut stopping: oneshot::Receiver<()>, + ready: oneshot::Sender<Result<(), String>>, +) -> Result<(), String> { + let prepared: Result<CommandLimits, String> = (|| { + let limits = adapter_settings()?.command_limits(&definition.port)?; + // Validate resources and grants without starting the user's executable. + worker_command(&definition)?; + Ok(limits) + })(); + let limits = match prepared { + Ok(limits) => limits, + Err(error) => { + let _ = ready.send(Err(error.clone())); + return Err(error); + } + }; + let _ = ready.send(Ok(())); + loop { + let request = tokio::select! { + biased; + _ = &mut stopping => break, + request = requests.recv() => request, + }; + let Some(mut request) = request else { break }; + if request.response.is_closed() { + continue; + } + let Some(completion) = request.command_lease.take().and_then(RequestLease::begin) else { + continue; + }; + let execution = execute(&definition, limits, &mut request, &mut stopping).await; + // execute has dropped I/O futures and reaped the child before replying. + drop(completion); + let _ = request.response.send(execution.reply); + if let Some(error) = execution.cleanup_error { + return Err(error); + } + if execution.stopped { + break; + } + } + requests.close(); + while let Some(request) = requests.recv().await { + let _ = request + .response + .send(Err("external command Adapter stopped".into())); + } + Ok(()) +} + +#[cfg(feature = "external")] +fn input(payload: &str) -> Result<Input, String> { + let node = crate::wire::parse(payload)?; + let fields = node.fields(&["args", "stdin"])?; + let arguments = fields + .get("args") + .ok_or("command args are missing")? + .list()?; + let mut bytes = 0_usize; + let arguments = arguments + .iter() + .map(|value| { + let value = value.text()?; + if value.contains('\0') { + return Err("command argument contains NUL".into()); + } + // Include terminators so empty arguments also consume the budget. + bytes = bytes.saturating_add(value.len()).saturating_add(1); + if bytes > ARGUMENT_BYTES { + return Err("command arguments exceed the 65536 byte limit".into()); + } + Ok(value.to_owned()) + }) + .collect::<Result<Vec<_>, String>>()?; + let stdin = fields + .get("stdin") + .filter(|value| !value.is_null()) + .map(|value| value.bytes()) + .transpose()?; + if stdin + .as_ref() + .is_some_and(|value| value.values().len() > STDIN_BYTES) + { + return Err("command stdin exceeds the 8388608 byte limit".into()); + } + Ok(Input { arguments, stdin }) +} + +#[cfg(feature = "external")] +fn spawn(definition: &Definition, payload: &str) -> Result<(Input, Child, Observation), String> { + let input = input(payload)?; + let (command, observation) = crate::external::observed_command(definition)?; + let observation = Observation::new(observation)?; + let mut command = tokio::process::Command::from(command); + // Resource arguments were mapped by the sandbox owner. Call arguments + // are opaque values, including absolute sandbox paths and shell syntax. + command + .args(&input.arguments) + .env_clear() + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .kill_on_drop(true); + let child = command + .spawn() + .map_err(|error| format!("cannot start external command: {error}"))?; + // Dropping Command closes the parent's copies of mapped pipe descriptors. + drop(command); + Ok((input, child, observation)) +} + +#[cfg(not(feature = "external"))] +fn spawn(_: &Definition, _: &str) -> Result<(Input, Child, Observation), String> { + Err("external command sandbox runtime feature is not enabled".into()) +} + +async fn execute( + definition: &Definition, + limits: CommandLimits, + request: &mut Request, + stopping: &mut oneshot::Receiver<()>, +) -> Execution { + let (input, mut child, mut observation) = match spawn(definition, &request.payload) { + Ok(prepared) => prepared, + Err(error) => { + return Execution { + reply: Err(error), + stopped: false, + cleanup_error: None, + }; + } + }; + let wrapper = child + .id() + .expect("newly spawned command launcher has a PID"); + let pipes = (child.stdin.take(), child.stdout.take(), child.stderr.take()); + let (Some(stdin), Some(stdout), Some(stderr)) = pipes else { + let cleanup_error = cleanup(&mut child, &mut observation, wrapper).await.err(); + return Execution { + reply: Err(cleanup_error + .clone() + .unwrap_or_else(|| "external command has incomplete pipes".into())), + stopped: false, + cleanup_error, + }; + }; + let mut stopped = false; + let result = { + let completion = async { + let (_, _, (stdout, stderr), status) = tokio::try_join!( + observation.started(wrapper), + write_stdin(stdin, input.stdin), + capture(stdout, stderr, limits.output_limit), + async { + child + .wait() + .await + .map_err(|error| format!("cannot wait for external command: {error}")) + }, + )?; + let code = status + .code() + .ok_or("external command sandbox terminated without an exit code")?; + output(code, stdout, stderr) + }; + tokio::select! { + biased; + _ = stopping => { + stopped = true; + Err("external command Adapter stopped".into()) + } + _ = request.response.closed() => Err("external command call cancelled".into()), + result = tokio::time::timeout(std::time::Duration::from_millis(limits.timeout_ms), completion) => { + result.unwrap_or_else(|_| Err("external command call timed out".into())) + } + } + }; + let cleanup_error = cleanup(&mut child, &mut observation, wrapper).await.err(); + let reply = match (result, &cleanup_error) { + (result, None) => result, + (Ok(_), Some(cleanup)) => Err(cleanup.clone()), + (Err(error), Some(cleanup)) => Err(format!("{error}; command cleanup failed: {cleanup}")), + }; + Execution { + reply, + stopped, + cleanup_error, + } +} + +async fn cleanup( + child: &mut Child, + observation: &mut Observation, + wrapper: u32, +) -> Result<(), String> { + let namespace = tokio::time::timeout(super::SHUTDOWN_GRACE, observation.terminate(wrapper)) + .await + .unwrap_or_else(|_| Err("external command namespace exceeded its cleanup deadline".into())); + // Reap the owned launcher even if namespace observation itself failed. + // Both stages have independent finite deadlines; neither error is lost. + let launcher = terminate_child(child).await; + match (namespace, launcher) { + (Err(namespace), Err(launcher)) => { + Err(format!("{namespace}; launcher cleanup failed: {launcher}")) + } + (namespace, launcher) => namespace.and(launcher), + } +} + +async fn write_stdin(mut stdin: ChildStdin, input: Option<Bytes>) -> Result<(), String> { + if let Some(input) = input { + match stdin.write_all(input.values()).await { + Ok(()) => {} + // Ordinary programs may intentionally stop reading before EOF. + Err(error) if error.kind() == std::io::ErrorKind::BrokenPipe => return Ok(()), + Err(error) => return Err(format!("cannot write external command stdin: {error}")), + } + } + drop(stdin); + Ok(()) +} + +async fn capture( + mut stdout: ChildStdout, + mut stderr: ChildStderr, + limit: usize, +) -> Result<(Vec<u8>, Vec<u8>), String> { + let mut output = (Vec::new(), Vec::new()); + let mut stdout_buffer = [0; 8192]; + let mut stderr_buffer = [0; 8192]; + let mut stdout_open = true; + let mut stderr_open = true; + while stdout_open || stderr_open { + let (read, is_stdout) = tokio::select! { + read = stdout.read(&mut stdout_buffer), if stdout_open => (read, true), + read = stderr.read(&mut stderr_buffer), if stderr_open => (read, false), + }; + let count = + read.map_err(|error| format!("cannot read external command output: {error}"))?; + if count == 0 { + if is_stdout { + stdout_open = false; + } else { + stderr_open = false; + } + continue; + } + if count > limit.saturating_sub(output.0.len() + output.1.len()) { + return Err("external command output exceeds output_limit".into()); + } + if is_stdout { + output.0.extend_from_slice(&stdout_buffer[..count]); + } else { + output.1.extend_from_slice(&stderr_buffer[..count]); + } + } + Ok(output) +} + +fn output(code: i32, stdout: Vec<u8>, stderr: Vec<u8>) -> Result<Reply, String> { + let mut encoder = crate::wire::Encoder::default(); + encoder.begin_object()?; + encoder.key("output")?; + encoder.begin_object()?; + encoder.key("code")?; + encoder.int(i64::from(code))?; + encoder.key("stdout")?; + encoder.bytes(&Bytes::new(stdout))?; + encoder.key("stderr")?; + encoder.bytes(&Bytes::new(stderr))?; + encoder.end()?; + encoder.end()?; + Ok(Reply::Result(encoder.finish()?.as_str().to_owned())) +} diff --git a/crates/dever-runtime/src/component/command/observation.rs b/crates/dever-runtime/src/component/command/observation.rs new file mode 100644 index 0000000..37feb32 --- /dev/null +++ b/crates/dever-runtime/src/component/command/observation.rs @@ -0,0 +1,135 @@ +#[cfg(all(feature = "external", target_os = "linux"))] +use tokio::io::AsyncReadExt; + +pub(super) struct Observation { + #[cfg(all(feature = "external", target_os = "linux"))] + startup: tokio::net::unix::pipe::Receiver, + #[cfg(all(feature = "external", target_os = "linux"))] + information: tokio::net::unix::pipe::Receiver, + #[cfg(all(feature = "external", target_os = "linux"))] + information_bytes: Vec<u8>, + #[cfg(all(feature = "external", target_os = "linux"))] + release: Option<std::fs::File>, + #[cfg(all(feature = "external", target_os = "linux"))] + namespace: Option<tokio::io::unix::AsyncFd<dever_sandbox::CommandNamespace>>, +} + +impl Observation { + #[cfg(all(feature = "external", target_os = "linux"))] + pub(super) fn new(observation: dever_sandbox::CommandObservation) -> Result<Self, String> { + use tokio::net::unix::pipe::Receiver; + Ok(Self { + startup: Receiver::from_file(observation.startup).map_err(|error| error.to_string())?, + information: Receiver::from_file(observation.namespace) + .map_err(|error| error.to_string())?, + information_bytes: Vec::new(), + release: Some(observation.release), + namespace: None, + }) + } + + #[cfg(all(feature = "external", not(target_os = "linux")))] + pub(super) fn new(_: dever_sandbox::CommandObservation) -> Result<Self, String> { + Err("external commands require Linux".into()) + } + + #[cfg(all(feature = "external", target_os = "linux"))] + async fn pin(&mut self, wrapper: u32) -> Result<(), String> { + if self.namespace.is_some() { + return Ok(()); + } + let mut buffer = [0; 1024]; + loop { + let count = self + .information + .read(&mut buffer) + .await + .map_err(|error| format!("cannot read command namespace identity: {error}"))?; + if count == 0 { + break; + } + if self.information_bytes.len() + count > 4096 { + return Err("command namespace identity exceeds its byte limit".into()); + } + self.information_bytes.extend_from_slice(&buffer[..count]); + } + let text = std::str::from_utf8(&self.information_bytes) + .map_err(|_| "command namespace identity is not UTF-8")?; + let node = crate::wire::parse(text)?; + let pid = node + .object()? + .get("child-pid") + .ok_or("command namespace identity is missing")? + .int()?; + let namespace = dever_sandbox::CommandNamespace::pin( + i32::try_from(pid).map_err(|_| "invalid command namespace PID")?, + wrapper, + )?; + self.namespace = Some( + tokio::io::unix::AsyncFd::new(namespace) + .map_err(|error| format!("cannot observe command namespace exit: {error}"))?, + ); + Ok(()) + } + + pub(super) async fn started(&mut self, wrapper: u32) -> Result<(), String> { + #[cfg(all(feature = "external", target_os = "linux"))] + { + self.pin(wrapper).await?; + // EOF releases bwrap only after its namespace is pinned. Neither + // normal cancellation nor a very short process can race PID reuse. + drop(self.release.take()); + let mut status = Vec::new(); + (&mut self.startup) + .take(4096) + .read_to_end(&mut status) + .await + .map_err(|error| format!("cannot read command startup status: {error}"))?; + match status.as_slice() { + b"R" => Ok(()), + [] => Err("external command sandbox failed before starting its guard".into()), + status => { + let message = status + .strip_prefix(b"RE") + .or_else(|| status.strip_prefix(b"E")); + Err(match message { + Some(message) => format!( + "external command startup failed: {}", + String::from_utf8_lossy(message) + ), + None => "invalid external command startup status".into(), + }) + } + } + } + #[cfg(not(all(feature = "external", target_os = "linux")))] + { + let _ = wrapper; + Err("external commands require Linux".into()) + } + } + + pub(super) async fn terminate(&mut self, wrapper: u32) -> Result<(), String> { + #[cfg(all(feature = "external", target_os = "linux"))] + { + self.pin(wrapper).await?; + let namespace = self + .namespace + .as_ref() + .expect("namespace pinned before cleanup"); + namespace.get_ref().terminate()?; + // pidfd readiness follows kernel namespace teardown, unlike pipe + // EOF, which occurs while the init is still closing its files. + let _exited = namespace + .readable() + .await + .map_err(|error| format!("cannot wait for command namespace exit: {error}"))?; + Ok(()) + } + #[cfg(not(all(feature = "external", target_os = "linux")))] + { + let _ = wrapper; + Err("external commands require Linux".into()) + } + } +} diff --git a/crates/dever-runtime/src/concurrent.rs b/crates/dever-runtime/src/concurrent.rs new file mode 100644 index 0000000..eac68f7 --- /dev/null +++ b/crates/dever-runtime/src/concurrent.rs @@ -0,0 +1,133 @@ +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Mutex, mpsc}; + +pub const MAX_WORKERS: usize = 256; +const MAX_BATCH_ITEMS: usize = 256; +const BATCHES_PER_WORKER: usize = 8; + +pub fn worker_count(workers: i64) -> Result<usize, &'static str> { + usize::try_from(workers) + .ok() + .filter(|count| (1..=MAX_WORKERS).contains(count)) + .ok_or("parallel worker count must be between 1 and 256") +} + +/// 单一生产者按需取值，有界队列防止 Stream 被提前收集；返回前回收全部工作线程。 +pub fn each<T: Send>( + values: impl IntoIterator<Item = T>, + workers: i64, + apply: impl Fn(T) -> Result<(), String> + Sync, +) -> Result<(), String> { + each_typed(values, workers, apply) +} + +pub fn each_typed<T: Send, E: From<String> + Send>( + values: impl IntoIterator<Item = T>, + workers: i64, + apply: impl Fn(T) -> Result<(), E> + Sync, +) -> Result<(), E> { + let workers = worker_count(workers).map_err(|error| E::from(error.to_owned()))?; + run_typed(values, workers, |value, _| apply(value)) +} + +/// 内存序列没有拉取副作用；限制每批元素数，同时给工作线程保留多批调度机会。 +pub fn each_slice<T: Sync>( + values: &[T], + workers: i64, + apply: impl Fn(&T) -> Result<(), String> + Sync, +) -> Result<(), String> { + each_slice_typed(values, workers, apply) +} + +pub fn each_slice_typed<T: Sync, E: From<String> + Send>( + values: &[T], + workers: i64, + apply: impl Fn(&T) -> Result<(), E> + Sync, +) -> Result<(), E> { + let workers = worker_count(workers).map_err(|error| E::from(error.to_owned()))?; + let batch = batch_size(values.len(), workers); + run_typed(values.chunks(batch), workers, |chunk, cancelled| { + for value in chunk { + if cancelled.load(Ordering::Acquire) { + break; + } + apply(value)?; + } + Ok(()) + }) +} + +pub(crate) fn batch_size(items: usize, workers: usize) -> usize { + (items / (workers * BATCHES_PER_WORKER)).clamp(1, MAX_BATCH_ITEMS) +} + +fn run_typed<T: Send, E: From<String> + Send>( + values: impl IntoIterator<Item = T>, + workers: usize, + apply: impl Fn(T, &AtomicBool) -> Result<(), E> + Sync, +) -> Result<(), E> { + let mut values = values.into_iter(); + let (minimum_items, maximum_items) = values.size_hint(); + let workers = match maximum_items { + Some(maximum_items) if maximum_items == minimum_items => workers.min(maximum_items), + _ => workers, + }; + if workers == 0 { + return Ok(()); + } + let (sender, receiver) = mpsc::sync_channel(workers); + let receiver = Mutex::new(receiver); + let cancelled = AtomicBool::new(false); + let failure = Mutex::new(None); + let record_failure = |error: E| { + cancelled.store(true, Ordering::Release); + let mut failure = failure.lock().expect("worker failure lock"); + if failure.is_none() { + *failure = Some(error); + } + }; + std::thread::scope(|scope| { + let mut handles = Vec::new(); + for _ in 0..workers { + let spawn = std::thread::Builder::new().spawn_scoped(scope, || { + loop { + let next = receiver.lock().expect("worker receiver lock").recv(); + let Ok(value) = next else { break }; + // 故障后继续清空队列直到发送端关闭，避免生产者卡在有界 send。 + if !cancelled.load(Ordering::Acquire) { + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + apply(value, &cancelled) + })) { + Ok(Ok(())) => {} + Ok(Err(error)) => record_failure(error), + Err(_) => record_failure(E::from("parallel worker panicked".into())), + } + } + } + }); + match spawn { + Ok(handle) => handles.push(handle), + Err(error) => { + record_failure(E::from(error.to_string())); + break; + } + } + } + while !cancelled.load(Ordering::Acquire) { + let Some(value) = values.next() else { break }; + if sender.send(value).is_err() { + break; + } + } + drop(sender); + for handle in handles { + if handle.join().is_err() { + record_failure(E::from("parallel worker panicked".into())); + } + } + }); + match failure.into_inner().expect("worker failure lock") { + Some(error) => Err(error), + None => Ok(()), + } +} diff --git a/crates/dever-runtime/src/config.rs b/crates/dever-runtime/src/config.rs new file mode 100644 index 0000000..86992cc --- /dev/null +++ b/crates/dever-runtime/src/config.rs @@ -0,0 +1,1467 @@ +use std::collections::BTreeMap; +use std::fs; +use std::path::{Component, Path, PathBuf}; +use std::sync::OnceLock; + +use serde::Deserialize; + +mod command; +mod compilation; +pub use command::CommandLimits; +pub use compilation::{CompilationBindings, CompilationSite, DatabaseDriver}; + +const DATA_DIRECTORIES: [&str; 5] = ["db", "upload", "log", "cache", "tmp"]; +const ADAPTER_FIELDS: &[&str] = &["use", "setting", "files", "command"]; +static SETTINGS: OnceLock<Settings> = OnceLock::new(); +pub const MAX_JOB_TIMEOUT_MS: u32 = 3_600_000; +pub const MAX_JOB_LEASE_MS: i64 = 2 * MAX_JOB_TIMEOUT_MS as i64; + +pub struct Settings { + executable_directory: PathBuf, + dever: Option<DeverSettings>, + lib: Vec<String>, + package: Option<PackageSettings>, + databases: BTreeMap<String, Database>, + tenant: Option<TenantSettings>, + http: Option<serde_json::Map<String, serde_json::Value>>, + adapter: Option<Box<serde_json::value::RawValue>>, + auth: AuthSettings, + sites: BTreeMap<String, SiteSettings>, + log: LogSettings, + runtime: Option<RuntimeSettings>, + job: Option<JobSettings>, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct PackageSettings { + registry: String, + #[serde(default, rename = "use")] + roots: Vec<String>, +} + +impl PackageSettings { + pub fn registry(&self) -> &str { + &self.registry + } + pub fn roots(&self) -> &[String] { + &self.roots + } +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct DeverSettings { + version: String, +} + +impl DeverSettings { + pub fn version(&self) -> &str { + &self.version + } +} + +#[derive(Clone, Debug, Default)] +pub struct AuthSettings { + providers: BTreeMap<String, AuthProvider>, +} + +#[derive(Clone, Debug)] +pub struct AuthProvider { + verify: String, + #[cfg_attr(not(feature = "api"), allow(dead_code))] + jwt_secret: crate::secret::Secret, + ttl_seconds: u64, + cookie: String, +} + +#[derive(Clone, Debug)] +pub struct SiteSettings { + path: String, + auth: String, + hosts: Vec<String>, + origin: Option<HttpOrigin>, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct HttpOrigin { + scheme: String, + host: String, + port: u16, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq)] +#[serde(rename_all = "lowercase")] +pub enum LogLevel { + Debug, + Info, + Warn, + Error, +} + +#[derive(Clone, Copy, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LogSettings { + #[serde(default = "default_log_level")] + pub level: LogLevel, +} + +impl Default for LogSettings { + fn default() -> Self { + Self { + level: default_log_level(), + } + } +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq)] +#[serde(rename_all = "lowercase")] +pub enum RuntimeMode { + Api, + Worker, + All, +} + +#[derive(Clone, Copy, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RuntimeSettings { + pub mode: RuntimeMode, + pub shutdown_ms: i64, +} + +#[derive(Clone, Copy, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct JobSettings { + pub workers: usize, + pub poll_ms: i64, + pub lease_ms: i64, + pub retry_base_ms: i64, + pub retry_max_ms: i64, +} + +impl JobSettings { + pub(crate) fn validate_lease( + &self, + database: &Database, + timeout_ms: i64, + ) -> Result<(), String> { + if !(1..=i64::from(MAX_JOB_TIMEOUT_MS)).contains(&timeout_ms) { + return Err("invalid Job timeout_ms".into()); + } + // A spare connection cannot bypass SQLite's single writer. The lease must + // cover the whole bounded handler even while its transaction blocks renewal. + if matches!(database, Database::Sqlite { .. }) && self.lease_ms <= timeout_ms { + return Err("job.lease_ms must exceed Job timeout_ms for SQLite".into()); + } + Ok(()) + } + + pub fn validate(self) -> Result<Self, String> { + if !(1..=256).contains(&self.workers) { + return Err("job.workers must be between 1 and 256".into()); + } + for (field, value, minimum, maximum) in [ + ("poll_ms", self.poll_ms, 1, 60_000), + ("lease_ms", self.lease_ms, 100, MAX_JOB_LEASE_MS), + ("retry_base_ms", self.retry_base_ms, 1, 86_400_000), + ("retry_max_ms", self.retry_max_ms, 1, 86_400_000), + ] { + if !(minimum..=maximum).contains(&value) { + return Err(format!( + "job.{field} must be between {minimum} and {maximum}" + )); + } + } + if self.retry_base_ms > self.retry_max_ms { + return Err("job.retry_base_ms exceeds retry_max_ms".into()); + } + Ok(self) + } + + pub fn for_test() -> Self { + Self { + workers: 1, + poll_ms: 1, + lease_ms: MAX_JOB_LEASE_MS, + retry_base_ms: 1000, + retry_max_ms: 300_000, + } + } +} + +impl AuthSettings { + pub fn provider(&self, key: &str) -> Option<&AuthProvider> { + self.providers.get(key) + } +} + +impl AuthProvider { + pub fn verify(&self) -> &str { + &self.verify + } + pub fn ttl_seconds(&self) -> u64 { + self.ttl_seconds + } + pub fn cookie(&self) -> &str { + &self.cookie + } + #[cfg(feature = "api")] + pub(crate) fn jwt_secret(&self) -> &crate::secret::Secret { + &self.jwt_secret + } +} + +impl SiteSettings { + pub fn path(&self) -> &str { + &self.path + } + pub fn auth(&self) -> &str { + &self.auth + } + pub fn hosts(&self) -> &[String] { + &self.hosts + } + pub fn origin(&self) -> Option<&HttpOrigin> { + self.origin.as_ref() + } +} + +impl HttpOrigin { + pub fn scheme(&self) -> &str { + &self.scheme + } + pub fn host(&self) -> &str { + &self.host + } + pub fn port(&self) -> u16 { + self.port + } +} + +#[derive(Clone, Debug)] +pub struct HttpSettings { + pub host: String, + pub port: i64, + pub limits: crate::http::Limits, + pub upload: UploadLimits, + pub upload_directory: PathBuf, +} + +#[derive(Clone, Copy, Debug, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct UploadLimits { + pub total_bytes: usize, + pub file_bytes: usize, + pub field_bytes: usize, + pub parts: usize, + pub fields: usize, + pub header_bytes: usize, + pub filename_bytes: usize, +} + +impl Default for UploadLimits { + fn default() -> Self { + Self { + total_bytes: 67_108_864, + file_bytes: 33_554_432, + field_bytes: 65_536, + parts: 32, + fields: 16, + header_bytes: 8192, + filename_bytes: 255, + } + } +} + +impl UploadLimits { + pub fn validate(self) -> Result<(), String> { + for (name, value, maximum) in [ + ("total_bytes", self.total_bytes, 1_073_741_824), + ("file_bytes", self.file_bytes, self.total_bytes), + ("field_bytes", self.field_bytes, 1_048_576), + ("parts", self.parts, 256), + ("fields", self.fields, self.parts), + ("header_bytes", self.header_bytes, 65_536), + ("filename_bytes", self.filename_bytes, 1024), + ] { + if value == 0 || value > maximum { + return Err(format!( + "http.upload.{name} must be between 1 and {maximum}" + )); + } + } + Ok(()) + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct HttpDocument { + host: String, + port: u16, + #[serde(default = "default_http_header_bytes")] + header_bytes: i64, + #[serde(default = "default_http_body_bytes")] + body_bytes: i64, + #[serde(default = "default_http_timeout_ms")] + timeout_ms: i64, + #[serde(default = "default_http_connections")] + connections: i64, + #[serde(default)] + upload: UploadLimits, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum Database { + Sqlite { + path: PathBuf, + tenant_directory: Option<PathBuf>, + max_connections: usize, + max_page_size: usize, + }, + Postgres { + url: String, + tenant_database_prefix: Option<String>, + tls: PostgresTls, + min_connections: usize, + max_connections: usize, + max_page_size: usize, + wait_timeout_ms: u64, + io_timeout_ms: u64, + }, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct TenantSettings { + database: String, + max_pools: usize, + idle_timeout_ms: u64, +} + +impl TenantSettings { + pub fn database(&self) -> &str { + &self.database + } + pub fn max_pools(&self) -> usize { + self.max_pools + } + pub fn idle_timeout_ms(&self) -> u64 { + self.idle_timeout_ms + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum PostgresTls { + System, + Disabled, +} + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct RuntimeProfile { + pub sqlite: bool, + pub postgres: bool, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Document { + #[serde(default, deserialize_with = "deserialize_present")] + dever: Option<DeverSettings>, + #[serde(default)] + lib: Vec<String>, + #[serde(default, deserialize_with = "deserialize_present")] + package: Option<PackageSettings>, + #[serde(default, deserialize_with = "deserialize_present")] + runtime: Option<RuntimeSettings>, + #[serde(default, deserialize_with = "deserialize_present")] + job: Option<JobSettings>, + #[serde(default, deserialize_with = "deserialize_adapter")] + adapter: Option<Box<serde_json::value::RawValue>>, + #[serde(default)] + auth: AuthDocument, + #[serde(default)] + sites: BTreeMap<String, SiteDocument>, + #[serde(default)] + performance: Option<serde_json::Map<String, serde_json::Value>>, + #[serde(default)] + log: LogSettings, + #[serde(default)] + http: Option<serde_json::Map<String, serde_json::Value>>, + #[serde(default, deserialize_with = "deserialize_database")] + database: Option<BTreeMap<String, DatabaseDocument>>, + #[serde(default, deserialize_with = "deserialize_present")] + tenant: Option<TenantDocument>, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct TenantDocument { + database: String, + #[serde(default = "default_tenant_max_pools")] + max_pools: usize, + #[serde(default = "default_tenant_idle_timeout_ms")] + idle_timeout_ms: u64, +} + +#[derive(Default, Deserialize)] +#[serde(deny_unknown_fields)] +struct AuthDocument { + #[serde(default)] + providers: BTreeMap<String, AuthProviderDocument>, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct AuthProviderDocument { + verify: String, + #[serde(rename = "jwtSecret")] + jwt_secret: String, + #[serde(rename = "ttlSeconds", default = "default_auth_ttl_seconds")] + ttl_seconds: u64, + #[serde(default = "default_auth_cookie")] + cookie: String, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct SiteDocument { + path: String, + auth: String, + #[serde(default)] + hosts: Vec<String>, + #[serde(default)] + origin: Option<String>, +} + +fn deserialize_present<'de, D, T>(deserializer: D) -> Result<Option<T>, D::Error> +where + D: serde::Deserializer<'de>, + T: Deserialize<'de>, +{ + T::deserialize(deserializer).map(Some) +} + +fn deserialize_adapter<'de, D>( + deserializer: D, +) -> Result<Option<Box<serde_json::value::RawValue>>, D::Error> +where + D: serde::Deserializer<'de>, +{ + // Preserve an explicit null so the shared wire validator rejects it as a wrong type. + Box::<serde_json::value::RawValue>::deserialize(deserializer).map(Some) +} + +#[derive(Deserialize)] +#[serde(tag = "type", rename_all = "lowercase", deny_unknown_fields)] +enum DatabaseDocument { + Sqlite { + path: PathBuf, + #[serde(default)] + tenant_directory: Option<PathBuf>, + #[serde(default = "default_sqlite_connections")] + max_connections: usize, + #[serde(default = "default_page_size")] + max_page_size: usize, + }, + Postgres { + url: String, + #[serde(default)] + tenant_database_prefix: Option<String>, + tls: TlsDocument, + #[serde(default)] + min_connections: usize, + #[serde(default = "default_postgres_connections")] + max_connections: usize, + #[serde(default = "default_page_size")] + max_page_size: usize, + #[serde(default = "default_postgres_wait_timeout_ms")] + wait_timeout_ms: u64, + #[serde(default = "default_postgres_io_timeout_ms")] + io_timeout_ms: u64, + }, +} + +#[derive(Clone, Copy, Deserialize)] +#[serde(rename_all = "lowercase")] +enum TlsDocument { + System, + Disabled, +} + +fn deserialize_database<'de, D>( + deserializer: D, +) -> Result<Option<BTreeMap<String, DatabaseDocument>>, D::Error> +where + D: serde::Deserializer<'de>, +{ + BTreeMap::deserialize(deserializer).map(Some) +} + +pub fn validate_dever_version(version: &str) -> Result<(), String> { + let mut parts = version.split('.'); + for _ in 0..3 { + let part = parts + .next() + .ok_or_else(|| "dever.version must be an exact major.minor.patch version".to_owned())?; + if part.is_empty() + || !part.bytes().all(|byte| byte.is_ascii_digit()) + || (part.len() > 1 && part.starts_with('0')) + || part.parse::<u64>().is_err() + { + return Err("dever.version must be an exact major.minor.patch version".into()); + } + } + if parts.next().is_some() { + return Err("dever.version must be an exact major.minor.patch version".into()); + } + Ok(()) +} + +impl Settings { + pub fn load_for_executable() -> Result<Self, String> { + Self::load(&executable_directory()?) + } + + pub fn load_project(project_root: &Path) -> Result<Self, String> { + Self::load(project_root) + } + + fn load(executable_directory: &Path) -> Result<Self, String> { + let path = executable_directory.join("config/setting.json"); + let text = fs::read_to_string(&path) + .map_err(|error| format!("cannot read '{}': {error}", path.display()))?; + crate::wire::parse(&text).map_err(|error| format!("invalid setting.json: {error}"))?; + let document: Document = serde_json::from_str(&text) + .map_err(|error| format!("invalid '{}': {error}", path.display()))?; + let Document { + dever, + lib, + package, + runtime, + job, + adapter, + auth, + sites, + performance, + log, + http, + database, + tenant, + } = document; + if let Some(dever) = &dever { + validate_dever_version(dever.version())?; + } + if let Some(runtime) = runtime + && !(1..=300_000).contains(&runtime.shutdown_ms) + { + return Err("runtime.shutdown_ms must be between 1 and 300000".into()); + } + if let Some(job) = job { + job.validate()?; + } + let _ = performance; + let auth = normalize_auth(auth)?; + let sites = normalize_sites(sites, &auth)?; + if database + .as_ref() + .is_some_and(|database| !database.contains_key("default")) + { + return Err("database.default is required".into()); + } + let mut databases = BTreeMap::new(); + for (name, database) in database.unwrap_or_default() { + compilation::validate_database_name(&name)?; + databases.insert( + name.clone(), + normalize_database(&name, database, executable_directory)?, + ); + } + let tenant = normalize_tenant(tenant, &databases)?; + Ok(Self { + executable_directory: executable_directory.to_path_buf(), + dever, + lib, + package, + databases, + tenant, + http, + adapter, + auth, + sites, + log, + runtime, + job, + }) + } + + /// HTTP 配置只在启动 API 时必需；普通程序可保留空 http 对象。 + pub fn http(&self) -> Result<HttpSettings, String> { + let document = self + .http + .clone() + .ok_or("http.host and http.port are required for dever.api.serve()")?; + let document: HttpDocument = serde_json::from_value(serde_json::Value::Object(document)) + .map_err(|error| format!("invalid http settings: {error}"))?; + if document.host.parse::<std::net::IpAddr>().is_err() { + return Err("http.host must be an explicit IPv4 or IPv6 address".into()); + } + // 验证上限后再监听；缺省只限制资源，不猜测监听地址或端口。 + for (field, value, minimum, maximum) in [ + ("header_bytes", document.header_bytes, 8192, 1_048_576), + ("body_bytes", document.body_bytes, 0, 67_108_864), + ("timeout_ms", document.timeout_ms, 1, 300_000), + ("connections", document.connections, 1, 65_536), + ] { + if !(minimum..=maximum).contains(&value) { + return Err(format!( + "http.{field} must be between {minimum} and {maximum}" + )); + } + } + document.upload.validate()?; + Ok(HttpSettings { + host: document.host, + port: i64::from(document.port), + upload: document.upload, + upload_directory: self.executable_directory.join("data/tmp"), + limits: crate::http::Limits { + header_bytes: document.header_bytes, + body_bytes: document.body_bytes, + timeout_ms: document.timeout_ms, + connections: document.connections, + http2: None, + }, + }) + } + + pub fn services(&self, api: bool, worker: bool) -> Result<RuntimeSettings, String> { + let runtime = self.runtime.unwrap_or(RuntimeSettings { + mode: match (api, worker) { + (true, true) => RuntimeMode::All, + (false, true) => RuntimeMode::Worker, + _ => RuntimeMode::Api, + }, + shutdown_ms: 30_000, + }); + if matches!(runtime.mode, RuntimeMode::Api | RuntimeMode::All) { + if !api { + return Err("runtime.mode requires an API declaration".into()); + } + self.http()?; + } + if matches!(runtime.mode, RuntimeMode::Worker | RuntimeMode::All) { + if !worker { + return Err("runtime.mode requires a Job declaration".into()); + } + self.job + .ok_or("job settings are required for worker mode")? + .validate()?; + } + Ok(runtime) + } + + pub fn jobs(&self) -> Result<JobSettings, String> { + self.job + .ok_or_else(|| "job settings are required".into()) + .and_then(JobSettings::validate) + } + + pub fn validate_job_connection( + &self, + explicit: Option<&str>, + root: &str, + timeout_ms: u32, + ) -> Result<(), String> { + let settings = self.jobs()?; + let (name, database) = self.resolve_database(explicit, root)?; + let capacity = match database { + Database::Sqlite { + max_connections, .. + } + | Database::Postgres { + max_connections, .. + } => *max_connections, + }; + // Every handler may own a transaction; control and heartbeat retain one spare slot. + if capacity <= settings.workers { + return Err(format!( + "database.{name}.max_connections must exceed job.workers to reserve a lease-control connection" + )); + } + settings + .validate_lease(database, i64::from(timeout_ms)) + .map_err(|error| format!("database.{name}: {error}")) + } + + /// Port-only programs may omit the file when every binding is automatic and has no setting. + pub fn load_adapter_settings() -> Result<Self, String> { + let directory = executable_directory()?; + Self::load_adapter_settings_at(&directory) + } + + pub(crate) fn load_adapter_settings_at(directory: &Path) -> Result<Self, String> { + match fs::metadata(directory.join("config/setting.json")) { + Ok(_) => Self::load(directory), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(Self { + executable_directory: directory.to_owned(), + dever: None, + lib: Vec::new(), + package: None, + databases: BTreeMap::new(), + tenant: None, + http: None, + adapter: None, + auth: AuthSettings::default(), + sites: BTreeMap::new(), + log: LogSettings::default(), + runtime: None, + job: None, + }), + Err(error) => Err(format!("cannot inspect setting.json: {error}")), + } + } + + pub fn validate_adapter_names(&self, identities: &[&str]) -> Result<(), String> { + if let Some(raw) = &self.adapter { + crate::wire::parse(raw.get())?.fields(identities)?; + } + Ok(()) + } + + pub fn select_adapter( + &self, + identity: &str, + candidates: &[&str], + ) -> Result<(usize, Option<String>), String> { + let node = self + .adapter + .as_ref() + .map(|raw| crate::wire::parse(raw.get())) + .transpose()?; + let binding = node + .as_ref() + .map(|node| node.object()) + .transpose()? + .and_then(|fields| fields.get(identity)); + let fields = binding + .map(|node| node.fields(ADAPTER_FIELDS)) + .transpose()?; + let selected = fields + .and_then(|fields| fields.get("use")) + .map(|node| node.text()) + .transpose()?; + let index = match selected { + Some(name) => candidates + .iter() + .position(|candidate| *candidate == name) + .ok_or_else(|| format!("unknown Adapter selection for Port '{identity}'"))?, + None if candidates.len() == 1 => 0, + None => return Err(format!("adapter.{identity}.use is required")), + }; + let setting = fields + .and_then(|fields| fields.get("setting")) + .map(|node| node.raw().to_owned()); + Ok((index, setting)) + } + + /// File access is a deployment grant for the selected Port, not Worker + /// business settings. Paths stay below the application and appear at /data. + #[cfg(feature = "external")] + pub fn external_grants( + &self, + identity: &str, + capabilities: &[String], + ) -> Result<Vec<dever_sandbox::Grant>, String> { + #[derive(Deserialize)] + #[serde(deny_unknown_fields)] + struct FileGrant { + path: PathBuf, + #[serde(default)] + write: bool, + } + let Some(adapter) = &self.adapter else { + return Ok(Vec::new()); + }; + let node = crate::wire::parse(adapter.get())?; + let Some(binding) = node.object()?.get(identity) else { + return Ok(Vec::new()); + }; + let binding = binding.fields(ADAPTER_FIELDS)?; + let Some(files) = binding.get("files") else { + return Ok(Vec::new()); + }; + if !capabilities.iter().any(|capability| capability == "file") { + return Err("Adapter file grants require its checked file capability".into()); + } + let files: BTreeMap<String, FileGrant> = serde_json::from_str(files.raw()) + .map_err(|_| "Adapter files must map names to {path, write} grants")?; + if files.len() > 64 { + return Err("Adapter has too many file grants".into()); + } + let mut grants = Vec::new(); + for (name, grant) in files { + if name.is_empty() + || name.len() > 64 + || !name + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_') + { + return Err("Adapter file grant names must contain lowercase letters, digits or underscores".into()); + } + if grant.path.as_os_str().is_empty() + || grant.path.is_absolute() + || grant + .path + .components() + .any(|part| !matches!(part, Component::Normal(_))) + { + return Err("Adapter file grant path must be relative to the application".into()); + } + let executable_cache = Path::new("data/cache/lib"); + if grant.write + && (grant.path.starts_with(executable_cache) + || executable_cache.starts_with(&grant.path)) + { + return Err( + "Adapter writable file grant overlaps the executable resource cache".into(), + ); + } + grants.push(dever_sandbox::Grant { + source: self.executable_directory.join(grant.path), + destination: Path::new("/data").join(name), + writable: grant.write, + }); + } + Ok(grants) + } + + pub fn initialize_data_directories(&self) -> Result<(), String> { + for name in DATA_DIRECTORIES { + let path = self.executable_directory.join("data").join(name); + fs::create_dir_all(&path) + .map_err(|error| format!("cannot create '{}': {error}", path.display()))?; + } + Ok(()) + } + + pub fn database(&self, name: &str) -> Option<&Database> { + self.databases.get(name) + } + + /// The launcher consumes this value before dispatch. Runtime keeps it only so + /// one strict setting.json contract can also be read by standalone artifacts. + pub fn dever_version(&self) -> Option<&str> { + self.dever.as_ref().map(DeverSettings::version) + } + + pub fn external_libs(&self) -> &[String] { + &self.lib + } + + pub fn packages(&self) -> Option<&PackageSettings> { + self.package.as_ref() + } + + pub fn tenant(&self) -> Option<&TenantSettings> { + self.tenant.as_ref() + } + + pub fn auth(&self) -> &AuthSettings { + &self.auth + } + + pub fn sites(&self) -> &BTreeMap<String, SiteSettings> { + &self.sites + } + + pub fn log(&self) -> LogSettings { + self.log + } + + pub fn site_for_directory<'a>( + &'a self, + directory: &[&str], + ) -> Result<Option<(&'a str, &'a SiteSettings)>, String> { + compilation::select_site(&self.sites, directory, |site| &site.path) + } + + pub fn databases(&self) -> impl Iterator<Item = (&String, &Database)> { + self.databases.iter() + } + + pub fn resolve_database( + &self, + explicit: Option<&str>, + package_root: &str, + ) -> Result<(&str, &Database), String> { + compilation::resolve_database(&self.databases, explicit, package_root) + } + + pub fn profile(&self) -> RuntimeProfile { + let mut profile = RuntimeProfile::default(); + for database in self.databases.values() { + match database { + Database::Sqlite { .. } => profile.sqlite = true, + Database::Postgres { .. } => profile.postgres = true, + } + } + profile + } + + pub fn validate_transaction_bindings( + &self, + transactions: &[&[(Option<&str>, &str)]], + ) -> Result<(), String> { + for transaction in transactions { + let mut connection = None; + for (explicit, package_root) in *transaction { + let name = self.resolve_database(*explicit, package_root)?.0; + if connection + .replace(name) + .is_some_and(|current| current != name) + { + return Err("a transaction resolves to multiple database connections".into()); + } + } + } + Ok(()) + } + + pub fn validate_scoped_database_bindings( + &self, + bindings: &[(Option<&str>, &str, bool)], + transactions: &[&[(Option<&str>, &str, bool)]], + ) -> Result<(), String> { + for (explicit, root, tenant_scoped) in bindings { + let (name, database) = self.resolve_database(*explicit, root)?; + if self.tenant.is_some() && *tenant_scoped { + match database { + Database::Sqlite { + tenant_directory: Some(_), + .. + } + | Database::Postgres { + tenant_database_prefix: Some(_), + .. + } => {} + Database::Sqlite { .. } => { + return Err(format!( + "database.{name}.tenant_directory is required by a tenant Model" + )); + } + Database::Postgres { .. } => { + return Err(format!( + "database.{name}.tenant_database_prefix is required by a tenant Model" + )); + } + } + } + } + for transaction in transactions { + let mut selected: Option<(&str, bool)> = None; + for (explicit, root, tenant_scoped) in *transaction { + let name = self.resolve_database(*explicit, root)?.0; + if let Some((current_name, current_scope)) = selected { + if current_name != name { + return Err( + "a transaction resolves to multiple database connections".into() + ); + } + if self.tenant.is_some() && current_scope != *tenant_scoped { + return Err("a transaction cannot mix global and tenant Models".into()); + } + } else { + selected = Some((name, *tenant_scoped)); + } + } + } + Ok(()) + } + + pub(crate) fn prepare_databases( + &self, + bundled: RuntimeProfile, + bindings: &[(Option<&str>, &str, bool)], + transactions: &[&[(Option<&str>, &str, bool)]], + ) -> Result<(), String> { + let actual = self.profile(); + if actual.sqlite && !bundled.sqlite { + return Err("setting.json requires SQLite support; rebuild the application".into()); + } + if actual.postgres && !bundled.postgres { + return Err("setting.json requires PostgreSQL support; rebuild the application".into()); + } + for (explicit, package_root, _) in bindings { + self.resolve_database(*explicit, package_root)?; + } + self.validate_scoped_database_bindings(bindings, transactions)?; + self.initialize_data_directories() + } + + #[cfg(any(feature = "sqlite", feature = "postgres"))] + pub(crate) fn validate_database_entry(&self) -> Result<(), String> { + if self.tenant.is_some() { + return Err("this application entry does not support tenant database settings".into()); + } + Ok(()) + } +} + +pub fn executable_directory() -> Result<PathBuf, String> { + let executable = std::env::current_exe() + .map_err(|error| format!("cannot locate the executable: {error}"))?; + executable + .parent() + .map(Path::to_path_buf) + .ok_or_else(|| "the executable has no parent directory".to_owned()) +} + +pub fn bootstrap( + bundled: RuntimeProfile, + bindings: &[(Option<&str>, &str)], + transactions: &[&[(Option<&str>, &str)]], +) -> Result<(), String> { + let scoped_bindings = bindings + .iter() + .map(|(explicit, root)| (*explicit, *root, false)) + .collect::<Vec<_>>(); + let scoped_transactions = transactions + .iter() + .map(|transaction| { + transaction + .iter() + .map(|(explicit, root)| (*explicit, *root, false)) + .collect::<Vec<_>>() + }) + .collect::<Vec<_>>(); + let scoped_transaction_refs = scoped_transactions + .iter() + .map(Vec::as_slice) + .collect::<Vec<_>>(); + bootstrap_scoped(bundled, &scoped_bindings, &scoped_transaction_refs) +} + +pub fn bootstrap_scoped( + bundled: RuntimeProfile, + bindings: &[(Option<&str>, &str, bool)], + transactions: &[&[(Option<&str>, &str, bool)]], +) -> Result<(), String> { + let settings = Settings::load_for_executable()?; + settings.prepare_databases(bundled, bindings, transactions)?; + #[cfg(feature = "sqlite")] + crate::sqlite::initialize(&settings)?; + #[cfg(feature = "postgres")] + crate::postgres::initialize(&settings)?; + SETTINGS + .set(settings) + .map_err(|_| "runtime settings were initialized more than once".to_owned()) +} + +pub fn settings() -> &'static Settings { + SETTINGS + .get() + .expect("runtime settings are initialized before entry") +} + +pub fn try_settings() -> Option<&'static Settings> { + SETTINGS.get() +} + +pub enum CurrentSettings { + Process(&'static Settings), + #[cfg(feature = "api")] + Application(std::sync::Arc<Settings>), +} + +impl std::ops::Deref for CurrentSettings { + type Target = Settings; + fn deref(&self) -> &Settings { + match self { + Self::Process(settings) => settings, + #[cfg(feature = "api")] + Self::Application(settings) => settings, + } + } +} + +pub fn try_current_settings() -> Option<CurrentSettings> { + #[cfg(feature = "api")] + if let Some(session) = crate::application::current() { + return Some(CurrentSettings::Application(session.settings().clone())); + } + SETTINGS.get().map(CurrentSettings::Process) +} + +pub fn current_settings() -> CurrentSettings { + try_current_settings().expect("runtime settings are initialized before entry") +} + +pub fn http_settings() -> Result<HttpSettings, String> { + match try_current_settings() { + Some(settings) => settings.http(), + None => Settings::load_for_executable()?.http(), + } +} + +const fn default_http_header_bytes() -> i64 { + 16_384 +} + +const fn default_http_body_bytes() -> i64 { + 1_048_576 +} + +const fn default_http_timeout_ms() -> i64 { + 30_000 +} + +const fn default_http_connections() -> i64 { + 64 +} + +const fn default_tenant_max_pools() -> usize { + 64 +} + +const fn default_tenant_idle_timeout_ms() -> u64 { + 300_000 +} + +fn normalize_database( + name: &str, + database: DatabaseDocument, + executable_directory: &Path, +) -> Result<Database, String> { + match database { + DatabaseDocument::Sqlite { + path, + tenant_directory, + max_connections, + max_page_size, + } => { + validate_limit(name, "max_connections", max_connections, 1, 64)?; + validate_limit(name, "max_page_size", max_page_size, 1, 10_000)?; + validate_relative_path(name, &path)?; + if let Some(directory) = &tenant_directory { + validate_relative_database_path(name, "tenant_directory", directory)?; + } + Ok(Database::Sqlite { + path: executable_directory.join(path), + tenant_directory: tenant_directory.map(|path| executable_directory.join(path)), + max_connections, + max_page_size, + }) + } + DatabaseDocument::Postgres { + url, + tenant_database_prefix, + tls, + min_connections, + max_connections, + max_page_size, + wait_timeout_ms, + io_timeout_ms, + } => { + if url.is_empty() { + return Err(format!("database.{name}.url must not be empty")); + } + validate_limit(name, "max_connections", max_connections, 1, 256)?; + if min_connections > max_connections { + return Err(format!( + "database.{name}.min_connections must not exceed max_connections" + )); + } + validate_limit(name, "max_page_size", max_page_size, 1, 10_000)?; + validate_duration(name, "wait_timeout_ms", wait_timeout_ms)?; + validate_duration(name, "io_timeout_ms", io_timeout_ms)?; + if let Some(prefix) = &tenant_database_prefix + && (!valid_name(prefix) || prefix.len() > 43) + { + return Err(format!( + "database.{name}.tenant_database_prefix must use at most 43 lower_snake_case bytes" + )); + } + Ok(Database::Postgres { + url, + tenant_database_prefix, + tls: match tls { + TlsDocument::System => PostgresTls::System, + TlsDocument::Disabled => PostgresTls::Disabled, + }, + min_connections, + max_connections, + max_page_size, + wait_timeout_ms, + io_timeout_ms, + }) + } + } +} + +fn normalize_tenant( + document: Option<TenantDocument>, + databases: &BTreeMap<String, Database>, +) -> Result<Option<TenantSettings>, String> { + let Some(document) = document else { + if databases.values().any(|database| match database { + Database::Sqlite { + tenant_directory, .. + } => tenant_directory.is_some(), + Database::Postgres { + tenant_database_prefix, + .. + } => tenant_database_prefix.is_some(), + }) { + return Err("tenant database targets require top-level tenant settings".into()); + } + return Ok(None); + }; + if !valid_name(&document.database) { + return Err("tenant.database must use lower_snake_case".into()); + } + if !databases.contains_key(&document.database) { + return Err(format!( + "tenant.database references unknown database '{}'", + document.database + )); + } + if !(1..=1024).contains(&document.max_pools) { + return Err("tenant.max_pools must be between 1 and 1024".into()); + } + if !(1_000..=86_400_000).contains(&document.idle_timeout_ms) { + return Err("tenant.idle_timeout_ms must be between 1000 and 86400000".into()); + } + Ok(Some(TenantSettings { + database: document.database, + max_pools: document.max_pools, + idle_timeout_ms: document.idle_timeout_ms, + })) +} + +fn validate_limit( + connection: &str, + field: &str, + value: usize, + minimum: usize, + maximum: usize, +) -> Result<(), String> { + if (minimum..=maximum).contains(&value) { + Ok(()) + } else { + Err(format!( + "database.{connection}.{field} must be between {minimum} and {maximum}" + )) + } +} + +fn validate_relative_path(connection: &str, path: &Path) -> Result<(), String> { + validate_relative_database_path(connection, "path", path) +} + +fn validate_relative_database_path( + connection: &str, + field: &str, + path: &Path, +) -> Result<(), String> { + let valid = !path.as_os_str().is_empty() + && path + .components() + .all(|component| matches!(component, Component::Normal(_) | Component::CurDir)); + if valid { + Ok(()) + } else { + Err(format!( + "database.{connection}.{field} must stay below the executable directory" + )) + } +} + +fn valid_name(name: &str) -> bool { + name.starts_with(|character: char| character.is_ascii_lowercase()) + && name.split('_').all(|part| { + !part.is_empty() + && part + .chars() + .all(|character| character.is_ascii_lowercase() || character.is_ascii_digit()) + }) +} + +fn normalize_auth(document: AuthDocument) -> Result<AuthSettings, String> { + let mut providers = BTreeMap::new(); + for (key, provider) in document.providers { + compilation::validate_provider(&key, &provider.verify)?; + if !(32..=4096).contains(&provider.jwt_secret.len()) { + return Err(format!( + "auth.providers.{key}.jwtSecret must contain 32 to 4096 bytes" + )); + } + if !(60..=604_800).contains(&provider.ttl_seconds) { + return Err(format!( + "auth.providers.{key}.ttlSeconds must be between 60 and 604800" + )); + } + validate_cookie_token(&provider.cookie) + .map_err(|error| format!("auth.providers.{key}.cookie {error}"))?; + providers.insert( + key, + AuthProvider { + verify: provider.verify, + jwt_secret: crate::secret::Secret::from_input(provider.jwt_secret.into_bytes()), + ttl_seconds: provider.ttl_seconds, + cookie: provider.cookie, + }, + ); + } + Ok(AuthSettings { providers }) +} + +fn normalize_sites( + documents: BTreeMap<String, SiteDocument>, + auth: &AuthSettings, +) -> Result<BTreeMap<String, SiteSettings>, String> { + let mut sites = BTreeMap::new(); + for (key, site) in documents { + compilation::validate_site( + &key, + &site.path, + &site.auth, + auth.provider(&site.auth).is_some(), + )?; + let origin = site + .origin + .as_deref() + .map(normalize_http_origin) + .transpose() + .map_err(|error| format!("sites.{key}.origin: {error}"))?; + let mut hosts = Vec::with_capacity(site.hosts.len().max(1)); + for host in site.hosts { + let normalized = normalize_authority_host(&host) + .map_err(|error| format!("sites.{key}.hosts: {error}"))?; + if hosts.iter().any(|existing| existing == &normalized) { + return Err(format!( + "sites.{key}.hosts contains duplicate host '{normalized}'" + )); + } + hosts.push(normalized); + } + if let Some(origin) = &origin { + if hosts.is_empty() { + hosts.push(origin.host.clone()); + } else if !hosts.iter().any(|host| host == &origin.host) { + return Err(format!( + "sites.{key}.origin host must be present in sites.{key}.hosts" + )); + } + } + sites.insert( + key, + SiteSettings { + path: site.path, + auth: site.auth, + hosts, + origin, + }, + ); + } + + compilation::validate_site_paths( + sites + .iter() + .map(|(key, site)| (key.as_str(), site.path.as_str())), + )?; + Ok(sites) +} + +pub(crate) fn normalize_authority_host(authority: &str) -> Result<String, String> { + let authority = authority + .parse::<hyper::http::uri::Authority>() + .map_err(|_| "host must be an HTTP authority without a scheme or path".to_owned())?; + if authority.as_str().contains('@') { + return Err("host cannot contain credentials".into()); + } + Ok(authority.host().to_ascii_lowercase()) +} + +fn normalize_http_origin(value: &str) -> Result<HttpOrigin, String> { + let origin = value + .parse::<hyper::Uri>() + .map_err(|_| "must be an absolute HTTP or HTTPS origin".to_owned())?; + let scheme = origin + .scheme_str() + .filter(|scheme| matches!(*scheme, "http" | "https")) + .ok_or_else(|| "scheme must be http or https".to_owned())?; + if origin.path() != "/" || origin.query().is_some() { + return Err("must not contain a path, query, or fragment".into()); + } + let authority = origin + .authority() + .ok_or_else(|| "must contain a host".to_owned())?; + if authority.as_str().contains('@') { + return Err("cannot contain credentials".into()); + } + Ok(HttpOrigin { + scheme: scheme.to_owned(), + host: authority.host().to_ascii_lowercase(), + port: authority + .port_u16() + .unwrap_or(if scheme == "https" { 443 } else { 80 }), + }) +} + +fn validate_cookie_token(name: &str) -> Result<(), String> { + if name.is_empty() + || !name.bytes().all(|byte| { + byte.is_ascii_alphanumeric() + || matches!( + byte, + b'!' | b'#' + ..=b'\'' | b'*' | b'+' | b'-' | b'.' | b'^' | b'_' | b'`' | b'|' | b'~' + ) + }) + { + return Err("must be a valid Cookie name".into()); + } + Ok(()) +} + +const fn default_auth_ttl_seconds() -> u64 { + 86_400 +} + +fn default_auth_cookie() -> String { + "dever_session".into() +} + +const fn default_log_level() -> LogLevel { + LogLevel::Info +} + +const fn default_sqlite_connections() -> usize { + 1 +} + +const fn default_postgres_connections() -> usize { + 4 +} + +const fn default_page_size() -> usize { + 100 +} + +const fn default_postgres_wait_timeout_ms() -> u64 { + 5_000 +} + +const fn default_postgres_io_timeout_ms() -> u64 { + 30_000 +} + +fn validate_duration(connection: &str, field: &str, value: u64) -> Result<(), String> { + if (1..=300_000).contains(&value) { + Ok(()) + } else { + Err(format!( + "database.{connection}.{field} must be between 1 and 300000" + )) + } +} diff --git a/crates/dever-runtime/src/config/command.rs b/crates/dever-runtime/src/config/command.rs new file mode 100644 index 0000000..40ad32e --- /dev/null +++ b/crates/dever-runtime/src/config/command.rs @@ -0,0 +1,50 @@ +use super::{ADAPTER_FIELDS, Settings}; +use serde::Deserialize; + +#[derive(Clone, Copy, Debug, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct CommandLimits { + pub timeout_ms: u64, + pub output_limit: usize, +} + +impl Default for CommandLimits { + fn default() -> Self { + Self { + timeout_ms: 30_000, + output_limit: 1024 * 1024, + } + } +} + +impl Settings { + /// Deployment limits apply only to ordinary command Adapters. + pub fn command_limits(&self, identity: &str) -> Result<CommandLimits, String> { + let node = self + .adapter + .as_ref() + .map(|raw| crate::wire::parse(raw.get())) + .transpose()?; + let binding = node + .as_ref() + .map(|node| node.object()) + .transpose()? + .and_then(|fields| fields.get(identity)); + let command = binding + .map(|node| node.fields(ADAPTER_FIELDS)) + .transpose()? + .and_then(|fields| fields.get("command")); + let limits = command + .map(|node| serde_json::from_str::<CommandLimits>(node.raw())) + .transpose() + .map_err(|_| "Adapter command limits require timeout_ms and output_limit integers")? + .unwrap_or_default(); + if !(1..=3_600_000).contains(&limits.timeout_ms) { + return Err("Adapter command timeout_ms must be between 1 and 3600000".into()); + } + if !(1..=8 * 1024 * 1024).contains(&limits.output_limit) { + return Err("Adapter command output_limit must be between 1 and 8388608".into()); + } + Ok(limits) + } +} diff --git a/crates/dever-runtime/src/config/compilation.rs b/crates/dever-runtime/src/config/compilation.rs new file mode 100644 index 0000000..a4da1bb --- /dev/null +++ b/crates/dever-runtime/src/config/compilation.rs @@ -0,0 +1,232 @@ +//! The non-secret deployment contract consumed by the compiler. + +use super::{Database, RuntimeProfile, Settings}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct CompilationBindings { + pub databases: BTreeMap<String, DatabaseDriver>, + pub providers: BTreeMap<String, String>, + pub sites: BTreeMap<String, CompilationSite>, +} + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, Eq, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum DatabaseDriver { + Sqlite, + Postgres, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct CompilationSite { + pub path: String, + pub auth: String, +} + +impl Settings { + pub fn compilation_bindings(&self) -> CompilationBindings { + CompilationBindings { + databases: self + .databases + .iter() + .map(|(name, database)| { + ( + name.clone(), + match database { + Database::Sqlite { .. } => DatabaseDriver::Sqlite, + Database::Postgres { .. } => DatabaseDriver::Postgres, + }, + ) + }) + .collect(), + providers: self + .auth + .providers + .iter() + .map(|(key, provider)| (key.clone(), provider.verify.clone())) + .collect(), + sites: self + .sites + .iter() + .map(|(key, site)| { + ( + key.clone(), + CompilationSite { + path: site.path.clone(), + auth: site.auth.clone(), + }, + ) + }) + .collect(), + } + } +} + +impl CompilationBindings { + pub fn validate(&self) -> Result<(), String> { + if !self.databases.is_empty() && !self.databases.contains_key("default") { + return Err("database.default is required".into()); + } + for name in self.databases.keys() { + validate_database_name(name)?; + } + for (key, verify) in &self.providers { + validate_provider(key, verify)?; + } + for (key, site) in &self.sites { + validate_site( + key, + &site.path, + &site.auth, + self.providers.contains_key(&site.auth), + )?; + } + validate_site_paths( + self.sites + .iter() + .map(|(key, site)| (key.as_str(), site.path.as_str())), + ) + } + + pub fn site_for_directory( + &self, + directory: &[&str], + ) -> Result<Option<(&str, &CompilationSite)>, String> { + select_site(&self.sites, directory, |site| &site.path) + } + + pub fn resolve_database( + &self, + explicit: Option<&str>, + package_root: &str, + ) -> Result<(&str, &DatabaseDriver), String> { + resolve_database(&self.databases, explicit, package_root) + } + + pub fn profile(&self) -> RuntimeProfile { + RuntimeProfile { + sqlite: self + .databases + .values() + .any(|driver| *driver == DatabaseDriver::Sqlite), + postgres: self + .databases + .values() + .any(|driver| *driver == DatabaseDriver::Postgres), + } + } +} + +pub(super) fn resolve_database<'a, T>( + databases: &'a BTreeMap<String, T>, + explicit: Option<&str>, + package_root: &str, +) -> Result<(&'a str, &'a T), String> { + let name = match explicit { + Some(name) => name, + None if databases.contains_key(package_root) => package_root, + None => "default", + }; + databases + .get_key_value(name) + .map(|(name, database)| (name.as_str(), database)) + .ok_or_else(|| format!("database connection '{name}' is not configured")) +} + +pub(super) fn select_site<'a, T>( + sites: &'a BTreeMap<String, T>, + directory: &[&str], + path: impl Fn(&T) -> &str, +) -> Result<Option<(&'a str, &'a T)>, String> { + let mut matches = sites.iter().filter(|(_, site)| { + let path = path(site); + path.is_empty() + || path.split('/').count() <= directory.len() + && path + .split('/') + .zip(directory) + .all(|(expected, actual)| expected == *actual) + }); + let selected = matches.next(); + if matches.next().is_some() { + return Err("API directory matches multiple sites".into()); + } + Ok(selected.map(|(key, site)| (key.as_str(), site))) +} + +pub(super) fn validate_database_name(name: &str) -> Result<(), String> { + if !super::valid_name(name) { + return Err(format!( + "database connection name '{name}' must use lower_snake_case" + )); + } + Ok(()) +} + +pub(super) fn validate_provider(key: &str, verify: &str) -> Result<(), String> { + if !super::valid_name(key) { + return Err(format!("auth provider '{key}' must use lower_snake_case")); + } + let parts = verify.split('.').collect::<Vec<_>>(); + if parts.len() != 3 || parts.iter().any(|part| !super::valid_name(part)) { + return Err(format!( + "auth.providers.{key}.verify must be component.domain.function" + )); + } + Ok(()) +} + +pub(super) fn validate_site( + key: &str, + path: &str, + auth: &str, + provider_exists: bool, +) -> Result<(), String> { + if !super::valid_name(key) { + return Err(format!("site '{key}' must use lower_snake_case")); + } + if !path.is_empty() { + if path.starts_with('/') || path.ends_with('/') { + return Err(format!( + "sites.{key}.path must be relative to the api directory" + )); + } + if path.split('/').any(|part| !super::valid_name(part)) { + return Err(format!( + "sites.{key}.path segments must use lower_snake_case" + )); + } + } + if !provider_exists { + return Err(format!( + "sites.{key}.auth references unknown provider '{auth}'" + )); + } + Ok(()) +} + +pub(super) fn validate_site_paths<'a>( + sites: impl Iterator<Item = (&'a str, &'a str)>, +) -> Result<(), String> { + let mut entries = sites.collect::<Vec<_>>(); + entries.sort_by_key(|(_, path)| *path); + for adjacent in entries.windows(2) { + let [(left_key, left), (right_key, right)] = adjacent else { + unreachable!() + }; + if left.is_empty() + || left == right + || right + .strip_prefix(*left) + .is_some_and(|suffix| suffix.starts_with('/')) + { + return Err(format!( + "sites.{left_key}.path and sites.{right_key}.path overlap; site ownership must be unambiguous" + )); + } + } + Ok(()) +} diff --git a/crates/dever-runtime/src/cron.rs b/crates/dever-runtime/src/cron.rs new file mode 100644 index 0000000..5951831 --- /dev/null +++ b/crates/dever-runtime/src/cron.rs @@ -0,0 +1,114 @@ +//! Bounded, timezone-independent five-field UTC schedules shared with the compiler. +use chrono::{Datelike, Timelike}; + +#[derive(Clone, Debug)] +pub struct Cron { + fields: [u64; 5], + day_any: bool, + weekday_any: bool, +} + +impl Cron { + pub fn parse(text: &str) -> Result<Self, String> { + if text.len() > 512 { + return Err("cron exceeds 512 bytes".into()); + } + let parts = text.split_whitespace().collect::<Vec<_>>(); + if parts.len() != 5 { + return Err("cron requires five UTC fields".into()); + } + let mut fields = [0; 5]; + for (index, (min, max)) in [(0, 59), (0, 23), (1, 31), (1, 12), (0, 6)] + .into_iter() + .enumerate() + { + fields[index] = field(parts[index], min, max)?; + } + let day_any = fields[2] == ((1u64 << 32) - 2); + let weekday_any = fields[4] == 127; + if weekday_any && !day_any { + let possible = [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] + .into_iter() + .enumerate() + .any(|(month, days)| { + fields[3] & (1 << (month + 1)) != 0 + && fields[2] & ((1u64 << (days + 1)) - 2) != 0 + }); + if !possible { + return Err("cron has no possible UTC calendar date".into()); + } + } + Ok(Self { + fields, + day_any, + weekday_any, + }) + } + + pub fn fingerprint(&self) -> String { + format!( + "cron-v1:{:?}:{}:{}", + self.fields, self.day_any, self.weekday_any + ) + } + + pub fn matches_minute(&self, minute: i64) -> bool { + let Some(seconds) = minute.checked_mul(60) else { + return false; + }; + let Some(date) = chrono::DateTime::from_timestamp(seconds, 0) else { + return false; + }; + let matches = |index: usize, value: u32| self.fields[index] & (1u64 << value) != 0; + let day = matches(2, date.day()); + let weekday = matches(4, date.weekday().num_days_from_sunday()); + let calendar = match (self.day_any, self.weekday_any) { + (false, false) => day || weekday, + (true, false) => weekday, + (false, true) => day, + (true, true) => true, + }; + matches(0, date.minute()) && matches(1, date.hour()) && matches(3, date.month()) && calendar + } +} + +fn field(text: &str, minimum: u32, maximum: u32) -> Result<u64, String> { + let mut mask = 0; + for part in text.split(',') { + let (range, step) = match part.split_once('/') { + Some((range, step)) => (range, number(step, 1, maximum + 1)?), + None => (part, 1), + }; + let (first, last) = if range == "*" { + (minimum, maximum) + } else if let Some((first, last)) = range.split_once('-') { + ( + number(first, minimum, maximum)?, + number(last, minimum, maximum)?, + ) + } else { + let value = number(range, minimum, maximum)?; + (value, if part.contains('/') { maximum } else { value }) + }; + if first > last { + return Err("cron range is reversed".into()); + } + for value in (first..=last).step_by(step as usize) { + mask |= 1u64 << value; + } + } + if mask == 0 { + return Err("empty cron field".into()); + } + Ok(mask) +} + +fn number(text: &str, minimum: u32, maximum: u32) -> Result<u32, String> { + if text.is_empty() || !text.bytes().all(|byte| byte.is_ascii_digit()) { + return Err("cron fields require unsigned decimal numbers".into()); + } + text.parse::<u32>() + .ok() + .filter(|value| (minimum..=maximum).contains(value)) + .ok_or_else(|| "cron field is outside its allowed range".into()) +} diff --git a/crates/dever-runtime/src/crypto.rs b/crates/dever-runtime/src/crypto.rs new file mode 100644 index 0000000..23ff11c --- /dev/null +++ b/crates/dever-runtime/src/crypto.rs @@ -0,0 +1,74 @@ +use argon2::password_hash::{Error, SaltString}; +use argon2::{Algorithm, Argon2, Params, PasswordHash, PasswordHasher, PasswordVerifier, Version}; +use hmac::{Hmac, Mac}; +use sha2::{Digest, Sha256}; +use subtle::ConstantTimeEq; +use zeroize::Zeroizing; + +use crate::{bytes::Bytes, secret::Secret}; + +const MAX_TOKEN_BYTES: i64 = 1024; +const INVALID_HASH: &str = "invalid or unsupported password hash"; + +pub fn token(length: i64) -> Result<Secret, String> { + if !(16..=MAX_TOKEN_BYTES).contains(&length) { + return Err("token length must be between 16 and 1024 bytes".into()); + } + let mut bytes = Zeroizing::new(vec![0; length as usize]); + getrandom::getrandom(&mut bytes).map_err(|_| "secure random source unavailable")?; + Ok(Secret(bytes)) +} + +pub fn password_hash(password: &Secret) -> Result<String, String> { + let mut salt = [0u8; 16]; + getrandom::getrandom(&mut salt).map_err(|_| "secure random source unavailable")?; + let salt = SaltString::encode_b64(&salt).map_err(|_| "password hashing failed")?; + Argon2::new(Algorithm::Argon2id, Version::V0x13, Params::default()) + .hash_password(&password.0, &salt) + .map(|hash| hash.to_string()) + .map_err(|_| "password hashing failed".into()) +} + +pub fn password_verify(password: &Secret, encoded: &str) -> Result<bool, String> { + // Bound parsing and resource costs before invoking the password verifier: + // that verifier otherwise accepts PHC-controlled Argon2 parameters. + if encoded.len() > 256 { + return Err(INVALID_HASH.into()); + } + let hash = PasswordHash::new(encoded).map_err(|_| INVALID_HASH)?; + let params = Params::try_from(&hash).map_err(|_| INVALID_HASH)?; + let limits = Params::default(); + if hash.algorithm.as_str() != "argon2id" + || hash.version != Some(19) + || hash.salt.is_none() + || hash.hash.is_none() + || params.m_cost() > limits.m_cost() + || params.t_cost() > limits.t_cost() + || params.p_cost() > limits.p_cost() + || params.output_len().unwrap_or(Params::DEFAULT_OUTPUT_LEN) > Params::DEFAULT_OUTPUT_LEN + || !params.keyid().is_empty() + || !params.data().is_empty() + { + return Err(INVALID_HASH.into()); + } + match Argon2::default().verify_password(&password.0, &hash) { + Ok(()) => Ok(true), + Err(Error::Password) => Ok(false), + Err(_) => Err(INVALID_HASH.into()), + } +} + +pub fn sha256(bytes: &Bytes) -> Bytes { + Bytes::new(Sha256::digest(bytes.values()).to_vec()) +} + +pub fn hmac_sha256(key: &Secret, bytes: &Bytes) -> Bytes { + let mut mac = Hmac::<Sha256>::new_from_slice(&key.0).expect("HMAC accepts any key length"); + mac.update(bytes.values()); + Bytes::new(mac.finalize().into_bytes().to_vec()) +} + +/// Content comparison is constant-time for equal lengths; lengths are public. +pub fn constant_time_eq(left: &Bytes, right: &Bytes) -> bool { + left.values().ct_eq(right.values()).into() +} diff --git a/crates/dever-runtime/src/database.rs b/crates/dever-runtime/src/database.rs new file mode 100644 index 0000000..f176a67 --- /dev/null +++ b/crates/dever-runtime/src/database.rs @@ -0,0 +1,969 @@ +use crate::config::Database as DatabaseConfig; +use crate::orm::{Error, ErrorKind, Row, RowStream, Value}; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; + +#[derive(Clone)] +pub struct Database { + key: DatabaseKey, + backend: Backend, +} + +/// One application entry's database owners. Native process roots retain their +/// existing global initialization; repeated embedded roots own this session. +pub struct Session { + settings: Arc<crate::config::Settings>, + databases: std::collections::BTreeMap<String, Database>, +} + +impl Session { + pub fn load( + bundled: crate::config::RuntimeProfile, + bindings: &[(Option<&str>, &str, bool)], + transactions: &[&[(Option<&str>, &str, bool)]], + ) -> Result<Self, String> { + let settings = crate::config::Settings::load_for_executable()?; + settings.validate_database_entry()?; + Self::from_settings(Arc::new(settings), bundled, bindings, transactions) + } + + pub fn from_settings( + settings: Arc<crate::config::Settings>, + bundled: crate::config::RuntimeProfile, + bindings: &[(Option<&str>, &str, bool)], + transactions: &[&[(Option<&str>, &str, bool)]], + ) -> Result<Self, String> { + settings.prepare_databases(bundled, bindings, transactions)?; + let mut databases = std::collections::BTreeMap::new(); + #[cfg(feature = "sqlite")] + for (name, database) in crate::sqlite::configured_databases(&settings)? { + databases.insert( + name.clone(), + Database { + key: DatabaseKey { + connection: name, + tenant_id: None, + }, + backend: Backend::Sqlite(database), + }, + ); + } + #[cfg(feature = "postgres")] + for (name, database) in crate::postgres::configured_databases(&settings)? { + databases.insert( + name.clone(), + Database { + key: DatabaseKey { + connection: name, + tenant_id: None, + }, + backend: Backend::Postgres(database), + }, + ); + } + Ok(Self { + settings, + databases, + }) + } + + pub fn database(&self, explicit: Option<&str>, root: &str) -> Result<Database, Error> { + let (name, _) = self + .settings + .resolve_database(explicit, root) + .map_err(|message| Error::new(ErrorKind::Pool, message))?; + self.databases.get(name).cloned().ok_or_else(|| { + Error::new( + ErrorKind::Pool, + format!("database '{name}' is not initialized"), + ) + }) + } + + pub async fn prepare(&self) -> Result<(), Error> { + for database in self.databases.values() { + database.prepare_pool().await?; + } + Ok(()) + } + + /// Called after the entry's structured scope has drained, before its + /// runtime is destroyed. Stop every pool before awaiting any driver. + pub async fn close(&self) -> Result<(), Error> { + self.stop_accepting(); + let mut failure: Option<Error> = None; + for database in self.databases.values() { + if let Err(error) = database.close_pool().await { + failure = Some(match failure { + Some(previous) => previous.cause(error), + None => error, + }); + } + } + failure.map_or(Ok(()), Err) + } + + pub(crate) fn stop_accepting(&self) { + for database in self.databases.values() { + database.stop_accepting(); + } + } +} + +#[derive(Clone)] +enum Backend { + #[cfg(feature = "sqlite")] + Sqlite(Arc<crate::sqlite::Database>), + #[cfg(feature = "postgres")] + Postgres(Arc<crate::postgres::Database>), +} + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub struct DatabaseKey { + connection: String, + tenant_id: Option<i64>, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ModelScope { + Global, + Tenant, +} + +#[derive(Clone, Copy)] +pub struct StorageBinding { + explicit: Option<&'static str>, + package_root: &'static str, + scope: ModelScope, +} + +impl StorageBinding { + pub const fn new( + explicit: Option<&'static str>, + package_root: &'static str, + scope: ModelScope, + ) -> Self { + Self { + explicit, + package_root, + scope, + } + } + + pub async fn database(&self) -> Result<Database, Error> { + let settings = crate::config::current_settings(); + if self.scope == ModelScope::Global || settings.tenant().is_none() { + return database_for(self.explicit, self.package_root); + } + let (connection, _) = settings + .resolve_database(self.explicit, self.package_root) + .map_err(|error| Error::new(ErrorKind::Pool, error))?; + crate::tenant::database(connection).await + } + + pub(crate) fn selector(&self) -> (Option<&str>, &str) { + (self.explicit, self.package_root) + } +} + +impl DatabaseKey { + pub fn connection(&self) -> &str { + &self.connection + } + pub fn tenant_id(&self) -> Option<i64> { + self.tenant_id + } +} + +pub enum Transaction { + #[cfg(feature = "sqlite")] + Sqlite { + key: DatabaseKey, + lease: Arc<crate::sqlite::Database>, + transaction: crate::sqlite::Transaction, + }, + #[cfg(feature = "postgres")] + Postgres { + key: DatabaseKey, + lease: Arc<crate::postgres::Database>, + transaction: crate::postgres::Transaction, + }, +} + +/// An internal shared transaction owner for foreign coroutine frames. Sequential +/// operations borrow the same transaction; finish takes it exactly once. No +/// task-local ambient transaction is installed or inherited by spawned tasks. +pub struct TransactionState(tokio::sync::Mutex<Option<Transaction>>); + +pub struct TransactionLease<'a> { + transaction: tokio::sync::MutexGuard<'a, Option<Transaction>>, + completed: bool, +} + +impl std::ops::Deref for TransactionLease<'_> { + type Target = Transaction; + fn deref(&self) -> &Self::Target { + self.transaction + .as_ref() + .expect("validated live transaction lease") + } +} + +impl TransactionLease<'_> { + /// The driver returned a known result. Cancellation before this point must + /// discard the shared transaction even if another frame still owns an alias. + pub fn complete(&mut self) { + self.completed = true; + } +} + +impl Drop for TransactionLease<'_> { + fn drop(&mut self) { + if !self.completed { + self.transaction.take(); + } + } +} + +impl TransactionState { + pub fn new(transaction: Transaction) -> Self { + Self(tokio::sync::Mutex::new(Some(transaction))) + } + + pub async fn borrow(state: Option<&Self>) -> Result<Option<TransactionLease<'_>>, Error> { + let Some(state) = state else { + return Ok(None); + }; + let guard = state.0.lock().await; + if guard.is_none() { + return Err(Error::new( + ErrorKind::Database, + "transaction was already finished", + )); + } + Ok(Some(TransactionLease { + transaction: guard, + completed: false, + })) + } + + pub async fn finish(&self, commit: bool) -> Result<(), Error> { + let transaction = + self.0.lock().await.take().ok_or_else(|| { + Error::new(ErrorKind::Database, "transaction was already finished") + })?; + if commit { + transaction.commit().await + } else { + transaction.rollback().await + } + } +} + +pub enum Executor<'a> { + Pool(Database), + #[cfg(feature = "sqlite")] + Sqlite(&'a crate::sqlite::Transaction), + #[cfg(feature = "postgres")] + Postgres(&'a crate::postgres::Transaction), +} + +pub struct Sql { + pub sqlite: &'static str, + pub postgres: &'static str, +} + +pub struct OwnedSql { + pub sqlite: String, + pub postgres: String, +} + +pub fn relation_sql( + prefix: &'static str, + before_limit: &'static str, + suffix: &'static str, + ids: usize, +) -> OwnedSql { + let sqlite = (1..=ids) + .map(|index| format!("?{index}")) + .collect::<Vec<_>>() + .join(", "); + let postgres = (1..=ids) + .map(|index| format!("${index}")) + .collect::<Vec<_>>() + .join(", "); + OwnedSql { + sqlite: format!("{prefix}{sqlite}{before_limit}?{}{suffix}", ids + 1), + postgres: format!("{prefix}{postgres}{before_limit}${}{suffix}", ids + 1), + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct Field { + pub name: String, + pub ty: String, + pub nullable: bool, + pub generated: bool, + pub default: Option<String>, + pub rename_from: Option<String>, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct Index { + pub name: String, + pub fields: Vec<String>, + pub unique: bool, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct Migration { + pub name: String, + pub revision: String, + pub drops: Vec<String>, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct Schema { + pub model: String, + pub table: String, + pub revision: String, + pub seed_revision: String, + pub fields: Vec<Field>, + pub indexes: Vec<Index>, + pub migrations: Vec<Migration>, +} + +pub(crate) fn validate_migration_order(previous: &Schema, desired: &Schema) -> Result<(), Error> { + let previous_names = previous + .migrations + .iter() + .map(|migration| migration.name.as_str()) + .collect::<Vec<_>>(); + let retained = desired + .migrations + .iter() + .map(|migration| migration.name.as_str()) + .filter(|name| previous_names.contains(name)) + .collect::<Vec<_>>(); + if retained != previous_names { + return Err(Error::migration(format!( + "applied migrations for '{}' changed order", + desired.model, + ))); + } + Ok(()) +} + +pub struct SchemaSql { + pub sqlite: String, + pub postgres: String, +} + +pub struct Seed { + pub sql: Sql, + pub parameters: Vec<Value>, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum MigrationPhase { + Before, + After, +} + +pub struct DataMigration { + pub name: String, + pub phase: MigrationPhase, + pub sql: Sql, + pub parameters: Vec<Value>, +} + +pub struct PostgresConstraint { + pub name: String, + pub definition: String, + pub foreign_key: bool, +} + +pub struct PostgresColumn { + pub name: String, + pub definition: String, + pub sql_type: String, + pub default: Option<String>, +} + +pub struct Model { + pub schema: Schema, + pub create_table: SchemaSql, + pub create_temporary_table: String, + pub create_indexes: Vec<SchemaSql>, + pub postgres_columns: Vec<PostgresColumn>, + pub postgres_constraints: Vec<PostgresConstraint>, + pub seeds: Vec<Seed>, + pub data_migrations: Vec<DataMigration>, +} + +impl Database { + pub fn name(&self) -> &str { + self.key.connection() + } + + pub fn key(&self) -> &DatabaseKey { + &self.key + } + + pub fn max_page_size(&self) -> usize { + match &self.backend { + #[cfg(feature = "sqlite")] + Backend::Sqlite(database) => database.max_page_size(), + #[cfg(feature = "postgres")] + Backend::Postgres(database) => database.max_page_size(), + } + } + + pub async fn begin(&self) -> Result<Transaction, Error> { + match &self.backend { + #[cfg(feature = "sqlite")] + Backend::Sqlite(database) => Ok(Transaction::Sqlite { + key: self.key.clone(), + lease: Arc::clone(database), + transaction: database.begin().await?, + }), + #[cfg(feature = "postgres")] + Backend::Postgres(database) => Ok(Transaction::Postgres { + key: self.key.clone(), + lease: Arc::clone(database), + transaction: database.begin().await?, + }), + } + } + + pub async fn execute(&self, sql: Sql, parameters: Vec<Value>) -> Result<usize, Error> { + match &self.backend { + #[cfg(feature = "sqlite")] + Backend::Sqlite(database) => database.execute(sql.sqlite, parameters).await, + #[cfg(feature = "postgres")] + Backend::Postgres(database) => database.execute(sql.postgres, parameters).await, + } + } + + pub async fn query(&self, sql: Sql, parameters: Vec<Value>) -> Result<Vec<Row>, Error> { + match &self.backend { + #[cfg(feature = "sqlite")] + Backend::Sqlite(database) => database.query(sql.sqlite, parameters).await, + #[cfg(feature = "postgres")] + Backend::Postgres(database) => database.query(sql.postgres, parameters).await, + } + } + + pub async fn query_bounded( + &self, + sql: Sql, + parameters: Vec<Value>, + maximum: usize, + ) -> Result<Vec<Row>, Error> { + match &self.backend { + #[cfg(feature = "sqlite")] + Backend::Sqlite(database) => { + database + .query_bounded(sql.sqlite, parameters, maximum) + .await + } + #[cfg(feature = "postgres")] + Backend::Postgres(database) => { + database + .query_bounded(sql.postgres, parameters, maximum) + .await + } + } + } + + pub async fn query_owned( + &self, + sql: OwnedSql, + parameters: Vec<Value>, + ) -> Result<Vec<Row>, Error> { + match &self.backend { + #[cfg(feature = "sqlite")] + Backend::Sqlite(database) => database.query_owned(sql.sqlite, parameters).await, + #[cfg(feature = "postgres")] + Backend::Postgres(database) => database.query_owned(sql.postgres, parameters).await, + } + } + + pub async fn stream( + &self, + sql: Sql, + parameters: Vec<Value>, + capacity: usize, + ) -> Result<RowStream<Row>, Error> { + let stream = match &self.backend { + #[cfg(feature = "sqlite")] + Backend::Sqlite(database) => database.stream(sql.sqlite, parameters, capacity).await?, + #[cfg(feature = "postgres")] + Backend::Postgres(database) => { + database.stream(sql.postgres, parameters, capacity).await? + } + }; + Ok(stream.hold_database(self.lease_owner())) + } + + pub async fn migrate(&self, model: Model) -> Result<(), Error> { + match &self.backend { + #[cfg(feature = "sqlite")] + Backend::Sqlite(database) => database.migrate(model).await, + #[cfg(feature = "postgres")] + Backend::Postgres(database) => database.migrate(model).await, + } + } + + pub async fn migrate_foreign_keys( + &self, + model: String, + table: String, + constraints: Vec<PostgresConstraint>, + ) -> Result<(), Error> { + let _ = (&model, &table, &constraints); + match &self.backend { + #[cfg(feature = "sqlite")] + Backend::Sqlite(_) => Ok(()), + #[cfg(feature = "postgres")] + Backend::Postgres(database) => { + database + .migrate_foreign_keys(model, table, constraints) + .await + } + } + } + + fn lease_owner(&self) -> Arc<dyn Send + Sync> { + match &self.backend { + #[cfg(feature = "sqlite")] + Backend::Sqlite(database) => Arc::clone(database) as Arc<dyn Send + Sync>, + #[cfg(feature = "postgres")] + Backend::Postgres(database) => Arc::clone(database) as Arc<dyn Send + Sync>, + } + } + + pub(crate) fn exclusively_held(&self) -> bool { + match &self.backend { + #[cfg(feature = "sqlite")] + Backend::Sqlite(database) => Arc::strong_count(database) == 1, + #[cfg(feature = "postgres")] + Backend::Postgres(database) => Arc::strong_count(database) == 1, + } + } + + pub(crate) async fn prepare_pool(&self) -> Result<(), Error> { + match &self.backend { + #[cfg(feature = "sqlite")] + Backend::Sqlite(_) => Ok(()), + #[cfg(feature = "postgres")] + Backend::Postgres(database) => database.prepare().await, + } + } + + pub(crate) async fn close_pool(&self) -> Result<(), Error> { + match &self.backend { + #[cfg(feature = "sqlite")] + Backend::Sqlite(database) => { + database.close(); + Ok(()) + } + #[cfg(feature = "postgres")] + Backend::Postgres(database) => database.close().await, + } + } + + pub(crate) fn stop_accepting(&self) { + match &self.backend { + #[cfg(feature = "sqlite")] + Backend::Sqlite(database) => database.close(), + #[cfg(feature = "postgres")] + Backend::Postgres(database) => database.stop_accepting(), + } + } +} + +impl Transaction { + pub fn key(&self) -> &DatabaseKey { + match self { + #[cfg(feature = "sqlite")] + Self::Sqlite { key, .. } => key, + #[cfg(feature = "postgres")] + Self::Postgres { key, .. } => key, + } + } + + pub async fn commit(self) -> Result<(), Error> { + match self { + #[cfg(feature = "sqlite")] + Self::Sqlite { + transaction, lease, .. + } => { + let result = transaction.commit().await; + drop(lease); + result + } + #[cfg(feature = "postgres")] + Self::Postgres { + transaction, lease, .. + } => { + let result = transaction.commit().await; + drop(lease); + result + } + } + } + + pub async fn rollback(self) -> Result<(), Error> { + match self { + #[cfg(feature = "sqlite")] + Self::Sqlite { + transaction, lease, .. + } => { + let result = transaction.rollback().await; + drop(lease); + result + } + #[cfg(feature = "postgres")] + Self::Postgres { + transaction, lease, .. + } => { + let result = transaction.rollback().await; + drop(lease); + result + } + } + } +} + +impl<'a> Executor<'a> { + pub fn new(database: Database, transaction: Option<&'a Transaction>) -> Result<Self, Error> { + let Some(transaction) = transaction else { + return Ok(Self::Pool(database)); + }; + if transaction.key() != database.key() { + return Err(Error::new( + ErrorKind::Database, + format!( + "transaction for database key {:?} cannot access database key {:?}", + transaction.key(), + database.key() + ), + )); + } + match (&database.backend, transaction) { + #[cfg(feature = "sqlite")] + (Backend::Sqlite(_), Transaction::Sqlite { transaction, .. }) => { + Ok(Self::Sqlite(transaction)) + } + #[cfg(feature = "postgres")] + (Backend::Postgres(_), Transaction::Postgres { transaction, .. }) => { + Ok(Self::Postgres(transaction)) + } + #[allow(unreachable_patterns)] + _ => Err(Error::new( + ErrorKind::Database, + "transaction database driver does not match the Model driver", + )), + } + } + + pub fn max_page_size(&self) -> usize { + match self { + Self::Pool(database) => database.max_page_size(), + #[cfg(feature = "sqlite")] + Self::Sqlite(transaction) => { + crate::sqlite::Executor::Transaction(transaction).max_page_size() + } + #[cfg(feature = "postgres")] + Self::Postgres(transaction) => { + crate::postgres::Executor::Transaction(transaction).max_page_size() + } + } + } + + pub async fn execute(&self, sql: Sql, parameters: Vec<Value>) -> Result<usize, Error> { + match self { + Self::Pool(database) => database.execute(sql, parameters).await, + #[cfg(feature = "sqlite")] + Self::Sqlite(transaction) => { + crate::sqlite::Executor::Transaction(transaction) + .execute(sql.sqlite, parameters) + .await + } + #[cfg(feature = "postgres")] + Self::Postgres(transaction) => { + crate::postgres::Executor::Transaction(transaction) + .execute(sql.postgres, parameters) + .await + } + } + } + + pub async fn query(&self, sql: Sql, parameters: Vec<Value>) -> Result<Vec<Row>, Error> { + match self { + Self::Pool(database) => database.query(sql, parameters).await, + #[cfg(feature = "sqlite")] + Self::Sqlite(transaction) => { + crate::sqlite::Executor::Transaction(transaction) + .query(sql.sqlite, parameters) + .await + } + #[cfg(feature = "postgres")] + Self::Postgres(transaction) => { + crate::postgres::Executor::Transaction(transaction) + .query(sql.postgres, parameters) + .await + } + } + } + + pub async fn query_bounded( + &self, + sql: Sql, + parameters: Vec<Value>, + maximum: usize, + ) -> Result<Vec<Row>, Error> { + match self { + Self::Pool(database) => database.query_bounded(sql, parameters, maximum).await, + #[cfg(feature = "sqlite")] + Self::Sqlite(transaction) => { + transaction + .query_bounded(sql.sqlite, parameters, maximum) + .await + } + #[cfg(feature = "postgres")] + Self::Postgres(transaction) => { + transaction + .query_bounded(sql.postgres, parameters, maximum) + .await + } + } + } + + pub async fn query_owned( + &self, + sql: OwnedSql, + parameters: Vec<Value>, + ) -> Result<Vec<Row>, Error> { + match self { + Self::Pool(database) => database.query_owned(sql, parameters).await, + #[cfg(feature = "sqlite")] + Self::Sqlite(transaction) => transaction.query_owned(sql.sqlite, parameters).await, + #[cfg(feature = "postgres")] + Self::Postgres(transaction) => transaction.query_owned(sql.postgres, parameters).await, + } + } + + pub async fn stream( + &self, + sql: Sql, + parameters: Vec<Value>, + capacity: usize, + ) -> Result<RowStream<Row>, Error> { + match self { + Self::Pool(database) => database.stream(sql, parameters, capacity).await, + #[cfg(feature = "sqlite")] + Self::Sqlite(_) => Err(Error::new( + ErrorKind::Database, + "row streams cannot escape a transaction", + )), + #[cfg(feature = "postgres")] + Self::Postgres(_) => Err(Error::new( + ErrorKind::Database, + "row streams cannot escape a transaction", + )), + } + } +} + +pub fn database_for(explicit: Option<&str>, package_root: &str) -> Result<Database, Error> { + #[cfg(feature = "api")] + if let Some(session) = crate::application::current() { + return session.databases().database(explicit, package_root); + } + let (name, config) = crate::config::settings() + .resolve_database(explicit, package_root) + .map_err(|error| Error::new(ErrorKind::Pool, error))?; + let backend = match config { + DatabaseConfig::Sqlite { .. } => { + #[cfg(feature = "sqlite")] + { + Backend::Sqlite(crate::sqlite::database(name)?) + } + #[cfg(not(feature = "sqlite"))] + { + return Err(Error::new(ErrorKind::Pool, "SQLite support is not bundled")); + } + } + DatabaseConfig::Postgres { .. } => { + #[cfg(feature = "postgres")] + { + Backend::Postgres(crate::postgres::database(name)?) + } + #[cfg(not(feature = "postgres"))] + { + return Err(Error::new( + ErrorKind::Pool, + "PostgreSQL support is not bundled", + )); + } + } + }; + Ok(Database { + key: DatabaseKey { + connection: name.to_owned(), + tenant_id: None, + }, + backend, + }) +} + +pub(crate) async fn tenant_database( + connection: &str, + tenant_id: i64, + create: bool, +) -> Result<Database, Error> { + let settings = crate::config::current_settings(); + let config = settings.database(connection).ok_or_else(|| { + Error::new( + ErrorKind::Pool, + format!("database connection '{connection}' is not configured"), + ) + })?; + let backend = match config { + DatabaseConfig::Sqlite { + tenant_directory: Some(directory), + max_connections, + max_page_size, + .. + } => { + #[cfg(not(feature = "sqlite"))] + let _ = (directory, max_connections, max_page_size, create); + #[cfg(feature = "sqlite")] + { + if create { + std::fs::create_dir_all(directory).map_err(|error| { + Error::migration(format!( + "cannot create tenant database directory: {error}" + )) + })?; + } + let path = directory.join(format!("tenant_{tenant_id}.db")); + if !create && !path.is_file() { + return Err(Error::migration("tenant database is not provisioned")); + } + Backend::Sqlite(Arc::new(crate::sqlite::Database::open( + path, + *max_connections, + *max_page_size, + )?)) + } + #[cfg(not(feature = "sqlite"))] + { + return Err(Error::new(ErrorKind::Pool, "SQLite support is not bundled")); + } + } + DatabaseConfig::Postgres { + url, + tenant_database_prefix: Some(prefix), + tls, + min_connections, + max_connections, + max_page_size, + wait_timeout_ms, + io_timeout_ms, + .. + } => { + #[cfg(not(feature = "postgres"))] + let _ = ( + url, + prefix, + tls, + min_connections, + max_connections, + max_page_size, + wait_timeout_ms, + io_timeout_ms, + create, + ); + #[cfg(feature = "postgres")] + { + let database_name = format!("{prefix}_{tenant_id}"); + if create { + crate::postgres::create_database( + url, + &database_name, + *tls, + std::time::Duration::from_millis(*wait_timeout_ms), + std::time::Duration::from_millis(*io_timeout_ms), + ) + .await?; + } + Backend::Postgres(Arc::new(crate::postgres::Database::open( + crate::postgres::ConnectionOptions { + url, + database_name: Some(&database_name), + tls: *tls, + min_connections: *min_connections, + max_connections: *max_connections, + max_page_size: *max_page_size, + wait_timeout: std::time::Duration::from_millis(*wait_timeout_ms), + io_timeout: std::time::Duration::from_millis(*io_timeout_ms), + }, + )?)) + } + #[cfg(not(feature = "postgres"))] + { + return Err(Error::new( + ErrorKind::Pool, + "PostgreSQL support is not bundled", + )); + } + } + DatabaseConfig::Sqlite { + tenant_directory: None, + .. + } => { + return Err(Error::new( + ErrorKind::Pool, + format!( + "database.{connection}.tenant_directory is required for tenant database mode" + ), + )); + } + DatabaseConfig::Postgres { + tenant_database_prefix: None, + .. + } => { + return Err(Error::new( + ErrorKind::Pool, + format!( + "database.{connection}.tenant_database_prefix is required for tenant database mode" + ), + )); + } + }; + Ok(Database { + key: DatabaseKey { + connection: connection.to_owned(), + tenant_id: Some(tenant_id), + }, + backend, + }) +} + +pub async fn prepare() -> Result<(), Error> { + #[cfg(feature = "postgres")] + crate::postgres::prepare().await?; + Ok(()) +} + +pub async fn shutdown() -> Result<(), Error> { + crate::tenant::shutdown().await?; + #[cfg(feature = "sqlite")] + crate::sqlite::shutdown(); + #[cfg(feature = "postgres")] + crate::postgres::shutdown().await?; + Ok(()) +} diff --git a/crates/dever-runtime/src/endpoint.rs b/crates/dever-runtime/src/endpoint.rs new file mode 100644 index 0000000..f181369 --- /dev/null +++ b/crates/dever-runtime/src/endpoint.rs @@ -0,0 +1,51 @@ +use std::future::Future; +use std::sync::{Arc, Mutex}; +use tokio::sync::Semaphore; + +#[derive(Debug)] +pub(crate) struct Endpoint<T> { + socket: Mutex<Option<Arc<T>>>, + closed: Semaphore, +} + +impl<T> Endpoint<T> { + pub(crate) fn new(socket: T) -> Self { + Self { + socket: Mutex::new(Some(Arc::new(socket))), + closed: Semaphore::new(0), + } + } + + pub(crate) fn get(&self) -> Result<Arc<T>, String> { + self.socket + .lock() + .expect("endpoint lock poisoned") + .clone() + .ok_or_else(|| "resource is closed".into()) + } + + pub(crate) fn close(&self) -> Result<(), String> { + self.take().map(|_| ()) + } + + pub(crate) fn take(&self) -> Result<Arc<T>, String> { + let socket = self.socket.lock().expect("endpoint lock poisoned").take(); + self.closed.close(); + socket.ok_or_else(|| "resource is closed".into()) + } + + pub(crate) async fn while_open<R>( + &self, + operation: impl Future<Output = Result<R, String>>, + ) -> Result<R, String> { + tokio::select! { + biased; + _ = self.closed.acquire() => Err("resource is closed".into()), + result = operation => result, + } + } + + pub(crate) async fn closed(&self) { + let _closed = self.closed.acquire().await; + } +} diff --git a/crates/dever-runtime/src/external.rs b/crates/dever-runtime/src/external.rs new file mode 100644 index 0000000..4eb0924 --- /dev/null +++ b/crates/dever-runtime/src/external.rs @@ -0,0 +1,701 @@ +//! Safe extraction primitives for compiler-owned external Worker resources. +//! +//! The packager supplies verified bytes; this module never downloads, searches +//! PATH, follows links, or executes a resource. A caller can therefore use +//! the same primitive for an embedded resource table and for a machine-cache +//! hit without creating a writable shared execution directory. + +use std::fs::{self, File, OpenOptions}; +use std::io::{self, Read, Write}; +use std::path::{Component, Path, PathBuf}; + +use serde::Deserialize; +use sha2::{Digest, Sha256}; + +#[derive(Clone, Copy, Debug)] +pub struct Resource<'a> { + pub path: &'a str, + pub bytes: &'a [u8], + pub sha256: &'a str, + pub executable: bool, +} + +pub(crate) struct PreparedBundle { + directory: PathBuf, + resources: Vec<Resource<'static>>, +} + +pub fn prepare(root: &Path, digest: &str, resources: &[Resource<'static>]) -> Result<(), String> { + let session = + crate::component::current().ok_or("external resources require a Dever invocation")?; + let directory = extract(root, digest, resources)?; + session + .settings + .get_or_init(|| { + crate::config::Settings::load_adapter_settings_at(root).map(std::sync::Arc::new) + }) + .as_ref() + .map_err(Clone::clone)?; + session + .resources + .set(PreparedBundle { + directory, + resources: resources.to_vec(), + }) + .map_err(|_| "external resources were initialized twice".into()) +} + +/// A managed Worker never obtains an interpreter from PATH. Its executable, +/// arguments and working directory all come from its verified resource tree. +#[derive(Debug)] +pub struct WorkerLaunch { + pub executable: PathBuf, + pub arguments: Vec<std::ffi::OsString>, + pub working_directory: PathBuf, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct LaunchManifest { + format: String, + ecosystem: String, + entry: String, + executable: String, + arguments: Vec<LaunchArgument>, + working_directory: String, +} + +#[derive(Deserialize)] +#[serde(untagged)] +enum LaunchArgument { + Resource(ResourceArgument), + Literal(LiteralArgument), +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ResourceArgument { + resource: String, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct LiteralArgument { + literal: String, +} + +pub fn worker_command( + definition: &crate::component::Definition, +) -> Result<std::process::Command, String> { + sandbox_launch(definition, |launch| launch.command()) +} + +pub(crate) fn observed_command( + definition: &crate::component::Definition, +) -> Result<(std::process::Command, dever_sandbox::CommandObservation), String> { + sandbox_launch(definition, |launch| launch.observed_command()) +} + +fn sandbox_launch<T>( + definition: &crate::component::Definition, + build: impl FnOnce(dever_sandbox::Launch<'_>) -> Result<T, String>, +) -> Result<T, String> { + let session = + crate::component::current().ok_or("external resources require a Dever invocation")?; + let bundle = session + .resources + .get() + .ok_or("external Worker resources and sandbox were not prepared")?; + let launch = verified_worker_launch( + &bundle.directory, + &bundle.resources, + &definition.entry, + &definition.ecosystem, + )?; + let settings = crate::component::adapter_settings()? + .external_grants(&definition.port, &definition.capabilities)?; + if definition + .capabilities + .iter() + .any(|capability| capability == "gpu") + { + return Err("GPU Worker requires a supported device isolation backend".into()); + } + let arguments = launch.arguments; + build(dever_sandbox::Launch { + assets: &bundle.directory.join("sandbox"), + worker: &launch.working_directory, + executable: &launch.executable, + arguments: &arguments, + working_directory: &launch.working_directory, + capabilities: dever_sandbox::Capabilities { + network: definition + .capabilities + .iter() + .any(|capability| capability == "network"), + process: definition + .capabilities + .iter() + .any(|capability| capability == "process"), + }, + grants: &settings, + }) +} + +pub fn verified_worker_launch( + directory: &Path, + resources: &[Resource<'_>], + entry: &str, + ecosystem: &str, +) -> Result<WorkerLaunch, String> { + let (suffix, format, tree) = match ecosystem { + "exec" | "pip" | "npm" | "go" => ("worker", "dever-worker-launch-v1", "workers/"), + "command" => ("command", "dever-command-launch-v1", "commands/"), + _ => return Err("invalid managed Worker ecosystem".into()), + }; + let manifest_path = format!("{entry}.dever-{suffix}.json"); + let manifest = resources + .iter() + .find(|resource| resource.path == manifest_path) + .ok_or("managed Worker launch manifest is missing")?; + if manifest.executable { + return Err("managed Worker manifest must not be executable".into()); + } + let text = std::str::from_utf8(manifest.bytes) + .map_err(|_| "managed Worker manifest must be UTF-8 JSON")?; + crate::wire::parse(text) + .map_err(|error| format!("invalid managed Worker manifest: {error}"))?; + let manifest: LaunchManifest = serde_json::from_str(text) + .map_err(|error| format!("invalid managed Worker manifest: {error}"))?; + if manifest.format != format || manifest.ecosystem != ecosystem || manifest.entry != entry { + return Err("managed Worker launch identity differs from its checked Adapter".into()); + } + validate_relative_path(&manifest.working_directory)?; + if !manifest.working_directory.starts_with(tree) || manifest.arguments.len() > 64 { + return Err("invalid managed Worker execution tree or argument count".into()); + } + let prefix = format!("{}/", manifest.working_directory); + let resource_path = |path: &str| -> Result<PathBuf, String> { + validate_relative_path(path)?; + if !path.starts_with(&prefix) || !resources.iter().any(|resource| resource.path == path) { + return Err("managed Worker argument escapes its verified execution tree".into()); + } + Ok(directory.join(path)) + }; + resource_path(&manifest.executable)?; + // Recheck the complete tree once, including executable bits, at every + // Worker start/restart before any launch path is exposed to the supervisor. + let executable = verified_entry(directory, resources, &manifest.executable)?; + let mut total_bytes = 0_usize; + let mut arguments = Vec::with_capacity(manifest.arguments.len()); + for argument in manifest.arguments { + let argument = match argument { + LaunchArgument::Resource(argument) => { + resource_path(&argument.resource)?.into_os_string() + } + LaunchArgument::Literal(argument) => { + if argument.literal.contains('\0') { + return Err("managed Worker argument contains NUL".into()); + } + argument.literal.into() + } + }; + total_bytes = total_bytes.saturating_add(argument.as_encoded_bytes().len()); + if total_bytes > 65_536 { + return Err("managed Worker arguments exceed the byte budget".into()); + } + arguments.push(argument); + } + let working_directory = directory.join(manifest.working_directory); + if !working_directory.is_dir() { + return Err("managed Worker execution directory is missing".into()); + } + reject_symlink_ancestors(&working_directory)?; + Ok(WorkerLaunch { + executable, + arguments, + working_directory, + }) +} + +pub fn verified_entry( + directory: &Path, + resources: &[Resource<'_>], + entry: &str, +) -> Result<PathBuf, String> { + let resource = resources + .iter() + .find(|resource| resource.path == entry) + .ok_or("external Adapter entry is not in the verified manifest")?; + if !resource.executable { + return Err("external Adapter entry is not executable in the verified manifest".into()); + } + verify_directory(directory, resources)?; + let path = directory.join(entry); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + if fs::metadata(&path) + .map_err(|error| error.to_string())? + .permissions() + .mode() + & 0o100 + == 0 + { + return Err("external Adapter entry has no owner execute permission".into()); + } + } + path.canonicalize() + .map_err(|error| format!("cannot resolve verified external Adapter entry: {error}")) +} + +pub fn bundle_digest(resources: &[Resource<'_>]) -> String { + let mut digest = Sha256::new(); + for resource in resources { + digest.update(resource.path.as_bytes()); + digest.update([0]); + digest.update(resource.sha256.as_bytes()); + digest.update([0]); + digest.update((resource.bytes.len() as u64).to_le_bytes()); + digest.update([u8::from(resource.executable)]); + } + digest + .finalize() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +pub fn extract(root: &Path, digest: &str, resources: &[Resource<'_>]) -> Result<PathBuf, String> { + validate_digest(digest)?; + let mut paths = std::collections::BTreeSet::new(); + for resource in resources { + validate_resource(resource)?; + if !paths.insert(resource.path) { + return Err(format!( + "duplicate external resource path '{}'", + resource.path + )); + } + } + if bundle_digest(resources) != digest { + return Err("external resource bundle identity does not match its manifest".into()); + } + ensure_cache_directory(root)?; + let target = root.join("data/cache/lib").join(digest); + if target.exists() { + verify_directory(&target, resources)?; + return Ok(target); + } + let parent = target.parent().ok_or("external resource has no parent")?; + let staging = parent.join(format!( + ".staging-{}-{}", + std::process::id(), + unique_suffix() + )); + create_private_directory(&staging)?; + let result = (|| { + let mut paths = std::collections::BTreeSet::new(); + for resource in resources { + validate_resource(resource)?; + if !paths.insert(resource.path) { + return Err(format!( + "duplicate external resource path '{}'", + resource.path + )); + } + let path = staging.join(resource.path); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).map_err(|error| error.to_string())?; + } + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options.open(&path).map_err(|error| error.to_string())?; + file.write_all(resource.bytes) + .map_err(|error| error.to_string())?; + file.sync_all().map_err(|error| error.to_string())?; + #[cfg(unix)] + if resource.executable { + set_executable(&file)?; + } + } + sync_directory(&staging)?; + match fs::rename(&staging, &target) { + Ok(()) => {} + Err(error) + if matches!( + error.kind(), + io::ErrorKind::AlreadyExists | io::ErrorKind::DirectoryNotEmpty + ) => + { + verify_directory(&target, resources)?; + let _ = fs::remove_dir_all(&staging); + } + Err(error) => return Err(error.to_string()), + } + sync_directory(parent)?; + Ok(()) + })(); + if result.is_err() { + let _ = fs::remove_dir_all(&staging); + } + result + .map(|()| target) + .map_err(|error: String| format!("cannot publish external resources: {error}")) +} + +fn verify_directory(root: &Path, resources: &[Resource<'_>]) -> Result<(), String> { + reject_symlink_ancestors(root)?; + let metadata = fs::symlink_metadata(root).map_err(|error| error.to_string())?; + if !metadata.is_dir() { + return Err("external resource cache entry is not a directory".into()); + } + let parent = root + .parent() + .ok_or("external resource cache has no parent")?; + validate_owner( + root, + &fs::metadata(parent).map_err(|error| error.to_string())?, + true, + )?; + let mut expected = std::collections::BTreeSet::new(); + for resource in resources { + validate_resource(resource)?; + if !expected.insert(resource.path) { + return Err("duplicate external resource path".into()); + } + let path = root.join(resource.path); + reject_symlink_ancestors(&path)?; + let metadata = fs::symlink_metadata(&path).map_err(|error| { + format!( + "cannot inspect external resource '{}': {error}", + resource.path + ) + })?; + if metadata.file_type().is_symlink() || !metadata.is_file() { + return Err(format!( + "external resource '{}' is not a regular file", + resource.path + )); + } + validate_owner( + &path, + &fs::metadata(root).map_err(|error| error.to_string())?, + true, + )?; + if !verify_resource_file(&path, resource)? { + return Err(format!( + "external resource '{}' failed integrity verification", + resource.path + )); + } + } + let mut actual = Vec::new(); + collect_files(root, root, &mut actual)?; + if actual.iter().any(|path| !expected.contains(path.as_str())) || actual.len() != expected.len() + { + return Err("external resource cache contains files outside the verified manifest".into()); + } + Ok(()) +} + +fn verify_resource_file(path: &Path, resource: &Resource<'_>) -> Result<bool, String> { + let mut file = File::open(path).map_err(|error| error.to_string())?; + if file.metadata().map_err(|error| error.to_string())?.len() != resource.bytes.len() as u64 { + return Ok(false); + } + let mut digest = Sha256::new(); + let mut buffer = [0_u8; 64 * 1024]; + let mut offset = 0; + loop { + let read = file.read(&mut buffer).map_err(|error| error.to_string())?; + if read == 0 { + break; + } + let Some(expected) = resource.bytes.get(offset..offset + read) else { + return Ok(false); + }; + if expected != &buffer[..read] { + return Ok(false); + } + digest.update(&buffer[..read]); + offset += read; + } + let actual = digest + .finalize() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::<String>(); + Ok(offset == resource.bytes.len() && actual == resource.sha256) +} + +fn collect_files(root: &Path, directory: &Path, files: &mut Vec<String>) -> Result<(), String> { + for entry in fs::read_dir(directory).map_err(|error| error.to_string())? { + let entry = entry.map_err(|error| error.to_string())?; + let path = entry.path(); + let metadata = fs::symlink_metadata(&path).map_err(|error| error.to_string())?; + if metadata.file_type().is_symlink() { + return Err(format!( + "external resource cache contains a symbolic link: {}", + path.display() + )); + } + if metadata.is_dir() { + validate_owner( + &path, + &fs::metadata(root).map_err(|error| error.to_string())?, + false, + )?; + collect_files(root, &path, files)?; + } else if metadata.is_file() { + let relative = path + .strip_prefix(root) + .map_err(|error| error.to_string())? + .to_str() + .ok_or("external resource path is not UTF-8")?; + files.push(relative.replace(std::path::MAIN_SEPARATOR, "/")); + } else { + return Err(format!( + "external resource cache contains a non-regular entry: {}", + path.display() + )); + } + } + Ok(()) +} + +fn ensure_cache_directory(root: &Path) -> Result<(), String> { + reject_symlink_ancestors(root)?; + if !root.exists() { + create_private_directory(root)?; + } + if !fs::symlink_metadata(root) + .map_err(|error| error.to_string())? + .is_dir() + { + return Err("external application root must be a directory".into()); + } + let owner = fs::metadata(root).map_err(|error| error.to_string())?; + validate_owner(root, &owner, false)?; + let data = root.join("data"); + let cache = data.join("cache"); + let lib = cache.join("lib"); + for path in [&data, &cache, &lib] { + match fs::symlink_metadata(path) { + Ok(metadata) if metadata.file_type().is_symlink() => { + return Err(format!( + "external resource directory '{}' is a symbolic link", + path.display() + )); + } + Ok(metadata) if !metadata.is_dir() => { + return Err(format!( + "external resource directory '{}' is not a directory", + path.display() + )); + } + Ok(_) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => { + let mut builder = fs::DirBuilder::new(); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + match builder.create(path) { + Ok(()) => {} + Err(create_error) if create_error.kind() == io::ErrorKind::AlreadyExists => { + let metadata = + fs::symlink_metadata(path).map_err(|error| error.to_string())?; + if !metadata.is_dir() || metadata.file_type().is_symlink() { + return Err( + "external resource directory changed during creation".into() + ); + } + } + Err(create_error) => { + return Err(format!( + "cannot create external resource cache '{}': {create_error}", + path.display() + )); + } + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(path, fs::Permissions::from_mode(0o700)) + .map_err(|permission_error| permission_error.to_string())?; + } + } + Err(error) => { + return Err(format!( + "cannot inspect external resource directory '{}': {error}", + path.display() + )); + } + } + validate_owner(path, &owner, path == &lib)?; + } + Ok(()) +} + +fn validate_resource(resource: &Resource<'_>) -> Result<(), String> { + validate_relative_path(resource.path)?; + if sha256(resource.bytes) != resource.sha256 { + return Err(format!( + "external resource '{}' has an invalid digest", + resource.path + )); + } + Ok(()) +} + +fn validate_relative_path(value: &str) -> Result<(), String> { + let path = Path::new(value); + if value.is_empty() + || value.contains(['\\', ':', '\0']) + || path.is_absolute() + || path + .components() + .any(|component| !matches!(component, Component::Normal(_))) + { + return Err(format!( + "external resource path '{value}' is not safely relative" + )); + } + Ok(()) +} + +fn reject_symlink_ancestors(path: &Path) -> Result<(), String> { + let mut ancestor = PathBuf::new(); + for component in path.components() { + ancestor.push(component); + match fs::symlink_metadata(&ancestor) { + Ok(metadata) if metadata.file_type().is_symlink() => { + return Err("external resource path crosses a symbolic link".into()); + } + Ok(_) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => break, + Err(error) => return Err(format!("cannot inspect external resource path: {error}")), + } + } + Ok(()) +} + +fn validate_digest(value: &str) -> Result<(), String> { + if value.len() != 64 + || !value + .bytes() + .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) + { + return Err("external resource digest must be lowercase SHA-256".into()); + } + Ok(()) +} + +fn sha256(bytes: &[u8]) -> String { + let mut digest = Sha256::new(); + digest.update(bytes); + digest + .finalize() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +fn create_private_directory(path: &Path) -> Result<(), String> { + let mut builder = fs::DirBuilder::new(); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + builder.create(path).map_err(|error| error.to_string()) +} + +fn validate_owner(path: &Path, owner: &fs::Metadata, private: bool) -> Result<(), String> { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let metadata = fs::symlink_metadata(path).map_err(|error| error.to_string())?; + let prohibited = if private { 0o077 } else { 0o022 }; + if metadata.uid() != owner.uid() || metadata.mode() & prohibited != 0 { + return Err(format!( + "external resource path '{}' has an untrusted owner or permissions", + path.display() + )); + } + } + #[cfg(not(unix))] + { + let _ = (path, owner, private); + } + Ok(()) +} + +fn sync_directory(path: &Path) -> Result<(), String> { + File::open(path) + .map_err(|error| error.to_string())? + .sync_all() + .map_err(|error| error.to_string()) +} + +#[cfg(unix)] +fn set_executable(file: &File) -> Result<(), String> { + use std::os::unix::fs::PermissionsExt; + let mut permissions = file + .metadata() + .map_err(|error| error.to_string())? + .permissions(); + permissions.set_mode(0o700); + file.set_permissions(permissions) + .map_err(|error| error.to_string()) +} + +fn unique_suffix() -> u64 { + use std::sync::atomic::{AtomicU64, Ordering}; + static NEXT: AtomicU64 = AtomicU64::new(0); + NEXT.fetch_add(1, Ordering::Relaxed) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn extraction_is_verified_and_rejects_path_escape() { + let root = std::env::temp_dir().join(format!("dever-external-{}", std::process::id())); + let bytes = b"worker"; + let digest = sha256(bytes); + let resources = [Resource { + path: "worker.bin", + bytes, + sha256: &digest, + executable: false, + }]; + let bundle = bundle_digest(&resources); + let directory = extract(&root, &bundle, &resources).unwrap(); + assert_eq!(fs::read(directory.join("worker.bin")).unwrap(), bytes); + fs::write(directory.join("unexpected.bin"), b"unexpected").unwrap(); + assert!(extract(&root, &bundle, &resources).is_err()); + assert!( + extract( + &root, + &bundle, + &[Resource { + path: "../escape", + bytes, + sha256: &digest, + executable: false + }] + ) + .is_err() + ); + let _ = fs::remove_dir_all(root); + } +} diff --git a/crates/dever-runtime/src/http/client.rs b/crates/dever-runtime/src/http/client.rs new file mode 100644 index 0000000..95c81c2 --- /dev/null +++ b/crates/dever-runtime/src/http/client.rs @@ -0,0 +1,548 @@ +use std::sync::Arc; +use std::time::Duration; + +use hyper::body::Incoming; +use hyper_util::rt::TokioIo; +use tokio::sync::{Mutex, OwnedSemaphorePermit, mpsc}; +use tokio::time::Instant; + +use super::client_body::{RequestBody, download, hold_upload, upload}; +use super::client_driver::{self, Control, Driver, Entry, Http2Entry, Http2Lease, Http2Pool, Idle}; +use super::{Budget, Header, Limits, Request, Response, message}; +use crate::{async_stream::AsyncStream, bytes::Bytes, task, tls::ClientTls}; + +#[derive(Clone, Copy, Debug)] +pub struct PoolLimits { + pub idle_ms: i64, + pub chunk_bytes: i64, + pub read_ms: i64, +} + +#[derive(Clone, Debug)] +pub struct StreamRequest { + pub method: String, + pub target: String, + pub headers: Vec<Header>, + pub body: AsyncStream<Result<Bytes, String>>, +} + +#[derive(Clone, Debug)] +pub struct StreamResponse { + pub status: i64, + pub headers: Vec<Header>, + pub body: AsyncStream<Result<Bytes, String>>, +} + +struct State { + authority: String, + scheme: &'static str, + host: String, + port: i64, + tls: Option<ClientTls>, + budget: Budget, + chunks: usize, + read_timeout: Duration, + control: Arc<Control>, + idle: Arc<Idle>, + http2: Option<Arc<Http2Pool>>, + queue: mpsc::Sender<Driver>, + driver: Mutex<Option<task::Task<()>>>, +} + +impl Drop for State { + fn drop(&mut self) { + self.control.close(); + } +} + +#[derive(Clone)] +pub struct HttpClient(Arc<State>); + +impl std::fmt::Debug for HttpClient { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("HttpClient(<resource>)") + } +} + +pub async fn client( + origin: &str, + tls: ClientTls, + limits: Limits, + pool: PoolLimits, +) -> Result<HttpClient, String> { + let budget = limits.check()?; + let idle_timeout = task::positive_duration(pool.idle_ms)?; + let read_timeout = task::positive_duration(pool.read_ms)?; + let chunks = usize::try_from(pool.chunk_bytes) + .ok() + .filter(|size| *size > 0) + .ok_or("HTTP chunk_bytes must be positive")?; + let uri: hyper::Uri = origin + .parse() + .map_err(|error| format!("invalid HTTP origin: {error}"))?; + let scheme = match uri.scheme_str() { + Some("http") => "http", + Some("https") => "https", + _ => return Err("HTTP origin must use http or https".into()), + }; + let secure = scheme == "https"; + let authority = uri + .authority() + .ok_or("HTTP origin requires a host")? + .as_str() + .to_owned(); + if authority.contains('@') + || origin.contains('#') + || uri.query().is_some() + || !matches!(uri.path(), "" | "/") + { + return Err( + "HTTP client origin cannot contain credentials, path, query or fragment".into(), + ); + } + let host = uri + .host() + .ok_or("HTTP origin requires a host")? + .trim_matches(['[', ']']) + .to_owned(); + let port = i64::from(uri.port_u16().unwrap_or(if secure { 443 } else { 80 })); + let connections = limits.connections as usize; + let request_capacity = budget + .http2 + .map(|http2| connections * http2.streams as usize) + .unwrap_or(connections); + let control = Control::new(request_capacity, connections); + let idle = Idle::new(idle_timeout); + let http2 = budget + .http2 + .map(|limits| Http2Pool::new(limits.streams as usize, idle_timeout)); + let (queue, driver) = + client_driver::start(control.clone(), idle.clone(), http2.clone(), connections).await?; + Ok(HttpClient(Arc::new(State { + authority, + scheme, + host, + port, + tls: secure.then(|| super::protocol::client_tls(tls, budget.http2)), + budget, + chunks, + read_timeout, + control, + idle, + http2, + queue, + driver: Mutex::new(Some(driver)), + }))) +} + +impl HttpClient { + pub(super) fn control(&self) -> &Arc<Control> { + &self.0.control + } + + pub(super) fn chunk_limit(&self) -> usize { + self.0.chunks + } + + pub(super) fn read_timeout(&self) -> Duration { + self.0.read_timeout + } + + fn encode( + &self, + mut request: Request, + ) -> Result<hyper::Request<http_body_util::Full<bytes::Bytes>>, String> { + if !request + .headers + .iter() + .any(|header| header.name.eq_ignore_ascii_case("host")) + { + request.headers.push(Header { + name: "host".into(), + value: Bytes::from_string(self.0.authority.clone()), + }); + } + let mut request = message::encode_request(request, self.0.budget)?; + if self.0.http2.is_some() { + message::http2_request(&mut request, self.0.scheme)?; + } + Ok(request) + } + + async fn http1_connection(&self) -> Result<Entry, String> { + while let Some(mut entry) = self.0.idle.take() { + if entry.sender.ready().await.is_ok() { + return Ok(entry); + } + // Discard a closed idle transport before sending any request bytes. + } + let permit = self + .0 + .control + .connections + .clone() + .acquire_owned() + .await + .map_err(|_| "HTTP client is closed".to_owned())?; + let transport = + crate::transport::connect(&self.0.host, self.0.port, self.0.tls.as_ref()).await?; + super::protocol::check_alpn(&transport, self.0.budget.http2)?; + let (sender, connection) = hyper::client::conn::http1::Builder::new() + .max_buf_size(self.0.budget.headers) + .handshake(TokioIo::new(transport)) + .await + .map_err(|error| error.to_string())?; + let (abort, driver) = client_driver::driver(connection, permit); + // The connection permit bounds queued plus running drivers together. + self.0 + .queue + .try_send(driver) + .map_err(|_| "HTTP client driver is closed".to_owned())?; + Ok(Entry { sender, abort }) + } + + async fn new_http2_connection( + &self, + permit: OwnedSemaphorePermit, + ) -> Result<Arc<Http2Entry>, String> { + let limits = self.0.budget.http2.expect("HTTP/2 client limits"); + let pool = self.0.http2.as_ref().expect("HTTP/2 client pool"); + let transport = + crate::transport::connect(&self.0.host, self.0.port, self.0.tls.as_ref()).await?; + super::protocol::check_alpn(&transport, self.0.budget.http2)?; + let (driver, handshake) = client_driver::http2_handshake( + transport, + permit, + limits, + self.0.budget.headers, + self.0.budget.timeout, + Arc::clone(pool), + ); + self.0 + .queue + .try_send(driver) + .map_err(|_| "HTTP client driver is closed".to_owned())?; + handshake + .await + .map_err(|_| "HTTP client driver closed during HTTP/2 handshake".to_owned())? + } + + async fn http2_connection(&self) -> Result<(Arc<Http2Entry>, OwnedSemaphorePermit), String> { + let pool = self.0.http2.as_ref().expect("HTTP/2 client pool"); + loop { + let changed = pool.changed(); + tokio::pin!(changed); + changed.as_mut().enable(); + if let Some(connection) = pool.take() { + return Ok(connection); + } + let connection = self.0.control.connections.clone().acquire_owned(); + tokio::pin!(connection); + tokio::select! { + biased; + _ = changed => continue, + permit = &mut connection => { + let permit = permit.map_err(|_| "HTTP client is closed".to_owned())?; + let entry = self.new_http2_connection(permit).await?; + if let Some(stream) = entry.acquire() { + return Ok((entry, stream)); + } + } + } + } + } + + async fn exchange( + &self, + request: hyper::Request<RequestBody>, + ) -> Result<StreamResponse, String> { + if self.0.http2.is_some() { + self.exchange_http2(request).await + } else { + self.exchange_http1(request).await + } + } + + async fn exchange_http1( + &self, + request: hyper::Request<RequestBody>, + ) -> Result<StreamResponse, String> { + self.0 + .control + .while_open(async { + tokio::time::timeout(self.0.budget.timeout, async { + let permit = self + .0 + .control + .requests + .clone() + .acquire_owned() + .await + .map_err(|_| "HTTP client is closed".to_owned())?; + let mut entry = self.http1_connection().await?; + let response = entry + .sender + .send_request(request) + .await + .map_err(|error| error.to_string())?; + let lease = Lease::Http1 { + entry: Some(entry), + client: self.clone(), + _permit: permit, + }; + self.response(response, lease) + }) + .await + .map_err(|_| "HTTP request timed out".to_owned())? + }) + .await + } + + async fn exchange_http2( + &self, + request: hyper::Request<RequestBody>, + ) -> Result<StreamResponse, String> { + self.0 + .control + .while_open(async { + tokio::time::timeout(self.0.budget.timeout, async { + let (head, body) = request.into_parts(); + let (body, upload_lease) = hold_upload(body); + let mut pending = Some(hyper::Request::from_parts(head, body)); + loop { + let request_permit = self + .0 + .control + .requests + .clone() + .acquire_owned() + .await + .map_err(|_| "HTTP client is closed".to_owned())?; + let (entry, stream_permit) = self.http2_connection().await?; + let stream = Http2Lease::new(entry.clone(), stream_permit, request_permit); + let mut sender = entry.sender.clone(); + if sender.ready().await.is_err() { + entry.drain(); + continue; + } + upload_lease.replace(stream.clone()); + let request = pending.take().expect("pending HTTP/2 request"); + let response = match sender.try_send_request(request).await { + Ok(response) => response, + Err(mut error) => { + if sender.is_closed() { + entry.drain(); + } + if let Some(request) = error.take_message() { + upload_lease.clear(); + drop(stream); + pending = Some(request); + continue; + } + return Err(error.into_error().to_string()); + } + }; + let lease = Lease::Http2 { + client: self.clone(), + _stream: stream, + }; + return self.response(response, lease); + } + }) + .await + .map_err(|_| "HTTP request timed out".to_owned())? + }) + .await + } + + fn response( + &self, + response: hyper::Response<Incoming>, + lease: Lease, + ) -> Result<StreamResponse, String> { + if response.status() == hyper::StatusCode::SWITCHING_PROTOCOLS { + return Err("HTTP client upgrades require the WebSocket API".into()); + } + if message::header_size(response.headers()) + 32 > self.0.budget.headers { + return Err("HTTP response headers exceed header_bytes".into()); + } + let (head, body) = response.into_parts(); + Ok(StreamResponse { + status: i64::from(head.status.as_u16()), + headers: message::decode_headers(head.headers), + body: download(body, lease), + }) + } +} + +pub(super) enum Lease { + Http1 { + entry: Option<Entry>, + client: HttpClient, + _permit: OwnedSemaphorePermit, + }, + Http2 { + client: HttpClient, + _stream: Http2Lease, + }, +} + +impl Lease { + pub fn client(&self) -> &HttpClient { + match self { + Self::Http1 { client, .. } | Self::Http2 { client, .. } => client, + } + } + + pub async fn complete(&mut self) { + let Self::Http1 { entry, client, .. } = self else { + return; + }; + let mut entry = entry.take().expect("active HTTP lease"); + if !client.control().closed.is_closed() && entry.sender.ready().await.is_ok() { + client.0.idle.put(entry); + } + } +} + +pub(super) async fn send_http2( + address: &str, + port: i64, + mut request: hyper::Request<http_body_util::Full<bytes::Bytes>>, + budget: Budget, +) -> Result<Response, String> { + use http_body_util::BodyExt; + + message::http2_request(&mut request, "http")?; + let deadline = Instant::now() + budget.timeout; + let timeout_ms = i64::try_from(budget.timeout.as_millis()) + .map_err(|_| "HTTP timeout is too large".to_owned())?; + let transport = tokio::time::timeout_at( + deadline, + crate::net::connect_stream_timeout(address, port, timeout_ms), + ) + .await + .map_err(|_| "HTTP request timed out".to_owned())??; + let (mut owner, spawner, fault) = task::scoped_owner(())?; + let limits = budget.http2.expect("HTTP/2 send limits"); + let builder = limits.client_builder(super::executor::Http2Executor(spawner), budget.headers); + let handshake = { + let handshake = tokio::time::timeout_at( + deadline, + owner.enter(builder.handshake(TokioIo::new(transport))), + ); + tokio::pin!(handshake); + tokio::select! { + result = &mut handshake => match result { + Ok(result) => result.map_err(|error| error.to_string()), + Err(_) => Err("HTTP request timed out".to_owned()), + }, + error = fault.wait() => Err(error), + } + }; + let (mut sender, connection) = match handshake { + Ok(connection) => connection, + Err(error) => { + let _ = owner.stop().await; + return Err(error); + } + }; + let request = request.map(|body| body.map_err(|never| match never {}).boxed_unsync()); + let exchange = async { + let response = sender + .send_request(request) + .await + .map_err(|error| error.to_string())?; + if response.status() == hyper::StatusCode::SWITCHING_PROTOCOLS { + return Err("HTTP upgrades are not supported by send".into()); + } + if message::header_size(response.headers()) + 32 > budget.headers { + return Err("HTTP response headers exceed header_bytes".into()); + } + let (head, body) = response.into_parts(); + let body = message::collect_body(body, budget.body) + .await + .map_err(|status| format!("HTTP response body rejected: {status}"))?; + Ok(Response { + status: i64::from(head.status.as_u16()), + headers: message::decode_headers(head.headers), + body, + }) + }; + let result = { + let connection = owner.enter(connection); + tokio::pin!(connection, exchange); + match tokio::time::timeout_at(deadline, async { + tokio::select! { + result = &mut exchange => result, + result = &mut connection => { + result.map_err(|error| error.to_string())?; + exchange.await + } + error = fault.wait() => Err(error), + } + }) + .await + { + Ok(result) => result, + Err(_) => Err("HTTP request timed out".to_owned()), + } + }; + let cleanup = owner.stop().await; + match result { + Ok(response) => { + cleanup?; + Ok(response) + } + Err(error) => Err(error), + } +} + +pub async fn open(client: &HttpClient, request: Request) -> Result<StreamResponse, String> { + use http_body_util::BodyExt; + let request = client + .encode(request)? + .map(|body| body.map_err(|never| match never {}).boxed_unsync()); + client.exchange(request).await +} + +pub async fn open_stream( + client: &HttpClient, + request: StreamRequest, +) -> Result<StreamResponse, String> { + let head = client.encode(Request { + method: request.method, + target: request.target, + headers: request.headers, + body: Bytes::new(Vec::new()), + })?; + let request = head.map(|_| upload(request.body, client.chunk_limit(), client.read_timeout())); + client.exchange(request).await +} + +pub async fn request(client: &HttpClient, request: Request) -> Result<Response, String> { + tokio::time::timeout(client.0.budget.timeout, async { + let response = open(client, request).await?; + let _owner = response.body.close_on_drop(); + let mut body = Bytes::new(Vec::new()); + while let Some(chunk) = response.body.pull().await { + let chunk = chunk?; + if body.values().len() + chunk.values().len() > client.0.budget.body { + return Err("HTTP response body exceeds body_bytes".into()); + } + body = body.concat(&chunk); + } + Ok(Response { + status: response.status, + headers: response.headers, + body, + }) + }) + .await + .map_err(|_| "HTTP request timed out".to_owned())? +} + +pub async fn close_client(client: &HttpClient) -> Result<(), String> { + client.0.control.close(); + task::stop_resource(&mut *client.0.driver.lock().await).await +} diff --git a/crates/dever-runtime/src/http/client_body.rs b/crates/dever-runtime/src/http/client_body.rs new file mode 100644 index 0000000..20d1431 --- /dev/null +++ b/crates/dever-runtime/src/http/client_body.rs @@ -0,0 +1,167 @@ +use std::io; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; +use std::task::{Context, Poll}; +use std::time::Duration; + +use futures_util::stream; +use http_body_util::{BodyExt, StreamBody, combinators::UnsyncBoxBody}; +use hyper::body::{Body, Frame, Incoming}; + +use super::client::Lease; +use super::client_driver::Http2Lease; +use crate::{async_stream::AsyncStream, bytes::Bytes, resource::Pull}; + +pub(super) type RequestBody = UnsyncBoxBody<bytes::Bytes, io::Error>; + +struct LeasedRequestBody { + body: RequestBody, + _lease: UploadLease, +} + +#[derive(Clone)] +pub(super) struct UploadLease(Arc<Mutex<Option<Http2Lease>>>); + +impl UploadLease { + pub(super) fn replace(&self, lease: Http2Lease) { + *self.0.lock().expect("HTTP/2 upload lease lock") = Some(lease); + } + + pub(super) fn clear(&self) { + self.0.lock().expect("HTTP/2 upload lease lock").take(); + } +} + +impl Body for LeasedRequestBody { + type Data = bytes::Bytes; + type Error = io::Error; + + fn poll_frame( + mut self: Pin<&mut Self>, + context: &mut Context<'_>, + ) -> Poll<Option<Result<Frame<Self::Data>, Self::Error>>> { + Pin::new(&mut self.body).poll_frame(context) + } + + fn is_end_stream(&self) -> bool { + self.body.is_end_stream() + } + + fn size_hint(&self) -> hyper::body::SizeHint { + self.body.size_hint() + } +} + +pub(super) fn hold_upload(body: RequestBody) -> (RequestBody, UploadLease) { + let lease = UploadLease(Arc::new(Mutex::new(None))); + let body = LeasedRequestBody { + body, + _lease: lease.clone(), + } + .boxed_unsync(); + (body, lease) +} + +pub(super) fn upload( + source: AsyncStream<Result<Bytes, String>>, + maximum: usize, + timeout: Duration, +) -> RequestBody { + let owner = source.close_on_drop(); + StreamBody::new(stream::unfold( + (source, owner), + move |(source, owner)| async move { + let chunk = tokio::time::timeout(timeout, source.pull()).await; + let result = match chunk { + Ok(Some(Ok(bytes))) if bytes.values().len() <= maximum => { + Ok(Frame::data(bytes.into_http())) + } + Ok(Some(Ok(_))) => Err(io::Error::other("HTTP upload chunk exceeds chunk_bytes")), + Ok(Some(Err(error))) => Err(io::Error::other(error)), + Ok(None) => return None, + Err(_) => Err(io::Error::new( + io::ErrorKind::TimedOut, + "HTTP upload timed out", + )), + }; + if result.is_err() { + source.close(); + } + Some((result, (source, owner))) + }, + )) + .boxed_unsync() +} + +struct Download { + body: Incoming, + pending: bytes::Bytes, + lease: Lease, +} + +pub(super) fn download(body: Incoming, lease: Lease) -> AsyncStream<Result<Bytes, String>> { + AsyncStream::new(stream::unfold( + Download { + body, + pending: bytes::Bytes::new(), + lease, + }, + |mut state| async move { + let client = state.lease.client().clone(); + let next = client + .control() + .while_open(async { + tokio::time::timeout(client.read_timeout(), state.next()) + .await + .map_err(|_| "HTTP response read timed out".to_owned())? + }) + .await; + match next { + Ok(Some((chunk, last))) => { + let item = if last { + Pull::Last(Ok(chunk)) + } else { + Pull::Item(Ok(chunk)) + }; + Some((item, state)) + } + Ok(None) => None, + Err(error) => Some((Pull::Last(Err(error)), state)), + } + }, + )) +} + +impl Download { + async fn next(&mut self) -> Result<Option<(Bytes, bool)>, String> { + while self.pending.is_empty() { + if self.body.is_end_stream() { + self.lease.complete().await; + return Ok(None); + } + let Some(frame) = self.body.frame().await else { + self.lease.complete().await; + return Ok(None); + }; + let frame = frame.map_err(|error| error.to_string())?; + match frame.into_data() { + Ok(bytes) => self.pending = bytes, + Err(frame) + if frame + .trailers_ref() + .is_some_and(|trailers| !trailers.is_empty()) => + { + return Err("HTTP response trailers are not supported".into()); + } + Err(_) => {} + } + } + let length = self.pending.len().min(self.lease.client().chunk_limit()); + let chunk = Bytes::from_http(self.pending.split_to(length)); + let last = self.pending.is_empty() && self.body.is_end_stream(); + if last { + self.lease.complete().await; + } + Ok(Some((chunk, last))) + } +} diff --git a/crates/dever-runtime/src/http/client_driver.rs b/crates/dever-runtime/src/http/client_driver.rs new file mode 100644 index 0000000..c9ea187 --- /dev/null +++ b/crates/dever-runtime/src/http/client_driver.rs @@ -0,0 +1,443 @@ +use std::future::Future; +use std::pin::Pin; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use futures_util::future::{AbortHandle, Abortable}; +use futures_util::{StreamExt, stream::FuturesUnordered}; +use hyper_util::rt::TokioIo; +use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore, mpsc, oneshot}; +use tokio::time::Instant; + +use super::client_body::RequestBody; +use super::executor::Http2Executor; +use super::protocol::Http2Limits; +use crate::transport::Transport; + +pub(super) type Driver = Pin<Box<dyn Future<Output = ()> + Send>>; + +pub(super) struct Entry { + pub sender: hyper::client::conn::http1::SendRequest<RequestBody>, + pub abort: AbortHandle, +} + +impl Drop for Entry { + fn drop(&mut self) { + self.abort.abort(); + } +} + +pub(super) struct Idle { + entries: Mutex<Vec<(Instant, Entry)>>, + changed: Notify, + timeout: Duration, +} + +impl Idle { + pub fn new(timeout: Duration) -> Arc<Self> { + Arc::new(Self { + entries: Mutex::new(Vec::new()), + changed: Notify::new(), + timeout, + }) + } + + pub fn take(&self) -> Option<Entry> { + let mut entries = self.entries.lock().expect("pool state lock"); + entries.retain(|(deadline, entry)| *deadline > Instant::now() && !entry.sender.is_closed()); + entries.pop().map(|(_, entry)| entry) + } + + pub fn put(&self, entry: Entry) { + self.entries + .lock() + .expect("pool state lock") + .push((Instant::now() + self.timeout, entry)); + self.changed.notify_one(); + } + + fn deadline(&self) -> Option<Instant> { + let mut entries = self.entries.lock().expect("pool state lock"); + entries.retain(|(deadline, entry)| *deadline > Instant::now() && !entry.sender.is_closed()); + entries.iter().map(|(deadline, _)| *deadline).min() + } +} + +pub(super) struct Control { + pub closed: Semaphore, + pub requests: Arc<Semaphore>, + pub connections: Arc<Semaphore>, +} + +impl Control { + pub fn new(requests: usize, connections: usize) -> Arc<Self> { + Arc::new(Self { + closed: Semaphore::new(0), + requests: Arc::new(Semaphore::new(requests)), + connections: Arc::new(Semaphore::new(connections)), + }) + } + + pub fn close(&self) { + self.closed.close(); + self.requests.close(); + self.connections.close(); + } + + pub async fn while_open<T>( + &self, + future: impl Future<Output = Result<T, String>>, + ) -> Result<T, String> { + tokio::select! { + biased; + _ = self.closed.acquire() => Err("HTTP client is closed".into()), + result = future => result, + } + } +} + +pub(super) struct Http2Entry { + pub sender: hyper::client::conn::http2::SendRequest<RequestBody>, + streams: Arc<Semaphore>, + accepting: AtomicBool, + active: AtomicUsize, + idle_deadline: Mutex<Option<Instant>>, + abort: AbortHandle, + changed: Arc<Notify>, + idle_timeout: Duration, +} + +impl Http2Entry { + fn new( + sender: hyper::client::conn::http2::SendRequest<RequestBody>, + streams: usize, + abort: AbortHandle, + changed: Arc<Notify>, + idle_timeout: Duration, + ) -> Arc<Self> { + Arc::new(Self { + sender, + streams: Arc::new(Semaphore::new(streams)), + accepting: AtomicBool::new(true), + active: AtomicUsize::new(0), + idle_deadline: Mutex::new(Some(Instant::now() + idle_timeout)), + abort, + changed, + idle_timeout, + }) + } + + pub fn acquire(self: &Arc<Self>) -> Option<OwnedSemaphorePermit> { + if !self.accepting.load(Ordering::Acquire) || self.sender.is_closed() { + self.drain(); + return None; + } + let permit = self.streams.clone().try_acquire_owned().ok()?; + if !self.accepting.load(Ordering::Acquire) || self.sender.is_closed() { + self.drain(); + return None; + } + self.active.fetch_add(1, Ordering::AcqRel); + *self.idle_deadline.lock().expect("HTTP/2 pool state lock") = None; + Some(permit) + } + + fn release(&self) { + let previous = self.active.fetch_sub(1, Ordering::AcqRel); + debug_assert!(previous > 0, "active HTTP/2 stream count"); + if previous == 1 && self.accepting.load(Ordering::Acquire) { + *self.idle_deadline.lock().expect("HTTP/2 pool state lock") = + Some(Instant::now() + self.idle_timeout); + } + self.changed.notify_waiters(); + } + + pub fn drain(&self) { + if self.accepting.swap(false, Ordering::AcqRel) { + self.streams.close(); + *self.idle_deadline.lock().expect("HTTP/2 pool state lock") = None; + self.changed.notify_waiters(); + } + } + + fn retire(&self) { + self.drain(); + self.abort.abort(); + } + + fn deadline(&self) -> Option<Instant> { + *self.idle_deadline.lock().expect("HTTP/2 pool state lock") + } +} + +pub(super) struct Http2Pool { + entries: Mutex<Vec<Arc<Http2Entry>>>, + changed: Arc<Notify>, + streams: usize, + idle_timeout: Duration, +} + +impl Http2Pool { + pub fn new(streams: usize, idle_timeout: Duration) -> Arc<Self> { + Arc::new(Self { + entries: Mutex::new(Vec::new()), + changed: Arc::new(Notify::new()), + streams, + idle_timeout, + }) + } + + pub fn take(&self) -> Option<(Arc<Http2Entry>, OwnedSemaphorePermit)> { + let mut entries = self.entries.lock().expect("HTTP/2 pool state lock"); + entries.retain(|entry| entry.accepting.load(Ordering::Acquire)); + entries + .iter() + .find_map(|entry| entry.acquire().map(|permit| (Arc::clone(entry), permit))) + } + + pub fn add( + &self, + sender: hyper::client::conn::http2::SendRequest<RequestBody>, + abort: AbortHandle, + ) -> Arc<Http2Entry> { + let entry = Http2Entry::new( + sender, + self.streams, + abort, + Arc::clone(&self.changed), + self.idle_timeout, + ); + self.entries + .lock() + .expect("HTTP/2 pool state lock") + .push(Arc::clone(&entry)); + self.changed.notify_waiters(); + entry + } + + pub fn changed(&self) -> tokio::sync::futures::OwnedNotified { + self.changed.clone().notified_owned() + } + + fn expire_and_deadline(&self) -> Option<Instant> { + let now = Instant::now(); + let mut entries = self.entries.lock().expect("HTTP/2 pool state lock"); + entries.retain(|entry| { + if !entry.accepting.load(Ordering::Acquire) || entry.sender.is_closed() { + entry.drain(); + return false; + } + if entry.active.load(Ordering::Acquire) == 0 + && entry.deadline().is_some_and(|deadline| deadline <= now) + { + entry.retire(); + return false; + } + true + }); + entries.iter().filter_map(|entry| entry.deadline()).min() + } + + fn close(&self) { + let entries = std::mem::take(&mut *self.entries.lock().expect("HTTP/2 pool state lock")); + for entry in entries { + entry.retire(); + } + self.changed.notify_waiters(); + } +} + +#[derive(Clone)] +pub(super) struct Http2Lease { + _owner: Arc<Http2LeaseOwner>, +} + +struct Http2LeaseOwner { + entry: Arc<Http2Entry>, + stream: Option<OwnedSemaphorePermit>, + request: Option<OwnedSemaphorePermit>, +} + +impl Http2Lease { + pub fn new( + entry: Arc<Http2Entry>, + stream: OwnedSemaphorePermit, + request: OwnedSemaphorePermit, + ) -> Self { + Self { + _owner: Arc::new(Http2LeaseOwner { + entry, + stream: Some(stream), + request: Some(request), + }), + } + } +} + +impl Drop for Http2LeaseOwner { + fn drop(&mut self) { + self.stream.take(); + self.request.take(); + self.entry.release(); + } +} + +struct DriverOwner { + control: Arc<Control>, + idle: Arc<Idle>, + http2: Option<Arc<Http2Pool>>, +} + +impl Drop for DriverOwner { + fn drop(&mut self) { + self.control.close(); + self.idle.entries.lock().expect("pool state lock").clear(); + if let Some(http2) = &self.http2 { + http2.close(); + } + } +} + +pub(super) async fn start( + control: Arc<Control>, + idle: Arc<Idle>, + http2: Option<Arc<Http2Pool>>, + capacity: usize, +) -> Result<(mpsc::Sender<Driver>, crate::task::Task<()>), String> { + let (queue, mut receiver) = mpsc::channel::<Driver>(capacity); + // The parent may finish before this future is first polled. + let owner = DriverOwner { + control: control.clone(), + idle: idle.clone(), + http2: http2.clone(), + }; + let task = crate::task::run(async move { + let _owner = owner; + let mut drivers = FuturesUnordered::<Driver>::new(); + loop { + let deadline = [ + idle.deadline(), + http2.as_ref().and_then(|pool| pool.expire_and_deadline()), + ] + .into_iter() + .flatten() + .min(); + let expire = async { + match deadline { + Some(deadline) => tokio::time::sleep_until(deadline).await, + None => std::future::pending().await, + } + }; + let http2_changed = async { + match &http2 { + Some(pool) => pool.changed().await, + None => std::future::pending().await, + } + }; + tokio::select! { + biased; + _ = control.closed.acquire() => return Ok(()), + Some(()) = drivers.next(), if !drivers.is_empty() => {}, + _ = idle.changed.notified() => {}, + _ = http2_changed => {}, + _ = expire => {}, + next = receiver.recv() => match next { + Some(driver) => drivers.push(driver), + None => return Ok(()), + }, + } + } + }) + .await?; + Ok((queue, task)) +} + +pub(super) fn driver( + connection: impl Future<Output = Result<(), hyper::Error>> + Send + 'static, + permit: tokio::sync::OwnedSemaphorePermit, +) -> (AbortHandle, Driver) { + let (abort, registration) = AbortHandle::new_pair(); + let driver = Box::pin(async move { + let _permit = permit; + // Protocol/I/O failures are observed through SendRequest/Incoming. + let _completion = Abortable::new(connection, registration).await; + }); + (abort, driver) +} + +async fn drive_http2<C>( + connection: C, + mut owner: crate::task::ScopedOwner, + fault: crate::task::ScopeFault, + entry: Arc<Http2Entry>, + registration: futures_util::future::AbortRegistration, +) where + C: Future<Output = Result<(), hyper::Error>> + Send + 'static, +{ + let completion = { + let connection = Abortable::new(owner.enter(connection), registration); + tokio::pin!(connection); + tokio::select! { + result = &mut connection => Some(result), + _ = fault.wait() => None, + } + }; + entry.drain(); + match completion { + Some(Ok(Ok(()))) => { + let _ = owner.finish().await; + } + Some(Ok(Err(_))) | Some(Err(_)) | None => { + let _ = owner.stop().await; + } + } +} + +pub(super) fn http2_handshake( + transport: Transport, + permit: OwnedSemaphorePermit, + limits: Http2Limits, + headers: usize, + timeout: Duration, + pool: Arc<Http2Pool>, +) -> (Driver, oneshot::Receiver<Result<Arc<Http2Entry>, String>>) { + let (result, receiver) = oneshot::channel(); + let driver = Box::pin(async move { + let (mut owner, spawner, fault) = match crate::task::scoped_owner(permit) { + Ok(owner) => owner, + Err(error) => { + let _ = result.send(Err(error)); + return; + } + }; + let handshake = { + let builder = limits.client_builder(Http2Executor(spawner), headers); + let handshake = tokio::time::timeout( + timeout, + owner.enter(builder.handshake(TokioIo::new(transport))), + ); + tokio::pin!(handshake); + tokio::select! { + result = &mut handshake => match result { + Ok(result) => result.map_err(|error| error.to_string()), + Err(_) => Err("HTTP/2 handshake timed out".to_owned()), + }, + error = fault.wait() => Err(error), + } + }; + let (sender, connection) = match handshake { + Ok(connection) => connection, + Err(error) => { + let _ = result.send(Err(error)); + let _ = owner.stop().await; + return; + } + }; + let (abort, registration) = AbortHandle::new_pair(); + let entry = pool.add(sender, abort); + let _ = result.send(Ok(Arc::clone(&entry))); + drive_http2(connection, owner, fault, entry, registration).await; + }); + (driver, receiver) +} diff --git a/crates/dever-runtime/src/http/executor.rs b/crates/dever-runtime/src/http/executor.rs new file mode 100644 index 0000000..f8e036b --- /dev/null +++ b/crates/dever-runtime/src/http/executor.rs @@ -0,0 +1,17 @@ +use std::future::Future; + +use hyper::rt::Executor; + +use crate::task::ScopedSpawner; + +#[derive(Clone)] +pub(super) struct Http2Executor(pub(super) ScopedSpawner); + +impl<F> Executor<F> for Http2Executor +where + F: Future<Output = ()> + Send + 'static, +{ + fn execute(&self, future: F) { + self.0.spawn(future); + } +} diff --git a/crates/dever-runtime/src/http/live.rs b/crates/dever-runtime/src/http/live.rs new file mode 100644 index 0000000..2bb4e52 --- /dev/null +++ b/crates/dever-runtime/src/http/live.rs @@ -0,0 +1,289 @@ +use std::future::Future; +use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, +}; + +use hyper::service::service_fn; +use hyper_util::rt::TokioIo; +use tokio::sync::Mutex; + +use super::reply::{HttpReply, LiveLimits, Outgoing, Session}; +use super::write_timeout::WriteTimeout; +use super::{Budget, Limits, Request, RequestContext, read_request, reject}; +use crate::{net, task}; + +/// HTTP/1 每条连接最多一个活跃 handler；升级后仍占用同一个连接和任务名额。 +pub async fn serve_live<F, Fut>( + route: F, + listener: net::Listener, + limits: Limits, + live: LiveLimits, +) -> Result<(), String> +where + F: Fn(Request, HttpReply) -> Fut + Clone + Send + Sync + 'static, + Fut: Future<Output = Result<(), String>> + Send + 'static, +{ + serve_transport(route, listener, limits, live, None).await +} + +pub async fn serve_live_tls<F, Fut>( + route: F, + listener: net::Listener, + limits: Limits, + live: LiveLimits, + tls: crate::tls::ServerTls, +) -> Result<(), String> +where + F: Fn(Request, HttpReply) -> Fut + Clone + Send + Sync + 'static, + Fut: Future<Output = Result<(), String>> + Send + 'static, +{ + serve_transport(route, listener, limits, live, Some(tls)).await +} + +async fn serve_transport<F, Fut>( + route: F, + listener: net::Listener, + limits: Limits, + live: LiveLimits, + tls: Option<crate::tls::ServerTls>, +) -> Result<(), String> +where + F: Fn(Request, HttpReply) -> Fut + Clone + Send + Sync + 'static, + Fut: Future<Output = Result<(), String>> + Send + 'static, +{ + let budget = limits.check()?; + live.check()?; + let tls = tls.map(|tls| super::protocol::server_tls(tls, budget.http2)); + task::parallel_each_stream( + net::incoming(listener.clone(), true), + limits.connections, + move |connection| { + let route = route.clone(); + let tls = tls.clone(); + let listener = listener.clone(); + async move { + let stream = connection?; + let stream = match crate::tls::accept(stream, tls.as_ref(), limits.timeout_ms).await + { + Ok(stream) => stream, + Err(_) => return Ok(()), + }; + if super::protocol::check_alpn(&stream, budget.http2).is_err() { + return Ok(()); + } + if let Some(http2) = budget.http2 { + return serve_http2(route, listener, stream, budget, live, http2).await; + } + let handlers = Arc::new(Mutex::new(Some(task::Group::new(1)?))); + let upgraded = Arc::new(AtomicBool::new(false)); + let service = { + let handlers = handlers.clone(); + let upgraded = upgraded.clone(); + service_fn(move |request| { + dispatch( + request, + route.clone(), + budget, + live, + handlers.clone(), + upgraded.clone(), + ) + }) + }; + let connection = super::server_builder(budget) + .serve_connection( + TokioIo::new(WriteTimeout::new(stream, budget.timeout)), + service, + ) + .with_upgrades(); + tokio::pin!(connection); + let result = tokio::select! { + result = &mut connection => result, + _ = listener.closed() => { + connection.as_mut().graceful_shutdown(); + connection.await + } + }; + let group = handlers + .lock() + .await + .take() + .expect("connection owns handlers"); + if result.is_ok() && upgraded.load(Ordering::Acquire) { + group.wait().await?; + } else { + group.stop().await?; + } + super::connection_result(result) + } + }, + ) + .await +} + +async fn serve_http2<F, Fut>( + route: F, + listener: net::Listener, + stream: crate::transport::Transport, + budget: Budget, + live: LiveLimits, + http2: super::Http2Limits, +) -> Result<(), String> +where + F: Fn(Request, HttpReply) -> Fut + Clone + Send + Sync + 'static, + Fut: Future<Output = Result<(), String>> + Send + 'static, +{ + let (mut owner, spawner, fault) = task::scoped_owner(())?; + let first_request = super::server_h2::FirstRequest::new(); + let service = { + let first_request = first_request.clone(); + service_fn(move |request| { + first_request.mark(); + let route = route.clone(); + dispatch_http2(request, route, budget, live) + }) + }; + let mut builder = http2.server_builder(super::executor::Http2Executor(spawner), budget.headers); + super::server_h2::configure_liveness(&mut builder, budget.timeout); + let connection = builder.serve_connection( + hyper_util::rt::TokioIo::new(WriteTimeout::new(stream, budget.timeout)), + service, + ); + let result = owner + .enter(super::server_h2::drive( + connection, + &listener, + &first_request, + budget.timeout, + &fault, + |connection| connection.graceful_shutdown(), + )) + .await; + super::server_h2::complete(owner, result).await +} + +async fn dispatch<F, Fut>( + mut request: hyper::Request<hyper::body::Incoming>, + route: F, + budget: Budget, + live: LiveLimits, + handlers: Arc<Mutex<Option<task::Group>>>, + upgraded: Arc<AtomicBool>, +) -> Result<hyper::Response<Outgoing>, std::convert::Infallible> +where + F: Fn(Request, HttpReply) -> Fut + Send + 'static, + Fut: Future<Output = Result<(), String>> + Send + 'static, +{ + let upgrade = if request.headers().contains_key(hyper::header::UPGRADE) { + let upgrade = hyper::upgrade::on(&mut request); + let mut handshake = hyper::Request::new(()); + *handshake.method_mut() = request.method().clone(); + *handshake.uri_mut() = request.uri().clone(); + *handshake.version_mut() = request.version(); + *handshake.headers_mut() = request.headers().clone(); + Some((handshake, upgrade)) + } else { + None + }; + let request = match read_request(request, budget).await { + Ok(request) => request, + Err(status) => return Ok(reject(status, budget).map(Outgoing::Full)), + }; + let context = RequestContext::new("http/1.1", request.target.clone(), None); + let (session, head) = Session::new(budget, live, upgrade, upgraded); + let reply = session.reply(); + let state = reply.clone(); + let handler_context = context.clone(); + let started = handlers + .lock() + .await + .as_mut() + .expect("active connection") + .run(async move { + let result = session.run(route(request, reply)).await; + if let Err(error) = result { + handler_context.report(error); + } + Ok(()) + }) + .await; + if let Err(error) = started { + context.report(error); + return Ok(reject(hyper::StatusCode::INTERNAL_SERVER_ERROR, budget).map(Outgoing::Full)); + } + match tokio::time::timeout(budget.timeout, head).await { + Ok(Ok(response)) => Ok(response), + Ok(Err(_)) => { + if state.fault().is_none() { + context.report("HTTP live handler ended without a response"); + } + Ok(reject(hyper::StatusCode::INTERNAL_SERVER_ERROR, budget).map(Outgoing::Full)) + } + Err(_) => Ok(reject(hyper::StatusCode::GATEWAY_TIMEOUT, budget).map(Outgoing::Full)), + } +} + +async fn dispatch_http2<F, Fut>( + request: hyper::Request<hyper::body::Incoming>, + route: F, + budget: Budget, + live: LiveLimits, +) -> Result<hyper::Response<Outgoing>, std::convert::Infallible> +where + F: Fn(Request, HttpReply) -> Fut + Send + 'static, + Fut: Future<Output = Result<(), String>> + Send + 'static, +{ + if request.method() == hyper::Method::CONNECT + || request.headers().contains_key(hyper::header::UPGRADE) + { + return Ok(reject(hyper::StatusCode::NOT_IMPLEMENTED, budget).map(Outgoing::Full)); + } + let request = match read_request(request, budget).await { + Ok(request) => request, + Err(status) => return Ok(reject(status, budget).map(Outgoing::Full)), + }; + let context = RequestContext::new("http/2", request.target.clone(), None); + let (session, head) = Session::new(budget, live, None, Arc::new(AtomicBool::new(false))); + let reply = session.reply(); + let state = reply.clone(); + let mut handler = match task::Group::new(1) { + Ok(handler) => handler, + Err(error) => { + context.report(error); + return Ok(reject(hyper::StatusCode::INTERNAL_SERVER_ERROR, budget).map(Outgoing::Full)); + } + }; + let handler_context = context.clone(); + let started = handler + .run(async move { + let result = session.run(route(request, reply)).await; + if let Err(error) = result { + handler_context.report(error); + } + Ok(()) + }) + .await; + if let Err(error) = started { + context.report(error); + return Ok(reject(hyper::StatusCode::INTERNAL_SERVER_ERROR, budget).map(Outgoing::Full)); + } + match tokio::time::timeout(budget.timeout, head).await { + Ok(Ok(response)) => Ok(response.map(|body| body.with_handler(handler))), + Ok(Err(_)) => { + if let Err(error) = handler.wait().await { + context.report(error); + } else if state.fault().is_none() { + context.report("HTTP live handler ended without a response"); + } + Ok(reject(hyper::StatusCode::INTERNAL_SERVER_ERROR, budget).map(Outgoing::Full)) + } + Err(_) => match handler.stop().await { + Ok(()) => Ok(reject(hyper::StatusCode::GATEWAY_TIMEOUT, budget).map(Outgoing::Full)), + Err(error) => { + context.report(error); + Ok(reject(hyper::StatusCode::GATEWAY_TIMEOUT, budget).map(Outgoing::Full)) + } + }, + } +} diff --git a/crates/dever-runtime/src/http/message.rs b/crates/dever-runtime/src/http/message.rs new file mode 100644 index 0000000..2ba8f14 --- /dev/null +++ b/crates/dever-runtime/src/http/message.rs @@ -0,0 +1,208 @@ +use http_body_util::{BodyExt, Full, LengthLimitError, Limited}; +use hyper::body::{Body, Incoming}; +use hyper::header::{CONTENT_LENGTH, HOST, TRANSFER_ENCODING, UPGRADE}; +use hyper::{HeaderMap, Method, StatusCode, Uri}; + +use super::{Budget, Header, Request, Response}; +use crate::bytes::Bytes; + +#[derive(Clone, Copy, Eq, PartialEq)] +enum MessageKind { + Request, + Response, +} + +pub(super) fn header_size(headers: &HeaderMap) -> usize { + headers + .iter() + .map(|(name, value)| name.as_str().len() + value.as_bytes().len() + 4) + .sum::<usize>() + + 2 +} + +pub(super) fn decode_headers(headers: HeaderMap) -> Vec<Header> { + headers + .iter() + .map(|(name, value)| Header { + name: name.to_string(), + value: Bytes::new(value.as_bytes().to_vec()), + }) + .collect() +} + +fn encode_headers( + headers: Vec<Header>, + budget: Budget, + kind: MessageKind, +) -> Result<HeaderMap, String> { + let mut result = HeaderMap::new(); + let mut length = 2; + for header in headers { + length += header.name.len() + header.value.values().len() + 4; + if length > budget.headers { + return Err("HTTP headers exceed header_bytes".into()); + } + let name = hyper::header::HeaderName::from_bytes(header.name.as_bytes()) + .map_err(|error| error.to_string())?; + if name == CONTENT_LENGTH || name == TRANSFER_ENCODING { + return Err("HTTP framing headers are owned by the engine".into()); + } + if name == UPGRADE { + return Err("HTTP upgrades are not supported yet".into()); + } + let value = hyper::header::HeaderValue::from_maybe_shared(header.value.into_http()) + .map_err(|error| error.to_string())?; + if budget.http2.is_some() { + check_http2_header(name.as_str(), value.as_bytes(), kind)?; + } + result + .try_append(name, value) + .map_err(|error| error.to_string())?; + } + Ok(result) +} + +fn check_http2_header(name: &str, value: &[u8], kind: MessageKind) -> Result<(), String> { + if matches!( + name, + "connection" | "keep-alive" | "proxy-connection" | "transfer-encoding" | "upgrade" + ) || (name == "te" + && (kind == MessageKind::Response || !value.eq_ignore_ascii_case(b"trailers"))) + { + return Err(format!("HTTP/2 forbids connection-specific header {name}")); + } + Ok(()) +} + +// Source 继续使用 origin-form；scheme/authority 只在 HTTP/2 线协议边界补齐。 +pub(super) fn http2_request<B>( + request: &mut hyper::Request<B>, + scheme: &str, +) -> Result<(), String> { + let host = request + .headers() + .get(HOST) + .ok_or("HTTP request requires Host")? + .to_str() + .map_err(|error| error.to_string())?; + let authority = host + .parse::<hyper::http::uri::Authority>() + .map_err(|error| error.to_string())?; + if authority.as_str().contains('@') { + return Err("HTTP/2 authority cannot contain credentials".into()); + } + let mut uri = request.uri().clone().into_parts(); + uri.scheme = Some( + scheme + .parse::<hyper::http::uri::Scheme>() + .map_err(|error| error.to_string())?, + ); + uri.authority = Some(authority); + *request.uri_mut() = Uri::from_parts(uri).map_err(|error| error.to_string())?; + *request.version_mut() = hyper::Version::HTTP_2; + Ok(()) +} + +fn check_message( + headers: &HeaderMap, + body: &Bytes, + start_bytes: usize, + budget: Budget, +) -> Result<(), String> { + // 预留引擎生成的 Content-Length 和 Date；不截断用户内容。 + if header_size(headers) + start_bytes + 80 > budget.headers { + return Err("HTTP headers exceed header_bytes".into()); + } + if body.values().len() > budget.body { + return Err("HTTP body exceeds body_bytes".into()); + } + Ok(()) +} + +pub(super) fn encode_request( + request: Request, + budget: Budget, +) -> Result<hyper::Request<Full<bytes::Bytes>>, String> { + let method = + Method::from_bytes(request.method.as_bytes()).map_err(|error| error.to_string())?; + let uri = request + .target + .parse::<Uri>() + .map_err(|error| error.to_string())?; + if method == Method::CONNECT + || uri.scheme().is_some() + || uri.authority().is_some() + || !(request.target.starts_with('/') + || (method == Method::OPTIONS && request.target == "*")) + { + return Err( + "HTTP send requires an origin-form target or OPTIONS *; CONNECT is not supported" + .into(), + ); + } + let headers = encode_headers(request.headers, budget, MessageKind::Request)?; + if headers.get_all(HOST).iter().count() != 1 { + return Err("HTTP request requires exactly one Host header".into()); + } + let start_bytes = request.method.len() + request.target.len() + 14; + check_message(&headers, &request.body, start_bytes, budget)?; + let mut result = hyper::Request::new(Full::new(request.body.into_http())); + *result.method_mut() = method; + *result.uri_mut() = uri; + *result.headers_mut() = headers; + Ok(result) +} + +pub(super) fn encode_response( + response: Response, + budget: Budget, +) -> Result<hyper::Response<Full<bytes::Bytes>>, String> { + let status = u16::try_from(response.status) + .ok() + .filter(|status| (200..=599).contains(status)) + .ok_or("HTTP response status must be between 200 and 599")?; + let status = StatusCode::from_u16(status).map_err(|error| error.to_string())?; + let no_body = matches!( + status, + StatusCode::NO_CONTENT | StatusCode::RESET_CONTENT | StatusCode::NOT_MODIFIED + ); + if no_body && !response.body.values().is_empty() { + return Err("HTTP 204, 205 and 304 responses cannot contain a body".into()); + } + let headers = encode_headers(response.headers, budget, MessageKind::Response)?; + check_message(&headers, &response.body, 32, budget)?; + let mut result = hyper::Response::new(Full::new(response.body.into_http())); + *result.status_mut() = status; + *result.headers_mut() = headers; + Ok(result) +} + +pub(super) async fn collect_body(body: Incoming, limit: usize) -> Result<Bytes, StatusCode> { + if body.size_hint().lower() > limit as u64 { + return Err(StatusCode::PAYLOAD_TOO_LARGE); + } + let mut incoming = Limited::new(body, limit); + let mut body = Bytes::new(Vec::new()); + // 单帧直接共享，多帧逐步扩容；不为大量微小帧保留一份元数据队列。 + while let Some(frame) = incoming.frame().await { + let frame = frame.map_err(|error| { + if error.is::<LengthLimitError>() { + StatusCode::PAYLOAD_TOO_LARGE + } else { + StatusCode::BAD_REQUEST + } + })?; + match frame.into_data() { + Ok(bytes) => body = body.concat(&Bytes::from_http(bytes)), + Err(frame) + if frame + .trailers_ref() + .is_some_and(|trailers| !trailers.is_empty()) => + { + return Err(StatusCode::NOT_IMPLEMENTED); + } + Err(_) => {} + } + } + Ok(body) +} diff --git a/crates/dever-runtime/src/http/mod.rs b/crates/dever-runtime/src/http/mod.rs new file mode 100644 index 0000000..f56b82c --- /dev/null +++ b/crates/dever-runtime/src/http/mod.rs @@ -0,0 +1,550 @@ +mod client; +mod client_body; +mod client_driver; +mod executor; +mod live; +mod message; +mod protocol; +pub(crate) mod reply; +mod server_h2; +mod write_timeout; + +pub use protocol::Http2Limits; + +pub use client::{ + HttpClient, PoolLimits, StreamRequest, StreamResponse, client, close_client, open, open_stream, + request, +}; +pub use live::{serve_live, serve_live_tls}; +pub use reply::{HttpReply, LiveLimits, finish, respond, start, write}; + +use std::convert::Infallible; +use std::future::Future; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use http_body_util::Full; +use hyper::body::Incoming; +use hyper::service::service_fn; +use hyper::{Method, StatusCode}; +use hyper_util::rt::{TokioIo, TokioTimer}; + +use crate::bytes::Bytes; +use crate::{net, task}; +use message::{collect_body, decode_headers, encode_request, encode_response, header_size}; +use write_timeout::WriteTimeout; + +static NEXT_REQUEST_ID: AtomicU64 = AtomicU64::new(1); +tokio::task_local! { static CURRENT_REQUEST: RequestContext; } + +#[cfg(feature = "wire")] +pub(crate) fn request_id() -> u64 { + CURRENT_REQUEST + .try_with(|request| request.id) + .unwrap_or_else(|_| NEXT_REQUEST_ID.fetch_add(1, Ordering::Relaxed)) +} + +#[cfg(feature = "wire")] +pub(crate) fn client_address() -> Option<String> { + CURRENT_REQUEST + .try_with(|request| request.client_address.clone()) + .ok() + .flatten() +} + +#[derive(Clone)] +struct RequestContext { + id: u64, + protocol: &'static str, + target: String, + client_address: Option<String>, +} + +impl RequestContext { + fn new(protocol: &'static str, target: String, client_address: Option<String>) -> Self { + Self { + id: NEXT_REQUEST_ID.fetch_add(1, Ordering::Relaxed), + protocol, + target, + client_address, + } + } + + fn report(&self, error: impl Into<String>) { + crate::log::error( + error, + vec![ + crate::log::Field { + name: "boundary".into(), + value: "http".into(), + }, + crate::log::Field { + name: "request_id".into(), + value: self.id.to_string(), + }, + crate::log::Field { + name: "protocol".into(), + value: self.protocol.into(), + }, + crate::log::Field { + name: "target".into(), + value: self.target.clone(), + }, + ], + ); + } +} + +#[derive(Clone, Debug)] +pub struct Header { + pub name: String, + pub value: Bytes, +} + +#[derive(Clone, Debug)] +pub struct Request { + pub method: String, + pub target: String, + pub headers: Vec<Header>, + pub body: Bytes, +} + +/// Server-only body ownership. Source-level HTTP values remain bounded Request values. +pub struct IncomingRequest { + pub head: Request, + pub body: Incoming, +} + +#[derive(Clone, Debug)] +pub struct Response { + pub status: i64, + pub headers: Vec<Header>, + pub body: Bytes, +} + +#[derive(Clone, Copy, Debug)] +pub struct Limits { + pub header_bytes: i64, + pub body_bytes: i64, + pub timeout_ms: i64, + pub connections: i64, + pub http2: Option<Http2Limits>, +} + +#[derive(Clone, Copy)] +struct Budget { + headers: usize, + body: usize, + timeout: Duration, + http2: Option<Http2Limits>, +} + +impl Limits { + fn check(self) -> Result<Budget, String> { + let headers = usize::try_from(self.header_bytes) + .ok() + .filter(|value| *value >= 8192) + .ok_or("HTTP header_bytes must be at least 8192")?; + let body = + usize::try_from(self.body_bytes).map_err(|_| "HTTP body_bytes must be nonnegative")?; + let millis = u64::try_from(self.timeout_ms) + .ok() + .filter(|value| *value > 0) + .ok_or("HTTP timeout_ms must be positive")?; + if !(1..=65536).contains(&self.connections) { + return Err("HTTP connections must be between 1 and 65536".into()); + } + if let Some(http2) = self.http2 { + http2.check(self.connections, headers)?; + } + Ok(Budget { + headers, + body, + timeout: Duration::from_millis(millis), + http2: self.http2, + }) + } +} + +/// 连接由现有任务组持有，取消 serve 会沿 Scope 排空连接和 handler 子任务。 +pub async fn serve<F, Fut>(route: F, listener: net::Listener, limits: Limits) -> Result<(), String> +where + F: Fn(Request) -> Fut + Clone + Send + Sync + 'static, + Fut: Future<Output = Result<Response, String>> + Send + 'static, +{ + serve_buffered(route, listener, limits, None).await +} + +pub async fn serve_tls<F, Fut>( + route: F, + listener: net::Listener, + limits: Limits, + tls: crate::tls::ServerTls, +) -> Result<(), String> +where + F: Fn(Request) -> Fut + Clone + Send + Sync + 'static, + Fut: Future<Output = Result<Response, String>> + Send + 'static, +{ + serve_buffered(route, listener, limits, Some(tls)).await +} + +async fn serve_buffered<F, Fut>( + route: F, + listener: net::Listener, + limits: Limits, + tls: Option<crate::tls::ServerTls>, +) -> Result<(), String> +where + F: Fn(Request) -> Fut + Clone + Send + Sync + 'static, + Fut: Future<Output = Result<Response, String>> + Send + 'static, +{ + let budget = limits.check()?; + serve_transport( + move |incoming: IncomingRequest| { + let route = route.clone(); + async move { + let mut request = incoming.head; + request.body = match tokio::time::timeout( + budget.timeout, + collect_body(incoming.body, budget.body), + ) + .await + { + Ok(Ok(body)) => body, + Ok(Err(status)) => { + return Ok(Response { + status: status.as_u16().into(), + headers: Vec::new(), + body: Bytes::new(Vec::new()), + }); + } + Err(_) => { + return Ok(Response { + status: 408, + headers: Vec::new(), + body: Bytes::new(Vec::new()), + }); + } + }; + route(request).await + } + }, + listener, + limits, + tls, + ) + .await +} + +pub async fn serve_incoming<F, Fut>( + route: F, + listener: net::Listener, + limits: Limits, +) -> Result<(), String> +where + F: Fn(IncomingRequest) -> Fut + Clone + Send + Sync + 'static, + Fut: Future<Output = Result<Response, String>> + Send + 'static, +{ + serve_transport(route, listener, limits, None).await +} + +async fn serve_transport<F, Fut>( + route: F, + listener: net::Listener, + limits: Limits, + tls: Option<crate::tls::ServerTls>, +) -> Result<(), String> +where + F: Fn(IncomingRequest) -> Fut + Clone + Send + Sync + 'static, + Fut: Future<Output = Result<Response, String>> + Send + 'static, +{ + let budget = limits.check()?; + let tls = tls.map(|tls| protocol::server_tls(tls, budget.http2)); + task::parallel_each_stream( + net::incoming(listener.clone(), true), + limits.connections, + move |connection| { + let route = route.clone(); + let tls = tls.clone(); + let listener = listener.clone(); + async move { + let stream = connection?; + let client_address = stream + .peer_addr() + .ok() + .map(|address| address.ip().to_string()); + let stream = match crate::tls::accept(stream, tls.as_ref(), limits.timeout_ms).await + { + Ok(stream) => stream, + Err(_) => return Ok(()), // A peer's TLS failure does not terminate the listener. + }; + if protocol::check_alpn(&stream, budget.http2).is_err() { + return Ok(()); + } + if let Some(http2) = budget.http2 { + return serve_http2(route, listener, stream, budget, http2, client_address) + .await; + } + let service = service_fn(move |request| { + dispatch(request, route.clone(), budget, client_address.clone()) + }); + let connection = server_builder(budget).serve_connection( + TokioIo::new(WriteTimeout::new(stream, budget.timeout)), + service, + ); + tokio::pin!(connection); + let result = tokio::select! { + result = &mut connection => result, + _ = listener.closed() => { + connection.as_mut().graceful_shutdown(); + connection.await + } + }; + connection_result(result) + } + }, + ) + .await +} + +async fn serve_http2<F, Fut>( + route: F, + listener: net::Listener, + stream: crate::transport::Transport, + budget: Budget, + http2: Http2Limits, + client_address: Option<String>, +) -> Result<(), String> +where + F: Fn(IncomingRequest) -> Fut + Clone + Send + Sync + 'static, + Fut: Future<Output = Result<Response, String>> + Send + 'static, +{ + let (mut owner, spawner, fault) = task::scoped_owner(())?; + let first_request = server_h2::FirstRequest::new(); + let service = { + let first_request = first_request.clone(); + service_fn(move |request| { + first_request.mark(); + let route = route.clone(); + dispatch(request, route, budget, client_address.clone()) + }) + }; + let mut builder = http2.server_builder(executor::Http2Executor(spawner), budget.headers); + server_h2::configure_liveness(&mut builder, budget.timeout); + let connection = builder.serve_connection( + TokioIo::new(WriteTimeout::new(stream, budget.timeout)), + service, + ); + let result = owner + .enter(server_h2::drive( + connection, + &listener, + &first_request, + budget.timeout, + &fault, + |connection| connection.graceful_shutdown(), + )) + .await; + server_h2::complete(owner, result).await +} + +fn server_builder(budget: Budget) -> hyper::server::conn::http1::Builder { + let mut builder = hyper::server::conn::http1::Builder::new(); + builder + .timer(TokioTimer::new()) + .header_read_timeout(budget.timeout) + .max_buf_size(budget.headers); + builder +} + +fn connection_result(result: Result<(), hyper::Error>) -> Result<(), String> { + // 对端断开、畸形报文和 I/O 超时只结束该连接；请求故障已在 dispatch 边界处理。 + let _ = result; + Ok(()) +} + +async fn dispatch<F, Fut>( + request: hyper::Request<Incoming>, + route: F, + budget: Budget, + client_address: Option<String>, +) -> Result<hyper::Response<Full<bytes::Bytes>>, Infallible> +where + F: Fn(IncomingRequest) -> Fut, + Fut: Future<Output = Result<Response, String>>, +{ + if request.method() == Method::CONNECT || request.headers().contains_key(hyper::header::UPGRADE) + { + return Ok(reject(StatusCode::NOT_IMPLEMENTED, budget)); + } + let request = match read_incoming(request, budget) { + Ok(request) => request, + Err(status) => return Ok(reject(status, budget)), + }; + let context = RequestContext::new( + if budget.http2.is_some() { + "http/2" + } else { + "http/1.1" + }, + request.head.target.clone(), + client_address, + ); + let response = match tokio::time::timeout( + budget.timeout, + CURRENT_REQUEST.scope(context.clone(), route(request)), + ) + .await + { + Ok(Ok(response)) => response, + Ok(Err(error)) => { + context.report(error); + return Ok(reject(StatusCode::INTERNAL_SERVER_ERROR, budget)); + } + Err(_) => return Ok(reject(StatusCode::GATEWAY_TIMEOUT, budget)), + }; + match encode_response(response, budget) { + Ok(response) => Ok(response), + Err(error) => { + context.report(error); + Ok(reject(StatusCode::INTERNAL_SERVER_ERROR, budget)) + } + } +} + +async fn read_request( + request: hyper::Request<Incoming>, + budget: Budget, +) -> Result<Request, StatusCode> { + let incoming = read_incoming(request, budget)?; + let mut request = incoming.head; + request.body = tokio::time::timeout(budget.timeout, collect_body(incoming.body, budget.body)) + .await + .map_err(|_| StatusCode::REQUEST_TIMEOUT)??; + Ok(request) +} + +fn read_incoming( + mut request: hyper::Request<Incoming>, + budget: Budget, +) -> Result<IncomingRequest, StatusCode> { + if request.method() == Method::CONNECT { + return Err(StatusCode::NOT_IMPLEMENTED); + } + let host_count = request + .headers() + .get_all(hyper::header::HOST) + .iter() + .count(); + let target = if request.version() == hyper::Version::HTTP_2 { + let authority = request + .uri() + .authority() + .ok_or(StatusCode::BAD_REQUEST)? + .as_str() + .to_owned(); + if host_count > 1 { + return Err(StatusCode::BAD_REQUEST); + } + if let Some(host) = request.headers().get(hyper::header::HOST) { + if !host.as_bytes().eq_ignore_ascii_case(authority.as_bytes()) { + return Err(StatusCode::BAD_REQUEST); + } + } else { + let host = hyper::header::HeaderValue::from_str(&authority) + .map_err(|_| StatusCode::BAD_REQUEST)?; + request.headers_mut().insert(hyper::header::HOST, host); + } + request + .uri() + .path_and_query() + .map(|value| value.as_str()) + .unwrap_or("/") + .to_owned() + } else { + if request.version() == hyper::Version::HTTP_11 && host_count != 1 { + return Err(StatusCode::BAD_REQUEST); + } + request.uri().to_string() + }; + let start_bytes = request.method().as_str().len() + target.len() + 14; + if header_size(request.headers()) + start_bytes > budget.headers { + return Err(StatusCode::REQUEST_HEADER_FIELDS_TOO_LARGE); + } + let (head, body) = request.into_parts(); + Ok(IncomingRequest { + head: Request { + method: head.method.to_string(), + target, + headers: decode_headers(head.headers), + body: Bytes::new(Vec::new()), + }, + body, + }) +} + +fn reject(status: StatusCode, budget: Budget) -> hyper::Response<Full<bytes::Bytes>> { + let mut response = hyper::Response::new(Full::new(bytes::Bytes::new())); + *response.status_mut() = status; + if budget.http2.is_none() { + response.headers_mut().insert( + hyper::header::CONNECTION, + hyper::header::HeaderValue::from_static("close"), + ); + } + response +} + +/// 单次请求独占驱动 future；完成、错误和取消都直接释放连接，不产生后台任务。 +pub async fn send( + address: &str, + port: i64, + request: Request, + limits: Limits, +) -> Result<Response, String> { + let budget = limits.check()?; + let request = encode_request(request, budget)?; + if budget.http2.is_some() { + return client::send_http2(address, port, request, budget).await; + } + tokio::time::timeout(budget.timeout, async { + let stream = net::connect_stream_timeout(address, port, limits.timeout_ms).await?; + let (mut sender, connection) = hyper::client::conn::http1::Builder::new() + .max_buf_size(budget.headers) + .handshake(TokioIo::new(stream)) + .await + .map_err(|error| error.to_string())?; + let exchange = async { + let response = sender + .send_request(request) + .await + .map_err(|error| error.to_string())?; + if response.status() == StatusCode::SWITCHING_PROTOCOLS { + return Err("HTTP upgrades are not supported by send".into()); + } + if header_size(response.headers()) + 32 > budget.headers { + return Err("HTTP response headers exceed header_bytes".into()); + } + let (head, body) = response.into_parts(); + let body = collect_body(body, budget.body) + .await + .map_err(|status| format!("HTTP response body rejected: {status}"))?; + Ok(Response { + status: i64::from(head.status.as_u16()), + headers: decode_headers(head.headers), + body, + }) + }; + tokio::pin!(exchange, connection); + tokio::select! { + result = &mut exchange => result, + result = &mut connection => { + result.map_err(|error| error.to_string())?; + // Connection: close 的最后一帧可能已入队，继续取完响应。 + exchange.await + } + } + }) + .await + .map_err(|_| "HTTP request timed out".to_owned())? +} diff --git a/crates/dever-runtime/src/http/protocol.rs b/crates/dever-runtime/src/http/protocol.rs new file mode 100644 index 0000000..d075d82 --- /dev/null +++ b/crates/dever-runtime/src/http/protocol.rs @@ -0,0 +1,97 @@ +use crate::{ + tls::{ClientTls, ServerTls}, + transport::Transport, +}; + +#[derive(Clone, Copy, Debug)] +pub struct Http2Limits { + pub streams: i64, + pub stream_window_bytes: i64, + pub connection_window_bytes: i64, +} + +impl Http2Limits { + pub(super) fn check(self, connections: i64, headers: usize) -> Result<(), String> { + if !(1..=65536).contains(&self.streams) || connections * self.streams > 65536 { + return Err("HTTP/2 connections * streams must be between 1 and 65536".into()); + } + if !(1..=i32::MAX as i64).contains(&self.stream_window_bytes) { + return Err("HTTP/2 stream_window_bytes must be between 1 and 2147483647".into()); + } + if !(65535..=i32::MAX as i64).contains(&self.connection_window_bytes) { + return Err( + "HTTP/2 connection_window_bytes must be between 65535 and 2147483647".into(), + ); + } + if u32::try_from(headers).is_err() { + return Err("HTTP/2 header_bytes must fit an unsigned 32-bit limit".into()); + } + Ok(()) + } + + pub(super) fn client_builder<E: Clone>( + self, + executor: E, + headers: usize, + ) -> hyper::client::conn::http2::Builder<E> { + let mut builder = hyper::client::conn::http2::Builder::new(executor); + builder + .timer(hyper_util::rt::TokioTimer::new()) + .adaptive_window(false) + .initial_stream_window_size(self.stream_window_bytes as u32) + .initial_connection_window_size(self.connection_window_bytes as u32) + .initial_max_send_streams(self.streams as usize) + .max_concurrent_streams(0) + .max_header_list_size(headers as u32) + .max_frame_size(FRAME_BYTES) + .max_send_buf_size(FRAME_BYTES as usize); + builder + } + + pub(super) fn server_builder<E>( + self, + executor: E, + headers: usize, + ) -> hyper::server::conn::http2::Builder<E> { + let mut builder = hyper::server::conn::http2::Builder::new(executor); + builder + .timer(hyper_util::rt::TokioTimer::new()) + .adaptive_window(false) + .initial_stream_window_size(self.stream_window_bytes as u32) + .initial_connection_window_size(self.connection_window_bytes as u32) + .max_concurrent_streams(self.streams as u32) + .max_header_list_size(headers as u32) + .max_frame_size(FRAME_BYTES) + .max_send_buf_size(FRAME_BYTES as usize); + builder + } +} + +const FRAME_BYTES: u32 = 16384; + +// HTTP 单独派生 ALPN；不能修改 WSS 和其他 TLS 使用者共享的配置。 +pub(super) fn client_tls(tls: ClientTls, http2: Option<Http2Limits>) -> ClientTls { + let mut config = (*tls.0).clone(); + config.alpn_protocols = vec![alpn(http2).to_vec()]; + ClientTls(std::sync::Arc::new(config)) +} + +pub(super) fn server_tls(tls: ServerTls, http2: Option<Http2Limits>) -> ServerTls { + let mut config = (*tls.0).clone(); + config.alpn_protocols = vec![alpn(http2).to_vec()]; + ServerTls(std::sync::Arc::new(config)) +} + +fn alpn(http2: Option<Http2Limits>) -> &'static [u8] { + if http2.is_some() { b"h2" } else { b"http/1.1" } +} + +pub(super) fn check_alpn(stream: &Transport, http2: Option<Http2Limits>) -> Result<(), String> { + if let Transport::Tls(stream) = stream + && http2.is_some() + && stream.get_ref().1.alpn_protocol() != Some(b"h2") + { + return Err("HTTP/2 TLS peer did not negotiate h2".into()); + } + Ok(()) +} diff --git a/crates/dever-runtime/src/http/reply.rs b/crates/dever-runtime/src/http/reply.rs new file mode 100644 index 0000000..b1e0c68 --- /dev/null +++ b/crates/dever-runtime/src/http/reply.rs @@ -0,0 +1,456 @@ +use std::future::Future; +use std::pin::Pin; +use std::sync::{ + Arc, Mutex, + atomic::{AtomicBool, Ordering}, +}; +use std::task::{Context, Poll}; +use std::time::Duration; + +use http_body_util::Full; +use hyper::body::{Body, Frame, SizeHint}; +use tokio::sync::{Mutex as AsyncMutex, Semaphore, mpsc, oneshot}; +use tokio::time::Sleep; + +use super::{Budget, Header, Response, message::encode_response}; +use crate::{bytes::Bytes, task, websocket}; + +#[derive(Clone, Copy, Debug)] +pub struct LiveLimits { + pub chunk_bytes: i64, + pub idle_ms: i64, + pub heartbeat_ms: i64, +} + +impl LiveLimits { + pub(super) fn check(self) -> Result<(), String> { + if self.chunk_bytes <= 0 || usize::try_from(self.chunk_bytes).is_err() { + return Err("HTTP chunk_bytes must be positive".into()); + } + if self.idle_ms <= 0 || self.heartbeat_ms <= 0 || self.heartbeat_ms >= self.idle_ms { + return Err("HTTP live limits require 0 < heartbeat_ms < idle_ms".into()); + } + Ok(()) + } +} + +type Head = hyper::Response<Outgoing>; +enum Mode { + Pending, + Streaming { + sender: mpsc::Sender<bytes::Bytes>, + sse: bool, + }, + Finished, + Upgraded(Option<websocket::WebSocket>), +} + +struct State { + mode: Mode, + head: Option<oneshot::Sender<Head>>, + upgrade: Option<(hyper::Request<()>, hyper::upgrade::OnUpgrade)>, + fault: Option<String>, + body_cancelled: bool, +} + +struct Shared { + state: Mutex<State>, + writer: AsyncMutex<()>, + closed: Semaphore, + budget: Budget, + live: LiveLimits, + upgraded: Arc<AtomicBool>, +} + +impl Shared { + fn close_body(&self, cancelled: bool) { + if cancelled { + self.state + .lock() + .expect("reply lock poisoned") + .body_cancelled = true; + } + self.closed.close(); + } +} + +#[derive(Clone)] +pub struct HttpReply(Arc<Shared>); + +impl std::fmt::Debug for HttpReply { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("HttpReply(<resource>)") + } +} + +impl HttpReply { + pub(crate) fn chunk_limit(&self) -> usize { + self.0.live.chunk_bytes as usize + } + + pub(crate) fn fault(&self) -> Option<String> { + self.0 + .state + .lock() + .expect("reply lock poisoned") + .fault + .clone() + } + + fn publish(&self, prepare: impl FnOnce() -> (Head, Mode)) -> Result<(), String> { + let mut state = self.0.state.lock().expect("reply lock poisoned"); + if !matches!(state.mode, Mode::Pending) { + return Err("HTTP response has already started".into()); + } + let head = state.head.take().ok_or("HTTP response is closed")?; + // 只有真正提交的正文才能拥有取消信号；拒绝重复提交时不能关闭已有响应。 + let (response, mode) = prepare(); + state.upgrade.take(); + state.mode = mode; + // SendError 会析构响应正文；先释放状态锁，正文 Drop 才能登记传输取消。 + drop(state); + if head.send(response).is_err() { + // 空正文从创建起就是 end-stream，不能只依赖 BufferedBody::drop 判断取消。 + self.0.close_body(true); + return Err("HTTP peer is closed".into()); + } + Ok(()) + } + + pub(crate) async fn upgrade( + &self, + limits: websocket::Limits, + ) -> Result<websocket::WebSocket, String> { + limits.check()?; + let (head, response, upgrade) = { + let mut state = self.0.state.lock().expect("reply lock poisoned"); + if !matches!(state.mode, Mode::Pending) { + return Err("HTTP response has already started".into()); + } + let (request, upgrade) = state.upgrade.take().ok_or("HTTP upgrade is unavailable")?; + let response = + tokio_tungstenite::tungstenite::handshake::server::create_response(&request) + .map_err(|error| error.to_string())?; + let response = response.map(|()| Outgoing::Full(Full::new(bytes::Bytes::new()))); + state.mode = Mode::Upgraded(None); + self.0.upgraded.store(true, Ordering::Release); + let head = state.head.take().ok_or("HTTP response is closed")?; + (head, response, upgrade) + }; + if head.send(response).is_err() { + self.0.close_body(true); + return Err("HTTP peer is closed".into()); + } + let stream = tokio::time::timeout(self.0.budget.timeout, upgrade) + .await + .map_err(|_| "WebSocket upgrade timed out")? + .map_err(|error| error.to_string())?; + // 取回已知的底层 I/O 和预读帧；升级后不再继承 HTTP 写入期限。 + let parts = stream + .downcast::<hyper_util::rt::TokioIo<super::write_timeout::WriteTimeout>>() + .map_err(|_| "WebSocket upgrade transport does not match the HTTP server")?; + let socket = websocket::from_server( + parts.io.into_inner().into_inner(), + parts.read_buf.to_vec(), + limits, + ) + .await?; + let mut state = self.0.state.lock().expect("reply lock poisoned"); + state.mode = Mode::Upgraded(Some(socket.clone())); + Ok(socket) + } +} + +pub(super) struct Session(HttpReply); + +impl Session { + pub(super) fn new( + budget: Budget, + live: LiveLimits, + upgrade: Option<(hyper::Request<()>, hyper::upgrade::OnUpgrade)>, + upgraded: Arc<AtomicBool>, + ) -> (Self, oneshot::Receiver<Head>) { + let (sender, receiver) = oneshot::channel(); + ( + Self(HttpReply(Arc::new(Shared { + state: Mutex::new(State { + mode: Mode::Pending, + head: Some(sender), + upgrade, + fault: None, + body_cancelled: false, + }), + writer: AsyncMutex::new(()), + closed: Semaphore::new(0), + budget, + live, + upgraded, + }))), + receiver, + ) + } + + pub(super) fn reply(&self) -> HttpReply { + self.0.clone() + } + + pub(super) async fn run( + self, + route: impl Future<Output = Result<(), String>>, + ) -> Result<(), String> { + let result = tokio::select! { + biased; + result = route => result, + _ = self.0.0.closed.acquire() => Ok(()), + }; + if let Err(error) = &result { + let mut state = self.0.0.state.lock().expect("reply lock poisoned"); + // 传输侧先取消正文时，随后被唤醒的 write 失败属于该流，不是程序故障。 + if state.body_cancelled { + return Ok(()); + } + state.fault = Some(error.clone()); + } + result + } +} + +impl Drop for Session { + fn drop(&mut self) { + let mut state = self.0.0.state.lock().expect("reply lock poisoned"); + // 函数返回或取消都释放正文发送端/升级连接；外部别名不能延长会话。 + if let Mode::Upgraded(Some(socket)) = &state.mode { + socket.abort(); + } + state.mode = Mode::Finished; + state.upgrade.take(); + state.head.take(); + } +} + +pub async fn respond(reply: &HttpReply, response: Response) -> Result<(), String> { + let _writer = reply.0.writer.lock().await; + let response = encode_response(response, reply.0.budget)?; + reply.publish(|| { + ( + response.map(|body| { + Outgoing::Buffered(BufferedBody { + body, + reply: reply.clone(), + }) + }), + Mode::Finished, + ) + }) +} + +pub async fn start(reply: &HttpReply, status: i64, headers: Vec<Header>) -> Result<(), String> { + start_stream(reply, status, headers, false).await +} + +pub(crate) async fn start_stream( + reply: &HttpReply, + status: i64, + headers: Vec<Header>, + sse: bool, +) -> Result<(), String> { + if matches!(status, 204 | 205 | 304) { + return Err("bodyless status requires http.respond".into()); + } + let _writer = reply.0.writer.lock().await; + let response = encode_response( + Response { + status, + headers, + body: Bytes::new(Vec::new()), + }, + reply.0.budget, + )?; + reply.publish(|| { + let (sender, receiver) = mpsc::channel(1); + let millis = if sse { + reply.0.live.heartbeat_ms + } else { + reply.0.live.idle_ms + }; + let interval = Duration::from_millis(millis as u64); + let body = StreamBody { + receiver, + reply: reply.clone(), + interval, + deadline: Box::pin(tokio::time::sleep(interval)), + sse, + ended: false, + }; + ( + response.map(|_| Outgoing::Stream(body)), + Mode::Streaming { sender, sse }, + ) + }) +} + +pub async fn write(reply: &HttpReply, bytes: Bytes) -> Result<(), String> { + write_chunk(reply, bytes, false).await +} + +pub(crate) async fn write_chunk(reply: &HttpReply, bytes: Bytes, sse: bool) -> Result<(), String> { + if bytes.values().len() > reply.chunk_limit() { + return Err("HTTP chunk exceeds chunk_bytes".into()); + } + tokio::time::timeout(reply.0.budget.timeout, async { + let _writer = reply.0.writer.lock().await; + let sender = { + let state = reply.0.state.lock().expect("reply lock poisoned"); + match &state.mode { + Mode::Streaming { sender, sse: mode } if *mode == sse => sender.clone(), + _ => return Err("HTTP reply is not an open stream of this kind".into()), + } + }; + sender + .send(bytes.into_http()) + .await + .map_err(|_| "HTTP peer is closed".into()) + }) + .await + .map_err(|_| "HTTP stream write timed out".to_owned())? +} + +pub async fn finish(reply: &HttpReply) -> Result<(), String> { + let _writer = reply.0.writer.lock().await; + let mut state = reply.0.state.lock().expect("reply lock poisoned"); + if !matches!(state.mode, Mode::Streaming { .. }) { + return Err("HTTP reply is not an open stream".into()); + } + state.mode = Mode::Finished; + Ok(()) +} + +pub(super) enum Outgoing { + Full(Full<bytes::Bytes>), + Buffered(BufferedBody), + Stream(StreamBody), + Handler(HandlerBody), +} + +impl Outgoing { + pub(super) fn with_handler(self, handler: task::Group) -> Self { + Self::Handler(HandlerBody { + body: Box::new(self), + _handler: handler, + }) + } +} + +pub(super) struct HandlerBody { + body: Box<Outgoing>, + _handler: task::Group, +} + +pub(super) struct BufferedBody { + body: Full<bytes::Bytes>, + reply: HttpReply, +} + +impl Drop for BufferedBody { + fn drop(&mut self) { + self.reply.0.close_body(!self.body.is_end_stream()); + } +} + +pub(super) struct StreamBody { + receiver: mpsc::Receiver<bytes::Bytes>, + reply: HttpReply, + interval: Duration, + deadline: Pin<Box<Sleep>>, + sse: bool, + ended: bool, +} + +impl Drop for StreamBody { + fn drop(&mut self) { + self.reply.0.close_body(!self.ended); + } +} + +impl Body for Outgoing { + type Data = bytes::Bytes; + type Error = std::io::Error; + + fn poll_frame( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll<Option<Result<Frame<Self::Data>, Self::Error>>> { + let body = match self.get_mut() { + Self::Stream(body) => body, + Self::Full(body) => { + return Pin::new(body) + .poll_frame(cx) + .map(|frame| frame.map(|frame| frame.map_err(|never| match never {}))); + } + Self::Buffered(body) => { + return Pin::new(&mut body.body) + .poll_frame(cx) + .map(|frame| frame.map(|frame| frame.map_err(|never| match never {}))); + } + Self::Handler(body) => return Pin::new(body.body.as_mut()).poll_frame(cx), + }; + if body.ended { + return Poll::Ready(None); + } + match body.receiver.poll_recv(cx) { + Poll::Ready(Some(bytes)) => { + body.deadline + .as_mut() + .reset(tokio::time::Instant::now() + body.interval); + return Poll::Ready(Some(Ok(Frame::data(bytes)))); + } + Poll::Ready(None) => { + body.ended = true; + let fault = body + .reply + .0 + .state + .lock() + .expect("reply lock poisoned") + .fault + .clone(); + return Poll::Ready(fault.map(|fault| Err(std::io::Error::other(fault)))); + } + Poll::Pending => {} + } + if body.deadline.as_mut().poll(cx).is_pending() { + return Poll::Pending; + } + body.deadline + .as_mut() + .reset(tokio::time::Instant::now() + body.interval); + if body.sse { + Poll::Ready(Some(Ok(Frame::data(bytes::Bytes::from_static(b":\n\n"))))) + } else { + body.ended = true; + body.reply.0.close_body(true); + Poll::Ready(Some(Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "HTTP stream idle timeout", + )))) + } + } + + fn is_end_stream(&self) -> bool { + match self { + Self::Full(body) => body.is_end_stream(), + Self::Buffered(body) => body.body.is_end_stream(), + Self::Stream(body) => body.ended, + Self::Handler(body) => body.body.is_end_stream(), + } + } + + fn size_hint(&self) -> SizeHint { + match self { + Self::Full(body) => body.size_hint(), + Self::Buffered(body) => body.body.size_hint(), + Self::Stream(_) => SizeHint::default(), + Self::Handler(body) => body.body.size_hint(), + } + } +} diff --git a/crates/dever-runtime/src/http/server_h2.rs b/crates/dever-runtime/src/http/server_h2.rs new file mode 100644 index 0000000..0a8d261 --- /dev/null +++ b/crates/dever-runtime/src/http/server_h2.rs @@ -0,0 +1,110 @@ +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use tokio::sync::Notify; + +use crate::{ + net, + task::{ScopeFault, ScopedOwner}, +}; + +pub(super) fn configure_liveness<E>( + builder: &mut hyper::server::conn::http2::Builder<E>, + timeout: Duration, +) { + builder + .keep_alive_interval(Some(timeout)) + .keep_alive_timeout(timeout); +} + +#[derive(Clone)] +pub(super) struct FirstRequest(Arc<FirstRequestState>); + +struct FirstRequestState { + seen: AtomicBool, + changed: Notify, +} + +impl FirstRequest { + pub(super) fn new() -> Self { + Self(Arc::new(FirstRequestState { + seen: AtomicBool::new(false), + changed: Notify::new(), + })) + } + + pub(super) fn mark(&self) { + if !self.0.seen.swap(true, Ordering::AcqRel) { + self.0.changed.notify_one(); + } + } + + async fn wait(&self) { + loop { + let changed = self.0.changed.notified(); + if self.0.seen.load(Ordering::Acquire) { + return; + } + changed.await; + } + } +} + +pub(super) async fn drive<C, Shutdown>( + connection: C, + listener: &net::Listener, + first_request: &FirstRequest, + timeout: Duration, + fault: &ScopeFault, + mut shutdown: Shutdown, +) -> Result<(), String> +where + C: Future<Output = Result<(), hyper::Error>>, + Shutdown: for<'a> FnMut(Pin<&'a mut C>), +{ + tokio::pin!(connection); + tokio::select! { + biased; + error = fault.wait() => return Err(error), + result = &mut connection => return super::connection_result(result), + _ = listener.closed() => { + shutdown(connection.as_mut()); + return drain(connection.as_mut(), fault).await; + } + _ = first_request.wait() => {} + _ = tokio::time::sleep(timeout) => return Ok(()), + } + tokio::select! { + biased; + error = fault.wait() => Err(error), + result = &mut connection => super::connection_result(result), + _ = listener.closed() => { + shutdown(connection.as_mut()); + drain(connection.as_mut(), fault).await + } + } +} + +async fn drain<C>(mut connection: Pin<&mut C>, fault: &ScopeFault) -> Result<(), String> +where + C: Future<Output = Result<(), hyper::Error>>, +{ + tokio::select! { + biased; + error = fault.wait() => Err(error), + result = connection.as_mut() => super::connection_result(result), + } +} + +pub(super) async fn complete(owner: ScopedOwner, result: Result<(), String>) -> Result<(), String> { + match result { + Ok(()) => owner.finish().await, + Err(error) => { + let _ = owner.stop().await; + Err(error) + } + } +} diff --git a/crates/dever-runtime/src/http/write_timeout.rs b/crates/dever-runtime/src/http/write_timeout.rs new file mode 100644 index 0000000..ddb78d9 --- /dev/null +++ b/crates/dever-runtime/src/http/write_timeout.rs @@ -0,0 +1,96 @@ +use std::future::Future; +use std::io::{self, IoSlice}; +use std::pin::Pin; +use std::task::{Context, Poll}; +use std::time::Duration; + +use crate::transport::Transport; +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; +use tokio::time::Sleep; + +// Header/body 的期限由协议层持有；这里仅限制无法取得进展的写操作。 +pub(super) struct WriteTimeout { + stream: Transport, + timeout: Duration, + deadline: Option<Pin<Box<Sleep>>>, +} + +impl WriteTimeout { + pub(super) fn into_inner(self) -> Transport { + self.stream + } + pub fn new(stream: Transport, timeout: Duration) -> Self { + Self { + stream, + timeout, + deadline: None, + } + } + + fn output<T>( + &mut self, + context: &mut Context<'_>, + write: impl FnOnce(Pin<&mut Transport>, &mut Context<'_>) -> Poll<io::Result<T>>, + ) -> Poll<io::Result<T>> { + match write(Pin::new(&mut self.stream), context) { + Poll::Ready(result) => { + self.deadline = None; + Poll::Ready(result) + } + Poll::Pending => { + let deadline = self + .deadline + .get_or_insert_with(|| Box::pin(tokio::time::sleep(self.timeout))); + match deadline.as_mut().poll(context) { + Poll::Ready(()) => Poll::Ready(Err(io::Error::new( + io::ErrorKind::TimedOut, + "HTTP write timed out", + ))), + Poll::Pending => Poll::Pending, + } + } + } + } +} + +impl AsyncRead for WriteTimeout { + fn poll_read( + mut self: Pin<&mut Self>, + context: &mut Context<'_>, + buffer: &mut ReadBuf<'_>, + ) -> Poll<io::Result<()>> { + Pin::new(&mut self.stream).poll_read(context, buffer) + } +} + +impl AsyncWrite for WriteTimeout { + fn poll_write( + mut self: Pin<&mut Self>, + context: &mut Context<'_>, + bytes: &[u8], + ) -> Poll<io::Result<usize>> { + self.output(context, |stream, context| stream.poll_write(context, bytes)) + } + + fn poll_write_vectored( + mut self: Pin<&mut Self>, + context: &mut Context<'_>, + buffers: &[IoSlice<'_>], + ) -> Poll<io::Result<usize>> { + self.output(context, |stream, context| { + stream.poll_write_vectored(context, buffers) + }) + } + + fn is_write_vectored(&self) -> bool { + self.stream.is_write_vectored() + } + + fn poll_flush(mut self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll<io::Result<()>> { + self.output(context, |stream, context| stream.poll_flush(context)) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll<io::Result<()>> { + self.output(context, |stream, context| stream.poll_shutdown(context)) + } +} diff --git a/crates/dever-runtime/src/job.rs b/crates/dever-runtime/src/job.rs new file mode 100644 index 0000000..3d98611 --- /dev/null +++ b/crates/dever-runtime/src/job.rs @@ -0,0 +1,343 @@ +//! Compiler-owned durable queue bridge. All persistence uses the ordinary database executor. +use crate::database::{Database, Executor, Sql, Transaction}; +use crate::orm::{self, Error, Row, Value}; +use crate::wire::Encoded; +use std::cell::RefCell; +use std::future::Future; + +pub mod resources; +pub mod store; +pub mod worker; + +pub use crate::time::Clock; + +tokio::task_local! { static EXECUTION_CONTEXT: ExecutionContext; } + +struct ExecutionContext { + identity: ExecutionIdentity, + permission_key: RefCell<Option<String>>, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ExecutionIdentity { + System { + tenant_id: Option<i64>, + }, + User { + tenant_id: Option<i64>, + provider: String, + site: String, + subject: String, + session: String, + tenant_claim: Option<String>, + permission_key: String, + }, +} + +impl ExecutionIdentity { + pub fn tenant_id(&self) -> Option<i64> { + match self { + Self::System { tenant_id } | Self::User { tenant_id, .. } => *tenant_id, + } + } + + fn validate(&self) -> Result<(), Error> { + if self.tenant_id().is_some_and(|tenant_id| tenant_id <= 0) { + return Err(Error::invalid_data( + "Job execution tenant id must be positive", + )); + } + if let Self::User { + provider, + site, + subject, + session, + tenant_claim, + permission_key, + .. + } = self + { + for (value, label) in [ + (provider.as_str(), "provider"), + (site.as_str(), "site"), + (subject.as_str(), "subject"), + (session.as_str(), "session"), + ] { + bounded(value, 256, &format!("Job execution {label}"))?; + } + if let Some(tenant_claim) = tenant_claim { + bounded(tenant_claim, 256, "Job execution tenant claim")?; + } + if !valid_permission_key(permission_key) { + return Err(Error::invalid_data( + "Job execution permission key is invalid", + )); + } + } + Ok(()) + } +} + +pub fn scope_execution<T>( + identity: ExecutionIdentity, + future: impl Future<Output = T>, +) -> impl Future<Output = T> { + // Pin before constructing the context wrapper so its poll frame does not + // carry another copy of a large business Future. + let future = Box::pin(future); + async move { + let permission_key = match &identity { + ExecutionIdentity::User { permission_key, .. } if !permission_key.is_empty() => { + Some(permission_key.clone()) + } + _ => None, + }; + EXECUTION_CONTEXT + .scope( + ExecutionContext { + identity, + permission_key: RefCell::new(permission_key), + }, + future, + ) + .await + } +} + +pub fn scope_system<T>( + tenant_id: Option<i64>, + future: impl Future<Output = T>, +) -> impl Future<Output = T> { + scope_execution(ExecutionIdentity::System { tenant_id }, future) +} + +pub fn execution_identity() -> Result<ExecutionIdentity, Error> { + EXECUTION_CONTEXT + .try_with(|context| { + let mut identity = context.identity.clone(); + if let ExecutionIdentity::User { permission_key, .. } = &mut identity { + *permission_key = context.permission_key.borrow().clone().unwrap_or_default(); + } + identity + }) + .map_err(|_| Error::invalid_data("trusted Job execution identity is required")) +} + +/// Compiled component ownership does not enable tenant isolation; deployment settings do. +pub async fn require_components( + identity: &ExecutionIdentity, + components: &[&str], + manifest: &[&str], +) -> Result<(), store::Outcome> { + if components.is_empty() || crate::config::current_settings().tenant().is_none() { + return Ok(()); + } + let tenant_id = identity + .tenant_id() + .ok_or(store::Outcome::InvalidIdentity)?; + crate::tenant::require_components(tenant_id, components, manifest) + .await + .map_err(|error| { + if error.kind() == orm::ErrorKind::InvalidData { + store::Outcome::ComponentDisabled + } else { + store::Outcome::Retry + } + }) +} + +#[cfg(feature = "api")] +pub fn bind_permission(permission_key: &str) -> Result<(), Error> { + if !valid_permission_key(permission_key) { + return Err(Error::invalid_data( + "Job execution permission key is invalid", + )); + } + EXECUTION_CONTEXT + .try_with(|context| match &context.identity { + ExecutionIdentity::User { .. } => { + let mut current = context.permission_key.borrow_mut(); + match current.as_deref() { + Some(value) if value != permission_key => Err(Error::invalid_data( + "Job execution permission cannot change within one request", + )), + Some(_) => Ok(()), + None => { + *current = Some(permission_key.to_owned()); + Ok(()) + } + } + } + ExecutionIdentity::System { .. } => Ok(()), + }) + .map_err(|_| Error::invalid_data("trusted Job execution identity is required"))? +} + +#[derive(Clone)] +pub struct Spec { + pub target: &'static str, + pub schema: &'static str, + pub attempts: u32, + pub timeout_ms: u32, +} + +impl Spec { + pub fn validate(&self) -> Result<(), Error> { + bounded(self.target, 1024, "Job target")?; + bounded(self.schema, 65536, "Job schema")?; + if !(1..=100).contains(&self.attempts) + || !(1..=crate::config::MAX_JOB_TIMEOUT_MS).contains(&self.timeout_ms) + { + return Err(Error::invalid_data("invalid Job retry or timeout policy")); + } + Ok(()) + } +} + +pub fn now() -> Result<i64, Error> { + crate::time::now().map_err(Error::invalid_data) +} + +fn bounded(value: &str, maximum: usize, label: &str) -> Result<(), Error> { + if value.is_empty() || value.len() > maximum { + return Err(Error::invalid_data(format!( + "{label} must contain 1..={maximum} bytes" + ))); + } + Ok(()) +} + +fn valid_permission_key(permission_key: &str) -> bool { + let parts = permission_key.split('.').collect::<Vec<_>>(); + parts.len() == 4 + && parts.iter().all(|part| { + part.starts_with(|character: char| character.is_ascii_lowercase()) + && part.split('_').all(|word| { + !word.is_empty() + && word.chars().all(|character| { + character.is_ascii_lowercase() || character.is_ascii_digit() + }) + }) + }) +} + +fn text(row: &Row, index: usize) -> Result<String, Error> { + orm::text(row.get(index)?.clone()) +} +fn int(row: &Row, index: usize) -> Result<i64, Error> { + orm::int(row.get(index)?.clone()) +} +fn t(value: impl Into<String>) -> Value { + Value::Text(value.into()) +} +fn n(value: i64) -> Value { + Value::Int(value) +} + +async fn finish_transaction<T>( + transaction: Transaction, + result: Result<T, Error>, +) -> Result<T, Error> { + match result { + Ok(value) => { + transaction.commit().await?; + Ok(value) + } + Err(error) => match transaction.rollback().await { + Ok(()) => Err(error), + Err(rollback) => Err(error.cause(rollback)), + }, + } +} + +pub async fn enqueue( + database: Database, + ambient: Option<&Transaction>, + spec: &Spec, + payload: Encoded, + key: &str, + run_at: i64, +) -> Result<crate::Id, Error> { + spec.validate()?; + bounded(key, 1024, "Job idempotency key")?; + // Business keys and scheduler keys occupy disjoint namespaces. + let key = format!("business:{key}"); + let tenant_id = database.key().tenant_id().unwrap_or(0); + let identity = execution_identity()?; + identity.validate()?; + if tenant_id > 0 && identity.tenant_id() != Some(tenant_id) { + return Err(Error::invalid_data( + "Job execution tenant does not match tenant storage", + )); + } + if let Some(transaction) = ambient { + let executor = Executor::new(database, Some(transaction))?; + return insert( + &executor, + tenant_id, + &identity, + spec, + payload.as_str(), + &key, + run_at, + ) + .await + .map(crate::Id); + } + let transaction = database.begin().await?; + let executor = Executor::new(database, Some(&transaction))?; + let result = insert( + &executor, + tenant_id, + &identity, + spec, + payload.as_str(), + &key, + run_at, + ) + .await; + finish_transaction(transaction, result).await.map(crate::Id) +} + +async fn insert( + executor: &Executor<'_>, + tenant_id: i64, + identity: &ExecutionIdentity, + spec: &Spec, + payload: &str, + key: &str, + run_at: i64, +) -> Result<String, Error> { + let id = orm::Uuid::new_v7()?.to_string(); + let (kind, scope_tenant_id, provider, site, subject, session, tenant_claim, permission_key) = + match identity { + ExecutionIdentity::System { tenant_id } => { + ("system", tenant_id.unwrap_or(0), "", "", "", "", "", "") + } + ExecutionIdentity::User { + tenant_id, + provider, + site, + subject, + session, + tenant_claim, + permission_key, + } => ( + "user", + tenant_id.unwrap_or(0), + provider.as_str(), + site.as_str(), + subject.as_str(), + session.as_str(), + tenant_claim.as_deref().unwrap_or(""), + permission_key.as_str(), + ), + }; + // RETURNING holds the conflicting row through this transaction. A separate + // SELECT could miss it if a worker completed the active job in between. + let rows = executor.query(Sql { + sqlite: "INSERT INTO _dever_jobs (id,tenant_id,scope_tenant_id,execution_kind,auth_provider,auth_site,auth_subject,auth_session,auth_tenant,auth_permission,target,schema,payload,dedupe_key,state,run_at,attempt,max_attempts,timeout_ms,claim_token,lease_until,error,updated_at) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13,?14,'pending',?15,0,?16,?17,'',0,'',?15) ON CONFLICT(tenant_id,scope_tenant_id,target,dedupe_key) WHERE state IN ('pending','running') DO UPDATE SET dedupe_key=excluded.dedupe_key RETURNING id", + postgres: "INSERT INTO _dever_jobs (id,tenant_id,scope_tenant_id,execution_kind,auth_provider,auth_site,auth_subject,auth_session,auth_tenant,auth_permission,target,schema,payload,dedupe_key,state,run_at,attempt,max_attempts,timeout_ms,claim_token,lease_until,error,updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,'pending',$15,0,$16,$17,'',0,'',$15) ON CONFLICT(tenant_id,scope_tenant_id,target,dedupe_key) WHERE state IN ('pending','running') DO UPDATE SET dedupe_key=excluded.dedupe_key RETURNING id", + }, vec![t(id), n(tenant_id), n(scope_tenant_id), t(kind), t(provider), t(site), t(subject), t(session), t(tenant_claim), t(permission_key), t(spec.target), t(spec.schema), t(payload), t(key), n(run_at), n(i64::from(spec.attempts)), n(i64::from(spec.timeout_ms))]).await?; + text(&orm::required_row(rows, "Job enqueue")?, 0) +} diff --git a/crates/dever-runtime/src/job/resources.rs b/crates/dever-runtime/src/job/resources.rs new file mode 100644 index 0000000..5a490a1 --- /dev/null +++ b/crates/dever-runtime/src/job/resources.rs @@ -0,0 +1,84 @@ +//! Application-owned Job bindings; storage and scheduling remain in their original owners. +use std::collections::{BTreeMap, BTreeSet}; + +use super::{Clock, store, worker::Binding}; +use crate::{database::Database, orm::Error}; + +pub struct Resources { + pub bindings: Vec<Binding>, + pub clock: Clock, +} + +impl Resources { + pub fn validate(&self, worker: bool) -> Result<(), String> { + let settings = crate::config::current_settings(); + for binding in &self.bindings { + binding.spec.validate().map_err(|error| error.to_string())?; + if worker && let Some((explicit, root)) = binding.selector() { + settings.validate_job_connection(explicit, root, binding.spec.timeout_ms)?; + } + } + Ok(()) + } + + pub async fn initialize(&self) -> Result<(), Error> { + let mut initialized = BTreeSet::new(); + for binding in &self.bindings { + if binding.uses_tenant_storage() { + continue; + } + let database = binding.database(None).await?; + if initialized.insert(database.key().clone()) { + store::initialize(database.clone()).await?; + } + Self::validate_schedule(binding, database).await?; + } + Ok(()) + } + + pub fn has_tenant_jobs(&self) -> bool { + self.bindings.iter().any(Binding::uses_tenant_storage) + } + + pub async fn migrate_tenant( + &self, + tenant_id: i64, + fingerprint: &str, + databases: &mut BTreeMap<String, Database>, + ) -> Result<(), Error> { + let settings = crate::config::current_settings(); + let mut initialized = BTreeSet::new(); + for binding in self + .bindings + .iter() + .filter(|binding| binding.uses_tenant_storage()) + { + let (explicit, root) = binding + .selector() + .ok_or_else(|| Error::migration("tenant Job requires scoped storage"))?; + let connection = settings + .resolve_database(explicit, root) + .map_err(Error::migration)? + .0 + .to_owned(); + if !databases.contains_key(&connection) { + let database = + crate::tenant::begin_migration(&connection, tenant_id, fingerprint).await?; + databases.insert(connection.clone(), database); + } + let database = databases[&connection].clone(); + if initialized.insert(connection) { + store::initialize(database.clone()).await?; + } + Self::validate_schedule(binding, database).await?; + } + Ok(()) + } + + async fn validate_schedule(binding: &Binding, database: Database) -> Result<(), Error> { + if let Some(schedule) = &binding.schedule { + store::validate_schedule(database, &binding.spec, schedule).await?; + } + Ok(()) + } +} diff --git a/crates/dever-runtime/src/job/store.rs b/crates/dever-runtime/src/job/store.rs new file mode 100644 index 0000000..0641ad3 --- /dev/null +++ b/crates/dever-runtime/src/job/store.rs @@ -0,0 +1,556 @@ +use super::*; + +/// A fencing token belongs to exactly one claim, including reclaim by the same process. +#[derive(Clone, Debug)] +pub struct Claim { + pub id: String, + /// Physical queue database tenant, not the execution scope. + pub tenant_id: Option<i64>, + pub execution: ExecutionIdentity, + pub target: String, + pub schema: String, + pub payload: String, + pub attempt: i64, + pub max_attempts: i64, + pub timeout_ms: i64, + pub token: String, +} + +#[derive(Clone, Copy, Debug)] +pub enum Outcome { + Success, + Retry, + Timeout, + UnknownTarget, + SchemaMismatch, + InvalidPayload, + InvalidPolicy, + IdentityRejected, + InvalidIdentity, + ComponentDisabled, +} + +impl Outcome { + fn error(self) -> &'static str { + match self { + Self::Success => "", + Self::Retry => "handler_failed", + Self::Timeout => "handler_timeout", + Self::UnknownTarget => "unknown_target", + Self::SchemaMismatch => "schema_mismatch", + Self::InvalidPayload => "invalid_payload", + Self::InvalidPolicy => "invalid_policy", + Self::IdentityRejected => "identity_rejected", + Self::InvalidIdentity => "invalid_identity", + Self::ComponentDisabled => "component_disabled", + } + } +} + +pub async fn initialize(database: Database) -> Result<(), Error> { + let tenant_id = database.key().tenant_id(); + let transaction = database.begin().await?; + let executor = Executor::new(database, Some(&transaction))?; + let result = initialize_in(&executor, tenant_id).await; + finish_transaction(transaction, result).await +} + +async fn initialize_in(executor: &Executor<'_>, tenant_id: Option<i64>) -> Result<(), Error> { + // Serialize private schema installation independently of Model migrations. + executor + .query( + Sql { + sqlite: "SELECT 1", + postgres: "SELECT pg_advisory_xact_lock(1684371045, 1785684595) IS NULL", + }, + vec![], + ) + .await?; + executor.execute(Sql { + sqlite: "CREATE TABLE IF NOT EXISTS _dever_job_version (singleton INTEGER PRIMARY KEY CHECK(singleton=1), version BIGINT NOT NULL)", + postgres: "CREATE TABLE IF NOT EXISTS _dever_job_version (singleton BIGINT PRIMARY KEY CHECK(singleton=1), version BIGINT NOT NULL)", + }, vec![]).await?; + let versions = executor + .query( + Sql { + sqlite: "SELECT version FROM _dever_job_version", + postgres: "SELECT version FROM _dever_job_version", + }, + vec![], + ) + .await?; + if let Some(version) = versions.first() { + if versions.len() != 1 { + return Err(Error::migration("invalid private Job schema version state")); + } + return match int(version, 0)? { + 5 => Ok(()), + 4 => migrate_v4(executor).await, + 3 => { + migrate_v3(executor).await?; + migrate_v4(executor).await + } + 2 => { + migrate_v2(executor).await?; + migrate_v3(executor).await?; + migrate_v4(executor).await + } + 1 => { + migrate_v1(executor, tenant_id).await?; + migrate_v2(executor).await?; + migrate_v3(executor).await?; + migrate_v4(executor).await + } + _ => Err(Error::migration("unsupported private Job schema version")), + }; + } + const JOBS: &str = "CREATE TABLE _dever_jobs (id TEXT PRIMARY KEY, tenant_id BIGINT NOT NULL, scope_tenant_id BIGINT NOT NULL, execution_kind TEXT NOT NULL CHECK(execution_kind IN ('system','user','legacy')), auth_provider TEXT NOT NULL, auth_site TEXT NOT NULL, auth_subject TEXT NOT NULL, auth_session TEXT NOT NULL, auth_tenant TEXT NOT NULL, auth_permission TEXT NOT NULL, target TEXT NOT NULL, schema TEXT NOT NULL, payload TEXT NOT NULL, dedupe_key TEXT NOT NULL, state TEXT NOT NULL CHECK(state IN ('pending','running','succeeded','dead','blocked')), run_at BIGINT NOT NULL, attempt BIGINT NOT NULL, max_attempts BIGINT NOT NULL, timeout_ms BIGINT NOT NULL, claim_token TEXT NOT NULL, lease_until BIGINT NOT NULL, error TEXT NOT NULL, updated_at BIGINT NOT NULL)"; + const ACTIVE: &str = "CREATE UNIQUE INDEX _dever_jobs_active ON _dever_jobs(tenant_id,scope_tenant_id,target,dedupe_key) WHERE state IN ('pending','running')"; + const DUE: &str = "CREATE INDEX _dever_jobs_due ON _dever_jobs(state,run_at,lease_until)"; + const SCHEDULE: &str = "CREATE TABLE _dever_job_schedule (identity TEXT PRIMARY KEY, fingerprint TEXT NOT NULL, last_minute BIGINT NOT NULL)"; + for sql in [ + JOBS, + ACTIVE, + DUE, + SCHEDULE, + "INSERT INTO _dever_job_version(singleton,version) VALUES (1,5)", + ] { + executor + .execute( + Sql { + sqlite: sql, + postgres: sql, + }, + vec![], + ) + .await?; + } + Ok(()) +} + +async fn migrate_v4(executor: &Executor<'_>) -> Result<(), Error> { + for sql in [ + "ALTER TABLE _dever_jobs ADD COLUMN auth_permission TEXT NOT NULL DEFAULT ''", + "UPDATE _dever_jobs SET state='blocked',error='authorization_contract_changed',claim_token='',lease_until=0 WHERE execution_kind='user' AND state IN ('pending','running')", + "ALTER TABLE _dever_jobs DROP COLUMN auth_capabilities", + "UPDATE _dever_job_version SET version=5 WHERE singleton=1", + ] { + executor + .execute( + Sql { + sqlite: sql, + postgres: sql, + }, + vec![], + ) + .await?; + } + Ok(()) +} + +async fn migrate_v3(executor: &Executor<'_>) -> Result<(), Error> { + for sql in [ + "ALTER TABLE _dever_jobs ADD COLUMN auth_capabilities TEXT NOT NULL DEFAULT '[]'", + "UPDATE _dever_jobs SET state='blocked',error='missing_execution_capabilities',claim_token='',lease_until=0 WHERE execution_kind='user' AND state IN ('pending','running')", + "UPDATE _dever_job_version SET version=4 WHERE singleton=1", + ] { + executor + .execute( + Sql { + sqlite: sql, + postgres: sql, + }, + vec![], + ) + .await?; + } + Ok(()) +} + +async fn migrate_v2(executor: &Executor<'_>) -> Result<(), Error> { + for sql in [ + "ALTER TABLE _dever_jobs ADD COLUMN scope_tenant_id BIGINT NOT NULL DEFAULT 0", + "ALTER TABLE _dever_jobs ADD COLUMN execution_kind TEXT NOT NULL DEFAULT 'legacy'", + "ALTER TABLE _dever_jobs ADD COLUMN auth_provider TEXT NOT NULL DEFAULT ''", + "ALTER TABLE _dever_jobs ADD COLUMN auth_site TEXT NOT NULL DEFAULT ''", + "ALTER TABLE _dever_jobs ADD COLUMN auth_subject TEXT NOT NULL DEFAULT ''", + "ALTER TABLE _dever_jobs ADD COLUMN auth_session TEXT NOT NULL DEFAULT ''", + "ALTER TABLE _dever_jobs ADD COLUMN auth_tenant TEXT NOT NULL DEFAULT ''", + "UPDATE _dever_jobs SET state='blocked',error='missing_execution_identity',claim_token='',lease_until=0 WHERE state IN ('pending','running')", + "DROP INDEX IF EXISTS _dever_jobs_active", + "CREATE UNIQUE INDEX _dever_jobs_active ON _dever_jobs(tenant_id,scope_tenant_id,target,dedupe_key) WHERE state IN ('pending','running')", + "UPDATE _dever_job_version SET version=3 WHERE singleton=1", + ] { + executor + .execute( + Sql { + sqlite: sql, + postgres: sql, + }, + vec![], + ) + .await?; + } + Ok(()) +} + +async fn migrate_v1(executor: &Executor<'_>, tenant_id: Option<i64>) -> Result<(), Error> { + if tenant_id.is_some() { + let rows = executor + .query( + Sql { + sqlite: "SELECT COUNT(*) FROM _dever_jobs", + postgres: "SELECT COUNT(*) FROM _dever_jobs", + }, + vec![], + ) + .await?; + if int(&orm::required_row(rows, "legacy Job row count")?, 0)? != 0 { + return Err(Error::migration( + "legacy Job rows have no tenant ownership; drain or explicitly migrate them before enabling tenant Jobs", + )); + } + } + for sql in [ + "ALTER TABLE _dever_jobs ADD COLUMN tenant_id BIGINT NOT NULL DEFAULT 0", + "DROP INDEX IF EXISTS _dever_jobs_active", + "CREATE UNIQUE INDEX _dever_jobs_active ON _dever_jobs(tenant_id,target,dedupe_key) WHERE state IN ('pending','running')", + "UPDATE _dever_job_version SET version=2 WHERE singleton=1", + ] { + executor + .execute( + Sql { + sqlite: sql, + postgres: sql, + }, + vec![], + ) + .await?; + } + Ok(()) +} + +pub async fn claim( + database: Database, + clock: &Clock, + lease_ms: i64, + available: usize, +) -> Result<Vec<Claim>, Error> { + if !(1..=256).contains(&available) + || !(100..=crate::config::MAX_JOB_LEASE_MS).contains(&lease_ms) + { + return Err(Error::invalid_data("invalid Job claim capacity or lease")); + } + let tenant_id = database.key().tenant_id().unwrap_or(0); + let transaction = database.begin().await?; + let executor = Executor::new(database, Some(&transaction))?; + let result = claim_in(&executor, tenant_id, clock, lease_ms, available).await; + finish_transaction(transaction, result).await +} + +async fn claim_in( + executor: &Executor<'_>, + expected_tenant: i64, + clock: &Clock, + lease_ms: i64, + available: usize, +) -> Result<Vec<Claim>, Error> { + let now = clock.now()?; + // Exhausted rows consume bounded cleanup work, never an execution slot. In + // particular, an empty claim must not hide eligible jobs behind dead rows. + executor.execute(Sql { + sqlite: "UPDATE _dever_jobs SET state='dead',error='attempts_exhausted',updated_at=?1 WHERE id IN (SELECT id FROM _dever_jobs WHERE attempt>=max_attempts AND ((state='pending' AND run_at<=?1) OR (state='running' AND lease_until<=?1)) ORDER BY run_at,id LIMIT ?2)", + postgres: "UPDATE _dever_jobs SET state='dead',error='attempts_exhausted',updated_at=$1 WHERE id IN (SELECT id FROM _dever_jobs WHERE attempt>=max_attempts AND ((state='pending' AND run_at<=$1) OR (state='running' AND lease_until<=$1)) ORDER BY run_at,id LIMIT $2 FOR UPDATE SKIP LOCKED)", + }, vec![n(now), n(available as i64)]).await?; + let rows = executor.query(Sql { + sqlite: "SELECT id,tenant_id,scope_tenant_id,execution_kind,auth_provider,auth_site,auth_subject,auth_session,auth_tenant,auth_permission,target,schema,payload,attempt,max_attempts,timeout_ms FROM _dever_jobs WHERE attempt<max_attempts AND ((state='pending' AND run_at<=?1) OR (state='running' AND lease_until<=?1)) ORDER BY run_at,id LIMIT ?2", + postgres: "SELECT id,tenant_id,scope_tenant_id,execution_kind,auth_provider,auth_site,auth_subject,auth_session,auth_tenant,auth_permission,target,schema,payload,attempt,max_attempts,timeout_ms FROM _dever_jobs WHERE attempt<max_attempts AND ((state='pending' AND run_at<=$1) OR (state='running' AND lease_until<=$1)) ORDER BY run_at,id LIMIT $2 FOR UPDATE SKIP LOCKED", + }, vec![n(now), n(available as i64)]).await?; + // BEGIN/row locking may wait. Start the lease only after ownership is locked. + let now = clock.now()?; + let lease_until = now + .checked_add(lease_ms) + .ok_or_else(|| Error::invalid_data("Job lease overflow"))?; + let mut claims = Vec::with_capacity(rows.len()); + for row in rows { + let id = text(&row, 0)?; + let tenant = int(&row, 1)?; + if tenant != expected_tenant { + return Err(Error::migration( + "Job tenant ownership does not match its physical database", + )); + } + let scope_tenant_id = int(&row, 2)?; + if scope_tenant_id < 0 { + return Err(Error::migration("Job execution tenant id is invalid")); + } + let execution = match text(&row, 3)?.as_str() { + "system" => ExecutionIdentity::System { + tenant_id: (scope_tenant_id > 0).then_some(scope_tenant_id), + }, + "user" => ExecutionIdentity::User { + tenant_id: (scope_tenant_id > 0).then_some(scope_tenant_id), + provider: text(&row, 4)?, + site: text(&row, 5)?, + subject: text(&row, 6)?, + session: text(&row, 7)?, + tenant_claim: match text(&row, 8)? { + value if value.is_empty() => None, + value => Some(value), + }, + permission_key: text(&row, 9)?, + }, + _ => { + return Err(Error::migration( + "active Job has no trusted execution identity", + )); + } + }; + execution.validate()?; + if tenant > 0 && execution.tenant_id() != Some(tenant) { + return Err(Error::migration( + "Job execution tenant does not match tenant storage", + )); + } + let token = orm::Uuid::new_v7()?.to_string(); + let changed = executor.execute(Sql { + sqlite: "UPDATE _dever_jobs SET state='running',attempt=attempt+1,claim_token=?2,lease_until=?3,updated_at=?4 WHERE id=?1 AND attempt<max_attempts AND ((state='pending' AND run_at<=?4) OR (state='running' AND lease_until<=?4))", + postgres: "UPDATE _dever_jobs SET state='running',attempt=attempt+1,claim_token=$2,lease_until=$3,updated_at=$4 WHERE id=$1 AND attempt<max_attempts AND ((state='pending' AND run_at<=$4) OR (state='running' AND lease_until<=$4))", + }, vec![t(&id), t(&token), n(lease_until), n(now)]).await?; + if changed == 1 { + claims.push(Claim { + id, + tenant_id: (tenant > 0).then_some(tenant), + execution, + target: text(&row, 10)?, + schema: text(&row, 11)?, + payload: text(&row, 12)?, + attempt: int(&row, 13)? + 1, + max_attempts: int(&row, 14)?, + timeout_ms: int(&row, 15)?, + token, + }); + } + } + Ok(claims) +} + +pub async fn renew( + database: Database, + claim: &Claim, + clock: &Clock, + lease_ms: i64, +) -> Result<bool, Error> { + if !(100..=crate::config::MAX_JOB_LEASE_MS).contains(&lease_ms) { + return Err(Error::invalid_data("invalid Job lease")); + } + let tenant_id = database.key().tenant_id().unwrap_or(0); + let transaction = database.begin().await?; + let executor = Executor::new(database, Some(&transaction))?; + let result = renew_in(&executor, tenant_id, claim, clock, lease_ms).await; + finish_transaction(transaction, result).await +} + +async fn lock_claim( + executor: &Executor<'_>, + expected_tenant: i64, + claim: &Claim, +) -> Result<bool, Error> { + let tenant_id = claim.tenant_id.unwrap_or(0); + if expected_tenant != tenant_id { + return Err(Error::migration( + "Job claim tenant does not match its physical database", + )); + } + // PostgreSQL needs the same ordering as SQLite's BEGIN IMMEDIATE: obtain the + // write lock before reading time, so lock wait cannot revive an expired lease. + executor.query(Sql { + sqlite: "SELECT id FROM _dever_jobs WHERE id=?1 AND claim_token=?2 AND tenant_id=?3 AND state='running'", + postgres: "SELECT id FROM _dever_jobs WHERE id=$1 AND claim_token=$2 AND tenant_id=$3 AND state='running' FOR UPDATE", + }, vec![t(&claim.id), t(&claim.token), n(tenant_id)]).await.map(|rows| !rows.is_empty()) +} + +async fn renew_in( + executor: &Executor<'_>, + expected_tenant: i64, + claim: &Claim, + clock: &Clock, + lease_ms: i64, +) -> Result<bool, Error> { + if !lock_claim(executor, expected_tenant, claim).await? { + return Ok(false); + } + let now = clock.now()?; + let until = now + .checked_add(lease_ms) + .ok_or_else(|| Error::invalid_data("Job lease overflow"))?; + executor.execute(Sql { + sqlite: "UPDATE _dever_jobs SET lease_until=?4,updated_at=?3 WHERE id=?1 AND claim_token=?2 AND state='running' AND lease_until>?3", + postgres: "UPDATE _dever_jobs SET lease_until=$4,updated_at=$3 WHERE id=$1 AND claim_token=$2 AND state='running' AND lease_until>$3", + }, vec![t(&claim.id), t(&claim.token), n(now), n(until)]).await.map(|count| count == 1) +} + +pub async fn complete( + database: Database, + claim: &Claim, + outcome: Outcome, + clock: &Clock, + retry_base_ms: i64, + retry_max_ms: i64, +) -> Result<bool, Error> { + if retry_base_ms < 1 || retry_max_ms < retry_base_ms || retry_max_ms > 86_400_000 { + return Err(Error::invalid_data("invalid Job retry bounds")); + } + let tenant_id = database.key().tenant_id().unwrap_or(0); + let transaction = database.begin().await?; + let executor = Executor::new(database, Some(&transaction))?; + let result = complete_in( + &executor, + tenant_id, + claim, + outcome, + clock, + retry_base_ms, + retry_max_ms, + ) + .await; + finish_transaction(transaction, result).await +} + +async fn complete_in( + executor: &Executor<'_>, + expected_tenant: i64, + claim: &Claim, + outcome: Outcome, + clock: &Clock, + retry_base_ms: i64, + retry_max_ms: i64, +) -> Result<bool, Error> { + if !lock_claim(executor, expected_tenant, claim).await? { + return Ok(false); + } + let now = clock.now()?; + let state = match outcome { + Outcome::Success => "succeeded", + Outcome::UnknownTarget + | Outcome::SchemaMismatch + | Outcome::InvalidPayload + | Outcome::InvalidPolicy + | Outcome::IdentityRejected + | Outcome::InvalidIdentity + | Outcome::ComponentDisabled => "blocked", + _ if claim.attempt >= claim.max_attempts => "dead", + _ => "pending", + }; + let exponent = u32::try_from(claim.attempt.saturating_sub(1)) + .unwrap_or(u32::MAX) + .min(62); + let delay = retry_base_ms + .saturating_mul(1i64 << exponent) + .min(retry_max_ms); + let run_at = now + .checked_add(delay) + .ok_or_else(|| Error::invalid_data("Job retry time overflow"))?; + executor.execute(Sql { + sqlite: "UPDATE _dever_jobs SET state=?4,error=?5,run_at=?6,lease_until=0,claim_token='',updated_at=?3 WHERE id=?1 AND claim_token=?2 AND state='running' AND lease_until>?3", + postgres: "UPDATE _dever_jobs SET state=$4,error=$5,run_at=$6,lease_until=0,claim_token='',updated_at=$3 WHERE id=$1 AND claim_token=$2 AND state='running' AND lease_until>$3", + }, vec![t(&claim.id), t(&claim.token), n(now), t(state), t(outcome.error()), n(run_at)]).await.map(|count| count == 1) +} + +pub async fn materialize( + database: Database, + spec: &Spec, + cron: &crate::cron::Cron, + now: i64, +) -> Result<usize, Error> { + spec.validate()?; + let tenant_id = database.key().tenant_id().unwrap_or(0); + let transaction = database.begin().await?; + let executor = Executor::new(database, Some(&transaction))?; + let result = materialize_in(&executor, tenant_id, spec, cron, now.div_euclid(60_000)).await; + finish_transaction(transaction, result).await +} + +/// Validate persisted schedule identity before any application listener starts. +pub async fn validate_schedule( + database: Database, + spec: &Spec, + cron: &crate::cron::Cron, +) -> Result<(), Error> { + let rows = database + .query( + Sql { + sqlite: "SELECT fingerprint FROM _dever_job_schedule WHERE identity=?1", + postgres: "SELECT fingerprint FROM _dever_job_schedule WHERE identity=$1", + }, + vec![t(spec.target)], + ) + .await?; + if let Some(row) = rows.first() { + validate_fingerprint(row, &cron.fingerprint())?; + } + Ok(()) +} + +fn validate_fingerprint(cursor: &Row, expected: &str) -> Result<(), Error> { + if text(cursor, 0)? != expected { + return Err(Error::migration("Job schedule fingerprint changed")); + } + Ok(()) +} + +async fn materialize_in( + executor: &Executor<'_>, + tenant_id: i64, + spec: &Spec, + cron: &crate::cron::Cron, + minute: i64, +) -> Result<usize, Error> { + let fingerprint = cron.fingerprint(); + executor.execute(Sql { + sqlite: "INSERT INTO _dever_job_schedule(identity,fingerprint,last_minute) VALUES (?1,?2,?3) ON CONFLICT DO NOTHING", + postgres: "INSERT INTO _dever_job_schedule(identity,fingerprint,last_minute) VALUES ($1,$2,$3) ON CONFLICT DO NOTHING", + }, vec![t(spec.target), t(&fingerprint), n(minute)]).await?; + let rows = executor.query(Sql { + sqlite: "SELECT fingerprint,last_minute FROM _dever_job_schedule WHERE identity=?1", + postgres: "SELECT fingerprint,last_minute FROM _dever_job_schedule WHERE identity=$1 FOR UPDATE", + }, vec![t(spec.target)]).await?; + let cursor = orm::required_row(rows, "Job schedule cursor")?; + validate_fingerprint(&cursor, &fingerprint)?; + let previous = int(&cursor, 1)?; + let end = minute.min(previous.saturating_add(256)); + let mut inserted = 0; + for slot in previous.saturating_add(1)..=end { + if cron.matches_minute(slot) { + let run_at = slot + .checked_mul(60_000) + .ok_or_else(|| Error::invalid_data("Job schedule time overflow"))?; + let identity = ExecutionIdentity::System { + tenant_id: (tenant_id > 0).then_some(tenant_id), + }; + insert( + executor, + tenant_id, + &identity, + spec, + "null", + &format!("schedule:{}:{slot}", spec.target), + run_at, + ) + .await?; + inserted += 1; + } + } + if end > previous { + executor + .execute( + Sql { + sqlite: "UPDATE _dever_job_schedule SET last_minute=?2 WHERE identity=?1", + postgres: "UPDATE _dever_job_schedule SET last_minute=$2 WHERE identity=$1", + }, + vec![t(spec.target), n(end)], + ) + .await?; + } + Ok(inserted) +} diff --git a/crates/dever-runtime/src/job/worker.rs b/crates/dever-runtime/src/job/worker.rs new file mode 100644 index 0000000..46ebcfa --- /dev/null +++ b/crates/dever-runtime/src/job/worker.rs @@ -0,0 +1,470 @@ +use std::collections::BTreeMap; +use std::future::Future; +use std::pin::Pin; + +use futures_util::{StreamExt, stream::FuturesUnordered}; + +use super::{Clock, Spec, store}; +use crate::{ + config::JobSettings, + database::{Database, ModelScope, StorageBinding}, + orm::ErrorKind, +}; + +#[derive(Clone)] +enum BindingStorage { + Fixed(Database), + Scoped(StorageBinding), +} + +#[derive(Clone)] +pub struct Binding { + storage: BindingStorage, + tenant: bool, + pub spec: Spec, + pub schedule: Option<crate::cron::Cron>, +} + +impl Binding { + pub fn fixed( + database: Database, + spec: &'static Spec, + schedule: Option<crate::cron::Cron>, + ) -> Self { + Self { + storage: BindingStorage::Fixed(database), + tenant: false, + spec: spec.clone(), + schedule, + } + } + + pub fn scoped( + storage: &'static StorageBinding, + scope: ModelScope, + spec: &'static Spec, + schedule: Option<crate::cron::Cron>, + ) -> Self { + Self::owned(*storage, scope, spec.clone(), schedule) + } + + pub fn owned( + storage: StorageBinding, + scope: ModelScope, + spec: Spec, + schedule: Option<crate::cron::Cron>, + ) -> Self { + Self { + storage: BindingStorage::Scoped(storage), + tenant: scope == ModelScope::Tenant, + spec, + schedule, + } + } + + pub(crate) fn uses_tenant_storage(&self) -> bool { + self.tenant && crate::config::current_settings().tenant().is_some() + } + + pub(crate) fn selector(&self) -> Option<(Option<&str>, &str)> { + match &self.storage { + BindingStorage::Fixed(_) => None, + BindingStorage::Scoped(storage) => Some(storage.selector()), + } + } + + pub(crate) async fn database( + &self, + tenant_id: Option<i64>, + ) -> Result<Database, crate::orm::Error> { + match &self.storage { + BindingStorage::Fixed(database) => Ok(database.clone()), + BindingStorage::Scoped(storage) => match tenant_id { + Some(tenant_id) => crate::tenant::scope(tenant_id, storage.database()).await, + None => storage.database().await, + }, + } + } +} + +async fn global_connections(bindings: &[Binding]) -> Result<Vec<Database>, String> { + let mut databases = BTreeMap::new(); + for binding in bindings + .iter() + .filter(|binding| !binding.uses_tenant_storage()) + { + let database = binding + .database(None) + .await + .map_err(|error| error.to_string())?; + databases.insert(database.key().clone(), database); + } + Ok(databases.into_values().collect()) +} + +async fn tenant_connections(bindings: &[Binding], tenant_id: i64) -> Result<Vec<Database>, String> { + let mut databases = BTreeMap::new(); + for binding in bindings + .iter() + .filter(|binding| binding.uses_tenant_storage()) + { + let database = match binding.database(Some(tenant_id)).await { + Ok(database) => database, + Err(error) if error.kind() == ErrorKind::PoolExhausted => continue, + Err(error) => return Err(error.to_string()), + }; + databases.insert(database.key().clone(), database); + } + Ok(databases.into_values().collect()) +} + +async fn schedule_global(bindings: &[Binding], clock: &Clock) -> Result<(), String> { + let now = clock.now().map_err(|error| error.to_string())?; + for binding in bindings + .iter() + .filter(|binding| !binding.uses_tenant_storage()) + { + if let Some(cron) = &binding.schedule { + let database = binding + .database(None) + .await + .map_err(|error| error.to_string())?; + store::materialize(database, &binding.spec, cron, now) + .await + .map_err(|error| error.to_string())?; + } + } + Ok(()) +} + +async fn schedule_tenant( + bindings: &[Binding], + clock: &Clock, + tenant_id: i64, +) -> Result<(), String> { + let now = clock.now().map_err(|error| error.to_string())?; + for binding in bindings + .iter() + .filter(|binding| binding.uses_tenant_storage()) + { + let Some(cron) = &binding.schedule else { + continue; + }; + let database = match binding.database(Some(tenant_id)).await { + Ok(database) => database, + Err(error) if error.kind() == ErrorKind::PoolExhausted => continue, + Err(error) => return Err(error.to_string()), + }; + store::materialize(database, &binding.spec, cron, now) + .await + .map_err(|error| error.to_string())?; + } + Ok(()) +} + +async fn execute<F, Fut>( + database: Database, + claim: store::Claim, + settings: JobSettings, + clock: Clock, + dispatch: F, +) -> Result<(), String> +where + F: Fn(store::Claim) -> Fut + Send + 'static, + Fut: Future<Output = store::Outcome> + Send + 'static, +{ + if database.key().tenant_id() != claim.tenant_id { + return Err("Job claim tenant does not match its physical database".into()); + } + let current_settings = crate::config::current_settings(); + let connection = current_settings + .database(database.name()) + .ok_or("Job database is not configured")?; + let outcome = if settings + .validate_lease(connection, claim.timeout_ms) + .is_err() + || !(1..=100).contains(&claim.max_attempts) + { + store::Outcome::InvalidPolicy + } else { + let identity = claim.execution.clone(); + let tenant_id = identity.tenant_id(); + let input = claim.clone(); + let task = crate::task::run(async move { + let execution = async move { + match tenant_id { + Some(tenant_id) => crate::tenant::scope(tenant_id, dispatch(input)).await, + None => dispatch(input).await, + } + }; + let outcome = crate::job::scope_execution(identity, execution).await; + Ok(outcome) + }) + .await?; + match crate::task::wait_lease(task, claim.timeout_ms, settings.lease_ms / 3, || async { + store::renew(database.clone(), &claim, &clock, settings.lease_ms) + .await + .map_err(|error| error.to_string()) + }) + .await? + { + crate::task::LeaseResult::Completed(outcome) => outcome, + crate::task::LeaseResult::Timeout => store::Outcome::Timeout, + crate::task::LeaseResult::Lost => return Ok(()), + } + }; + store::complete( + database, + &claim, + outcome, + &clock, + settings.retry_base_ms, + settings.retry_max_ms, + ) + .await + .map_err(|error| error.to_string())?; + Ok(()) +} + +async fn drain_connections<F, Fut>( + databases: Vec<Database>, + settings: JobSettings, + clock: &Clock, + limit: i64, + count: &mut i64, + dispatch: &F, +) -> Result<bool, String> +where + F: Fn(store::Claim) -> Fut + Clone + Send + 'static, + Fut: Future<Output = store::Outcome> + Send + 'static, +{ + let mut progress = false; + for database in databases { + if *count == limit { + return Ok(progress); + } + for claim in store::claim(database.clone(), clock, settings.lease_ms, 1) + .await + .map_err(|error| error.to_string())? + { + execute( + database.clone(), + claim, + settings, + clock.clone(), + dispatch.clone(), + ) + .await?; + *count += 1; + progress = true; + } + } + Ok(progress) +} + +pub async fn drain<F, Fut>( + bindings: Vec<Binding>, + clock: Clock, + limit: i64, + dispatch: F, +) -> Result<i64, String> +where + F: Fn(store::Claim) -> Fut + Clone + Send + 'static, + Fut: Future<Output = store::Outcome> + Send + 'static, +{ + if !(1..=10_000).contains(&limit) { + return Err("test Job drain limit must be between 1 and 10000".into()); + } + let settings = JobSettings::for_test(); + schedule_global(&bindings, &clock).await?; + let tenant_id = if bindings.iter().any(Binding::uses_tenant_storage) { + let tenant_id = crate::tenant::current_id().map_err(|error| error.to_string())?; + schedule_tenant(&bindings, &clock, tenant_id).await?; + Some(tenant_id) + } else { + None + }; + let mut count = 0; + loop { + let mut progress = drain_connections( + global_connections(&bindings).await?, + settings, + &clock, + limit, + &mut count, + &dispatch, + ) + .await?; + if count == limit { + return Ok(count); + } + if let Some(tenant_id) = tenant_id { + progress |= drain_connections( + tenant_connections(&bindings, tenant_id).await?, + settings, + &clock, + limit, + &mut count, + &dispatch, + ) + .await?; + } + if !progress { + return Ok(count); + } + } +} + +type ActiveJob = Pin<Box<dyn Future<Output = Result<(), String>> + Send>>; + +async fn claim_into<F, Fut>( + databases: Vec<Database>, + settings: JobSettings, + clock: &Clock, + dispatch: &F, + active: &mut FuturesUnordered<ActiveJob>, +) -> Result<(), String> +where + F: Fn(store::Claim) -> Fut + Clone + Send + 'static, + Fut: Future<Output = store::Outcome> + Send + 'static, +{ + for database in databases { + if crate::lifecycle::stopping() { + break; + } + let available = settings.workers.saturating_sub(active.len()); + if available == 0 { + break; + } + let claims = store::claim(database.clone(), clock, settings.lease_ms, available) + .await + .map_err(|error| error.to_string())?; + if crate::lifecycle::stopping() { + break; + } + for claim in claims { + let task = crate::task::run(execute( + database.clone(), + claim, + settings, + clock.clone(), + dispatch.clone(), + )) + .await?; + active.push(Box::pin(crate::task::wait(task))); + } + } + Ok(()) +} + +async fn claim_ready_tenants<F, Fut>( + bindings: &[Binding], + settings: JobSettings, + clock: &Clock, + dispatch: &F, + active: &mut FuturesUnordered<ActiveJob>, + after: &mut i64, +) -> Result<(), String> +where + F: Fn(store::Claim) -> Fut + Clone + Send + 'static, + Fut: Future<Output = store::Outcome> + Send + 'static, +{ + if active.len() >= settings.workers { + return Ok(()); + } + let tenant_ids = crate::tenant::ready_ids_after(*after, 256) + .await + .map_err(|error| error.to_string())?; + if tenant_ids.is_empty() { + *after = 0; + return Ok(()); + } + for tenant_id in tenant_ids { + if crate::lifecycle::stopping() || active.len() >= settings.workers { + break; + } + schedule_tenant(bindings, clock, tenant_id).await?; + claim_into( + tenant_connections(bindings, tenant_id).await?, + settings, + clock, + dispatch, + active, + ) + .await?; + *after = tenant_id; + } + Ok(()) +} + +pub async fn serve<F, Fut>( + bindings: Vec<Binding>, + clock: Clock, + settings: JobSettings, + dispatch: F, +) -> Result<(), String> +where + F: Fn(store::Claim) -> Fut + Clone + Send + 'static, + Fut: Future<Output = store::Outcome> + Send + 'static, +{ + let settings = settings.validate()?; + let has_tenant_jobs = bindings.iter().any(Binding::uses_tenant_storage); + let mut tenant_after = 0; + let mut prefer_tenant = has_tenant_jobs; + let mut active = FuturesUnordered::new(); + loop { + if crate::lifecycle::stopping() { + break; + } + schedule_global(&bindings, &clock).await?; + if prefer_tenant { + claim_ready_tenants( + &bindings, + settings, + &clock, + &dispatch, + &mut active, + &mut tenant_after, + ) + .await?; + claim_into( + global_connections(&bindings).await?, + settings, + &clock, + &dispatch, + &mut active, + ) + .await?; + } else { + claim_into( + global_connections(&bindings).await?, + settings, + &clock, + &dispatch, + &mut active, + ) + .await?; + if has_tenant_jobs { + claim_ready_tenants( + &bindings, + settings, + &clock, + &dispatch, + &mut active, + &mut tenant_after, + ) + .await?; + } + } + prefer_tenant = has_tenant_jobs && !prefer_tenant; + tokio::select! { + _ = crate::lifecycle::shutdown_requested() => break, + Some(result) = active.next(), if !active.is_empty() => result?, + _ = tokio::time::sleep(std::time::Duration::from_millis(settings.poll_ms as u64)) => {} + } + } + while let Some(result) = active.next().await { + result?; + } + Ok(()) +} diff --git a/crates/dever-runtime/src/lib.rs b/crates/dever-runtime/src/lib.rs new file mode 100644 index 0000000..63d0406 --- /dev/null +++ b/crates/dever-runtime/src/lib.rs @@ -0,0 +1,75 @@ +use std::io::{self, Write}; + +pub mod abi; +#[cfg(feature = "wire")] +pub mod api; +#[cfg(feature = "api")] +pub mod application; +pub mod async_stream; +#[cfg(feature = "api")] +pub mod auth; +pub mod bytes; +pub mod channel; +pub mod collections; +#[cfg(feature = "wire")] +pub mod component; +pub mod concurrent; +#[cfg(any(feature = "database", feature = "wire"))] +pub mod config; +pub mod cron; +#[cfg(feature = "crypto")] +pub mod crypto; +#[cfg(any(feature = "sqlite", feature = "postgres"))] +pub mod database; +mod endpoint; +#[cfg(feature = "external")] +pub mod external; +pub mod http; +#[cfg(any(feature = "sqlite", feature = "postgres"))] +pub mod job; +#[cfg(any(feature = "database", feature = "wire"))] +pub mod lifecycle; +pub mod log; +pub mod net; +pub mod number; +pub mod orm; +#[cfg(any(feature = "sqlite", feature = "postgres"))] +mod pool; +#[cfg(feature = "postgres")] +pub mod postgres; +pub mod process; +pub mod render; +pub mod resource; +pub mod secret; +#[cfg(feature = "sqlite")] +pub mod sqlite; +pub mod sse; +pub mod task; +#[cfg(any(feature = "sqlite", feature = "postgres"))] +pub mod tenant; +pub mod text; +pub mod time; +pub mod tls; +mod transport; +pub mod websocket; +#[cfg(feature = "wire")] +pub mod wire; + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct Id(pub String); + +/// 将故障格式化移出正常计算路径，避免每个数值操作携带重复格式化代码。 +#[cold] +#[inline(never)] +pub fn located_error(location: &str, error: impl std::fmt::Display) -> String { + format!("{location}: {error}") +} + +pub fn println(text: &str) -> io::Result<()> { + write_line(&mut io::stdout().lock(), text) +} + +pub fn write_line(output: &mut impl Write, text: &str) -> io::Result<()> { + writeln!(output, "{text}")?; + output.flush() +} diff --git a/crates/dever-runtime/src/lifecycle.rs b/crates/dever-runtime/src/lifecycle.rs new file mode 100644 index 0000000..f58836a --- /dev/null +++ b/crates/dever-runtime/src/lifecycle.rs @@ -0,0 +1,94 @@ +//! One application owner broadcasts stop-admission and joins its structured root. +use crate::config::{RuntimeMode, RuntimeSettings}; +use std::future::Future; +use std::sync::{Arc, OnceLock}; + +pub(crate) struct Lifecycle { + settings: RuntimeSettings, + stop: tokio::sync::watch::Sender<bool>, +} + +static LIFECYCLE: OnceLock<Arc<Lifecycle>> = OnceLock::new(); + +fn current() -> Option<Arc<Lifecycle>> { + #[cfg(feature = "api")] + if let Some(session) = crate::application::current() { + return session.lifecycle(); + } + LIFECYCLE.get().cloned() +} + +pub fn configure(api: bool, worker: bool) -> Result<(), String> { + let settings = crate::config::current_settings().services(api, worker)?; + let (stop, _) = tokio::sync::watch::channel(false); + let state = Arc::new(Lifecycle { settings, stop }); + #[cfg(feature = "api")] + if let Some(session) = crate::application::current() { + return session.set_lifecycle(state); + } + LIFECYCLE + .set(state) + .map_err(|_| "application lifecycle initialized twice".into()) +} + +pub fn api_enabled() -> bool { + current().is_none_or(|state| matches!(state.settings.mode, RuntimeMode::Api | RuntimeMode::All)) +} + +pub fn worker_enabled() -> bool { + current() + .is_some_and(|state| matches!(state.settings.mode, RuntimeMode::Worker | RuntimeMode::All)) +} + +pub fn stopping() -> bool { + current().is_some_and(|state| *state.stop.borrow()) +} + +pub async fn shutdown_requested() { + let Some(state) = current() else { + return std::future::pending().await; + }; + let mut receiver = state.stop.subscribe(); + while !*receiver.borrow_and_update() { + if receiver.changed().await.is_err() { + return; + } + } +} + +pub async fn run<F, T, E>(future: F) -> Result<T, E> +where + F: Future<Output = Result<T, E>> + Send + 'static, + T: Send + 'static, + E: From<String> + Send + 'static, +{ + let state = + current().ok_or_else(|| E::from("application lifecycle is not configured".into()))?; + #[cfg(unix)] + let mut terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) + .map_err(|error| E::from(format!("cannot register SIGTERM: {error}")))?; + #[cfg(unix)] + let mut interrupt = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::interrupt()) + .map_err(|error| E::from(format!("cannot register SIGINT: {error}")))?; + let signal = async { + #[cfg(unix)] + tokio::select! { _ = terminate.recv() => {}, _ = interrupt.recv() => {} } + #[cfg(not(unix))] + tokio::signal::ctrl_c() + .await + .map_err(|error| format!("cannot receive Ctrl-C: {error}"))?; + state.stop.send_replace(true); + Ok(()) + }; + let root = crate::task::run_typed(future).await.map_err(E::from)?; + crate::task::wait_shutdown(root, signal, state.settings.shutdown_ms).await +} + +pub async fn serve_both<A, W>(api: A, worker: W) -> Result<(), String> +where + A: Future<Output = Result<(), String>>, + W: Future<Output = Result<(), String>>, +{ + tokio::try_join!(api, worker)?; + Ok(()) +} diff --git a/crates/dever-runtime/src/log.rs b/crates/dever-runtime/src/log.rs new file mode 100644 index 0000000..c6242f7 --- /dev/null +++ b/crates/dever-runtime/src/log.rs @@ -0,0 +1,359 @@ +use std::collections::VecDeque; +use std::io::{self, BufWriter, Write}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Condvar, Mutex, MutexGuard, OnceLock}; + +const QUEUE_CAPACITY: usize = 128; +const WRITER_STACK_BYTES: usize = 128 * 1024; +const MAX_MESSAGE_BYTES: usize = 4 * 1024; +const MAX_FIELDS: usize = 16; +const MAX_FIELD_NAME_BYTES: usize = 128; +const MAX_FIELD_VALUE_BYTES: usize = 1024; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Level { + Debug, + Info, + Warn, + Error, +} + +impl Level { + fn name(self) -> &'static str { + match self { + Self::Debug => "debug", + Self::Info => "info", + Self::Warn => "warn", + Self::Error => "error", + } + } + + fn drops_when_full(self) -> bool { + matches!(self, Self::Debug | Self::Info) + } + + #[cfg(feature = "wire")] + fn enabled(self, configured: crate::config::LogLevel) -> bool { + let actual = match self { + Self::Debug => 0, + Self::Info => 1, + Self::Warn => 2, + Self::Error => 3, + }; + let minimum = match configured { + crate::config::LogLevel::Debug => 0, + crate::config::LogLevel::Info => 1, + crate::config::LogLevel::Warn => 2, + crate::config::LogLevel::Error => 3, + }; + actual >= minimum + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Field { + pub name: String, + pub value: String, +} + +struct Record { + level: Level, + message: String, + fields: Vec<Field>, + dropped: u64, +} + +enum Command { + Record(Record), + Flush, +} + +struct Logger { + queue: Mutex<Queue>, + available: Condvar, + space: Condvar, + flushed: Condvar, + flush_call: Mutex<()>, + dropped: AtomicU64, +} + +struct Queue { + commands: VecDeque<Command>, + closed: bool, + flushed: bool, +} + +struct WriterGuard<'a>(&'a Logger); + +impl Drop for WriterGuard<'_> { + fn drop(&mut self) { + self.0.close(); + } +} + +static LOGGER: OnceLock<Result<Arc<Logger>, String>> = OnceLock::new(); + +pub fn debug(message: impl Into<String>, fields: Vec<Field>) { + emit(Level::Debug, message, fields); +} + +pub fn info(message: impl Into<String>, fields: Vec<Field>) { + emit(Level::Info, message, fields); +} + +pub fn warn(message: impl Into<String>, fields: Vec<Field>) { + emit(Level::Warn, message, fields); +} + +pub fn error(message: impl Into<String>, fields: Vec<Field>) { + emit(Level::Error, message, fields); +} + +pub fn emit(level: Level, message: impl Into<String>, fields: Vec<Field>) { + #[cfg(feature = "wire")] + if crate::config::try_current_settings() + .is_some_and(|settings| !level.enabled(settings.log().level)) + { + return; + } + #[cfg(feature = "api")] + let fields = { + let mut fields = fields; + for field in crate::auth::log_fields() { + if !fields.iter().any(|existing| existing.name == field.name) { + fields.push(field); + } + } + fields + }; + let record = Record { + level, + message: truncate(message.into(), MAX_MESSAGE_BYTES), + fields: fields + .into_iter() + .take(MAX_FIELDS) + .map(|field| Field { + name: truncate(field.name, MAX_FIELD_NAME_BYTES), + value: truncate(field.value, MAX_FIELD_VALUE_BYTES), + }) + .collect(), + dropped: 0, + }; + match logger() { + Ok(logger) => logger.submit(record), + Err(error) => emergency(&Record { + level: Level::Error, + message: error.clone(), + fields: Vec::new(), + dropped: 0, + }), + } +} + +pub fn flush() { + if let Ok(logger) = logger() { + logger.flush(); + } +} + +pub fn dropped_events() -> u64 { + logger() + .as_ref() + .map(|logger| logger.dropped.load(Ordering::Relaxed)) + .unwrap_or_default() +} + +fn logger() -> &'static Result<Arc<Logger>, String> { + LOGGER.get_or_init(|| { + let logger = Arc::new(Logger::new()); + let writer = Arc::clone(&logger); + std::thread::Builder::new() + .name("dever-log".into()) + .stack_size(WRITER_STACK_BYTES) + .spawn(move || { + let mut output = BufWriter::new(io::stderr().lock()); + writer.write(&mut output); + }) + .map_err(|error| format!("cannot start Dever log writer: {error}"))?; + Ok(logger) + }) +} + +impl Logger { + fn new() -> Self { + Self { + queue: Mutex::new(Queue { + commands: VecDeque::with_capacity(QUEUE_CAPACITY), + closed: false, + flushed: false, + }), + available: Condvar::new(), + space: Condvar::new(), + flushed: Condvar::new(), + flush_call: Mutex::new(()), + dropped: AtomicU64::new(0), + } + } + + fn queue(&self) -> MutexGuard<'_, Queue> { + self.queue.lock().unwrap_or_else(|error| error.into_inner()) + } + + fn wait_for_space<'a>(&self, mut queue: MutexGuard<'a, Queue>) -> MutexGuard<'a, Queue> { + while !queue.closed && queue.commands.len() == QUEUE_CAPACITY { + queue = self + .space + .wait(queue) + .unwrap_or_else(|error| error.into_inner()); + } + queue + } + + fn submit(&self, mut record: Record) { + record.dropped = self.dropped.swap(0, Ordering::Relaxed); + let mut queue = self.queue(); + if !queue.closed && queue.commands.len() == QUEUE_CAPACITY && record.level.drops_when_full() + { + self.dropped + .fetch_add(record.dropped.saturating_add(1), Ordering::Relaxed); + return; + } + queue = self.wait_for_space(queue); + if queue.closed { + drop(queue); + emergency(&record); + return; + } + queue.commands.push_back(Command::Record(record)); + self.available.notify_one(); + } + + fn flush(&self) { + // A single pending flush pairs the fixed acknowledgement with its + // caller. Queue storage and waiting state never allocate after startup. + let _caller = self + .flush_call + .lock() + .unwrap_or_else(|error| error.into_inner()); + let mut queue = self.wait_for_space(self.queue()); + if queue.closed { + return; + } + queue.flushed = false; + queue.commands.push_back(Command::Flush); + self.available.notify_one(); + while !queue.flushed && !queue.closed { + queue = self + .flushed + .wait(queue) + .unwrap_or_else(|error| error.into_inner()); + } + } + + fn receive(&self) -> Option<Command> { + let mut queue = self.queue(); + while queue.commands.is_empty() && !queue.closed { + queue = self + .available + .wait(queue) + .unwrap_or_else(|error| error.into_inner()); + } + let command = queue.commands.pop_front(); + self.space.notify_one(); + command + } + + fn write(&self, output: &mut impl Write) { + let _guard = WriterGuard(self); + while let Some(command) = self.receive() { + match command { + Command::Record(record) => { + if write_record(output, &record).is_err() { + emergency(&record); + } + } + Command::Flush => { + let _ = output.flush(); + let mut queue = self.queue(); + queue.flushed = true; + self.flushed.notify_one(); + } + } + } + let _ = output.flush(); + } + + fn close(&self) { + let mut queue = self.queue(); + queue.closed = true; + queue.commands.clear(); + self.available.notify_all(); + self.space.notify_all(); + self.flushed.notify_all(); + } +} + +fn write_record(output: &mut impl Write, record: &Record) -> io::Result<()> { + write!( + output, + "{{\"level\":\"{}\",\"message\":", + record.level.name() + )?; + write_json_string(output, &record.message)?; + if record.dropped > 0 { + write!(output, ",\"dropped_events\":{}", record.dropped)?; + } + write!(output, ",\"fields\":{{")?; + for (index, field) in record.fields.iter().enumerate() { + if index > 0 { + output.write_all(b",")?; + } + write_json_string(output, &field.name)?; + output.write_all(b":")?; + write_json_string(output, &field.value)?; + } + output.write_all(b"}}\n")?; + if record.level == Level::Error { + output.flush()?; + } + Ok(()) +} + +fn write_json_string(output: &mut impl Write, value: &str) -> io::Result<()> { + output.write_all(b"\"")?; + for character in value.chars() { + match character { + '"' => output.write_all(b"\\\"")?, + '\\' => output.write_all(b"\\\\")?, + '\n' => output.write_all(b"\\n")?, + '\r' => output.write_all(b"\\r")?, + '\t' => output.write_all(b"\\t")?, + character if character.is_control() => write!(output, "\\u{:04x}", character as u32)?, + character => write!(output, "{character}")?, + } + } + output.write_all(b"\"") +} + +fn emergency(record: &Record) { + let mut output = io::stderr().lock(); + let _ = write_record(&mut output, record); +} + +fn truncate(mut value: String, limit: usize) -> String { + if value.len() <= limit { + return value; + } + let end = value + .char_indices() + .map(|(index, _)| index) + .take_while(|index| *index <= limit) + .last() + .unwrap_or(0); + value.truncate(end); + value +} + +#[cfg(test)] +#[path = "../../../test/runtime_logging.rs"] +mod tests; diff --git a/crates/dever-runtime/src/net.rs b/crates/dever-runtime/src/net.rs new file mode 100644 index 0000000..fd6b9ad --- /dev/null +++ b/crates/dever-runtime/src/net.rs @@ -0,0 +1,297 @@ +use std::future::Future; +use std::io::ErrorKind; +use std::net::{IpAddr, SocketAddr, ToSocketAddrs}; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use futures_util::stream; +use tokio::net::{TcpListener, TcpStream}; +use tokio::sync::Mutex as AsyncMutex; + +use crate::async_stream::AsyncStream; +use crate::bytes::Bytes; +use crate::endpoint::Endpoint; +use crate::resource::{Pull, read_limit}; + +#[derive(Clone, Debug)] +pub struct Listener(Arc<Endpoint<TcpListener>>); + +impl Listener { + pub fn close(&self) -> Result<(), String> { + self.0.close() + } +} + +#[derive(Debug)] +struct SocketState { + endpoint: Endpoint<TcpStream>, + reader: AsyncMutex<()>, + writer: AsyncMutex<()>, + timeout_millis: AtomicU64, +} + +#[derive(Clone, Debug)] +pub struct Socket(Arc<SocketState>); + +impl Socket { + fn new(stream: TcpStream) -> Self { + Self(Arc::new(SocketState { + endpoint: Endpoint::new(stream), + reader: AsyncMutex::new(()), + writer: AsyncMutex::new(()), + timeout_millis: AtomicU64::new(0), + })) + } + + pub fn close(&self) -> Result<(), String> { + self.0.endpoint.close() + } + + async fn operation<R>( + &self, + operation: impl Future<Output = Result<R, String>>, + ) -> Result<R, String> { + let operation = self.0.endpoint.while_open(operation); + match self.0.timeout_millis.load(Ordering::Relaxed) { + 0 => operation.await, + millis => timed(Duration::from_millis(millis), operation).await, + } + } +} + +pub async fn connect(host: &str, port: i64) -> Result<Socket, String> { + connect_stream(host, port).await.map(Socket::new) +} + +pub(crate) async fn connect_stream(host: &str, port: i64) -> Result<TcpStream, String> { + let addresses = addresses(host, port).await?; + let stream = TcpStream::connect(addresses.as_slice()) + .await + .map_err(|error| error.to_string())?; + prepare_stream(stream) +} + +pub async fn connect_timeout(address: &str, port: i64, millis: i64) -> Result<Socket, String> { + connect_stream_timeout(address, port, millis) + .await + .map(Socket::new) +} + +pub(crate) async fn connect_stream_timeout( + address: &str, + port: i64, + millis: i64, +) -> Result<TcpStream, String> { + let address = address + .parse::<IpAddr>() + .map_err(|_| "timed connect requires a numeric IP address")?; + let address = SocketAddr::new(address, port_number(port)?); + timed(timeout_duration(millis)?, async { + let stream = TcpStream::connect(address) + .await + .map_err(|error| error.to_string())?; + prepare_stream(stream) + }) + .await +} + +pub async fn listen(host: &str, port: i64) -> Result<Listener, String> { + let addresses = addresses(host, port).await?; + let listener = TcpListener::bind(addresses.as_slice()) + .await + .map_err(|error| error.to_string())?; + Ok(Listener(Arc::new(Endpoint::new(listener)))) +} + +pub async fn accept(listener: &Listener) -> Result<Socket, String> { + accept_stream(listener).await.map(Socket::new) +} + +async fn accept_stream(listener: &Listener) -> Result<TcpStream, String> { + let socket = listener.0.get()?; + listener + .0 + .while_open(async { + let (stream, _) = socket.accept().await.map_err(|error| error.to_string())?; + prepare_stream(stream) + }) + .await +} + +pub fn port(listener: &Listener) -> Result<i64, String> { + listener + .0 + .get()? + .local_addr() + .map(|address| i64::from(address.port())) + .map_err(|error| error.to_string()) +} + +fn prepare_stream(stream: TcpStream) -> Result<TcpStream, String> { + stream + .set_nodelay(true) + .map_err(|error| error.to_string())?; + Ok(stream) +} + +pub fn timeout(socket: &Socket, millis: i64) -> Result<(), String> { + let duration = timeout_duration(millis)?; + socket.0.endpoint.get()?; + socket + .0 + .timeout_millis + .store(duration.as_millis() as u64, Ordering::Relaxed); + Ok(()) +} + +pub async fn read(socket: &Socket, limit: i64) -> Result<Option<Bytes>, String> { + let limit = read_limit(limit)?; + let stream = socket.0.endpoint.get()?; + socket + .operation(async { + let _reader = socket.0.reader.lock().await; + // 等待读锁和可读事件后再分配缓冲区，避免等待者各占一份内存。 + stream.readable().await.map_err(|error| error.to_string())?; + let mut bytes = Vec::new(); + bytes + .try_reserve_exact(limit) + .map_err(|error| error.to_string())?; + bytes.resize(limit, 0); + loop { + match stream.try_read(&mut bytes) { + Ok(length) => { + bytes.truncate(length); + return Ok((length != 0).then(|| Bytes::new(bytes))); + } + Err(error) if error.kind() == ErrorKind::WouldBlock => { + stream.readable().await.map_err(|error| error.to_string())?; + } + Err(error) => return Err(error.to_string()), + } + } + }) + .await +} + +// 部分写入中断时先关闭连接，再释放写锁，阻止下一次写入接在残缺内容后。 +struct WriteProgress<'a> { + socket: &'a Socket, + partial: bool, +} + +impl Drop for WriteProgress<'_> { + fn drop(&mut self) { + if self.partial { + let _ = self.socket.close(); + } + } +} + +pub async fn write(socket: &Socket, bytes: &Bytes) -> Result<(), String> { + let stream = socket.0.endpoint.get()?; + socket + .operation(async { + let _writer = socket.0.writer.lock().await; + let mut progress = WriteProgress { + socket, + partial: false, + }; + let mut offset = 0; + while offset < bytes.values().len() { + stream.writable().await.map_err(|error| error.to_string())?; + match stream.try_write(&bytes.values()[offset..]) { + Ok(0) => return Err("socket write returned zero bytes".into()), + Ok(length) => { + offset += length; + progress.partial = offset < bytes.values().len(); + } + Err(error) if error.kind() == ErrorKind::WouldBlock => {} + Err(error) => return Err(error.to_string()), + } + } + Ok(()) + }) + .await +} + +pub fn connections(listener: Listener) -> AsyncStream<Result<Socket, String>> { + incoming(listener, false).map(|connection| connection.map(Socket::new)) +} + +pub(crate) fn incoming( + listener: Listener, + end_on_close: bool, +) -> AsyncStream<Result<TcpStream, String>> { + AsyncStream::new(stream::unfold(listener, move |listener| async move { + let result = tokio::select! { + biased; + _ = listener.closed(), if end_on_close => return None, + result = accept_stream(&listener) => result, + }; + let event = match result { + Ok(socket) => Pull::Item(Ok(socket)), + Err(error) => Pull::Last(Err(error)), + }; + Some((event, listener)) + })) +} + +impl Listener { + pub(crate) async fn closed(&self) { + self.0.closed().await; + } +} + +pub fn chunks(socket: Socket, limit: i64) -> Result<AsyncStream<Result<Bytes, String>>, String> { + read_limit(limit)?; + socket.0.endpoint.get()?; + Ok(AsyncStream::new(stream::unfold( + socket, + move |socket| async move { + let event = match read(&socket, limit).await { + Ok(Some(bytes)) => Pull::Item(Ok(bytes)), + Ok(None) => return None, + Err(error) => Pull::Last(Err(error)), + }; + Some((event, socket)) + }, + ))) +} + +async fn addresses(host: &str, port: i64) -> Result<Vec<SocketAddr>, String> { + let port = port_number(port)?; + if let Ok(address) = host.parse::<IpAddr>() { + return Ok(vec![SocketAddr::new(address, port)]); + } + let host = host.to_owned(); + // DNS 复用有界 blocking 通道及其结构化清理。 + crate::task::blocking(move || { + (host.as_str(), port) + .to_socket_addrs() + .map(Iterator::collect) + .map_err(|error| error.to_string()) + }) + .await +} + +fn port_number(port: i64) -> Result<u16, String> { + u16::try_from(port).map_err(|_| "port must be between 0 and 65535".into()) +} + +fn timeout_duration(millis: i64) -> Result<Duration, String> { + let millis = u64::try_from(millis) + .ok() + .filter(|millis| *millis > 0) + .ok_or("timeout must be a positive Int")?; + Ok(Duration::from_millis(millis)) +} + +async fn timed<T>( + duration: Duration, + operation: impl Future<Output = Result<T, String>>, +) -> Result<T, String> { + tokio::time::timeout(duration, operation) + .await + .map_err(|_| "network operation timed out".to_owned())? +} diff --git a/crates/dever-runtime/src/number.rs b/crates/dever-runtime/src/number.rs new file mode 100644 index 0000000..adb6f6e --- /dev/null +++ b/crates/dever-runtime/src/number.rs @@ -0,0 +1,208 @@ +use std::cell::RefCell; +use std::cmp::Ordering; +use std::fmt; + +use dec::{Context, Decimal128, Rounding}; + +thread_local! { + static CONTEXT: RefCell<Context<Decimal128>> = RefCell::new(Context::default()); +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct DecimalValue(Decimal128); + +impl Eq for DecimalValue {} + +impl PartialOrd for DecimalValue { + fn partial_cmp(&self, other: &Self) -> Option<Ordering> { + Some(self.cmp(other)) + } +} + +impl Ord for DecimalValue { + fn cmp(&self, other: &Self) -> Ordering { + self.0 + .partial_cmp(&other.0) + .expect("Decimal values are finite") + } +} + +impl fmt::Display for DecimalValue { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + let reduced = Context::<Decimal128>::default().reduce(self.0); + if reduced.is_zero() { + formatter.write_str("0") + } else { + write!(formatter, "{reduced}") + } + } +} + +impl DecimalValue { + pub const ZERO: Self = Self(Decimal128::ZERO); + + pub fn parse(text: &str) -> Result<Self, &'static str> { + CONTEXT.with_borrow_mut(|context| { + context.clear_status(); + let value = context.parse(text).map_err(|_| "invalid Decimal literal")?; + let value = finite_result(context, value)?; + if context.status().inexact() { + return Err("Decimal literal is not exactly representable in 34 digits"); + } + Ok(value) + }) + } + + pub fn from_int(value: i64) -> Self { + Self(Decimal128::from(value)) + } + + pub fn from_bytes(bytes: [u8; 16]) -> Result<Self, &'static str> { + let value = Decimal128::from_le_bytes(bytes); + if value.is_finite() { + Ok(Self(value)) + } else { + Err("Decimal must be finite") + } + } + + pub fn to_bytes(self) -> [u8; 16] { + self.0.to_le_bytes() + } + + pub fn checked_add(self, other: Self) -> Result<Self, &'static str> { + operation(|context| context.add(self.0, other.0)) + } + + pub fn checked_sub(self, other: Self) -> Result<Self, &'static str> { + operation(|context| context.sub(self.0, other.0)) + } + + pub fn checked_mul(self, other: Self) -> Result<Self, &'static str> { + operation(|context| context.mul(self.0, other.0)) + } + + pub fn checked_div(self, other: Self) -> Result<Self, &'static str> { + operation(|context| context.div(self.0, other.0)) + } + + pub fn checked_neg(self) -> Result<Self, &'static str> { + operation(|context| context.minus(self.0)) + } + + pub fn round(self, places: i64) -> Result<Self, &'static str> { + let exponent = places + .checked_neg() + .and_then(|value| i32::try_from(value).ok()) + .ok_or("Decimal rounding precision is out of range")?; + if !(-6176..=6111).contains(&exponent) { + return Err("Decimal rounding precision is out of range"); + } + operation(|context| { + let mut quantum = Decimal128::ONE; + context.set_exponent(&mut quantum, exponent); + context.quantize(self.0, quantum) + }) + } + + pub fn next_up(self) -> Option<Self> { + let value = Context::<Decimal128>::default().next_plus(self.0); + value.is_finite().then_some(Self(value)) + } + + pub fn next_down(self) -> Option<Self> { + let value = Context::<Decimal128>::default().next_minus(self.0); + value.is_finite().then_some(Self(value)) + } + + pub fn minimum() -> Self { + Self::parse("-9.999999999999999999999999999999999E+6144").expect("Decimal128 minimum") + } + + pub fn maximum() -> Self { + Self::parse("9.999999999999999999999999999999999E+6144").expect("Decimal128 maximum") + } +} + +fn operation( + apply: impl FnOnce(&mut Context<Decimal128>) -> Decimal128, +) -> Result<DecimalValue, &'static str> { + CONTEXT.with_borrow_mut(|context| { + context.clear_status(); + context.set_rounding(Rounding::HalfEven); + let value = apply(context); + finite_result(context, value) + }) +} + +fn finite_result( + context: &Context<Decimal128>, + value: Decimal128, +) -> Result<DecimalValue, &'static str> { + let status = context.status(); + if status.division_by_zero() || status.division_undefined() { + Err("Decimal division by zero") + } else if status.overflow() { + Err("Decimal overflow") + } else if status.underflow() { + Err("Decimal underflow") + } else if status.invalid_operation() || status.invalid_context() || !value.is_finite() { + Err("invalid Decimal operation") + } else { + Ok(DecimalValue(value)) + } +} + +pub fn int_add(left: i64, right: i64) -> Result<i64, &'static str> { + left.checked_add(right).ok_or("Int overflow") +} + +pub fn parse_int(text: &str) -> Option<i64> { + text.parse().ok() +} + +pub fn parse_decimal(text: &str) -> Option<DecimalValue> { + decimal_text(text) + .then(|| DecimalValue::parse(text).ok()) + .flatten() +} + +pub fn parse_float(text: &str) -> Option<f64> { + if !decimal_text(text) && !matches!(text, "NaN" | "inf" | "-inf") { + return None; + } + text.parse().ok() +} + +fn decimal_text(text: &str) -> bool { + text.bytes().any(|byte| byte.is_ascii_digit()) + && text + .bytes() + .all(|byte| byte.is_ascii_digit() || matches!(byte, b'+' | b'-' | b'.' | b'e' | b'E')) +} + +pub fn int_sub(left: i64, right: i64) -> Result<i64, &'static str> { + left.checked_sub(right).ok_or("Int overflow") +} + +pub fn int_mul(left: i64, right: i64) -> Result<i64, &'static str> { + left.checked_mul(right).ok_or("Int overflow") +} + +pub fn int_neg(value: i64) -> Result<i64, &'static str> { + value.checked_neg().ok_or("Int overflow") +} + +pub fn int_div(left: i64, right: i64) -> Result<i64, &'static str> { + if right == 0 { + return Err("Int division by zero"); + } + left.checked_div(right).ok_or("Int overflow") +} + +pub fn int_rem(left: i64, right: i64) -> Result<i64, &'static str> { + if right == 0 { + return Err("Int division by zero"); + } + left.checked_rem(right).ok_or("Int overflow") +} diff --git a/crates/dever-runtime/src/orm.rs b/crates/dever-runtime/src/orm.rs new file mode 100644 index 0000000..233b607 --- /dev/null +++ b/crates/dever-runtime/src/orm.rs @@ -0,0 +1,695 @@ +use std::fmt; +use std::pin::Pin; +use std::sync::Arc; +use std::sync::Mutex as StdMutex; + +use crate::number::DecimalValue; +use futures_util::{Stream, StreamExt, stream}; +use tokio::sync::{Mutex, Semaphore}; + +#[derive(Clone, Debug, PartialEq)] +pub enum Value { + Null, + Bool(bool), + Int(i64), + Float(f64), + Decimal(String), + Text(String), + Bytes(Vec<u8>), + Uuid(Uuid), +} + +#[derive(Clone, Debug, PartialEq)] +pub enum Related<T> { + Unloaded, + Loaded(Box<T>), +} + +impl<T: crate::render::Render> crate::render::Render for Related<T> { + fn render_to(&self, output: &mut String) { + match self { + Self::Unloaded => output.push_str("Related.Unloaded"), + Self::Loaded(value) => { + output.push_str("Related.Loaded("); + value.render_to(output); + output.push(')'); + } + } + } +} + +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct Uuid([u8; 16]); + +impl Uuid { + pub const fn from_bytes(value: [u8; 16]) -> Self { + Self(value) + } + + pub const fn into_bytes(self) -> [u8; 16] { + self.0 + } + + pub fn parse(text: &str) -> Result<Self, Error> { + if text.len() != 36 + || ![8, 13, 18, 23] + .into_iter() + .all(|index| text.as_bytes()[index] == b'-') + { + return Err(Error::invalid_data( + "Uuid must use the canonical 8-4-4-4-12 form", + )); + } + let mut output = [0; 16]; + let mut source = text.bytes().filter(|byte| *byte != b'-'); + for byte in &mut output { + let high = hex(source.next().expect("validated Uuid length"))?; + let low = hex(source.next().expect("validated Uuid length"))?; + *byte = (high << 4) | low; + } + Ok(Self(output)) + } + + #[cfg(feature = "database")] + pub fn new_v7() -> Result<Self, Error> { + let millis = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_err(|error| Error::new(ErrorKind::Database, error.to_string()))? + .as_millis(); + if millis > 0xffff_ffff_ffff { + return Err(Error::new( + ErrorKind::Database, + "system time exceeds the UUIDv7 timestamp range", + )); + } + let mut value = [0; 16]; + getrandom::getrandom(&mut value[6..]) + .map_err(|error| Error::new(ErrorKind::Database, error.to_string()))?; + let timestamp = (millis as u64).to_be_bytes(); + value[..6].copy_from_slice(×tamp[2..]); + value[6] = (value[6] & 0x0f) | 0x70; + value[8] = (value[8] & 0x3f) | 0x80; + Ok(Self(value)) + } +} + +impl fmt::Display for Uuid { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + for (index, byte) in self.0.iter().enumerate() { + if matches!(index, 4 | 6 | 8 | 10) { + formatter.write_str("-")?; + } + write!(formatter, "{byte:02x}")?; + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Row { + values: Vec<Value>, +} + +type RowProducer<T> = Pin<Box<dyn Stream<Item = Result<T, Error>> + Send>>; + +struct RowStreamState<T> { + producer: Mutex<Option<RowProducer<T>>>, + closed: Semaphore, + worker: StdMutex<Option<crate::task::ScopedOwner>>, + worker_fault: Option<crate::task::ScopeFault>, + _database_lease: Option<Arc<dyn Send + Sync>>, +} + +/// Internal database stream used by generated Model code. Dever exposes it as AsyncStream<T>. +pub struct RowStream<T>(Arc<RowStreamState<T>>); + +impl<T> Clone for RowStream<T> { + fn clone(&self) -> Self { + Self(Arc::clone(&self.0)) + } +} + +impl<T> fmt::Debug for RowStream<T> { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("RowStream(<resource>)") + } +} + +impl<T> RowStream<T> { + pub(crate) fn new(producer: impl Stream<Item = Result<T, Error>> + Send + 'static) -> Self { + Self(Arc::new(RowStreamState { + producer: Mutex::new(Some(Box::pin(producer))), + closed: Semaphore::new(0), + worker: StdMutex::new(None), + worker_fault: None, + _database_lease: None, + })) + } + + #[cfg(any(feature = "sqlite", feature = "postgres"))] + pub(crate) fn hold_database(mut self, lease: Arc<dyn Send + Sync>) -> Self { + Arc::get_mut(&mut self.0) + .expect("new database stream has one owner") + ._database_lease = Some(lease); + self + } + + pub async fn pull(&self) -> Result<Option<T>, Error> { + let mut producer = self.0.producer.lock().await; + let next = match producer.as_mut() { + Some(producer) => match &self.0.worker_fault { + Some(fault) => tokio::select! { + biased; + _ = self.0.closed.acquire() => None, + message = fault.wait() => Some(Err(Error::new(ErrorKind::Database, message))), + value = producer.next() => value, + }, + None => tokio::select! { + biased; + _ = self.0.closed.acquire() => None, + value = producer.next() => value, + }, + }, + None => None, + }; + let result = match next { + Some(Ok(value)) => Ok(Some(value)), + Some(Err(error)) => { + producer.take(); + Err(error) + } + None => { + producer.take(); + Ok(None) + } + }; + if result.as_ref().is_err() || matches!(&result, Ok(None)) { + drop(producer); + self.0.finish_worker(); + } + result + } + + pub fn close(&self) { + self.0.closed.close(); + if let Ok(mut producer) = self.0.producer.try_lock() { + producer.take(); + } + self.0.stop_worker(); + } + + pub fn close_on_drop(&self) -> CloseRowStreamOnDrop<T> { + CloseRowStreamOnDrop(self.clone()) + } +} + +impl<T: Send + 'static> RowStream<T> { + #[cfg(any(feature = "sqlite", feature = "postgres"))] + pub(crate) fn from_receiver( + receiver: tokio::sync::mpsc::Receiver<Result<T, Error>>, + worker: crate::task::ScopedOwner, + worker_fault: crate::task::ScopeFault, + ) -> Self { + Self(Arc::new(RowStreamState { + producer: Mutex::new(Some(Box::pin(stream::unfold( + receiver, + |mut receiver| async move { receiver.recv().await.map(|value| (value, receiver)) }, + )))), + closed: Semaphore::new(0), + worker: StdMutex::new(Some(worker)), + worker_fault: Some(worker_fault), + _database_lease: None, + })) + } + + pub fn map<U: Send + 'static>( + self, + transform: impl FnMut(T) -> Result<U, Error> + Send + 'static, + ) -> RowStream<U> { + let owner = self.close_on_drop(); + RowStream::new(stream::unfold( + (owner, transform), + |(owner, mut transform)| async move { + owner.0.pull().await.transpose().map(|value| { + let value = value.and_then(&mut transform); + (value, (owner, transform)) + }) + }, + )) + } +} + +impl<T> RowStreamState<T> { + fn finish_worker(&self) { + let owner = self + .worker + .lock() + .unwrap_or_else(|error| error.into_inner()) + .take(); + if let Some(owner) = owner { + owner.finish_in_background(); + } + } + + fn stop_worker(&self) { + let owner = self + .worker + .lock() + .unwrap_or_else(|error| error.into_inner()) + .take(); + if let Some(owner) = owner { + owner.stop_in_background(); + } + } +} + +impl<T> Drop for RowStreamState<T> { + fn drop(&mut self) { + self.closed.close(); + self.producer.get_mut().take(); + let owner = self + .worker + .get_mut() + .unwrap_or_else(|error| error.into_inner()) + .take(); + if let Some(owner) = owner { + owner.stop_in_background(); + } + } +} + +pub struct CloseRowStreamOnDrop<T>(RowStream<T>); + +impl<T> Drop for CloseRowStreamOnDrop<T> { + fn drop(&mut self) { + self.0.close(); + } +} + +impl Row { + pub fn new(values: Vec<Value>) -> Self { + Self { values } + } + + pub fn get(&self, index: usize) -> Result<&Value, Error> { + self.values.get(index).ok_or_else(|| { + Error::invalid_data(format!("database row has no column at index {index}")) + }) + } + + pub fn len(&self) -> usize { + self.values.len() + } + + pub fn is_empty(&self) -> bool { + self.values.is_empty() + } + + pub fn into_values(self) -> std::vec::IntoIter<Value> { + self.values.into_iter() + } +} + +pub fn required_row(rows: Vec<Row>, operation: &str) -> Result<Row, Error> { + let mut rows = rows.into_iter(); + let row = rows.next().ok_or_else(|| { + Error::invalid_data(format!("SQL operation '{operation}' returned no rows")) + })?; + if rows.next().is_some() { + return Err(Error::invalid_data(format!( + "SQL operation '{operation}' returned more than one row" + ))); + } + Ok(row) +} + +pub fn optional_row(rows: Vec<Row>, operation: &str) -> Result<Option<Row>, Error> { + let mut rows = rows.into_iter(); + let row = rows.next(); + if rows.next().is_some() { + return Err(Error::invalid_data(format!( + "SQL operation '{operation}' returned more than one row" + ))); + } + Ok(row) +} + +pub fn bounded_rows(rows: Vec<Row>, maximum: usize) -> Result<Vec<Row>, Error> { + if rows.len() > maximum { + let message = if maximum == 1 { + "database query returned more than one row".into() + } else { + format!("database query returned more than {maximum} rows") + }; + Err(Error::invalid_data(message)) + } else { + Ok(rows) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ErrorKind { + Pool, + PoolExhausted, + Connection, + Timeout, + Cancelled, + Database, + Constraint, + NotFound, + InvalidData, + Migration, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Error { + kind: ErrorKind, + message: String, +} + +impl Error { + pub fn new(kind: ErrorKind, message: impl Into<String>) -> Self { + Self { + kind, + message: message.into(), + } + } + + pub fn invalid_data(message: impl Into<String>) -> Self { + Self::new(ErrorKind::InvalidData, message) + } + + pub fn not_found(model: &str) -> Self { + Self::new(ErrorKind::NotFound, format!("{model} was not found")) + } + + pub fn migration(message: impl Into<String>) -> Self { + Self::new(ErrorKind::Migration, message) + } + + pub fn cause(mut self, error: impl fmt::Display) -> Self { + self.message.push_str("; caused by: "); + self.message.push_str(&error.to_string()); + self + } + + pub fn kind(&self) -> ErrorKind { + self.kind + } + + #[cfg(feature = "postgres")] + pub(crate) fn keeps_connection(&self) -> bool { + matches!( + self.kind, + ErrorKind::Constraint + | ErrorKind::NotFound + | ErrorKind::InvalidData + | ErrorKind::Migration + ) + } +} + +impl fmt::Display for Error { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.message) + } +} + +impl std::error::Error for Error {} + +pub fn boolean(value: Value) -> Result<bool, Error> { + match value { + Value::Bool(value) => Ok(value), + Value::Int(0) => Ok(false), + Value::Int(1) => Ok(true), + _ => Err(Error::invalid_data("expected a database Bool")), + } +} + +pub fn int(value: Value) -> Result<i64, Error> { + match value { + Value::Int(value) => Ok(value), + _ => Err(Error::invalid_data("expected a database Int")), + } +} + +pub fn float(value: Value) -> Result<f64, Error> { + match value { + Value::Float(value) => Ok(value), + Value::Int(value) => Ok(value as f64), + _ => Err(Error::invalid_data("expected a database Float")), + } +} + +pub fn decimal(value: Value) -> Result<DecimalValue, Error> { + match value { + Value::Decimal(value) | Value::Text(value) => decimal_text(&value) + .and_then(|value| DecimalValue::parse(&value).map_err(Error::invalid_data)), + _ => Err(Error::invalid_data("expected a database Decimal")), + } +} + +pub fn decimal_text(value: &str) -> Result<String, Error> { + if value.starts_with('D') { + decode_decimal_storage(value).map(|value| value.to_string()) + } else { + DecimalValue::parse(value) + .map(|value| value.to_string()) + .map_err(Error::invalid_data) + } +} + +pub fn decimal_value(value: DecimalValue, precision: u8, scale: u8) -> Result<Value, Error> { + decimal_storage(&value.to_string(), precision, scale).map(Value::Decimal) +} + +pub fn decimal_storage(text: &str, precision: u8, scale: u8) -> Result<String, Error> { + let (negative, mut digits) = scaled_decimal(text, precision, scale)?; + if negative { + for digit in &mut digits { + *digit = b'9' - (*digit - b'0'); + } + } + Ok(format!( + "D{}{}:{scale:02}", + if negative { '0' } else { '1' }, + String::from_utf8(digits).expect("ASCII decimal digits") + )) +} + +pub fn uuid(value: Value) -> Result<Uuid, Error> { + match value { + Value::Uuid(value) => Ok(value), + Value::Bytes(value) => <[u8; 16]>::try_from(value) + .map(Uuid::from_bytes) + .map_err(|_| Error::invalid_data("database Uuid must contain 16 bytes")), + _ => Err(Error::invalid_data("expected a database Uuid")), + } +} + +fn scaled_decimal(text: &str, precision: u8, scale: u8) -> Result<(bool, Vec<u8>), Error> { + let (negative, text) = match text.as_bytes().first() { + Some(b'-') => (true, &text[1..]), + Some(b'+') => (false, &text[1..]), + _ => (false, text), + }; + let (mantissa, exponent) = text + .find(['e', 'E']) + .map(|index| { + text[index + 1..] + .parse::<i32>() + .map(|exponent| (&text[..index], exponent)) + .map_err(|_| Error::invalid_data("invalid Decimal exponent")) + }) + .transpose()? + .unwrap_or((text, 0)); + let mut point = None; + let mut digits = Vec::with_capacity(mantissa.len() + usize::from(scale)); + for byte in mantissa.bytes() { + if byte == b'.' && point.is_none() { + point = Some(digits.len()); + } else if byte.is_ascii_digit() { + digits.push(byte); + } else { + return Err(Error::invalid_data("invalid Decimal value")); + } + } + if digits.is_empty() { + return Err(Error::invalid_data("invalid Decimal value")); + } + let integer_digits = i64::try_from(point.unwrap_or(digits.len())) + .map_err(|_| Error::invalid_data("Decimal value is too long"))? + .checked_add(i64::from(exponent)) + .ok_or_else(|| Error::invalid_data("Decimal exponent is out of range"))?; + let scaled_length = integer_digits + .checked_add(i64::from(scale)) + .ok_or_else(|| Error::invalid_data("Decimal scale is out of range"))?; + let first_nonzero = digits.iter().position(|digit| *digit != b'0'); + let Some(first_nonzero) = first_nonzero else { + return Ok((false, vec![b'0'; usize::from(precision)])); + }; + if scaled_length <= i64::try_from(first_nonzero).expect("index fits i64") { + return Err(Error::invalid_data(format!( + "Decimal has more than {scale} fractional digits" + ))); + } + let significant_length = scaled_length - i64::try_from(first_nonzero).expect("index fits i64"); + if significant_length > i64::from(precision) { + return Err(Error::invalid_data(format!( + "Decimal exceeds precision {precision} with scale {scale}" + ))); + } + let scaled_length = usize::try_from(scaled_length) + .map_err(|_| Error::invalid_data("Decimal scale is out of range"))?; + if scaled_length < digits.len() { + if digits[scaled_length..].iter().any(|digit| *digit != b'0') { + return Err(Error::invalid_data(format!( + "Decimal has more than {scale} fractional digits" + ))); + } + digits.truncate(scaled_length); + } else { + digits.resize(scaled_length, b'0'); + } + let mut magnitude = digits.split_off(first_nonzero); + let mut padded = vec![b'0'; usize::from(precision) - magnitude.len()]; + padded.append(&mut magnitude); + Ok((negative, padded)) +} + +fn decode_decimal_storage(text: &str) -> Result<DecimalValue, Error> { + let Some(encoded) = text.strip_prefix('D') else { + return Err(Error::invalid_data("invalid stored Decimal")); + }; + let (sign, encoded) = encoded + .split_at_checked(1) + .ok_or_else(|| Error::invalid_data("invalid stored Decimal"))?; + let (digits, scale) = encoded + .rsplit_once(':') + .ok_or_else(|| Error::invalid_data("invalid stored Decimal"))?; + let scale = scale + .parse::<usize>() + .map_err(|_| Error::invalid_data("invalid stored Decimal scale"))?; + if !matches!(sign, "0" | "1") + || digits.len() < scale + || !digits.bytes().all(|byte| byte.is_ascii_digit()) + { + return Err(Error::invalid_data("invalid stored Decimal")); + } + let digits = if sign == "0" { + digits + .bytes() + .map(|digit| char::from(b'9' - (digit - b'0'))) + .collect::<String>() + } else { + digits.to_owned() + }; + let point = digits.len() - scale; + let mut decimal = if scale == 0 { + digits + } else { + format!("{}.{}", &digits[..point], &digits[point..]) + }; + if sign == "0" { + decimal.insert(0, '-'); + } + DecimalValue::parse(&decimal).map_err(Error::invalid_data) +} + +fn hex(value: u8) -> Result<u8, Error> { + match value { + b'0'..=b'9' => Ok(value - b'0'), + b'a'..=b'f' => Ok(value - b'a' + 10), + b'A'..=b'F' => Ok(value - b'A' + 10), + _ => Err(Error::invalid_data("Uuid contains a non-hexadecimal digit")), + } +} + +pub fn text(value: Value) -> Result<String, Error> { + match value { + Value::Text(value) => Ok(value), + _ => Err(Error::invalid_data("expected database Text")), + } +} + +pub fn bytes(value: Value) -> Result<crate::bytes::Bytes, Error> { + match value { + Value::Bytes(value) => Ok(crate::bytes::Bytes::new(value)), + _ => Err(Error::invalid_data("expected database Bytes")), + } +} + +pub fn nullable<T>( + value: Value, + decode: impl FnOnce(Value) -> Result<T, Error>, +) -> Result<Option<T>, Error> { + match value { + Value::Null => Ok(None), + value => decode(value).map(Some), + } +} + +pub fn column(values: &mut std::vec::IntoIter<Value>, name: &str) -> Result<Value, Error> { + values + .next() + .ok_or_else(|| Error::invalid_data(format!("database row is missing column '{name}'"))) +} + +pub fn scalar(row: Row) -> Result<Value, Error> { + let mut values = row.into_values(); + let value = column(&mut values, "scalar")?; + if values.next().is_some() { + return Err(Error::invalid_data( + "database scalar query returned more than one column", + )); + } + Ok(value) +} + +pub fn pagination(page: i64, size: i64, maximum: usize) -> Result<(i64, i64, i64), Error> { + let maximum = i64::try_from(maximum) + .map_err(|_| Error::invalid_data("database page limit is too large"))?; + if page < 1 { + return Err(Error::invalid_data("page must be at least 1")); + } + if size < 1 || size > maximum { + return Err(Error::invalid_data(format!( + "page size must be between 1 and {maximum}" + ))); + } + let offset = page + .checked_sub(1) + .and_then(|page| page.checked_mul(size)) + .ok_or_else(|| Error::invalid_data("page offset is too large"))?; + Ok((page, size, offset)) +} + +pub fn cursor_size(size: i64, maximum: usize) -> Result<i64, Error> { + let maximum = i64::try_from(maximum) + .map_err(|_| Error::invalid_data("database cursor limit is too large"))?; + if size < 1 || size > maximum || size == i64::MAX { + return Err(Error::invalid_data(format!( + "cursor size must be between 1 and {maximum}" + ))); + } + Ok(size) +} + +pub fn relation_limit(maximum: usize) -> Result<i64, Error> { + i64::try_from(maximum).map_err(|_| Error::invalid_data("database relation limit is too large")) +} + +pub fn stream_capacity(size: i64, maximum: usize) -> Result<usize, Error> { + let maximum = i64::try_from(maximum) + .map_err(|_| Error::invalid_data("database stream limit is too large"))?; + if size < 1 || size > maximum { + return Err(Error::invalid_data(format!( + "stream buffer size must be between 1 and {maximum}" + ))); + } + usize::try_from(size).map_err(|_| Error::invalid_data("stream buffer size is too large")) +} + +pub fn affected(value: usize) -> Result<i64, Error> { + i64::try_from(value).map_err(|_| Error::invalid_data("affected row count is too large")) +} diff --git a/crates/dever-runtime/src/pool.rs b/crates/dever-runtime/src/pool.rs new file mode 100644 index 0000000..4590f22 --- /dev/null +++ b/crates/dever-runtime/src/pool.rs @@ -0,0 +1,34 @@ +#[cfg(feature = "sqlite")] +pub(crate) const WAIT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); +pub(crate) const CREATE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); +pub(crate) const RECYCLE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); + +pub(crate) struct PoolObjectGuard<T> { + object: Option<T>, + detach: fn(T), +} + +impl<T> PoolObjectGuard<T> { + pub(crate) fn new(object: T, detach: fn(T)) -> Self { + Self { + object: Some(object), + detach, + } + } + + pub(crate) fn object(&self) -> &T { + self.object.as_ref().expect("active pooled object guard") + } + + pub(crate) fn release(mut self) -> T { + self.object.take().expect("active pooled object guard") + } +} + +impl<T> Drop for PoolObjectGuard<T> { + fn drop(&mut self) { + if let Some(object) = self.object.take() { + (self.detach)(object); + } + } +} diff --git a/crates/dever-runtime/src/postgres.rs b/crates/dever-runtime/src/postgres.rs new file mode 100644 index 0000000..a8ea19f --- /dev/null +++ b/crates/dever-runtime/src/postgres.rs @@ -0,0 +1,922 @@ +use std::collections::BTreeMap; +use std::error::Error as _; +use std::future::Future; +use std::str::FromStr; +use std::sync::{Arc, OnceLock}; +use std::time::Duration; + +use deadpool_postgres::{ + Manager, ManagerConfig, Pool, PoolConfig, RecyclingMethod, Runtime, Timeouts, +}; +use futures_util::TryStreamExt; +use tokio_postgres::types::{FromSql, IsNull, ToSql, Type, to_sql_checked}; + +use crate::config::{Database as DatabaseConfig, PostgresTls, Settings}; +use crate::orm::{Error, ErrorKind, Row, RowStream, Uuid, Value}; +use crate::pool::{CREATE_TIMEOUT, PoolObjectGuard, RECYCLE_TIMEOUT}; + +mod migration; + +const STATEMENT_CACHE_CAPACITY: usize = 64; + +static DATABASES: OnceLock<BTreeMap<String, Arc<Database>>> = OnceLock::new(); + +pub struct Database { + pool: Pool, + min_connections: usize, + max_page_size: usize, + io_timeout: Duration, +} + +pub struct ConnectionOptions<'a> { + pub url: &'a str, + pub database_name: Option<&'a str>, + pub tls: PostgresTls, + pub min_connections: usize, + pub max_connections: usize, + pub max_page_size: usize, + pub wait_timeout: Duration, + pub io_timeout: Duration, +} + +pub struct Transaction { + connection: Option<deadpool_postgres::Object>, + max_page_size: usize, + io_timeout: Duration, +} + +pub enum Executor<'a> { + Pool(&'a Database), + Transaction(&'a Transaction), +} + +struct Parameter(Value); +struct NumericText(String); +#[derive(Debug)] +struct UuidBytes(Uuid); + +impl Database { + pub fn open(options: ConnectionOptions<'_>) -> Result<Self, Error> { + let ConnectionOptions { + url, + database_name, + tls, + min_connections, + max_connections, + max_page_size, + wait_timeout, + io_timeout, + } = options; + let mut config = tokio_postgres::Config::from_str(url) + .map_err(|error| Error::new(ErrorKind::Pool, error.to_string()))?; + if let Some(database_name) = database_name { + config.dbname(database_name); + } + config.connect_timeout(CREATE_TIMEOUT); + let pool = match tls { + PostgresTls::Disabled => { + config.ssl_mode(tokio_postgres::config::SslMode::Disable); + build_pool(config, tokio_postgres::NoTls, max_connections, wait_timeout)? + } + PostgresTls::System => { + config.ssl_mode(tokio_postgres::config::SslMode::Require); + let (tls, _) = tokio_postgres_rustls::MakeRustlsConnect::with_native_certs() + .map_err(|errors| { + Error::new( + ErrorKind::Pool, + errors + .into_iter() + .map(|error| error.to_string()) + .collect::<Vec<_>>() + .join("; "), + ) + })?; + build_pool(config, tls, max_connections, wait_timeout)? + } + }; + Ok(Self { + pool, + min_connections, + max_page_size, + io_timeout, + }) + } + + pub fn max_page_size(&self) -> usize { + self.max_page_size + } + + pub(crate) async fn prepare(&self) -> Result<(), Error> { + let mut connections = Vec::with_capacity(self.min_connections); + for _ in 0..self.min_connections { + connections.push(self.connection().await?); + } + Ok(()) + } + + pub async fn execute(&self, sql: &'static str, parameters: Vec<Value>) -> Result<usize, Error> { + let connection = guarded(self.connection().await?); + let result = io( + self.io_timeout, + execute_on(connection.object(), sql, parameters), + ) + .await; + finish(connection, result) + } + + pub async fn query( + &self, + sql: &'static str, + parameters: Vec<Value>, + ) -> Result<Vec<Row>, Error> { + let connection = guarded(self.connection().await?); + let result = io( + self.io_timeout, + query_on(connection.object(), sql, parameters), + ) + .await; + finish(connection, result) + } + + pub async fn query_bounded( + &self, + sql: &'static str, + parameters: Vec<Value>, + maximum: usize, + ) -> Result<Vec<Row>, Error> { + let connection = guarded(self.connection().await?); + let result = io( + self.io_timeout, + query_bounded_on(connection.object(), sql, parameters, maximum), + ) + .await; + finish(connection, result) + } + + pub async fn query_owned( + &self, + sql: String, + parameters: Vec<Value>, + ) -> Result<Vec<Row>, Error> { + let connection = guarded(self.connection().await?); + let result = io( + self.io_timeout, + query_owned_on(connection.object(), &sql, parameters), + ) + .await; + finish(connection, result) + } + + pub async fn stream( + &self, + sql: &'static str, + parameters: Vec<Value>, + capacity: usize, + ) -> Result<RowStream<Row>, Error> { + if capacity == 0 || capacity > self.max_page_size { + return Err(Error::invalid_data(format!( + "row stream capacity must be between 1 and {}", + self.max_page_size + ))); + } + let connection = self.connection().await?; + let io_timeout = self.io_timeout; + let (rows, receiver) = tokio::sync::mpsc::channel(capacity); + let (ready, initialized) = tokio::sync::oneshot::channel(); + let (worker, spawner, worker_fault) = + crate::task::scoped_owner(()).map_err(|error| Error::new(ErrorKind::Pool, error))?; + spawner.spawn(async move { + let connection = guarded(connection); + let mut ready = Some(ready); + let result = async { + let statement = io(io_timeout, prepare_cached(connection.object(), sql)).await?; + let parameters = parameters.into_iter().map(Parameter).collect::<Vec<_>>(); + let references = parameters + .iter() + .map(|value| value as &(dyn ToSql + Sync)) + .collect::<Vec<_>>(); + let source = io(io_timeout, async { + connection + .object() + .query_raw(&statement, references) + .await + .map_err(database_error) + }) + .await?; + tokio::pin!(source); + ready + .take() + .expect("pending row stream initialization") + .send(Ok(())) + .ok(); + while let Some(row) = io(io_timeout, async { + source.as_mut().try_next().await.map_err(database_error) + }) + .await? + { + if rows.send(decode_row(row)).await.is_err() { + return Err(Error::new( + ErrorKind::Cancelled, + "PostgreSQL row stream consumer closed", + )); + } + } + Ok::<_, Error>(()) + } + .await; + match result { + Ok(()) => drop(connection.release()), + Err(error) => { + if let Some(ready) = ready.take() { + let _ = ready.send(Err(error)); + } else { + let _ = rows.send(Err(error)).await; + } + } + } + }); + initialized + .await + .map_err(|_| Error::new(ErrorKind::Pool, "PostgreSQL row stream worker stopped"))??; + Ok(RowStream::from_receiver(receiver, worker, worker_fault)) + } + + pub async fn begin(&self) -> Result<Transaction, Error> { + let connection = guarded(self.connection().await?); + io(self.io_timeout, async { + connection + .object() + .batch_execute("BEGIN") + .await + .map_err(database_error) + }) + .await?; + Ok(Transaction { + connection: Some(connection.release()), + max_page_size: self.max_page_size, + io_timeout: self.io_timeout, + }) + } + + pub async fn migrate(&self, model: crate::database::Model) -> Result<(), Error> { + let connection = guarded(self.connection().await?); + let result = io( + self.io_timeout, + migration::apply(connection.object(), model), + ) + .await; + finish(connection, result) + } + + pub async fn migrate_foreign_keys( + &self, + model: String, + table: String, + constraints: Vec<crate::database::PostgresConstraint>, + ) -> Result<(), Error> { + let connection = guarded(self.connection().await?); + let result = io( + self.io_timeout, + migration::apply_foreign_keys(connection.object(), &model, &table, &constraints), + ) + .await; + finish(connection, result) + } + + async fn connection(&self) -> Result<deadpool_postgres::Object, Error> { + self.pool.get().await.map_err(pool_error) + } + + pub async fn close(&self) -> Result<(), Error> { + self.stop_accepting(); + self.drain().await + } + + pub(crate) fn stop_accepting(&self) { + self.pool.close(); + } + + async fn drain(&self) -> Result<(), Error> { + io(self.io_timeout, async { + while self.pool.status().size != 0 { + tokio::task::yield_now().await; + } + Ok(()) + }) + .await + } +} + +pub(crate) async fn create_database( + url: &str, + database_name: &str, + tls: PostgresTls, + wait_timeout: Duration, + io_timeout: Duration, +) -> Result<(), Error> { + if database_name.is_empty() + || database_name.len() > 63 + || !database_name + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_') + { + return Err(Error::migration("invalid PostgreSQL tenant database name")); + } + let database = Database::open(ConnectionOptions { + url, + database_name: None, + tls, + min_connections: 0, + max_connections: 1, + max_page_size: 1, + wait_timeout, + io_timeout, + })?; + let connection = guarded(database.connection().await?); + let sql = format!("CREATE DATABASE \"{database_name}\""); + let result = io(io_timeout, async { + match connection.object().batch_execute(&sql).await { + Ok(()) => Ok(()), + Err(error) if error.code().is_some_and(|code| code.code() == "42P04") => Ok(()), + Err(error) => Err(database_error(error)), + } + }) + .await; + let result = finish(connection, result); + database.close().await?; + result +} + +impl Transaction { + pub async fn execute(&self, sql: &'static str, parameters: Vec<Value>) -> Result<usize, Error> { + io( + self.io_timeout, + execute_on(self.connection(), sql, parameters), + ) + .await + } + + pub async fn query( + &self, + sql: &'static str, + parameters: Vec<Value>, + ) -> Result<Vec<Row>, Error> { + io( + self.io_timeout, + query_on(self.connection(), sql, parameters), + ) + .await + } + + pub async fn query_bounded( + &self, + sql: &'static str, + parameters: Vec<Value>, + maximum: usize, + ) -> Result<Vec<Row>, Error> { + io( + self.io_timeout, + query_bounded_on(self.connection(), sql, parameters, maximum), + ) + .await + } + + pub async fn query_owned( + &self, + sql: String, + parameters: Vec<Value>, + ) -> Result<Vec<Row>, Error> { + io( + self.io_timeout, + query_owned_on(self.connection(), &sql, parameters), + ) + .await + } + + pub async fn commit(mut self) -> Result<(), Error> { + self.finish("COMMIT").await + } + + pub async fn rollback(mut self) -> Result<(), Error> { + self.finish("ROLLBACK").await + } + + fn connection(&self) -> &deadpool_postgres::Object { + self.connection + .as_ref() + .expect("open PostgreSQL transaction") + } + + async fn finish(&mut self, sql: &'static str) -> Result<(), Error> { + let connection = guarded(self.connection.take().expect("open PostgreSQL transaction")); + let result = io(self.io_timeout, async { + connection + .object() + .batch_execute(sql) + .await + .map_err(database_error) + }) + .await; + finish(connection, result) + } +} + +impl Drop for Transaction { + fn drop(&mut self) { + if let Some(connection) = self.connection.take() { + drop(deadpool_postgres::Object::take(connection)); + } + } +} + +fn guarded(connection: deadpool_postgres::Object) -> PoolObjectGuard<deadpool_postgres::Object> { + PoolObjectGuard::new(connection, detach) +} + +fn detach(connection: deadpool_postgres::Object) { + drop(deadpool_postgres::Object::take(connection)); +} + +impl Executor<'_> { + pub fn max_page_size(&self) -> usize { + match self { + Self::Pool(database) => database.max_page_size(), + Self::Transaction(transaction) => transaction.max_page_size, + } + } + + pub async fn execute(&self, sql: &'static str, parameters: Vec<Value>) -> Result<usize, Error> { + match self { + Self::Pool(database) => database.execute(sql, parameters).await, + Self::Transaction(transaction) => transaction.execute(sql, parameters).await, + } + } + + pub async fn query( + &self, + sql: &'static str, + parameters: Vec<Value>, + ) -> Result<Vec<Row>, Error> { + match self { + Self::Pool(database) => database.query(sql, parameters).await, + Self::Transaction(transaction) => transaction.query(sql, parameters).await, + } + } +} + +pub fn initialize(settings: &Settings) -> Result<(), String> { + DATABASES + .set(configured_databases(settings)?) + .map_err(|_| "PostgreSQL databases were initialized more than once".to_owned()) +} + +pub(crate) fn configured_databases( + settings: &Settings, +) -> Result<BTreeMap<String, Arc<Database>>, String> { + let mut databases = BTreeMap::new(); + for (name, config) in settings.databases() { + if let DatabaseConfig::Postgres { + url, + tls, + min_connections, + max_connections, + max_page_size, + wait_timeout_ms, + io_timeout_ms, + .. + } = config + { + let database = Database::open(ConnectionOptions { + url, + database_name: None, + tls: *tls, + min_connections: *min_connections, + max_connections: *max_connections, + max_page_size: *max_page_size, + wait_timeout: Duration::from_millis(*wait_timeout_ms), + io_timeout: Duration::from_millis(*io_timeout_ms), + }) + .map_err(|error| format!("cannot initialize database.{name}: {error}"))?; + databases.insert(name.clone(), Arc::new(database)); + } + } + Ok(databases) +} + +pub async fn prepare() -> Result<(), Error> { + for database in DATABASES + .get() + .expect("PostgreSQL databases are initialized before prepare") + .values() + { + database.prepare().await?; + } + Ok(()) +} + +pub async fn shutdown() -> Result<(), Error> { + let databases = DATABASES + .get() + .expect("PostgreSQL databases are initialized before shutdown") + .values() + .cloned() + .collect::<Vec<_>>(); + for database in &databases { + database.stop_accepting(); + } + let mut first_error = None; + for database in databases { + if let Err(error) = database.drain().await + && first_error.is_none() + { + first_error = Some(error); + } + } + first_error.map_or(Ok(()), Err) +} + +pub fn database(name: &str) -> Result<Arc<Database>, Error> { + DATABASES + .get() + .and_then(|databases| databases.get(name)) + .cloned() + .ok_or_else(|| { + Error::new( + ErrorKind::Pool, + format!("PostgreSQL database '{name}' is not initialized"), + ) + }) +} + +fn build_pool<T>( + config: tokio_postgres::Config, + tls: T, + max_connections: usize, + wait_timeout: Duration, +) -> Result<Pool, Error> +where + T: tokio_postgres::tls::MakeTlsConnect<tokio_postgres::Socket> + Clone + Send + Sync + 'static, + T::Stream: Send + Sync, + T::TlsConnect: Send + Sync, + <T::TlsConnect as tokio_postgres::tls::TlsConnect<tokio_postgres::Socket>>::Future: Send, +{ + let manager = Manager::from_config( + config, + tls, + ManagerConfig { + recycling_method: RecyclingMethod::Fast, + }, + ); + let mut pool_config = PoolConfig::new(max_connections); + pool_config.timeouts = Timeouts { + wait: Some(wait_timeout), + create: Some(CREATE_TIMEOUT), + recycle: Some(RECYCLE_TIMEOUT), + }; + Pool::builder(manager) + .config(pool_config) + .runtime(Runtime::Tokio1) + .build() + .map_err(|error| Error::new(ErrorKind::Pool, error.to_string())) +} + +async fn execute_on( + connection: &deadpool_postgres::Object, + sql: &'static str, + parameters: Vec<Value>, +) -> Result<usize, Error> { + let statement = prepare_cached(connection, sql).await?; + let parameters = parameters.into_iter().map(Parameter).collect::<Vec<_>>(); + let references = parameters + .iter() + .map(|value| value as &(dyn ToSql + Sync)) + .collect::<Vec<_>>(); + connection + .execute(&statement, &references) + .await + .map(|affected| affected as usize) + .map_err(database_error) +} + +async fn query_on( + connection: &deadpool_postgres::Object, + sql: &'static str, + parameters: Vec<Value>, +) -> Result<Vec<Row>, Error> { + let statement = prepare_cached(connection, sql).await?; + let parameters = parameters.into_iter().map(Parameter).collect::<Vec<_>>(); + let references = parameters + .iter() + .map(|value| value as &(dyn ToSql + Sync)) + .collect::<Vec<_>>(); + connection + .query(&statement, &references) + .await + .map_err(database_error)? + .into_iter() + .map(decode_row) + .collect() +} + +async fn query_bounded_on( + connection: &deadpool_postgres::Object, + sql: &'static str, + parameters: Vec<Value>, + maximum: usize, +) -> Result<Vec<Row>, Error> { + let statement = prepare_cached(connection, sql).await?; + let parameters = parameters.into_iter().map(Parameter).collect::<Vec<_>>(); + let references = parameters + .iter() + .map(|value| value as &(dyn ToSql + Sync)) + .collect::<Vec<_>>(); + let rows = connection + .query_raw(&statement, references) + .await + .map_err(database_error)?; + tokio::pin!(rows); + let mut output = Vec::with_capacity(maximum.saturating_add(1)); + while output.len() <= maximum { + let Some(row) = rows.try_next().await.map_err(database_error)? else { + break; + }; + output.push(decode_row(row)?); + } + crate::orm::bounded_rows(output, maximum) +} + +async fn query_owned_on( + connection: &deadpool_postgres::Object, + sql: &str, + parameters: Vec<Value>, +) -> Result<Vec<Row>, Error> { + let statement = connection.prepare(sql).await.map_err(database_error)?; + let parameters = parameters.into_iter().map(Parameter).collect::<Vec<_>>(); + let references = parameters + .iter() + .map(|value| value as &(dyn ToSql + Sync)) + .collect::<Vec<_>>(); + connection + .query(&statement, &references) + .await + .map_err(database_error)? + .into_iter() + .map(decode_row) + .collect() +} + +async fn prepare_cached( + connection: &deadpool_postgres::Object, + sql: &'static str, +) -> Result<tokio_postgres::Statement, Error> { + let statement = connection + .prepare_cached(sql) + .await + .map_err(database_error)?; + if connection.statement_cache.size() > STATEMENT_CACHE_CAPACITY { + connection.statement_cache.remove(sql, &[]); + } + Ok(statement) +} + +fn decode_row(row: tokio_postgres::Row) -> Result<Row, Error> { + let mut values = Vec::with_capacity(row.len()); + for (index, column) in row.columns().iter().enumerate() { + let value = match *column.type_() { + Type::BOOL => optional(&row, index, |value: bool| Value::Bool(value))?, + Type::INT2 => optional(&row, index, |value: i16| Value::Int(i64::from(value)))?, + Type::INT4 => optional(&row, index, |value: i32| Value::Int(i64::from(value)))?, + Type::INT8 => optional(&row, index, Value::Int)?, + Type::FLOAT4 => optional(&row, index, |value: f32| Value::Float(f64::from(value)))?, + Type::FLOAT8 => optional(&row, index, Value::Float)?, + Type::BYTEA => optional(&row, index, Value::Bytes)?, + Type::NUMERIC => optional(&row, index, |value: NumericText| Value::Decimal(value.0))?, + Type::UUID => optional(&row, index, |value: UuidBytes| Value::Uuid(value.0))?, + Type::TEXT | Type::VARCHAR | Type::BPCHAR | Type::NAME => { + optional(&row, index, Value::Text)? + } + ref ty => { + return Err(Error::invalid_data(format!( + "unsupported PostgreSQL column type '{ty}'" + ))); + } + }; + values.push(value); + } + Ok(Row::new(values)) +} + +fn optional<T>( + row: &tokio_postgres::Row, + index: usize, + convert: impl FnOnce(T) -> Value, +) -> Result<Value, Error> +where + T: for<'a> tokio_postgres::types::FromSql<'a>, +{ + row.try_get::<_, Option<T>>(index) + .map(|value| value.map(convert).unwrap_or(Value::Null)) + .map_err(database_error) +} + +impl ToSql for Parameter { + fn to_sql( + &self, + ty: &Type, + output: &mut bytes::BytesMut, + ) -> Result<IsNull, Box<dyn std::error::Error + Send + Sync>> { + match &self.0 { + Value::Null => Ok(IsNull::Yes), + Value::Bool(value) => value.to_sql(ty, output), + Value::Int(value) => value.to_sql(ty, output), + Value::Float(value) => value.to_sql(ty, output), + Value::Decimal(value) => crate::orm::decimal_text(value)?.to_sql(ty, output), + Value::Text(value) => value.to_sql(ty, output), + Value::Bytes(value) => value.to_sql(ty, output), + Value::Uuid(value) => UuidBytes(*value).to_sql(ty, output), + } + } + + fn accepts(_: &Type) -> bool { + true + } + + to_sql_checked!(); +} + +impl<'a> FromSql<'a> for NumericText { + fn from_sql(_: &Type, raw: &'a [u8]) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> { + decode_numeric(raw) + .map(Self) + .map_err(|message| std::io::Error::new(std::io::ErrorKind::InvalidData, message).into()) + } + + fn accepts(ty: &Type) -> bool { + *ty == Type::NUMERIC + } +} + +impl<'a> FromSql<'a> for UuidBytes { + fn from_sql(_: &Type, raw: &'a [u8]) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> { + raw.try_into().map(Uuid::from_bytes).map(Self).map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + "PostgreSQL UUID is not 16 bytes", + ) + .into() + }) + } + + fn accepts(ty: &Type) -> bool { + *ty == Type::UUID + } +} + +impl ToSql for UuidBytes { + fn to_sql( + &self, + _: &Type, + output: &mut bytes::BytesMut, + ) -> Result<IsNull, Box<dyn std::error::Error + Send + Sync>> { + output.extend_from_slice(&self.0.into_bytes()); + Ok(IsNull::No) + } + + fn accepts(ty: &Type) -> bool { + *ty == Type::UUID + } + + to_sql_checked!(); +} + +fn decode_numeric(raw: &[u8]) -> Result<String, &'static str> { + if raw.len() < 8 || !(raw.len() - 8).is_multiple_of(2) { + return Err("invalid PostgreSQL NUMERIC payload"); + } + let read = |offset: usize| i16::from_be_bytes([raw[offset], raw[offset + 1]]); + let digit_count = read(0); + let weight = read(2); + let sign = u16::from_be_bytes([raw[4], raw[5]]); + let scale = read(6); + if digit_count < 0 + || scale < 0 + || raw.len() != 8 + usize::try_from(digit_count).unwrap_or(usize::MAX) * 2 + || !matches!(sign, 0x0000 | 0x4000) + { + return Err("unsupported PostgreSQL NUMERIC payload"); + } + let digits = raw[8..] + .as_chunks::<2>() + .0 + .iter() + .map(|digit| u16::from_be_bytes([digit[0], digit[1]])) + .collect::<Vec<_>>(); + if digits.iter().any(|digit| *digit >= 10_000) { + return Err("invalid PostgreSQL NUMERIC digit"); + } + let digit_at = |exponent: i16| { + let index = i32::from(weight) - i32::from(exponent); + usize::try_from(index) + .ok() + .and_then(|index| digits.get(index)) + .copied() + .unwrap_or(0) + }; + let mut text = String::new(); + if sign == 0x4000 && digits.iter().any(|digit| *digit != 0) { + text.push('-'); + } + if weight >= 0 { + for exponent in (0..=weight).rev() { + let digit = digit_at(exponent); + if exponent == weight { + text.push_str(&digit.to_string()); + } else { + use std::fmt::Write as _; + write!(text, "{digit:04}").expect("string formatting"); + } + } + } else { + text.push('0'); + } + let scale = usize::try_from(scale).expect("validated NUMERIC scale"); + if scale > 0 { + text.push('.'); + let groups = scale.div_ceil(4); + for group in 1..=groups { + use std::fmt::Write as _; + write!(text, "{:04}", digit_at(-(group as i16))).expect("string formatting"); + } + text.truncate(text.len() - (groups * 4 - scale)); + } + Ok(text) +} + +impl std::fmt::Debug for Parameter { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("Parameter(<value>)") + } +} + +fn database_error(error: tokio_postgres::Error) -> Error { + let kind = match error.code() { + Some( + &tokio_postgres::error::SqlState::UNIQUE_VIOLATION + | &tokio_postgres::error::SqlState::FOREIGN_KEY_VIOLATION + | &tokio_postgres::error::SqlState::CHECK_VIOLATION + | &tokio_postgres::error::SqlState::NOT_NULL_VIOLATION, + ) => ErrorKind::Constraint, + Some(code) if code == &tokio_postgres::error::SqlState::QUERY_CANCELED => { + ErrorKind::Cancelled + } + Some( + &tokio_postgres::error::SqlState::ADMIN_SHUTDOWN + | &tokio_postgres::error::SqlState::CRASH_SHUTDOWN + | &tokio_postgres::error::SqlState::CANNOT_CONNECT_NOW + | &tokio_postgres::error::SqlState::DATABASE_DROPPED, + ) => ErrorKind::Connection, + Some(code) if code.code().starts_with("08") => ErrorKind::Connection, + _ if error.is_closed() => ErrorKind::Connection, + _ => ErrorKind::Database, + }; + let mut result = Error::new(kind, error.to_string()); + if let Some(source) = error.source() { + result = result.cause(source); + } + result +} + +async fn io<T>( + timeout: Duration, + operation: impl Future<Output = Result<T, Error>>, +) -> Result<T, Error> { + tokio::time::timeout(timeout, operation) + .await + .map_err(|_| Error::new(ErrorKind::Timeout, "PostgreSQL I/O timed out"))? +} + +fn finish<T>( + connection: PoolObjectGuard<deadpool_postgres::Object>, + result: Result<T, Error>, +) -> Result<T, Error> { + if match &result { + Ok(_) => true, + Err(error) => error.keeps_connection(), + } { + drop(connection.release()); + } + result +} + +fn pool_error(error: deadpool_postgres::PoolError) -> Error { + let kind = match &error { + deadpool_postgres::PoolError::Timeout(deadpool_postgres::TimeoutType::Wait) => { + ErrorKind::PoolExhausted + } + deadpool_postgres::PoolError::Timeout(_) => ErrorKind::Timeout, + deadpool_postgres::PoolError::Backend(_) => ErrorKind::Connection, + deadpool_postgres::PoolError::Closed => ErrorKind::Connection, + _ => ErrorKind::Pool, + }; + Error::new(kind, error.to_string()) +} diff --git a/crates/dever-runtime/src/postgres/migration.rs b/crates/dever-runtime/src/postgres/migration.rs new file mode 100644 index 0000000..3b03df5 --- /dev/null +++ b/crates/dever-runtime/src/postgres/migration.rs @@ -0,0 +1,660 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use tokio_postgres::types::ToSql; + +use crate::database::{MigrationPhase, Model, PostgresConstraint, Schema}; +use crate::orm::Error; + +use super::{database_error, execute_on}; + +const CREATE_HISTORY: &str = "\ +CREATE TABLE IF NOT EXISTS _dever_history (\ + model TEXT NOT NULL,\ + kind TEXT NOT NULL,\ + name TEXT NOT NULL,\ + revision TEXT NOT NULL,\ + definition TEXT NOT NULL,\ + applied_at BIGINT NOT NULL DEFAULT ((extract(epoch from clock_timestamp()) * 1000)::bigint),\ + PRIMARY KEY (model, kind, name)\ +)"; + +pub(super) async fn apply( + connection: &deadpool_postgres::Object, + model: Model, +) -> Result<(), Error> { + connection + .batch_execute("BEGIN") + .await + .map_err(database_error)?; + let result = apply_locked(connection, &model).await; + match result { + Ok(()) => connection + .batch_execute("COMMIT") + .await + .map_err(database_error), + Err(error) => match connection.batch_execute("ROLLBACK").await { + Ok(()) => Err(error), + Err(rollback) => Err(error.cause(rollback)), + }, + } +} + +pub(super) async fn apply_foreign_keys( + connection: &deadpool_postgres::Object, + model: &str, + table: &str, + constraints: &[PostgresConstraint], +) -> Result<(), Error> { + connection + .batch_execute("BEGIN") + .await + .map_err(database_error)?; + let result = apply_foreign_keys_locked(connection, model, table, constraints).await; + match result { + Ok(()) => connection + .batch_execute("COMMIT") + .await + .map_err(database_error), + Err(error) => match connection.batch_execute("ROLLBACK").await { + Ok(()) => Err(error), + Err(rollback) => Err(error.cause(rollback)), + }, + } +} + +async fn apply_foreign_keys_locked( + connection: &deadpool_postgres::Object, + model: &str, + table: &str, + constraints: &[PostgresConstraint], +) -> Result<(), Error> { + connection + .query_one( + "SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", + &[&model], + ) + .await + .map_err(database_error)?; + for constraint in constraints { + let exists = connection + .query_one( + "SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_constraint con JOIN pg_catalog.pg_class rel ON rel.oid = con.conrelid JOIN pg_catalog.pg_namespace nsp ON nsp.oid = rel.relnamespace WHERE nsp.nspname = current_schema() AND rel.relname = $1 AND con.conname = $2 AND con.contype = 'f')", + &[&table, &constraint.name], + ) + .await + .map_err(database_error)? + .get::<_, bool>(0); + if !exists { + alter_table( + connection, + table, + &format!( + "ADD CONSTRAINT {} {}", + quote(&constraint.name), + constraint.definition + ), + ) + .await?; + } + } + store_catalog_history(connection, model, table).await +} + +async fn apply_locked(connection: &deadpool_postgres::Object, model: &Model) -> Result<(), Error> { + connection + .query_one( + "SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", + &[&model.schema.model], + ) + .await + .map_err(database_error)?; + connection + .batch_execute(CREATE_HISTORY) + .await + .map_err(database_error)?; + let table_exists = table_exists(connection, &model.schema.table).await?; + let previous = load_schema(connection, &model.schema.model).await?; + let mut pending = BTreeSet::new(); + match (table_exists, previous.as_ref()) { + (false, None) => { + connection + .batch_execute(&model.create_table.postgres) + .await + .map_err(database_error)?; + } + (true, Some(previous)) => { + validate_migrations(connection, &model.schema).await?; + crate::database::validate_migration_order(previous, &model.schema)?; + validate_catalog(connection, previous).await?; + let applied = load_migrations(connection, &model.schema.model).await?; + pending = model + .schema + .migrations + .iter() + .filter(|migration| !applied.contains_key(&migration.name)) + .map(|migration| migration.name.as_str()) + .collect(); + apply_data(connection, model, &pending, MigrationPhase::Before).await?; + if previous.revision != model.schema.revision { + migrate_schema(connection, previous, model).await?; + } + } + (true, None) => { + return Err(Error::migration(format!( + "table '{}' exists without Dever schema history", + model.schema.table + ))); + } + (false, Some(_)) => { + return Err(Error::migration(format!( + "Dever schema history exists but table '{}' is missing", + model.schema.table + ))); + } + } + for index in &model.create_indexes { + connection + .batch_execute(&index.postgres) + .await + .map_err(database_error)?; + } + apply_data(connection, model, &pending, MigrationPhase::After).await?; + if previous + .as_ref() + .is_none_or(|schema| schema.seed_revision != model.schema.seed_revision) + { + for seed in &model.seeds { + execute_on(connection, seed.sql.postgres, seed.parameters.clone()).await?; + } + } + store_history(connection, &model.schema).await +} + +async fn apply_data( + connection: &deadpool_postgres::Object, + model: &Model, + pending: &BTreeSet<&str>, + phase: MigrationPhase, +) -> Result<(), Error> { + for statement in &model.data_migrations { + if statement.phase == phase && pending.contains(statement.name.as_str()) { + execute_on( + connection, + statement.sql.postgres, + statement.parameters.clone(), + ) + .await?; + } + } + Ok(()) +} + +async fn migrate_schema( + connection: &deadpool_postgres::Object, + previous: &Schema, + model: &Model, +) -> Result<(), Error> { + let applied = load_migrations(connection, &model.schema.model).await?; + let allowed_drops = model + .schema + .migrations + .iter() + .filter(|migration| !applied.contains_key(&migration.name)) + .flat_map(|migration| migration.drops.iter().cloned()) + .collect::<BTreeSet<_>>(); + let previous_fields = previous + .fields + .iter() + .map(|field| (field.name.as_str(), field)) + .collect::<BTreeMap<_, _>>(); + let columns = model + .postgres_columns + .iter() + .map(|column| (column.name.as_str(), column)) + .collect::<BTreeMap<_, _>>(); + let mut mapped = Vec::new(); + let mut consumed = BTreeSet::new(); + for target in &model.schema.fields { + let source = previous_fields + .get(target.name.as_str()) + .copied() + .or_else(|| { + target + .rename_from + .as_deref() + .and_then(|name| previous_fields.get(name).copied()) + }); + if let Some(source) = source { + if !compatible_types(&source.ty, &target.ty) { + return Err(Error::migration(format!( + "field '{}.{}' cannot change type from '{}' to '{}' automatically", + model.schema.model, target.name, source.ty, target.ty + ))); + } + if !consumed.insert(source.name.as_str()) { + return Err(Error::migration(format!( + "field '{}.{}' reuses migration source '{}'", + model.schema.model, target.name, source.name + ))); + } + } + mapped.push((target, source)); + } + for source in &previous.fields { + if !consumed.contains(source.name.as_str()) && !allowed_drops.contains(&source.name) { + return Err(Error::migration(format!( + "field '{}.{}' was removed without a named migrate drop", + model.schema.model, source.name + ))); + } + } + + drop_dever_constraints(connection, &model.schema.table).await?; + for (target, source) in &mapped { + if let Some(source) = source + && source.name != target.name + { + alter_table( + connection, + &model.schema.table, + &format!( + "RENAME COLUMN {} TO {}", + quote(&source.name), + quote(&target.name) + ), + ) + .await?; + } + } + for source in &previous.fields { + if !consumed.contains(source.name.as_str()) { + alter_table( + connection, + &model.schema.table, + &format!("DROP COLUMN {}", quote(&source.name)), + ) + .await?; + } + } + for (target, source) in mapped { + let column = columns + .get(target.name.as_str()) + .expect("generated PostgreSQL column"); + let Some(source) = source else { + alter_table( + connection, + &model.schema.table, + &format!("ADD COLUMN {}", column.definition), + ) + .await?; + continue; + }; + let previous_type = postgres_type(&source.ty); + if previous_type != column.sql_type { + alter_table( + connection, + &model.schema.table, + &format!( + "ALTER COLUMN {} TYPE {} USING {}::{}", + quote(&target.name), + column.sql_type, + quote(&target.name), + column.sql_type, + ), + ) + .await?; + } + if source.nullable != target.nullable { + alter_table( + connection, + &model.schema.table, + &format!( + "ALTER COLUMN {} {} NOT NULL", + quote(&target.name), + if target.nullable { "DROP" } else { "SET" }, + ), + ) + .await?; + } + if source.default != target.default || source.generated != target.generated { + let action = column + .default + .as_ref() + .map(|default| format!("SET DEFAULT {default}")) + .unwrap_or_else(|| "DROP DEFAULT".into()); + alter_table( + connection, + &model.schema.table, + &format!("ALTER COLUMN {} {action}", quote(&target.name)), + ) + .await?; + } + } + for constraint in model + .postgres_constraints + .iter() + .filter(|constraint| !constraint.foreign_key) + { + alter_table( + connection, + &model.schema.table, + &format!( + "ADD CONSTRAINT {} {}", + quote(&constraint.name), + constraint.definition + ), + ) + .await?; + } + for index in &previous.indexes { + connection + .batch_execute(&format!("DROP INDEX IF EXISTS {}", quote(&index.name))) + .await + .map_err(database_error)?; + } + Ok(()) +} + +async fn drop_dever_constraints( + connection: &deadpool_postgres::Object, + table: &str, +) -> Result<(), Error> { + let constraints = connection + .query( + "SELECT con.conname FROM pg_catalog.pg_constraint con JOIN pg_catalog.pg_class rel ON rel.oid = con.conrelid JOIN pg_catalog.pg_namespace nsp ON nsp.oid = rel.relnamespace WHERE nsp.nspname = current_schema() AND rel.relname = $1 AND left(con.conname, 7) = '_dever_'", + &[&table], + ) + .await + .map_err(database_error)?; + for row in constraints { + alter_table( + connection, + table, + &format!("DROP CONSTRAINT {}", quote(row.get::<_, &str>(0))), + ) + .await?; + } + Ok(()) +} + +async fn alter_table( + connection: &deadpool_postgres::Object, + table: &str, + operation: &str, +) -> Result<(), Error> { + connection + .batch_execute(&format!("ALTER TABLE {} {operation}", quote(table))) + .await + .map_err(database_error) +} + +fn compatible_types(previous: &str, desired: &str) -> bool { + previous == desired + || (previous.starts_with("Text") && desired.starts_with("Text")) + || (previous.starts_with("Bytes") && desired.starts_with("Bytes")) + || (previous == "Int" && desired == "Float") +} + +fn postgres_type(logical: &str) -> String { + if logical == "Bool" { + "BOOLEAN".into() + } else if logical == "Float" { + "DOUBLE PRECISION".into() + } else if logical.starts_with("Bytes") { + "BYTEA".into() + } else if let Some(decimal) = logical + .strip_prefix("Decimal(") + .and_then(|value| value.strip_suffix(')')) + { + format!("NUMERIC({decimal})") + } else if logical == "Uuid" { + "UUID".into() + } else if logical == "Int" + || logical == "DateTime" + || logical == "Date" + || logical == "Time" + || logical == "Duration" + || logical.starts_with("id:") + { + "BIGINT".into() + } else { + "TEXT".into() + } +} + +fn quote(identifier: &str) -> String { + format!("\"{}\"", identifier.replace('"', "\"\"")) +} + +async fn table_exists(connection: &deadpool_postgres::Object, table: &str) -> Result<bool, Error> { + connection + .query_one( + "SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = current_schema() AND c.relname = $1 AND c.relkind = 'r')", + &[&table], + ) + .await + .map(|row| row.get(0)) + .map_err(database_error) +} + +async fn load_schema( + connection: &deadpool_postgres::Object, + model: &str, +) -> Result<Option<Schema>, Error> { + let row = connection + .query_opt( + "SELECT definition FROM _dever_history WHERE model = $1 AND kind = 'schema' AND name = 'current'", + &[&model], + ) + .await + .map_err(database_error)?; + row.map(|row| { + serde_json::from_str(row.get(0)) + .map_err(|error| Error::migration(format!("invalid Dever schema history: {error}"))) + }) + .transpose() +} + +async fn validate_catalog( + connection: &deadpool_postgres::Object, + schema: &Schema, +) -> Result<(), Error> { + let recorded = load_catalog(connection, &schema.model) + .await? + .ok_or_else(|| { + Error::migration(format!( + "Model '{}' has no recorded database catalog", + schema.model + )) + })?; + if catalog(connection, &schema.table).await? != recorded { + return Err(Error::migration(format!( + "table '{}' does not match its recorded Dever schema", + schema.table + ))); + } + Ok(()) +} + +async fn load_catalog( + connection: &deadpool_postgres::Object, + model: &str, +) -> Result<Option<String>, Error> { + connection + .query_opt( + "SELECT definition FROM _dever_history WHERE model = $1 AND kind = 'catalog' AND name = 'current'", + &[&model], + ) + .await + .map(|row| row.map(|row| row.get(0))) + .map_err(database_error) +} + +async fn catalog(connection: &deadpool_postgres::Object, table: &str) -> Result<String, Error> { + let columns = connection + .query( + "SELECT attribute.attname, pg_catalog.format_type(attribute.atttypid, attribute.atttypmod), attribute.attnotnull, attribute.attidentity::text, pg_catalog.pg_get_expr(default_value.adbin, default_value.adrelid) FROM pg_catalog.pg_attribute attribute JOIN pg_catalog.pg_class relation ON relation.oid = attribute.attrelid JOIN pg_catalog.pg_namespace namespace ON namespace.oid = relation.relnamespace LEFT JOIN pg_catalog.pg_attrdef default_value ON default_value.adrelid = relation.oid AND default_value.adnum = attribute.attnum WHERE namespace.nspname = current_schema() AND relation.relname = $1 AND attribute.attnum > 0 AND NOT attribute.attisdropped ORDER BY attribute.attnum", + &[&table], + ) + .await + .map_err(database_error)? + .into_iter() + .map(|row| { + ( + row.get::<_, String>(0), + row.get::<_, String>(1), + row.get::<_, bool>(2), + row.get::<_, String>(3), + row.get::<_, Option<String>>(4), + ) + }) + .collect::<Vec<_>>(); + let constraints = connection + .query( + "SELECT constraint_.conname, constraint_.contype::text, pg_catalog.pg_get_constraintdef(constraint_.oid, false) FROM pg_catalog.pg_constraint constraint_ JOIN pg_catalog.pg_class relation ON relation.oid = constraint_.conrelid JOIN pg_catalog.pg_namespace namespace ON namespace.oid = relation.relnamespace WHERE namespace.nspname = current_schema() AND relation.relname = $1 ORDER BY constraint_.conname", + &[&table], + ) + .await + .map_err(database_error)? + .into_iter() + .map(|row| { + ( + row.get::<_, String>(0), + row.get::<_, String>(1), + row.get::<_, String>(2), + ) + }) + .collect::<Vec<_>>(); + let indexes = connection + .query( + "SELECT index_relation.relname, pg_catalog.pg_get_indexdef(index_relation.oid) FROM pg_catalog.pg_index index_ JOIN pg_catalog.pg_class table_relation ON table_relation.oid = index_.indrelid JOIN pg_catalog.pg_class index_relation ON index_relation.oid = index_.indexrelid JOIN pg_catalog.pg_namespace namespace ON namespace.oid = table_relation.relnamespace WHERE namespace.nspname = current_schema() AND table_relation.relname = $1 AND left(index_relation.relname, 11) = '_dever_idx_' ORDER BY index_relation.relname", + &[&table], + ) + .await + .map_err(database_error)? + .into_iter() + .map(|row| (row.get::<_, String>(0), row.get::<_, String>(1))) + .collect::<Vec<_>>(); + serde_json::to_string(&(columns, constraints, indexes)) + .map_err(|error| Error::migration(format!("cannot encode PostgreSQL catalog: {error}"))) +} + +async fn validate_migrations( + connection: &deadpool_postgres::Object, + schema: &Schema, +) -> Result<(), Error> { + let applied = load_migrations(connection, &schema.model).await?; + let desired = schema + .migrations + .iter() + .map(|migration| (migration.name.as_str(), migration)) + .collect::<BTreeMap<_, _>>(); + for (name, revision) in applied { + let Some(migration) = desired.get(name.as_str()) else { + return Err(Error::migration(format!( + "applied migration '{}.{}' was removed from source", + schema.model, name + ))); + }; + if migration.revision != revision { + return Err(Error::migration(format!( + "applied migration '{}.{}' was modified", + schema.model, name + ))); + } + } + Ok(()) +} + +async fn load_migrations( + connection: &deadpool_postgres::Object, + model: &str, +) -> Result<BTreeMap<String, String>, Error> { + connection + .query( + "SELECT name, revision FROM _dever_history WHERE model = $1 AND kind = 'migration'", + &[&model], + ) + .await + .map_err(database_error) + .map(|rows| { + rows.into_iter() + .map(|row| (row.get(0), row.get(1))) + .collect() + }) +} + +async fn store_history( + connection: &deadpool_postgres::Object, + schema: &Schema, +) -> Result<(), Error> { + let definition = serde_json::to_string(schema).map_err(|error| { + Error::migration(format!("cannot encode Dever schema history: {error}")) + })?; + store_history_row( + connection, + &schema.model, + "schema", + "current", + &schema.revision, + &definition, + ) + .await?; + for migration in &schema.migrations { + store_history_row( + connection, + &schema.model, + "migration", + &migration.name, + &migration.revision, + "", + ) + .await?; + } + store_history_row( + connection, + &schema.model, + "seed", + "current", + &schema.seed_revision, + "", + ) + .await?; + store_catalog_history(connection, &schema.model, &schema.table).await +} + +async fn store_catalog_history( + connection: &deadpool_postgres::Object, + model: &str, + table: &str, +) -> Result<(), Error> { + store_history_row( + connection, + model, + "catalog", + "current", + "actual", + &catalog(connection, table).await?, + ) + .await +} + +async fn store_history_row( + connection: &deadpool_postgres::Object, + model: &str, + kind: &str, + name: &str, + revision: &str, + definition: &str, +) -> Result<(), Error> { + let parameters: [&(dyn ToSql + Sync); 5] = [&model, &kind, &name, &revision, &definition]; + connection + .execute( + "INSERT INTO _dever_history (model, kind, name, revision, definition) VALUES ($1, $2, $3, $4, $5) \ + ON CONFLICT (model, kind, name) DO UPDATE SET revision = EXCLUDED.revision, definition = EXCLUDED.definition, applied_at = (extract(epoch from clock_timestamp()) * 1000)::bigint WHERE _dever_history.kind <> 'migration'", + ¶meters, + ) + .await + .map(|_| ()) + .map_err(database_error) +} diff --git a/crates/dever-runtime/src/process.rs b/crates/dever-runtime/src/process.rs new file mode 100644 index 0000000..38804e2 --- /dev/null +++ b/crates/dever-runtime/src/process.rs @@ -0,0 +1,14 @@ +use crate::collections::List; + +pub fn arguments() -> Result<List<String>, String> { + // argv[0] 是宿主可执行文件路径，不属于应用参数。 + std::env::args_os() + .skip(1) + .map(|argument| { + argument + .into_string() + .map_err(|_| "process argument is not valid UTF-8".into()) + }) + .collect::<Result<Vec<_>, _>>() + .map(List::new) +} diff --git a/crates/dever-runtime/src/render.rs b/crates/dever-runtime/src/render.rs new file mode 100644 index 0000000..525373c --- /dev/null +++ b/crates/dever-runtime/src/render.rs @@ -0,0 +1,165 @@ +use std::fmt::{Display, Write}; + +use crate::Id; +use crate::bytes::Bytes; +use crate::collections::{List, Map, MapEntry}; +use crate::number::DecimalValue; +use crate::resource::{Resource, Stream}; + +pub trait Render { + fn render_to(&self, output: &mut String); + + fn render(&self) -> String { + let mut output = String::new(); + self.render_to(&mut output); + output + } +} + +fn write_display(output: &mut String, value: impl Display) { + write!(output, "{value}").expect("writing to a String cannot fail"); +} + +impl Render for String { + fn render_to(&self, output: &mut String) { + output.push_str(self); + } +} +impl Render for bool { + fn render_to(&self, output: &mut String) { + write_display(output, self); + } +} +impl Render for i64 { + fn render_to(&self, output: &mut String) { + write_display(output, self); + } +} +impl Render for f64 { + fn render_to(&self, output: &mut String) { + write_display(output, self); + } +} +impl Render for DecimalValue { + fn render_to(&self, output: &mut String) { + write_display(output, self); + } +} + +impl Render for crate::orm::Uuid { + fn render_to(&self, output: &mut String) { + use std::fmt::Write as _; + write!(output, "{self}").expect("string formatting"); + } +} +impl Render for Id { + fn render_to(&self, output: &mut String) { + output.push_str(&self.0); + } +} +impl Render for Bytes { + fn render_to(&self, output: &mut String) { + write!(output, "{:?}", self.values()).expect("writing to a String cannot fail"); + } +} +impl<T> Render for Resource<T> { + fn render_to(&self, output: &mut String) { + output.push_str("<resource>"); + } +} +impl<T> Render for Stream<T> { + fn render_to(&self, output: &mut String) { + output.push_str("<stream>"); + } +} +impl<T> Render for crate::async_stream::AsyncStream<T> { + fn render_to(&self, output: &mut String) { + output.push_str("<async-stream>"); + } +} +impl Render for crate::net::Socket { + fn render_to(&self, output: &mut String) { + output.push_str("<resource>"); + } +} +impl Render for crate::net::Listener { + fn render_to(&self, output: &mut String) { + output.push_str("<resource>"); + } +} + +impl Render for crate::http::HttpReply { + fn render_to(&self, output: &mut String) { + output.push_str("HttpReply(<resource>)"); + } +} +impl Render for crate::websocket::WebSocket { + fn render_to(&self, output: &mut String) { + output.push_str("WebSocket(<resource>)"); + } +} + +impl Render for crate::tls::ClientTls { + fn render_to(&self, output: &mut String) { + output.push_str("ClientTls(<resource>)"); + } +} +impl Render for crate::tls::ServerTls { + fn render_to(&self, output: &mut String) { + output.push_str("ServerTls(<resource>)"); + } +} +impl Render for crate::http::HttpClient { + fn render_to(&self, output: &mut String) { + output.push_str("HttpClient(<resource>)"); + } +} + +impl<T> Render for crate::channel::Channel<T> { + fn render_to(&self, output: &mut String) { + output.push_str("Channel(<resource>)"); + } +} +impl<T: Render> Render for Option<T> { + fn render_to(&self, output: &mut String) { + match self { + Some(value) => value.render_to(output), + None => output.push_str("null"), + } + } +} +impl<T: Render> Render for List<T> { + fn render_to(&self, output: &mut String) { + output.push('['); + for (index, value) in self.values().iter().enumerate() { + if index != 0 { + output.push_str(", "); + } + value.render_to(output); + } + output.push(']'); + } +} +impl<K: Render, V: Render> Render for Map<K, V> { + fn render_to(&self, output: &mut String) { + output.push('{'); + for (index, (key, value)) in self.pairs().enumerate() { + if index != 0 { + output.push_str(", "); + } + key.render_to(output); + output.push_str(" = "); + value.render_to(output); + } + output.push('}'); + } +} +impl<K: Render, V: Render> Render for MapEntry<K, V> { + fn render_to(&self, output: &mut String) { + output.push_str("{key = "); + self.key.render_to(output); + output.push_str(", value = "); + self.value.render_to(output); + output.push('}'); + } +} diff --git a/crates/dever-runtime/src/resource.rs b/crates/dever-runtime/src/resource.rs new file mode 100644 index 0000000..b076a2a --- /dev/null +++ b/crates/dever-runtime/src/resource.rs @@ -0,0 +1,178 @@ +use std::fs::{File, OpenOptions}; +use std::io::{Read, Write}; +use std::sync::{Arc, Mutex}; + +use crate::bytes::Bytes; + +pub enum Pull<T> { + Item(T), + Last(T), + End, +} + +struct StreamState<T> { + producer: Option<Box<dyn FnMut() -> Pull<T>>>, +} + +pub struct Stream<T>(Arc<Mutex<StreamState<T>>>); + +impl<T> Clone for Stream<T> { + fn clone(&self) -> Self { + Self(Arc::clone(&self.0)) + } +} + +impl<T> std::fmt::Debug for Stream<T> { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("Stream(<resource>)") + } +} + +impl<T: 'static> Stream<T> { + pub fn new(producer: impl FnMut() -> Pull<T> + 'static) -> Self { + Self(Arc::new(Mutex::new(StreamState { + producer: Some(Box::new(producer)), + }))) + } + + fn pull_next(&self) -> Pull<T> { + let mut state = self.0.lock().expect("stream lock poisoned"); + let Some(producer) = state.producer.as_mut() else { + return Pull::End; + }; + let result = producer.as_mut()(); + if matches!(&result, Pull::Last(_) | Pull::End) { + state.producer = None; + } + result + } + + pub fn pull(&self) -> Option<T> { + match self.pull_next() { + Pull::Item(value) | Pull::Last(value) => Some(value), + Pull::End => None, + } + } + + pub fn close(&self) { + self.0.lock().expect("stream lock poisoned").producer = None; + } + + pub fn map<U: 'static>(self, mut transform: impl FnMut(T) -> U + 'static) -> Stream<U> { + Stream::new(move || match self.pull_next() { + Pull::Item(value) => Pull::Item(transform(value)), + Pull::Last(value) => Pull::Last(transform(value)), + Pull::End => Pull::End, + }) + } +} + +#[derive(Debug)] +pub struct Resource<T>(Arc<Mutex<Option<T>>>); + +impl<T> Clone for Resource<T> { + fn clone(&self) -> Self { + Self(Arc::clone(&self.0)) + } +} + +impl<T> Resource<T> { + pub fn new(value: T) -> Self { + Self(Arc::new(Mutex::new(Some(value)))) + } + + pub fn close(&self) -> Result<(), String> { + self.0 + .lock() + .map_err(|_| "resource lock failed".to_owned())? + .take() + .map(|_| ()) + .ok_or_else(|| "resource is closed".to_owned()) + } + + pub fn is_open(&self) -> Result<bool, String> { + self.0 + .lock() + .map(|resource| resource.is_some()) + .map_err(|_| "resource lock failed".to_owned()) + } + + fn with<R>(&self, operation: impl FnOnce(&mut T) -> std::io::Result<R>) -> Result<R, String> { + let mut guard = self + .0 + .lock() + .map_err(|_| "resource lock failed".to_owned())?; + let resource = guard + .as_mut() + .ok_or_else(|| "resource is closed".to_owned())?; + operation(resource).map_err(|error| error.to_string()) + } +} + +pub type FileHandle = Resource<File>; + +pub fn file_open(path: &str) -> Result<FileHandle, String> { + File::open(path) + .map(Resource::new) + .map_err(|error| error.to_string()) +} + +pub fn file_create(path: &str) -> Result<FileHandle, String> { + OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .map(Resource::new) + .map_err(|error| error.to_string()) +} + +pub fn read<T: Read>(resource: &Resource<T>, limit: i64) -> Result<Option<Bytes>, String> { + let limit = read_limit(limit)?; + resource.with(|source| { + let mut bytes = Vec::new(); + bytes + .try_reserve_exact(limit) + .map_err(std::io::Error::other)?; + bytes.resize(limit, 0); + let length = source.read(&mut bytes)?; + bytes.truncate(length); + Ok((length != 0).then(|| Bytes::new(bytes))) + }) +} + +pub fn read_chunks<T: Read + 'static>( + resource: Resource<T>, + limit: i64, +) -> Result<Stream<Result<Bytes, String>>, String> { + read_limit(limit)?; + if !resource.is_open()? { + return Err("resource is closed".to_owned()); + } + Ok(Stream::new(move || match read(&resource, limit) { + Ok(Some(bytes)) => Pull::Item(Ok(bytes)), + Ok(None) => Pull::End, + Err(message) => Pull::Last(Err(message)), + })) +} + +pub(crate) fn read_limit(limit: i64) -> Result<usize, String> { + usize::try_from(limit) + .ok() + .filter(|limit| *limit > 0) + .ok_or_else(|| "read size must be a positive Int".to_owned()) +} + +pub fn write<T: Write>(resource: &Resource<T>, bytes: &Bytes) -> Result<(), String> { + resource.with(|output| { + output.write_all(bytes.values())?; + output.flush() + }) +} + +pub fn stdout_write(text: &str) -> Result<(), String> { + let mut output = std::io::stdout().lock(); + output + .write_all(text.as_bytes()) + .and_then(|_| output.flush()) + .map_err(|error| format!("standard output failed: {error}")) +} diff --git a/crates/dever-runtime/src/secret.rs b/crates/dever-runtime/src/secret.rs new file mode 100644 index 0000000..134c4d5 --- /dev/null +++ b/crates/dever-runtime/src/secret.rs @@ -0,0 +1,21 @@ +use std::fmt; +use zeroize::Zeroizing; + +/// Opaque source value. Only approved runtime sinks may inspect its bytes. +#[derive(Clone)] +// The payload is consumed only when an approved crypto sink is linked. +#[cfg_attr(not(feature = "crypto"), allow(dead_code))] +pub struct Secret(pub(crate) Zeroizing<Vec<u8>>); + +impl Secret { + /// Compiler-owned input decoding and test construction; no source-level reveal. + pub fn from_input(bytes: Vec<u8>) -> Self { + Self(Zeroizing::new(bytes)) + } +} + +impl fmt::Debug for Secret { + fn fmt(&self, output: &mut fmt::Formatter<'_>) -> fmt::Result { + output.write_str("Secret([REDACTED])") + } +} diff --git a/crates/dever-runtime/src/sqlite.rs b/crates/dever-runtime/src/sqlite.rs new file mode 100644 index 0000000..9b1dc99 --- /dev/null +++ b/crates/dever-runtime/src/sqlite.rs @@ -0,0 +1,578 @@ +use std::collections::BTreeMap; +use std::sync::{Arc, OnceLock}; + +use deadpool_sqlite::rusqlite::types::{Value as SqliteValue, ValueRef}; +use deadpool_sqlite::{Config, Hook, HookError, Pool, PoolConfig, Runtime, Timeouts}; +use tokio::sync::OnceCell; + +use crate::config::{Database as DatabaseConfig, Settings}; +use crate::orm::{Error, ErrorKind, Row, RowStream, Value}; +use crate::pool::{CREATE_TIMEOUT, PoolObjectGuard, RECYCLE_TIMEOUT, WAIT_TIMEOUT}; + +mod migration; + +const CONNECTION_SETUP: &str = "\ +PRAGMA foreign_keys = ON;\n\ +PRAGMA synchronous = NORMAL;\n\ +PRAGMA busy_timeout = 5000;"; + +static DATABASES: OnceLock<BTreeMap<String, Arc<Database>>> = OnceLock::new(); + +pub struct Database { + pool: Pool, + max_page_size: usize, + initialized: OnceCell<()>, +} + +pub struct Transaction { + connection: Option<deadpool_sqlite::Object>, + max_page_size: usize, +} + +pub enum Executor<'a> { + Pool(&'a Database), + Transaction(&'a Transaction), +} + +impl Database { + pub fn open( + path: impl Into<std::path::PathBuf>, + max_connections: usize, + max_page_size: usize, + ) -> Result<Self, Error> { + let config = Config::new(path); + let manager = deadpool_sqlite::Manager::from_config(&config, Runtime::Tokio1); + let mut pool_config = PoolConfig::new(max_connections); + pool_config.timeouts = Timeouts { + wait: Some(WAIT_TIMEOUT), + create: Some(CREATE_TIMEOUT), + recycle: Some(RECYCLE_TIMEOUT), + }; + let pool = Pool::builder(manager) + .config(pool_config) + .runtime(Runtime::Tokio1) + .post_create(Hook::async_fn(|connection, _| { + Box::pin(async move { + connection + .interact(|connection| connection.execute_batch(CONNECTION_SETUP)) + .await + .map_err(|error| HookError::message(error.to_string()))? + .map_err(HookError::Backend) + }) + })) + .build() + .map_err(|error| Error::new(ErrorKind::Pool, error.to_string()))?; + Ok(Self { + pool, + max_page_size, + initialized: OnceCell::new(), + }) + } + + pub fn max_page_size(&self) -> usize { + self.max_page_size + } + + pub(crate) fn close(&self) { + self.pool.close(); + } + + pub async fn execute(&self, sql: &'static str, parameters: Vec<Value>) -> Result<usize, Error> { + self.prepare().await?; + let connection = guarded(self.connection().await?); + let result = execute_on(connection.object(), sql, parameters).await; + drop(connection.release()); + result + } + + pub async fn query( + &self, + sql: &'static str, + parameters: Vec<Value>, + ) -> Result<Vec<Row>, Error> { + self.prepare().await?; + let connection = guarded(self.connection().await?); + let result = query_on(connection.object(), sql, parameters).await; + drop(connection.release()); + result + } + + pub async fn query_bounded( + &self, + sql: &'static str, + parameters: Vec<Value>, + maximum: usize, + ) -> Result<Vec<Row>, Error> { + self.prepare().await?; + let connection = guarded(self.connection().await?); + let result = query_bounded_on(connection.object(), sql, parameters, maximum).await; + drop(connection.release()); + result + } + + pub async fn query_owned( + &self, + sql: String, + parameters: Vec<Value>, + ) -> Result<Vec<Row>, Error> { + self.prepare().await?; + let connection = guarded(self.connection().await?); + let result = query_owned_on(connection.object(), sql, parameters).await; + drop(connection.release()); + result + } + + pub async fn stream( + &self, + sql: &'static str, + parameters: Vec<Value>, + capacity: usize, + ) -> Result<RowStream<Row>, Error> { + self.prepare().await?; + if capacity == 0 || capacity > self.max_page_size { + return Err(Error::invalid_data(format!( + "row stream capacity must be between 1 and {}", + self.max_page_size + ))); + } + let connection = self.connection().await?; + let (rows, receiver) = tokio::sync::mpsc::channel(capacity); + let (ready, initialized) = tokio::sync::oneshot::channel(); + let (worker, spawner, worker_fault) = + crate::task::scoped_owner(()).map_err(|error| Error::new(ErrorKind::Pool, error))?; + spawner.spawn(async move { + let connection = guarded(connection); + let result = connection + .object() + .interact(move |connection| stream_rows(connection, sql, parameters, rows, ready)) + .await; + if matches!(result, Ok(true)) { + drop(connection.release()); + } + }); + initialized + .await + .map_err(|_| Error::new(ErrorKind::Pool, "SQLite row stream worker stopped"))??; + Ok(RowStream::from_receiver(receiver, worker, worker_fault)) + } + + pub async fn execute_batch(&self, sql: String) -> Result<(), Error> { + self.prepare().await?; + let connection = guarded(self.connection().await?); + let result = connection + .object() + .interact(move |connection| connection.execute_batch(&sql)) + .await + .map_err(interact_error)? + .map_err(database_error); + drop(connection.release()); + result + } + + pub async fn begin(&self) -> Result<Transaction, Error> { + self.begin_with("BEGIN IMMEDIATE").await + } + + async fn begin_with(&self, statement: &'static str) -> Result<Transaction, Error> { + self.prepare().await?; + let connection = guarded(self.connection().await?); + batch_on(connection.object(), statement).await?; + Ok(Transaction { + connection: Some(connection.release()), + max_page_size: self.max_page_size, + }) + } + + pub async fn migrate(&self, model: crate::database::Model) -> Result<(), Error> { + self.prepare().await?; + let connection = guarded(self.connection().await?); + let result = migration::apply(connection.object(), model).await; + if result.is_ok() { + drop(connection.release()); + } + result + } + + async fn prepare(&self) -> Result<(), Error> { + self.initialized + .get_or_try_init(|| async { + let connection = guarded(self.connection().await?); + let result = connection + .object() + .interact(|connection| connection.pragma_update(None, "journal_mode", "WAL")) + .await + .map_err(interact_error)? + .map_err(database_error); + if result.is_ok() { + drop(connection.release()); + } + result + }) + .await + .copied() + } + + async fn connection(&self) -> Result<deadpool_sqlite::Object, Error> { + self.pool + .get() + .await + .map_err(|error| Error::new(ErrorKind::Pool, error.to_string())) + } +} + +impl Transaction { + pub async fn execute(&self, sql: &'static str, parameters: Vec<Value>) -> Result<usize, Error> { + execute_on(self.connection(), sql, parameters).await + } + + pub async fn query( + &self, + sql: &'static str, + parameters: Vec<Value>, + ) -> Result<Vec<Row>, Error> { + query_on(self.connection(), sql, parameters).await + } + + pub async fn query_bounded( + &self, + sql: &'static str, + parameters: Vec<Value>, + maximum: usize, + ) -> Result<Vec<Row>, Error> { + query_bounded_on(self.connection(), sql, parameters, maximum).await + } + + pub async fn query_owned( + &self, + sql: String, + parameters: Vec<Value>, + ) -> Result<Vec<Row>, Error> { + query_owned_on(self.connection(), sql, parameters).await + } + + pub async fn commit(mut self) -> Result<(), Error> { + self.finish("COMMIT").await + } + + pub async fn rollback(mut self) -> Result<(), Error> { + self.finish("ROLLBACK").await + } + + fn connection(&self) -> &deadpool_sqlite::Object { + self.connection.as_ref().expect("open SQLite transaction") + } + + async fn finish(&mut self, sql: &'static str) -> Result<(), Error> { + let connection = guarded(self.connection.take().expect("open SQLite transaction")); + let result = batch_on(connection.object(), sql).await; + if result.is_ok() { + drop(connection.release()); + } + result + } +} + +impl Drop for Transaction { + fn drop(&mut self) { + if let Some(connection) = self.connection.take() { + drop(deadpool_sqlite::Object::take(connection)); + } + } +} + +fn guarded(connection: deadpool_sqlite::Object) -> PoolObjectGuard<deadpool_sqlite::Object> { + PoolObjectGuard::new(connection, detach) +} + +fn detach(connection: deadpool_sqlite::Object) { + drop(deadpool_sqlite::Object::take(connection)); +} + +impl Executor<'_> { + pub fn max_page_size(&self) -> usize { + match self { + Self::Pool(database) => database.max_page_size(), + Self::Transaction(transaction) => transaction.max_page_size, + } + } + + pub async fn execute(&self, sql: &'static str, parameters: Vec<Value>) -> Result<usize, Error> { + match self { + Self::Pool(database) => database.execute(sql, parameters).await, + Self::Transaction(transaction) => transaction.execute(sql, parameters).await, + } + } + + pub async fn query( + &self, + sql: &'static str, + parameters: Vec<Value>, + ) -> Result<Vec<Row>, Error> { + match self { + Self::Pool(database) => database.query(sql, parameters).await, + Self::Transaction(transaction) => transaction.query(sql, parameters).await, + } + } +} + +pub fn initialize(settings: &Settings) -> Result<(), String> { + DATABASES + .set(configured_databases(settings)?) + .map_err(|_| "SQLite databases were initialized more than once".to_owned()) +} + +pub(crate) fn configured_databases( + settings: &Settings, +) -> Result<BTreeMap<String, Arc<Database>>, String> { + let mut databases = BTreeMap::new(); + for (name, config) in settings.databases() { + if let DatabaseConfig::Sqlite { + path, + max_connections, + max_page_size, + .. + } = config + { + let database = Database::open(path, *max_connections, *max_page_size) + .map_err(|error| format!("cannot initialize database.{name}: {error}"))?; + databases.insert(name.clone(), Arc::new(database)); + } + } + Ok(databases) +} + +pub fn database(name: &str) -> Result<Arc<Database>, Error> { + DATABASES + .get() + .and_then(|databases| databases.get(name)) + .cloned() + .ok_or_else(|| { + Error::new( + ErrorKind::Pool, + format!("SQLite database '{name}' is not initialized"), + ) + }) +} + +pub fn shutdown() { + if let Some(databases) = DATABASES.get() { + for database in databases.values() { + database.close(); + } + } +} + +pub fn database_for(explicit: Option<&str>, package_root: &str) -> Result<Arc<Database>, Error> { + let (name, config) = crate::config::settings() + .resolve_database(explicit, package_root) + .map_err(|error| Error::new(ErrorKind::Pool, error))?; + if !matches!(config, DatabaseConfig::Sqlite { .. }) { + return Err(Error::new( + ErrorKind::Pool, + format!("database connection '{name}' is not SQLite"), + )); + } + database(name) +} + +fn sqlite_values(parameters: Vec<Value>) -> Vec<SqliteValue> { + parameters + .into_iter() + .map(|value| match value { + Value::Null => SqliteValue::Null, + Value::Bool(value) => SqliteValue::Integer(i64::from(value)), + Value::Int(value) => SqliteValue::Integer(value), + Value::Float(value) => SqliteValue::Real(value), + Value::Decimal(value) => SqliteValue::Text(value), + Value::Text(value) => SqliteValue::Text(value), + Value::Bytes(value) => SqliteValue::Blob(value), + Value::Uuid(value) => SqliteValue::Blob(value.into_bytes().to_vec()), + }) + .collect() +} + +async fn execute_on( + connection: &deadpool_sqlite::Object, + sql: &'static str, + parameters: Vec<Value>, +) -> Result<usize, Error> { + connection + .interact(move |connection| { + let parameters = sqlite_values(parameters); + connection.execute(sql, deadpool_sqlite::rusqlite::params_from_iter(parameters)) + }) + .await + .map_err(interact_error)? + .map_err(database_error) +} + +async fn query_on( + connection: &deadpool_sqlite::Object, + sql: &'static str, + parameters: Vec<Value>, +) -> Result<Vec<Row>, Error> { + connection + .interact(move |connection| { + let parameters = sqlite_values(parameters); + let mut statement = connection.prepare_cached(sql)?; + let column_count = statement.column_count(); + let mut rows = + statement.query(deadpool_sqlite::rusqlite::params_from_iter(parameters))?; + let mut output = Vec::new(); + while let Some(row) = rows.next()? { + let values = (0..column_count) + .map(|index| value(index, row.get_ref(index)?)) + .collect::<Result<Vec<_>, _>>()?; + output.push(Row::new(values)); + } + Ok(output) + }) + .await + .map_err(interact_error)? + .map_err(database_error) +} + +async fn query_bounded_on( + connection: &deadpool_sqlite::Object, + sql: &'static str, + parameters: Vec<Value>, + maximum: usize, +) -> Result<Vec<Row>, Error> { + let rows = connection + .interact(move |connection| { + let parameters = sqlite_values(parameters); + let mut statement = connection.prepare_cached(sql)?; + let column_count = statement.column_count(); + let mut rows = + statement.query(deadpool_sqlite::rusqlite::params_from_iter(parameters))?; + let mut output = Vec::with_capacity(maximum.saturating_add(1)); + while output.len() <= maximum { + let Some(row) = rows.next()? else { + break; + }; + let values = (0..column_count) + .map(|index| value(index, row.get_ref(index)?)) + .collect::<Result<Vec<_>, _>>()?; + output.push(Row::new(values)); + } + Ok(output) + }) + .await + .map_err(interact_error)? + .map_err(database_error)?; + crate::orm::bounded_rows(rows, maximum) +} + +async fn query_owned_on( + connection: &deadpool_sqlite::Object, + sql: String, + parameters: Vec<Value>, +) -> Result<Vec<Row>, Error> { + connection + .interact(move |connection| { + let parameters = sqlite_values(parameters); + let mut statement = connection.prepare(&sql)?; + let column_count = statement.column_count(); + let mut rows = + statement.query(deadpool_sqlite::rusqlite::params_from_iter(parameters))?; + let mut output = Vec::new(); + while let Some(row) = rows.next()? { + let values = (0..column_count) + .map(|index| value(index, row.get_ref(index)?)) + .collect::<Result<Vec<_>, _>>()?; + output.push(Row::new(values)); + } + Ok(output) + }) + .await + .map_err(interact_error)? + .map_err(database_error) +} + +fn stream_rows( + connection: &mut deadpool_sqlite::rusqlite::Connection, + sql: &'static str, + parameters: Vec<Value>, + rows: tokio::sync::mpsc::Sender<Result<Row, Error>>, + ready: tokio::sync::oneshot::Sender<Result<(), Error>>, +) -> bool { + let parameters = sqlite_values(parameters); + let mut statement = match connection.prepare_cached(sql) { + Ok(statement) => statement, + Err(error) => { + let _ = ready.send(Err(database_error(error))); + return false; + } + }; + let column_count = statement.column_count(); + let mut source = match statement.query(deadpool_sqlite::rusqlite::params_from_iter(parameters)) + { + Ok(source) => source, + Err(error) => { + let _ = ready.send(Err(database_error(error))); + return false; + } + }; + if ready.send(Ok(())).is_err() { + return true; + } + loop { + let row = match source.next() { + Ok(Some(row)) => row, + Ok(None) => return true, + Err(error) => { + let _ = rows.blocking_send(Err(database_error(error))); + return false; + } + }; + let values = (0..column_count) + .map(|index| value(index, row.get_ref(index)?)) + .collect::<Result<Vec<_>, deadpool_sqlite::rusqlite::Error>>(); + let item = values.map(Row::new).map_err(database_error); + let healthy = item.is_ok(); + if rows.blocking_send(item).is_err() { + return true; + } + if !healthy { + return false; + } + } +} + +async fn batch_on(connection: &deadpool_sqlite::Object, sql: &'static str) -> Result<(), Error> { + connection + .interact(move |connection| connection.execute_batch(sql)) + .await + .map_err(interact_error)? + .map_err(database_error) +} + +fn value(column: usize, value: ValueRef<'_>) -> Result<Value, deadpool_sqlite::rusqlite::Error> { + Ok(match value { + ValueRef::Null => Value::Null, + ValueRef::Integer(value) => Value::Int(value), + ValueRef::Real(value) => Value::Float(value), + ValueRef::Text(value) => Value::Text( + std::str::from_utf8(value) + .map_err(|error| deadpool_sqlite::rusqlite::Error::Utf8Error(column, error))? + .to_owned(), + ), + ValueRef::Blob(value) => Value::Bytes(value.to_vec()), + }) +} + +fn interact_error(error: deadpool_sqlite::InteractError) -> Error { + Error::new(ErrorKind::Pool, error.to_string()) +} + +fn database_error(error: deadpool_sqlite::rusqlite::Error) -> Error { + let kind = match &error { + deadpool_sqlite::rusqlite::Error::SqliteFailure(failure, _) + if failure.code == deadpool_sqlite::rusqlite::ErrorCode::ConstraintViolation => + { + ErrorKind::Constraint + } + _ => ErrorKind::Database, + }; + Error::new(kind, error.to_string()) +} diff --git a/crates/dever-runtime/src/sqlite/migration.rs b/crates/dever-runtime/src/sqlite/migration.rs new file mode 100644 index 0000000..1c4d0a6 --- /dev/null +++ b/crates/dever-runtime/src/sqlite/migration.rs @@ -0,0 +1,483 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use deadpool_sqlite::rusqlite::{Connection, OptionalExtension, params}; + +use crate::database::{Field, Migration, MigrationPhase, Model, Schema}; +use crate::orm::Error; + +use super::{database_error, interact_error, sqlite_values}; + +const CREATE_HISTORY: &str = "\ +CREATE TABLE IF NOT EXISTS _dever_history (\ + model TEXT NOT NULL,\ + kind TEXT NOT NULL,\ + name TEXT NOT NULL,\ + revision TEXT NOT NULL,\ + definition TEXT NOT NULL,\ + applied_at INTEGER NOT NULL DEFAULT (unixepoch()),\ + PRIMARY KEY (model, kind, name)\ +)"; + +pub(super) async fn apply(connection: &deadpool_sqlite::Object, model: Model) -> Result<(), Error> { + connection + .interact(move |connection| transaction(connection, model)) + .await + .map_err(interact_error)? +} + +fn transaction(connection: &mut Connection, model: Model) -> Result<(), Error> { + connection + .execute_batch("BEGIN IMMEDIATE; PRAGMA defer_foreign_keys = ON;") + .map_err(database_error)?; + let result = apply_locked(connection, &model); + let result = match result { + Ok(()) => match connection.execute_batch("COMMIT") { + Ok(()) => Ok(()), + Err(commit) => rollback(connection, database_error(commit)), + }, + Err(error) => rollback(connection, error), + }; + connection.flush_prepared_statement_cache(); + result +} + +fn rollback(connection: &Connection, error: Error) -> Result<(), Error> { + match connection.execute_batch("ROLLBACK") { + Ok(()) => Err(error), + Err(rollback) => Err(error.cause(rollback)), + } +} + +fn apply_locked(connection: &Connection, model: &Model) -> Result<(), Error> { + connection + .execute_batch(CREATE_HISTORY) + .map_err(database_error)?; + let table_exists = table_exists(connection, &model.schema.table)?; + let previous = load_schema(connection, &model.schema.model)?; + match (table_exists, previous.as_ref()) { + (false, None) => create_model(connection, model)?, + (true, Some(previous)) => update_model(connection, model, previous)?, + (true, None) => { + return Err(Error::migration(format!( + "table '{}' exists without Dever schema history", + model.schema.table + ))); + } + (false, Some(_)) => { + return Err(Error::migration(format!( + "Dever schema history exists but table '{}' is missing", + model.schema.table + ))); + } + } + apply_seed(connection, model, previous.as_ref())?; + store_history(connection, &model.schema)?; + Ok(()) +} + +fn create_model(connection: &Connection, model: &Model) -> Result<(), Error> { + connection + .execute_batch(&model.create_table.sqlite) + .map_err(database_error)?; + create_indexes(connection, &model.create_indexes) +} + +fn update_model(connection: &Connection, model: &Model, previous: &Schema) -> Result<(), Error> { + if previous.table != model.schema.table { + return Err(Error::migration(format!( + "Model '{}' cannot rename table '{}' to '{}' automatically", + model.schema.model, previous.table, model.schema.table + ))); + } + validate_migrations(connection, &model.schema)?; + crate::database::validate_migration_order(previous, &model.schema)?; + validate_catalog(connection, previous)?; + let pending = unapplied_migrations(connection, &model.schema)?; + apply_data(connection, model, &pending, MigrationPhase::Before)?; + if previous.revision == model.schema.revision { + create_indexes(connection, &model.create_indexes)?; + } else if fields_match(&previous.fields, &model.schema.fields) { + synchronize_indexes(connection, previous, model)?; + } else { + rebuild_table(connection, previous, model, &pending)?; + } + apply_data(connection, model, &pending, MigrationPhase::After) +} + +fn apply_data( + connection: &Connection, + model: &Model, + pending: &[Migration], + phase: MigrationPhase, +) -> Result<(), Error> { + for statement in &model.data_migrations { + if statement.phase != phase + || !pending + .iter() + .any(|migration| migration.name == statement.name) + { + continue; + } + let parameters = sqlite_values(statement.parameters.clone()); + connection + .execute( + statement.sql.sqlite, + deadpool_sqlite::rusqlite::params_from_iter(parameters), + ) + .map_err(database_error)?; + } + Ok(()) +} + +fn rebuild_table( + connection: &Connection, + previous: &Schema, + model: &Model, + new_migrations: &[Migration], +) -> Result<(), Error> { + let allowed_drops = new_migrations + .iter() + .flat_map(|migration| migration.drops.iter().cloned()) + .collect::<BTreeSet<_>>(); + let previous_fields = previous + .fields + .iter() + .map(|field| (field.name.as_str(), field)) + .collect::<BTreeMap<_, _>>(); + let mut copied = Vec::new(); + let mut consumed = BTreeSet::new(); + for target in &model.schema.fields { + let source = previous_fields + .get(target.name.as_str()) + .copied() + .or_else(|| { + target + .rename_from + .as_deref() + .and_then(|name| previous_fields.get(name).copied()) + }); + let Some(source) = source else { + continue; + }; + if !compatible_types(&source.ty, &target.ty) { + return Err(Error::migration(format!( + "field '{}.{}' cannot change type from '{}' to '{}' automatically", + model.schema.model, target.name, source.ty, target.ty + ))); + } + if !consumed.insert(source.name.as_str()) { + return Err(Error::migration(format!( + "field '{}.{}' reuses migration source '{}'", + model.schema.model, target.name, source.name + ))); + } + copied.push((target.name.as_str(), source.name.as_str())); + } + for source in &previous.fields { + if !consumed.contains(source.name.as_str()) && !allowed_drops.contains(&source.name) { + return Err(Error::migration(format!( + "field '{}.{}' was removed without a named migrate drop", + model.schema.model, source.name + ))); + } + } + + connection + .execute_batch(&model.create_temporary_table) + .map_err(database_error)?; + let targets = copied + .iter() + .map(|(target, _)| quote(target)) + .collect::<Vec<_>>() + .join(", "); + let sources = copied + .iter() + .map(|(_, source)| quote(source)) + .collect::<Vec<_>>() + .join(", "); + let temporary = temporary_table(&model.schema); + connection + .execute_batch(&format!( + "INSERT INTO {} ({targets}) SELECT {sources} FROM {};\nDROP TABLE {};\nALTER TABLE {} RENAME TO {};", + quote(&temporary), + quote(&model.schema.table), + quote(&model.schema.table), + quote(&temporary), + quote(&model.schema.table), + )) + .map_err(database_error)?; + create_indexes(connection, &model.create_indexes)?; + let mut foreign_keys = connection + .prepare("PRAGMA foreign_key_check") + .map_err(database_error)?; + if foreign_keys + .query([]) + .map_err(database_error)? + .next() + .map_err(database_error)? + .is_some() + { + return Err(Error::migration(format!( + "Model '{}' migration would leave invalid foreign keys", + model.schema.model + ))); + } + Ok(()) +} + +fn synchronize_indexes( + connection: &Connection, + previous: &Schema, + model: &Model, +) -> Result<(), Error> { + for index in &previous.indexes { + connection + .execute_batch(&format!("DROP INDEX IF EXISTS {}", quote(&index.name))) + .map_err(database_error)?; + } + create_indexes(connection, &model.create_indexes) +} + +fn create_indexes( + connection: &Connection, + indexes: &[crate::database::SchemaSql], +) -> Result<(), Error> { + for index in indexes { + connection + .execute_batch(&index.sqlite) + .map_err(database_error)?; + } + Ok(()) +} + +fn apply_seed( + connection: &Connection, + model: &Model, + previous: Option<&Schema>, +) -> Result<(), Error> { + if previous.is_some_and(|schema| schema.seed_revision == model.schema.seed_revision) { + return Ok(()); + } + for seed in &model.seeds { + let parameters = sqlite_values(seed.parameters.clone()); + connection + .execute( + seed.sql.sqlite, + deadpool_sqlite::rusqlite::params_from_iter(parameters), + ) + .map_err(database_error)?; + } + Ok(()) +} + +fn validate_catalog(connection: &Connection, schema: &Schema) -> Result<(), Error> { + let recorded = load_catalog(connection, &schema.model)?.ok_or_else(|| { + Error::migration(format!( + "Model '{}' has no recorded database catalog", + schema.model + )) + })?; + if catalog(connection, &schema.table)? != recorded { + return Err(Error::migration(format!( + "table '{}' does not match its recorded Dever schema", + schema.table + ))); + } + Ok(()) +} + +fn validate_migrations(connection: &Connection, schema: &Schema) -> Result<(), Error> { + let applied = load_migrations(connection, &schema.model)?; + let desired = schema + .migrations + .iter() + .map(|migration| (migration.name.as_str(), migration)) + .collect::<BTreeMap<_, _>>(); + for (name, revision) in applied { + let Some(migration) = desired.get(name.as_str()) else { + return Err(Error::migration(format!( + "applied migration '{}.{}' was removed from source", + schema.model, name + ))); + }; + if migration.revision != revision { + return Err(Error::migration(format!( + "applied migration '{}.{}' was modified", + schema.model, name + ))); + } + } + Ok(()) +} + +fn unapplied_migrations(connection: &Connection, schema: &Schema) -> Result<Vec<Migration>, Error> { + let applied = load_migrations(connection, &schema.model)?; + Ok(schema + .migrations + .iter() + .filter(|migration| !applied.contains_key(&migration.name)) + .cloned() + .collect()) +} + +fn table_exists(connection: &Connection, table: &str) -> Result<bool, Error> { + connection + .query_row( + "SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = ?1", + [table], + |_| Ok(()), + ) + .optional() + .map(|value| value.is_some()) + .map_err(database_error) +} + +fn load_schema(connection: &Connection, model: &str) -> Result<Option<Schema>, Error> { + let definition = connection + .query_row( + "SELECT definition FROM _dever_history WHERE model = ?1 AND kind = 'schema' AND name = 'current'", + [model], + |row| row.get::<_, String>(0), + ) + .optional() + .map_err(database_error)?; + definition + .map(|definition| { + serde_json::from_str(&definition) + .map_err(|error| Error::migration(format!("invalid Dever schema history: {error}"))) + }) + .transpose() +} + +fn load_catalog(connection: &Connection, model: &str) -> Result<Option<String>, Error> { + connection + .query_row( + "SELECT definition FROM _dever_history WHERE model = ?1 AND kind = 'catalog' AND name = 'current'", + [model], + |row| row.get(0), + ) + .optional() + .map_err(database_error) +} + +fn catalog(connection: &Connection, table: &str) -> Result<String, Error> { + let table_sql = connection + .query_row( + "SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = ?1", + [table], + |row| row.get::<_, String>(0), + ) + .map_err(database_error)?; + let mut statement = connection + .prepare( + "SELECT name, sql FROM sqlite_schema WHERE type = 'index' AND tbl_name = ?1 AND substr(name, 1, 11) = '_dever_idx_' ORDER BY name", + ) + .map_err(database_error)?; + let indexes = statement + .query_map([table], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + }) + .map_err(database_error)? + .collect::<Result<Vec<_>, _>>() + .map_err(database_error)?; + serde_json::to_string(&(table_sql, indexes)) + .map_err(|error| Error::migration(format!("cannot encode SQLite catalog: {error}"))) +} + +fn load_migrations( + connection: &Connection, + model: &str, +) -> Result<BTreeMap<String, String>, Error> { + let mut statement = connection + .prepare( + "SELECT name, revision FROM _dever_history WHERE model = ?1 AND kind = 'migration'", + ) + .map_err(database_error)?; + statement + .query_map([model], |row| Ok((row.get(0)?, row.get(1)?))) + .map_err(database_error)? + .collect::<Result<BTreeMap<_, _>, _>>() + .map_err(database_error) +} + +fn store_history(connection: &Connection, schema: &Schema) -> Result<(), Error> { + let definition = serde_json::to_string(schema).map_err(|error| { + Error::migration(format!("cannot encode Dever schema history: {error}")) + })?; + store_history_row( + connection, + &schema.model, + "schema", + "current", + &schema.revision, + &definition, + )?; + for migration in &schema.migrations { + store_history_row( + connection, + &schema.model, + "migration", + &migration.name, + &migration.revision, + "", + )?; + } + store_history_row( + connection, + &schema.model, + "seed", + "current", + &schema.seed_revision, + "", + )?; + store_history_row( + connection, + &schema.model, + "catalog", + "current", + &schema.revision, + &catalog(connection, &schema.table)?, + ) +} + +fn store_history_row( + connection: &Connection, + model: &str, + kind: &str, + name: &str, + revision: &str, + definition: &str, +) -> Result<(), Error> { + connection + .execute( + "INSERT INTO _dever_history (model, kind, name, revision, definition) VALUES (?1, ?2, ?3, ?4, ?5) \ + ON CONFLICT (model, kind, name) DO UPDATE SET revision = excluded.revision, definition = excluded.definition, applied_at = unixepoch() WHERE _dever_history.kind <> 'migration'", + params![model, kind, name, revision, definition], + ) + .map(|_| ()) + .map_err(database_error) +} + +fn fields_match(previous: &[Field], desired: &[Field]) -> bool { + previous == desired +} + +fn compatible_types(previous: &str, desired: &str) -> bool { + previous == desired + || (previous.starts_with("Text") && desired.starts_with("Text")) + || (previous.starts_with("Bytes") && desired.starts_with("Bytes")) + || (previous == "Int" && desired == "Float") + || (sqlite_integer_reference(previous) && sqlite_integer_reference(desired)) +} + +fn sqlite_integer_reference(ty: &str) -> bool { + ty == "Int" || ty.starts_with("id:") +} + +pub fn temporary_table(schema: &Schema) -> String { + format!("_dever_{}_{}", schema.table, &schema.revision[..8]) +} + +fn quote(identifier: &str) -> String { + format!("\"{}\"", identifier.replace('"', "\"\"")) +} diff --git a/crates/dever-runtime/src/sse.rs b/crates/dever-runtime/src/sse.rs new file mode 100644 index 0000000..4bfe17e --- /dev/null +++ b/crates/dever-runtime/src/sse.rs @@ -0,0 +1,108 @@ +use crate::bytes::Bytes; +use crate::http::{Header, HttpReply}; + +#[derive(Clone, Debug)] +pub struct Event { + pub event: String, + pub data: String, + pub id: Option<String>, + pub retry_ms: Option<i64>, +} + +pub async fn start(reply: &HttpReply, mut headers: Vec<Header>) -> Result<(), String> { + if reply.chunk_limit() < 3 { + return Err("SSE chunk_bytes must fit a three-byte heartbeat".into()); + } + if headers + .iter() + .any(|header| header.name.eq_ignore_ascii_case("content-type")) + { + return Err("SSE owns the Content-Type header".into()); + } + headers.push(Header { + name: "content-type".into(), + value: Bytes::from_string("text/event-stream; charset=utf-8".into()), + }); + if !headers + .iter() + .any(|header| header.name.eq_ignore_ascii_case("cache-control")) + { + headers.push(Header { + name: "cache-control".into(), + value: Bytes::from_string("no-cache".into()), + }); + } + crate::http::reply::start_stream(reply, 200, headers, true).await +} + +pub async fn send(reply: &HttpReply, event: Event) -> Result<(), String> { + let bytes = encode(event, reply.chunk_limit())?; + crate::http::reply::write_chunk(reply, bytes, true).await +} + +// 按 EventSource 的行语义编码；先计算展开长度，避免超过单块上限后才分配。 +pub fn encode(event: Event, limit: usize) -> Result<Bytes, String> { + if event.event.contains(['\r', '\n']) { + return Err("SSE event cannot contain a line break".into()); + } + if event + .id + .as_ref() + .is_some_and(|id| id.contains(['\r', '\n', '\0'])) + { + return Err("SSE id cannot contain a line break or NUL".into()); + } + if event.retry_ms.is_some_and(|retry| retry < 0) { + return Err("SSE retry_ms must be nonnegative".into()); + } + let retry = event.retry_ms.map(|retry| retry.to_string()); + // split_terminator 丢弃末尾空行，会改变 data 尾部换行；这里保留每一行。 + let newlines = event.data.bytes().filter(|byte| *byte == b'\n').count(); + let lines = || { + event + .data + .split('\n') + .enumerate() + .flat_map(|(index, line)| { + let line = if index < newlines { + line.strip_suffix('\r').unwrap_or(line) + } else { + line + }; + line.split('\r') + }) + }; + let length = 1 + + lines().map(|line| 7 + line.len()).sum::<usize>() + + if event.event.is_empty() { + 0 + } else { + 8 + event.event.len() + } + + event.id.as_ref().map_or(0, |id| 5 + id.len()) + + retry.as_ref().map_or(0, |retry| 8 + retry.len()); + if length > limit { + return Err("SSE event exceeds chunk_bytes".into()); + } + let mut encoded = String::with_capacity(length); + let mut field = |name: &str, value: &str| { + encoded.push_str(name); + encoded.push_str(": "); + encoded.push_str(value); + encoded.push('\n'); + }; + if !event.event.is_empty() { + field("event", &event.event); + } + if let Some(id) = event.id { + field("id", &id); + } + if let Some(retry) = retry { + field("retry", &retry); + } + for line in lines() { + field("data", line); + } + encoded.push('\n'); + Ok(Bytes::from_string(encoded)) +} diff --git a/crates/dever-runtime/src/task.rs b/crates/dever-runtime/src/task.rs new file mode 100644 index 0000000..f94c7e9 --- /dev/null +++ b/crates/dever-runtime/src/task.rs @@ -0,0 +1,1179 @@ +use std::collections::{HashMap, VecDeque}; +use std::future::Future; +use std::pin::Pin; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +use tokio::runtime::Builder; +use tokio::sync::{OwnedSemaphorePermit, Semaphore}; +use tokio::task::{AbortHandle, JoinError, JoinHandle, JoinSet}; + +mod scoped; + +pub(crate) use scoped::{ScopeFault, ScopedOwner, ScopedSpawner, scoped_owner}; +#[cfg(feature = "wire")] +pub(crate) use scoped::{ScopedCompletion, scoped_completion}; + +tokio::task_local! { + static CURRENT_SCOPE: Scope; +} + +pub const MAX_ASYNC_WORKERS: usize = 64; +pub const MAX_BLOCKING_WORKERS: usize = 64; +pub const DEFAULT_MAX_BLOCKING_WORKERS: usize = 8; +pub const DEFAULT_TASKS: usize = 4_096; +pub const MAX_TASKS: usize = 65_536; +pub const MAX_GROUP_TASKS: usize = 65_536; + +pub fn run_entry<F, T>(future: F) -> Result<T, String> +where + F: Future<Output = Result<T, String>>, +{ + run_boxed_entry(RuntimeConfig::default(), Box::pin(future)) +} + +pub fn run_entry_typed<F, T, E>(future: F) -> Result<T, E> +where + F: Future<Output = Result<T, E>>, + E: From<String>, +{ + run_boxed_entry(RuntimeConfig::default(), Box::pin(future)) +} + +/// Source-configured programs establish one runtime, including when called from +/// synchronous parallel handlers. Rust embedding retains run_entry_with. +pub fn start_entry<F>(config: RuntimeConfig, future: F) -> Result<(), String> +where + F: Future<Output = Result<(), String>>, +{ + static STARTED: AtomicBool = AtomicBool::new(false); + validate_runtime_config(config)?; + if STARTED.swap(true, Ordering::AcqRel) { + return Err("the program runtime can only be started once".into()); + } + run_boxed_entry(config, Box::pin(future)) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RuntimeConfig { + pub worker_threads: usize, + pub max_blocking_threads: usize, + pub task_capacity: usize, +} + +impl Default for RuntimeConfig { + fn default() -> Self { + Self { + worker_threads: default_async_workers(), + max_blocking_threads: default_blocking_workers(), + task_capacity: DEFAULT_TASKS, + } + } +} + +pub fn run_entry_with<F, T>(config: RuntimeConfig, future: F) -> Result<T, String> +where + F: Future<Output = Result<T, String>>, +{ + run_boxed_entry(config, Box::pin(future)) +} + +pub fn run_entry_with_typed<F, T, E>(config: RuntimeConfig, future: F) -> Result<T, E> +where + F: Future<Output = Result<T, E>>, + E: From<String>, +{ + run_boxed_entry(config, Box::pin(future)) +} + +// Pin once at each public entry before forwarding a potentially large Future +// through synchronous cleanup adapters and task-local scopes. +fn run_boxed_entry<F, T, E>(config: RuntimeConfig, future: Pin<Box<F>>) -> Result<T, E> +where + F: Future<Output = Result<T, E>>, + E: From<String>, +{ + run_scoped_entry(config, future, |result, resources| { + std::future::ready(merge_resource_result(result, resources)) + }) +} + +fn merge_resource_result<T, E>(result: Result<T, E>, resources: Result<(), String>) -> Result<T, E> +where + E: From<String>, +{ + match result { + Ok(value) => resources.map(|()| value).map_err(E::from), + Err(primary) => Err(primary), + } +} + +/// Close entry-owned resources after descendants drain, while their runtime +/// is still alive. Cleanup must not start new business tasks in the closed scope. +pub fn run_entry_with_typed_cleanup<F, T, E, C, Cleanup>( + config: RuntimeConfig, + future: F, + cleanup: C, +) -> Result<T, E> +where + F: Future<Output = Result<T, E>>, + E: From<String>, + C: FnOnce(Result<T, E>) -> Cleanup, + Cleanup: Future<Output = Result<T, E>>, +{ + run_scoped_entry(config, Box::pin(future), |result, resources| { + cleanup(merge_resource_result(result, resources)) + }) +} + +/// Stop invocation Workers before cancelling descendants, then let the caller +/// append cleanup diagnostics using its concrete fault representation. +pub fn run_entry_with_typed_resource_cleanup<F, T, E, C, Cleanup>( + config: RuntimeConfig, + future: F, + cleanup: C, +) -> Result<T, E> +where + F: Future<Output = Result<T, E>>, + E: From<String>, + C: FnOnce(Result<T, E>, Result<(), String>) -> Cleanup, + Cleanup: Future<Output = Result<T, E>>, +{ + run_scoped_entry(config, Box::pin(future), cleanup) +} + +fn run_scoped_entry<F, T, E, C, Cleanup>( + config: RuntimeConfig, + future: Pin<Box<F>>, + cleanup: C, +) -> Result<T, E> +where + F: Future<Output = Result<T, E>>, + E: From<String>, + C: FnOnce(Result<T, E>, Result<(), String>) -> Cleanup, + Cleanup: Future<Output = Result<T, E>>, +{ + if tokio::runtime::Handle::try_current().is_ok() { + return Err(E::from( + "an async Dever entry cannot start inside another runtime".into(), + )); + } + validate_runtime_config(config).map_err(E::from)?; + let runtime = Builder::new_multi_thread() + .worker_threads(config.worker_threads) + .max_blocking_threads(config.max_blocking_threads) + .enable_time() + .enable_io() + .build() + .map_err(|error| E::from(format!("cannot start async runtime: {error}")))?; + runtime.block_on(async { + let scope = Scope::root(config); + #[cfg(feature = "wire")] + let components = crate::component::Session::new(scope.clone()); + let execution = CURRENT_SCOPE.scope(scope.clone(), async { + let result = future.await; + #[cfg(feature = "wire")] + let resources = crate::component::close().await; + #[cfg(not(feature = "wire"))] + let resources = Ok(()); + scope.cancel_and_wait().await; + cleanup(result, resources).await + }); + #[cfg(feature = "wire")] + let execution = crate::component::scope_resources(Some(components), execution); + execution.await + }) +} + +pub struct Task<T, E = String> { + body: Option<AbortHandle>, + supervisor: Option<JoinHandle<(usize, TaskOutcome<T, E>)>>, + registration: Option<Registration>, +} + +impl<T, E> Task<T, E> { + fn request_stop(&mut self) { + if let Some(body) = self.body.take() { + body.abort(); + } + } + + async fn finish(mut self, stopping: bool) -> Result<TaskOutcome<T, E>, E> + where + E: From<String>, + { + if stopping { + self.request_stop(); + } else { + self.body.take(); + } + let supervisor = self.supervisor.take().expect("task consumed exactly once"); + let result = supervisor.await; + self.registration + .take() + .expect("task registration") + .finish(); + result + .map(|(_, outcome)| outcome) + .map_err(|error| E::from(join_error(error))) + } +} + +impl<T, E> Drop for Task<T, E> { + fn drop(&mut self) { + self.request_stop(); + } +} + +pub async fn run<F, T>(future: F) -> Result<Task<T>, String> +where + F: Future<Output = Result<T, String>> + Send + 'static, + T: Send + 'static, +{ + run_typed(future).await +} + +pub async fn run_typed<F, T, E>(future: F) -> Result<Task<T, E>, String> +where + F: Future<Output = Result<T, E>> + Send + 'static, + T: Send + 'static, + E: Send + 'static, +{ + let parent = current_scope()?; + run_in(&parent, future) +} + +pub(crate) fn run_in<F, T, E>(parent: &Scope, future: F) -> Result<Task<T, E>, String> +where + F: Future<Output = Result<T, E>> + Send + 'static, + T: Send + 'static, + E: Send + 'static, +{ + let permit = parent + .limits + .task_slots + .clone() + .try_acquire_owned() + .map_err(|_| "async task capacity is exhausted".to_owned())?; + let started = start_task(parent, future, permit); + let body = started.body.abort_handle(); + let registration = Registration { + scope: parent.clone(), + id: started.id, + }; + Ok(Task { + body: Some(body), + supervisor: Some(tokio::spawn(supervise(started))), + registration: Some(registration), + }) +} + +pub async fn wait<T, E>(task: Task<T, E>) -> Result<T, E> +where + E: From<String>, +{ + waited_result(task.finish(false).await) +} + +pub async fn stop<E>(task: Task<(), E>) -> Result<(), E> +where + E: From<String>, +{ + stopped_result(task.finish(true).await) +} + +// Resource close may itself be cancelled. Keep the supervisor in its owner until +// completion so another close caller can resume the same cleanup operation. +pub(crate) async fn stop_resource(slot: &mut Option<Task<()>>) -> Result<(), String> { + let Some(task) = slot.as_mut() else { + return Ok(()); + }; + task.request_stop(); + let result = task.supervisor.as_mut().expect("resource task").await; + stopped_result(slot.take().expect("resource task").completed(result)) +} + +#[cfg(feature = "wire")] +pub(crate) async fn wait_resource(slot: &mut Option<Task<()>>) -> Result<(), String> { + let Some(task) = slot.as_mut() else { + return Ok(()); + }; + let result = task.supervisor.as_mut().expect("resource task").await; + waited_result(slot.take().expect("resource task").completed(result)) +} + +/// The deadline ends execution, then cleanup completes before the caller resumes. +pub async fn wait_timeout<T, E>(mut task: Task<T, E>, millis: i64) -> Result<Option<T>, E> +where + E: From<String>, +{ + let duration = positive_duration(millis).map_err(E::from)?; + let completed = tokio::select! { + biased; + result = task.supervisor.as_mut().expect("owned task") => Some(result), + _ = tokio::time::sleep(duration) => None, + }; + match completed { + Some(result) => waited_result(task.completed(result)).map(Some), + None => discard_result(task.finish(true).await).map(|()| None), + } +} + +#[cfg(any(feature = "sqlite", feature = "postgres"))] +pub(crate) enum LeaseResult<T> { + Completed(T), + Timeout, + Lost, +} + +/// Retain the supervisor while renewing a lease; every exit joins cancelled descendants. +#[cfg(any(feature = "sqlite", feature = "postgres"))] +pub(crate) async fn wait_lease<T, F, Fut>( + mut task: Task<T>, + timeout_ms: i64, + period_ms: i64, + mut renew: F, +) -> Result<LeaseResult<T>, String> +where + F: FnMut() -> Fut, + Fut: Future<Output = Result<bool, String>>, +{ + let deadline = tokio::time::sleep(positive_duration(timeout_ms)?); + tokio::pin!(deadline); + loop { + tokio::select! { + biased; + result = task.supervisor.as_mut().expect("owned task") => return waited_result(task.completed(result)).map(LeaseResult::Completed), + _ = &mut deadline => { discard_result(task.finish(true).await)?; return Ok(LeaseResult::Timeout); } + _ = tokio::time::sleep(positive_duration(period_ms)?) => {} + } + let renewed = tokio::select! { + biased; + result = task.supervisor.as_mut().expect("owned task") => return waited_result(task.completed(result)).map(LeaseResult::Completed), + _ = &mut deadline => { discard_result(task.finish(true).await)?; return Ok(LeaseResult::Timeout); } + result = renew() => result, + }; + if !matches!(renewed, Ok(true)) { + discard_result(task.finish(true).await)?; + return renewed.map(|_| LeaseResult::Lost); + } + } +} + +/// The source root remains owned while a lifecycle requests cooperative shutdown. +pub async fn wait_shutdown<T, E, F>( + mut task: Task<T, E>, + shutdown: F, + grace_ms: i64, +) -> Result<T, E> +where + E: From<String>, + F: Future<Output = Result<(), String>>, +{ + tokio::select! { + biased; + result = task.supervisor.as_mut().expect("owned task") => return waited_result(task.completed(result)), + signal = shutdown => { + if let Err(error) = signal { + discard_result(task.finish(true).await)?; + return Err(E::from(error)); + } + } + } + wait_timeout(task, grace_ms) + .await? + .ok_or_else(|| E::from("application shutdown deadline exceeded".into())) +} + +pub async fn race<T, E>(mut tasks: Vec<Task<T, E>>) -> Result<T, E> +where + E: From<String>, +{ + if tasks.len() < 2 { + return Err(E::from("race requires at least two tasks".into())); + } + let (result, winner, _) = futures_util::future::select_all( + tasks + .iter_mut() + .map(|task| task.supervisor.as_mut().expect("owned task")), + ) + .await; + let result = waited_result(tasks.remove(winner).completed(result)); + for task in &mut tasks { + task.request_stop(); + } + let mut cleanup = Ok(()); + for task in tasks { + if let Err(error) = discard_result(task.finish(true).await) + && cleanup.is_ok() + { + cleanup = Err(error); + } + } + let value = result?; + cleanup?; + Ok(value) +} + +impl<T, E> Task<T, E> { + fn completed( + mut self, + result: Result<(usize, TaskOutcome<T, E>), JoinError>, + ) -> Result<TaskOutcome<T, E>, E> + where + E: From<String>, + { + self.body.take(); + self.supervisor.take(); + self.registration + .take() + .expect("task registration") + .finish(); + result + .map(|(_, outcome)| outcome) + .map_err(|error| E::from(join_error(error))) + } +} + +fn discard_result<T, E>(result: Result<TaskOutcome<T, E>, E>) -> Result<(), E> +where + E: From<String>, +{ + match result? { + TaskOutcome::Finished(result) => result.map(|_| ()), + TaskOutcome::Stopped => Ok(()), + TaskOutcome::Panicked(error) => Err(E::from(error)), + } +} + +pub fn positive_duration(millis: i64) -> Result<std::time::Duration, String> { + let millis = u64::try_from(millis) + .ok() + .filter(|value| *value > 0) + .ok_or_else(|| "duration must be positive".to_owned())?; + Ok(std::time::Duration::from_millis(millis)) +} + +pub fn ticks(millis: i64) -> Result<crate::async_stream::AsyncStream<i64>, String> { + let period = positive_duration(millis)?; + current_scope()?; + let mut timer = tokio::time::interval_at(tokio::time::Instant::now() + period, period); + timer.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + Ok(crate::async_stream::AsyncStream::new( + futures_util::stream::unfold((timer, 0_i64), |(mut timer, count)| async move { + timer.tick().await; + let next = count.checked_add(1)?; + Some((crate::resource::Pull::Item(next), (timer, next))) + }), + )) +} + +pub fn configured(workers: i64, blocking: i64, tasks: i64) -> Result<RuntimeConfig, String> { + let number = + |value| usize::try_from(value).map_err(|_| "runtime limits must be positive".to_owned()); + let config = RuntimeConfig { + worker_threads: number(workers)?, + max_blocking_threads: number(blocking)?, + task_capacity: number(tasks)?, + }; + validate_runtime_config(config)?; + Ok(config) +} + +pub async fn parallel<F, T>(apply: F) -> Result<T, String> +where + F: FnOnce() -> Result<T, String> + Send + 'static, + T: Send + 'static, +{ + parallel_typed(apply).await +} + +pub async fn parallel_typed<F, T, E>(apply: F) -> Result<T, E> +where + F: FnOnce() -> Result<T, E> + Send + 'static, + T: Send + 'static, + E: From<String> + Send + 'static, +{ + start_blocking(apply).await?.wait().await +} + +pub async fn blocking<F, T>(apply: F) -> Result<T, String> +where + F: FnOnce() -> Result<T, String> + Send + 'static, + T: Send + 'static, +{ + blocking_typed(apply).await +} + +pub async fn blocking_typed<F, T, E>(apply: F) -> Result<T, E> +where + F: FnOnce() -> Result<T, E> + Send + 'static, + T: Send + 'static, + E: From<String> + Send + 'static, +{ + parallel_typed(apply).await +} + +pub async fn parallel_each<I, T, F>(values: I, workers: i64, apply: F) -> Result<(), String> +where + I: IntoIterator<Item = T>, + I::IntoIter: Send, + T: Send + 'static, + F: Fn(T) -> Result<(), String> + Send + Sync + 'static, +{ + parallel_each_typed(values, workers, apply).await +} + +pub async fn parallel_each_typed<I, T, F, E>(values: I, workers: i64, apply: F) -> Result<(), E> +where + I: IntoIterator<Item = T>, + I::IntoIter: Send, + T: Send + 'static, + F: Fn(T) -> Result<(), E> + Send + Sync + 'static, + E: From<String> + Send + 'static, +{ + let workers = + crate::concurrent::worker_count(workers).map_err(|error| E::from(error.to_owned()))?; + let mut values = values.into_iter(); + let (minimum, maximum) = values.size_hint(); + let exact_items = maximum.filter(|maximum| *maximum == minimum); + let concurrency = workers + .min( + current_runtime_limits() + .map_err(E::from)? + .max_blocking_workers, + ) + .min(exact_items.unwrap_or(usize::MAX)); + if concurrency == 0 { + return Ok(()); + } + let batch_size = exact_items + .map(|items| crate::concurrent::batch_size(items, concurrency)) + .unwrap_or(1); + let apply = Arc::new(apply); + let cancelled = Arc::new(AtomicBool::new(false)); + let mut jobs: VecDeque<ScopedBlocking<(), E>> = VecDeque::new(); + let mut first_error = None; + + loop { + if jobs.len() >= concurrency { + let result = jobs + .pop_front() + .expect("non-empty parallel job queue") + .wait() + .await; + record_parallel_result(result, &cancelled, &mut first_error); + } + if cancelled.load(Ordering::Acquire) { + break; + } + let permit = acquire_blocking_slot().await.map_err(E::from)?; + let batch: Vec<_> = values.by_ref().take(batch_size).collect(); + if batch.is_empty() { + break; + } + let apply = Arc::clone(&apply); + let cancelled = Arc::clone(&cancelled); + jobs.push_back(start_blocking_with_permit(permit, move || { + for value in batch { + if cancelled.load(Ordering::Acquire) { + break; + } + if let Err(error) = apply(value) { + cancelled.store(true, Ordering::Release); + return Err(error); + } + } + Ok(()) + })?); + } + while let Some(job) = jobs.pop_front() { + record_parallel_result(job.wait().await, &cancelled, &mut first_error); + } + first_error.map_or(Ok(()), Err) +} + +pub async fn sleep(milliseconds: i64) -> Result<(), &'static str> { + let milliseconds = + u64::try_from(milliseconds).map_err(|_| "sleep duration must not be negative")?; + tokio::time::sleep(std::time::Duration::from_millis(milliseconds)).await; + Ok(()) +} + +pub async fn parallel_each_stream<T, F, Fut>( + stream: crate::async_stream::AsyncStream<T>, + limit: i64, + apply: F, +) -> Result<(), String> +where + T: Send + 'static, + F: Fn(T) -> Fut, + Fut: Future<Output = Result<(), String>> + Send + 'static, +{ + parallel_each_stream_typed(stream, limit, apply).await +} + +pub async fn parallel_each_stream_typed<T, F, Fut, E>( + stream: crate::async_stream::AsyncStream<T>, + limit: i64, + apply: F, +) -> Result<(), E> +where + T: Send + 'static, + F: Fn(T) -> Fut, + Fut: Future<Output = Result<(), E>> + Send + 'static, + E: From<String> + Send + 'static, +{ + let _owner = stream.close_on_drop(); + let mut group = Group::<E>::new_typed(limit).map_err(E::from)?; + let result = async { + loop { + group.reap_completed().await?; + if group.tasks.len() == group.limit { + group.join_one().await?; + continue; + } + // 拉取前预留容量；生产者不会在任务组之外积压下一项。 + let permit = task_slots() + .map_err(E::from)? + .try_acquire_owned() + .map_err(|_| E::from("async task capacity is exhausted".to_owned()))?; + let next = stream.pull(); + tokio::pin!(next); + let value = loop { + tokio::select! { + biased; + result = group.tasks.join_next(), if !group.tasks.is_empty() => { + group.complete(result.expect("non-empty group"), false)?; + } + value = &mut next => break value, + } + }; + let Some(value) = value else { + return Ok(()); + }; + group.start(apply(value), permit); + } + } + .await; + match result { + Ok(()) => group.wait().await, + Err(error) => group.stop_and_drain(Some(error)).await, + } +} + +pub struct Group<E: 'static = String> { + limit: usize, + tasks: JoinSet<(usize, TaskOutcome<(), E>)>, + bodies: HashMap<usize, AbortHandle>, + scope: Scope, +} + +impl<E> Group<E> +where + E: From<String> + Send + 'static, +{ + pub fn new_typed(limit: i64) -> Result<Self, String> { + let limit = usize::try_from(limit) + .ok() + .filter(|limit| (1..=MAX_GROUP_TASKS).contains(limit)) + .ok_or_else(|| "task group limit must be between 1 and 65536".to_owned())?; + Ok(Self { + limit, + tasks: JoinSet::new(), + bodies: HashMap::new(), + scope: current_scope()?, + }) + } + + pub async fn run<F>(&mut self, future: F) -> Result<(), E> + where + F: Future<Output = Result<(), E>> + Send + 'static, + { + if let Err(error) = self.reap_completed().await { + return self.stop_and_drain(Some(error)).await; + } + if self.tasks.len() >= self.limit + && let Err(error) = self.join_one().await + { + return self.stop_and_drain(Some(error)).await; + } + let permit = task_slots() + .map_err(E::from)? + .try_acquire_owned() + .map_err(|_| E::from("async task capacity is exhausted".to_owned()))?; + self.start(future, permit); + Ok(()) + } + + fn start<F>(&mut self, future: F, permit: OwnedSemaphorePermit) + where + F: Future<Output = Result<(), E>> + Send + 'static, + { + let started = start_task(&self.scope, future, permit); + self.bodies.insert(started.id, started.body.abort_handle()); + self.tasks.spawn(supervise(started)); + } + + pub async fn wait(mut self) -> Result<(), E> { + while !self.tasks.is_empty() { + if let Err(error) = self.join_one().await { + return self.stop_and_drain(Some(error)).await; + } + } + Ok(()) + } + + pub async fn stop(mut self) -> Result<(), E> { + self.stop_and_drain(None).await + } + + async fn reap_completed(&mut self) -> Result<(), E> { + while let Some(result) = self.tasks.try_join_next() { + self.complete(result, false)?; + } + Ok(()) + } + + async fn join_one(&mut self) -> Result<(), E> { + match self.tasks.join_next().await { + Some(result) => self.complete(result, false), + None => Ok(()), + } + } + + fn complete( + &mut self, + result: Result<(usize, TaskOutcome<(), E>), JoinError>, + stopping: bool, + ) -> Result<(), E> { + let (id, outcome) = result.map_err(|error| E::from(join_error(error)))?; + self.bodies.remove(&id); + self.scope.unregister(id); + if stopping { + stopped_result(Ok(outcome)) + } else { + waited_result(Ok(outcome)) + } + } + + fn request_stop(&mut self) { + for body in self.bodies.values() { + body.abort(); + } + } + + async fn stop_and_drain(&mut self, mut first_error: Option<E>) -> Result<(), E> { + self.request_stop(); + while let Some(result) = self.tasks.join_next().await { + if let Err(error) = self.complete(result, true) { + first_error.get_or_insert(error); + } + } + for id in self.bodies.keys().copied().collect::<Vec<_>>() { + self.scope.unregister(id); + } + self.bodies.clear(); + first_error.map_or(Ok(()), Err) + } +} + +impl Group<String> { + pub fn new(limit: i64) -> Result<Self, String> { + Self::new_typed(limit) + } +} + +impl<E: 'static> Drop for Group<E> { + fn drop(&mut self) { + for body in self.bodies.values() { + body.abort(); + } + self.tasks.detach_all(); + } +} + +struct StartedTask<T, E> { + id: usize, + body: JoinHandle<Result<T, E>>, + scope: Scope, + completion: Completion, + _permit: OwnedSemaphorePermit, +} + +fn start_task<F, T, E>(parent: &Scope, future: F, permit: OwnedSemaphorePermit) -> StartedTask<T, E> +where + F: Future<Output = Result<T, E>> + Send + 'static, + T: Send + 'static, + E: Send + 'static, +{ + let scope = parent.child(); + #[cfg(feature = "wire")] + let future = crate::component::scope_resources(crate::component::current(), future); + #[cfg(feature = "api")] + let future = crate::application::scope_resources(crate::application::current(), future); + let body = tokio::spawn(CURRENT_SCOPE.scope(scope.clone(), future)); + let completion = Completion::new(); + let id = parent.register(Some(body.abort_handle()), completion.clone()); + StartedTask { + id, + body, + scope, + completion, + _permit: permit, + } +} + +async fn supervise<T, E>(started: StartedTask<T, E>) -> (usize, TaskOutcome<T, E>) +where + E: Send + 'static, +{ + supervise_with(started, |_, _| {}).await +} + +async fn supervise_with<T, E>( + started: StartedTask<T, E>, + complete: impl FnOnce(usize, &TaskOutcome<T, E>), +) -> (usize, TaskOutcome<T, E>) +where + E: Send + 'static, +{ + let _complete = CompleteOnDrop(started.completion); + let outcome = match started.body.await { + Ok(result) => TaskOutcome::Finished(result), + Err(error) if error.is_cancelled() => TaskOutcome::Stopped, + Err(error) => TaskOutcome::Panicked(join_error(error)), + }; + started.scope.cancel_and_wait().await; + complete(started.id, &outcome); + (started.id, outcome) +} + +enum TaskOutcome<T, E> { + Finished(Result<T, E>), + Stopped, + Panicked(String), +} + +fn waited_result<T, E>(result: Result<TaskOutcome<T, E>, E>) -> Result<T, E> +where + E: From<String>, +{ + match result? { + TaskOutcome::Finished(result) => result, + TaskOutcome::Stopped => Err(E::from("async task was stopped".into())), + TaskOutcome::Panicked(error) => Err(E::from(error)), + } +} + +fn stopped_result<E>(result: Result<TaskOutcome<(), E>, E>) -> Result<(), E> +where + E: From<String>, +{ + match result? { + TaskOutcome::Finished(result) => result, + TaskOutcome::Stopped => Ok(()), + TaskOutcome::Panicked(error) => Err(E::from(error)), + } +} + +struct ScopedBlocking<T, E> { + id: usize, + scope: Scope, + handle: Option<JoinHandle<(usize, Result<T, E>)>>, +} + +impl<T: Send + 'static, E: From<String> + Send + 'static> ScopedBlocking<T, E> { + async fn wait(mut self) -> Result<T, E> { + let handle = self.handle.take().expect("blocking task awaited once"); + let result = handle.await; + self.scope.unregister(self.id); + match result { + Ok((_, result)) => result, + Err(error) => Err(E::from(join_error(error))), + } + } +} + +async fn start_blocking<F, T, E>(apply: F) -> Result<ScopedBlocking<T, E>, E> +where + F: FnOnce() -> Result<T, E> + Send + 'static, + T: Send + 'static, + E: From<String> + Send + 'static, +{ + start_blocking_with_permit(acquire_blocking_slot().await.map_err(E::from)?, apply) +} + +fn start_blocking_with_permit<F, T, E>( + permit: OwnedSemaphorePermit, + apply: F, +) -> Result<ScopedBlocking<T, E>, E> +where + F: FnOnce() -> Result<T, E> + Send + 'static, + T: Send + 'static, + E: From<String> + Send + 'static, +{ + let scope = current_scope().map_err(E::from)?; + let completion = Completion::new(); + let id = scope.register(None, completion.clone()); + #[cfg(feature = "api")] + let session = crate::application::current(); + #[cfg(feature = "wire")] + let components = crate::component::current(); + let handle = tokio::task::spawn_blocking(move || { + #[cfg(feature = "api")] + let apply = || crate::application::with_resources(session, apply); + #[cfg(feature = "wire")] + let apply = || crate::component::with_resources(components, apply); + let _permit = permit; + let _complete = CompleteOnDrop(completion); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(apply)) + .unwrap_or_else(|_| Err(E::from("blocking task panicked".to_owned()))); + (id, result) + }); + Ok(ScopedBlocking { + id, + scope, + handle: Some(handle), + }) +} + +async fn acquire_blocking_slot() -> Result<OwnedSemaphorePermit, String> { + blocking_slots()? + .acquire_owned() + .await + .map_err(|_| "blocking task capacity is closed".to_owned()) +} + +#[derive(Clone)] +pub(crate) struct Scope { + state: Arc<ScopeState>, + limits: RuntimeLimits, +} + +struct ScopeState { + registry: Mutex<ScopeRegistry>, +} + +#[derive(Default)] +struct ScopeRegistry { + next_id: usize, + work: HashMap<usize, ScopedWork>, +} + +struct ScopedWork { + abort: Option<AbortHandle>, + completion: Completion, +} + +impl Scope { + fn root(config: RuntimeConfig) -> Self { + Self::new(RuntimeLimits::new(config)) + } + + fn child(&self) -> Self { + Self::new(self.limits.clone()) + } + + fn new(limits: RuntimeLimits) -> Self { + Self { + state: Arc::new(ScopeState { + registry: Mutex::new(ScopeRegistry::default()), + }), + limits, + } + } + + fn register(&self, abort: Option<AbortHandle>, completion: Completion) -> usize { + let mut registry = self + .state + .registry + .lock() + .unwrap_or_else(|error| error.into_inner()); + let id = registry.next_id; + registry.next_id = registry.next_id.wrapping_add(1); + registry.work.insert(id, ScopedWork { abort, completion }); + id + } + + fn unregister(&self, id: usize) { + self.state + .registry + .lock() + .unwrap_or_else(|error| error.into_inner()) + .work + .remove(&id); + } + + async fn cancel_and_wait(&self) { + loop { + let work = { + let mut registry = self + .state + .registry + .lock() + .unwrap_or_else(|error| error.into_inner()); + std::mem::take(&mut registry.work) + }; + if work.is_empty() { + return; + } + for child in work.values() { + if let Some(abort) = &child.abort { + abort.abort(); + } + } + for child in work.into_values() { + child.completion.wait().await; + } + } + } + + async fn wait(&self) { + loop { + let completions = self + .state + .registry + .lock() + .unwrap_or_else(|error| error.into_inner()) + .work + .values() + .map(|work| work.completion.clone()) + .collect::<Vec<_>>(); + if completions.is_empty() { + return; + } + for completion in completions { + completion.wait().await; + } + } + } +} + +struct Registration { + scope: Scope, + id: usize, +} + +impl Registration { + fn finish(self) { + self.scope.unregister(self.id); + } +} + +#[derive(Clone)] +struct Completion(Arc<Semaphore>); + +impl Completion { + fn new() -> Self { + Self(Arc::new(Semaphore::new(0))) + } + + fn complete(&self) { + self.0.close(); + } + + async fn wait(self) { + let _ = self.0.acquire().await; + } +} + +struct CompleteOnDrop(Completion); + +impl Drop for CompleteOnDrop { + fn drop(&mut self) { + self.0.complete(); + } +} + +fn current_scope() -> Result<Scope, String> { + CURRENT_SCOPE + .try_with(Clone::clone) + .map_err(|_| "async operation requires a Dever task scope".to_owned()) +} + +#[cold] +fn join_error(error: JoinError) -> String { + if error.is_cancelled() { + "async task was stopped".into() + } else if error.is_panic() { + "async task panicked".into() + } else { + error.to_string() + } +} + +#[derive(Clone)] +struct RuntimeLimits { + task_slots: Arc<Semaphore>, + blocking_slots: Arc<Semaphore>, + max_blocking_workers: usize, +} + +impl RuntimeLimits { + fn new(config: RuntimeConfig) -> Self { + Self { + task_slots: Arc::new(Semaphore::new(config.task_capacity)), + blocking_slots: Arc::new(Semaphore::new(config.max_blocking_threads)), + max_blocking_workers: config.max_blocking_threads, + } + } +} + +fn record_parallel_result<E>( + result: Result<(), E>, + cancelled: &AtomicBool, + first_error: &mut Option<E>, +) { + if let Err(error) = result { + cancelled.store(true, Ordering::Release); + first_error.get_or_insert(error); + } +} + +fn task_slots() -> Result<Arc<Semaphore>, String> { + Ok(current_runtime_limits()?.task_slots) +} + +fn blocking_slots() -> Result<Arc<Semaphore>, String> { + Ok(current_runtime_limits()?.blocking_slots) +} + +fn current_runtime_limits() -> Result<RuntimeLimits, String> { + Ok(current_scope()?.limits) +} + +fn default_async_workers() -> usize { + available_parallelism().min(MAX_ASYNC_WORKERS) +} + +fn default_blocking_workers() -> usize { + available_parallelism() + .min(DEFAULT_MAX_BLOCKING_WORKERS) + .min(MAX_BLOCKING_WORKERS) +} + +fn available_parallelism() -> usize { + std::thread::available_parallelism() + .map(usize::from) + .unwrap_or(1) +} + +fn validate_runtime_config(config: RuntimeConfig) -> Result<(), String> { + if !(1..=MAX_ASYNC_WORKERS).contains(&config.worker_threads) { + return Err(format!( + "async worker count must be between 1 and {MAX_ASYNC_WORKERS}" + )); + } + if !(1..=MAX_BLOCKING_WORKERS).contains(&config.max_blocking_threads) { + return Err(format!( + "blocking worker count must be between 1 and {MAX_BLOCKING_WORKERS}" + )); + } + if !(1..=MAX_TASKS).contains(&config.task_capacity) { + return Err(format!( + "async task capacity must be between 1 and {MAX_TASKS}" + )); + } + Ok(()) +} diff --git a/crates/dever-runtime/src/task/scoped.rs b/crates/dever-runtime/src/task/scoped.rs new file mode 100644 index 0000000..ac34588 --- /dev/null +++ b/crates/dever-runtime/src/task/scoped.rs @@ -0,0 +1,270 @@ +use std::future::Future; +use std::sync::{Arc, Mutex}; + +use tokio::sync::{Notify, oneshot}; + +use super::{ + CURRENT_SCOPE, CompleteOnDrop, Completion, Registration, Scope, TaskOutcome, start_task, + supervise_with, +}; + +/// Completion of an externally supervised resource still belongs to the +/// requesting task: cancellation must drain it without aborting its cleanup. +#[cfg(feature = "wire")] +pub(crate) fn scoped_completion() -> Result<ScopedCompletion, String> { + let scope = super::current_scope()?; + let completion = Completion::new(); + let id = scope.register(None, completion.clone()); + Ok(ScopedCompletion { + registration: Some(Registration { scope, id }), + completion, + }) +} + +#[cfg(feature = "wire")] +pub(crate) struct ScopedCompletion { + registration: Option<Registration>, + completion: Completion, +} + +#[cfg(feature = "wire")] +impl Drop for ScopedCompletion { + fn drop(&mut self) { + if let Some(registration) = self.registration.take() { + registration.finish(); + } + self.completion.complete(); + } +} + +pub(crate) fn scoped_owner<G>(guard: G) -> Result<(ScopedOwner, ScopedSpawner, ScopeFault), String> +where + G: Send + 'static, +{ + let parent = super::current_scope()?; + Ok(scoped_owner_in(parent, guard)) +} + +fn scoped_owner_in<G>(parent: Scope, guard: G) -> (ScopedOwner, ScopedSpawner, ScopeFault) +where + G: Send + 'static, +{ + let scope = parent.child(); + let completion = Completion::new(); + let registration = Registration { + id: parent.register(None, completion.clone()), + scope: parent, + }; + let admission = Arc::new(Mutex::new(true)); + let fault = ScopeFault::new(); + let task_slots = Arc::clone(&scope.limits.task_slots); + ( + ScopedOwner { + scope: scope.clone(), + admission: Arc::clone(&admission), + fault: fault.clone(), + registration: Some(registration), + completion: Some(completion), + guard: Some(Box::new(guard)), + }, + ScopedSpawner { + scope, + admission, + fault: fault.clone(), + task_slots, + }, + fault, + ) +} + +pub(crate) struct ScopedOwner { + scope: Scope, + admission: Arc<Mutex<bool>>, + fault: ScopeFault, + registration: Option<Registration>, + completion: Option<Completion>, + guard: Option<Box<dyn Send>>, +} + +impl ScopedOwner { + pub(crate) async fn enter<F>(&mut self, future: F) -> F::Output + where + F: Future, + { + CURRENT_SCOPE.scope(self.scope.clone(), future).await + } + + pub(crate) async fn finish(mut self) -> Result<(), String> { + self.start_cleanup(false, true) + .expect("scoped owner cleaned once") + .await + .unwrap_or_else(|_| Err("scoped task cleanup stopped".into())) + } + + pub(crate) async fn stop(mut self) -> Result<(), String> { + self.start_cleanup(true, true) + .expect("scoped owner cleaned once") + .await + .unwrap_or_else(|_| Err("scoped task cleanup stopped".into())) + } + + pub(crate) fn finish_in_background(mut self) { + let _ = self.start_cleanup(false, false); + } + + pub(crate) fn stop_in_background(mut self) { + let _ = self.start_cleanup(true, false); + } + + fn close_admission(&self) { + *self + .admission + .lock() + .unwrap_or_else(|error| error.into_inner()) = false; + } + + fn start_cleanup( + &mut self, + stopping: bool, + cancel_if_unobserved: bool, + ) -> Option<oneshot::Receiver<Result<(), String>>> { + self.close_admission(); + let completion = self.completion.take()?; + let (mut sender, receiver) = oneshot::channel(); + let scope = self.scope.clone(); + let registration = self.registration.take(); + let guard = self.guard.take(); + let fault = self.fault.clone(); + tokio::spawn(async move { + let complete = CompleteOnDrop(completion); + if stopping { + scope.cancel_and_wait().await; + } else if cancel_if_unobserved { + tokio::select! { + _ = scope.wait() => {} + _ = sender.closed() => scope.cancel_and_wait().await, + } + } else { + scope.wait().await; + } + drop(guard); + if let Some(registration) = registration { + registration.finish(); + } + let result = fault.result(); + drop(complete); + let _ = sender.send(result); + }); + Some(receiver) + } +} + +impl Drop for ScopedOwner { + fn drop(&mut self) { + let _ = self.start_cleanup(true, true); + } +} + +#[derive(Clone)] +pub(crate) struct ScopedSpawner { + scope: Scope, + admission: Arc<Mutex<bool>>, + fault: ScopeFault, + task_slots: Arc<tokio::sync::Semaphore>, +} + +impl ScopedSpawner { + pub(crate) fn spawn<F>(&self, future: F) + where + F: Future<Output = ()> + Send + 'static, + { + let accepting = self + .admission + .lock() + .unwrap_or_else(|error| error.into_inner()); + if !*accepting { + return; + } + let permit = match Arc::clone(&self.task_slots) + .try_acquire_owned() + .map_err(|_| "async task capacity is exhausted".to_owned()) + { + Ok(permit) => permit, + Err(error) => { + self.fault.report(error); + return; + } + }; + let started = start_task( + &self.scope, + async move { + future.await; + Ok(()) + }, + permit, + ); + let scope = self.scope.clone(); + let fault = self.fault.clone(); + tokio::spawn(supervise_with(started, move |id, outcome| { + match outcome { + TaskOutcome::Finished(Err(error)) | TaskOutcome::Panicked(error) => { + fault.report(error.clone()); + } + TaskOutcome::Finished(Ok(())) | TaskOutcome::Stopped => {} + } + scope.unregister(id); + })); + } +} + +#[derive(Clone)] +pub(crate) struct ScopeFault(Arc<FaultState>); + +struct FaultState { + first: Mutex<Option<String>>, + changed: Notify, +} + +impl ScopeFault { + fn new() -> Self { + Self(Arc::new(FaultState { + first: Mutex::new(None), + changed: Notify::new(), + })) + } + + pub(crate) fn report(&self, error: String) { + let mut first = self + .0 + .first + .lock() + .unwrap_or_else(|error| error.into_inner()); + if first.is_none() { + *first = Some(error); + drop(first); + self.0.changed.notify_one(); + } + } + + pub(crate) async fn wait(&self) -> String { + loop { + let changed = self.0.changed.notified(); + if let Some(error) = self.current() { + return error; + } + changed.await; + } + } + + fn result(&self) -> Result<(), String> { + self.current().map_or(Ok(()), Err) + } + + fn current(&self) -> Option<String> { + self.0 + .first + .lock() + .unwrap_or_else(|error| error.into_inner()) + .clone() + } +} diff --git a/crates/dever-runtime/src/tenant.rs b/crates/dever-runtime/src/tenant.rs new file mode 100644 index 0000000..df389d8 --- /dev/null +++ b/crates/dever-runtime/src/tenant.rs @@ -0,0 +1,483 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::future::Future; +use std::sync::{Arc, OnceLock}; +use std::time::{Duration, Instant}; + +use tokio::sync::Mutex; + +use crate::database::{Database, OwnedSql, Sql}; +use crate::orm::{self, Error, ErrorKind, Value}; + +tokio::task_local! { + static CURRENT_TENANT: i64; +} + +static MANAGER: OnceLock<Arc<Manager>> = OnceLock::new(); + +pub(crate) struct Manager { + control: Database, + schema_fingerprint: String, + max_pools: usize, + idle_timeout: Duration, + state: Mutex<State>, +} + +struct State { + accepting: bool, + pools: BTreeMap<(String, i64), PoolEntry>, +} + +struct PoolEntry { + database: Database, + last_used: Instant, + ready: bool, +} + +impl State { + async fn retire_pool(&mut self, key: &(String, i64)) -> Result<(), Error> { + let Some(entry) = self.pools.get_mut(key) else { + return Ok(()); + }; + entry.ready = false; + let result = entry.database.close_pool().await; + self.pools.remove(key); + result + } +} + +pub async fn scope<T>(tenant_id: i64, future: impl Future<Output = T>) -> T { + CURRENT_TENANT.scope(tenant_id, future).await +} + +pub fn current_id() -> Result<i64, Error> { + CURRENT_TENANT + .try_with(|tenant_id| *tenant_id) + .map_err(|_| Error::new(ErrorKind::InvalidData, "tenant context is required")) + .and_then(|tenant_id| { + if tenant_id > 0 { + Ok(tenant_id) + } else { + Err(Error::new( + ErrorKind::InvalidData, + "tenant id must be positive", + )) + } + }) +} + +pub fn schema_fingerprint() -> Result<String, Error> { + Ok(manager()?.schema_fingerprint.clone()) +} + +pub async fn initialize(schema_fingerprint: &str) -> Result<(), Error> { + if schema_fingerprint.is_empty() || schema_fingerprint.len() > 65_536 { + return Err(Error::migration("invalid tenant schema fingerprint")); + } + let configuration = crate::config::current_settings(); + let settings = configuration + .tenant() + .ok_or_else(|| Error::new(ErrorKind::Pool, "tenant settings are not configured"))?; + let control = crate::database::database_for(Some(settings.database()), "")?; + control.execute(Sql { + sqlite: "CREATE TABLE IF NOT EXISTS _dever_tenant_storage (tenant_id INTEGER PRIMARY KEY, state TEXT NOT NULL, schema_fingerprint TEXT NOT NULL, updated_at INTEGER NOT NULL)", + postgres: "CREATE TABLE IF NOT EXISTS _dever_tenant_storage (tenant_id BIGINT PRIMARY KEY, state TEXT NOT NULL, schema_fingerprint TEXT NOT NULL, updated_at BIGINT NOT NULL)", + }, Vec::new()).await?; + control.execute(Sql { + sqlite: "CREATE TABLE IF NOT EXISTS _dever_tenant_component (tenant_id INTEGER NOT NULL, component TEXT NOT NULL, disabled_at INTEGER NOT NULL, PRIMARY KEY (tenant_id, component), FOREIGN KEY (tenant_id) REFERENCES _dever_tenant_storage (tenant_id) ON DELETE CASCADE)", + postgres: "CREATE TABLE IF NOT EXISTS _dever_tenant_component (tenant_id BIGINT NOT NULL, component TEXT NOT NULL, disabled_at BIGINT NOT NULL, PRIMARY KEY (tenant_id, component), FOREIGN KEY (tenant_id) REFERENCES _dever_tenant_storage (tenant_id) ON DELETE CASCADE)", + }, Vec::new()).await?; + let manager = Arc::new(Manager { + control, + schema_fingerprint: schema_fingerprint.to_owned(), + max_pools: settings.max_pools(), + idle_timeout: Duration::from_millis(settings.idle_timeout_ms()), + state: Mutex::new(State { + accepting: true, + pools: BTreeMap::new(), + }), + }); + #[cfg(feature = "api")] + if let Some(session) = crate::application::current() { + return session.set_tenant(manager); + } + MANAGER + .set(manager) + .map_err(|_| Error::migration("tenant storage was initialized more than once")) +} + +pub async fn database(connection: &str) -> Result<Database, Error> { + let tenant_id = current_id()?; + let manager = manager()?; + manager.require_ready(tenant_id).await?; + let key = (connection.to_owned(), tenant_id); + let mut state = manager.state.lock().await; + if !state.accepting { + return Err(Error::new( + ErrorKind::Pool, + "tenant storage is shutting down", + )); + } + if let Some(entry) = state.pools.get_mut(&key) + && entry.ready + { + entry.last_used = Instant::now(); + return Ok(entry.database.clone()); + } + // An interrupted preparation remains owned, but must never become a usable pool. + state.retire_pool(&key).await?; + let now = Instant::now(); + let expired = state + .pools + .iter() + .filter(|(_, entry)| { + entry.database.exclusively_held() + && now.duration_since(entry.last_used) >= manager.idle_timeout + }) + .map(|(key, _)| key.clone()) + .collect::<Vec<_>>(); + for expired_key in expired { + state.retire_pool(&expired_key).await?; + } + let mut reusable = state + .pools + .iter() + .filter(|(_, entry)| entry.database.exclusively_held()) + .map(|(key, entry)| (key.clone(), entry.last_used)) + .collect::<Vec<_>>(); + reusable.sort_by_key(|(_, last_used)| *last_used); + for (reusable_key, _) in reusable { + if state.pools.len() < manager.max_pools { + break; + } + state.retire_pool(&reusable_key).await?; + } + if state.pools.len() >= manager.max_pools { + return Err(Error::new( + ErrorKind::PoolExhausted, + "tenant database pool limit reached", + )); + } + let database = crate::database::tenant_database(connection, tenant_id, false).await?; + state.pools.insert( + key.clone(), + PoolEntry { + database: database.clone(), + last_used: now, + ready: false, + }, + ); + let preparation = async { + database.prepare_pool().await?; + manager.require_local_marker(&database, tenant_id).await + } + .await; + if let Err(error) = preparation { + let cleanup = state.retire_pool(&key).await; + return Err(match cleanup { + Ok(()) => error, + Err(cleanup) => error.cause(cleanup), + }); + } + state + .pools + .get_mut(&key) + .expect("registered tenant pool") + .ready = true; + Ok(database) +} + +pub async fn ready_ids_after(after: i64, limit: usize) -> Result<Vec<i64>, Error> { + if after < 0 || !(1..=256).contains(&limit) { + return Err(Error::invalid_data("invalid tenant worker page")); + } + let manager = manager()?; + let rows = manager.control.query(Sql { + sqlite: "SELECT tenant_id FROM _dever_tenant_storage WHERE state='ready' AND schema_fingerprint=?1 AND tenant_id>?2 ORDER BY tenant_id LIMIT ?3", + postgres: "SELECT tenant_id FROM _dever_tenant_storage WHERE state='ready' AND schema_fingerprint=$1 AND tenant_id>$2 ORDER BY tenant_id LIMIT $3", + }, vec![ + Value::Text(manager.schema_fingerprint.clone()), + Value::Int(after), + Value::Int(limit as i64), + ]).await?; + rows.into_iter() + .map(|row| { + let tenant_id = orm::int(row.get(0)?.clone())?; + if tenant_id <= 0 { + return Err(Error::migration( + "tenant registry contains an invalid tenant id", + )); + } + Ok(tenant_id) + }) + .collect() +} + +pub async fn disable_component( + tenant_id: i64, + component: &str, + tenant_components: &[&str], +) -> Result<(), Error> { + require_tenant_component(component, tenant_components)?; + let manager = manager()?; + manager.require_ready(tenant_id).await?; + let disabled_at = crate::time::now().map_err(Error::migration)?; + manager.control.execute(Sql { + sqlite: "INSERT INTO _dever_tenant_component (tenant_id,component,disabled_at) VALUES (?1,?2,?3) ON CONFLICT(tenant_id,component) DO UPDATE SET disabled_at=excluded.disabled_at", + postgres: "INSERT INTO _dever_tenant_component (tenant_id,component,disabled_at) VALUES ($1,$2,$3) ON CONFLICT(tenant_id,component) DO UPDATE SET disabled_at=excluded.disabled_at", + }, vec![ + Value::Int(tenant_id), + Value::Text(component.to_owned()), + Value::Int(disabled_at), + ]).await.map(|_| ()) +} + +pub async fn enable_component( + tenant_id: i64, + component: &str, + tenant_components: &[&str], +) -> Result<(), Error> { + require_tenant_component(component, tenant_components)?; + let manager = manager()?; + manager.require_ready(tenant_id).await?; + manager + .control + .execute( + Sql { + sqlite: "DELETE FROM _dever_tenant_component WHERE tenant_id=?1 AND component=?2", + postgres: "DELETE FROM _dever_tenant_component WHERE tenant_id=$1 AND component=$2", + }, + vec![Value::Int(tenant_id), Value::Text(component.to_owned())], + ) + .await + .map(|_| ()) +} + +pub async fn require_components( + tenant_id: i64, + components: &[&str], + tenant_components: &[&str], +) -> Result<(), Error> { + let components = components.iter().copied().collect::<BTreeSet<_>>(); + for component in &components { + require_tenant_component(component, tenant_components)?; + } + if components.is_empty() { + return Ok(()); + } + + let manager = manager()?; + manager.require_ready(tenant_id).await?; + let sqlite_placeholders = (2..=components.len() + 1) + .map(|index| format!("?{index}")) + .collect::<Vec<_>>() + .join(","); + let postgres_placeholders = (2..=components.len() + 1) + .map(|index| format!("${index}")) + .collect::<Vec<_>>() + .join(","); + let mut parameters = Vec::with_capacity(components.len() + 1); + parameters.push(Value::Int(tenant_id)); + parameters.extend( + components + .iter() + .map(|component| Value::Text((*component).to_owned())), + ); + let rows = manager.control.query_owned(OwnedSql { + sqlite: format!("SELECT component FROM _dever_tenant_component WHERE tenant_id=?1 AND component IN ({sqlite_placeholders}) ORDER BY component LIMIT 1"), + postgres: format!("SELECT component FROM _dever_tenant_component WHERE tenant_id=$1 AND component IN ({postgres_placeholders}) ORDER BY component LIMIT 1"), + }, parameters).await?; + if let Some(row) = orm::optional_row(rows, "tenant component lookup")? { + let component = orm::text(row.get(0)?.clone())?; + return Err(Error::invalid_data(format!( + "tenant component '{component}' is disabled" + ))); + } + Ok(()) +} + +pub async fn begin_migration( + connection: &str, + tenant_id: i64, + schema_fingerprint: &str, +) -> Result<Database, Error> { + if tenant_id <= 0 { + return Err(Error::migration("tenant id must be positive")); + } + let manager = manager()?; + if schema_fingerprint != manager.schema_fingerprint { + return Err(Error::migration( + "tenant migration schema fingerprint does not match this application", + )); + } + manager + .write_state(tenant_id, "migrating", schema_fingerprint) + .await?; + let database = crate::database::tenant_database(connection, tenant_id, true).await?; + #[cfg(feature = "api")] + if let Some(session) = crate::application::current() { + session.own_migration_database(database.clone()); + } + if let Err(error) = database.prepare_pool().await { + return Err(match database.close_pool().await { + Ok(()) => error, + Err(cleanup) => error.cause(cleanup), + }); + } + Ok(database) +} + +pub async fn mark_database_ready( + database: &Database, + tenant_id: i64, + schema_fingerprint: &str, +) -> Result<(), Error> { + let manager = manager()?; + if schema_fingerprint != manager.schema_fingerprint { + return Err(Error::migration( + "tenant migration schema fingerprint does not match this application", + )); + } + if database.key().tenant_id() != Some(tenant_id) { + return Err(Error::migration( + "tenant migration database key does not match tenant id", + )); + } + database.execute(Sql { + sqlite: "CREATE TABLE IF NOT EXISTS _dever_tenant_ready (marker INTEGER PRIMARY KEY CHECK (marker = 1), tenant_id INTEGER NOT NULL, schema_fingerprint TEXT NOT NULL)", + postgres: "CREATE TABLE IF NOT EXISTS _dever_tenant_ready (marker BIGINT PRIMARY KEY CHECK (marker = 1), tenant_id BIGINT NOT NULL, schema_fingerprint TEXT NOT NULL)", + }, Vec::new()).await?; + database.execute(Sql { + sqlite: "INSERT INTO _dever_tenant_ready (marker,tenant_id,schema_fingerprint) VALUES (1,?1,?2) ON CONFLICT(marker) DO UPDATE SET tenant_id=excluded.tenant_id,schema_fingerprint=excluded.schema_fingerprint", + postgres: "INSERT INTO _dever_tenant_ready (marker,tenant_id,schema_fingerprint) VALUES (1,$1,$2) ON CONFLICT(marker) DO UPDATE SET tenant_id=excluded.tenant_id,schema_fingerprint=excluded.schema_fingerprint", + }, vec![Value::Int(tenant_id), Value::Text(schema_fingerprint.to_owned())]).await?; + Ok(()) +} + +pub async fn finish_migration(tenant_id: i64, schema_fingerprint: &str) -> Result<(), Error> { + let manager = manager()?; + if schema_fingerprint != manager.schema_fingerprint { + return Err(Error::migration( + "tenant migration schema fingerprint does not match this application", + )); + } + manager + .write_state(tenant_id, "ready", schema_fingerprint) + .await +} + +pub async fn shutdown() -> Result<(), Error> { + let Some(manager) = MANAGER.get() else { + return Ok(()); + }; + manager.close().await +} + +impl Manager { + pub(crate) async fn close(&self) -> Result<(), Error> { + let databases = { + let mut state = self.state.lock().await; + state.accepting = false; + std::mem::take(&mut state.pools) + .into_values() + .map(|entry| entry.database) + .collect::<Vec<_>>() + }; + for database in &databases { + database.stop_accepting(); + } + let mut first_error = None; + for database in databases { + if let Err(error) = database.close_pool().await + && first_error.is_none() + { + first_error = Some(error); + } + } + first_error.map_or(Ok(()), Err) + } +} + +fn manager() -> Result<Arc<Manager>, Error> { + #[cfg(feature = "api")] + if let Some(session) = crate::application::current() { + return session + .tenant() + .ok_or_else(|| Error::new(ErrorKind::Pool, "tenant storage is not initialized")); + } + MANAGER + .get() + .cloned() + .ok_or_else(|| Error::new(ErrorKind::Pool, "tenant storage is not initialized")) +} + +fn require_tenant_component(component: &str, tenant_components: &[&str]) -> Result<(), Error> { + if tenant_components.contains(&component) { + Ok(()) + } else { + Err(Error::invalid_data(format!( + "component '{component}' is not a tenant component" + ))) + } +} + +impl Manager { + async fn require_ready(&self, tenant_id: i64) -> Result<(), Error> { + let rows = self.control.query(Sql { + sqlite: "SELECT state, schema_fingerprint FROM _dever_tenant_storage WHERE tenant_id = ?1", + postgres: "SELECT state, schema_fingerprint FROM _dever_tenant_storage WHERE tenant_id = $1", + }, vec![Value::Int(tenant_id)]).await?; + let row = orm::optional_row(rows, "tenant storage lookup")? + .ok_or_else(|| Error::migration("tenant storage is not provisioned"))?; + let state = orm::text(row.get(0)?.clone())?; + let fingerprint = orm::text(row.get(1)?.clone())?; + if state != "ready" { + return Err(Error::migration("tenant storage is not ready")); + } + if fingerprint != self.schema_fingerprint { + return Err(Error::migration( + "tenant storage schema fingerprint is stale", + )); + } + Ok(()) + } + + async fn write_state( + &self, + tenant_id: i64, + state: &str, + fingerprint: &str, + ) -> Result<(), Error> { + let updated_at = crate::time::now().map_err(Error::migration)?; + self.control.execute(Sql { + sqlite: "INSERT INTO _dever_tenant_storage (tenant_id,state,schema_fingerprint,updated_at) VALUES (?1,?2,?3,?4) ON CONFLICT(tenant_id) DO UPDATE SET state=excluded.state,schema_fingerprint=excluded.schema_fingerprint,updated_at=excluded.updated_at", + postgres: "INSERT INTO _dever_tenant_storage (tenant_id,state,schema_fingerprint,updated_at) VALUES ($1,$2,$3,$4) ON CONFLICT(tenant_id) DO UPDATE SET state=excluded.state,schema_fingerprint=excluded.schema_fingerprint,updated_at=excluded.updated_at", + }, vec![ + Value::Int(tenant_id), + Value::Text(state.to_owned()), + Value::Text(fingerprint.to_owned()), + Value::Int(updated_at), + ]).await.map(|_| ()) + } + + async fn require_local_marker(&self, database: &Database, tenant_id: i64) -> Result<(), Error> { + let rows = database.query(Sql { + sqlite: "SELECT tenant_id, schema_fingerprint FROM _dever_tenant_ready WHERE marker = 1", + postgres: "SELECT tenant_id, schema_fingerprint FROM _dever_tenant_ready WHERE marker = 1", + }, Vec::new()).await.map_err(|error| { + Error::migration(format!("tenant database ready marker is missing: {error}")) + })?; + let row = orm::optional_row(rows, "tenant database ready marker")? + .ok_or_else(|| Error::migration("tenant database ready marker is missing"))?; + if orm::int(row.get(0)?.clone())? != tenant_id { + return Err(Error::migration( + "tenant database ready marker has the wrong tenant id", + )); + } + if orm::text(row.get(1)?.clone())? != self.schema_fingerprint { + return Err(Error::migration( + "tenant database ready marker has a stale schema fingerprint", + )); + } + Ok(()) + } +} diff --git a/crates/dever-runtime/src/text.rs b/crates/dever-runtime/src/text.rs new file mode 100644 index 0000000..e08fdbd --- /dev/null +++ b/crates/dever-runtime/src/text.rs @@ -0,0 +1,58 @@ +use crate::collections::List; + +pub fn codepoint(value: &str) -> Option<i64> { + let mut chars = value.chars(); + let first = chars.next()?; + chars + .next() + .is_none() + .then_some(i64::from(u32::from(first))) +} + +pub fn from_codepoint(value: i64) -> Option<String> { + char::from_u32(u32::try_from(value).ok()?).map(String::from) +} + +pub fn at(value: &str, index: i64) -> Option<String> { + value + .chars() + .nth(usize::try_from(index).ok()?) + .map(String::from) +} + +/// Text 的位置按 Unicode scalar 计数；Bytes API 单独使用字节偏移。 +pub fn slice(value: &str, start: i64, end: i64) -> Option<String> { + let start = usize::try_from(start).ok()?; + let end = usize::try_from(end).ok()?; + if start > end { + return None; + } + let mut boundaries = value + .char_indices() + .map(|(index, _)| index) + .chain([value.len()]); + let first = boundaries.nth(start)?; + let last = if start == end { + first + } else { + boundaries.nth(end - start - 1)? + }; + Some(value[first..last].to_owned()) +} + +pub fn index_of(value: &str, pattern: &str) -> Option<i64> { + value + .find(pattern) + .map(|offset| value[..offset].chars().count() as i64) +} + +pub fn split(value: &str, separator: &str) -> List<String> { + List::new(pieces(value, separator).collect()) +} + +pub fn pieces<'a>(value: &'a str, separator: &'a str) -> impl Iterator<Item = String> + 'a { + value + .split(separator) + .filter(move |piece| !separator.is_empty() || !piece.is_empty()) + .map(str::to_owned) +} diff --git a/crates/dever-runtime/src/time.rs b/crates/dever-runtime/src/time.rs new file mode 100644 index 0000000..1b74497 --- /dev/null +++ b/crates/dever-runtime/src/time.rs @@ -0,0 +1,188 @@ +use chrono::{Datelike, Timelike}; +use std::sync::OnceLock; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +const MILLIS_PER_DAY: i64 = 86_400_000; + +/// A worker and its application test share this clock; advancing it never sleeps. +#[derive(Clone, Default)] +pub struct Clock(Option<std::sync::Arc<std::sync::atomic::AtomicI64>>); + +impl Clock { + pub fn for_test() -> Self { + Self(Some(std::sync::Arc::new( + std::sync::atomic::AtomicI64::new(0), + ))) + } + + pub fn now(&self) -> Result<i64, crate::orm::Error> { + match &self.0 { + Some(clock) => Ok(clock.load(std::sync::atomic::Ordering::SeqCst)), + None => now().map_err(crate::orm::Error::invalid_data), + } + } + + pub fn advance(&self, millis: i64) -> Result<(), String> { + if millis < 0 { + return Err("test clock cannot move backwards".into()); + } + let clock = self + .0 + .as_ref() + .ok_or("production clock cannot be advanced")?; + clock + .fetch_update( + std::sync::atomic::Ordering::SeqCst, + std::sync::atomic::Ordering::SeqCst, + |current| current.checked_add(millis), + ) + .map(|_| ()) + .map_err(|_| "test clock overflow".into()) + } +} + +pub fn now() -> Result<i64, String> { + let value = unix_millis()?; + datetime(value)?; + Ok(value) +} + +fn datetime(value: i64) -> Result<chrono::DateTime<chrono::Utc>, String> { + chrono::DateTime::from_timestamp_millis(value) + .filter(|value| (0..=9999).contains(&value.year())) + .ok_or_else(|| "DateTime is outside the four-digit UTC calendar range".into()) +} + +pub fn parse_datetime(text: &str) -> Result<i64, String> { + if let Some(fraction) = text.get(19..).and_then(|tail| tail.strip_prefix('.')) + && fraction.bytes().take_while(u8::is_ascii_digit).count() > 3 + { + return Err("DateTime requires at most three fractional digits".into()); + } + let parsed = chrono::DateTime::parse_from_rfc3339(text) + .map_err(|_| "invalid RFC3339 DateTime".to_owned())?; + if parsed.nanosecond() >= 1_000_000_000 || parsed.nanosecond() % 1_000_000 != 0 { + return Err("DateTime requires millisecond precision and no leap seconds".into()); + } + let value = parsed.timestamp_millis(); + datetime(value)?; + Ok(value) +} + +pub fn format_datetime(value: i64) -> Result<String, String> { + let value = datetime(value)?; + let precision = if value.timestamp_subsec_millis() == 0 { + chrono::SecondsFormat::Secs + } else { + chrono::SecondsFormat::Millis + }; + Ok(value.to_rfc3339_opts(precision, true)) +} + +pub fn parse_date(text: &str) -> Result<i64, String> { + if text.len() != 10 + || text.as_bytes()[4] != b'-' + || text.as_bytes()[7] != b'-' + || !text + .bytes() + .enumerate() + .all(|(index, byte)| matches!(index, 4 | 7) || byte.is_ascii_digit()) + { + return Err("Date must use YYYY-MM-DD".into()); + } + let date = chrono::NaiveDate::parse_from_str(text, "%Y-%m-%d") + .map_err(|_| "invalid calendar Date".to_owned())?; + Ok(date + .and_hms_opt(0, 0, 0) + .expect("midnight") + .and_utc() + .timestamp_millis()) +} + +pub fn format_date(value: i64) -> Result<String, String> { + if value.rem_euclid(MILLIS_PER_DAY) != 0 { + return Err("Date must be UTC midnight milliseconds".into()); + } + Ok(datetime(value)?.format("%Y-%m-%d").to_string()) +} + +pub fn parse_time(text: &str) -> Result<i64, String> { + let bytes = text.as_bytes(); + if !matches!(bytes.len(), 8 | 12) + || bytes[2] != b':' + || bytes[5] != b':' + || (bytes.len() == 12 && bytes[8] != b'.') + || !bytes + .iter() + .enumerate() + .all(|(index, byte)| matches!(index, 2 | 5 | 8) || byte.is_ascii_digit()) + { + return Err("Time must use HH:MM:SS[.sss]".into()); + } + let time = chrono::NaiveTime::parse_from_str(text, "%H:%M:%S%.f") + .map_err(|_| "invalid clock Time".to_owned())?; + if time.nanosecond() >= 1_000_000_000 { + return Err("Time does not accept leap seconds".into()); + } + Ok(i64::from(time.num_seconds_from_midnight()) * 1000 + + i64::from(time.nanosecond() / 1_000_000)) +} + +pub fn format_time(value: i64) -> Result<String, String> { + if !(0..MILLIS_PER_DAY).contains(&value) { + return Err("Time must be milliseconds within one day".into()); + } + let time = chrono::NaiveTime::from_num_seconds_from_midnight_opt( + (value / 1000) as u32, + (value % 1000) as u32 * 1_000_000, + ) + .expect("validated time of day"); + Ok(time + .format(if value % 1000 == 0 { + "%H:%M:%S" + } else { + "%H:%M:%S%.3f" + }) + .to_string()) +} + +pub fn duration(millis: i64) -> i64 { + millis +} + +pub fn add(value: i64, duration: i64) -> Result<i64, String> { + value + .checked_add(duration) + .ok_or_else(|| "DateTime addition overflow".into()) +} + +pub fn subtract(value: i64, duration: i64) -> Result<i64, String> { + value + .checked_sub(duration) + .ok_or_else(|| "DateTime subtraction overflow".into()) +} + +pub fn difference(later: i64, earlier: i64) -> Result<i64, String> { + later + .checked_sub(earlier) + .ok_or_else(|| "Duration difference overflow".into()) +} + +pub fn unix_millis() -> Result<i64, String> { + let elapsed = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|error| error.to_string())?; + i64::try_from(elapsed.as_millis()).map_err(|_| "clock value is out of Int range".into()) +} + +pub fn monotonic_nanos() -> Result<i64, String> { + static ORIGIN: OnceLock<Instant> = OnceLock::new(); + i64::try_from(ORIGIN.get_or_init(Instant::now).elapsed().as_nanos()) + .map_err(|_| "clock value is out of Int range".into()) +} + +pub fn sleep(millis: i64) -> Result<(), String> { + let millis = u64::try_from(millis).map_err(|_| "sleep duration must not be negative")?; + std::thread::sleep(Duration::from_millis(millis)); + Ok(()) +} diff --git a/crates/dever-runtime/src/tls.rs b/crates/dever-runtime/src/tls.rs new file mode 100644 index 0000000..72bd19e --- /dev/null +++ b/crates/dever-runtime/src/tls.rs @@ -0,0 +1,109 @@ +use std::sync::{Arc, OnceLock}; + +use rustls::pki_types::{CertificateDer, PrivateKeyDer, ServerName, pem::PemObject}; +use rustls::{ClientConfig, RootCertStore, ServerConfig}; + +use crate::{bytes::Bytes, transport::Transport}; + +#[derive(Clone)] +pub struct ClientTls(pub(crate) Arc<ClientConfig>); + +#[derive(Clone)] +pub struct ServerTls(pub(crate) Arc<ServerConfig>); + +pub fn system() -> ClientTls { + static CONFIG: OnceLock<Arc<ClientConfig>> = OnceLock::new(); + ClientTls( + CONFIG + .get_or_init(|| { + let roots = RootCertStore { + roots: webpki_roots::TLS_SERVER_ROOTS.to_vec(), + }; + client_config(roots) + }) + .clone(), + ) +} + +/// Custom trust replaces public roots; it never disables certificate/name verification. +pub fn client(ca_pem: &Bytes) -> Result<ClientTls, String> { + let certificates = certificates(ca_pem)?; + let mut roots = RootCertStore::empty(); + for certificate in certificates { + roots.add(certificate).map_err(|error| error.to_string())?; + } + Ok(ClientTls(client_config(roots))) +} + +fn client_config(roots: RootCertStore) -> Arc<ClientConfig> { + let mut config = ClientConfig::builder() + .with_root_certificates(roots) + .with_no_client_auth(); + // Share trust material and keep rustls's session-cache capacity small. + config.resumption = rustls::client::Resumption::in_memory_sessions(32); + Arc::new(config) +} + +pub fn server(cert_pem: &Bytes, key_pem: &Bytes) -> Result<ServerTls, String> { + let key = PrivateKeyDer::from_pem_slice(key_pem.values()).map_err(|error| error.to_string())?; + let mut config = ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(certificates(cert_pem)?, key) + .map_err(|error| error.to_string())?; + config.session_storage = rustls::server::ServerSessionMemoryCache::new(32); + config.send_tls13_tickets = 1; + Ok(ServerTls(Arc::new(config))) +} + +fn certificates(pem: &Bytes) -> Result<Vec<CertificateDer<'static>>, String> { + let certificates = CertificateDer::pem_slice_iter(pem.values()) + .collect::<Result<Vec<_>, _>>() + .map_err(|error| error.to_string())?; + if certificates.is_empty() { + return Err("TLS PEM contains no certificates".into()); + } + Ok(certificates) +} + +pub(crate) async fn connect( + stream: tokio::net::TcpStream, + host: &str, + config: &ClientTls, +) -> Result<Transport, String> { + let name = ServerName::try_from(host.to_owned()).map_err(|error| error.to_string())?; + let stream = tokio_rustls::TlsConnector::from(config.0.clone()) + .connect(name, stream) + .await + .map_err(|error| error.to_string())?; + Ok(Transport::Tls(Box::new(stream.into()))) +} + +pub(crate) async fn accept( + stream: tokio::net::TcpStream, + config: Option<&ServerTls>, + millis: i64, +) -> Result<Transport, String> { + let Some(config) = config else { + return Ok(Transport::Tcp(stream)); + }; + let stream = tokio::time::timeout( + crate::task::positive_duration(millis)?, + tokio_rustls::TlsAcceptor::from(config.0.clone()).accept(stream), + ) + .await + .map_err(|_| "TLS handshake timed out".to_owned())? + .map_err(|error| error.to_string())?; + Ok(Transport::Tls(Box::new(stream.into()))) +} + +impl std::fmt::Debug for ClientTls { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("ClientTls(<resource>)") + } +} + +impl std::fmt::Debug for ServerTls { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("ServerTls(<resource>)") + } +} diff --git a/crates/dever-runtime/src/transport.rs b/crates/dever-runtime/src/transport.rs new file mode 100644 index 0000000..5336d92 --- /dev/null +++ b/crates/dever-runtime/src/transport.rs @@ -0,0 +1,72 @@ +use std::io::{self, IoSlice}; +use std::pin::Pin; +use std::task::{Context, Poll}; + +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; +use tokio::net::TcpStream; + +// TLS state is large; plain TCP keeps its direct, allocation-free representation. +pub(crate) enum Transport { + Tcp(TcpStream), + Tls(Box<tokio_rustls::TlsStream<TcpStream>>), +} + +macro_rules! dispatch { + ($value:expr, $method:ident $(, $argument:expr)*) => { + match $value { + Transport::Tcp(stream) => Pin::new(stream).$method($($argument),*), + Transport::Tls(stream) => Pin::new(stream.as_mut()).$method($($argument),*), + } + }; +} + +impl AsyncRead for Transport { + fn poll_read( + self: Pin<&mut Self>, + context: &mut Context<'_>, + buffer: &mut ReadBuf<'_>, + ) -> Poll<io::Result<()>> { + dispatch!(self.get_mut(), poll_read, context, buffer) + } +} + +impl AsyncWrite for Transport { + fn poll_write( + self: Pin<&mut Self>, + context: &mut Context<'_>, + bytes: &[u8], + ) -> Poll<io::Result<usize>> { + dispatch!(self.get_mut(), poll_write, context, bytes) + } + fn poll_write_vectored( + self: Pin<&mut Self>, + context: &mut Context<'_>, + buffers: &[IoSlice<'_>], + ) -> Poll<io::Result<usize>> { + dispatch!(self.get_mut(), poll_write_vectored, context, buffers) + } + fn is_write_vectored(&self) -> bool { + match self { + Self::Tcp(stream) => stream.is_write_vectored(), + Self::Tls(stream) => stream.is_write_vectored(), + } + } + fn poll_flush(self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll<io::Result<()>> { + dispatch!(self.get_mut(), poll_flush, context) + } + fn poll_shutdown(self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll<io::Result<()>> { + dispatch!(self.get_mut(), poll_shutdown, context) + } +} + +pub(crate) async fn connect( + host: &str, + port: i64, + tls: Option<&crate::tls::ClientTls>, +) -> Result<Transport, String> { + let stream = crate::net::connect_stream(host, port).await?; + match tls { + Some(config) => crate::tls::connect(stream, host, config).await, + None => Ok(Transport::Tcp(stream)), + } +} diff --git a/crates/dever-runtime/src/websocket.rs b/crates/dever-runtime/src/websocket.rs new file mode 100644 index 0000000..7ad0bac --- /dev/null +++ b/crates/dever-runtime/src/websocket.rs @@ -0,0 +1,328 @@ +use std::future::Future; +use std::sync::Arc; +use std::time::Duration; + +use futures_util::{ + SinkExt, StreamExt, + stream::{self, SplitSink, SplitStream}, +}; +use tokio::sync::Mutex; +use tokio_tungstenite::{ + WebSocketStream, + tungstenite::{ + self, + protocol::{CloseFrame, Role, WebSocketConfig, frame::coding::CloseCode}, + }, +}; + +use crate::transport::Transport; +use crate::{ + async_stream::AsyncStream, bytes::Bytes, endpoint::Endpoint, http::HttpReply, net, + resource::Pull, +}; + +#[derive(Clone, Copy, Debug)] +pub struct Limits { + pub message_bytes: i64, + pub idle_ms: i64, + pub write_ms: i64, +} + +impl Limits { + pub(crate) fn check(self) -> Result<WebSocketConfig, String> { + let maximum = usize::try_from(self.message_bytes) + .ok() + .filter(|value| *value > 0 && value.checked_add(14).is_some()) + .ok_or("WebSocket message_bytes must be positive and leave space for a frame header")?; + if self.idle_ms <= 0 || self.write_ms <= 0 { + return Err("WebSocket idle_ms and write_ms must be positive".into()); + } + Ok(WebSocketConfig::default() + .read_buffer_size(4096) + .write_buffer_size(0) + .max_write_buffer_size(maximum.max(125) + 14) + .max_message_size(Some(maximum)) + .max_frame_size(Some(maximum.max(125)))) + } +} + +#[derive(Clone, Debug)] +pub enum Message { + Text(String), + Binary(Bytes), + Ping(Bytes), + Pong(Bytes), +} + +type Wire = WebSocketStream<Transport>; +struct Parts { + reader: Mutex<SplitStream<Wire>>, + writer: Mutex<SplitSink<Wire, tungstenite::Message>>, +} + +#[derive(Clone)] +pub struct WebSocket { + endpoint: Arc<Endpoint<Parts>>, + limits: Limits, +} + +impl std::fmt::Debug for WebSocket { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("WebSocket(<resource>)") + } +} + +impl WebSocket { + fn new(stream: Wire, limits: Limits) -> Self { + let (writer, reader) = stream.split(); + Self { + endpoint: Arc::new(Endpoint::new(Parts { + reader: Mutex::new(reader), + writer: Mutex::new(writer), + })), + limits, + } + } + + pub(crate) fn abort(&self) { + let _closed = self.endpoint.close(); + } + + async fn operation<T>( + &self, + millis: i64, + operation: impl Future<Output = Result<T, String>>, + ) -> Result<T, String> { + tokio::time::timeout( + Duration::from_millis(millis as u64), + self.endpoint.while_open(operation), + ) + .await + .map_err(|_| "WebSocket operation timed out".to_owned())? + } +} + +// 中途取消或失败的帧写入不能重试；关闭整个传输，避免留下半帧。 +struct AbortOnDrop<'a>(Option<&'a WebSocket>); +impl Drop for AbortOnDrop<'_> { + fn drop(&mut self) { + if let Some(socket) = self.0 { + socket.abort(); + } + } +} + +pub async fn accept(reply: &HttpReply, limits: Limits) -> Result<WebSocket, String> { + reply.upgrade(limits).await +} + +pub(crate) async fn from_server( + stream: Transport, + buffered: Vec<u8>, + limits: Limits, +) -> Result<WebSocket, String> { + let stream = + WebSocketStream::from_partially_read(stream, buffered, Role::Server, Some(limits.check()?)) + .await; + Ok(WebSocket::new(stream, limits)) +} + +/// address 是数字 IP；target 是 / 开头的路径，握手、掩码与响应校验交给 tungstenite。 +pub async fn connect( + address: &str, + port: i64, + target: &str, + limits: Limits, +) -> Result<WebSocket, String> { + let config = limits.check()?; + if !target.starts_with('/') || target.contains('#') { + return Err("WebSocket target must be an origin-form path without a fragment".into()); + } + tokio::time::timeout(Duration::from_millis(limits.write_ms as u64), async { + let socket = net::connect_stream_timeout(address, port, limits.write_ms).await?; + let host = if address.contains(':') { + format!("[{address}]") + } else { + address.to_owned() + }; + let uri = format!("ws://{host}:{port}{target}"); + let (stream, _) = + tokio_tungstenite::client_async_with_config(uri, Transport::Tcp(socket), Some(config)) + .await + .map_err(|error| error.to_string())?; + Ok(WebSocket::new(stream, limits)) + }) + .await + .map_err(|_| "WebSocket handshake timed out".to_owned())? +} + +pub async fn open( + url: &str, + tls: crate::tls::ClientTls, + limits: Limits, +) -> Result<WebSocket, String> { + let config = limits.check()?; + let uri: hyper::Uri = url + .parse() + .map_err(|error| format!("invalid WebSocket URL: {error}"))?; + if url.contains('#') + || uri + .authority() + .is_some_and(|authority| authority.as_str().contains('@')) + { + return Err("WebSocket URL cannot contain credentials or a fragment".into()); + } + let secure = match uri.scheme_str() { + Some("ws") => false, + Some("wss") => true, + _ => return Err("WebSocket URL must use ws or wss".into()), + }; + let host = uri + .host() + .ok_or("WebSocket URL requires a host")? + .trim_matches(['[', ']']); + let port = i64::from(uri.port_u16().unwrap_or(if secure { 443 } else { 80 })); + tokio::time::timeout(crate::task::positive_duration(limits.write_ms)?, async { + let transport = crate::transport::connect(host, port, secure.then_some(&tls)).await?; + let (stream, _) = tokio_tungstenite::client_async_with_config(url, transport, Some(config)) + .await + .map_err(|error| error.to_string())?; + Ok(WebSocket::new(stream, limits)) + }) + .await + .map_err(|_| "WebSocket handshake timed out".to_owned())? +} + +pub async fn send(socket: &WebSocket, message: Message) -> Result<(), String> { + let message = match message { + Message::Text(text) => tungstenite::Message::Text(text.into()), + Message::Binary(bytes) => tungstenite::Message::Binary(bytes.into_http()), + Message::Ping(bytes) => tungstenite::Message::Ping(bytes.into_http()), + Message::Pong(bytes) => tungstenite::Message::Pong(bytes.into_http()), + }; + let limit = if message.is_ping() || message.is_pong() { + 125 + } else { + socket.limits.message_bytes as usize + }; + if message.len() > limit { + return Err("WebSocket message exceeds its size limit".into()); + } + let parts = socket.endpoint.get()?; + write_wire(socket, &parts, Some(message)).await +} + +async fn write_wire( + socket: &WebSocket, + parts: &Parts, + message: Option<tungstenite::Message>, +) -> Result<(), String> { + socket + .operation(socket.limits.write_ms, async { + let mut writer = parts.writer.lock().await; + // 等待方向锁时尚未写入；此时取消不应中断另一个任务正在发送的帧。 + let mut guard = AbortOnDrop(Some(socket)); + match message { + Some(message) => writer.send(message).await, + None => writer.flush().await, + } + .map_err(|error| error.to_string())?; + guard.0 = None; + Ok(()) + }) + .await +} + +pub async fn receive(socket: &WebSocket) -> Result<Option<Message>, String> { + let parts = socket.endpoint.get()?; + let incoming = socket + .operation(socket.limits.idle_ms, async { + parts + .reader + .lock() + .await + .next() + .await + .transpose() + .map_err(|error| error.to_string()) + }) + .await; + let incoming = match incoming { + Ok(incoming) => incoming, + Err(error) => { + socket.abort(); + return Err(error); + } + }; + use tungstenite::Message as WireMessage; + let message = match incoming { + Some(WireMessage::Text(text)) => Message::Text(text.to_string()), + Some(WireMessage::Binary(bytes)) => Message::Binary(Bytes::from_http(bytes)), + Some(WireMessage::Pong(bytes)) => Message::Pong(Bytes::from_http(bytes)), + Some(WireMessage::Ping(bytes)) => { + write_wire(socket, &parts, None).await?; + Message::Ping(Bytes::from_http(bytes)) + } + Some(WireMessage::Close(_)) => { + let _guard = AbortOnDrop(Some(socket)); + write_wire(socket, &parts, None).await?; + return Ok(None); + } + None => { + socket.abort(); + return Ok(None); + } + Some(WireMessage::Frame(_)) => unreachable!("tungstenite never returns raw frames"), + }; + Ok(Some(message)) +} + +pub fn messages(socket: WebSocket) -> AsyncStream<Result<Message, String>> { + AsyncStream::new(stream::unfold(socket, |socket| async move { + let value = match receive(&socket).await { + Ok(Some(message)) => Pull::Item(Ok(message)), + Ok(None) => return None, + Err(error) => Pull::Last(Err(error)), + }; + Some((value, socket)) + })) +} + +pub async fn close(socket: &WebSocket, code: i64, reason: &str) -> Result<(), String> { + let code = u16::try_from(code) + .map(CloseCode::from) + .ok() + .filter(|code| code.is_allowed()) + .ok_or("invalid WebSocket close code")?; + if reason.len() > 123 { + return Err("WebSocket close reason exceeds 123 UTF-8 bytes".into()); + } + // 先唤醒并禁止普通读写，再由此操作独占最后的传输所有权，完成关闭握手。 + let parts = socket.endpoint.take()?; + tokio::time::timeout( + Duration::from_millis(socket.limits.write_ms as u64), + async { + parts + .writer + .lock() + .await + .send(tungstenite::Message::Close(Some(CloseFrame { + code, + reason: reason.to_owned().into(), + }))) + .await + .map_err(|error| error.to_string())?; + while let Some(message) = parts.reader.lock().await.next().await { + if matches!( + message.map_err(|error| error.to_string())?, + tungstenite::Message::Close(_) + ) { + return Ok(()); + } + } + Ok(()) + }, + ) + .await + .map_err(|_| "WebSocket close handshake timed out".to_owned())? +} diff --git a/crates/dever-runtime/src/wire.rs b/crates/dever-runtime/src/wire.rs new file mode 100644 index 0000000..07bb3dc --- /dev/null +++ b/crates/dever-runtime/src/wire.rs @@ -0,0 +1,348 @@ +//! Strict wire boundary. Parsed nodes never escape into generated business signatures. +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt; + +use base64::Engine as _; +use serde::Deserializer; +use serde::de::{self, MapAccess, SeqAccess, Visitor}; +use serde_json::value::RawValue; + +pub const MAX_BYTES: usize = 16 * 1024 * 1024; +pub const MAX_DEPTH: usize = 64; +pub const MAX_ELEMENTS: usize = 65_536; + +pub struct Node<'a> { + raw: &'a str, + kind: Kind<'a>, +} + +enum Kind<'a> { + Null, + Bool(bool), + Number, + Text(String), + List(Vec<Node<'a>>), + Object(BTreeMap<String, Node<'a>>), +} + +pub fn parse(text: &str) -> Result<Node<'_>, String> { + if text.len() > MAX_BYTES { + return Err("wire JSON exceeds byte limit".into()); + } + let raw: &RawValue = serde_json::from_str(text).map_err(|error| error.to_string())?; + let mut remaining = MAX_ELEMENTS; + parse_node(raw, 0, &mut remaining) +} + +fn parse_node<'a>( + raw: &'a RawValue, + depth: usize, + remaining: &mut usize, +) -> Result<Node<'a>, String> { + if depth > MAX_DEPTH { + return Err("wire JSON exceeds depth limit".into()); + } + *remaining = remaining + .checked_sub(1) + .ok_or("wire JSON exceeds element limit")?; + let text = raw.get(); + let kind = match text.as_bytes()[0] { + b'n' => Kind::Null, + b't' => Kind::Bool(true), + b'f' => Kind::Bool(false), + b'"' => Kind::Text(serde_json::from_str(text).map_err(|error| error.to_string())?), + b'[' | b'{' => { + if depth == MAX_DEPTH { + return Err("wire JSON exceeds depth limit".into()); + } + let mut decoder = serde_json::Deserializer::from_str(text); + let kind = decoder + .deserialize_any(Container { depth, remaining }) + .map_err(|error| error.to_string())?; + decoder.end().map_err(|error| error.to_string())?; + kind + } + _ => Kind::Number, + }; + Ok(Node { raw: text, kind }) +} + +struct Container<'b> { + depth: usize, + remaining: &'b mut usize, +} + +impl<'de> Visitor<'de> for Container<'_> { + type Value = Kind<'de>; + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a bounded JSON container") + } + fn visit_seq<A: SeqAccess<'de>>(self, mut sequence: A) -> Result<Self::Value, A::Error> { + let mut values = Vec::new(); + while let Some(raw) = sequence.next_element::<&'de RawValue>()? { + values + .push(parse_node(raw, self.depth + 1, self.remaining).map_err(de::Error::custom)?); + } + Ok(Kind::List(values)) + } + fn visit_map<A: MapAccess<'de>>(self, mut object: A) -> Result<Self::Value, A::Error> { + let mut values = BTreeMap::new(); + while let Some(key) = object.next_key::<String>()? { + if values.contains_key(&key) { + return Err(de::Error::custom("duplicate JSON field")); + } + let raw = object.next_value::<&'de RawValue>()?; + values.insert( + key, + parse_node(raw, self.depth + 1, self.remaining).map_err(de::Error::custom)?, + ); + } + Ok(Kind::Object(values)) + } +} + +impl<'a> Node<'a> { + pub fn bytes(&self) -> Result<crate::bytes::Bytes, String> { + base64::engine::general_purpose::STANDARD + .decode(self.text()?) + .map(crate::bytes::Bytes::new) + .map_err(|_| "invalid wire Bytes base64".into()) + } + + pub fn is_null(&self) -> bool { + matches!(self.kind, Kind::Null) + } + pub fn raw(&self) -> &'a str { + self.raw + } + pub fn text(&self) -> Result<&str, String> { + match &self.kind { + Kind::Text(value) => Ok(value), + _ => Err("expected wire string".into()), + } + } + pub fn boolean(&self) -> Result<bool, String> { + match self.kind { + Kind::Bool(value) => Ok(value), + _ => Err("expected wire Bool".into()), + } + } + pub fn int(&self) -> Result<i64, String> { + if !matches!(self.kind, Kind::Number) { + return Err("expected wire integer".into()); + } + self.raw + .parse() + .map_err(|_| "wire integer is not an exact i64".into()) + } + pub fn float(&self) -> Result<f64, String> { + if !matches!(self.kind, Kind::Number) { + return Err("expected wire Float".into()); + } + let value: f64 = self.raw.parse().map_err(|_| "invalid wire Float")?; + if !value.is_finite() { + return Err("non-finite wire Float".into()); + } + Ok(value) + } + pub fn list(&self) -> Result<&[Node<'a>], String> { + match &self.kind { + Kind::List(values) => Ok(values), + _ => Err("expected wire List".into()), + } + } + pub fn fields(&self, names: &[&str]) -> Result<&BTreeMap<String, Node<'a>>, String> { + let fields = self.object()?; + if fields.keys().any(|name| !names.contains(&name.as_str())) { + return Err("unknown wire record field".into()); + } + Ok(fields) + } + pub fn object(&self) -> Result<&BTreeMap<String, Node<'a>>, String> { + match &self.kind { + Kind::Object(fields) => Ok(fields), + _ => Err("expected wire record".into()), + } + } +} + +/// Only validated JSON can be placed in an API envelope without re-encoding its numbers. +pub struct Encoded(String); +impl Encoded { + pub fn as_str(&self) -> &str { + &self.0 + } + pub fn null() -> Self { + Self("null".into()) + } +} + +enum Frame { + Array(usize), + Object { + names: BTreeSet<String>, + awaiting_value: bool, + }, +} + +#[derive(Default)] +pub struct Encoder { + output: String, + stack: Vec<Frame>, + elements: usize, + failed: bool, +} + +impl Encoder { + pub fn bytes(&mut self, value: &crate::bytes::Bytes) -> Result<(), String> { + let length = base64::encoded_len(value.values().len(), true) + .and_then(|length| length.checked_add(2)) + .ok_or("wire JSON exceeds byte limit")?; + if length > MAX_BYTES.saturating_sub(self.output.len()) { + return Err("wire JSON exceeds byte limit".into()); + } + self.text(&base64::engine::general_purpose::STANDARD.encode(value.values())) + } + + fn append(&mut self, text: &str) -> Result<(), String> { + if text.len() > MAX_BYTES.saturating_sub(self.output.len()) { + // Frame bookkeeping may already have advanced before an append fails. + // Such a partial write must never become a validated Encoded value. + self.failed = true; + return Err("wire JSON exceeds byte limit".into()); + } + self.output.push_str(text); + Ok(()) + } + fn value(&mut self) -> Result<(), String> { + if self.elements == MAX_ELEMENTS { + return Err("wire JSON exceeds element limit".into()); + } + self.elements += 1; + let comma = match self.stack.last_mut() { + Some(Frame::Array(count)) => { + *count += 1; + *count > 1 + } + Some(Frame::Object { awaiting_value, .. }) if *awaiting_value => { + *awaiting_value = false; + false + } + Some(Frame::Object { .. }) => return Err("wire object requires a field name".into()), + None if self.elements != 1 => return Err("wire JSON has multiple roots".into()), + None => false, + }; + if comma { + self.append(",")?; + } + Ok(()) + } + pub fn null(&mut self) -> Result<(), String> { + self.scalar("null") + } + pub fn boolean(&mut self, value: bool) -> Result<(), String> { + self.scalar(if value { "true" } else { "false" }) + } + pub fn int(&mut self, value: i64) -> Result<(), String> { + self.scalar(&value.to_string()) + } + pub fn float(&mut self, value: f64) -> Result<(), String> { + if !value.is_finite() { + return Err("non-finite wire Float".into()); + } + self.scalar(&serde_json::to_string(&value).map_err(|error| error.to_string())?) + } + pub fn text(&mut self, value: &str) -> Result<(), String> { + if value.len() > MAX_BYTES.saturating_sub(self.output.len()) { + return Err("wire JSON exceeds byte limit".into()); + } + self.scalar(&serde_json::to_string(value).map_err(|error| error.to_string())?) + } + fn scalar(&mut self, text: &str) -> Result<(), String> { + self.value()?; + self.append(text) + } + pub fn json(&mut self, text: &str) -> Result<(), String> { + if text.len() > MAX_BYTES.saturating_sub(self.output.len()) { + return Err("wire JSON exceeds byte limit".into()); + } + let raw: &RawValue = serde_json::from_str(text).map_err(|error| error.to_string())?; + let mut remaining = MAX_ELEMENTS - self.elements; + parse_node(raw, self.stack.len(), &mut remaining)?; + self.value()?; + self.elements = MAX_ELEMENTS - remaining; + self.append(raw.get()) + } + pub fn begin_array(&mut self) -> Result<(), String> { + self.begin(false) + } + pub fn begin_object(&mut self) -> Result<(), String> { + self.begin(true) + } + fn begin(&mut self, object: bool) -> Result<(), String> { + if self.stack.len() >= MAX_DEPTH { + return Err("wire JSON exceeds depth limit".into()); + } + self.value()?; + self.append(if object { "{" } else { "[" })?; + self.stack.push(if object { + Frame::Object { + names: BTreeSet::new(), + awaiting_value: false, + } + } else { + Frame::Array(0) + }); + Ok(()) + } + pub fn key(&mut self, name: &str) -> Result<(), String> { + if name.len() > MAX_BYTES.saturating_sub(self.output.len()) { + return Err("wire JSON exceeds byte limit".into()); + } + let Some(Frame::Object { + names, + awaiting_value, + }) = self.stack.last_mut() + else { + return Err("wire field outside record".into()); + }; + if *awaiting_value { + return Err("wire field has no value".into()); + } + if !names.insert(name.to_owned()) { + return Err("duplicate JSON field".into()); + } + *awaiting_value = true; + let comma = names.len() > 1; + if comma { + self.append(",")?; + } + self.append(&serde_json::to_string(name).map_err(|error| error.to_string())?)?; + self.append(":") + } + pub fn end(&mut self) -> Result<(), String> { + if matches!( + self.stack.last(), + Some(Frame::Object { + awaiting_value: true, + .. + }) + ) { + return Err("incomplete wire container".into()); + } + match self.stack.pop() { + Some(Frame::Array(_)) => self.append("]"), + Some(Frame::Object { + awaiting_value: false, + .. + }) => self.append("}"), + _ => Err("incomplete wire container".into()), + } + } + pub fn finish(self) -> Result<Encoded, String> { + if self.failed || !self.stack.is_empty() || self.elements == 0 { + return Err("incomplete wire JSON".into()); + } + Ok(Encoded(self.output)) + } +} diff --git a/crates/dever-sandbox/Cargo.toml b/crates/dever-sandbox/Cargo.toml new file mode 100644 index 0000000..a210c21 --- /dev/null +++ b/crates/dever-sandbox/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "dever-sandbox" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +publish.workspace = true + +[lints] +workspace = true + +[dependencies] +sha2.workspace = true +goblin = { version = "=0.10.7", default-features = false, features = ["elf32", "elf64", "endian_fd", "std"] } + +[target.'cfg(target_os = "linux")'.dependencies] +seccompiler = { version = "=0.5.0", default-features = false } +libc = "=0.2.189" +command-fds = "=0.3.3" +nix = { version = "=0.31.3", default-features = false } +rustix = { version = "=1.1.2", features = ["fs", "process"] } + +[[bin]] +name = "dever-sandbox-guard" +path = "src/main.rs" + +[[test]] +name = "isolation" +path = "../../test/dever-sandbox-tests/isolation.rs" diff --git a/crates/dever-sandbox/src/assets.rs b/crates/dever-sandbox/src/assets.rs new file mode 100644 index 0000000..24a9ba5 --- /dev/null +++ b/crates/dever-sandbox/src/assets.rs @@ -0,0 +1,182 @@ +//! Validate the bootstrap dependency closure before executing any asset. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::io::Read; +use std::path::Path; + +use goblin::elf::{Elf, dynamic, header, program_header}; +use sha2::{Digest, Sha256}; + +const MAX_ASSET_BYTES: usize = 32 * 1024 * 1024; +const MAX_ASSET_FILES: usize = 128; + +/// The first executable must be independent of every host loader and library. +pub fn validate_bwrap(bytes: &[u8]) -> Result<String, String> { + validate_bwrap_for_target(bytes, host_target()) +} + +fn host_target() -> &'static str { + if cfg!(target_arch = "aarch64") { + "linux-aarch64" + } else { + "linux-x86_64" + } +} + +fn target_abi(target: &str) -> Result<(u16, &'static str, &'static str), String> { + match target { + "linux-x86_64" => Ok(( + header::EM_X86_64, + "ld-linux-x86-64.so.2", + "/lib64/ld-linux-x86-64.so.2", + )), + "linux-aarch64" => Ok(( + header::EM_AARCH64, + "ld-linux-aarch64.so.1", + "/lib/ld-linux-aarch64.so.1", + )), + _ => Err(format!("unsupported sandbox target '{target}'")), + } +} + +pub fn validate_bwrap_for_target(bytes: &[u8], target: &str) -> Result<String, String> { + let elf = Elf::parse(bytes).map_err(|error| format!("invalid sandbox bwrap ELF: {error}"))?; + let (machine, _, _) = target_abi(target)?; + if bytes.len() > MAX_ASSET_BYTES + || !elf.is_64 + || !elf.little_endian + || elf.header.e_machine != machine + || !matches!(elf.header.e_type, header::ET_DYN | header::ET_EXEC) + || elf + .program_headers + .iter() + .any(|segment| segment.p_type == program_header::PT_INTERP) + || elf.dynamic.as_ref().is_some_and(|table| { + table + .dyns + .iter() + .any(|entry| entry.d_tag == dynamic::DT_NEEDED) + }) + || !elf.rpaths.is_empty() + || !elf.runpaths.is_empty() + { + return Err("sandbox bwrap must be a static target ELF without PT_INTERP, DT_NEEDED or library search paths".into()); + } + Ok(format!("{:x}", Sha256::digest(bytes))) +} + +/// Used both before signing a pack and when launching its verified extraction. +/// Content authentication belongs to the release/bundle owner; this validates +/// that the loader can resolve every dependency without a host-library search. +pub fn validate_assets(files: &[(String, Vec<u8>)]) -> Result<(), String> { + validate_assets_for_target(files, host_target()) +} + +pub fn validate_assets_for_target(files: &[(String, Vec<u8>)], target: &str) -> Result<(), String> { + let (machine, loader_name, interpreter) = target_abi(target)?; + let total = files + .iter() + .try_fold(0usize, |total, (_, bytes)| total.checked_add(bytes.len())) + .ok_or("sandbox asset size overflow")?; + if files.len() > MAX_ASSET_FILES || total > MAX_ASSET_BYTES { + return Err("sandbox assets exceed their file or byte budget".into()); + } + let mut supplied = BTreeMap::new(); + for (name, bytes) in files { + let valid = matches!(name.as_str(), "bin/bwrap" | "bin/guard") + || name.strip_prefix("lib/").is_some_and(|name| { + !name.is_empty() && name != "." && name != ".." && !name.contains(['/', '\\', '\0']) + }); + if !valid || supplied.insert(name.as_str(), bytes.as_slice()).is_some() { + return Err("sandbox asset has an invalid or repeated path".into()); + } + } + let loader = format!("lib/{loader_name}"); + for required in ["bin/bwrap", "bin/guard", &loader] { + if !supplied.contains_key(required) { + return Err(format!("sandbox asset '{required}' is missing")); + } + } + validate_bwrap_for_target(supplied["bin/bwrap"], target)?; + let mut reached = BTreeSet::new(); + // The shared OS ABI set also serves packaged language runtimes. Validate + // every supplied library even when the two bootstrap helpers do not use it. + let mut remaining = supplied + .keys() + .map(|name| (*name).to_owned()) + .collect::<Vec<_>>(); + while let Some(name) = remaining.pop() { + if !reached.insert(name.clone()) { + continue; + } + let bytes = supplied + .get(name.as_str()) + .ok_or_else(|| format!("sandbox dependency '{name}' is missing"))?; + let elf = + Elf::parse(bytes).map_err(|error| format!("invalid sandbox ELF asset: {error}"))?; + if !elf.is_64 + || !elf.little_endian + || elf.header.e_machine != machine + || !matches!(elf.header.e_type, header::ET_DYN | header::ET_EXEC) + { + return Err("sandbox ELF asset has an incompatible target or object type".into()); + } + // Reject paths that can outrank the explicit bootstrap library path. + if !elf.rpaths.is_empty() || !elf.runpaths.is_empty() { + return Err("sandbox bootstrap assets must not declare RPATH or RUNPATH".into()); + } + if let Some(actual) = elf.interpreter + && actual != interpreter + { + return Err("sandbox ELF interpreter differs from the packaged loader".into()); + } + for library in elf.libraries { + if library.is_empty() || library.contains(['/', '\\']) || matches!(library, "." | "..") + { + return Err("sandbox dependency is not a plain library name".into()); + } + remaining.push(format!("lib/{library}")); + } + } + Ok(()) +} + +pub(super) fn verify_directory(root: &Path) -> Result<String, String> { + let mut files = Vec::new(); + let mut remaining = MAX_ASSET_BYTES; + for directory in ["bin", "lib"] { + let directory_path = super::real_path(&root.join(directory))?; + for entry in fs::read_dir(directory_path).map_err(|error| error.to_string())? { + let entry = entry.map_err(|error| error.to_string())?; + let path = super::real_path(&entry.path())?; + if !entry + .file_type() + .map_err(|error| error.to_string())? + .is_file() + || files.len() >= MAX_ASSET_FILES + { + return Err("sandbox assets contain a non-regular file or too many files".into()); + } + let mut bytes = Vec::new(); + fs::File::open(path) + .and_then(|file| file.take(remaining as u64 + 1).read_to_end(&mut bytes)) + .map_err(|error| format!("cannot read sandbox asset: {error}"))?; + remaining = remaining + .checked_sub(bytes.len()) + .ok_or("sandbox assets exceed their byte budget")?; + let name = entry + .file_name() + .into_string() + .map_err(|_| "sandbox asset name is not UTF-8")?; + files.push((format!("{directory}/{name}"), bytes)); + } + } + validate_assets(&files)?; + let bytes = &files + .iter() + .find(|(name, _)| name == "bin/bwrap") + .ok_or("sandbox bwrap is missing")? + .1; + Ok(format!("{:x}", Sha256::digest(bytes))) +} diff --git a/crates/dever-sandbox/src/filter.rs b/crates/dever-sandbox/src/filter.rs new file mode 100644 index 0000000..d9ae810 --- /dev/null +++ b/crates/dever-sandbox/src/filter.rs @@ -0,0 +1,322 @@ +//! Fixed allowlist for packaged Linux runtimes. Mounts own file visibility; +//! this filter owns syscall categories that could bypass that visibility. + +use super::Capabilities; +use seccompiler::{ + BpfProgram, SeccompAction, SeccompCmpArgLen, SeccompCmpOp, SeccompCondition, SeccompFilter, + SeccompRule, +}; +use std::collections::BTreeMap; + +fn condition(argument: u8, op: SeccompCmpOp, value: u64) -> Result<SeccompCondition, String> { + SeccompCondition::new(argument, SeccompCmpArgLen::Qword, op, value) + .map_err(|error| error.to_string()) +} + +fn rule(conditions: Vec<SeccompCondition>) -> Result<SeccompRule, String> { + SeccompRule::new(conditions).map_err(|error| error.to_string()) +} + +fn install( + rules: BTreeMap<i64, Vec<SeccompRule>>, + default: SeccompAction, + matched: SeccompAction, +) -> Result<(), String> { + let architecture = std::env::consts::ARCH + .try_into() + .map_err(|error: seccompiler::BackendError| error.to_string())?; + let filter = SeccompFilter::new(rules, default, matched, architecture) + .map_err(|error| error.to_string())?; + let program: BpfProgram = filter + .try_into() + .map_err(|error: seccompiler::BackendError| error.to_string())?; + seccompiler::apply_filter(&program) + .map_err(|error| format!("cannot enforce Worker syscall policy: {error}")) +} + +pub(super) fn apply(capabilities: Capabilities) -> Result<(), String> { + // clone3 carries flags behind a pointer. ENOSYS makes supported libc use + // clone, whose scalar flags can be checked without user-memory races. + install( + BTreeMap::from([(libc::SYS_clone3, vec![])]), + SeccompAction::Allow, + SeccompAction::Errno(libc::ENOSYS as u32), + )?; + let mut rules = BTreeMap::new(); + for syscall in [ + libc::SYS_read, + libc::SYS_write, + libc::SYS_readv, + libc::SYS_writev, + libc::SYS_pread64, + libc::SYS_pwrite64, + libc::SYS_preadv, + libc::SYS_pwritev, + libc::SYS_close, + libc::SYS_close_range, + libc::SYS_lseek, + libc::SYS_fcntl, + libc::SYS_dup, + libc::SYS_dup3, + libc::SYS_pipe2, + libc::SYS_openat, + libc::SYS_fstat, + libc::SYS_statx, + libc::SYS_faccessat, + libc::SYS_faccessat2, + libc::SYS_readlinkat, + // copy2/copytree inspect extended metadata on their visible source. + // These calls only read mounted files; writes remain independently + // governed by the read-only inputs and explicit writable grants. + libc::SYS_listxattr, + libc::SYS_llistxattr, + libc::SYS_flistxattr, + libc::SYS_getxattr, + libc::SYS_lgetxattr, + libc::SYS_fgetxattr, + libc::SYS_getdents64, + libc::SYS_getcwd, + libc::SYS_chdir, + libc::SYS_fchdir, + libc::SYS_mkdirat, + libc::SYS_unlinkat, + libc::SYS_renameat, + libc::SYS_renameat2, + libc::SYS_linkat, + libc::SYS_symlinkat, + libc::SYS_ftruncate, + libc::SYS_truncate, + libc::SYS_fchmod, + libc::SYS_fchmodat, + libc::SYS_fchown, + libc::SYS_fchownat, + libc::SYS_flock, + libc::SYS_fsync, + libc::SYS_fdatasync, + libc::SYS_sync_file_range, + libc::SYS_fallocate, + libc::SYS_fstatfs, + libc::SYS_statfs, + libc::SYS_umask, + libc::SYS_utimensat, + libc::SYS_sendfile, + libc::SYS_copy_file_range, + libc::SYS_mmap, + libc::SYS_mprotect, + libc::SYS_munmap, + libc::SYS_mremap, + libc::SYS_madvise, + libc::SYS_brk, + libc::SYS_msync, + libc::SYS_rt_sigaction, + libc::SYS_rt_sigprocmask, + libc::SYS_rt_sigreturn, + libc::SYS_rt_sigsuspend, + libc::SYS_rt_sigtimedwait, + libc::SYS_sigaltstack, + libc::SYS_tgkill, + libc::SYS_kill, + libc::SYS_getpid, + libc::SYS_getppid, + libc::SYS_gettid, + libc::SYS_getuid, + libc::SYS_geteuid, + libc::SYS_getgid, + libc::SYS_getegid, + libc::SYS_getgroups, + libc::SYS_getpgid, + libc::SYS_getsid, + libc::SYS_setpgid, + libc::SYS_setsid, + libc::SYS_set_tid_address, + libc::SYS_set_robust_list, + libc::SYS_get_robust_list, + libc::SYS_futex, + libc::SYS_rseq, + libc::SYS_sched_yield, + libc::SYS_sched_getaffinity, + libc::SYS_sched_getparam, + libc::SYS_sched_getscheduler, + libc::SYS_clock_gettime, + libc::SYS_clock_getres, + libc::SYS_clock_nanosleep, + libc::SYS_nanosleep, + libc::SYS_gettimeofday, + libc::SYS_times, + libc::SYS_getrusage, + libc::SYS_getrlimit, + libc::SYS_prlimit64, + libc::SYS_uname, + libc::SYS_sysinfo, + libc::SYS_getrandom, + libc::SYS_epoll_create1, + libc::SYS_epoll_ctl, + libc::SYS_epoll_pwait, + libc::SYS_epoll_pwait2, + libc::SYS_ppoll, + libc::SYS_pselect6, + libc::SYS_eventfd2, + libc::SYS_timerfd_create, + libc::SYS_timerfd_settime, + libc::SYS_timerfd_gettime, + libc::SYS_inotify_init1, + libc::SYS_inotify_add_watch, + libc::SYS_inotify_rm_watch, + libc::SYS_recvfrom, + libc::SYS_sendto, + libc::SYS_recvmsg, + libc::SYS_sendmsg, + libc::SYS_recvmmsg, + libc::SYS_sendmmsg, + libc::SYS_shutdown, + libc::SYS_getsockopt, + libc::SYS_setsockopt, + libc::SYS_getsockname, + libc::SYS_getpeername, + libc::SYS_execve, + libc::SYS_exit, + libc::SYS_exit_group, + libc::SYS_wait4, + libc::SYS_waitid, + libc::SYS_clone3, + ] { + rules.insert(syscall, vec![]); + } + #[cfg(target_arch = "x86_64")] + for syscall in [ + libc::SYS_arch_prctl, + libc::SYS_open, + libc::SYS_stat, + libc::SYS_lstat, + libc::SYS_newfstatat, + libc::SYS_access, + libc::SYS_readlink, + libc::SYS_pipe, + libc::SYS_dup2, + libc::SYS_poll, + libc::SYS_select, + libc::SYS_epoll_wait, + libc::SYS_epoll_create, + libc::SYS_mkdir, + libc::SYS_rmdir, + libc::SYS_unlink, + libc::SYS_rename, + libc::SYS_link, + libc::SYS_symlink, + libc::SYS_chmod, + libc::SYS_chown, + libc::SYS_time, + libc::SYS_alarm, + libc::SYS_getpgrp, + ] { + rules.insert(syscall, vec![]); + } + #[cfg(target_arch = "aarch64")] + { + rules.insert(libc::SYS_newfstatat, vec![]); + } + + // New namespaces and CLONE_PARENT could escape the owned PID lifecycle. + let forbidden_clone = (libc::CLONE_NEWCGROUP + | libc::CLONE_NEWIPC + | libc::CLONE_NEWNET + | libc::CLONE_NEWNS + | libc::CLONE_NEWPID + | libc::CLONE_NEWUSER + | libc::CLONE_NEWUTS + | libc::CLONE_PARENT + | libc::CLONE_UNTRACED) as u64; + let mut clone = vec![condition(0, SeccompCmpOp::MaskedEq(forbidden_clone), 0)?]; + if !capabilities.process { + let required = (libc::CLONE_THREAD | libc::CLONE_VM | libc::CLONE_SIGHAND) as u64; + clone.push(condition( + 0, + SeccompCmpOp::MaskedEq(required | 255), + required, + )?); + } + rules.insert(libc::SYS_clone, vec![rule(clone)?]); + if capabilities.process { + // POSIX_SPAWN_RESETIDS restores the caller's real identity before exec. + // Permit that exact reset only; real/saved IDs and group membership + // cannot change. Linux uid_t/gid_t consume the low 32 argument bits. + for (syscall, identity) in [ + (libc::SYS_setresuid, rustix::process::getuid().as_raw()), + (libc::SYS_setresgid, rustix::process::getgid().as_raw()), + ] { + let conditions = [u32::MAX, identity, u32::MAX] + .into_iter() + .enumerate() + .map(|(argument, value)| { + SeccompCondition::new( + argument as u8, + SeccompCmpArgLen::Dword, + SeccompCmpOp::Eq, + u64::from(value), + ) + .map_err(|error| error.to_string()) + }) + .collect::<Result<Vec<_>, _>>()?; + rules.insert(syscall, vec![rule(conditions)?]); + } + } + #[cfg(target_arch = "x86_64")] + if capabilities.process { + rules.insert(libc::SYS_fork, vec![]); + rules.insert(libc::SYS_vfork, vec![]); + } + // Anonymous socket pairs cannot reach another process. They are needed by + // language runtime event loops even without the network capability. + rules.insert( + libc::SYS_socketpair, + vec![rule(vec![condition( + 0, + SeccompCmpOp::Eq, + libc::AF_UNIX as u64, + )?])?], + ); + if capabilities.network { + let mut sockets = Vec::new(); + for domain in [libc::AF_INET, libc::AF_INET6] { + for kind in [libc::SOCK_STREAM, libc::SOCK_DGRAM] { + sockets.push(rule(vec![ + condition(0, SeccompCmpOp::Eq, domain as u64)?, + condition( + 1, + SeccompCmpOp::MaskedEq( + !(libc::SOCK_NONBLOCK | libc::SOCK_CLOEXEC) as u32 as u64, + ), + kind as u64, + )?, + ])?); + } + } + rules.insert(libc::SYS_socket, sockets); + for syscall in [ + libc::SYS_connect, + libc::SYS_bind, + libc::SYS_listen, + libc::SYS_accept, + libc::SYS_accept4, + ] { + rules.insert(syscall, vec![]); + } + } + // ioctl truncates request to unsigned int. Compare that same width so a + // caller cannot hide a forbidden request behind nonzero upper bits. + let terminal_requests = [libc::TIOCSTI, libc::TIOCLINUX] + .into_iter() + .map(|request| { + SeccompCondition::new(1, SeccompCmpArgLen::Dword, SeccompCmpOp::Ne, request) + .map_err(|error| error.to_string()) + }) + .collect::<Result<Vec<_>, _>>()?; + rules.insert(libc::SYS_ioctl, vec![rule(terminal_requests)?]); + // The allowlist excludes namespace, mount, ptrace/process_vm, io_uring, + // device creation, kernel administration, BPF and all x32 syscall numbers. + install( + rules, + SeccompAction::Errno(libc::EPERM as u32), + SeccompAction::Allow, + ) +} diff --git a/crates/dever-sandbox/src/lib.rs b/crates/dever-sandbox/src/lib.rs new file mode 100644 index 0000000..306b258 --- /dev/null +++ b/crates/dever-sandbox/src/lib.rs @@ -0,0 +1,321 @@ +//! The fixed Linux execution boundary shared by Lib runtime and build owners. +//! Callers verify signed assets and own every child; this crate never discovers tools. + +use std::collections::BTreeSet; +use std::ffi::OsString; +use std::path::{Component, Path, PathBuf}; +use std::process::Command; + +mod assets; +mod mounts; +mod observation; +mod trusted_helper; +pub use assets::{ + validate_assets, validate_assets_for_target, validate_bwrap, validate_bwrap_for_target, +}; +pub use observation::CommandNamespace; +pub use trusted_helper::{APPARMOR_PROFILE, TRUSTED_HELPER_ROOT, validate_host_policy}; + +/// Executable modes are part of the bootstrap resource contract. The private +/// loader is needed by the guard and packaged language runtimes. Authoring and +/// embedding can run on a different host; target validation belongs to assets. +pub fn asset_executable(path: &str) -> bool { + matches!( + path, + "bin/bwrap" | "bin/guard" | "lib/ld-linux-x86-64.so.2" | "lib/ld-linux-aarch64.so.1" + ) +} + +#[cfg(target_os = "linux")] +mod filter; + +#[derive(Clone, Copy, Debug, Default)] +pub struct Capabilities { + pub network: bool, + pub process: bool, +} + +/// The source is a parent-validated deployment grant, never a Worker request. +#[derive(Debug)] +pub struct Grant { + pub source: PathBuf, + pub destination: PathBuf, + pub writable: bool, +} + +pub struct Launch<'a> { + pub assets: &'a Path, + pub worker: &'a Path, + pub executable: &'a Path, + pub arguments: &'a [OsString], + pub working_directory: &'a Path, + pub capabilities: Capabilities, + pub grants: &'a [Grant], +} + +/// Guard exec confirmation and PID-namespace lifetime, separate from user I/O. +pub struct CommandObservation { + pub startup: std::fs::File, + pub namespace: std::fs::File, + pub release: std::fs::File, +} + +/// A private ELF closure cannot honor additional host preload libraries. +/// Applies to compiler launches as well as the sandbox bootstrap. +pub fn require_no_loader_preload() -> Result<(), String> { + match std::fs::symlink_metadata("/etc/ld.so.preload") { + Ok(_) => { + Err("private executable does not support host loader preload configuration".into()) + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(format!( + "cannot inspect host loader preload configuration: {error}" + )), + } +} + +impl Launch<'_> { + pub fn command(&self) -> Result<Command, String> { + self.command_with_tools(None, None) + } + + #[cfg(target_os = "linux")] + pub fn observed_command(&self) -> Result<(Command, CommandObservation), String> { + use std::os::fd::OwnedFd; + let (startup, startup_writer) = std::io::pipe().map_err(|error| error.to_string())?; + let (namespace, namespace_writer) = std::io::pipe().map_err(|error| error.to_string())?; + let (blocked, release) = std::io::pipe().map_err(|error| error.to_string())?; + let command = self.command_with_tools( + None, + Some([ + std::fs::File::from(OwnedFd::from(startup_writer)), + std::fs::File::from(OwnedFd::from(namespace_writer)), + std::fs::File::from(OwnedFd::from(blocked)), + ]), + )?; + Ok(( + command, + CommandObservation { + startup: OwnedFd::from(startup).into(), + namespace: OwnedFd::from(namespace).into(), + release: OwnedFd::from(release).into(), + }, + )) + } + + #[cfg(not(target_os = "linux"))] + pub fn observed_command(&self) -> Result<(Command, CommandObservation), String> { + Err("external command sandbox is not supported on this platform".into()) + } + + /// Only the explicit package-build owner may supply this verified signed + /// root. Ordinary Worker launches never acquire a compiler or /usr mount. + pub fn command_for_build(&self, tool_root: &Path) -> Result<Command, String> { + if self.capabilities.network || !self.capabilities.process { + return Err( + "package builds require isolated network and child-process capability".into(), + ); + } + self.command_with_tools(Some(tool_root), None) + } + + fn command_with_tools( + &self, + tool_root: Option<&Path>, + observation: Option<[std::fs::File; 3]>, + ) -> Result<Command, String> { + if !cfg!(all( + target_os = "linux", + any(target_arch = "x86_64", target_arch = "aarch64") + )) { + return Err("external Worker sandbox is not supported on this platform".into()); + } + let assets = real_path(self.assets)?; + let bwrap_hash = assets::verify_directory(&assets)?; + require_no_loader_preload()?; + let worker = real_path(self.worker)?; + let tool_root = tool_root.map(real_path).transpose()?; + let executable = mapped_path(&worker, self.executable)?; + let working_directory = mapped_path(&worker, self.working_directory)?; + let loader = assets.join("lib").join(loader_name()); + for file in [ + &loader, + &assets.join("bin/bwrap"), + &assets.join("bin/guard"), + ] { + if !real_path(file)?.is_file() { + return Err("sandbox executable asset is not a regular file".into()); + } + } + // A direct static exec is required for the trusted AppArmor attachment. + let bwrap = trusted_helper::executable(&assets, &bwrap_hash)?; + let mut command = Command::new(bwrap); + command.env_clear().args([ + "--unshare-all", + "--die-with-parent", + "--new-session", + "--cap-drop", + "ALL", + ]); + if self.capabilities.network { + command.arg("--share-net"); + } + let mut mounts = mounts::Mounts::default(); + let startup_fd = if let Some([startup, namespace, blocked]) = observation { + let startup = mounts.pass(startup)?; + let namespace = mounts.pass(namespace)?; + let blocked = mounts.pass(blocked)?; + command + .arg("--info-fd") + .arg(namespace.to_string()) + .arg("--block-fd") + .arg(blocked.to_string()); + Some(startup) + } else { + None + }; + mounts.bind(&mut command, &assets, Path::new("/sandbox"), false)?; + mounts.bind(&mut command, &assets.join("lib"), Path::new("/lib"), false)?; + command.args(["--symlink", "lib", "/lib64"]); + mounts.bind(&mut command, &worker, Path::new("/worker"), false)?; + if let Some(root) = &tool_root { + mounts.bind(&mut command, &root.join("usr"), Path::new("/usr"), false)?; + command.args(["--symlink", "usr/bin", "/bin"]); + } + command.args([ + "--proc", "/proc", "--dev", "/dev", "--tmpfs", "/tmp", "--dir", "/data", + ]); + if self.capabilities.network { + // DNS is an OS service input, never application configuration. + // Expose only these bounded resolver files, not the host's /etc. + for name in ["resolv.conf", "hosts"] { + let source = Path::new("/etc") + .join(name) + .canonicalize() + .map_err(|error| format!("cannot resolve network resolver input: {error}"))?; + let source = real_path(&source)?; + let metadata = std::fs::metadata(&source).map_err(|error| error.to_string())?; + if !metadata.is_file() || metadata.len() > 65_536 { + return Err("network resolver input is not a bounded regular file".into()); + } + mounts.bind(&mut command, &source, &Path::new("/etc").join(name), false)?; + } + } + let mut destinations = BTreeSet::new(); + for grant in self.grants { + let source = real_path(&grant.source)?; + if grant.writable + && [&assets, &worker] + .into_iter() + .chain(tool_root.iter()) + .any(|protected| { + source.starts_with(protected) || protected.starts_with(&source) + }) + { + return Err("sandbox writable grant overlaps executable inputs".into()); + } + let destination = &grant.destination; + normalized_absolute(destination)?; + if !destination.starts_with("/data") || destination == Path::new("/data") { + return Err("sandbox file grant must have a path below /data".into()); + } + if destinations.iter().any(|prior: &&Path| { + prior.starts_with(destination) || destination.starts_with(prior) + }) { + return Err("sandbox file grants overlap".into()); + } + destinations.insert(destination.as_path()); + mounts.bind(&mut command, &source, destination, grant.writable)?; + } + let flags = + u8::from(self.capabilities.network) | (u8::from(self.capabilities.process) << 1); + command + .arg("--chdir") + .arg(working_directory) + .args(["--", "/sandbox/bin/guard"]); + if let Some(fd) = startup_fd { + command.arg("--observe").arg(fd.to_string()); + } + command.arg(flags.to_string()).arg(executable); + for argument in self.arguments { + // Only resource arguments are absolute; fixed flags stay literal. + // Other absolute arguments cannot silently expose a host pathname. + let path = Path::new(argument); + command.arg(if path.is_absolute() { + self.argument_path(path)?.into_os_string() + } else { + argument.clone() + }); + } + mounts.attach(&mut command)?; + Ok(command) + } + + /// Build manifests need the same names as command arguments. Output files + /// may not exist yet; their owning mount was already validated at launch. + pub fn argument_path(&self, path: &Path) -> Result<PathBuf, String> { + normalized_absolute(path)?; + for grant in self.grants { + if let Ok(relative) = path.strip_prefix(&grant.source) { + return Ok(grant.destination.join(relative)); + } + } + let relative = path + .strip_prefix(self.worker) + .map_err(|_| "sandbox argument escapes its owned mounts")?; + Ok(Path::new("/worker").join(relative)) + } +} + +pub fn loader_name() -> &'static str { + if cfg!(target_arch = "aarch64") { + "ld-linux-aarch64.so.1" + } else { + "ld-linux-x86-64.so.2" + } +} + +fn normalized_absolute(path: &Path) -> Result<(), String> { + if !path.is_absolute() + || path + .components() + .any(|part| !matches!(part, Component::RootDir | Component::Normal(_))) + { + return Err("sandbox path must be absolute and normalized".into()); + } + Ok(()) +} + +fn real_path(path: &Path) -> Result<PathBuf, String> { + normalized_absolute(path)?; + for ancestor in path.ancestors() { + let metadata = std::fs::symlink_metadata(ancestor) + .map_err(|error| format!("cannot inspect sandbox input: {error}"))?; + if metadata.file_type().is_symlink() { + return Err("sandbox input must not contain symbolic links".into()); + } + } + Ok(path.to_owned()) +} + +fn mapped_path(worker: &Path, path: &Path) -> Result<PathBuf, String> { + let path = real_path(path)?; + let relative = path + .strip_prefix(worker) + .map_err(|_| "sandbox execution path escapes the Worker tree")?; + Ok(Path::new("/worker").join(relative)) +} + +/// Called only by the single-threaded guard immediately before exec. The filter +/// is inherited by exec and every subsequently created thread or child. +pub fn restrict(capabilities: Capabilities) -> Result<(), String> { + #[cfg(target_os = "linux")] + { + filter::apply(capabilities) + } + #[cfg(not(target_os = "linux"))] + { + let _ = capabilities; + Err("sandbox is not supported on this platform".into()) + } +} diff --git a/crates/dever-sandbox/src/main.rs b/crates/dever-sandbox/src/main.rs new file mode 100644 index 0000000..8055886 --- /dev/null +++ b/crates/dever-sandbox/src/main.rs @@ -0,0 +1,79 @@ +//! Private signed helper. It accepts a fixed capability mask, never filter code. + +#[cfg(target_os = "linux")] +fn execute() -> Result<(), String> { + use std::io::Write; + use std::os::unix::process::CommandExt; + let mut arguments = std::env::args_os().skip(1); + let first = arguments.next(); + let mut observation = if first.as_deref() == Some(std::ffi::OsStr::new("--observe")) { + let fd = arguments + .next() + .and_then(|value| value.to_str().and_then(|value| value.parse::<i32>().ok())) + .filter(|fd| *fd >= 3) + .ok_or("invalid sandbox observation descriptor")?; + // Opening proc creates a safe owned CLOEXEC duplicate. Close the + // inherited raw descriptor before exec so neither channel leaks. + let file = std::fs::OpenOptions::new() + .write(true) + .open(format!("/proc/self/fd/{fd}")) + .map_err(|error| format!("cannot open sandbox observation: {error}"))?; + nix::unistd::close(fd) + .map_err(|error| format!("cannot close inherited observation: {error}"))?; + Some(file) + } else { + None + }; + let first = if observation.is_some() { + arguments.next() + } else { + first + }; + let result = (|| { + let flags = first + .and_then(|value| value.to_str().and_then(|value| value.parse::<u8>().ok())) + .filter(|value| *value <= 3) + .ok_or("invalid sandbox capability mask")?; + let executable = arguments.next().ok_or("missing sandbox executable")?; + if !std::path::Path::new(&executable).starts_with("/worker") { + return Err("sandbox executable must be inside its Worker tree".into()); + } + let mut command = std::process::Command::new(executable); + command.args(arguments).env_clear(); + dever_sandbox::restrict(dever_sandbox::Capabilities { + network: flags & 1 != 0, + process: flags & 2 != 0, + })?; + if let Some(observation) = &mut observation { + observation + .write_all(b"R") + .map_err(|error| format!("cannot confirm sandbox guard: {error}"))?; + } + Err(format!( + "cannot execute sandboxed Worker: {}", + command.exec() + )) + })(); + if let (Some(observation), Err(error)) = (&mut observation, &result) { + observation + .write_all(b"E") + .and_then(|_| observation.write_all(error.as_bytes())) + .map_err(|write| format!("{error}; cannot report sandbox startup: {write}"))?; + } + result +} + +#[cfg(not(target_os = "linux"))] +fn execute() -> Result<(), String> { + Err("sandbox is not supported on this platform".into()) +} + +fn main() -> std::process::ExitCode { + match execute() { + Ok(()) => std::process::ExitCode::SUCCESS, + Err(error) => { + eprintln!("{error}"); + std::process::ExitCode::FAILURE + } + } +} diff --git a/crates/dever-sandbox/src/mounts.rs b/crates/dever-sandbox/src/mounts.rs new file mode 100644 index 0000000..4afca7d --- /dev/null +++ b/crates/dever-sandbox/src/mounts.rs @@ -0,0 +1,87 @@ +//! Pin deployment grants until bwrap has mounted the exact checked inode. + +use std::path::Path; +use std::process::Command; + +#[derive(Default)] +pub(super) struct Mounts { + #[cfg(target_os = "linux")] + descriptors: Vec<command_fds::FdMapping>, +} + +impl Mounts { + #[cfg(target_os = "linux")] + pub(super) fn pass(&mut self, file: std::fs::File) -> Result<i32, String> { + let child_fd = + i32::try_from(self.descriptors.len()).map_err(|_| "too many sandbox descriptors")? + 3; + self.descriptors.push(command_fds::FdMapping { + parent_fd: file.into(), + child_fd, + }); + Ok(child_fd) + } + + #[cfg(not(target_os = "linux"))] + pub(super) fn pass(&mut self, _: std::fs::File) -> Result<i32, String> { + Err("sandbox descriptors are not supported on this platform".into()) + } + + #[cfg(target_os = "linux")] + pub(super) fn bind( + &mut self, + command: &mut Command, + source: &Path, + destination: &Path, + writable: bool, + ) -> Result<(), String> { + use rustix::fs::{Mode, OFlags, ResolveFlags, openat2}; + let descriptor = openat2( + rustix::fs::CWD, + source, + OFlags::PATH | OFlags::CLOEXEC, + Mode::empty(), + ResolveFlags::NO_SYMLINKS | ResolveFlags::NO_MAGICLINKS, + ) + .map_err(|error| format!("cannot pin sandbox mount without following links: {error}"))?; + let child_fd = + i32::try_from(self.descriptors.len()).map_err(|_| "too many sandbox mounts")? + 3; + command + .arg(if writable { + "--bind-fd" + } else { + "--ro-bind-fd" + }) + .arg(child_fd.to_string()) + .arg(destination); + self.descriptors.push(command_fds::FdMapping { + parent_fd: descriptor, + child_fd, + }); + Ok(()) + } + + #[cfg(target_os = "linux")] + pub(super) fn attach(self, command: &mut Command) -> Result<(), String> { + use command_fds::CommandFdExt; + command + .fd_mappings(self.descriptors) + .map_err(|error| format!("cannot pass sandbox mount descriptors: {error}"))?; + Ok(()) + } + + #[cfg(not(target_os = "linux"))] + pub(super) fn bind( + &mut self, + _: &mut Command, + _: &Path, + _: &Path, + _: bool, + ) -> Result<(), String> { + Err("sandbox mounts are not supported on this platform".into()) + } + + #[cfg(not(target_os = "linux"))] + pub(super) fn attach(self, _: &mut Command) -> Result<(), String> { + Err("sandbox mounts are not supported on this platform".into()) + } +} diff --git a/crates/dever-sandbox/src/observation.rs b/crates/dever-sandbox/src/observation.rs new file mode 100644 index 0000000..845d10b --- /dev/null +++ b/crates/dever-sandbox/src/observation.rs @@ -0,0 +1,48 @@ +//! A pinned namespace init is the completion boundary for all descendants. +#[cfg(target_os = "linux")] +use std::os::fd::{AsRawFd, RawFd}; + +pub struct CommandNamespace { + #[cfg(target_os = "linux")] + descriptor: std::fs::File, +} + +impl CommandNamespace { + #[cfg(target_os = "linux")] + pub fn pin(pid: i32, wrapper: u32) -> Result<Self, String> { + use rustix::process::{Pid, PidfdFlags, pidfd_open}; + let pid = Pid::from_raw(pid).ok_or("invalid sandbox namespace PID")?; + let descriptor = pidfd_open(pid, PidfdFlags::NONBLOCK) + .map_err(|error| format!("cannot pin sandbox namespace: {error}"))?; + // --block-fd holds the namespace before guard/target execution. If + // setup failed and its PID was reused, do not acquire authority over + // that process: only this bwrap's direct child belongs to this launch. + let status = std::fs::read_to_string(format!("/proc/{}/status", pid.as_raw_pid())) + .map_err(|error| format!("cannot verify sandbox namespace parent: {error}"))?; + let parent = status + .lines() + .find_map(|line| line.strip_prefix("PPid:")) + .and_then(|value| value.trim().parse::<u32>().ok()); + if parent != Some(wrapper) { + return Err("sandbox namespace no longer belongs to its launcher".into()); + } + Ok(Self { + descriptor: descriptor.into(), + }) + } + + #[cfg(target_os = "linux")] + pub fn terminate(&self) -> Result<(), String> { + match rustix::process::pidfd_send_signal(&self.descriptor, rustix::process::Signal::KILL) { + Ok(()) | Err(rustix::io::Errno::SRCH) => Ok(()), + Err(error) => Err(format!("cannot stop owned command namespace: {error}")), + } + } +} + +#[cfg(target_os = "linux")] +impl AsRawFd for CommandNamespace { + fn as_raw_fd(&self) -> RawFd { + self.descriptor.as_raw_fd() + } +} diff --git a/crates/dever-sandbox/src/trusted_helper.rs b/crates/dever-sandbox/src/trusted_helper.rs new file mode 100644 index 0000000..40c1076 --- /dev/null +++ b/crates/dever-sandbox/src/trusted_helper.rs @@ -0,0 +1,130 @@ +//! The only host integration is a signed, administrator-installed static helper. +use std::path::{Path, PathBuf}; + +pub const TRUSTED_HELPER_ROOT: &str = "/opt/dever/sandbox"; +pub const APPARMOR_PROFILE: &str = include_str!("../../../sdk/apparmor/dever-sandbox"); + +#[cfg(all(test, target_os = "linux"))] +#[path = "../../../test/dever-sandbox-tests/trusted_helper.rs"] +mod tests; + +pub(super) fn executable(assets: &Path, expected: &str) -> Result<PathBuf, String> { + #[cfg(target_os = "linux")] + if rustix::process::geteuid().as_raw() != 0 && validate_host_policy()? { + let path = Path::new(TRUSTED_HELPER_ROOT).join(expected).join("bwrap"); + verify_helper(&path, expected)?; + require_profiles()?; + return Ok(path); + } + let _ = expected; + Ok(assets.join("bin/bwrap")) +} + +/// Validate the host prerequisite shared by non-root launch and real-host +/// policy installation. This only reads kernel state; administrators own it. +pub fn validate_host_policy() -> Result<bool, String> { + #[cfg(target_os = "linux")] + { + host_policy(|path| std::fs::read_to_string(path)) + } + #[cfg(not(target_os = "linux"))] + { + Ok(false) + } +} + +#[cfg(target_os = "linux")] +fn host_policy(read: impl Fn(&str) -> std::io::Result<String>) -> Result<bool, String> { + let restricted = policy_flag(&read, "apparmor_restrict_unprivileged_userns")?.unwrap_or(false); + if !restricted { + return Ok(false); + } + if policy_flag(&read, "apparmor_restrict_unprivileged_unconfined")? != Some(true) { + return Err("Dever AppArmor deployment requires kernel.apparmor_restrict_unprivileged_unconfined=1 while user namespace restriction is enabled; an administrator must approve strict profile transitions before installing or loading the Dever policy".into()); + } + Ok(true) +} + +#[cfg(target_os = "linux")] +fn policy_flag( + read: &impl Fn(&str) -> std::io::Result<String>, + name: &str, +) -> Result<Option<bool>, String> { + match read(&format!("/proc/sys/kernel/{name}")) { + Ok(value) => match value.trim() { + "0" => Ok(Some(false)), + "1" => Ok(Some(true)), + _ => Err(format!( + "invalid AppArmor kernel flag '{name}': expected 0 or 1" + )), + }, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(format!( + "cannot inspect AppArmor kernel flag '{name}': {error}" + )), + } +} + +#[cfg(target_os = "linux")] +fn verify_helper(path: &Path, expected: &str) -> Result<(), String> { + use std::io::Read; + use std::os::unix::fs::MetadataExt; + for (index, ancestor) in path.ancestors().enumerate() { + let metadata = std::fs::symlink_metadata(ancestor).map_err(|error| format!( + "trusted sandbox helper is missing or inaccessible at '{}': {error}; install the matching signed Dever sandbox assets", path.display()))?; + if metadata.file_type().is_symlink() + || metadata.uid() != 0 + || metadata.mode() & 0o022 != 0 + || (index == 0 + && (!metadata.is_file() + || metadata.mode() & 0o6111 != 0o111 + || metadata.nlink() != 1)) + || (index != 0 && !metadata.is_dir()) + { + return Err("trusted sandbox helper requires real root-owned paths without shared writes, setuid or setgid".into()); + } + } + match rustix::fs::lgetxattr(path, "security.capability", &mut [0u8; 64]) { + Err(rustix::io::Errno::NODATA | rustix::io::Errno::OPNOTSUPP) => {} + Ok(_) => return Err("trusted sandbox helper must not have file capabilities".into()), + Err(error) => { + return Err(format!( + "cannot inspect sandbox helper capabilities: {error}" + )); + } + } + let mut bytes = Vec::new(); + std::fs::File::open(path) + .and_then(|file| file.take(32 * 1024 * 1024 + 1).read_to_end(&mut bytes)) + .map_err(|error| format!("cannot read trusted sandbox helper: {error}"))?; + if super::validate_bwrap(&bytes)? != expected { + return Err("trusted sandbox helper differs from the embedded signed asset".into()); + } + Ok(()) +} + +#[cfg(target_os = "linux")] +fn require_profiles() -> Result<(), String> { + // The profiles list requires policy-admin privilege; the policy filesystem + // exposes each profile's name/mode to ordinary users without changing state. + let entries = std::fs::read_dir("/sys/kernel/security/apparmor/policy/profiles") + .map_err(|error| format!("cannot verify the Dever AppArmor profiles: {error}; an administrator must load /etc/apparmor.d/dever-sandbox"))?; + let mut required = std::collections::BTreeSet::from(["dever-bwrap", "dever-bwrap-child"]); + for entry in entries.take(4096) { + let entry = entry.map_err(|error| format!("cannot inspect AppArmor profiles: {error}"))?; + let name = std::fs::read_to_string(entry.path().join("name")) + .map_err(|error| format!("cannot read AppArmor profile name: {error}"))?; + if required.contains(name.trim()) { + let mode = std::fs::read_to_string(entry.path().join("mode")) + .map_err(|error| format!("cannot read AppArmor profile mode: {error}"))?; + if mode.trim() != "enforce" { + return Err("Dever AppArmor profiles must be loaded in enforce mode".into()); + } + required.remove(name.trim()); + } + if required.is_empty() { + return Ok(()); + } + } + Err("Dever AppArmor profile is not loaded; an administrator must load /etc/apparmor.d/dever-sandbox".into()) +} diff --git a/docs/superpowers/plans/2026-09-02-dever-language-mvp.md b/docs/superpowers/plans/2026-09-02-dever-language-mvp.md deleted file mode 100644 index d112170..0000000 --- a/docs/superpowers/plans/2026-09-02-dever-language-mvp.md +++ /dev/null @@ -1,830 +0,0 @@ -# Dever Language MVP Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Build the first executable Dever vertical slice in which a human reviews multilingual natural semantics, an external AI can submit only typed candidates, a deterministic refiner controls promotion, and one trusted temperature-alarm graph generates a Rust service and a React simulator. - -**Architecture:** The repository is a Rust workspace plus two React applications. `dever-core` owns platform-neutral semantics, `dever-refiner` owns candidate validation and atomic promotion, `dever-ir` owns the executable representation, and each backend only lowers that IR. The temperature example depends on those public packages; no core package may depend on the example. Generated Rust and React live only under `examples/temperature-alarm/build/` and never become trusted source. - -**Tech Stack:** Stable Rust (edition 2021), Cargo, `serde`, `serde_json`, `sha2`, Axum/Tokio for local HTTP adapters, React, TypeScript, Vite, Vitest, Testing Library, and npm lockfiles. - ---- - -## Execution Constraints - -- The current machine has Node.js `24.15.0`, npm `11.12.1`, and Corepack `0.34.6`. -- The current machine does **not** have `rustc`, `cargo`, or `rustup`. Stop at Task 0 and request authorization before installing a Rust toolchain. Toolchain installation is external setup, not a repository change. -- Do not run `npm run build`, workspace-wide tests, or end-to-end/integration tests by default. Run only the targeted commands listed under each task. Before the final service/browser exercise, explain that it starts local processes and obtain confirmation. -- Do not create a Git commit unless the user explicitly requests it. Commit steps are intentionally omitted from this plan. -- Keep permanent tests below repository-root `test/`. Do not place lasting tests beside production source. -- Add packages only when their implementation task begins. Do not scaffold all future directories in Task 1. -- Do not add provider systems, plugin registries, databases, authentication, deployment files, queues, caches, or compatibility branches. - -## Acceptance Traceability - -| Design acceptance criterion | Owning task(s) | -|---|---| -| Same IDs/hash in Chinese and English; locale edits do not affect behavior | 3 | -| Deterministic candidate diff and atomic human approval | 4, 5 | -| Invalid candidates enter the code scrap yard with exact reasons | 4, 5 | -| Trusted graph lowers to Dever IR and generated Rust/React | 6, 7, 8 | -| Temperature, hysteresis, disconnect, recovery, and quarantine behavior | 6, 7, 11 | -| Backend diagnostics map to stable semantic IDs | 7, 8 | -| Rebuild needs no AI/network once toolchains and locked dependencies exist | 11 | -| Generated output is disposable; core has no temperature concepts | 1, 11 | - -## Repository Map - -Create each path only in its owning task. - -```text -/data/project/dever/ - AGENTS.md AI/human engineering rules for this repository - .gitignore generated output and local cache exclusions - Cargo.toml Rust workspace membership and shared dependency versions - Cargo.lock locked compiler/workbench-server dependencies - crates/ - dever-core/ semantic types, stable IDs, canonical JSON, behavior hash - dever-refiner/ isolated candidate graph, validators, diff, approval, scrap - dever-ir/ normalized executable IR and lowering - dever-backend-rust/ deterministic Rust service generator and source maps - dever-backend-react/ deterministic React simulator generator and bindings - dever-workbench/ local project store and HTTP adapter - dever-cli/ thin command-line composition root - apps/ - dever-workbench/ manually maintained multilingual React review surface - examples/ - temperature-alarm/ - semantic/program.json trusted language-neutral graph - locale/zh-CN.json Simplified Chinese presentation only - locale/en-US.json English presentation only - candidates/ reviewed sample candidate inputs - build/ ignored generated Rust/React output - test/ - dever-tests/ Rust contract and acceptance tests - fixtures/ reusable valid and invalid semantic inputs - workbench/ Vitest UI tests -``` - -Dependency direction: - -```text -temperature example -> CLI/workbench -> refiner -> core -temperature example -> CLI -> backends -> IR -> refiner -> core -generated React client -----------------> generated Rust HTTP contract - -core -X-> temperature example -IR -X-> Rust syntax / React / locale text -``` - -## Milestone 0: Toolchain Gate - -### Task 0: Verify prerequisites without changing the machine - -**Files:** None. - -- [x] Run: - - ```bash - cd /data/project/dever - node --version - npm --version - rustc --version - cargo --version - ``` - - Expected now: Node and npm report the versions above; `rustc` fails with `command not found`. Do not reinterpret that as a repository defect. - -- [x] Ask the user for permission to install the stable Rust toolchain with the minimal profile. After authorization, use the then-current official Rust installation procedure and restart the shell environment. - -- [x] Re-run `rustc --version && cargo --version` and record the exact versions in the implementation session log. Expected: both commands succeed. Do not pin a historical compiler version merely to match this plan. - - Verified on 2026-09-02: `rustc 1.98.0`, `cargo 1.98.0`, `rustup 1.29.1`, target `stable-x86_64-unknown-linux-gnu`, and GCC `13.3.0`. The command runner does not export `HOME`, so implementation commands use `/root/.cargo/bin/cargo` and `/root/.cargo/bin/rustc` explicitly instead of changing shell startup files. - -## Milestone 1: Trusted Semantic Kernel - -### Task 1: Establish repository rules and the smallest Rust workspace - -**Files:** - -- Create: `AGENTS.md` -- Create: `.gitignore` -- Create: `Cargo.toml` -- Create: `crates/dever-core/Cargo.toml` -- Create: `crates/dever-core/src/lib.rs` -- Create: `test/dever-tests/Cargo.toml` -- Create: `test/dever-tests/src/lib.rs` -- Create: `test/dever-tests/tests/workspace_smoke.rs` - -- [ ] Write `AGENTS.md` with these executable repository rules: - - ```markdown - # Dever Repository Rules - - - Trusted behavior lives only in the canonical semantic graph. - - AI output is an untrusted typed candidate until deterministic validation and explicit human approval both succeed. - - Do not add implicit fallback, retry, default-value, capability, allocation, or error-suppression behavior. - - `dever-core` and `dever-ir` must remain independent of Rust syntax, React, locale wording, AI providers, and examples. - - Example packages may depend on core packages; core packages must never depend on examples. - - Generated files belong under `examples/*/build/` and are not trusted or committed. - - Permanent tests belong under repository-root `test/`. - - Prefer explicit data and small functions. Add an abstraction only for a stable invariant or real shared behavior. - - Do not commit, install dependencies, start services, or run broad builds/tests without the user's authorization when required by the active session instructions. - ``` - -- [ ] Write `.gitignore` with only owned generated/local paths: - - ```gitignore - /target/ - /apps/dever-workbench/node_modules/ - /apps/dever-workbench/dist/ - /examples/*/build/ - *.log - ``` - -- [ ] Create a virtual Cargo workspace containing only `dever-core` and the root test crate. Use edition 2021 and centralize exact dependency versions under `[workspace.dependencies]`; initially add only `serde`, `serde_json`, and `sha2` because canonical storage and hashing require them. - -- [ ] Keep `test/dever-tests/src/lib.rs` empty and put this deliberately failing smoke test in `test/dever-tests/tests/workspace_smoke.rs`: - - ```rust - #[test] - fn workspace_smoke_test() { - assert_eq!(dever_core::FORMAT_VERSION, 1); - } - ``` - -- [ ] Run `cargo test -p dever-tests workspace_smoke_test -- --exact`. Expected: compilation fails because `FORMAT_VERSION` does not exist. - -- [ ] Add exactly this public constant to `crates/dever-core/src/lib.rs`: - - ```rust - pub const FORMAT_VERSION: u32 = 1; - ``` - -- [ ] Re-run the same targeted test. Expected: `1 passed; 0 failed`. - -- [ ] Inspect `cargo tree -p dever-core`. Expected: only serialization/hash dependencies and their transitive dependencies; no HTTP, UI, AI, or temperature package. - -### Task 2: Implement stable IDs, typed semantic objects, and canonical behavior hashes - -**Files:** - -- Create: `crates/dever-core/src/id.rs` -- Create: `crates/dever-core/src/model.rs` -- Create: `crates/dever-core/src/canonical_json.rs` -- Create: `crates/dever-core/src/behavior_hash.rs` -- Modify: `crates/dever-core/src/lib.rs` -- Create: `test/dever-tests/tests/core_semantics.rs` -- Create: `test/fixtures/minimal-program.json` -- Create: `test/fixtures/minimal-program-reordered.json` -- Create: `test/fixtures/duplicate-key.json` - -- [ ] Define `SemanticId` as a validated value object. Its accepted grammar is lowercase ASCII segments separated by one dot: `[a-z][a-z0-9_]*(\.[a-z][a-z0-9_]*)+`. It must reject whitespace, locale text, empty segments, uppercase letters, and leading digits. Expose `parse`, `as_str`, `Display`, `Serialize`, and `Deserialize`; keep its inner `String` private. - -- [ ] Add targeted tests named: - - ```text - stable_id_accepts_namespaced_ascii_identifier - stable_id_rejects_localized_or_ambiguous_identity - canonical_loader_rejects_duplicate_object_keys - canonical_loader_rejects_non_integer_numbers - equivalent_unordered_objects_have_identical_behavior_hashes - changed_transition_has_a_different_behavior_hash - ``` - - The minimal fixtures must contain no temperature terms. Use a generic switch machine with `state.off` and `state.on` so this test cannot accidentally couple core to the example. - -- [ ] Run each new test individually, for example: - - ```bash - cargo test -p dever-tests stable_id_accepts_namespaced_ascii_identifier -- --exact - ``` - - Expected before implementation: unresolved imports or failed assertions. - -- [ ] Implement the minimum platform-neutral model, using `BTreeMap`/`BTreeSet` for semantically unordered collections and `Vec` only where order is behaviorally meaningful: - - ```rust - pub struct SemanticGraph { - pub format_version: u32, - pub revision: u64, - pub project: Project, - pub modules: BTreeMap<SemanticId, Module>, - pub objects: BTreeMap<SemanticId, SemanticObject>, - pub resources: ResourcePolicy, - } - - pub enum SemanticObject { - ScalarType(ScalarType), - RangeType(RangeType), - RecordType(RecordType), - EnumType(EnumType), - OptionalType(OptionalType), - Contract(Contract), - Capability(Capability), - Effect(Effect), - Input(Input), - Output(Output), - Operation(Operation), - State(State), - Event(Event), - Transition(Transition), - StateMachine(StateMachine), - UiScreen(UiScreen), - UiControl(UiControl), - UiBinding(UiBinding), - UiInteraction(UiInteraction), - Scenario(Scenario), - } - ``` - - Every reference field is a `SemanticId`. Integer ranges use inclusive lower and upper bounds. Transition guards are limited to `Always` and integer intervals in this MVP; this restriction makes overlap and coverage validation deterministic. Actions are limited to state assignment, output assignment, and declared capability invocation. Allocation is represented only by `ResourcePolicy`, never inferred from backend containers. - - Each state machine declares exactly one owning module. MVP values are immutable and have explicit copy or owned value semantics; histories and quarantine buffers are owned by one runtime instance and bounded by `ResourcePolicy`. Shared mutable ownership is outside the MVP rather than silently delegated to a backend. - -- [ ] Implement a streaming JSON loader that rejects duplicate keys before typed deserialization. Reject fractional/exponent numbers in semantic files; the MVP normalized numeric representation is signed or unsigned base-10 integer JSON. - -- [ ] Implement canonical JSON output by recursively sorting every JSON object key lexically, emitting no insignificant whitespace, preserving declared `Vec` order, and terminating with no newline. Never derive a behavior hash from raw input bytes. - -- [ ] Implement: - - ```rust - pub fn load_semantic_graph(input: &[u8]) -> Result<SemanticGraph, SemanticLoadError>; - pub fn canonical_behavior_bytes(graph: &SemanticGraph) -> Result<Vec<u8>, CanonicalError>; - pub fn behavior_hash(graph: &SemanticGraph) -> Result<String, CanonicalError>; - ``` - - `behavior_hash` returns lowercase hexadecimal SHA-256. Locale data is absent from `SemanticGraph`, so it cannot affect this hash by construction. - -- [ ] Re-run only the six named tests. Expected: each reports `1 passed; 0 failed`. - -- [ ] Refactor pass: verify no semantic validator has entered `dever-core`; core owns representation and canonical identity only. - -### Task 3: Add separate locale catalogs and deterministic natural-semantic projections - -**Files:** - -- Create: `crates/dever-workbench/Cargo.toml` -- Create: `crates/dever-workbench/src/lib.rs` -- Create: `crates/dever-workbench/src/presentation.rs` -- Modify: `Cargo.toml` -- Create: `test/dever-tests/tests/presentation.rs` -- Create: `test/fixtures/locale/en-US.json` -- Create: `test/fixtures/locale/zh-CN.json` - -- [ ] Add `dever-workbench` only now. It may depend on `dever-core`; `dever-core` must not depend on it. - -- [ ] Define the locale file as a presentation-only catalog: - - ```json - { - "locale": "en-US", - "terms": { - "project.switch": "Switch", - "state.off": "Off", - "state.on": "On" - }, - "templates": { - "state.initial": "Initial state: {state}", - "transition.on_event": "When {event}, change from {from} to {to}." - } - } - ``` - - The Chinese fixture must contain the same term IDs and template keys with Chinese values. Missing keys are errors; there is no silent fallback to English. - -- [ ] Add tests named: - - ```text - chinese_and_english_views_keep_the_same_semantic_ids - chinese_and_english_views_keep_the_same_behavior_hash - locale_only_edit_does_not_change_behavior_hash - missing_locale_entry_is_an_explicit_error - ``` - -- [ ] Run each test before implementation. Expected: unresolved presentation API. - -- [ ] Implement a `NaturalDocument` made only of ordered sections and `NaturalStatement { semantic_ids, message_key, arguments }`. The server-side presenter derives statements mechanically from typed graph objects. Locale rendering substitutes only declared arguments and must return `PresentationError::MissingTerm` or `MissingTemplate` instead of inventing text. - -- [ ] Re-run only the four named tests. Expected: each passes. - -- [ ] Run `cargo tree -p dever-core` and inspect the output. Expected: it contains no `dever-workbench`; core remains independent. - -## Milestone 2: Deterministic Refinement - -### Task 4: Apply typed candidates in isolation and derive deterministic semantic diffs - -**Files:** - -- Create: `crates/dever-refiner/Cargo.toml` -- Create: `crates/dever-refiner/src/lib.rs` -- Create: `crates/dever-refiner/src/candidate.rs` -- Create: `crates/dever-refiner/src/diff.rs` -- Create: `crates/dever-refiner/src/validation/mod.rs` -- Modify: `Cargo.toml` -- Create: `test/dever-tests/tests/refiner_candidates.rs` -- Create: `test/fixtures/candidates/valid-add-transition.json` -- Create: `test/fixtures/candidates/stale-base.json` - -- [ ] Define a typed patch envelope: - - ```rust - pub struct CandidatePatch { - pub proposal_id: SemanticId, - pub base_revision: u64, - pub base_behavior_hash: String, - pub provenance: BTreeSet<SemanticId>, - pub operations: Vec<PatchOperation>, - } - - pub enum PatchOperation { - AddObject { - id: SemanticId, - object: SemanticObject, - evidence: BTreeSet<SemanticId>, - }, - ReplaceObject { - id: SemanticId, - expected_hash: String, - object: SemanticObject, - evidence: BTreeSet<SemanticId>, - }, - RemoveObject { - id: SemanticId, - expected_hash: String, - evidence: BTreeSet<SemanticId>, - }, - } - ``` - - There is no raw source insertion operation and no natural-language operation. - -- [ ] Add tests named: - - ```text - valid_patch_changes_only_an_isolated_candidate_graph - equivalent_patch_inputs_produce_identical_diff_hashes - stale_base_revision_is_rejected_before_operations_apply - replace_requires_the_expected_object_hash - failed_multi_operation_patch_leaves_trusted_graph_unchanged - ``` - -- [ ] Run each test individually. Expected before implementation: failure. - -- [ ] Implement `prepare_candidate(trusted, patch) -> Result<PreparedCandidate, RefinementFailure>`. Clone the trusted graph once, check base revision/hash before any operation, apply all operations to the clone, and derive added/removed/changed object IDs by comparing canonical object hashes. Sort diff entries by stable ID. - -- [ ] Hash the canonical semantic diff and expose it as `review_hash`. This is the value later bound to human approval. - -- [ ] Re-run the five candidate tests. Expected: each passes and the trusted fixture bytes/hash remain unchanged after every failure. - -### Task 5: Validate candidates, bind approval, and persist rejected work in the code scrap yard - -**Files:** - -- Create: `crates/dever-refiner/src/validation/references.rs` -- Create: `crates/dever-refiner/src/validation/types.rs` -- Create: `crates/dever-refiner/src/validation/state_machines.rs` -- Create: `crates/dever-refiner/src/validation/effects.rs` -- Create: `crates/dever-refiner/src/validation/resources.rs` -- Create: `crates/dever-refiner/src/approval.rs` -- Create: `crates/dever-refiner/src/scrap.rs` -- Modify: `crates/dever-refiner/src/lib.rs` -- Create: `test/dever-tests/tests/refiner_validation.rs` -- Create: `test/dever-tests/tests/refiner_approval.rs` -- Create: `test/fixtures/candidates/invalid-broken-reference.json` -- Create: `test/fixtures/candidates/invalid-duplicate-transition.json` -- Create: `test/fixtures/candidates/invalid-effect.json` -- Create: `test/fixtures/candidates/invalid-unjustified-always-guard.json` - -- [ ] Give every diagnostic a stable code, severity, primary semantic ID, deterministic message arguments, and related IDs. Start with these exact error codes: - - ```text - D_REF_MISSING - D_TYPE_MISMATCH - D_STATE_UNKNOWN - D_TRANSITION_OVERLAP - D_TRANSITION_GAP - D_RULE_DUPLICATE - D_EFFECT_UNDECLARED - D_FAILURE_UNDECLARED - D_FALLBACK_UNJUSTIFIED - D_OBJECT_UNREACHABLE - D_DEPENDENCY_INCOMPLETE - D_ALLOCATION_FORBIDDEN - D_BASE_STALE - D_APPROVAL_MISMATCH - ``` - -- [ ] Add one focused failing test for each of the four required rejection examples plus: - - ```text - valid_integer_intervals_are_exhaustive_and_non_overlapping - forbidden_dynamic_allocation_is_rejected - state_machine_requires_one_explicit_owner_module - approval_must_match_candidate_review_hash - valid_human_approval_promotes_the_whole_candidate_atomically - rejected_candidate_is_serialized_as_scrap_not_trusted_semantics - scrap_candidate_requires_revalidation_against_current_base - ``` - -- [ ] Run each test individually. Expected before implementation: failure. - -- [ ] Implement a fixed validation pipeline. Use an explicit ordered array of validator functions so output ordering cannot vary. Do not introduce a plugin registry: - - ```rust - const VALIDATORS: &[Validator] = &[ - validate_references, - validate_types, - validate_state_machines, - validate_effects_and_failures, - validate_reachability_and_dependencies, - validate_resources, - ]; - ``` - -- [ ] Treat an `Always` guard or other fallback as valid only when its patch operation cites at least one existing `Contract` semantic ID. An AI explanation string is never evidence. - -- [ ] Define `HumanApproval { candidate_id, base_behavior_hash, review_hash }`. `promote` must verify all three fields, increment revision once, and return a new trusted graph. It must have no partial-write API. - -- [ ] Define `ScrapRecord { sequence, candidate, base_behavior_hash, diagnostics, validation_evidence }`. The refiner returns the record plus canonical bytes without performing filesystem I/O. Task 9's `ProjectStore` persists those bytes under its configured scrap directory. The trusted-graph loader must reject scrap-record format. - -- [ ] Re-run only the named validation/approval tests. Expected: each passes. - -- [ ] Refactor pass: diagnostics contain no localized prose and validators contain no temperature IDs or thresholds. - -## Milestone 3: Executable IR and Rust Backend - -### Task 6: Add normalized Dever IR and the temperature example semantics - -**Files:** - -- Create: `crates/dever-ir/Cargo.toml` -- Create: `crates/dever-ir/src/lib.rs` -- Create: `crates/dever-ir/src/lower.rs` -- Modify: `Cargo.toml` -- Create: `examples/temperature-alarm/semantic/program.json` -- Create: `examples/temperature-alarm/locale/zh-CN.json` -- Create: `examples/temperature-alarm/locale/en-US.json` -- Create: `examples/temperature-alarm/candidates/invalid-fallback.json` -- Create: `test/dever-tests/tests/temperature_semantics.rs` -- Create: `test/dever-tests/tests/ir_lowering.rs` - -- [ ] Encode the approved example contract exactly: - - ```text - accepted temperature: -40..=125 integer Celsius - initial: Normal, alarm off - Normal + -40..=79: Normal, alarm off - Normal + 80..=125: Alarm, alarm on - Alarm + -40..=74: Normal, alarm off - Alarm + 75..=125: Alarm, alarm on - Offline + -40..=79: Normal, alarm off - Offline + 80..=125: Alarm, alarm on - any state + disconnect: Offline, alarm on - invalid value/type: preserve state and output; append quarantine event - history capacity: 64 - quarantine capacity: 32 - dynamic allocation policy: bounded - ``` - - Give every contract, state, event, transition, capability, effect, UI control, binding, and scenario its own stable ID. Keep all temperature terms out of Rust source outside the example fixtures and backend output tests. - -- [ ] Add targeted tests named: - - ```text - approved_temperature_graph_passes_every_refiner_validator - temperature_transition_intervals_are_exhaustive - temperature_locales_share_ids_and_behavior_hash - lowering_is_independent_of_locale_catalog - lowering_sorts_unordered_objects_by_stable_id - ir_contains_no_rust_or_react_constructs - ``` - -- [ ] Run each test individually; expect failure before IR implementation. - -- [ ] Define IR as normalized integer/state tables, explicit effects, bounded resources, HTTP-neutral operations, and UI-neutral screen/control intents. Retain `source_id: SemanticId` on every executable instruction. Do not store Rust names, JSX names, URL paths, or localized strings in IR. - -- [ ] Add `dever-core` and `dever-refiner` as `dever-ir` dependencies. Implement: - - ```rust - pub fn lower( - graph: &SemanticGraph, - validation: &ValidationReport, - ) -> Result<ExecutableProgram, LoweringError>; - ``` - - Reject a report whose validated behavior hash differs from the graph's current behavior hash. `dever-ir` consumes the report but does not rerun or redefine refiner rules. - -- [ ] Re-run the six tests. Expected: each passes. - -- [ ] Run `rg -n -i 'temperature|celsius|alarm|react|jsx|rust' crates/dever-core crates/dever-refiner crates/dever-ir`. Expected: no temperature-domain hits and no backend constructs in core/IR; occurrences in comments describing forbidden coupling must be removed rather than exempted. - -### Task 7: Generate a pure state machine plus a thin Rust HTTP service - -**Files:** - -- Create: `crates/dever-backend-rust/Cargo.toml` -- Create: `crates/dever-backend-rust/src/lib.rs` -- Create: `crates/dever-backend-rust/src/generator.rs` -- Create: `crates/dever-backend-rust/src/source_map.rs` -- Create: `crates/dever-backend-rust/src/templates.rs` -- Modify: `Cargo.toml` -- Create: `test/dever-tests/tests/rust_backend.rs` -- Create: `test/dever-tests/tests/generated_runtime.rs` - -- [ ] Add only `dever-core` and `dever-ir` as internal dependencies. Generator implementation may use `serde_json`; it must not depend on the temperature example crate or path. - -- [ ] Define one backend result: - - ```rust - pub struct GeneratedProject { - pub files: BTreeMap<PathBuf, Vec<u8>>, - pub source_map: SourceMap, - } - - pub fn generate(program: &ExecutableProgram) -> Result<GeneratedProject, RustBackendError>; - ``` - - Pure generation returns bytes and performs no filesystem writes. A separate CLI adapter writes them. - -- [ ] Add tests named: - - ```text - rust_generation_is_byte_for_byte_deterministic - rust_source_map_covers_every_generated_transition - generated_machine_matches_all_temperature_boundary_cases - invalid_reading_preserves_state_and_output - invalid_reading_appends_one_bounded_quarantine_event - disconnect_enters_offline_and_enables_alarm - offline_recovery_uses_the_approved_eighty_degree_boundary - ``` - -- [ ] Run the generator tests first. Expected: unresolved backend API. - -- [ ] Generate two layers: - - ```text - src/machine.rs deterministic state/event reducer with no HTTP concepts - src/api.rs JSON envelope parsing and snapshot shaping - src/main.rs Axum/Tokio composition root and local bind address - Cargo.toml exact dependency versions from one backend manifest constant - Cargo.lock deterministic backend asset matching the exact manifest versions - source-map.json generated line spans -> Dever stable semantic IDs - ``` - - `machine.rs` must validate the input envelope before dispatch. Invalid type/range values append a bounded quarantine record and do not call the reducer. When a bounded buffer is full, remove the oldest entry because that retention rule is declared in the semantic resource policy; do not invent any other recovery. - -- [ ] Generate this exact service boundary. Parse the event request initially as `serde_json::Value` so a wrong-typed reading can become a declared quarantine event instead of disappearing inside framework rejection: - - ```text - GET /api/snapshot - POST /api/events {"kind":"reading","value":80} - POST /api/events {"kind":"reading","value":"broken"} - POST /api/events {"kind":"disconnect"} - ``` - - A snapshot contains sequence, last accepted temperature, stable state ID, stable output IDs and values, bounded history, and bounded quarantine events. A quarantine event contains rejected JSON value, violated contract ID, source capability ID, and sequence. It contains no invented replacement reading. - -- [ ] Compile and execute generated-runtime tests from a temporary directory under `target/`, never under the trusted example source. Use one test name at a time. Expected: all seven targeted tests pass. - -- [ ] Verify generated diagnostics by intentionally compiling one copied output with a known generated-token corruption in the temporary test directory. Assert that `source-map.json` maps the affected generated span to the expected transition stable ID. Remove the corrupted temporary directory through the test harness. - -- [ ] Refactor pass: template fragments are small functions grouped by generated file; no single string builder owns machine logic, HTTP wiring, manifest generation, and source maps together. - -## Milestone 4: React Backend and Simulator - -### Task 8: Generate a React simulator from UI intent without moving domain rules into the browser - -**Files:** - -- Create: `crates/dever-backend-react/Cargo.toml` -- Create: `crates/dever-backend-react/src/lib.rs` -- Create: `crates/dever-backend-react/src/generator.rs` -- Create: `crates/dever-backend-react/src/source_map.rs` -- Create: `crates/dever-backend-react/src/templates.rs` -- Modify: `Cargo.toml` -- Create: `test/dever-tests/tests/react_backend.rs` - -- [ ] Define `generate(&ExecutableProgram) -> Result<GeneratedProject, ReactBackendError>` with the same side-effect-free output shape as the Rust backend. Keep the types separate in each backend rather than adding a premature generic backend framework. - -- [ ] Add targeted Rust generator tests named: - - ```text - react_generation_is_byte_for_byte_deterministic - generated_client_uses_only_declared_service_operations - generated_ui_contains_all_approved_controls_and_views - generated_ui_contains_no_temperature_transition_thresholds - react_source_map_covers_controls_bindings_and_interactions - ``` - -- [ ] Run each test before implementation. Expected: failure. - -- [ ] Generate exactly these maintained boundaries: - - ```text - package.json exact locked React/Vite dependency declarations - package-lock.json deterministic backend asset emitted with exact dependency versions - index.html - tsconfig.json - vite.config.ts - src/main.tsx - src/App.tsx screen composition only - src/api.ts generated request/response bindings - src/scenarios.ts approved input sequences, never state transitions - src/styles.css - source-map.json - ``` - - The browser may display state returned by the service but must never reproduce the 75/80 state rules. The valid slider bounds may be present because they are an input/UI contract, not domain transition logic. Fault buttons send a disconnect event, an out-of-range number, or a wrong-typed JSON value through the generated binding. - -- [ ] Implement a restrained simulator layout: current state and alarm are primary signals; controls, scenario run state, history, and quarantine are dense unframed sections; use familiar Lucide icons with tooltips for icon-only controls; ensure narrow screens stack without overlap. Do not add marketing content or decorative assets. - -- [ ] Re-run the five Rust generator tests. Expected: each passes. - -- [ ] Validate `package.json` and `package-lock.json` consistency with `npm install --package-lock-only --ignore-scripts --offline` in a temporary generated directory after package installation has been explicitly authorized. Expected: no lockfile change. Do not run `npm run build`; rendered interaction is verified in Task 11. - -## Milestone 5: Multilingual Natural Semantic Workbench - -### Task 9: Add a local project store and narrow workbench API - -**Files:** - -- Create: `crates/dever-workbench/src/store.rs` -- Create: `crates/dever-workbench/src/api.rs` -- Create: `crates/dever-workbench/src/server.rs` -- Modify: `crates/dever-workbench/src/lib.rs` -- Modify: `crates/dever-workbench/Cargo.toml` -- Create: `crates/dever-cli/Cargo.toml` -- Create: `crates/dever-cli/src/main.rs` -- Modify: `Cargo.toml` -- Create: `test/dever-tests/tests/workbench_api.rs` - -- [ ] Add Axum/Tokio only to `dever-workbench` and `dever-cli`. Add the refiner and both backend crates as explicit internal `dever-workbench` dependencies for validation/promotion/generation orchestration. Do not leak HTTP types into core, refiner, IR, or backends. - -- [ ] Implement a `ProjectStore` with explicit paths for trusted graph, locale directory, candidate inbox, scrap directory, and build output. A single process lock serializes local writes. Persist approved graph updates by writing a sibling temporary file, syncing it, and atomically renaming it; failure leaves the prior graph intact. - -- [ ] Expose only the MVP endpoints: - - ```text - GET /api/project?locale=zh-CN - GET /api/candidates - POST /api/candidates - POST /api/candidates/{id}/validate - POST /api/candidates/{id}/approve - POST /api/candidates/{id}/reject - GET /api/scrap - POST /api/generate - GET /api/runtime/snapshot - ``` - - External AI tools use the same typed `POST /api/candidates` contract as every other client. There is no model endpoint, prompt storage, provider selection, or AI approval path. The runtime endpoint proxies the explicitly supplied generated-service URL and returns a typed `not_connected` status when no runtime URL was supplied; it must not return an invented empty snapshot. - -- [ ] Add focused tests named: - - ```text - external_client_can_submit_only_a_typed_candidate - project_response_contains_natural_document_ids_and_behavior_hash - approval_rejects_a_changed_review_hash - successful_approval_atomically_replaces_the_trusted_graph - rejected_candidate_is_visible_only_in_scrap - generation_writes_only_below_the_configured_build_directory - ``` - -- [ ] Run each API test using Axum's in-process service interface; do not start a TCP listener. Expected before implementation: failure, then `1 passed` per test after implementation. - -- [ ] Implement a thin CLI with these commands and no business logic: - - ```text - dever check <project-directory> - dever render <project-directory> --locale <locale> - dever refine <project-directory> <candidate-file> - dever generate <project-directory> - dever workbench <project-directory> --listen 127.0.0.1:<port> [--runtime <generated-service-url>] - ``` - - CLI handlers only parse arguments, call package APIs, and shape diagnostics. Invalid arguments return non-zero status with one usage message; do not add fallback paths or interactive guessing. - -- [ ] Run one targeted CLI contract test for each command through the Rust test crate. Do not run the full workspace test suite. - -### Task 10: Build the manually maintained React review surface - -**Files:** - -- Create: `apps/dever-workbench/package.json` -- Create: `apps/dever-workbench/package-lock.json` -- Create: `apps/dever-workbench/tsconfig.json` -- Create: `apps/dever-workbench/vite.config.ts` -- Create: `apps/dever-workbench/index.html` -- Create: `apps/dever-workbench/src/main.tsx` -- Create: `apps/dever-workbench/src/App.tsx` -- Create: `apps/dever-workbench/src/api/client.ts` -- Create: `apps/dever-workbench/src/domain/types.ts` -- Create: `apps/dever-workbench/src/hooks/useProjectReview.ts` -- Create: `apps/dever-workbench/src/components/ProjectNavigation.tsx` -- Create: `apps/dever-workbench/src/components/SemanticSummary.tsx` -- Create: `apps/dever-workbench/src/components/DecisionTable.tsx` -- Create: `apps/dever-workbench/src/components/StateMachineView.tsx` -- Create: `apps/dever-workbench/src/components/CapabilityEffects.tsx` -- Create: `apps/dever-workbench/src/components/CandidateDiff.tsx` -- Create: `apps/dever-workbench/src/components/ScrapDiagnostics.tsx` -- Create: `apps/dever-workbench/src/components/RuntimeQuarantine.tsx` -- Create: `apps/dever-workbench/src/components/ApprovalBar.tsx` -- Create: `apps/dever-workbench/src/styles.css` -- Create: `test/workbench/project-review.test.tsx` -- Create: `test/workbench/locale-switch.test.tsx` -- Create: `test/workbench/approval.test.tsx` - -- [ ] Install only exact versions of `react`, `react-dom`, and `lucide-react`; install exact dev versions of TypeScript, Vite, Vitest, jsdom, and Testing Library packages. Use npm's generated lockfile. Do not add a router, global store, CSS framework, or data-fetching library: the MVP has one work surface and one focused state hook. - -- [ ] First write UI tests proving: - - ```text - project summary and stable IDs render without implementation code - Chinese/English switch changes labels but not displayed behavior hash - decision table exposes transitions, capabilities, effects, and resources - candidate diff groups added/changed/removed semantic objects - approve button is disabled until validation succeeds - approval submits the exact displayed review hash - scrap reasons render stable diagnostic code plus localized arguments - runtime quarantine shows events or an explicit not-connected state - narrow viewport content has no horizontal document overflow - ``` - -- [ ] Run the new tests one file at a time. Expected before components exist: module resolution failure. - - ```bash - npm test -- --run ../../test/workbench/project-review.test.tsx - npm test -- --run ../../test/workbench/locale-switch.test.tsx - npm test -- --run ../../test/workbench/approval.test.tsx - ``` - - Configure Vitest explicitly to include repository-root `test/workbench` and keep production source free of permanent tests. - -- [ ] Implement one quiet operational work surface with persistent project navigation, a compact locale selector, tabbed semantic views, and a bottom approval bar. Keep presentation components stateless. `useProjectReview` alone owns loading, candidate selection, validation, approval, rejection, and refresh side effects. `api/client.ts` alone owns fetch calls and response decoding. - -- [ ] Do not display generated Rust/TypeScript in the normal interface. An expert diagnostic link may show stable IDs, IR/source-map diagnostics, and hashes, but it cannot edit trusted state. - -- [ ] Re-run the three targeted test files and `npm run typecheck`. Expected: all targeted tests pass and TypeScript reports no errors. Do not run `npm run build`. - -- [ ] Refactor pass: repeated rendering comes from typed configuration or focused components; no copied fetch/validation/loading/error flow; no nested cards, oversized headings, clipped buttons, or unexplained icon-only actions. - -## Milestone 6: Vertical-Slice Closure - -### Task 11: Prove all approved behavior through one explicit acceptance matrix - -**Files:** - -- Create: `test/dever-tests/tests/acceptance_matrix.rs` -- Create: `test/acceptance/README.md` -- Modify only if a test exposes an in-scope defect: files owned by Tasks 2-10 - -- [ ] Encode the state-machine matrix as table-driven tests, not repeated test bodies: - - ```text - Normal + 79 -> Normal, alarm off - Normal + 80 -> Alarm, alarm on - Alarm + 75 -> Alarm, alarm on - Alarm + 74 -> Normal, alarm off - Offline + 79 -> Normal, alarm off - Offline + 80 -> Alarm, alarm on - any + disconnect -> Offline, alarm on - any + -41 -> unchanged, one quarantine event - any + 126 -> unchanged, one quarantine event - any + "broken" -> unchanged, one quarantine event - ``` - -- [ ] Add deterministic artifact checks: - - ```text - generate twice -> identical relative paths and bytes - edit only zh-CN text -> identical behavior hash and backend bytes - no AI/network call appears in check, render, refine, approve, lower, or generate - every generated diagnostic/source-map entry resolves to a trusted stable ID - Cargo metadata shows no core/refiner/IR/backend/workbench dependency sourced from `examples/` - build output paths are ignored and absent from `git ls-files` - ``` - -- [ ] Run each new Rust acceptance test individually. Do not run `cargo test --workspace`. - -- [ ] Run `rg -n '(TODO|FIXME|unimplemented!|todo!|panic!\("not implemented|fallback|default reading|retry)' crates apps examples --glob '!examples/*/build/**'`. Expected: no placeholder implementation or invented fallback/retry. Review legitimate contract text containing “fallback” manually rather than hiding it with an exclusion. - -- [ ] Run `git diff --check` and `git status --short`. Expected: no whitespace errors; only planned trusted source/docs/lockfiles are tracked, while `examples/temperature-alarm/build/` is ignored. - -- [ ] Before starting any local server or browser automation, explain the exact ports/processes and ask for confirmation because this is service-affecting verification. - -- [ ] After confirmation, generate the example, start the generated Rust service and workbench on loopback-only unused ports, and exercise the manual slider, all three automatic scenarios, disconnect, out-of-range, and malformed inputs. Use Playwright screenshots at desktop and mobile widths to verify nonblank rendering, stable layout, visible state/alarm changes, history, and quarantine. Stop both processes before finishing. - -- [ ] Record only observed commands/results in `test/acceptance/README.md`. Mark checks as passed, failed, skipped, or blocked; never claim the browser flow passed from unit tests alone. - -- [ ] Final self-review against all 14 design acceptance criteria. Any unmet item keeps the MVP incomplete. Optional LLVM, self-hosting, MCU, extra UI frameworks, package registry, authentication, and deployment work remain explicitly out of scope. - -## Implementation Handoff - -This plan is intentionally sequential: later milestones consume stable contracts from earlier ones. Do not parallelize core model, refiner, and IR ownership. Within a later milestone, UI-only work may proceed independently only after the corresponding HTTP/IR contract is fixed. - -At execution time choose one mode: - -1. **Subagent-Driven (recommended):** one bounded implementation worker per task, followed by main-agent integration and targeted verification; no more than two agents active and no overlapping file ownership. -2. **Inline Execution:** the main agent executes each checkbox in order, pausing at the Rust-installation gate and the final local-service/browser gate. - -In both modes, no commit is created without a separate explicit user request. diff --git a/docs/superpowers/specs/2026-09-02-dever-language-mvp-design.md b/docs/superpowers/specs/2026-09-02-dever-language-mvp-design.md deleted file mode 100644 index 29f7d7c..0000000 --- a/docs/superpowers/specs/2026-09-02-dever-language-mvp-design.md +++ /dev/null @@ -1,333 +0,0 @@ -# Dever Language MVP Design - -- Date: 2026-09-02 -- Status: Written specification approved by the user on 2026-09-02 -- Scope: First executable vertical slice - -## 1. Summary - -Dever is an AI-oriented, general-purpose language system whose primary development surface is a multilingual natural semantic workbench rather than conventional source code. - -An AI may propose implementations, but it cannot directly modify the trusted program. A deterministic refiner validates typed semantic candidates and promotes only accepted semantics into the trusted semantic graph. Humans review semantic changes, capabilities, state transitions, and resource effects. The accepted graph is lowered to a language-independent Dever IR and then compiled through replaceable platform backends. - -The MVP validates this model end to end with a simulated temperature alarm controller. It generates a Rust service and a React web interface without requiring physical hardware. - -## 2. Goals - -The MVP must prove that: - -1. A program can be represented and reviewed as natural semantics instead of generated implementation code. -2. Multiple interface languages can display the same program without changing its behavior. -3. AI-produced changes can remain untrusted candidates until a deterministic refiner accepts them. -4. Unjustified or invalid candidate logic can be isolated in a code scrap yard. -5. Runtime values that violate a declared contract can be isolated without inventing fallback behavior. -6. One trusted semantic graph can generate an executable Rust service and a React interface. -7. The language semantics remain independent of Rust, React, natural-language wording, and any AI provider. -8. An approved project can build deterministically without an AI model or network connection. - -## 3. Non-Goals - -The MVP will not provide: - -- a complete general-purpose standard library; -- a package registry or public package manager; -- a direct LLVM backend; -- a self-hosted compiler; -- physical microcontroller flashing or hardware drivers; -- additional web frameworks beyond the React backend; -- native desktop or mobile UI backends; -- databases, authentication, or deployment infrastructure; -- advanced generics, macros, unrestricted concurrency, or a complete borrow checker; -- formal worst-case execution-time proof; -- a built-in AI chat interface or dependency on a specific model provider. - -These exclusions limit the first implementation, not the long-term language direction. - -## 4. Core Principles - -### 4.1 AI proposes; deterministic systems decide - -AI tools operate outside the trusted compiler boundary. They submit typed candidate changes and may use diagnostics to revise them. Only deterministic validation rules can promote a candidate into the trusted program. - -### 4.2 The semantic graph is the source of truth - -Natural-language views, diagrams, generated Rust, generated React, Dever IR, and machine code are projections or build artifacts. None may independently redefine program behavior. - -### 4.3 Natural language is presentation, not identity - -Every semantic object has an opaque stable ID. Localized labels and sentences never serve as symbol identity. Changing a translation cannot change executable behavior. - -### 4.4 No implicit behavior - -I/O, allocation, failure, state transitions, permissions, and fallback policies must be declared. Missing behavior produces a validation error rather than an AI-generated default. - -### 4.5 Backends are replaceable - -Rust and React are first-generation targets, not parts of the Dever language specification. Dever semantics and IR must not refer to Rust syntax, React hooks, JSX, or a particular runtime implementation. - -## 5. System Boundaries - -| Component | Responsibility | Must not own | -|---|---|---| -| AI adapter boundary | Accept typed semantic candidate patches from external AI tools | Model-specific prompts inside the compiler | -| Natural semantic workbench | Display, edit, compare, and approve semantic objects in the selected locale | Executable semantics hidden in UI-only state | -| Dever Core | Stable IDs, types, states, rules, capabilities, effects, constraints, and semantic hashing | Temperature-specific or platform-specific behavior | -| Dever Refiner | Deterministic validation, candidate promotion, and rejection diagnostics | Probabilistic acceptance decisions | -| Dever IR | Stable, language-independent executable representation | Rust, React, or localized wording | -| Rust backend | Bootstrap native/service output | Authority to change Dever semantics | -| React backend | Bootstrap web UI output | Authority to change Dever UI semantics | -| Temperature example | Domain rules and simulated device adapters | Compiler or refiner internals | - -Dependencies flow from examples and backends toward public Dever interfaces. Dever Core must never depend on the temperature example. - -## 6. Trusted Semantic Model - -The MVP semantic model contains these first-class objects: - -- project and module; -- scalar, record, enum, range, and optional types; -- input, output, and operation; -- capability and effect; -- state machine, state, event, transition, guard, and action; -- invariant, precondition, postcondition, and resource constraint; -- UI screen, field, control, binding, and interaction; -- candidate patch, provenance reference, validation evidence, and approval; -- code-scrap record and runtime-quarantine event. - -Each object has a stable ID, kind, typed properties, references to other stable IDs, and optional localized presentation data. - -For the MVP, the graph is persisted as canonical UTF-8 JSON with an explicit format version. Object keys are serialized in lexical order; unordered semantic collections are sorted by stable ID; ordered collections retain their declared semantic order; duplicate keys are invalid; and numbers use one normalized representation. Equivalent semantic graphs therefore produce identical behavior hashes. Localized labels and presentation preferences are stored separately and excluded from the behavior hash so translation edits cannot appear as executable changes. - -The workbench is the normal editing surface. The serialized form is intended for storage, tooling, and version control, not routine manual programming. - -## 7. Multilingual Natural Semantic Workbench - -The workbench is a manually implemented React application in the MVP. It is not generated by Dever itself. - -It provides: - -- project and module navigation; -- natural semantic summaries; -- rule and decision tables; -- state-machine views; -- capability and effect views; -- resource-constraint views; -- UI structure and interaction views; -- candidate semantic diffs; -- code-scrap diagnostics; -- runtime-quarantine events; -- approval state and validation evidence. - -The initial locale packs are Simplified Chinese and English. Both are deterministic renderers of the same semantic objects. The architecture permits later locale packs without changing Dever Core. - -Built-in phrases come from controlled locale templates. Domain terms may have localized labels, but their stable IDs and types remain unchanged. AI may propose translations, but translation is never part of compilation. - -The default experience contains no conventional implementation code. An optional expert diagnostic view may expose lowered forms, but it is not a trusted authoring surface. - -## 8. Candidate and Refinement Workflow - -External AI tools submit candidate patches containing typed operations such as: - -- add or remove a semantic object; -- update a typed property; -- connect or disconnect stable-ID references; -- add a state transition or rule; -- provide an implementation candidate for an approved operation; -- attach provenance to a requirement or approved contract. - -The refiner applies candidates only in an isolated candidate graph. It then performs deterministic checks for: - -- valid schema and stable-ID references; -- type correctness; -- valid states and transitions; -- authorized capabilities and effects; -- declared failure behavior; -- contract provenance for branches and fallbacks; -- duplicate or conflicting rules; -- unreachable semantic objects; -- dependency closure; -- MVP resource constraints, including forbidden dynamic allocation. - -A candidate is promoted atomically only when every required check passes and a human approves the semantic diff. Partial promotion is forbidden because selected fragments may not form a coherent program. - -AI-generated explanations are not validation evidence. Review summaries are derived mechanically from the candidate and trusted semantic graphs. - -## 9. Human and AI Collaboration - -Humans own: - -- goals and acceptance behavior; -- module boundaries and public operations; -- domain types, invariants, and state transitions; -- capability grants and resource policies; -- approval of semantic changes; -- explicit authorization of unsafe or platform-specific operations. - -AI tools own: - -- candidate implementations within approved boundaries; -- candidate tests and examples; -- revisions in response to deterministic diagnostics; -- internal optimizations that preserve approved contracts. - -The workbench review artifact must show at least: - -- the requirement or contract being implemented; -- added, removed, and changed semantic objects; -- state-machine changes; -- capability and effect changes; -- resource-policy changes; -- affected modules; -- validation evidence; -- rejected alternatives and their reasons when relevant. - -Routine review focuses on semantics and boundaries. High-risk operations can require an expanded expert review. - -## 10. Memory and Execution Model - -Dever has no mandatory tracing garbage collector. It distinguishes memory allocation from memory reclamation: - -- ownership and regions determine deterministic lifetime; -- transfer, temporary use, exclusive mutation, copying, sharing, and regional storage are explicit semantic choices; -- the refiner infers local ownership where the result is unique; -- ambiguous cross-module or long-lived ownership requires an explicit semantic decision; -- dynamic allocation is an effect that a target or module may forbid. - -The MVP needs only the subset required by the temperature example: value types, immutable data, explicit state ownership, and bounded event records. A complete general-purpose ownership and lifetime system is outside MVP scope. - -The Rust service may use heap storage where declared. A future microcontroller target can reject all allocation effects. React output executes within the browser's JavaScript runtime and therefore follows that platform's memory behavior without changing Dever Core semantics. - -## 11. Bootstrap and Backend Strategy - -The initial compiler, refiner, and service runtime are implemented in Rust. - -The first backend path is: - -1. Trusted semantic graph to Dever IR. -2. Dever IR to generated Rust. -3. Generated Rust to native output through the external Rust toolchain. - -Generated Rust is disposable build output. It is not trusted source and is not presented for routine human review. - -The first web path is: - -1. Trusted UI semantics to React and TypeScript build output. -2. Trusted service semantics to the Rust service. -3. Generated client bindings connect the two outputs without redefining domain behavior. - -The long-term path adds direct LLVM lowering and gradually rewrites Dever Core and compiler components in Dever. Rust remains a bootstrap implementation until self-hosting is reproducible. The Dever specification must remain independent throughout. - -## 12. The Two Isolation Areas - -### 12.1 Code scrap yard - -The code scrap yard is a build-time and review-time store for rejected candidates. A record includes the candidate, semantic base version, rejection reasons, provenance, and validation evidence. - -Scrap content: - -- never enters the trusted graph; -- is never compiled or deployed; -- cannot be promoted without revalidation against the current trusted graph; -- may be retained for audit or discarded by an explicit retention policy. - -### 12.2 Runtime quarantine - -Runtime quarantine receives values or events that violate an input contract. It is not a silent fallback channel. Each quarantine event records the rejected input, contract violation, source capability, and time or sequence metadata available on the target. - -The target policy determines whether quarantine uses a bounded buffer, callback, external report, or immediate stop. The MVP Rust service uses a bounded in-memory event list. A future microcontroller backend may use a fixed-size ring buffer or fail-stop behavior. - -Neither isolation area is a tracing-GC memory heap. - -## 13. Temperature Alarm Vertical Slice - -### 13.1 Domain contract - -| Semantic item | Approved behavior | -|---|---| -| Valid temperature | From -40 degrees Celsius through 125 degrees Celsius, inclusive | -| Initial state | Normal | -| Enter alarm | A valid reading greater than or equal to 80 degrees Celsius | -| Leave alarm | A valid reading below 75 degrees Celsius | -| Hysteresis interval | From 75 inclusive to below 80 retains the prior Normal or Alarm state | -| Sensor disconnect | Enter Offline and immediately enable the alarm | -| Recovery from Offline | A valid reading at or above 80 enters Alarm; a valid reading below 80 enters Normal | -| Invalid reading | Keep the current domain state and alarm output, reject the reading, and create a runtime-quarantine event | -| Alarm output | A simulated alarm capability displayed in the web UI | - -All thresholds and failure actions are explicit approved rules. The AI and refiner may not invent default readings, retries, state resets, or alarm suppression. - -### 13.2 Simulator behavior - -The example provides: - -- a manual temperature slider restricted to valid values; -- an automatic normal-fluctuation scenario; -- an automatic heating scenario that enters Alarm; -- an automatic cooling scenario that clears Alarm below 75 degrees; -- a sensor-disconnect injection; -- out-of-range and malformed-reading injections; -- current temperature, state, alarm output, state history, and quarantine views. - -No physical hardware is required. Simulated temperature input and alarm output implement example-owned capability adapters. They remain outside Dever Core. - -### 13.3 Generated outputs - -The trusted example semantics generate: - -- a Rust service that owns the state machine and enforces the approved contract; -- a React interface that drives simulations and displays trusted state and quarantine events; -- generated bindings between the interface and service; -- semantic source maps that relate backend diagnostics to stable semantic IDs. - -Generated files live under the example build output and are not committed as trusted source. - -## 14. Logical Repository Structure - -The MVP is one independent repository at `/data/project/dever` so tightly coupled compiler, workbench, backend, and example changes can be reviewed together. Logical packages remain separate: - -- `dever-core`; -- `dever-refiner`; -- `dever-ir`; -- `dever-backend-rust`; -- `dever-backend-react`; -- `dever-workbench`; -- `examples/temperature-alarm`. - -This list defines ownership boundaries, not a requirement to scaffold every directory before its implementation phase begins. - -Removing `examples/temperature-alarm` must not prevent the core, refiner, IR, backends, or workbench from building independently. - -## 15. Acceptance Criteria - -The MVP is accepted only when all of the following hold: - -1. Chinese and English views render the same stable semantic IDs and behavior hash. -2. A locale-only edit leaves the behavior hash unchanged. -3. A valid typed candidate produces a deterministic semantic diff and can be atomically approved. -4. An unjustified fallback, duplicate rule, invalid effect, or broken reference is rejected into the code scrap yard with a specific reason. -5. Approved semantics generate a working Rust service and React interface. -6. The manual slider drives valid readings through the generated service. -7. Automatic heating enters Alarm at 80 degrees or above. -8. Automatic cooling leaves Alarm only below 75 degrees. -9. Sensor disconnect enters Offline and enables the alarm. -10. Invalid readings preserve the current domain state and create runtime-quarantine events. -11. Generated backend diagnostics can be related to Dever stable semantic IDs. -12. With the declared toolchain and locked dependencies already available locally, the approved project can rebuild without access to an AI model or network service. -13. Generated Rust and React are disposable and do not become trusted source. -14. Dever Core contains no temperature-domain concepts and remains usable without the example. - -## 16. Principal Risks and Controls - -| Risk | Control | -|---|---| -| Natural wording becomes ambiguous | Controlled deterministic locale templates render typed semantic objects | -| AI explanation disagrees with behavior | Review artifacts are mechanically derived from semantic graphs | -| A second AI merely approves the first AI | Final acceptance is performed only by deterministic rules and human approval | -| Selected snippets form an incoherent program | Candidate promotion is atomic and dependency-closed | -| Rust semantics leak into Dever | Dever Core and IR define independent types, ownership, effects, and contracts | -| Generated code becomes the maintained source | Generated outputs are disposable and excluded from trusted project state | -| The MVP grows into a full language implementation | The non-goals are enforced until the vertical slice passes its acceptance criteria | - -## 17. Completion of the Design Phase - -After this document is reviewed and approved, the next artifact is an implementation plan that decomposes the vertical slice into independently testable milestones. No implementation should begin from this design document alone without that plan. diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..494092e --- /dev/null +++ b/examples/README.md @@ -0,0 +1,63 @@ +# Dever 示例 + +`examples/` 只保留两个入口：`cms/` 是当前维护的应用示例，`old/` 是旧语言合同和回归材料的归档。旧目录不符合当前 component/domain/role 应用布局，不应直接执行 `dever check/run/build`。 + +CMS 同时维护 [Dever 源码](cms/dever/module/news/article/api/admin/manage.dever)和 [Markdown 源码](cms/md/module/news/article/api/admin/manage.dever.md)。两套实现使用同一目录与业务合同： + +```text +module/ + platform/tenant/ 平台租户 + user/account/ 凭据、登录、bootstrap CMD + user/membership/ 租户成员 + user/session/ 登录会话 + user/authorization/ 核心角色授权的业务入口 + content/category/ 分类 + content/revision/ 文章编辑与修订快照 + media/asset/ 上传、Storage Port/Adapter + publishing/schedule/ 定时发布与 Job + audit/operation/ 操作审计 + system/health/ 健康 CMD + news/article/ + app/editing.dever + app/publication.dever + api/admin/manage.dever + api/front/browse.dever + domain.dever + model.dever + model/publication.dever +test/ + content/category/lifecycle.dever + news/article/publish.dever + user/account/bootstrap.dever + user/account/validation.dever + user/session/lifecycle.dever +``` + +Markdown 项目使用相同的目录和 API/CMD 声明，仅把源码扩展名换成 `.dever.md` 并补齐文档合同。 + +编译器从 API/CMD/Job 声明生成入口。平台租户、账号、成员与会话是 `global type`，保存在共享平台库；文章、分类、修订、媒体、排期和审计属于当前租户物理库。认证 verify 只读取 global Model 并建立只读用户与租户身份；接口权限由核心根据当前站点和角色自动校验，后续 API 才进入可信 tenant scope。一个职责文件可包含一组内聚方法，没有按函数拆文件。 + +文章通过 `/news/article/admin/manage/create` 创建，`list/detail` 查询，`remove` 删除草稿，`publish` 发布。编辑调用 `/content/revision/admin/manage/create`，传入 `selected_article_id`、`expected_version`、标题正文和可空分类/媒体 ID；版本冲突返回 409，文章更新和修订快照在同一事务内完成。文章不声明绕过这些规则的自动 REST 写接口。列表 SQL 直接返回同领域 `app.ArticleView`，不暴露私有 Model。 + +`/publishing/schedule/admin/manage/schedule` 接收文章 ID 和 `publish_at`；创建时验证作者与草稿状态，按真实到期时间入队。每个排期有独立任务键，同刻文章不会合并。任务执行仍由核心复验身份和权限；发布、审计和排期状态一并提交或回滚，完成任务重放不会重复写入。手动重复发布返回 409。 + +在仓库根目录执行： + +```bash +dever check examples/cms/dever +dever check examples/cms/md +dever test examples/cms/dever +dever test examples/cms/md +dever fmt examples/cms/dever --check +dever fmt examples/cms/md --check +dever run examples/cms/dever -- user.account.bootstrap '{"tenant_key":"tenant-one","tenant_name":"Tenant One","email":"owner@example.com","display_name":"Owner","password":"change-me-now"}' +dever tenant migrate examples/cms/dever 1 +dever tenant owner examples/cms/dever 1 admin 1 +dever tenant owner examples/cms/dever 1 front 1 +``` + +`run/build` 只读取各项目的 `config/setting.json`，运行数据写入各自 `data/`。先用 bootstrap CMD 建立平台租户、业务 Owner membership 与 Argon2 密码，再用返回的真实 `tenant_id` 执行 `tenant migrate`，并用返回的 `tenant_id/user_id` 分别 provision admin/front 核心 RBAC Owner；示例中的 `1` 只是首次空库的常见结果，不能在生产脚本里硬编码。两个站点各自通过同站点的 `user/authorization/<site>/manage` 接口管理普通角色。不带 CMD 参数时启动 admin/front HTTP API。`test` 不读取部署配置；每个涉及 Model 的测试使用独立临时 SQLite，并在结束后清理。 + +CMS 的五个应用测试验证分类、发布规则、账号引导/规范化和会话生命周期的正常业务输出，位于各项目根 `test/`；拒绝、过期等负向行为，以及带真实登录的 HTTP、并发编辑、上传、Job 重试/回滚验收见 [业务验收说明](../test/cms/README.md)。普通测试不伪造认证上下文。编译器和 CLI 的 Rust 回归测试仍统一位于仓库根 `test/`，两者都不混入业务 `module/`。部署前必须替换示例 JWT secret，并把两个站点的 `origin` 和 Host 改成真实 HTTPS 入口；`origin` 描述反向代理外部地址，不是 Dever 内部监听地址。 + +旧示例仍可供编译器维护者查阅历史语义，但不能作为新 Dever 应用的目录、可见性或调用方式范本。 diff --git a/examples/cms/dever/config/setting.json b/examples/cms/dever/config/setting.json new file mode 100644 index 0000000..c3cc00a --- /dev/null +++ b/examples/cms/dever/config/setting.json @@ -0,0 +1,56 @@ +{ + "runtime": { + "mode": "all", + "shutdown_ms": 30000 + }, + "job": { + "workers": 1, + "poll_ms": 1000, + "lease_ms": 60000, + "retry_base_ms": 1000, + "retry_max_ms": 300000 + }, + "http": { + "host": "127.0.0.1", + "port": 8080 + }, + "database": { + "default": { + "type": "sqlite", + "path": "data/db/platform.db", + "tenant_directory": "data/tenants", + "max_connections": 4, + "max_page_size": 100 + } + }, + "tenant": { + "database": "default", + "max_pools": 32, + "idle_timeout_ms": 300000 + }, + "auth": { + "providers": { + "session": { + "verify": "user.account.verify", + "jwtSecret": "replace-this-example-secret-before-deployment-2026", + "ttlSeconds": 3600, + "cookie": "cms_session" + } + } + }, + "sites": { + "admin": { + "path": "admin", + "auth": "session", + "origin": "https://cms.example.com" + }, + "front": { + "path": "front", + "auth": "session", + "origin": "https://cms.example.com" + } + }, + "log": { + "level": "info" + } +} diff --git a/examples/cms/dever/data/.gitkeep b/examples/cms/dever/data/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/examples/cms/dever/data/.gitkeep @@ -0,0 +1 @@ + diff --git a/examples/cms/dever/module/audit/operation/app.dever b/examples/cms/dever/module/audit/operation/app.dever new file mode 100644 index 0000000..f50c645 --- /dev/null +++ b/examples/cms/dever/module/audit/operation/app.dever @@ -0,0 +1,10 @@ +record(kind: model.OperationKind, resource: Text, resource_id: Text) () { + stored = model.create( + { + kind = kind + resource = resource + resource_id = resource_id + } + ) + discarded = stored +} diff --git a/examples/cms/dever/module/audit/operation/model.dever b/examples/cms/dever/module/audit/operation/model.dever new file mode 100644 index 0000000..351556a --- /dev/null +++ b/examples/cms/dever/module/audit/operation/model.dever @@ -0,0 +1,14 @@ +type OperationKind { + Login = "登录" + Logout = "登出" + Publish = "发布" + Upload = "上传" + Access = "授权" +} + +type Operation { + actor_id: user.account.model.principal.id? + kind: OperationKind index + resource: Text(1, 120) + resource_id: Text(1, 120) +} diff --git a/examples/cms/dever/module/content/category/api/admin/manage.dever b/examples/cms/dever/module/content/category/api/admin/manage.dever new file mode 100644 index 0000000..491e23b --- /dev/null +++ b/examples/cms/dever/module/content/category/api/admin/manage.dever @@ -0,0 +1,3 @@ +post create = app.create + +delete disable = app.disable diff --git a/examples/cms/dever/module/content/category/app.dever b/examples/cms/dever/module/content/category/app.dever new file mode 100644 index 0000000..22c0743 --- /dev/null +++ b/examples/cms/dever/module/content/category/app.dever @@ -0,0 +1,41 @@ +type CategoryView { + id: model.id + name: Text + slug: Text + status: Text +} + +create(name: Text) (category: CategoryView) { + normalized = domain.normalize_name(name) + domain.require_name(normalized) + stored = model.create( + { + name = normalized + slug = domain.normalize_slug(normalized) + } + ) + category = CategoryView { + id = stored.id + name = stored.name + slug = stored.slug + status = "active" + } +} + +disable(id: model.id) (ok: Bool) { + changed = model.update( + id, + { + status = model.CategoryStatus.Disabled + } + ) + discarded = changed + ok = true +} + +require_active(id: model.id) () { + stored = model.get(id) + domain.require_active(stored.status == model.CategoryStatus.Active) +} + +require_active(id: null) () {} diff --git a/examples/cms/dever/module/content/category/domain.dever b/examples/cms/dever/module/content/category/domain.dever new file mode 100644 index 0000000..c7dcfc0 --- /dev/null +++ b/examples/cms/dever/module/content/category/domain.dever @@ -0,0 +1,23 @@ +normalize_name(value: Text) (name: Text) pure { + name = text.trim(value) +} + +normalize_slug(value: Text) (slug: Text) pure { + slug = text.lower(text.trim(value)) +} + +require_name(value: Text) () pure { + check_name(value != "") +} + +check_name(value: true) () pure {} + +check_name(value: false) () pure { + fail(dever.api.Error.Invalid) +} + +require_active(active: true) () pure {} + +require_active(active: false) () pure { + fail(dever.api.Error.Conflict) +} diff --git a/examples/cms/dever/module/content/category/model.dever b/examples/cms/dever/module/content/category/model.dever new file mode 100644 index 0000000..0380ce6 --- /dev/null +++ b/examples/cms/dever/module/content/category/model.dever @@ -0,0 +1,10 @@ +type CategoryStatus { + Active = "启用" + Disabled = "禁用" +} + +type Category { + name: Text(1, 80) + slug: Text(1, 100) unique + status: CategoryStatus default CategoryStatus.Active index +} diff --git a/examples/cms/dever/module/content/revision/api/admin/manage.dever b/examples/cms/dever/module/content/revision/api/admin/manage.dever new file mode 100644 index 0000000..5ea4a25 --- /dev/null +++ b/examples/cms/dever/module/content/revision/api/admin/manage.dever @@ -0,0 +1 @@ +post create = app.create diff --git a/examples/cms/dever/module/content/revision/app.dever b/examples/cms/dever/module/content/revision/app.dever new file mode 100644 index 0000000..c2ade52 --- /dev/null +++ b/examples/cms/dever/module/content/revision/app.dever @@ -0,0 +1,45 @@ +type RevisionView { + id: model.id + article_id: news.article.model.id + version: Int + title: Text + body: Text + status: Text +} + +transaction create( + selected_article_id: news.article.model.id, + expected_version: Int, + title: Text, + body: Text, + category_id: content.category.model.id?, + asset_id: media.asset.model.id? +) (revision: RevisionView) { + edited = news.article.edit( + selected_article_id, + expected_version, + title, + body, + category_id, + asset_id + ) + stored = model.create( + { + article_id = selected_article_id + version = edited.version + title = edited.title + body = edited.body + category_id = edited.category_id + asset_id = edited.asset_id + status = model.RevisionStatus.Draft + } + ) + revision = RevisionView { + id = stored.id + article_id = stored.article_id + version = stored.version + title = stored.title + body = stored.body + status = "draft" + } +} diff --git a/examples/cms/dever/module/content/revision/model.dever b/examples/cms/dever/module/content/revision/model.dever new file mode 100644 index 0000000..76efa0f --- /dev/null +++ b/examples/cms/dever/module/content/revision/model.dever @@ -0,0 +1,16 @@ +type RevisionStatus { + Draft = "草稿" + Published = "已发布" +} + +type Revision { + article_id: news.article.model.id index + version: Int + title: Text(1, 160) + body: Text + category_id: content.category.model.id? + asset_id: media.asset.model.id? + status: RevisionStatus default RevisionStatus.Draft index +} + +unique(article_id, version) diff --git a/examples/cms/dever/module/media/asset/adapter.dever b/examples/cms/dever/module/media/asset/adapter.dever new file mode 100644 index 0000000..0bb2c05 --- /dev/null +++ b/examples/cms/dever/module/media/asset/adapter.dever @@ -0,0 +1,3 @@ +port.store(file: Upload) (key: Uuid) { + key = dever.storage.put(file) +} diff --git a/examples/cms/dever/module/media/asset/api/admin/manage.dever b/examples/cms/dever/module/media/asset/api/admin/manage.dever new file mode 100644 index 0000000..94a63f6 --- /dev/null +++ b/examples/cms/dever/module/media/asset/api/admin/manage.dever @@ -0,0 +1 @@ +post upload = app.store diff --git a/examples/cms/dever/module/media/asset/app.dever b/examples/cms/dever/module/media/asset/app.dever new file mode 100644 index 0000000..3c67272 --- /dev/null +++ b/examples/cms/dever/module/media/asset/app.dever @@ -0,0 +1,43 @@ +type StoreResult { + id: model.id + storage_key: Uuid + filename: Text + content_type: Text + size: Int +} + +store(file: Upload) (result: StoreResult) { + filename = dever.api.upload_filename(file) + content_type = dever.api.upload_content_type(file) + size = dever.api.upload_size(file) + key = port.store(file) + asset = model.create( + { + storage_key = key + filename = filename + content_type = content_type + size = size + } + ) + audit.operation.record(audit.operation.model.OperationKind.Upload, "asset", filename) + result = StoreResult { + id = asset.id + storage_key = asset.storage_key + filename = asset.filename + content_type = asset.content_type + size = asset.size + } +} + +require_ready(id: model.id) () { + stored = model.get(id) + ready(stored.status == model.AssetStatus.Ready) +} + +require_ready(id: null) () {} + +ready(available: true) () {} + +ready(available: false) () { + fail(dever.api.Error.Conflict) +} diff --git a/examples/cms/dever/module/media/asset/model.dever b/examples/cms/dever/module/media/asset/model.dever new file mode 100644 index 0000000..715867b --- /dev/null +++ b/examples/cms/dever/module/media/asset/model.dever @@ -0,0 +1,12 @@ +type AssetStatus { + Ready = "就绪" + Removed = "已删除" +} + +type Asset { + storage_key: Uuid unique + filename: Text(1, 255) + content_type: Text(1, 160) + size: Int + status: AssetStatus default AssetStatus.Ready index +} diff --git a/examples/cms/dever/module/media/asset/port.dever b/examples/cms/dever/module/media/asset/port.dever new file mode 100644 index 0000000..252efd2 --- /dev/null +++ b/examples/cms/dever/module/media/asset/port.dever @@ -0,0 +1 @@ +store(file: Upload) (key: Uuid) fails dever.storage.PutResult diff --git a/examples/cms/dever/module/news/article/api/admin/manage.dever b/examples/cms/dever/module/news/article/api/admin/manage.dever new file mode 100644 index 0000000..3ce98a5 --- /dev/null +++ b/examples/cms/dever/module/news/article/api/admin/manage.dever @@ -0,0 +1,9 @@ +get list = app.list + +get detail = app.detail + +post create = app.create + +delete remove = app.remove + +post publish = app.publish diff --git a/examples/cms/dever/module/news/article/api/front/browse.dever b/examples/cms/dever/module/news/article/api/front/browse.dever new file mode 100644 index 0000000..5b2584c --- /dev/null +++ b/examples/cms/dever/module/news/article/api/front/browse.dever @@ -0,0 +1,3 @@ +get list = app.published + +get detail = app.published_detail diff --git a/examples/cms/dever/module/news/article/app/editing.dever b/examples/cms/dever/module/news/article/app/editing.dever new file mode 100644 index 0000000..99409e6 --- /dev/null +++ b/examples/cms/dever/module/news/article/app/editing.dever @@ -0,0 +1,114 @@ +type ArticleView { + id: model.id + slug: Text + title: Text + body: Text + version: Int + category_id: content.category.model.id? + asset_id: media.asset.model.id? +} + +type ArticlePage { + rows: List<ArticleView> + total: Int +} + +create( + slug: Text, + title: Text, + body: Text, + category_id: content.category.model.id?, + asset_id: media.asset.model.id? +) (article: ArticleView) { + normalized_slug = domain.normalize_slug(slug) + normalized_title = domain.normalize_title(title) + domain.require_content(normalized_slug, normalized_title, body) + content.category.require_active(category_id) + media.asset.require_ready(asset_id) + stored = model.create( + { + author_id = user.account.current_user() + slug = normalized_slug + title = normalized_title + body = body + category_id = category_id + asset_id = asset_id + } + ) + article = detail(stored.id) +} + +list(size: Int) (articles: ArticlePage) { + author = user.account.current_user() + articles = ArticlePage { + rows = model.owned_articles(author, domain.page_size(size)) + total = model.count( + { + where = author_id == author + } + ) + } +} + +detail(id: model.id) (article: ArticleView) { + stored = model.get(id) + domain.require_owner(dever.auth.owns_user(stored.author_id)) + article = ArticleView { + id = stored.id + slug = stored.slug + title = stored.title + body = stored.body + version = stored.version + category_id = stored.category_id + asset_id = stored.asset_id + } +} + +remove(id: model.id) (removed: Int) { + stored = model.get(id) + domain.require_owner(dever.auth.owns_user(stored.author_id)) + domain.require_draft(stored.status == model.ArticleStatus.Draft) + removed = model.delete( + { + where = id == stored.id and version == stored.version + } + ) + domain.require_updated(removed) +} + +edit( + id: model.id, + expected_version: Int, + title: Text, + body: Text, + category_id: content.category.model.id?, + asset_id: media.asset.model.id? +) (article: ArticleView) { + stored = model.get(id) + domain.require_owner(dever.auth.owns_user(stored.author_id)) + domain.require_draft(stored.status == model.ArticleStatus.Draft) + normalized_title = domain.normalize_title(title) + domain.require_content(stored.slug, normalized_title, body) + content.category.require_active(category_id) + media.asset.require_ready(asset_id) + changed = model.update( + { + where = id == stored.id and version == expected_version and status == model.ArticleStatus.Draft + }, + { + title = normalized_title + body = body + category_id = category_id + asset_id = asset_id + version = expected_version + 1 + } + ) + domain.require_updated(changed) + article = detail(stored.id) +} + +require_schedulable(article_id: model.id) () { + stored = model.get(article_id) + domain.require_owner(dever.auth.owns_user(stored.author_id)) + domain.require_draft(stored.status == model.ArticleStatus.Draft) +} diff --git a/examples/cms/dever/module/news/article/app/publication.dever b/examples/cms/dever/module/news/article/app/publication.dever new file mode 100644 index 0000000..ab8465b --- /dev/null +++ b/examples/cms/dever/module/news/article/app/publication.dever @@ -0,0 +1,95 @@ +transaction publish(article_id: model.id) (publication: model.publication.Publication) { + stored = model.get(article_id) + domain.require_owner(dever.auth.owns_user(stored.author_id)) + domain.require_draft(stored.status == model.ArticleStatus.Draft) + publication = publish_draft(article_id) +} + +published(size: Int) (articles: model.publication.Page) { + articles = model.publication.list( + { + order = [published_at.desc, id.desc] + size = domain.page_size(size) + } + ) +} + +published_detail(id: model.publication.id) (article: model.publication.Publication) { + article = model.publication.get(id) + domain.require_published(text.trim(article.body) != "") +} + +transaction publish_scheduled(article_id: model.id) ( + publication: model.publication.Publication +) { + stored = model.get(article_id) + publication = scheduled_publication(stored.status, article_id) +} + +scheduled_publication(status: model.ArticleStatus.Published, article_id: model.id) ( + publication: model.publication.Publication +) { + stored = model.get(article_id) + publication = existing_publication( + model.publication.first( + { + where = slug == stored.slug + } + ) + ) +} + +scheduled_publication(status: model.ArticleStatus.Draft, article_id: model.id) ( + publication: model.publication.Publication +) { + publication = publish_draft(article_id) +} + +publish_draft(article_id: model.id) (publication: model.publication.Publication) { + stored = model.get(article_id) + domain.require_draft(stored.status == model.ArticleStatus.Draft) + content.category.require_active(stored.category_id) + media.asset.require_ready(stored.asset_id) + published_at = dever.time.now() + changed = model.update( + { + where = id == stored.id and version == stored.version and status == model.ArticleStatus.Draft + }, + { + status = model.ArticleStatus.Published + published_at = published_at + version = stored.version + 1 + } + ) + domain.require_updated(changed) + audit.operation.record( + audit.operation.model.OperationKind.Publish, + "article", + stored.slug + ) + publication = model.publication.upsert( + { + slug = stored.slug + }, + { + title = stored.title + body = stored.body + published_at = published_at + }, + { + title = stored.title + body = stored.body + published_at = published_at + } + ) +} + +existing_publication(stored: model.publication.Publication) ( + publication: model.publication.Publication +) { + publication = stored +} + +existing_publication(stored: null) (publication: model.publication.Publication) { + fail(dever.api.Error.NotFound) +} diff --git a/examples/cms/dever/module/news/article/domain.dever b/examples/cms/dever/module/news/article/domain.dever new file mode 100644 index 0000000..809b22f --- /dev/null +++ b/examples/cms/dever/module/news/article/domain.dever @@ -0,0 +1,49 @@ +normalize_slug(value: Text) (slug: Text) pure { + slug = text.lower(text.trim(value)) +} + +normalize_title(value: Text) (title: Text) pure { + title = text.trim(value) +} + +page_size(value: Int >= 1 and <= 100) (size: Int) pure { + size = value +} + +page_size(value: other) (size: Int) pure { + fail(dever.api.Error.Invalid) +} + +require_draft(draft: true) () pure {} + +require_draft(draft: false) () pure { + fail(dever.api.Error.Conflict) +} + +require_owner(owned: true) () pure {} + +require_owner(owned: false) () pure { + fail(dever.api.Error.Forbidden) +} + +require_published(published: true) () pure {} + +require_published(published: false) () pure { + fail(dever.api.Error.NotFound) +} + +require_updated(changed: 1) () pure {} + +require_updated(changed: other) () pure { + fail(dever.api.Error.Conflict) +} + +require_content(slug: Text, title: Text, body: Text) () pure { + require_valid(slug != "" and title != "" and text.trim(body) != "") +} + +require_valid(valid: true) () pure {} + +require_valid(valid: false) () pure { + fail(dever.api.Error.Invalid) +} diff --git a/examples/cms/dever/module/news/article/model.dever b/examples/cms/dever/module/news/article/model.dever new file mode 100644 index 0000000..664a965 --- /dev/null +++ b/examples/cms/dever/module/news/article/model.dever @@ -0,0 +1,25 @@ +type ArticleStatus { + Draft = "草稿" + Published = "已发布" +} + +type Article { + private owner author_id: user.account.model.principal.id = dever.auth.user_id() + slug: Text(1, 180) unique + title: Text(1, 160) + body: Text + category_id: content.category.model.id? + asset_id: media.asset.model.id? + private version: Int default 1 + private status: ArticleStatus default ArticleStatus.Draft index + private published_at: DateTime? +} + +index(author_id, created_at) + +sql owned_articles(author: user.account.model.principal.id, size: Int) ( + articles: List<app.ArticleView> +) { + sqlite = "SELECT id, slug, title, body, version, category_id, asset_id FROM article WHERE author_id = ?1 ORDER BY id DESC LIMIT ?2" + postgres = "SELECT id, slug, title, body, version, category_id, asset_id FROM article WHERE author_id = $1 ORDER BY id DESC LIMIT $2" +} diff --git a/examples/cms/dever/module/news/article/model/publication.dever b/examples/cms/dever/module/news/article/model/publication.dever new file mode 100644 index 0000000..1ff42f4 --- /dev/null +++ b/examples/cms/dever/module/news/article/model/publication.dever @@ -0,0 +1,6 @@ +type Publication { + slug: Text(1, 180) unique + title: Text(1, 160) + body: Text + published_at: DateTime index +} diff --git a/examples/cms/dever/module/platform/tenant/app.dever b/examples/cms/dever/module/platform/tenant/app.dever new file mode 100644 index 0000000..74ff4cf --- /dev/null +++ b/examples/cms/dever/module/platform/tenant/app.dever @@ -0,0 +1,41 @@ +type TenantView { + id: model.id + key: Text + name: Text +} + +provision(key: Text, name: Text) (tenant: TenantView) { + stored = model.create( + { + key = domain.normalize_key(key) + name = domain.normalize_name(name) + } + ) + tenant = TenantView { + id = stored.id + key = stored.key + name = stored.name + } +} + +active(tenant_key: Text) (tenant: TenantView) { + stored = model.first( + { + where = key == domain.normalize_key(tenant_key) + } + ) + tenant = active_view(stored) +} + +active_view(stored: model.Tenant) (tenant: TenantView) { + domain.require_active(stored.status == model.TenantStatus.Active) + tenant = TenantView { + id = stored.id + key = stored.key + name = stored.name + } +} + +active_view(stored: null) (tenant: TenantView) { + fail(dever.api.Error.Unauthorized) +} diff --git a/examples/cms/dever/module/platform/tenant/domain.dever b/examples/cms/dever/module/platform/tenant/domain.dever new file mode 100644 index 0000000..62c78ea --- /dev/null +++ b/examples/cms/dever/module/platform/tenant/domain.dever @@ -0,0 +1,13 @@ +normalize_key(value: Text) (key: Text) pure { + key = text.lower(text.trim(value)) +} + +normalize_name(value: Text) (name: Text) pure { + name = text.trim(value) +} + +require_active(active: true) () pure {} + +require_active(active: false) () pure { + fail(dever.api.Error.Unauthorized) +} diff --git a/examples/cms/dever/module/platform/tenant/model.dever b/examples/cms/dever/module/platform/tenant/model.dever new file mode 100644 index 0000000..0648b14 --- /dev/null +++ b/examples/cms/dever/module/platform/tenant/model.dever @@ -0,0 +1,11 @@ +type TenantStatus { + Active = "启用" + Disabled = "禁用" +} + +global type Tenant { + uuid: Uuid generated unique + key: Text(1, 64) unique + name: Text(1, 100) + status: TenantStatus default TenantStatus.Active index +} diff --git a/examples/cms/dever/module/publishing/schedule/api/admin/manage.dever b/examples/cms/dever/module/publishing/schedule/api/admin/manage.dever new file mode 100644 index 0000000..98a524a --- /dev/null +++ b/examples/cms/dever/module/publishing/schedule/api/admin/manage.dever @@ -0,0 +1 @@ +post schedule = app.create diff --git a/examples/cms/dever/module/publishing/schedule/app.dever b/examples/cms/dever/module/publishing/schedule/app.dever new file mode 100644 index 0000000..2cbd7ac --- /dev/null +++ b/examples/cms/dever/module/publishing/schedule/app.dever @@ -0,0 +1,76 @@ +type PublishRequest { + schedule_id: model.id +} + +type ScheduleView { + id: model.id + article_id: news.article.model.id + publish_at: DateTime + status: Text +} + +transaction create(article_id: news.article.model.id, publish_at: DateTime) ( + schedule: ScheduleView +) { + news.article.require_schedulable(article_id) + stored = model.create( + { + article_id = article_id + publish_at = publish_at + } + ) + queued = dever.job.enqueue_at( + job.publish, + PublishRequest { + schedule_id = stored.id + }, + uuid.to_text(stored.uuid), + publish_at + ) + discarded = queued + schedule = ScheduleView { + id = stored.id + article_id = stored.article_id + publish_at = stored.publish_at + status = "pending" + } +} + +transaction publish(input: PublishRequest) () { + scheduled = model.get(input.schedule_id) + complete(scheduled.status, scheduled.id) +} + +complete(status: model.ScheduleStatus.Completed, schedule_id: model.id) () {} + +complete(status: model.ScheduleStatus.Cancelled, schedule_id: model.id) () {} + +complete(status: model.ScheduleStatus.Pending, schedule_id: model.id) () { + scheduled = model.get(schedule_id) + domain.require_due( + model.exists( + { + where = id == schedule_id and publish_at <= dever.time.now() + } + ) + ) + changed = model.update( + { + where = id == scheduled.id and status == model.ScheduleStatus.Pending + }, + { + status = model.ScheduleStatus.Completed + } + ) + publish_claimed(changed, scheduled.article_id) +} + +publish_claimed(changed: 0, article_id: news.article.model.id) () {} + +publish_claimed(changed: 1, article_id: news.article.model.id) () { + publication = news.article.publish_scheduled(article_id) +} + +publish_claimed(changed: other, article_id: news.article.model.id) () { + fail(dever.api.Error.Conflict) +} diff --git a/examples/cms/dever/module/publishing/schedule/domain.dever b/examples/cms/dever/module/publishing/schedule/domain.dever new file mode 100644 index 0000000..264f740 --- /dev/null +++ b/examples/cms/dever/module/publishing/schedule/domain.dever @@ -0,0 +1,5 @@ +require_due(valid: true) () pure {} + +require_due(valid: false) () pure { + fail(dever.api.Error.Invalid) +} diff --git a/examples/cms/dever/module/publishing/schedule/job.dever b/examples/cms/dever/module/publishing/schedule/job.dever new file mode 100644 index 0000000..d6e95d7 --- /dev/null +++ b/examples/cms/dever/module/publishing/schedule/job.dever @@ -0,0 +1,5 @@ +database default + +job publish(input: app.PublishRequest) () retry(3) timeout(30000) { + app.publish(input) +} diff --git a/examples/cms/dever/module/publishing/schedule/model.dever b/examples/cms/dever/module/publishing/schedule/model.dever new file mode 100644 index 0000000..c24cd9f --- /dev/null +++ b/examples/cms/dever/module/publishing/schedule/model.dever @@ -0,0 +1,12 @@ +type ScheduleStatus { + Pending = "待发布" + Completed = "已完成" + Cancelled = "已取消" +} + +type Schedule { + uuid: Uuid generated unique + article_id: news.article.model.id unique + publish_at: DateTime index + status: ScheduleStatus default ScheduleStatus.Pending index +} diff --git a/examples/cms/dever/module/system/health/api.dever b/examples/cms/dever/module/system/health/api.dever new file mode 100644 index 0000000..795781c --- /dev/null +++ b/examples/cms/dever/module/system/health/api.dever @@ -0,0 +1 @@ +cmd ping = app.ping diff --git a/examples/cms/dever/module/system/health/app.dever b/examples/cms/dever/module/system/health/app.dever new file mode 100644 index 0000000..b7e6e67 --- /dev/null +++ b/examples/cms/dever/module/system/health/app.dever @@ -0,0 +1,9 @@ +type HealthView { + status: Text +} + +ping() (result: HealthView) { + result = HealthView { + status = "ready" + } +} diff --git a/examples/cms/dever/module/user/account/api.dever b/examples/cms/dever/module/user/account/api.dever new file mode 100644 index 0000000..7a12000 --- /dev/null +++ b/examples/cms/dever/module/user/account/api.dever @@ -0,0 +1 @@ +cmd bootstrap = app.bootstrap diff --git a/examples/cms/dever/module/user/account/api/admin/session.dever b/examples/cms/dever/module/user/account/api/admin/session.dever new file mode 100644 index 0000000..36e5f79 --- /dev/null +++ b/examples/cms/dever/module/user/account/api/admin/session.dever @@ -0,0 +1,3 @@ +public post login = app.login + +post logout = app.logout diff --git a/examples/cms/dever/module/user/account/api/front/session.dever b/examples/cms/dever/module/user/account/api/front/session.dever new file mode 100644 index 0000000..36e5f79 --- /dev/null +++ b/examples/cms/dever/module/user/account/api/front/session.dever @@ -0,0 +1,3 @@ +public post login = app.login + +post logout = app.logout diff --git a/examples/cms/dever/module/user/account/app.dever b/examples/cms/dever/module/user/account/app.dever new file mode 100644 index 0000000..c3bd7c4 --- /dev/null +++ b/examples/cms/dever/module/user/account/app.dever @@ -0,0 +1,157 @@ +type AccountView { + id: model.principal.id + uuid: Uuid + email: Text + display_name: Text +} + +type BootstrapView { + tenant_id: platform.tenant.model.id + user_id: model.principal.id + credential_id: model.id +} + +type LoginView { + account: AccountView + tenant_id: platform.tenant.model.id + site: Text +} + +type LogoutView { + revoked: Bool +} + +type Identity { + id: Text + user_id: model.principal.id? + tenant_id: platform.tenant.model.id? +} + +bootstrap( + tenant_key: Text, + tenant_name: Text, + email: Text, + display_name: Text, + password: Secret +) (result: BootstrapView) { + tenant = platform.tenant.provision(tenant_key, tenant_name) + input = domain.prepare(email, display_name) + hash = blocking(dever.crypto.password_hash(password)) + principal = model.principal.create( + { + display_name = input.name + } + ) + stored = model.create( + { + principal_id = principal.id + email = input.address + password_hash = hash + } + ) + user.membership.grant_owner(principal.id, tenant.id) + result = BootstrapView { + tenant_id = tenant.id + user_id = principal.id + credential_id = stored.id + } +} + +login(credential_id: model.id, password: Secret, tenant: Text) (result: LoginView) { + domain.require_active( + model.exists( + { + where = id == credential_id + } + ) + ) + stored = model.get(credential_id) + principal = model.principal.get(stored.principal_id) + selected = platform.tenant.active(tenant) + domain.require_active(stored.status == model.AccountStatus.Active) + domain.require_active(principal.status == model.principal.PrincipalStatus.Active) + domain.require_active( + blocking(dever.crypto.password_verify(password, stored.password_hash)) + ) + authenticated = establish_login(credential_id, selected.key, dever.site.key()) + dever.auth.issue_cookie( + authenticated.subject, + uuid.to_text(authenticated.session_id), + authenticated.tenant_key + ) + result = authenticated.result +} + +transaction establish_login(credential_id: model.id, tenant: Text, site: Text) ( + result: LoginView, + subject: Text, + session_id: Uuid, + tenant_key: Text +) { + stored = model.get(credential_id) + principal = model.principal.get(stored.principal_id) + selected = platform.tenant.active(tenant) + domain.require_active(stored.status == model.AccountStatus.Active) + domain.require_active(principal.status == model.principal.PrincipalStatus.Active) + user.membership.require_active(principal.id, selected.id) + session = user.session.open( + uuid.to_text(principal.uuid), + principal.id, + selected.id, + selected.key, + site + ) + result = LoginView { + account = AccountView { + id = principal.id + uuid = principal.uuid + email = stored.email + display_name = principal.display_name + } + tenant_id = selected.id + site = site + } + subject = uuid.to_text(principal.uuid) + session_id = session.uuid + tenant_key = selected.key +} + +logout() (result: LogoutView) { + user.session.revoke(uuid.parse(dever.auth.session())) + dever.auth.clear_cookie() + result = LogoutView { + revoked = true + } +} + +verify(claims: dever.auth.Claims) (identity: Identity) { + session = user.session.authenticate(uuid.parse(claims.session), claims.site) + domain.require_active(session.subject == claims.subject) + domain.require_tenant(session.tenant_key, claims.tenant) + selected = platform.tenant.active(session.tenant_key) + domain.require_active(selected.id == session.tenant_id) + active(session.user_id) + user.membership.require_active(session.user_id, session.tenant_id) + identity = Identity { + id = claims.subject + user_id = session.user_id + tenant_id = session.tenant_id + } +} + +active(user_id: model.principal.id) () { + principal = model.principal.get(user_id) + domain.require_active(principal.status == model.principal.PrincipalStatus.Active) +} + +current_user() (id: model.principal.id) { + id = require_user(dever.auth.user_id()) +} + +require_user(value: model.principal.id) (id: model.principal.id) { + id = value +} + +require_user(value: null) (id: model.principal.id) { + fail(dever.api.Error.Unauthorized) +} diff --git a/examples/cms/dever/module/user/account/domain.dever b/examples/cms/dever/module/user/account/domain.dever new file mode 100644 index 0000000..f6be1b5 --- /dev/null +++ b/examples/cms/dever/module/user/account/domain.dever @@ -0,0 +1,49 @@ +type AccountInput { + address: Text + name: Text +} + +normalize_email(email: Text) (address: Text) pure { + address = text.lower(text.trim(email)) +} + +normalize_name(display_name: Text) (name: Text) pure { + name = text.trim(display_name) +} + +prepare(email: Text, display_name: Text) (input: AccountInput) pure { + address = normalize_email(email) + name = normalize_name(display_name) + require( + text.contains(address, "@") and not text.starts_with(address, "@") and not text.ends_with( + address, + "@" + ), + "email must contain a local part and domain" + ) + require(not text.is_empty(name), "display name cannot be blank") + input = AccountInput { + address = address + name = name + } +} + +require(valid: true, message: Text) () pure {} + +require(valid: false, message: Text) () pure { + fail(dever.api.Error.Invalid) +} + +require_active(active: true) () pure {} + +require_active(active: false) () pure { + fail(dever.api.Error.Unauthorized) +} + +require_tenant(expected: Text, actual: Text) () pure { + require_active(expected == actual) +} + +require_tenant(expected: Text, actual: null) () pure { + fail(dever.api.Error.Unauthorized) +} diff --git a/examples/cms/dever/module/user/account/model.dever b/examples/cms/dever/module/user/account/model.dever new file mode 100644 index 0000000..0228fa7 --- /dev/null +++ b/examples/cms/dever/module/user/account/model.dever @@ -0,0 +1,11 @@ +type AccountStatus { + Active = "启用" + Disabled = "禁用" +} + +global type Account { + principal_id: user.account.model.principal.id unique + email: Text(254) unique + private password_hash: Text(255) + status: AccountStatus default AccountStatus.Active index +} diff --git a/examples/cms/dever/module/user/account/model/principal.dever b/examples/cms/dever/module/user/account/model/principal.dever new file mode 100644 index 0000000..b662b44 --- /dev/null +++ b/examples/cms/dever/module/user/account/model/principal.dever @@ -0,0 +1,10 @@ +type PrincipalStatus { + Active = "启用" + Disabled = "禁用" +} + +global type Principal { + uuid: Uuid generated unique + display_name: Text(1, 64) + status: PrincipalStatus default PrincipalStatus.Active index +} diff --git a/examples/cms/dever/module/user/authorization/api/admin/manage.dever b/examples/cms/dever/module/user/authorization/api/admin/manage.dever new file mode 100644 index 0000000..e918e84 --- /dev/null +++ b/examples/cms/dever/module/user/authorization/api/admin/manage.dever @@ -0,0 +1,9 @@ +get permissions = app.permissions + +post save = app.save + +post grant = app.grant + +delete revoke = app.revoke + +delete disable = app.disable diff --git a/examples/cms/dever/module/user/authorization/api/front/manage.dever b/examples/cms/dever/module/user/authorization/api/front/manage.dever new file mode 100644 index 0000000..e918e84 --- /dev/null +++ b/examples/cms/dever/module/user/authorization/api/front/manage.dever @@ -0,0 +1,9 @@ +get permissions = app.permissions + +post save = app.save + +post grant = app.grant + +delete revoke = app.revoke + +delete disable = app.disable diff --git a/examples/cms/dever/module/user/authorization/app.dever b/examples/cms/dever/module/user/authorization/app.dever new file mode 100644 index 0000000..65b39a4 --- /dev/null +++ b/examples/cms/dever/module/user/authorization/app.dever @@ -0,0 +1,25 @@ +permissions() (values: List<dever.auth.Permission>) { + values = dever.auth.permissions() +} + +save(id: Text, name: Text, all_permissions: Bool, permission_keys: List<Text>) ( + ok: Bool +) { + dever.auth.save_role(id, name, all_permissions, permission_keys) + ok = true +} + +grant(user_id: Int, role_id: Text) (ok: Bool) { + dever.auth.grant_role(user_id, role_id) + ok = true +} + +revoke(user_id: Int, role_id: Text) (ok: Bool) { + dever.auth.revoke_role(user_id, role_id) + ok = true +} + +disable(role_id: Text) (ok: Bool) { + dever.auth.disable_role(role_id) + ok = true +} diff --git a/examples/cms/dever/module/user/membership/app.dever b/examples/cms/dever/module/user/membership/app.dever new file mode 100644 index 0000000..024b7b4 --- /dev/null +++ b/examples/cms/dever/module/user/membership/app.dever @@ -0,0 +1,31 @@ +grant_owner( + account_id: user.account.model.principal.id, + selected_tenant_id: platform.tenant.model.id +) () { + stored = model.create( + { + user_id = account_id + tenant_id = selected_tenant_id + role = model.MembershipRole.Owner + } + ) + discarded = stored +} + +require_active( + account_id: user.account.model.principal.id, + selected_tenant_id: platform.tenant.model.id +) () { + stored = model.first( + { + where = user_id == account_id and tenant_id == selected_tenant_id and status == model.MembershipStatus.Active + } + ) + active(stored) +} + +active(stored: model.Membership) () {} + +active(stored: null) () { + fail(dever.api.Error.Unauthorized) +} diff --git a/examples/cms/dever/module/user/membership/model.dever b/examples/cms/dever/module/user/membership/model.dever new file mode 100644 index 0000000..76b9ef5 --- /dev/null +++ b/examples/cms/dever/module/user/membership/model.dever @@ -0,0 +1,19 @@ +type MembershipRole { + Owner = "所有者" + Editor = "编辑" + Reader = "读者" +} + +type MembershipStatus { + Active = "启用" + Disabled = "禁用" +} + +global type Membership { + user_id: user.account.model.principal.id + tenant_id: platform.tenant.model.id + role: MembershipRole index + status: MembershipStatus default MembershipStatus.Active index +} + +unique(user_id, tenant_id) diff --git a/examples/cms/dever/module/user/session/app.dever b/examples/cms/dever/module/user/session/app.dever new file mode 100644 index 0000000..439f5f8 --- /dev/null +++ b/examples/cms/dever/module/user/session/app.dever @@ -0,0 +1,94 @@ +type SessionView { + uuid: Uuid +} + +type AuthSession { + subject: Text + user_id: user.account.model.principal.id + tenant_id: platform.tenant.model.id + tenant_key: Text +} + +open( + subject: Text, + account_id: user.account.model.principal.id, + selected_tenant_id: platform.tenant.model.id, + selected_tenant_key: Text, + site_key: Text +) (session: SessionView) { + stored = model.create( + { + subject = subject + user_id = account_id + tenant_id = selected_tenant_id + tenant_key = selected_tenant_key + site = site_key + expires_at = dever.time.add(dever.time.now(), dever.time.duration(3600000)) + } + ) + session = SessionView { + uuid = stored.uuid + } +} + +authenticate(session_id: Uuid, site_key: Text) (result: AuthSession) { + stored = model.first( + { + where = uuid == session_id and site == site_key and revoked == false and expires_at > dever.time.now( + ) + } + ) + result = auth_session(stored) +} + +authenticate(session_id: null, site_key: Text) (result: AuthSession) { + fail(dever.api.Error.Unauthorized) +} + +revoke(session_id: Uuid) () { + stored = model.first( + { + where = uuid == session_id and revoked == false + } + ) + revoke_found(stored) +} + +revoke(session_id: null) () { + fail(dever.api.Error.Unauthorized) +} + +auth_session(stored: model.Session) (result: AuthSession) { + result = AuthSession { + subject = stored.subject + user_id = stored.user_id + tenant_id = stored.tenant_id + tenant_key = stored.tenant_key + } +} + +auth_session(stored: null) (result: AuthSession) { + fail(dever.api.Error.Unauthorized) +} + +revoke_found(stored: model.Session) () { + changed = model.update( + stored.id, + { + revoked = true + } + ) + discarded = changed +} + +revoke_found(stored: null) () { + fail(dever.api.Error.Unauthorized) +} + +cleanup() () { + removed = model.delete( + { + where = revoked == true or expires_at <= dever.time.now() + } + ) +} diff --git a/examples/cms/dever/module/user/session/job.dever b/examples/cms/dever/module/user/session/job.dever new file mode 100644 index 0000000..734c1b4 --- /dev/null +++ b/examples/cms/dever/module/user/session/job.dever @@ -0,0 +1,7 @@ +database default + +job cleanup() () retry(3) timeout(30000) { + app.cleanup() +} + +schedule cleanup = "0 * * * *" diff --git a/examples/cms/dever/module/user/session/model.dever b/examples/cms/dever/module/user/session/model.dever new file mode 100644 index 0000000..5cc7044 --- /dev/null +++ b/examples/cms/dever/module/user/session/model.dever @@ -0,0 +1,12 @@ +global type Session { + uuid: Uuid generated unique + subject: Text(1, 128) + user_id: user.account.model.principal.id + tenant_id: platform.tenant.model.id + tenant_key: Text(1, 64) + site: Text(1, 32) + expires_at: DateTime index + revoked: Bool default false index +} + +index(user_id, tenant_id, site) diff --git a/examples/cms/dever/test/content/category/lifecycle.dever b/examples/cms/dever/test/content/category/lifecycle.dever new file mode 100644 index 0000000..fe6f909 --- /dev/null +++ b/examples/cms/dever/test/content/category/lifecycle.dever @@ -0,0 +1,13 @@ +lifecycle() () { + category = app.create(" Dever News ") + assert_eq(category.name, "Dever News") + assert_eq(category.slug, "dever news") + assert_eq(category.status, "active") + app.require_active(category.id) + app.require_active(null) + assert(app.disable(category.id)) + independent = app.create("Announcements") + assert(independent.id != category.id) + assert_eq(independent.slug, "announcements") + app.require_active(independent.id) +} diff --git a/examples/cms/dever/test/news/article/publish.dever b/examples/cms/dever/test/news/article/publish.dever new file mode 100644 index 0000000..0fe2703 --- /dev/null +++ b/examples/cms/dever/test/news/article/publish.dever @@ -0,0 +1,11 @@ +publish() () { + assert_eq(domain.normalize_slug(" News-Article "), "news-article") + assert_eq(domain.normalize_title(" First article "), "First article") + assert_eq(domain.page_size(1), 1) + assert_eq(domain.page_size(100), 100) + domain.require_owner(true) + domain.require_draft(true) + domain.require_updated(1) + domain.require_content("news", "Title", "Body") + domain.require_published(true) +} diff --git a/examples/cms/dever/test/user/account/bootstrap.dever b/examples/cms/dever/test/user/account/bootstrap.dever new file mode 100644 index 0000000..fce9726 --- /dev/null +++ b/examples/cms/dever/test/user/account/bootstrap.dever @@ -0,0 +1,16 @@ +bootstrap() () { + created = user.account.bootstrap( + "tenant-one", + "Tenant One", + " owner@example.com ", + " Owner ", + secret("correct horse battery staple") + ) + user.membership.require_active(created.user_id, created.tenant_id) + user.account.active(created.user_id) + tenant = platform.tenant.active("tenant-one") + assert_eq(tenant.id, created.tenant_id) + assert_eq(tenant.key, "tenant-one") + unrelated = platform.tenant.provision("tenant-two", "Tenant Two") + assert(unrelated.id != created.tenant_id) +} diff --git a/examples/cms/dever/test/user/account/validation.dever b/examples/cms/dever/test/user/account/validation.dever new file mode 100644 index 0000000..d3be11d --- /dev/null +++ b/examples/cms/dever/test/user/account/validation.dever @@ -0,0 +1,8 @@ +validation() () { + prepared = domain.prepare(" OWNER@EXAMPLE.COM ", " Owner ") + assert_eq(prepared.address, "owner@example.com") + assert_eq(prepared.name, "Owner") + unicode_name = domain.prepare("editor@example.com", " 编辑员 ") + assert_eq(unicode_name.address, "editor@example.com") + assert_eq(unicode_name.name, "编辑员") +} diff --git a/examples/cms/dever/test/user/session/lifecycle.dever b/examples/cms/dever/test/user/session/lifecycle.dever new file mode 100644 index 0000000..74b8f35 --- /dev/null +++ b/examples/cms/dever/test/user/session/lifecycle.dever @@ -0,0 +1,37 @@ +lifecycle() () { + account = user.account.bootstrap( + "sessions", + "Sessions", + "session@example.com", + "Session user", + secret("correct horse battery staple") + ) + session = app.open( + "session-subject", + account.user_id, + account.tenant_id, + "sessions", + "admin" + ) + authenticated = app.authenticate(session.uuid, "admin") + assert_eq(authenticated.subject, "session-subject") + assert_eq(authenticated.user_id, account.user_id) + assert_eq(authenticated.tenant_id, account.tenant_id) + assert_eq(authenticated.tenant_key, "sessions") + app.revoke(session.uuid) + dever.test.advance_clock(3600000) + app.cleanup() + replacement = app.open( + "new-subject", + account.user_id, + account.tenant_id, + "sessions", + "front" + ) + assert(replacement.uuid != session.uuid) + current = app.authenticate(replacement.uuid, "front") + assert_eq(current.subject, "new-subject") + assert_eq(current.user_id, account.user_id) + assert_eq(current.tenant_id, account.tenant_id) + assert_eq(current.tenant_key, "sessions") +} diff --git a/examples/cms/md/config/setting.json b/examples/cms/md/config/setting.json new file mode 100644 index 0000000..c3cc00a --- /dev/null +++ b/examples/cms/md/config/setting.json @@ -0,0 +1,56 @@ +{ + "runtime": { + "mode": "all", + "shutdown_ms": 30000 + }, + "job": { + "workers": 1, + "poll_ms": 1000, + "lease_ms": 60000, + "retry_base_ms": 1000, + "retry_max_ms": 300000 + }, + "http": { + "host": "127.0.0.1", + "port": 8080 + }, + "database": { + "default": { + "type": "sqlite", + "path": "data/db/platform.db", + "tenant_directory": "data/tenants", + "max_connections": 4, + "max_page_size": 100 + } + }, + "tenant": { + "database": "default", + "max_pools": 32, + "idle_timeout_ms": 300000 + }, + "auth": { + "providers": { + "session": { + "verify": "user.account.verify", + "jwtSecret": "replace-this-example-secret-before-deployment-2026", + "ttlSeconds": 3600, + "cookie": "cms_session" + } + } + }, + "sites": { + "admin": { + "path": "admin", + "auth": "session", + "origin": "https://cms.example.com" + }, + "front": { + "path": "front", + "auth": "session", + "origin": "https://cms.example.com" + } + }, + "log": { + "level": "info" + } +} diff --git a/examples/cms/md/module/audit/operation/app.dever.md b/examples/cms/md/module/audit/operation/app.dever.md new file mode 100644 index 0000000..ee4ce48 --- /dev/null +++ b/examples/cms/md/module/audit/operation/app.dever.md @@ -0,0 +1,34 @@ +# 操作审计应用 + +审计应用写入最小化操作摘要。 + +- 包：`audit.operation.app` +- 公开类型：无 +- 公开方法： + - `record` +- 使用： + - `audit.operation.record(kind, resource, resource_id)` + +## 记录操作 + +保存操作类型、资源和资源标识。 + +- 函数：`record` +- 输入： + - `kind: audit.operation.model.OperationKind`：操作类型 + - `resource: Text`：资源类型 + - `resource_id: Text`：资源 ID +- 输出：无 + +```dever +record(kind: model.OperationKind, resource: Text, resource_id: Text) () { + stored = model.create( + { + kind = kind + resource = resource + resource_id = resource_id + } + ) + discarded = stored +} +``` diff --git a/examples/cms/md/module/audit/operation/model.dever.md b/examples/cms/md/module/audit/operation/model.dever.md new file mode 100644 index 0000000..bd8d7ba --- /dev/null +++ b/examples/cms/md/module/audit/operation/model.dever.md @@ -0,0 +1,52 @@ +# 操作审计模型 + +审计记录保存敏感操作摘要，不保存密码、Token 或请求全文。 + +- 包：`audit.operation.model` +- 公开类型： + - `OperationKind` + - `Operation` +- 公开方法：无 +- 使用：无 + +## 操作类型 + +审计操作分类。 + +- 类型：`OperationKind` +- 分支： + - `Login`：登录 + - `Logout`：登出 + - `Publish`：发布 + - `Upload`：上传 + - `Access`：授权 + +```dever +type OperationKind { + Login = "登录" + Logout = "登出" + Publish = "发布" + Upload = "上传" + Access = "授权" +} +``` + +## 操作记录 + +审计记录只保存资源摘要。 + +- 类型：`Operation` +- 字段： + - `actor_id: user.account.model.principal.id?`：操作者 + - `kind: OperationKind`：操作类型 + - `resource: Text(1, 120)`：资源类型 + - `resource_id: Text(1, 120)`：资源 ID + +```dever +type Operation { + actor_id: user.account.model.principal.id? + kind: OperationKind index + resource: Text(1, 120) + resource_id: Text(1, 120) +} +``` diff --git a/examples/cms/md/module/content/category/api/admin/manage.dever.md b/examples/cms/md/module/content/category/api/admin/manage.dever.md new file mode 100644 index 0000000..f9e1fe5 --- /dev/null +++ b/examples/cms/md/module/content/category/api/admin/manage.dever.md @@ -0,0 +1,28 @@ +# 后台分类接口 + +后台分类接口创建和停用分类。 + +- 包：`content.category.api.admin.manage` +- 公开类型：无 +- 公开方法：无 +- 使用：无 + +## 创建分类 + +创建分类委托给应用。 + +- 声明：`post create` + +```dever +post create = app.create +``` + +## 停用分类 + +停用分类委托给应用。 + +- 声明：`delete disable` + +```dever +delete disable = app.disable +``` diff --git a/examples/cms/md/module/content/category/app.dever.md b/examples/cms/md/module/content/category/app.dever.md new file mode 100644 index 0000000..1118f9f --- /dev/null +++ b/examples/cms/md/module/content/category/app.dever.md @@ -0,0 +1,105 @@ +# 内容分类应用 + +分类应用负责创建和停用分类。 + +- 包：`content.category.app` +- 公开类型： + - `CategoryView` +- 公开方法： + - `create` + - `disable` + - `require_active` +- 使用： + - `content.category.create(name)` + - `content.category.disable(id)` + - `content.category.require_active(id)` + +## 分类视图 + +返回分类公开字段。 + +- 类型：`CategoryView` +- 字段： + - `id: model.id`：记录标识 + - `name: Text`：该声明的业务值 + - `slug: Text`：规范化标识 + - `status: Text`：业务状态 + +```dever +type CategoryView { + id: model.id + name: Text + slug: Text + status: Text +} +``` + +## 创建分类 + +创建规范化分类。 + +- 函数：`create` +- 输入： + - `name: Text`：该声明的业务值 +- 输出： + - `category: CategoryView`：该声明的业务值 + +```dever +create(name: Text) (category: CategoryView) { + normalized = domain.normalize_name(name) + domain.require_name(normalized) + stored = model.create( + { + name = normalized + slug = domain.normalize_slug(normalized) + } + ) + category = CategoryView { + id = stored.id + name = stored.name + slug = stored.slug + status = "active" + } +} +``` + +## 停用分类 + +停用已有分类。 + +- 函数：`disable` +- 输入： + - `id: content.category.model.id`：记录标识 +- 输出： + - `ok: Bool`：该声明的业务值 + +```dever +disable(id: model.id) (ok: Bool) { + changed = model.update( + id, + { + status = model.CategoryStatus.Disabled + } + ) + discarded = changed + ok = true +} +``` + +## require_active + +分类应用负责创建和停用分类。 + +- 函数：`require_active` +- 输入： + - `id: content.category.model.id?`：记录标识 +- 输出：无 + +```dever +require_active(id: model.id) () { + stored = model.get(id) + domain.require_active(stored.status == model.CategoryStatus.Active) +} + +require_active(id: null) () {} +``` diff --git a/examples/cms/md/module/content/category/domain.dever.md b/examples/cms/md/module/content/category/domain.dever.md new file mode 100644 index 0000000..c8ed2b0 --- /dev/null +++ b/examples/cms/md/module/content/category/domain.dever.md @@ -0,0 +1,89 @@ +# 内容分类规则 + +分类名称和标识在领域规则中统一规范化。 + +- 包：`content.category.domain` +- 公开类型：无 +- 公开方法：无 +- 使用：无 + +## 规范名称 + +去除分类名称两端空白。 + +- 函数：`normalize_name` +- 输入： + - `value: Text`：该声明的业务值 +- 输出： + - `name: Text`：该声明的业务值 + +```dever +normalize_name(value: Text) (name: Text) pure { + name = text.trim(value) +} +``` + +## 规范标识 + +统一分类标识大小写。 + +- 函数：`normalize_slug` +- 输入： + - `value: Text`：该声明的业务值 +- 输出： + - `slug: Text`：规范化标识 + +```dever +normalize_slug(value: Text) (slug: Text) pure { + slug = text.lower(text.trim(value)) +} +``` + +## 检查名称 + +拒绝空分类名称。 + +- 函数：`require_name` +- 输入： + - `value: Text`：该声明的业务值 +- 输出：无 + +```dever +require_name(value: Text) () pure { + check_name(value != "") +} +``` + +## 检查名称结果 + +有效名称继续执行，无效名称返回标准错误。 + +- 函数：`check_name` +- 输入： + - `value: Bool`：该声明的业务值 +- 输出：无 + +```dever +check_name(value: true) () pure {} + +check_name(value: false) () pure { + fail(dever.api.Error.Invalid) +} +``` + +## require_active + +分类名称和标识在领域规则中统一规范化。 + +- 函数：`require_active` +- 输入： + - `active: Bool`：该声明的业务值 +- 输出：无 + +```dever +require_active(active: true) () pure {} + +require_active(active: false) () pure { + fail(dever.api.Error.Conflict) +} +``` diff --git a/examples/cms/md/module/content/category/model.dever.md b/examples/cms/md/module/content/category/model.dever.md new file mode 100644 index 0000000..a71ba53 --- /dev/null +++ b/examples/cms/md/module/content/category/model.dever.md @@ -0,0 +1,44 @@ +# 内容分类模型 + +内容分类保存标题、标识和状态。 + +- 包：`content.category.model` +- 公开类型： + - `CategoryStatus` + - `Category` +- 公开方法：无 +- 使用：无 + +## 分类状态 + +状态枚举。 + +- 类型：`CategoryStatus` +- 分支： + - `Active`：启用 + - `Disabled`：禁用 + +```dever +type CategoryStatus { + Active = "启用" + Disabled = "禁用" +} +``` + +## 分类 + +分类记录。 + +- 类型：`Category` +- 字段： + - `name: Text(1, 80)`：名称 + - `slug: Text(1, 100)`：标识 + - `status: CategoryStatus`：状态 + +```dever +type Category { + name: Text(1, 80) + slug: Text(1, 100) unique + status: CategoryStatus default CategoryStatus.Active index +} +``` diff --git a/examples/cms/md/module/content/revision/api/admin/manage.dever.md b/examples/cms/md/module/content/revision/api/admin/manage.dever.md new file mode 100644 index 0000000..cdac8c8 --- /dev/null +++ b/examples/cms/md/module/content/revision/api/admin/manage.dever.md @@ -0,0 +1,18 @@ +# 后台修订接口 + +后台修订接口创建文章修订记录。 + +- 包：`content.revision.api.admin.manage` +- 公开类型：无 +- 公开方法：无 +- 使用：无 + +## 创建修订 + +创建修订委托给应用。 + +- 声明：`post create` + +```dever +post create = app.create +``` diff --git a/examples/cms/md/module/content/revision/app.dever.md b/examples/cms/md/module/content/revision/app.dever.md new file mode 100644 index 0000000..62d481d --- /dev/null +++ b/examples/cms/md/module/content/revision/app.dever.md @@ -0,0 +1,89 @@ +# 内容修订应用 + +修订应用创建文章历史版本并返回公开视图。 + +- 包：`content.revision.app` +- 公开类型： + - `RevisionView` +- 公开方法： + - `create` +- 使用： + - `content.revision.create(selected_article_id, expected_version, title, body, category_id, asset_id)` + +## 修订视图 + +不暴露存储实现细节的修订结果。 + +- 类型：`RevisionView` +- 字段： + - `id: model.id`：记录标识 + - `article_id: news.article.model.id`：文章标识 + - `version: Int`：当前编辑版本 + - `title: Text`：文章标题 + - `body: Text`：正文 + - `status: Text`：业务状态 + +```dever +type RevisionView { + id: model.id + article_id: news.article.model.id + version: Int + title: Text + body: Text + status: Text +} +``` + +## 创建修订 + +按文章已有版本数生成下一个版本。 + +- 函数：`create` +- 输入： + - `selected_article_id: news.article.model.id`：待编辑文章标识 + - `expected_version: Int`：调用者读取的编辑版本 + - `title: Text`：文章标题 + - `body: Text`：正文 + - `category_id: content.category.model.id?`：可选分类引用 + - `asset_id: media.asset.model.id?`：可选媒体引用 +- 输出： + - `revision: RevisionView`：该声明的业务值 + +```dever +transaction create( + selected_article_id: news.article.model.id, + expected_version: Int, + title: Text, + body: Text, + category_id: content.category.model.id?, + asset_id: media.asset.model.id? +) (revision: RevisionView) { + edited = news.article.edit( + selected_article_id, + expected_version, + title, + body, + category_id, + asset_id + ) + stored = model.create( + { + article_id = selected_article_id + version = edited.version + title = edited.title + body = edited.body + category_id = edited.category_id + asset_id = edited.asset_id + status = model.RevisionStatus.Draft + } + ) + revision = RevisionView { + id = stored.id + article_id = stored.article_id + version = stored.version + title = stored.title + body = stored.body + status = "draft" + } +} +``` diff --git a/examples/cms/md/module/content/revision/model.dever.md b/examples/cms/md/module/content/revision/model.dever.md new file mode 100644 index 0000000..0997405 --- /dev/null +++ b/examples/cms/md/module/content/revision/model.dever.md @@ -0,0 +1,62 @@ +# 内容修订模型 + +修订记录保存文章历史内容和版本。 + +- 包：`content.revision.model` +- 公开类型： + - `RevisionStatus` + - `Revision` +- 公开方法：无 +- 使用：无 + +## 修订状态 + +修订状态枚举。 + +- 类型：`RevisionStatus` +- 分支： + - `Draft`：该声明的业务值 + - `Published`：该声明的业务值 + +```dever +type RevisionStatus { + Draft = "草稿" + Published = "已发布" +} +``` + +## 修订记录 + +文章的一个历史版本。 + +- 类型：`Revision` +- 字段： + - `article_id: news.article.model.id`：文章标识 + - `version: Int`：当前编辑版本 + - `title: Text(1, 160)`：文章标题 + - `body: Text`：正文 + - `category_id: content.category.model.id?`：可选分类引用 + - `asset_id: media.asset.model.id?`：可选媒体引用 + - `status: RevisionStatus`：业务状态 + +```dever +type Revision { + article_id: news.article.model.id index + version: Int + title: Text(1, 160) + body: Text + category_id: content.category.model.id? + asset_id: media.asset.model.id? + status: RevisionStatus default RevisionStatus.Draft index +} +``` + +## 版本唯一约束 + +同一文章的版本号唯一。 + +- 声明：`unique(article_id, version)` + +```dever +unique(article_id, version) +``` diff --git a/examples/cms/md/module/media/asset/adapter.dever.md b/examples/cms/md/module/media/asset/adapter.dever.md new file mode 100644 index 0000000..34cd937 --- /dev/null +++ b/examples/cms/md/module/media/asset/adapter.dever.md @@ -0,0 +1,24 @@ +# 媒体本地适配器 + +本地适配器使用受控 data/upload 路径保存上传内容。 + +- 包：`media.asset.adapter` +- 公开类型：无 +- 公开方法：无 +- 使用：无 + +## 本地存储 + +适配器实现媒体存储端口。 + +- 函数：`port.store` +- 输入： + - `file: Upload`：上传文件 +- 输出： + - `key: Uuid`：存储键 + +```dever +port.store(file: Upload) (key: Uuid) { + key = dever.storage.put(file) +} +``` diff --git a/examples/cms/md/module/media/asset/api/admin/manage.dever.md b/examples/cms/md/module/media/asset/api/admin/manage.dever.md new file mode 100644 index 0000000..57b8671 --- /dev/null +++ b/examples/cms/md/module/media/asset/api/admin/manage.dever.md @@ -0,0 +1,18 @@ +# 后台媒体接口 + +后台媒体接口只开放上传动作。 + +- 包：`media.asset.api.admin.manage` +- 公开类型：无 +- 公开方法：无 +- 使用：无 + +## 上传媒体 + +上传请求委托给媒体应用。 + +- 声明：`post upload` + +```dever +post upload = app.store +``` diff --git a/examples/cms/md/module/media/asset/app.dever.md b/examples/cms/md/module/media/asset/app.dever.md new file mode 100644 index 0000000..f966c24 --- /dev/null +++ b/examples/cms/md/module/media/asset/app.dever.md @@ -0,0 +1,107 @@ +# 媒体资源应用 + +媒体应用通过 Storage Port 接收上传并保存元数据。 + +- 包：`media.asset.app` +- 公开类型： + - `StoreResult` +- 公开方法： + - `store` + - `require_ready` + - `ready` +- 使用： + - `media.asset.store(file)` + - `media.asset.require_ready(id)` + - `media.asset.ready(available)` + +## 保存结果 + +返回资源公开元数据。 + +- 类型：`StoreResult` +- 字段： + - `id: model.id`：记录标识 + - `storage_key: Uuid`：该声明的业务值 + - `filename: Text`：该声明的业务值 + - `content_type: Text`：该声明的业务值 + - `size: Int`：受限页大小 + +```dever +type StoreResult { + id: model.id + storage_key: Uuid + filename: Text + content_type: Text + size: Int +} +``` + +## 保存上传 + +文件内容只能通过 Port 交给 Adapter。 + +- 函数：`store` +- 输入： + - `file: Upload`：该声明的业务值 +- 输出： + - `result: StoreResult`：该声明的业务值 + +```dever +store(file: Upload) (result: StoreResult) { + filename = dever.api.upload_filename(file) + content_type = dever.api.upload_content_type(file) + size = dever.api.upload_size(file) + key = port.store(file) + asset = model.create( + { + storage_key = key + filename = filename + content_type = content_type + size = size + } + ) + audit.operation.record(audit.operation.model.OperationKind.Upload, "asset", filename) + result = StoreResult { + id = asset.id + storage_key = asset.storage_key + filename = asset.filename + content_type = asset.content_type + size = asset.size + } +} +``` + +## require_ready + +媒体应用通过 Storage Port 接收上传并保存元数据。 + +- 函数：`require_ready` +- 输入： + - `id: media.asset.model.id?`：记录标识 +- 输出：无 + +```dever +require_ready(id: model.id) () { + stored = model.get(id) + ready(stored.status == model.AssetStatus.Ready) +} + +require_ready(id: null) () {} +``` + +## ready + +媒体应用通过 Storage Port 接收上传并保存元数据。 + +- 函数：`ready` +- 输入： + - `available: Bool`：该声明的业务值 +- 输出：无 + +```dever +ready(available: true) () {} + +ready(available: false) () { + fail(dever.api.Error.Conflict) +} +``` diff --git a/examples/cms/md/module/media/asset/model.dever.md b/examples/cms/md/module/media/asset/model.dever.md new file mode 100644 index 0000000..fc00704 --- /dev/null +++ b/examples/cms/md/module/media/asset/model.dever.md @@ -0,0 +1,48 @@ +# 媒体资源模型 + +媒体模型只保存受控存储键和文件元数据。 + +- 包：`media.asset.model` +- 公开类型： + - `AssetStatus` + - `Asset` +- 公开方法：无 +- 使用：无 + +## 资源状态 + +资源生命周期状态。 + +- 类型：`AssetStatus` +- 分支： + - `Ready`：就绪 + - `Removed`：已删除 + +```dever +type AssetStatus { + Ready = "就绪" + Removed = "已删除" +} +``` + +## 资源记录 + +资源元数据不包含上传内容本身。 + +- 类型：`Asset` +- 字段： + - `storage_key: Uuid`：存储键 + - `filename: Text(1, 255)`：文件名 + - `content_type: Text(1, 160)`：内容类型 + - `size: Int`：大小 + - `status: AssetStatus`：状态 + +```dever +type Asset { + storage_key: Uuid unique + filename: Text(1, 255) + content_type: Text(1, 160) + size: Int + status: AssetStatus default AssetStatus.Ready index +} +``` diff --git a/examples/cms/md/module/media/asset/port.dever.md b/examples/cms/md/module/media/asset/port.dever.md new file mode 100644 index 0000000..b0bcf59 --- /dev/null +++ b/examples/cms/md/module/media/asset/port.dever.md @@ -0,0 +1,23 @@ +# 媒体存储端口 + +媒体应用通过端口隔离存储实现。 + +- 包：`media.asset.port` +- 公开类型：无 +- 公开方法：无 +- 使用：无 + +## 保存上传 + +端口只声明上传内容到存储的契约。 + +- 函数：`store` +- 输入： + - `file: Upload`：上传文件 +- 输出： + - `key: Uuid`：存储键 +- 允许失败：`dever.storage.PutResult` + +```dever +store(file: Upload) (key: Uuid) fails dever.storage.PutResult +``` diff --git a/examples/cms/md/module/news/article/api/admin/manage.dever.md b/examples/cms/md/module/news/article/api/admin/manage.dever.md new file mode 100644 index 0000000..0c09baf --- /dev/null +++ b/examples/cms/md/module/news/article/api/admin/manage.dever.md @@ -0,0 +1,58 @@ +# 后台文章接口 + +普通管理使用 Model REST，发布保留为业务动作。 + +- 包：`news.article.api.admin.manage` +- 公开类型：无 +- 公开方法：无 +- 使用：无 + +## get list + +普通管理使用 Model REST，发布保留为业务动作。 + +- 声明：`get list` + +```dever +get list = app.list +``` + +## get detail + +普通管理使用 Model REST，发布保留为业务动作。 + +- 声明：`get detail` + +```dever +get detail = app.detail +``` + +## post create + +普通管理使用 Model REST，发布保留为业务动作。 + +- 声明：`post create` + +```dever +post create = app.create +``` + +## delete remove + +普通管理使用 Model REST，发布保留为业务动作。 + +- 声明：`delete remove` + +```dever +delete remove = app.remove +``` + +## 发布文章 + +本节定义对应的源码合同。 + +- 声明：`post publish` + +```dever +post publish = app.publish +``` diff --git a/examples/cms/md/module/news/article/api/front/browse.dever.md b/examples/cms/md/module/news/article/api/front/browse.dever.md new file mode 100644 index 0000000..0e561d1 --- /dev/null +++ b/examples/cms/md/module/news/article/api/front/browse.dever.md @@ -0,0 +1,28 @@ +# 前台文章接口 + +前台只查询已发布内容，不开放自动 REST 写路由。 + +- 包：`news.article.api.front.browse` +- 公开类型：无 +- 公开方法：无 +- 使用：无 + +## 文章列表 + +本节定义对应的源码合同。 + +- 声明：`get list` + +```dever +get list = app.published +``` + +## 文章详情 + +本节定义对应的源码合同。 + +- 声明：`get detail` + +```dever +get detail = app.published_detail +``` diff --git a/examples/cms/md/module/news/article/app/editing.dever.md b/examples/cms/md/module/news/article/app/editing.dever.md new file mode 100644 index 0000000..4974d7c --- /dev/null +++ b/examples/cms/md/module/news/article/app/editing.dever.md @@ -0,0 +1,244 @@ +# news.article.app.editing + +文章创建、所有权查询与带版本的编辑；所有写入校验分类和媒体引用。 + +- 包：`news.article.app.editing` +- 公开类型： + - `ArticleView` + - `ArticlePage` +- 公开方法： + - `create` + - `list` + - `detail` + - `remove` + - `edit` + - `require_schedulable` +- 使用： + - `news.article.create(slug, title, body, category_id, asset_id)` + - `news.article.list(size)` + - `news.article.detail(id)` + - `news.article.remove(id)` + - `news.article.edit(id, expected_version, title, body, category_id, asset_id)` + - `news.article.require_schedulable(article_id)` + +## ArticleView + +文章创建、所有权查询与带版本的编辑；所有写入校验分类和媒体引用。 + +- 类型：`ArticleView` +- 字段： + - `id: model.id`：记录标识 + - `slug: Text`：规范化标识 + - `title: Text`：文章标题 + - `body: Text`：正文 + - `version: Int`：当前编辑版本 + - `category_id: content.category.model.id?`：可选分类引用 + - `asset_id: media.asset.model.id?`：可选媒体引用 + +```dever +type ArticleView { + id: model.id + slug: Text + title: Text + body: Text + version: Int + category_id: content.category.model.id? + asset_id: media.asset.model.id? +} +``` + +## ArticlePage + +文章创建、所有权查询与带版本的编辑；所有写入校验分类和媒体引用。 + +- 类型：`ArticlePage` +- 字段： + - `rows: List<ArticleView>`：页面记录 + - `total: Int`：所有权范围内总数 + +```dever +type ArticlePage { + rows: List<ArticleView> + total: Int +} +``` + +## create + +文章创建、所有权查询与带版本的编辑；所有写入校验分类和媒体引用。 + +- 函数：`create` +- 输入： + - `slug: Text`：规范化标识 + - `title: Text`：文章标题 + - `body: Text`：正文 + - `category_id: content.category.model.id?`：可选分类引用 + - `asset_id: media.asset.model.id?`：可选媒体引用 +- 输出： + - `article: ArticleView`：该声明的业务值 + +```dever +create( + slug: Text, + title: Text, + body: Text, + category_id: content.category.model.id?, + asset_id: media.asset.model.id? +) (article: ArticleView) { + normalized_slug = domain.normalize_slug(slug) + normalized_title = domain.normalize_title(title) + domain.require_content(normalized_slug, normalized_title, body) + content.category.require_active(category_id) + media.asset.require_ready(asset_id) + stored = model.create( + { + author_id = user.account.current_user() + slug = normalized_slug + title = normalized_title + body = body + category_id = category_id + asset_id = asset_id + } + ) + article = detail(stored.id) +} +``` + +## list + +文章创建、所有权查询与带版本的编辑；所有写入校验分类和媒体引用。 + +- 函数：`list` +- 输入： + - `size: Int`：受限页大小 +- 输出： + - `articles: ArticlePage`：该声明的业务值 + +```dever +list(size: Int) (articles: ArticlePage) { + author = user.account.current_user() + articles = ArticlePage { + rows = model.owned_articles(author, domain.page_size(size)) + total = model.count( + { + where = author_id == author + } + ) + } +} +``` + +## detail + +文章创建、所有权查询与带版本的编辑；所有写入校验分类和媒体引用。 + +- 函数：`detail` +- 输入： + - `id: news.article.model.id`：记录标识 +- 输出： + - `article: ArticleView`：该声明的业务值 + +```dever +detail(id: model.id) (article: ArticleView) { + stored = model.get(id) + domain.require_owner(dever.auth.owns_user(stored.author_id)) + article = ArticleView { + id = stored.id + slug = stored.slug + title = stored.title + body = stored.body + version = stored.version + category_id = stored.category_id + asset_id = stored.asset_id + } +} +``` + +## remove + +文章创建、所有权查询与带版本的编辑；所有写入校验分类和媒体引用。 + +- 函数：`remove` +- 输入： + - `id: news.article.model.id`：记录标识 +- 输出： + - `removed: Int`：该声明的业务值 + +```dever +remove(id: model.id) (removed: Int) { + stored = model.get(id) + domain.require_owner(dever.auth.owns_user(stored.author_id)) + domain.require_draft(stored.status == model.ArticleStatus.Draft) + removed = model.delete( + { + where = id == stored.id and version == stored.version + } + ) + domain.require_updated(removed) +} +``` + +## edit + +文章创建、所有权查询与带版本的编辑；所有写入校验分类和媒体引用。 + +- 函数：`edit` +- 输入： + - `id: news.article.model.id`：记录标识 + - `expected_version: Int`：调用者读取的编辑版本 + - `title: Text`：文章标题 + - `body: Text`：正文 + - `category_id: content.category.model.id?`：可选分类引用 + - `asset_id: media.asset.model.id?`：可选媒体引用 +- 输出： + - `article: ArticleView`：该声明的业务值 + +```dever +edit( + id: model.id, + expected_version: Int, + title: Text, + body: Text, + category_id: content.category.model.id?, + asset_id: media.asset.model.id? +) (article: ArticleView) { + stored = model.get(id) + domain.require_owner(dever.auth.owns_user(stored.author_id)) + domain.require_draft(stored.status == model.ArticleStatus.Draft) + normalized_title = domain.normalize_title(title) + domain.require_content(stored.slug, normalized_title, body) + content.category.require_active(category_id) + media.asset.require_ready(asset_id) + changed = model.update( + { + where = id == stored.id and version == expected_version and status == model.ArticleStatus.Draft + }, + { + title = normalized_title + body = body + category_id = category_id + asset_id = asset_id + version = expected_version + 1 + } + ) + domain.require_updated(changed) + article = detail(stored.id) +} +``` + +## require_schedulable + +文章创建、所有权查询与带版本的编辑；所有写入校验分类和媒体引用。 + +- 函数：`require_schedulable` +- 输入： + - `article_id: news.article.model.id`：文章标识 +- 输出：无 + +```dever +require_schedulable(article_id: model.id) () { + stored = model.get(article_id) + domain.require_owner(dever.auth.owns_user(stored.author_id)) + domain.require_draft(stored.status == model.ArticleStatus.Draft) +} +``` diff --git a/examples/cms/md/module/news/article/app/publication.dever.md b/examples/cms/md/module/news/article/app/publication.dever.md new file mode 100644 index 0000000..0c0a264 --- /dev/null +++ b/examples/cms/md/module/news/article/app/publication.dever.md @@ -0,0 +1,203 @@ +# news.article.app.publication + +手动发布和定时任务共用原子发布流程，发布快照与审计同步提交。 + +- 包：`news.article.app.publication` +- 公开类型：无 +- 公开方法： + - `publish` + - `published` + - `published_detail` + - `publish_scheduled` + - `scheduled_publication` + - `publish_draft` + - `existing_publication` +- 使用： + - `news.article.publish(article_id)` + - `news.article.published(size)` + - `news.article.published_detail(id)` + - `news.article.publish_scheduled(article_id)` + - `news.article.scheduled_publication(status, article_id)` + - `news.article.publish_draft(article_id)` + - `news.article.existing_publication(stored)` + +## publish + +手动发布和定时任务共用原子发布流程，发布快照与审计同步提交。 + +- 函数：`publish` +- 输入： + - `article_id: news.article.model.id`：文章标识 +- 输出： + - `publication: news.article.model.publication.Publication`：该声明的业务值 + +```dever +transaction publish(article_id: model.id) (publication: model.publication.Publication) { + stored = model.get(article_id) + domain.require_owner(dever.auth.owns_user(stored.author_id)) + domain.require_draft(stored.status == model.ArticleStatus.Draft) + publication = publish_draft(article_id) +} +``` + +## published + +手动发布和定时任务共用原子发布流程，发布快照与审计同步提交。 + +- 函数：`published` +- 输入： + - `size: Int`：受限页大小 +- 输出： + - `articles: news.article.model.publication.Page`：该声明的业务值 + +```dever +published(size: Int) (articles: model.publication.Page) { + articles = model.publication.list( + { + order = [published_at.desc, id.desc] + size = domain.page_size(size) + } + ) +} +``` + +## published_detail + +手动发布和定时任务共用原子发布流程，发布快照与审计同步提交。 + +- 函数：`published_detail` +- 输入： + - `id: news.article.model.publication.id`：记录标识 +- 输出： + - `article: news.article.model.publication.Publication`：该声明的业务值 + +```dever +published_detail(id: model.publication.id) (article: model.publication.Publication) { + article = model.publication.get(id) + domain.require_published(text.trim(article.body) != "") +} +``` + +## publish_scheduled + +手动发布和定时任务共用原子发布流程，发布快照与审计同步提交。 + +- 函数：`publish_scheduled` +- 输入： + - `article_id: news.article.model.id`：文章标识 +- 输出： + - `publication: news.article.model.publication.Publication`：该声明的业务值 + +```dever +transaction publish_scheduled(article_id: model.id) ( + publication: model.publication.Publication +) { + stored = model.get(article_id) + publication = scheduled_publication(stored.status, article_id) +} +``` + +## scheduled_publication + +手动发布和定时任务共用原子发布流程，发布快照与审计同步提交。 + +- 函数：`scheduled_publication` +- 输入： + - `status: news.article.model.ArticleStatus`：业务状态 + - `article_id: news.article.model.id`：文章标识 +- 输出： + - `publication: news.article.model.publication.Publication`：该声明的业务值 + +```dever +scheduled_publication(status: model.ArticleStatus.Published, article_id: model.id) ( + publication: model.publication.Publication +) { + stored = model.get(article_id) + publication = existing_publication( + model.publication.first( + { + where = slug == stored.slug + } + ) + ) +} + +scheduled_publication(status: model.ArticleStatus.Draft, article_id: model.id) ( + publication: model.publication.Publication +) { + publication = publish_draft(article_id) +} +``` + +## publish_draft + +手动发布和定时任务共用原子发布流程，发布快照与审计同步提交。 + +- 函数：`publish_draft` +- 输入： + - `article_id: news.article.model.id`：文章标识 +- 输出： + - `publication: news.article.model.publication.Publication`：该声明的业务值 + +```dever +publish_draft(article_id: model.id) (publication: model.publication.Publication) { + stored = model.get(article_id) + domain.require_draft(stored.status == model.ArticleStatus.Draft) + content.category.require_active(stored.category_id) + media.asset.require_ready(stored.asset_id) + published_at = dever.time.now() + changed = model.update( + { + where = id == stored.id and version == stored.version and status == model.ArticleStatus.Draft + }, + { + status = model.ArticleStatus.Published + published_at = published_at + version = stored.version + 1 + } + ) + domain.require_updated(changed) + audit.operation.record( + audit.operation.model.OperationKind.Publish, + "article", + stored.slug + ) + publication = model.publication.upsert( + { + slug = stored.slug + }, + { + title = stored.title + body = stored.body + published_at = published_at + }, + { + title = stored.title + body = stored.body + published_at = published_at + } + ) +} +``` + +## existing_publication + +手动发布和定时任务共用原子发布流程，发布快照与审计同步提交。 + +- 函数：`existing_publication` +- 输入： + - `stored: news.article.model.publication.Publication?`：该声明的业务值 +- 输出： + - `publication: news.article.model.publication.Publication`：该声明的业务值 + +```dever +existing_publication(stored: model.publication.Publication) ( + publication: model.publication.Publication +) { + publication = stored +} + +existing_publication(stored: null) (publication: model.publication.Publication) { + fail(dever.api.Error.NotFound) +} +``` diff --git a/examples/cms/md/module/news/article/domain.dever.md b/examples/cms/md/module/news/article/domain.dever.md new file mode 100644 index 0000000..f5aa854 --- /dev/null +++ b/examples/cms/md/module/news/article/domain.dever.md @@ -0,0 +1,162 @@ +# 文章规则 + +文章字段规范化、分页边界和发布状态规则集中在纯 Domain。 + +- 包：`news.article.domain` +- 公开类型：无 +- 公开方法：无 +- 使用：无 + +## 规范文章标识 + +本节定义对应的源码合同。 + +- 函数：`normalize_slug` +- 输入： + - `value: Text`：该声明的业务值 +- 输出： + - `slug: Text`：规范化标识 + +```dever +normalize_slug(value: Text) (slug: Text) pure { + slug = text.lower(text.trim(value)) +} +``` + +## 规范标题 + +本节定义对应的源码合同。 + +- 函数：`normalize_title` +- 输入： + - `value: Text`：该声明的业务值 +- 输出： + - `title: Text`：文章标题 + +```dever +normalize_title(value: Text) (title: Text) pure { + title = text.trim(value) +} +``` + +## 限制分页大小 + +本节定义对应的源码合同。 + +- 函数：`page_size` +- 输入： + - `value: Int`：该声明的业务值 +- 输出： + - `size: Int`：受限页大小 + +```dever +page_size(value: Int >= 1 and <= 100) (size: Int) pure { + size = value +} + +page_size(value: other) (size: Int) pure { + fail(dever.api.Error.Invalid) +} +``` + +## 检查草稿状态 + +本节定义对应的源码合同。 + +- 函数：`require_draft` +- 输入： + - `draft: Bool`：该声明的业务值 +- 输出：无 + +```dever +require_draft(draft: true) () pure {} + +require_draft(draft: false) () pure { + fail(dever.api.Error.Conflict) +} +``` + +## 检查文章所有权 + +发布操作只允许文章作者执行。 + +- 函数：`require_owner` +- 输入： + - `owned: Bool`：该声明的业务值 +- 输出：无 + +```dever +require_owner(owned: true) () pure {} + +require_owner(owned: false) () pure { + fail(dever.api.Error.Forbidden) +} +``` + +## require_published + +文章字段规范化、分页边界和发布状态规则集中在纯 Domain。 + +- 函数：`require_published` +- 输入： + - `published: Bool`：该声明的业务值 +- 输出：无 + +```dever +require_published(published: true) () pure {} + +require_published(published: false) () pure { + fail(dever.api.Error.NotFound) +} +``` + +## require_updated + +文章字段规范化、分页边界和发布状态规则集中在纯 Domain。 + +- 函数：`require_updated` +- 输入： + - `changed: Int`：条件更新影响行数 +- 输出：无 + +```dever +require_updated(changed: 1) () pure {} + +require_updated(changed: other) () pure { + fail(dever.api.Error.Conflict) +} +``` + +## require_content + +文章字段规范化、分页边界和发布状态规则集中在纯 Domain。 + +- 函数：`require_content` +- 输入： + - `slug: Text`：规范化标识 + - `title: Text`：文章标题 + - `body: Text`：正文 +- 输出：无 + +```dever +require_content(slug: Text, title: Text, body: Text) () pure { + require_valid(slug != "" and title != "" and text.trim(body) != "") +} +``` + +## require_valid + +文章字段规范化、分页边界和发布状态规则集中在纯 Domain。 + +- 函数：`require_valid` +- 输入： + - `valid: Bool`：该声明的业务值 +- 输出：无 + +```dever +require_valid(valid: true) () pure {} + +require_valid(valid: false) () pure { + fail(dever.api.Error.Invalid) +} +``` diff --git a/examples/cms/md/module/news/article/model.dever.md b/examples/cms/md/module/news/article/model.dever.md new file mode 100644 index 0000000..956994b --- /dev/null +++ b/examples/cms/md/module/news/article/model.dever.md @@ -0,0 +1,86 @@ +# 文章模型 + +草稿位于租户数据库，REST 由可信用户 owner 限定到作者范围。 + +- 包：`news.article.model` +- 公开类型： + - `ArticleStatus` + - `Article` +- 公开方法：无 +- 使用：无 + +## 文章状态 + +本节定义对应的源码合同。 + +- 类型：`ArticleStatus` +- 分支： + - `Draft`：该声明的业务值 + - `Published`：该声明的业务值 + +```dever +type ArticleStatus { + Draft = "草稿" + Published = "已发布" +} +``` + +## 文章记录 + +本节定义对应的源码合同。 + +- 类型：`Article` +- 字段： + - `private author_id: user.account.model.principal.id`：可信作者标识 + - `slug: Text(1, 180)`：规范化标识 + - `title: Text(1, 160)`：文章标题 + - `body: Text`：正文 + - `category_id: content.category.model.id?`：可选分类引用 + - `asset_id: media.asset.model.id?`：可选媒体引用 + - `private version: Int`：当前编辑版本 + - `private status: ArticleStatus`：业务状态 + - `private published_at: DateTime?`：实际发布时间 + +```dever +type Article { + private owner author_id: user.account.model.principal.id = dever.auth.user_id() + slug: Text(1, 180) unique + title: Text(1, 160) + body: Text + category_id: content.category.model.id? + asset_id: media.asset.model.id? + private version: Int default 1 + private status: ArticleStatus default ArticleStatus.Draft index + private published_at: DateTime? +} +``` + +## 作者索引 + +本节定义对应的源码合同。 + +- 声明：`index(author_id, created_at)` + +```dever +index(author_id, created_at) +``` + +## sql owned_articles + +草稿位于租户数据库，REST 由可信用户 owner 限定到作者范围。 + +- 声明：`sql owned_articles` +- 输入： + - `author: user.account.model.principal.id`：该声明的业务值 + - `size: Int`：受限页大小 +- 输出： + - `articles: List<app.ArticleView>`：该声明的业务值 + +```dever +sql owned_articles(author: user.account.model.principal.id, size: Int) ( + articles: List<app.ArticleView> +) { + sqlite = "SELECT id, slug, title, body, version, category_id, asset_id FROM article WHERE author_id = ?1 ORDER BY id DESC LIMIT ?2" + postgres = "SELECT id, slug, title, body, version, category_id, asset_id FROM article WHERE author_id = $1 ORDER BY id DESC LIMIT $2" +} +``` diff --git a/examples/cms/md/module/news/article/model/publication.dever.md b/examples/cms/md/module/news/article/model/publication.dever.md new file mode 100644 index 0000000..dcb3e78 --- /dev/null +++ b/examples/cms/md/module/news/article/model/publication.dever.md @@ -0,0 +1,29 @@ +# 文章公开读模型 + +发布动作把可公开内容写入无私密字段的租户内读模型。 + +- 包：`news.article.model.publication` +- 公开类型： + - `Publication` +- 公开方法：无 +- 使用：无 + +## 发布记录 + +本节定义对应的源码合同。 + +- 类型：`Publication` +- 字段： + - `slug: Text(1, 180)`：文章标识 + - `title: Text(1, 160)`：标题 + - `body: Text`：正文 + - `published_at: DateTime`：发布时间 + +```dever +type Publication { + slug: Text(1, 180) unique + title: Text(1, 160) + body: Text + published_at: DateTime index +} +``` diff --git a/examples/cms/md/module/platform/tenant/app.dever.md b/examples/cms/md/module/platform/tenant/app.dever.md new file mode 100644 index 0000000..7045521 --- /dev/null +++ b/examples/cms/md/module/platform/tenant/app.dever.md @@ -0,0 +1,106 @@ +# 租户应用 + +提供平台初始化和登录期租户选择。 + +- 包：`platform.tenant.app` +- 公开类型： + - `TenantView` +- 公开方法： + - `provision` + - `active` + - `active_view` +- 使用： + - `platform.tenant.provision(key, name)` + - `platform.tenant.active(tenant_key)` + - `platform.tenant.active_view(stored)` + +## 租户视图 + +本节定义对应的源码合同。 + +- 类型：`TenantView` +- 字段： + - `id: model.id`：租户 ID + - `key: Text`：租户键 + - `name: Text`：名称 + +```dever +type TenantView { + id: model.id + key: Text + name: Text +} +``` + +## 创建租户 + +本节定义对应的源码合同。 + +- 函数：`provision` +- 输入： + - `key: Text`：租户键 + - `name: Text`：名称 +- 输出： + - `tenant: TenantView`：租户 + +```dever +provision(key: Text, name: Text) (tenant: TenantView) { + stored = model.create( + { + key = domain.normalize_key(key) + name = domain.normalize_name(name) + } + ) + tenant = TenantView { + id = stored.id + key = stored.key + name = stored.name + } +} +``` + +## 查找启用租户 + +本节定义对应的源码合同。 + +- 函数：`active` +- 输入： + - `tenant_key: Text`：租户键 +- 输出： + - `tenant: TenantView`：租户 + +```dever +active(tenant_key: Text) (tenant: TenantView) { + stored = model.first( + { + where = key == domain.normalize_key(tenant_key) + } + ) + tenant = active_view(stored) +} +``` + +## 映射启用租户 + +本节定义对应的源码合同。 + +- 函数：`active_view` +- 输入： + - `stored: platform.tenant.model.Tenant?`：查询结果 +- 输出： + - `tenant: TenantView`：租户 + +```dever +active_view(stored: model.Tenant) (tenant: TenantView) { + domain.require_active(stored.status == model.TenantStatus.Active) + tenant = TenantView { + id = stored.id + key = stored.key + name = stored.name + } +} + +active_view(stored: null) (tenant: TenantView) { + fail(dever.api.Error.Unauthorized) +} +``` diff --git a/examples/cms/md/module/platform/tenant/domain.dever.md b/examples/cms/md/module/platform/tenant/domain.dever.md new file mode 100644 index 0000000..f40148c --- /dev/null +++ b/examples/cms/md/module/platform/tenant/domain.dever.md @@ -0,0 +1,57 @@ +# 租户规则 + +租户键、名称和状态规则只在本领域使用。 + +- 包：`platform.tenant.domain` +- 公开类型：无 +- 公开方法：无 +- 使用：无 + +## 规范租户键 + +本节定义对应的源码合同。 + +- 函数：`normalize_key` +- 输入： + - `value: Text`：原值 +- 输出： + - `key: Text`：规范值 + +```dever +normalize_key(value: Text) (key: Text) pure { + key = text.lower(text.trim(value)) +} +``` + +## 规范租户名 + +本节定义对应的源码合同。 + +- 函数：`normalize_name` +- 输入： + - `value: Text`：原值 +- 输出： + - `name: Text`：规范值 + +```dever +normalize_name(value: Text) (name: Text) pure { + name = text.trim(value) +} +``` + +## 检查启用状态 + +本节定义对应的源码合同。 + +- 函数：`require_active` +- 输入： + - `active: Bool`：是否启用 +- 输出：无 + +```dever +require_active(active: true) () pure {} + +require_active(active: false) () pure { + fail(dever.api.Error.Unauthorized) +} +``` diff --git a/examples/cms/md/module/platform/tenant/model.dever.md b/examples/cms/md/module/platform/tenant/model.dever.md new file mode 100644 index 0000000..428eb07 --- /dev/null +++ b/examples/cms/md/module/platform/tenant/model.dever.md @@ -0,0 +1,46 @@ +# 租户模型 + +控制库保存租户目录，租户业务数据不放在本表。 + +- 包：`platform.tenant.model` +- 公开类型： + - `TenantStatus` + - `Tenant` +- 公开方法：无 +- 使用：无 + +## 租户状态 + +本节定义对应的源码合同。 + +- 类型：`TenantStatus` +- 分支： + - `Active`：启用 + - `Disabled`：禁用 + +```dever +type TenantStatus { + Active = "启用" + Disabled = "禁用" +} +``` + +## 租户记录 + +本节定义对应的源码合同。 + +- 类型：`Tenant` +- 字段： + - `uuid: Uuid`：稳定标识 + - `key: Text(1, 64)`：租户键 + - `name: Text(1, 100)`：租户名称 + - `status: TenantStatus`：状态 + +```dever +global type Tenant { + uuid: Uuid generated unique + key: Text(1, 64) unique + name: Text(1, 100) + status: TenantStatus default TenantStatus.Active index +} +``` diff --git a/examples/cms/md/module/publishing/schedule/api/admin/manage.dever.md b/examples/cms/md/module/publishing/schedule/api/admin/manage.dever.md new file mode 100644 index 0000000..2a2ee5b --- /dev/null +++ b/examples/cms/md/module/publishing/schedule/api/admin/manage.dever.md @@ -0,0 +1,18 @@ +# 后台发布计划接口 + +后台接口创建发布计划。 + +- 包：`publishing.schedule.api.admin.manage` +- 公开类型：无 +- 公开方法：无 +- 使用：无 + +## 创建计划 + +创建请求委托给计划应用。 + +- 声明：`post schedule` + +```dever +post schedule = app.create +``` diff --git a/examples/cms/md/module/publishing/schedule/app.dever.md b/examples/cms/md/module/publishing/schedule/app.dever.md new file mode 100644 index 0000000..0b087fc --- /dev/null +++ b/examples/cms/md/module/publishing/schedule/app.dever.md @@ -0,0 +1,166 @@ +# 发布计划应用 + +发布计划应用在事务中写入计划并提交持久化 Job。 + +- 包：`publishing.schedule.app` +- 公开类型： + - `PublishRequest` + - `ScheduleView` +- 公开方法： + - `create` + - `publish` + - `complete` + - `publish_claimed` +- 使用： + - `publishing.schedule.create(article_id, publish_at)` + - `publishing.schedule.publish(input)` + - `publishing.schedule.complete(status, schedule_id)` + - `publishing.schedule.publish_claimed(changed, article_id)` + +## 发布请求 + +Job 只携带计划 ID。 + +- 类型：`PublishRequest` +- 字段： + - `schedule_id: model.id`：持久计划标识 + +```dever +type PublishRequest { + schedule_id: model.id +} +``` + +## 计划视图 + +返回计划的公开字段。 + +- 类型：`ScheduleView` +- 字段： + - `id: model.id`：记录标识 + - `article_id: news.article.model.id`：文章标识 + - `publish_at: DateTime`：计划执行时间 + - `status: Text`：业务状态 + +```dever +type ScheduleView { + id: model.id + article_id: news.article.model.id + publish_at: DateTime + status: Text +} +``` + +## 创建计划 + +业务记录、Job 和返回视图在同一事务内完成。 + +- 函数：`create` +- 输入： + - `article_id: news.article.model.id`：文章标识 + - `publish_at: DateTime`：计划执行时间 +- 输出： + - `schedule: ScheduleView`：该声明的业务值 + +```dever +transaction create(article_id: news.article.model.id, publish_at: DateTime) ( + schedule: ScheduleView +) { + news.article.require_schedulable(article_id) + stored = model.create( + { + article_id = article_id + publish_at = publish_at + } + ) + queued = dever.job.enqueue_at( + job.publish, + PublishRequest { + schedule_id = stored.id + }, + uuid.to_text(stored.uuid), + publish_at + ) + discarded = queued + schedule = ScheduleView { + id = stored.id + article_id = stored.article_id + publish_at = stored.publish_at + status = "pending" + } +} +``` + +## 执行计划 + +Worker 首先检查计划状态，再调用文章领域的定时发布入口。 + +- 函数：`publish` +- 输入： + - `input: PublishRequest`：经过类型检查的输入 +- 输出：无 + +```dever +transaction publish(input: PublishRequest) () { + scheduled = model.get(input.schedule_id) + complete(scheduled.status, scheduled.id) +} +``` + +## complete + +发布计划应用在事务中写入计划并提交持久化 Job。 + +- 函数：`complete` +- 输入： + - `status: publishing.schedule.model.ScheduleStatus`：业务状态 + - `schedule_id: publishing.schedule.model.id`：持久计划标识 +- 输出：无 + +```dever +complete(status: model.ScheduleStatus.Completed, schedule_id: model.id) () {} + +complete(status: model.ScheduleStatus.Cancelled, schedule_id: model.id) () {} + +complete(status: model.ScheduleStatus.Pending, schedule_id: model.id) () { + scheduled = model.get(schedule_id) + domain.require_due( + model.exists( + { + where = id == schedule_id and publish_at <= dever.time.now() + } + ) + ) + changed = model.update( + { + where = id == scheduled.id and status == model.ScheduleStatus.Pending + }, + { + status = model.ScheduleStatus.Completed + } + ) + publish_claimed(changed, scheduled.article_id) +} +``` + +## publish_claimed + +发布计划应用在事务中写入计划并提交持久化 Job。 + +- 函数：`publish_claimed` +- 输入： + - `changed: Int`：条件更新影响行数 + - `article_id: news.article.model.id`：文章标识 +- 输出：无 + +```dever +publish_claimed(changed: 0, article_id: news.article.model.id) () {} + +publish_claimed(changed: 1, article_id: news.article.model.id) () { + publication = news.article.publish_scheduled(article_id) +} + +publish_claimed(changed: other, article_id: news.article.model.id) () { + fail(dever.api.Error.Conflict) +} +``` diff --git a/examples/cms/md/module/publishing/schedule/domain.dever.md b/examples/cms/md/module/publishing/schedule/domain.dever.md new file mode 100644 index 0000000..6d376d6 --- /dev/null +++ b/examples/cms/md/module/publishing/schedule/domain.dever.md @@ -0,0 +1,25 @@ +# 发布计划规则 + +计划任务只允许待发布记录进入执行流程。 + +- 包：`publishing.schedule.domain` +- 公开类型：无 +- 公开方法：无 +- 使用：无 + +## require_due + +计划任务只允许待发布记录进入执行流程。 + +- 函数：`require_due` +- 输入： + - `valid: Bool`：该声明的业务值 +- 输出：无 + +```dever +require_due(valid: true) () pure {} + +require_due(valid: false) () pure { + fail(dever.api.Error.Invalid) +} +``` diff --git a/examples/cms/md/module/publishing/schedule/job.dever.md b/examples/cms/md/module/publishing/schedule/job.dever.md new file mode 100644 index 0000000..59a742f --- /dev/null +++ b/examples/cms/md/module/publishing/schedule/job.dever.md @@ -0,0 +1,33 @@ +# 发布计划任务 + +发布计划任务由持久化 Job 执行。 + +- 包：`publishing.schedule.job` +- 公开类型：无 +- 公开方法：无 +- 使用：无 + +## 队列连接 + +使用默认数据库连接。 + +- 声明：`database default` + +```dever +database default +``` + +## 发布任务 + +任务委托给发布计划应用。 + +- 函数：`publish` +- 输入： + - `input: publishing.schedule.PublishRequest`：发布请求 +- 输出：无 + +```dever +job publish(input: app.PublishRequest) () retry(3) timeout(30000) { + app.publish(input) +} +``` diff --git a/examples/cms/md/module/publishing/schedule/model.dever.md b/examples/cms/md/module/publishing/schedule/model.dever.md new file mode 100644 index 0000000..cacd578 --- /dev/null +++ b/examples/cms/md/module/publishing/schedule/model.dever.md @@ -0,0 +1,48 @@ +# 发布计划模型 + +发布计划保存文章的计划时间和幂等状态。 + +- 包：`publishing.schedule.model` +- 公开类型： + - `ScheduleStatus` + - `Schedule` +- 公开方法：无 +- 使用：无 + +## 计划状态 + +计划状态枚举。 + +- 类型：`ScheduleStatus` +- 分支： + - `Pending`：该声明的业务值 + - `Completed`：该声明的业务值 + - `Cancelled`：该声明的业务值 + +```dever +type ScheduleStatus { + Pending = "待发布" + Completed = "已完成" + Cancelled = "已取消" +} +``` + +## 发布计划 + +计划关联文章和执行时间。 + +- 类型：`Schedule` +- 字段： + - `uuid: Uuid`：计划独立去重标识 + - `article_id: news.article.model.id`：文章标识 + - `publish_at: DateTime`：计划执行时间 + - `status: ScheduleStatus`：业务状态 + +```dever +type Schedule { + uuid: Uuid generated unique + article_id: news.article.model.id unique + publish_at: DateTime index + status: ScheduleStatus default ScheduleStatus.Pending index +} +``` diff --git a/examples/cms/md/module/system/health/api.dever.md b/examples/cms/md/module/system/health/api.dever.md new file mode 100644 index 0000000..14753da --- /dev/null +++ b/examples/cms/md/module/system/health/api.dever.md @@ -0,0 +1,18 @@ +# 系统健康命令 + +声明本地健康检查命令。 + +- 包：`system.health.api` +- 公开类型：无 +- 公开方法：无 +- 使用：无 + +## ping + +将命令映射到健康检查 App。 + +- 声明：`cmd ping` + +```dever +cmd ping = app.ping +``` diff --git a/examples/cms/md/module/system/health/app.dever.md b/examples/cms/md/module/system/health/app.dever.md new file mode 100644 index 0000000..d0bcba1 --- /dev/null +++ b/examples/cms/md/module/system/health/app.dever.md @@ -0,0 +1,42 @@ +# 系统健康 + +提供无需业务状态的本地运行检查。 + +- 包：`system.health.app` +- 公开类型： + - `HealthView` +- 公开方法： + - `ping` +- 使用： + - `system.health.ping()` + +## HealthView + +返回健康检查的就绪状态。 + +- 类型：`HealthView` +- 字段： + - `status: Text`：状态 + +```dever +type HealthView { + status: Text +} +``` + +## 健康检查 + +返回固定的就绪状态。 + +- 函数：`ping` +- 输入：无 +- 输出： + - `result: HealthView`：健康状态 + +```dever +ping() (result: HealthView) { + result = HealthView { + status = "ready" + } +} +``` diff --git a/examples/cms/md/module/user/account/api.dever.md b/examples/cms/md/module/user/account/api.dever.md new file mode 100644 index 0000000..480d8dd --- /dev/null +++ b/examples/cms/md/module/user/account/api.dever.md @@ -0,0 +1,18 @@ +# 账户命令 + +首次部署通过显式命令建立租户、用户主体、凭据和所有者成员关系。 + +- 包：`user.account.api` +- 公开类型：无 +- 公开方法：无 +- 使用：无 + +## 初始化平台账户 + +本节定义对应的源码合同。 + +- 声明：`cmd bootstrap` + +```dever +cmd bootstrap = app.bootstrap +``` diff --git a/examples/cms/md/module/user/account/api/admin/session.dever.md b/examples/cms/md/module/user/account/api/admin/session.dever.md new file mode 100644 index 0000000..46b6077 --- /dev/null +++ b/examples/cms/md/module/user/account/api/admin/session.dever.md @@ -0,0 +1,28 @@ +# 后台会话接口 + +后台站点登录公开，登出要求认证。 + +- 包：`user.account.api.admin.session` +- 公开类型：无 +- 公开方法：无 +- 使用：无 + +## 登录 + +本节定义对应的源码合同。 + +- 声明：`post login` + +```dever +public post login = app.login +``` + +## 登出 + +本节定义对应的源码合同。 + +- 声明：`post logout` + +```dever +post logout = app.logout +``` diff --git a/examples/cms/md/module/user/account/api/front/session.dever.md b/examples/cms/md/module/user/account/api/front/session.dever.md new file mode 100644 index 0000000..bb490fb --- /dev/null +++ b/examples/cms/md/module/user/account/api/front/session.dever.md @@ -0,0 +1,28 @@ +# 前台会话接口 + +前台复用账户和会话，但由 front 站点签发独立受众令牌。 + +- 包：`user.account.api.front.session` +- 公开类型：无 +- 公开方法：无 +- 使用：无 + +## 登录 + +本节定义对应的源码合同。 + +- 声明：`post login` + +```dever +public post login = app.login +``` + +## 登出 + +本节定义对应的源码合同。 + +- 声明：`post logout` + +```dever +post logout = app.logout +``` diff --git a/examples/cms/md/module/user/account/app.dever.md b/examples/cms/md/module/user/account/app.dever.md new file mode 100644 index 0000000..b496274 --- /dev/null +++ b/examples/cms/md/module/user/account/app.dever.md @@ -0,0 +1,350 @@ +# 账户应用 + +账户应用拥有凭据、登录、会话签发和认证身份组装。 + +- 包：`user.account.app` +- 公开类型： + - `AccountView` + - `BootstrapView` + - `LoginView` + - `LogoutView` + - `Identity` +- 公开方法： + - `bootstrap` + - `login` + - `establish_login` + - `logout` + - `verify` + - `active` + - `current_user` + - `require_user` +- 使用： + - `user.account.bootstrap(tenant_key, tenant_name, email, display_name, password)` + - `user.account.login(credential_id, password, tenant)` + - `user.account.establish_login(credential_id, tenant, site)` + - `user.account.logout()` + - `user.account.verify(claims)` + - `user.account.active(user_id)` + - `user.account.current_user()` + - `user.account.require_user(value)` + +## 账户视图 + +本节定义对应的源码合同。 + +- 类型：`AccountView` +- 字段： + - `id: model.principal.id`：用户主体 + - `uuid: Uuid`：身份 UUID + - `email: Text`：邮箱 + - `display_name: Text`：显示名 + +```dever +type AccountView { + id: model.principal.id + uuid: Uuid + email: Text + display_name: Text +} +``` + +## 初始化结果 + +本节定义对应的源码合同。 + +- 类型：`BootstrapView` +- 字段： + - `tenant_id: platform.tenant.model.id`：租户 + - `user_id: model.principal.id`：用户主体 + - `credential_id: model.id`：凭据 ID + +```dever +type BootstrapView { + tenant_id: platform.tenant.model.id + user_id: model.principal.id + credential_id: model.id +} +``` + +## 登录结果 + +本节定义对应的源码合同。 + +- 类型：`LoginView` +- 字段： + - `account: AccountView`：账户 + - `tenant_id: platform.tenant.model.id`：租户 + - `site: Text`：站点 + +```dever +type LoginView { + account: AccountView + tenant_id: platform.tenant.model.id + site: Text +} +``` + +## 登出结果 + +本节定义对应的源码合同。 + +- 类型：`LogoutView` +- 字段： + - `revoked: Bool`：是否撤销 + +```dever +type LogoutView { + revoked: Bool +} +``` + +## 认证身份 + +本节定义对应的源码合同。 + +- 类型：`Identity` +- 字段： + - `id: Text`：JWT subject + - `user_id: model.principal.id?`：用户主体 + - `tenant_id: platform.tenant.model.id?`：租户 + +```dever +type Identity { + id: Text + user_id: model.principal.id? + tenant_id: platform.tenant.model.id? +} +``` + +## 初始化平台账户 + +本节定义对应的源码合同。 + +- 函数：`bootstrap` +- 输入： + - `tenant_key: Text`：租户键 + - `tenant_name: Text`：租户名称 + - `email: Text`：邮箱 + - `display_name: Text`：显示名 + - `password: Secret`：初始密码 +- 输出： + - `result: BootstrapView`：初始化结果 + +```dever +bootstrap( + tenant_key: Text, + tenant_name: Text, + email: Text, + display_name: Text, + password: Secret +) (result: BootstrapView) { + tenant = platform.tenant.provision(tenant_key, tenant_name) + input = domain.prepare(email, display_name) + hash = blocking(dever.crypto.password_hash(password)) + principal = model.principal.create( + { + display_name = input.name + } + ) + stored = model.create( + { + principal_id = principal.id + email = input.address + password_hash = hash + } + ) + user.membership.grant_owner(principal.id, tenant.id) + result = BootstrapView { + tenant_id = tenant.id + user_id = principal.id + credential_id = stored.id + } +} +``` + +## 登录 + +本节定义对应的源码合同。 + +- 函数：`login` +- 输入： + - `credential_id: user.account.model.id`：凭据 ID + - `password: Secret`：密码 + - `tenant: Text`：租户键 +- 输出： + - `result: LoginView`：登录结果 + +```dever +login(credential_id: model.id, password: Secret, tenant: Text) (result: LoginView) { + domain.require_active( + model.exists( + { + where = id == credential_id + } + ) + ) + stored = model.get(credential_id) + principal = model.principal.get(stored.principal_id) + selected = platform.tenant.active(tenant) + domain.require_active(stored.status == model.AccountStatus.Active) + domain.require_active(principal.status == model.principal.PrincipalStatus.Active) + domain.require_active( + blocking(dever.crypto.password_verify(password, stored.password_hash)) + ) + authenticated = establish_login(credential_id, selected.key, dever.site.key()) + dever.auth.issue_cookie( + authenticated.subject, + uuid.to_text(authenticated.session_id), + authenticated.tenant_key + ) + result = authenticated.result +} +``` + +## 建立登录会话 + +密码校验在事务外完成。本函数只负责在短事务内重新检查账户、租户和成员关系，再创建会话。 + +- 函数：`establish_login` +- 输入： + - `credential_id: user.account.model.id`：凭据 ID + - `tenant: Text`：租户键 + - `site: Text`：站点键 +- 输出： + - `result: LoginView`：登录视图 + - `subject: Text`：认证主体 + - `session_id: Uuid`：会话 UUID + - `tenant_key: Text`：租户键 + +```dever +transaction establish_login(credential_id: model.id, tenant: Text, site: Text) ( + result: LoginView, + subject: Text, + session_id: Uuid, + tenant_key: Text +) { + stored = model.get(credential_id) + principal = model.principal.get(stored.principal_id) + selected = platform.tenant.active(tenant) + domain.require_active(stored.status == model.AccountStatus.Active) + domain.require_active(principal.status == model.principal.PrincipalStatus.Active) + user.membership.require_active(principal.id, selected.id) + session = user.session.open( + uuid.to_text(principal.uuid), + principal.id, + selected.id, + selected.key, + site + ) + result = LoginView { + account = AccountView { + id = principal.id + uuid = principal.uuid + email = stored.email + display_name = principal.display_name + } + tenant_id = selected.id + site = site + } + subject = uuid.to_text(principal.uuid) + session_id = session.uuid + tenant_key = selected.key +} +``` + +## 登出 + +本节定义对应的源码合同。 + +- 函数：`logout` +- 输入：无 +- 输出： + - `result: LogoutView`：登出结果 + +```dever +logout() (result: LogoutView) { + user.session.revoke(uuid.parse(dever.auth.session())) + dever.auth.clear_cookie() + result = LogoutView { + revoked = true + } +} +``` + +## 验证认证声明 + +本节定义对应的源码合同。 + +- 函数：`verify` +- 输入： + - `claims: dever.auth.Claims`：已验签声明 +- 输出： + - `identity: Identity`：可信身份 + +```dever +verify(claims: dever.auth.Claims) (identity: Identity) { + session = user.session.authenticate(uuid.parse(claims.session), claims.site) + domain.require_active(session.subject == claims.subject) + domain.require_tenant(session.tenant_key, claims.tenant) + selected = platform.tenant.active(session.tenant_key) + domain.require_active(selected.id == session.tenant_id) + active(session.user_id) + user.membership.require_active(session.user_id, session.tenant_id) + identity = Identity { + id = claims.subject + user_id = session.user_id + tenant_id = session.tenant_id + } +} +``` + +## 验证用户主体 + +本节定义对应的源码合同。 + +- 函数：`active` +- 输入： + - `user_id: user.account.model.principal.id`：用户主体 +- 输出：无 + +```dever +active(user_id: model.principal.id) () { + principal = model.principal.get(user_id) + domain.require_active(principal.status == model.principal.PrincipalStatus.Active) +} +``` + +## 当前用户的类型化标识 + +读取入口已经验证的可信用户标识，保留 Model ID 类型；不重新查询全局会话数据库，也不接受请求传入的作者 ID。 + +- 函数：`current_user` +- 输入：无 +- 输出： + - `id: user.account.model.principal.id`：可信用户主体 + +```dever +current_user() (id: model.principal.id) { + id = require_user(dever.auth.user_id()) +} +``` + +## 要求可信用户标识 + +有可信用户时保留其名义类型；没有用户身份时明确拒绝。 + +- 函数：`require_user` +- 输入： + - `value: user.account.model.principal.id?`：已验证的可空用户标识 +- 输出： + - `id: user.account.model.principal.id`：可信用户主体 + +```dever +require_user(value: model.principal.id) (id: model.principal.id) { + id = value +} + +require_user(value: null) (id: model.principal.id) { + fail(dever.api.Error.Unauthorized) +} +``` diff --git a/examples/cms/md/module/user/account/domain.dever.md b/examples/cms/md/module/user/account/domain.dever.md new file mode 100644 index 0000000..ba8373e --- /dev/null +++ b/examples/cms/md/module/user/account/domain.dever.md @@ -0,0 +1,141 @@ +# 账户规则 + +账户输入和认证断言集中在纯 Domain。 + +- 包：`user.account.domain` +- 公开类型：无 +- 公开方法：无 +- 使用：无 + +## 账户输入 + +本节定义对应的源码合同。 + +- 类型：`AccountInput` +- 字段： + - `address: Text`：邮箱 + - `name: Text`：显示名 + +```dever +type AccountInput { + address: Text + name: Text +} +``` + +## 规范邮箱 + +本节定义对应的源码合同。 + +- 函数：`normalize_email` +- 输入： + - `email: Text`：邮箱 +- 输出： + - `address: Text`：规范邮箱 + +```dever +normalize_email(email: Text) (address: Text) pure { + address = text.lower(text.trim(email)) +} +``` + +## 规范显示名 + +本节定义对应的源码合同。 + +- 函数：`normalize_name` +- 输入： + - `display_name: Text`：显示名 +- 输出： + - `name: Text`：规范名称 + +```dever +normalize_name(display_name: Text) (name: Text) pure { + name = text.trim(display_name) +} +``` + +## 准备账户输入 + +本节定义对应的源码合同。 + +- 函数：`prepare` +- 输入： + - `email: Text`：邮箱 + - `display_name: Text`：显示名 +- 输出： + - `input: AccountInput`：规范输入 + +```dever +prepare(email: Text, display_name: Text) (input: AccountInput) pure { + address = normalize_email(email) + name = normalize_name(display_name) + require( + text.contains(address, "@") and not text.starts_with(address, "@") and not text.ends_with( + address, + "@" + ), + "email must contain a local part and domain" + ) + require(not text.is_empty(name), "display name cannot be blank") + input = AccountInput { + address = address + name = name + } +} +``` + +## 检查输入 + +本节定义对应的源码合同。 + +- 函数：`require` +- 输入： + - `valid: Bool`：是否有效 + - `message: Text`：内部原因 +- 输出：无 + +```dever +require(valid: true, message: Text) () pure {} + +require(valid: false, message: Text) () pure { + fail(dever.api.Error.Invalid) +} +``` + +## 检查认证状态 + +本节定义对应的源码合同。 + +- 函数：`require_active` +- 输入： + - `active: Bool`：是否有效 +- 输出：无 + +```dever +require_active(active: true) () pure {} + +require_active(active: false) () pure { + fail(dever.api.Error.Unauthorized) +} +``` + +## 检查租户声明 + +本节定义对应的源码合同。 + +- 函数：`require_tenant` +- 输入： + - `expected: Text`：可信租户键 + - `actual: Text?`：JWT 租户声明 +- 输出：无 + +```dever +require_tenant(expected: Text, actual: Text) () pure { + require_active(expected == actual) +} + +require_tenant(expected: Text, actual: null) () pure { + fail(dever.api.Error.Unauthorized) +} +``` diff --git a/examples/cms/md/module/user/account/model.dever.md b/examples/cms/md/module/user/account/model.dever.md new file mode 100644 index 0000000..cfac7f4 --- /dev/null +++ b/examples/cms/md/module/user/account/model.dever.md @@ -0,0 +1,46 @@ +# 账户凭据模型 + +控制库只在该模型保存密码哈希，公开身份由 Principal 承担。 + +- 包：`user.account.model` +- 公开类型： + - `AccountStatus` + - `Account` +- 公开方法：无 +- 使用：无 + +## 凭据状态 + +本节定义对应的源码合同。 + +- 类型：`AccountStatus` +- 分支： + - `Active`：启用 + - `Disabled`：禁用 + +```dever +type AccountStatus { + Active = "启用" + Disabled = "禁用" +} +``` + +## 账户凭据 + +本节定义对应的源码合同。 + +- 类型：`Account` +- 字段： + - `principal_id: user.account.model.principal.id`：用户主体 + - `email: Text(254)`：登录邮箱 + - `private password_hash: Text(255)`：私密密码哈希 + - `status: AccountStatus`：凭据状态 + +```dever +global type Account { + principal_id: user.account.model.principal.id unique + email: Text(254) unique + private password_hash: Text(255) + status: AccountStatus default AccountStatus.Active index +} +``` diff --git a/examples/cms/md/module/user/account/model/principal.dever.md b/examples/cms/md/module/user/account/model/principal.dever.md new file mode 100644 index 0000000..116f29e --- /dev/null +++ b/examples/cms/md/module/user/account/model/principal.dever.md @@ -0,0 +1,44 @@ +# 用户主体模型 + +Principal 是会话、成员关系和认证 Identity 共用的公开用户身份。 + +- 包：`user.account.model.principal` +- 公开类型： + - `PrincipalStatus` + - `Principal` +- 公开方法：无 +- 使用：无 + +## 主体状态 + +本节定义对应的源码合同。 + +- 类型：`PrincipalStatus` +- 分支： + - `Active`：启用 + - `Disabled`：禁用 + +```dever +type PrincipalStatus { + Active = "启用" + Disabled = "禁用" +} +``` + +## 用户主体 + +本节定义对应的源码合同。 + +- 类型：`Principal` +- 字段： + - `uuid: Uuid`：身份标识 + - `display_name: Text(1, 64)`：显示名称 + - `status: PrincipalStatus`：状态 + +```dever +global type Principal { + uuid: Uuid generated unique + display_name: Text(1, 64) + status: PrincipalStatus default PrincipalStatus.Active index +} +``` diff --git a/examples/cms/md/module/user/authorization/api/admin/manage.dever.md b/examples/cms/md/module/user/authorization/api/admin/manage.dever.md new file mode 100644 index 0000000..9e54db2 --- /dev/null +++ b/examples/cms/md/module/user/authorization/api/admin/manage.dever.md @@ -0,0 +1,58 @@ +# 后台授权接口 + +后台接口管理当前租户、当前站点的角色授权。 + +- 包：`user.authorization.api.admin.manage` +- 公开类型：无 +- 公开方法：无 +- 使用：无 + +## 权限目录 + +本节声明当前站点权限目录接口。 + +- 声明：`get permissions` + +```dever +get permissions = app.permissions +``` + +## 保存角色 + +本节声明保存当前站点角色及权限集合的接口。 + +- 声明：`post save` + +```dever +post save = app.save +``` + +## 授予角色 + +本节声明向用户授予当前站点角色的接口。 + +- 声明：`post grant` + +```dever +post grant = app.grant +``` + +## 撤销角色 + +本节声明从用户撤销当前站点角色的接口。 + +- 声明：`delete revoke` + +```dever +delete revoke = app.revoke +``` + +## 禁用角色 + +本节声明禁用当前站点角色的接口。 + +- 声明：`delete disable` + +```dever +delete disable = app.disable +``` diff --git a/examples/cms/md/module/user/authorization/api/front/manage.dever.md b/examples/cms/md/module/user/authorization/api/front/manage.dever.md new file mode 100644 index 0000000..5e1d1b9 --- /dev/null +++ b/examples/cms/md/module/user/authorization/api/front/manage.dever.md @@ -0,0 +1,58 @@ +# 前台授权接口 + +前台接口管理当前租户、当前站点的角色授权。 + +- 包：`user.authorization.api.front.manage` +- 公开类型：无 +- 公开方法：无 +- 使用：无 + +## 权限目录 + +本节声明当前站点权限目录接口。 + +- 声明：`get permissions` + +```dever +get permissions = app.permissions +``` + +## 保存角色 + +本节声明保存当前站点角色及权限集合的接口。 + +- 声明：`post save` + +```dever +post save = app.save +``` + +## 授予角色 + +本节声明向用户授予当前站点角色的接口。 + +- 声明：`post grant` + +```dever +post grant = app.grant +``` + +## 撤销角色 + +本节声明从用户撤销当前站点角色的接口。 + +- 声明：`delete revoke` + +```dever +delete revoke = app.revoke +``` + +## 禁用角色 + +本节声明禁用当前站点角色的接口。 + +- 声明：`delete disable` + +```dever +delete disable = app.disable +``` diff --git a/examples/cms/md/module/user/authorization/app.dever.md b/examples/cms/md/module/user/authorization/app.dever.md new file mode 100644 index 0000000..dcb8e82 --- /dev/null +++ b/examples/cms/md/module/user/authorization/app.dever.md @@ -0,0 +1,108 @@ +# 授权管理应用 + +授权管理应用只适配 Dever 核心角色存储，不保存业务权限副本。 + +- 包：`user.authorization.app` +- 公开类型：无 +- 公开方法： + - `permissions` + - `save` + - `grant` + - `revoke` + - `disable` +- 使用： + - `user.authorization.permissions()` + - `user.authorization.save(id, name, all_permissions, permission_keys)` + - `user.authorization.grant(user_id, role_id)` + - `user.authorization.revoke(user_id, role_id)` + - `user.authorization.disable(role_id)` + +## 权限目录 + +本节定义当前站点权限目录的读取入口。 + +- 函数：`permissions` +- 输入：无 +- 输出： + - `values: List<dever.auth.Permission>`：当前站点权限目录 + +```dever +permissions() (values: List<dever.auth.Permission>) { + values = dever.auth.permissions() +} +``` + +## 保存角色 + +本节定义当前站点角色及其权限集合的保存入口。 + +- 函数：`save` +- 输入： + - `id: Text`：角色 ID + - `name: Text`：角色名称 + - `all_permissions: Bool`：是否拥有当前站点全部权限 + - `permission_keys: List<Text>`：权限键 +- 输出： + - `ok: Bool`：保存成功 + +```dever +save(id: Text, name: Text, all_permissions: Bool, permission_keys: List<Text>) ( + ok: Bool +) { + dever.auth.save_role(id, name, all_permissions, permission_keys) + ok = true +} +``` + +## 授予角色 + +本节定义向用户授予当前站点角色的入口。 + +- 函数：`grant` +- 输入： + - `user_id: Int`：用户 ID + - `role_id: Text`：角色 ID +- 输出： + - `ok: Bool`：授予成功 + +```dever +grant(user_id: Int, role_id: Text) (ok: Bool) { + dever.auth.grant_role(user_id, role_id) + ok = true +} +``` + +## 撤销角色 + +本节定义从用户撤销当前站点角色的入口。 + +- 函数：`revoke` +- 输入： + - `user_id: Int`：用户 ID + - `role_id: Text`：角色 ID +- 输出： + - `ok: Bool`：撤销成功 + +```dever +revoke(user_id: Int, role_id: Text) (ok: Bool) { + dever.auth.revoke_role(user_id, role_id) + ok = true +} +``` + +## 禁用角色 + +本节定义禁用当前站点角色的入口。 + +- 函数：`disable` +- 输入： + - `role_id: Text`：角色 ID +- 输出： + - `ok: Bool`：禁用成功 + +```dever +disable(role_id: Text) (ok: Bool) { + dever.auth.disable_role(role_id) + ok = true +} +``` diff --git a/examples/cms/md/module/user/membership/app.dever.md b/examples/cms/md/module/user/membership/app.dever.md new file mode 100644 index 0000000..ee5884c --- /dev/null +++ b/examples/cms/md/module/user/membership/app.dever.md @@ -0,0 +1,81 @@ +# 成员应用 + +成员应用建立租户关系并验证有效成员。 + +- 包：`user.membership.app` +- 公开类型：无 +- 公开方法： + - `grant_owner` + - `require_active` + - `active` +- 使用： + - `user.membership.grant_owner(account_id, selected_tenant_id)` + - `user.membership.require_active(account_id, selected_tenant_id)` + - `user.membership.active(stored)` + +## 授予所有者 + +本节定义对应的源码合同。 + +- 函数：`grant_owner` +- 输入： + - `account_id: user.account.model.principal.id`：用户主体 + - `selected_tenant_id: platform.tenant.model.id`：租户 +- 输出：无 + +```dever +grant_owner( + account_id: user.account.model.principal.id, + selected_tenant_id: platform.tenant.model.id +) () { + stored = model.create( + { + user_id = account_id + tenant_id = selected_tenant_id + role = model.MembershipRole.Owner + } + ) + discarded = stored +} +``` + +## 验证有效成员 + +本节定义对应的源码合同。 + +- 函数：`require_active` +- 输入： + - `account_id: user.account.model.principal.id`：用户主体 + - `selected_tenant_id: platform.tenant.model.id`：租户 +- 输出：无 + +```dever +require_active( + account_id: user.account.model.principal.id, + selected_tenant_id: platform.tenant.model.id +) () { + stored = model.first( + { + where = user_id == account_id and tenant_id == selected_tenant_id and status == model.MembershipStatus.Active + } + ) + active(stored) +} +``` + +## 检查查询结果 + +本节定义对应的源码合同。 + +- 函数：`active` +- 输入： + - `stored: user.membership.model.Membership?`：成员查询结果 +- 输出：无 + +```dever +active(stored: model.Membership) () {} + +active(stored: null) () { + fail(dever.api.Error.Unauthorized) +} +``` diff --git a/examples/cms/md/module/user/membership/model.dever.md b/examples/cms/md/module/user/membership/model.dever.md new file mode 100644 index 0000000..b9f3aad --- /dev/null +++ b/examples/cms/md/module/user/membership/model.dever.md @@ -0,0 +1,75 @@ +# 租户成员模型 + +控制库保存用户主体在租户中的角色。 + +- 包：`user.membership.model` +- 公开类型： + - `MembershipRole` + - `MembershipStatus` + - `Membership` +- 公开方法：无 +- 使用：无 + +## 成员角色 + +本节定义对应的源码合同。 + +- 类型：`MembershipRole` +- 分支： + - `Owner`：所有者 + - `Editor`：编辑 + - `Reader`：读者 + +```dever +type MembershipRole { + Owner = "所有者" + Editor = "编辑" + Reader = "读者" +} +``` + +## 成员状态 + +本节定义对应的源码合同。 + +- 类型：`MembershipStatus` +- 分支： + - `Active`：启用 + - `Disabled`：禁用 + +```dever +type MembershipStatus { + Active = "启用" + Disabled = "禁用" +} +``` + +## 成员记录 + +本节定义对应的源码合同。 + +- 类型：`Membership` +- 字段： + - `user_id: user.account.model.principal.id`：用户主体 + - `tenant_id: platform.tenant.model.id`：租户 + - `role: MembershipRole`：角色 + - `status: MembershipStatus`：状态 + +```dever +global type Membership { + user_id: user.account.model.principal.id + tenant_id: platform.tenant.model.id + role: MembershipRole index + status: MembershipStatus default MembershipStatus.Active index +} +``` + +## 成员唯一约束 + +本节定义对应的源码合同。 + +- 声明：`unique(user_id, tenant_id)` + +```dever +unique(user_id, tenant_id) +``` diff --git a/examples/cms/md/module/user/session/app.dever.md b/examples/cms/md/module/user/session/app.dever.md new file mode 100644 index 0000000..f4d8636 --- /dev/null +++ b/examples/cms/md/module/user/session/app.dever.md @@ -0,0 +1,213 @@ +# 会话应用 + +会话应用创建、验证和撤销站点绑定的数据库会话。 + +- 包：`user.session.app` +- 公开类型： + - `SessionView` + - `AuthSession` +- 公开方法： + - `open` + - `authenticate` + - `revoke` + - `auth_session` + - `revoke_found` + - `cleanup` +- 使用： + - `user.session.open(subject, account_id, selected_tenant_id, selected_tenant_key, site_key)` + - `user.session.authenticate(session_id, site_key)` + - `user.session.revoke(session_id)` + - `user.session.auth_session(stored)` + - `user.session.revoke_found(stored)` + - `user.session.cleanup()` + +## 会话视图 + +本节定义对应的源码合同。 + +- 类型：`SessionView` +- 字段： + - `uuid: Uuid`：会话 UUID + +```dever +type SessionView { + uuid: Uuid +} +``` + +## 认证会话 + +本节定义对应的源码合同。 + +- 类型：`AuthSession` +- 字段： + - `subject: Text`：subject + - `user_id: user.account.model.principal.id`：用户主体 + - `tenant_id: platform.tenant.model.id`：租户 + - `tenant_key: Text`：租户声明 + +```dever +type AuthSession { + subject: Text + user_id: user.account.model.principal.id + tenant_id: platform.tenant.model.id + tenant_key: Text +} +``` + +## 创建会话 + +本节定义对应的源码合同。 + +- 函数：`open` +- 输入： + - `subject: Text`：subject + - `account_id: user.account.model.principal.id`：用户主体 + - `selected_tenant_id: platform.tenant.model.id`：租户 + - `selected_tenant_key: Text`：租户键 + - `site_key: Text`：站点 +- 输出： + - `session: SessionView`：会话 + +```dever +open( + subject: Text, + account_id: user.account.model.principal.id, + selected_tenant_id: platform.tenant.model.id, + selected_tenant_key: Text, + site_key: Text +) (session: SessionView) { + stored = model.create( + { + subject = subject + user_id = account_id + tenant_id = selected_tenant_id + tenant_key = selected_tenant_key + site = site_key + expires_at = dever.time.add(dever.time.now(), dever.time.duration(3600000)) + } + ) + session = SessionView { + uuid = stored.uuid + } +} +``` + +## 验证会话 + +本节定义对应的源码合同。 + +- 函数：`authenticate` +- 输入： + - `session_id: Uuid?`：会话 UUID + - `site_key: Text`：站点 +- 输出： + - `result: AuthSession`：认证会话 + +```dever +authenticate(session_id: Uuid, site_key: Text) (result: AuthSession) { + stored = model.first( + { + where = uuid == session_id and site == site_key and revoked == false and expires_at > dever.time.now( + ) + } + ) + result = auth_session(stored) +} + +authenticate(session_id: null, site_key: Text) (result: AuthSession) { + fail(dever.api.Error.Unauthorized) +} +``` + +## 撤销会话 + +本节定义对应的源码合同。 + +- 函数：`revoke` +- 输入： + - `session_id: Uuid?`：会话 UUID +- 输出：无 + +```dever +revoke(session_id: Uuid) () { + stored = model.first( + { + where = uuid == session_id and revoked == false + } + ) + revoke_found(stored) +} + +revoke(session_id: null) () { + fail(dever.api.Error.Unauthorized) +} +``` + +## 映射认证会话 + +本节定义对应的源码合同。 + +- 函数：`auth_session` +- 输入： + - `stored: user.session.model.Session?`：查询结果 +- 输出： + - `result: AuthSession`：认证会话 + +```dever +auth_session(stored: model.Session) (result: AuthSession) { + result = AuthSession { + subject = stored.subject + user_id = stored.user_id + tenant_id = stored.tenant_id + tenant_key = stored.tenant_key + } +} + +auth_session(stored: null) (result: AuthSession) { + fail(dever.api.Error.Unauthorized) +} +``` + +## 更新撤销状态 + +本节定义对应的源码合同。 + +- 函数：`revoke_found` +- 输入： + - `stored: user.session.model.Session?`：查询结果 +- 输出：无 + +```dever +revoke_found(stored: model.Session) () { + changed = model.update( + stored.id, + { + revoked = true + } + ) + discarded = changed +} + +revoke_found(stored: null) () { + fail(dever.api.Error.Unauthorized) +} +``` + +## 清理过期会话 + +清理已撤销或已过期的会话记录。 + +- 函数：`cleanup` +- 输入：无 +- 输出：无 + +```dever +cleanup() () { + removed = model.delete( + { + where = revoked == true or expires_at <= dever.time.now() + } + ) +} +``` diff --git a/examples/cms/md/module/user/session/job.dever.md b/examples/cms/md/module/user/session/job.dever.md new file mode 100644 index 0000000..c6a11da --- /dev/null +++ b/examples/cms/md/module/user/session/job.dever.md @@ -0,0 +1,42 @@ +# 会话清理任务 + +周期任务清理撤销或过期会话。 + +- 包：`user.session.job` +- 公开类型：无 +- 公开方法：无 +- 使用：无 + +## 队列连接 + +使用默认数据库连接。 + +- 声明：`database default` + +```dever +database default +``` + +## 清理任务 + +任务委托给会话应用。 + +- 函数：`cleanup` +- 输入：无 +- 输出：无 + +```dever +job cleanup() () retry(3) timeout(30000) { + app.cleanup() +} +``` + +## 每小时执行 + +按 UTC 每小时清理。 + +- 声明：`schedule cleanup` + +```dever +schedule cleanup = "0 * * * *" +``` diff --git a/examples/cms/md/module/user/session/model.dever.md b/examples/cms/md/module/user/session/model.dever.md new file mode 100644 index 0000000..b241726 --- /dev/null +++ b/examples/cms/md/module/user/session/model.dever.md @@ -0,0 +1,47 @@ +# 会话模型 + +控制库保存站点绑定、租户绑定且可撤销的登录会话。 + +- 包：`user.session.model` +- 公开类型： + - `Session` +- 公开方法：无 +- 使用：无 + +## 会话记录 + +本节定义对应的源码合同。 + +- 类型：`Session` +- 字段： + - `uuid: Uuid`：会话标识 + - `subject: Text(1, 128)`：JWT subject + - `user_id: user.account.model.principal.id`：用户主体 + - `tenant_id: platform.tenant.model.id`：租户 + - `tenant_key: Text(1, 64)`：JWT 租户声明 + - `site: Text(1, 32)`：站点 + - `expires_at: DateTime`：过期时间 + - `revoked: Bool`：撤销状态 + +```dever +global type Session { + uuid: Uuid generated unique + subject: Text(1, 128) + user_id: user.account.model.principal.id + tenant_id: platform.tenant.model.id + tenant_key: Text(1, 64) + site: Text(1, 32) + expires_at: DateTime index + revoked: Bool default false index +} +``` + +## 会话查询索引 + +本节定义对应的源码合同。 + +- 声明：`index(user_id, tenant_id, site)` + +```dever +index(user_id, tenant_id, site) +``` diff --git a/examples/cms/md/test/content/category/lifecycle.dever.md b/examples/cms/md/test/content/category/lifecycle.dever.md new file mode 100644 index 0000000..5e7e37a --- /dev/null +++ b/examples/cms/md/test/content/category/lifecycle.dever.md @@ -0,0 +1,32 @@ +# content.category.lifecycle + +验证分类创建、规范化、活动状态读取及停用对独立分类的隔离。停用分类的关联拒绝由 HTTP 验收覆盖。 + +- 包：`content.category.lifecycle` +- 公开类型：无 +- 公开方法：无 +- 使用：无 + +## lifecycle + +通过真实 App 操作分类，并核对独立分类的标识和可用状态。 + +- 函数：`lifecycle` +- 输入：无 +- 输出：无 + +```dever +lifecycle() () { + category = app.create(" Dever News ") + assert_eq(category.name, "Dever News") + assert_eq(category.slug, "dever news") + assert_eq(category.status, "active") + app.require_active(category.id) + app.require_active(null) + assert(app.disable(category.id)) + independent = app.create("Announcements") + assert(independent.id != category.id) + assert_eq(independent.slug, "announcements") + app.require_active(independent.id) +} +``` diff --git a/examples/cms/md/test/news/article/publish.dever.md b/examples/cms/md/test/news/article/publish.dever.md new file mode 100644 index 0000000..644afd7 --- /dev/null +++ b/examples/cms/md/test/news/article/publish.dever.md @@ -0,0 +1,30 @@ +# 文章发布测试 + +验证文章标识和标题规范化、分页边界，以及合法发布条件。数据库发布、权限拒绝和版本冲突由 HTTP 验收覆盖。 + +- 包：`news.article.publish` +- 公开类型：无 +- 公开方法：无 +- 使用：无 + +## 发布 + +通过真实 Domain 调用核对规范化结果和合法边界。 + +- 函数：`publish` +- 输入：无 +- 输出：无 + +```dever +publish() () { + assert_eq(domain.normalize_slug(" News-Article "), "news-article") + assert_eq(domain.normalize_title(" First article "), "First article") + assert_eq(domain.page_size(1), 1) + assert_eq(domain.page_size(100), 100) + domain.require_owner(true) + domain.require_draft(true) + domain.require_updated(1) + domain.require_content("news", "Title", "Body") + domain.require_published(true) +} +``` diff --git a/examples/cms/md/test/user/account/bootstrap.dever.md b/examples/cms/md/test/user/account/bootstrap.dever.md new file mode 100644 index 0000000..f4cd8ea --- /dev/null +++ b/examples/cms/md/test/user/account/bootstrap.dever.md @@ -0,0 +1,35 @@ +# 平台初始化测试 + +在运行器隔离的数据库中建立租户、用户主体和私密凭据，读取所有者成员关系并核对不同租户的标识。 + +- 包：`user.account.bootstrap` +- 公开类型：无 +- 公开方法：无 +- 使用：无 + +## 初始化平台账户 + +本节定义对应的源码合同。 + +- 函数：`bootstrap` +- 输入：无 +- 输出：无 + +```dever +bootstrap() () { + created = user.account.bootstrap( + "tenant-one", + "Tenant One", + " owner@example.com ", + " Owner ", + secret("correct horse battery staple") + ) + user.membership.require_active(created.user_id, created.tenant_id) + user.account.active(created.user_id) + tenant = platform.tenant.active("tenant-one") + assert_eq(tenant.id, created.tenant_id) + assert_eq(tenant.key, "tenant-one") + unrelated = platform.tenant.provision("tenant-two", "Tenant Two") + assert(unrelated.id != created.tenant_id) +} +``` diff --git a/examples/cms/md/test/user/account/validation.dever.md b/examples/cms/md/test/user/account/validation.dever.md new file mode 100644 index 0000000..30f4bdb --- /dev/null +++ b/examples/cms/md/test/user/account/validation.dever.md @@ -0,0 +1,27 @@ +# user.account.validation + +验证账户邮箱和显示名称规范化，包括中文名称。非法请求由 HTTP 验收覆盖。 + +- 包：`user.account.validation` +- 公开类型：无 +- 公开方法：无 +- 使用：无 + +## validation + +核对真实 Domain 返回的规范化字段。 + +- 函数：`validation` +- 输入：无 +- 输出：无 + +```dever +validation() () { + prepared = domain.prepare(" OWNER@EXAMPLE.COM ", " Owner ") + assert_eq(prepared.address, "owner@example.com") + assert_eq(prepared.name, "Owner") + unicode_name = domain.prepare("editor@example.com", " 编辑员 ") + assert_eq(unicode_name.address, "editor@example.com") + assert_eq(unicode_name.name, "编辑员") +} +``` diff --git a/examples/cms/md/test/user/session/lifecycle.dever.md b/examples/cms/md/test/user/session/lifecycle.dever.md new file mode 100644 index 0000000..29d814f --- /dev/null +++ b/examples/cms/md/test/user/session/lifecycle.dever.md @@ -0,0 +1,56 @@ +# 会话生命周期测试 + +在隔离数据库中核对会话身份字段，执行撤销和虚拟时钟清理，再验证独立新会话。旧会话拒绝由 HTTP 验收覆盖。 + +- 包：`user.session.lifecycle` +- 公开类型：无 +- 公开方法：无 +- 使用：无 + +## lifecycle + +通过真实 Session App 核对创建、读取和清理后的独立使用。 + +- 函数：`lifecycle` +- 输入：无 +- 输出：无 + +```dever +lifecycle() () { + account = user.account.bootstrap( + "sessions", + "Sessions", + "session@example.com", + "Session user", + secret("correct horse battery staple") + ) + session = app.open( + "session-subject", + account.user_id, + account.tenant_id, + "sessions", + "admin" + ) + authenticated = app.authenticate(session.uuid, "admin") + assert_eq(authenticated.subject, "session-subject") + assert_eq(authenticated.user_id, account.user_id) + assert_eq(authenticated.tenant_id, account.tenant_id) + assert_eq(authenticated.tenant_key, "sessions") + app.revoke(session.uuid) + dever.test.advance_clock(3600000) + app.cleanup() + replacement = app.open( + "new-subject", + account.user_id, + account.tenant_id, + "sessions", + "front" + ) + assert(replacement.uuid != session.uuid) + current = app.authenticate(replacement.uuid, "front") + assert_eq(current.subject, "new-subject") + assert_eq(current.user_id, account.user_id) + assert_eq(current.tenant_id, account.tenant_id) + assert_eq(current.tenant_key, "sessions") +} +``` diff --git a/examples/old/dever/async/config/setting.json b/examples/old/dever/async/config/setting.json new file mode 100644 index 0000000..d3a67b5 --- /dev/null +++ b/examples/old/dever/async/config/setting.json @@ -0,0 +1,10 @@ +{ + "database": { + "default": { + "type": "sqlite", + "path": "data/db/app.db", + "max_connections": 1, + "max_page_size": 100 + } + } +} diff --git a/examples/old/dever/async/data/.gitkeep b/examples/old/dever/async/data/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/examples/old/dever/async/data/.gitkeep @@ -0,0 +1 @@ + diff --git a/examples/old/dever/async/module/main.dever b/examples/old/dever/async/module/main.dever new file mode 100644 index 0000000..851067b --- /dev/null +++ b/examples/old/dever/async/module/main.dever @@ -0,0 +1,28 @@ +# Task、Group 和 Channel 都有明确所有者；示例不启动服务或访问网络。 + +pause() () { + dever.time.sleep(1) +} + +square(value: Int) (result: Int) pure { + result = value * value +} + +send_after(channel: Channel<Int>, value: Int) () { + dever.task.sleep(1) + send(channel, value) +} + +main() (task_value: Int?, grouped_value: Int?, computed: Int) { + messages = channel(Int, 1) + sender = run(send_after(messages, 7)) + task_value = receive(messages) + wait(sender) + workers = group(1) + run(workers, send_after(messages, 21)) + wait(workers) + grouped_value = receive(messages) + computed = parallel(square(12)) + blocking(pause()) + close(messages) +} diff --git a/examples/old/dever/async_tcp/config/setting.json b/examples/old/dever/async_tcp/config/setting.json new file mode 100644 index 0000000..d3a67b5 --- /dev/null +++ b/examples/old/dever/async_tcp/config/setting.json @@ -0,0 +1,10 @@ +{ + "database": { + "default": { + "type": "sqlite", + "path": "data/db/app.db", + "max_connections": 1, + "max_page_size": 100 + } + } +} diff --git a/examples/old/dever/async_tcp/data/.gitkeep b/examples/old/dever/async_tcp/data/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/examples/old/dever/async_tcp/data/.gitkeep @@ -0,0 +1 @@ + diff --git a/examples/old/dever/async_tcp/module/main.dever b/examples/old/dever/async_tcp/module/main.dever new file mode 100644 index 0000000..85eef2d --- /dev/null +++ b/examples/old/dever/async_tcp/module/main.dever @@ -0,0 +1,24 @@ + +connection(event: dever.net.ConnectResult.Connected(socket)) () recover("监听错误是流的最后一项，记录后结束服务") { + dever.net.timeout(socket, 5000) + dever.net.write(socket, dever.bytes.from_text("Hello from Dever\n")) + dever.net.close(socket) +} + +connection(event: dever.net.ConnectResult.Failed(message)) () recover("监听错误是流的最后一项，记录后结束服务") { + blocking(dever.io.println(message)) +} + +serve() () { + listener = dever.net.listen("127.0.0.1", 0) + # 随机分配本机端口；最多同时处理 32 条连接。 + port = dever.net.port(listener) + blocking(dever.io.println(int.to_text(port))) + stream = dever.net.connections(listener) + parallel_each(connection, stream, 32) + dever.net.close_listener(listener) +} + +main() () { + serve() +} diff --git a/examples/old/dever/backend/config/setting.json b/examples/old/dever/backend/config/setting.json new file mode 100644 index 0000000..d3a67b5 --- /dev/null +++ b/examples/old/dever/backend/config/setting.json @@ -0,0 +1,10 @@ +{ + "database": { + "default": { + "type": "sqlite", + "path": "data/db/app.db", + "max_connections": 1, + "max_page_size": 100 + } + } +} diff --git a/examples/old/dever/backend/data/.gitkeep b/examples/old/dever/backend/data/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/examples/old/dever/backend/data/.gitkeep @@ -0,0 +1 @@ + diff --git a/examples/old/dever/backend/module/main.dever b/examples/old/dever/backend/module/main.dever new file mode 100644 index 0000000..9b7418f --- /dev/null +++ b/examples/old/dever/backend/module/main.dever @@ -0,0 +1,69 @@ +# 自包含后端示例：随机回环端口，只请求一次，然后排空并关闭。 + +type Context { + prefix: Text + requests: Channel<Int> +} + +route(request: dever.http.Request, context: Context) ( + response: dever.http.Response +) { + send(context.requests, 1) + response = dever.http.Response { + status = 200 + headers = [dever.http.header("Content-Type", "text/plain; charset=utf-8")] + body = dever.bytes.from_text(context.prefix + request.target) + } +} + +serve(listener: dever.system.Listener, context: Context) () { + dever.system.http_serve(route, listener, dever.http.default_limits(), context) +} + +expired() () { + blocking(dever.io.println("服务排空超时，剩余任务已停止")) +} + +count(value: Int) (text: Text) { + text = "requests = " + int.to_text(value) +} + +count(value: null) (text: Text) { + text = "request channel ended" +} + +main() () { + listener = dever.net.listen("127.0.0.1", 0) + port = dever.net.port(listener) + requests = channel(Int, 1) + server = run( + serve( + listener, + Context { + prefix = "Hello " + requests = requests + } + ) + ) + client = dever.http.client( + "http://127.0.0.1:" + int.to_text(port), + dever.tls.system(), + dever.http.default_limits(), + dever.http.default_pool_limits() + ) + response = dever.http.request( + client, + dever.http.Request { + method = "GET" + target = "/Dever" + headers = [] + body = dever.bytes.from_text("") + } + ) + blocking(dever.io.println(dever.bytes.to_text(response.body))) + dever.net.close_listener(listener) + timeout(server, 2000, expired) + dever.http.close_client(client) + close(requests) + blocking(dever.io.println(count(receive(requests)))) +} diff --git a/examples/old/dever/contracts/config/setting.json b/examples/old/dever/contracts/config/setting.json new file mode 100644 index 0000000..d3a67b5 --- /dev/null +++ b/examples/old/dever/contracts/config/setting.json @@ -0,0 +1,10 @@ +{ + "database": { + "default": { + "type": "sqlite", + "path": "data/db/app.db", + "max_connections": 1, + "max_page_size": 100 + } + } +} diff --git a/examples/old/dever/contracts/data/.gitkeep b/examples/old/dever/contracts/data/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/examples/old/dever/contracts/data/.gitkeep @@ -0,0 +1 @@ + diff --git a/examples/old/dever/contracts/dever.api b/examples/old/dever/contracts/dever.api new file mode 100644 index 0000000..a09091e --- /dev/null +++ b/examples/old/dever/contracts/dever.api @@ -0,0 +1,4 @@ +dever-api 4 +function domain.port(Int) (port: domain.Port []) pure=true recover=None effects=[] errors=[domain.PortResult.Failed] recoveries={} +type domain.Port record construct=public {value: Int [1..65535]} +type domain.PortResult Ready(port: domain.Port []) | error Failed(message: Text []) diff --git a/examples/old/dever/contracts/module/domain.dever b/examples/old/dever/contracts/module/domain.dever new file mode 100644 index 0000000..3be27a1 --- /dev/null +++ b/examples/old/dever/contracts/module/domain.dever @@ -0,0 +1,19 @@ + +public type Port { + value: Int >= 1 and <= 65535 +} + +public type PortResult { + Ready(port: Port) + error Failed(message: Text) +} + +public port(value: Int >= 1 and <= 65535) (port: Port) pure { + port = Port { + value = value + } +} + +public port(value: other) (port: Port) pure { + fail(PortResult.Failed("port must be between 1 and 65535")) +} diff --git a/examples/old/dever/contracts/module/main.dever b/examples/old/dever/contracts/module/main.dever new file mode 100644 index 0000000..dde78cc --- /dev/null +++ b/examples/old/dever/contracts/module/main.dever @@ -0,0 +1,18 @@ + +main() (configured: domain.Port, invalid: domain.PortResult, fallback: domain.Port) pure { + configured = domain.Port { + value = 8080 + } + invalid = result(domain.port(0)) + fallback = configured_or_default(result(domain.port(-1))) +} + +configured_or_default(result: domain.PortResult.Ready(port)) (value: domain.Port) pure recover("optional local configuration uses port 8080") { + value = port +} + +configured_or_default(result: domain.PortResult.Failed(message)) (value: domain.Port) pure recover("optional local configuration uses port 8080") { + value = domain.Port { + value = 8080 + } +} diff --git a/examples/old/dever/core/config/setting.json b/examples/old/dever/core/config/setting.json new file mode 100644 index 0000000..d3a67b5 --- /dev/null +++ b/examples/old/dever/core/config/setting.json @@ -0,0 +1,10 @@ +{ + "database": { + "default": { + "type": "sqlite", + "path": "data/db/app.db", + "max_connections": 1, + "max_page_size": 100 + } + } +} diff --git a/examples/old/dever/core/data/.gitkeep b/examples/old/dever/core/data/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/examples/old/dever/core/data/.gitkeep @@ -0,0 +1 @@ + diff --git a/examples/old/dever/core/module/main.dever b/examples/old/dever/core/module/main.dever new file mode 100644 index 0000000..3dcd188 --- /dev/null +++ b/examples/old/dever/core/module/main.dever @@ -0,0 +1,134 @@ + +type Pair { + left: Int + right: Int +} + +type Number { + Present(value: Int) + Missing +} + +type Key { + First + Second +} + +number(value: Number.Present(payload)) (answer: Int) { + answer = payload +} + +number(value: other) (answer: Int) { + answer = 0 +} + +grade(value: Int < 0) (answer: Text) { + answer = "negative" +} + +grade(value: Int >= 0 and < 10) (answer: Text) { + answer = "small" +} + +grade(value: Int >= 10) (answer: Text) { + answer = "large" +} + +scale(value: Int, factor: Int) (answer: Int) { + answer = value * factor +} + +positive(value: Int) (answer: Bool) { + answer = value > 0 +} + +target(value: Int, expected: Int) (answer: Bool) { + answer = value == expected +} + +parts(value: Text) (first: Text, last: Text) { + first = text.trim(value) + last = text.lower(first) +} + +change(pair: Pair) (answer: Pair) { + pair.left = 99 + answer = pair +} + +map_key(entry: MapEntry<Text, Int>) (key: Text) { + key = entry.key +} + +optional(value: Int?) (answer: Int?) { + answer = value +} + +main() ( + original: Int, + changed: Int, + choice: Int, + category: Text, + exact: Bool, + quotient: Decimal, + third: Decimal, + rounded: Decimal, + integer_division: Int, + remainder: Int, + selected: List<Int>, + found: Int?, + total: Int, + empty_total: Decimal, + names: Text, + ordered_keys: List<Text>, + maps_equal: Bool, + nan_equal: Bool, + finite: Bool, + optionals: List<Int?>, + key_value: Int?, + identity: Bool +) { + pair = Pair { + right = 2 + left = 1 + } + saved = pair + pair = change(pair) + original = saved.left + changed = pair.left + choice = number(Number.Present(7)) + number(Number.Missing) + category = grade(12) + exact = 0.1 + 0.2 == 0.3 + quotient = 5 / 2 + third = 1 / 3 + rounded = decimal.round(2.345, 2) + integer_division = -5 // 2 + remainder = -5 % 2 + values = each(scale, [-1, 0, 1, 2], 3) + selected = filter(positive, values) + found = find(target, values, 3) + total = sum(scale, selected, 2) + empty_total = sum([]) + split = parts(" ABC ") + names = split.first + split.last + labels = { + "first" = 1 + "second" = 2 + } + original_labels = labels + labels = put(labels, "first", 3) + labels = remove(labels, "first") + labels = put(labels, "first", 1) + ordered_keys = each(map_key, entries(labels)) + maps_equal = labels == original_labels + nan = 0.0e0 / 0.0e0 + nan_equal = [nan] == [nan] + finite = math.is_finite(float.from_int(4)) + optionals = each(optional, [null, 1, null]) + keys = { + Key.First = 1 + Key.Second = 2 + } + key_value = get(keys, Key.Second) + identity = dever.id.from_text("stable") == dever.id.from_text("stable") +} diff --git a/examples/old/dever/hello/config/setting.json b/examples/old/dever/hello/config/setting.json new file mode 100644 index 0000000..d3a67b5 --- /dev/null +++ b/examples/old/dever/hello/config/setting.json @@ -0,0 +1,10 @@ +{ + "database": { + "default": { + "type": "sqlite", + "path": "data/db/app.db", + "max_connections": 1, + "max_page_size": 100 + } + } +} diff --git a/examples/old/dever/hello/data/.gitkeep b/examples/old/dever/hello/data/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/examples/old/dever/hello/data/.gitkeep @@ -0,0 +1 @@ + diff --git a/examples/old/dever/hello/module/app/user.dever b/examples/old/dever/hello/module/app/user.dever new file mode 100644 index 0000000..629e305 --- /dev/null +++ b/examples/old/dever/hello/module/app/user.dever @@ -0,0 +1,3 @@ +public greet() () { + dever.io.println("hello") +} diff --git a/examples/old/dever/hello/module/main.dever b/examples/old/dever/hello/module/main.dever new file mode 100644 index 0000000..3dd6558 --- /dev/null +++ b/examples/old/dever/hello/module/main.dever @@ -0,0 +1,3 @@ +main() () { + app.user.greet() +} diff --git a/examples/old/dever/http/config/setting.json b/examples/old/dever/http/config/setting.json new file mode 100644 index 0000000..d3a67b5 --- /dev/null +++ b/examples/old/dever/http/config/setting.json @@ -0,0 +1,10 @@ +{ + "database": { + "default": { + "type": "sqlite", + "path": "data/db/app.db", + "max_connections": 1, + "max_page_size": 100 + } + } +} diff --git a/examples/old/dever/http/data/.gitkeep b/examples/old/dever/http/data/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/examples/old/dever/http/data/.gitkeep @@ -0,0 +1 @@ + diff --git a/examples/old/dever/http/module/main.dever b/examples/old/dever/http/module/main.dever new file mode 100644 index 0000000..14ea928 --- /dev/null +++ b/examples/old/dever/http/module/main.dever @@ -0,0 +1,19 @@ + +route(request: dever.http.Request) (response: dever.http.Response) { + response = dever.http.Response { + status = 200 + headers = [dever.http.header("Content-Type", "text/plain; charset=utf-8")] + body = dever.bytes.from_text("Hello Dever: " + request.target) + } +} + +serve() () { + listener = dever.net.listen("127.0.0.1", 0) + port = dever.net.port(listener) + blocking(dever.io.println("HTTP listening on 127.0.0.1:" + int.to_text(port))) + dever.http.serve(route, listener, dever.http.default_limits()) +} + +main() () { + serve() +} diff --git a/examples/old/dever/library/config/setting.json b/examples/old/dever/library/config/setting.json new file mode 100644 index 0000000..d3a67b5 --- /dev/null +++ b/examples/old/dever/library/config/setting.json @@ -0,0 +1,10 @@ +{ + "database": { + "default": { + "type": "sqlite", + "path": "data/db/app.db", + "max_connections": 1, + "max_page_size": 100 + } + } +} diff --git a/examples/old/dever/library/data/.gitkeep b/examples/old/dever/library/data/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/examples/old/dever/library/data/.gitkeep @@ -0,0 +1 @@ + diff --git a/examples/old/dever/library/module/main.dever b/examples/old/dever/library/module/main.dever new file mode 100644 index 0000000..093d924 --- /dev/null +++ b/examples/old/dever/library/module/main.dever @@ -0,0 +1,11 @@ + +main() (json: Text) { + total = reduce(add, [1, 2, 3], 0) + json = dever.json.stringify( + dever.json.parse("{\"message\":\"你好 Dever\",\"total\":" + int.to_text(total) + "}") + ) +} + +add(value: Int, total: Int) (next: Int) { + next = total + value +} diff --git a/examples/old/dever/live_http/config/setting.json b/examples/old/dever/live_http/config/setting.json new file mode 100644 index 0000000..d3a67b5 --- /dev/null +++ b/examples/old/dever/live_http/config/setting.json @@ -0,0 +1,10 @@ +{ + "database": { + "default": { + "type": "sqlite", + "path": "data/db/app.db", + "max_connections": 1, + "max_page_size": 100 + } + } +} diff --git a/examples/old/dever/live_http/data/.gitkeep b/examples/old/dever/live_http/data/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/examples/old/dever/live_http/data/.gitkeep @@ -0,0 +1 @@ + diff --git a/examples/old/dever/live_http/module/main.dever b/examples/old/dever/live_http/module/main.dever new file mode 100644 index 0000000..160a9a0 --- /dev/null +++ b/examples/old/dever/live_http/module/main.dever @@ -0,0 +1,107 @@ + +route(request: dever.http.Request, reply: dever.system.HttpReply) () { + path(request.target, reply) +} + +path(target: "/events", reply: dever.system.HttpReply) () { + dever.sse.start(reply, []) + tick(1, reply) + tick(2, reply) + tick(3, reply) +} + +path(target: "/stream", reply: dever.system.HttpReply) () { + dever.http.start(reply, 200, [dever.http.header("Content-Type", "text/plain")]) + dever.http.write(reply, dever.bytes.from_text("first\n")) + dever.task.sleep(100) + dever.http.write(reply, dever.bytes.from_text("second\n")) + dever.http.finish(reply) +} + +path(target: "/ws", reply: dever.system.HttpReply) () { + upgraded(result(dever.websocket.accept(reply, dever.websocket.default_limits())), reply) +} + +path(target: other, reply: dever.system.HttpReply) () { + dever.http.respond( + reply, + dever.http.Response { + status = 200 + headers = [dever.http.header("Content-Type", "text/plain; charset=utf-8")] + body = dever.bytes.from_text("Dever: /stream, /events, /ws") + } + ) +} + +tick(index: Int, reply: dever.system.HttpReply) () { + dever.task.sleep(100) + dever.sse.send( + reply, + dever.sse.Event { + event = "tick" + data = int.to_text(index) + id = int.to_text(index) + retry_ms = 1000 + } + ) +} + +upgraded( + result: dever.websocket.ConnectResult.Connected(socket), + reply: dever.system.HttpReply +) () recover("WebSocket handshake errors are returned as HTTP 400") { + each(message, dever.websocket.messages(socket), socket) +} + +upgraded( + result: dever.websocket.ConnectResult.Failed(message), + reply: dever.system.HttpReply +) () recover("WebSocket handshake errors are returned as HTTP 400") { + dever.http.respond( + reply, + dever.http.Response { + status = 400 + headers = [] + body = dever.bytes.from_text(message) + } + ) +} + +message( + event: dever.websocket.MessageEvent.Read(value), + socket: dever.system.WebSocket +) () { + echo(value, socket) +} + +message( + event: dever.websocket.MessageEvent.Failed(message), + socket: dever.system.WebSocket +) () { + fail(dever.websocket.MessageEvent.Failed(message)) +} + +echo(value: dever.websocket.Message.Text(text), socket: dever.system.WebSocket) () { + dever.websocket.send(socket, dever.websocket.Message.Text(text)) +} + +echo(value: dever.websocket.Message.Binary(bytes), socket: dever.system.WebSocket) () { + dever.websocket.send(socket, dever.websocket.Message.Binary(bytes)) +} + +# Ping/Pong 已由协议引擎处理，应用只消费控制帧。 +echo(value: other, socket: dever.system.WebSocket) () {} + +main() () { + listener = dever.net.listen("127.0.0.1", 0) + port = dever.net.port(listener) + blocking( + dever.io.println("HTTP / SSE / WebSocket on 127.0.0.1:" + int.to_text(port)) + ) + dever.http.serve_live( + route, + listener, + dever.http.default_limits(), + dever.http.default_live_limits() + ) +} diff --git a/examples/old/dever/parallel/config/setting.json b/examples/old/dever/parallel/config/setting.json new file mode 100644 index 0000000..d3a67b5 --- /dev/null +++ b/examples/old/dever/parallel/config/setting.json @@ -0,0 +1,10 @@ +{ + "database": { + "default": { + "type": "sqlite", + "path": "data/db/app.db", + "max_connections": 1, + "max_page_size": 100 + } + } +} diff --git a/examples/old/dever/parallel/data/.gitkeep b/examples/old/dever/parallel/data/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/examples/old/dever/parallel/data/.gitkeep @@ -0,0 +1 @@ + diff --git a/examples/old/dever/parallel/module/main.dever b/examples/old/dever/parallel/module/main.dever new file mode 100644 index 0000000..d02e49c --- /dev/null +++ b/examples/old/dever/parallel/module/main.dever @@ -0,0 +1,10 @@ +# 最多两个工作线程；输出顺序不保证，parallel_each 返回时任务都已完成。 + +show(name: Text) () { + dever.io.println(name) +} + +main() () { + parallel_each(show, ["alpha", "beta", "gamma"], 2) + dever.io.println("done") +} diff --git a/examples/old/dever/privacy/config/setting.json b/examples/old/dever/privacy/config/setting.json new file mode 100644 index 0000000..d3a67b5 --- /dev/null +++ b/examples/old/dever/privacy/config/setting.json @@ -0,0 +1,10 @@ +{ + "database": { + "default": { + "type": "sqlite", + "path": "data/db/app.db", + "max_connections": 1, + "max_page_size": 100 + } + } +} diff --git a/examples/old/dever/privacy/data/.gitkeep b/examples/old/dever/privacy/data/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/examples/old/dever/privacy/data/.gitkeep @@ -0,0 +1 @@ + diff --git a/examples/old/dever/privacy/module/counter.dever b/examples/old/dever/privacy/module/counter.dever new file mode 100644 index 0000000..45f340e --- /dev/null +++ b/examples/old/dever/privacy/module/counter.dever @@ -0,0 +1,20 @@ +# private 隐藏表示，调用方通过操作使用 Counter；普通数据无需强行套这层封装。 + +public type Counter { + private value: Int +} + +public start() (counter: Counter) pure { + counter = Counter { + value = 0 + } +} + +public increment(counter: Counter) (next: Counter) pure { + next = counter + next.value = counter.value + 1 +} + +public read(counter: Counter) (value: Int) pure { + value = counter.value +} diff --git a/examples/old/dever/privacy/module/main.dever b/examples/old/dever/privacy/module/main.dever new file mode 100644 index 0000000..b07ad04 --- /dev/null +++ b/examples/old/dever/privacy/module/main.dever @@ -0,0 +1,7 @@ + +main() (before: Int, after: Int) pure { + original = counter.start() + changed = counter.increment(original) + before = counter.read(original) + after = counter.read(changed) +} diff --git a/examples/old/dever/rejected/coverage/module/main.dever b/examples/old/dever/rejected/coverage/module/main.dever new file mode 100644 index 0000000..13118a1 --- /dev/null +++ b/examples/old/dever/rejected/coverage/module/main.dever @@ -0,0 +1,9 @@ +# 预期 C009：缺少 false 分句。补齐后才能覆盖全部 Bool 输入。 + +label(enabled: true) (text: Text) { + text = "enabled" +} + +main() (text: Text) { + text = label(true) +} diff --git a/examples/old/dever/rejected/failure/module/main.dever b/examples/old/dever/rejected/failure/module/main.dever new file mode 100644 index 0000000..d0ab4b7 --- /dev/null +++ b/examples/old/dever/rejected/failure/module/main.dever @@ -0,0 +1,9 @@ +# 预期 C012：赋值后丢弃失败。把失败作为命名输出返回可通过。 + +type Result { + error Failed(message: Text) +} + +main(value: Result) () { + discarded = value +} diff --git a/examples/old/dever/rejected/private/module/main.dever b/examples/old/dever/rejected/private/module/main.dever new file mode 100644 index 0000000..0f56026 --- /dev/null +++ b/examples/old/dever/rejected/private/module/main.dever @@ -0,0 +1,5 @@ +# 预期 C006：另一个 package 不能读取私有字段。 + +public read(box: secret.Box) (value: Int) { + value = box.value +} diff --git a/examples/old/dever/rejected/private/module/secret.dever b/examples/old/dever/rejected/private/module/secret.dever new file mode 100644 index 0000000..5ab127f --- /dev/null +++ b/examples/old/dever/rejected/private/module/secret.dever @@ -0,0 +1,4 @@ + +public type Box { + private value: Int +} diff --git a/examples/old/dever/rejected/pure/module/main.dever b/examples/old/dever/rejected/pure/module/main.dever new file mode 100644 index 0000000..712766d --- /dev/null +++ b/examples/old/dever/rejected/pure/module/main.dever @@ -0,0 +1,5 @@ +# 预期 C013：输出属于副作用。需要输出时应移除 pure。 + +main() () pure { + dever.io.println("hello") +} diff --git a/examples/old/dever/rejected/range/module/main.dever b/examples/old/dever/rejected/range/module/main.dever new file mode 100644 index 0000000..0c2332f --- /dev/null +++ b/examples/old/dever/rejected/range/module/main.dever @@ -0,0 +1,11 @@ +# 预期 C011：0 不能构造合法端口。改为 8080 可通过。 + +type Port { + value: Int >= 1 and <= 65535 +} + +main() (port: Port) { + port = Port { + value = 0 + } +} diff --git a/examples/old/dever/sequences/config/setting.json b/examples/old/dever/sequences/config/setting.json new file mode 100644 index 0000000..d3a67b5 --- /dev/null +++ b/examples/old/dever/sequences/config/setting.json @@ -0,0 +1,10 @@ +{ + "database": { + "default": { + "type": "sqlite", + "path": "data/db/app.db", + "max_connections": 1, + "max_page_size": 100 + } + } +} diff --git a/examples/old/dever/sequences/data/.gitkeep b/examples/old/dever/sequences/data/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/examples/old/dever/sequences/data/.gitkeep @@ -0,0 +1 @@ + diff --git a/examples/old/dever/sequences/module/main.dever b/examples/old/dever/sequences/module/main.dever new file mode 100644 index 0000000..929baea --- /dev/null +++ b/examples/old/dever/sequences/module/main.dever @@ -0,0 +1,58 @@ +# handler 是具名函数；reduce_until 处理当前元素后停止，不处理后面的 100。 + +type Summary { + total: Int + count: Int +} + +double(value: Int) (result: Int) pure { + result = value * 2 +} + +map_values(transform: handler(value: Int) (result: Int), values: List<Int>) ( + results: List<Int> +) { + results = each(transform, values) +} + +add(value: Int, total: Int) (next: Int) pure { + next = total + value +} + +collect(value: Int, summary: Summary) (next: Summary, stop: Bool) pure { + next = Summary { + total = summary.total + value + count = summary.count + 1 + } + stop = next.total >= 6 +} + +main() ( + mapped: List<Int>, + total: Int, + stopped: Summary, + empty_total: Int, + bytes_total: Int, + first_value: Int?, + missing: Int? +) pure { + mapped = map_values(double, [1, 2, 3]) + total = reduce(add, [1, 2, 3], 0) + stopped = reduce_until( + collect, + [1, 2, 3, 100], + Summary { + total = 0 + count = 0 + } + ) + empty_total = reduce(add, [], 0) + bytes_total = reduce(add, dever.bytes.from_text("AB"), 0) + first_value = first(mapped) + missing = get( + { + "count" = 3 + }, + "unknown" + ) +} diff --git a/examples/old/dever/stream/config/setting.json b/examples/old/dever/stream/config/setting.json new file mode 100644 index 0000000..d3a67b5 --- /dev/null +++ b/examples/old/dever/stream/config/setting.json @@ -0,0 +1,10 @@ +{ + "database": { + "default": { + "type": "sqlite", + "path": "data/db/app.db", + "max_connections": 1, + "max_page_size": 100 + } + } +} diff --git a/examples/old/dever/stream/data/.gitkeep b/examples/old/dever/stream/data/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/examples/old/dever/stream/data/.gitkeep @@ -0,0 +1 @@ + diff --git a/examples/old/dever/stream/input.txt b/examples/old/dever/stream/input.txt new file mode 100644 index 0000000..5da849b --- /dev/null +++ b/examples/old/dever/stream/input.txt @@ -0,0 +1 @@ +ABC diff --git a/examples/old/dever/stream/module/main.dever b/examples/old/dever/stream/module/main.dever new file mode 100644 index 0000000..5e192b6 --- /dev/null +++ b/examples/old/dever/stream/module/main.dever @@ -0,0 +1,21 @@ + +sum_byte(value: Int, total: Int) (next: Int) { + next = total + value +} + +consume(event: dever.io.ReadEvent.Chunk(bytes), total: Int) (next: Int) { + next = total + reduce(sum_byte, bytes, 0) +} + +consume(event: dever.io.ReadEvent.Failed(message), total: Int) ( + next: Int +) { + fail(dever.io.ReadEvent.Failed(message)) +} + +main() (total: Int) { + file = dever.io.open("examples/old/dever/stream/input.txt") + stream = dever.io.chunks(file, 2) + total = reduce(consume, stream, 0) + close(stream) +} diff --git a/examples/old/dever/system/config/setting.json b/examples/old/dever/system/config/setting.json new file mode 100644 index 0000000..d3a67b5 --- /dev/null +++ b/examples/old/dever/system/config/setting.json @@ -0,0 +1,10 @@ +{ + "database": { + "default": { + "type": "sqlite", + "path": "data/db/app.db", + "max_connections": 1, + "max_page_size": 100 + } + } +} diff --git a/examples/old/dever/system/data/.gitkeep b/examples/old/dever/system/data/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/examples/old/dever/system/data/.gitkeep @@ -0,0 +1 @@ + diff --git a/examples/old/dever/system/module/main.dever b/examples/old/dever/system/module/main.dever new file mode 100644 index 0000000..cfd0aa3 --- /dev/null +++ b/examples/old/dever/system/module/main.dever @@ -0,0 +1,11 @@ +# 结果随启动参数及当前时间变化；失败以结果返回。 + +main() ( + arguments: List<Text>, + clock: Int, + invalid_sleep: dever.time.SleepResult +) { + arguments = dever.process.arguments() + clock = dever.time.unix_millis() + invalid_sleep = result(dever.time.sleep(-1)) +} diff --git a/examples/old/dever/text/config/setting.json b/examples/old/dever/text/config/setting.json new file mode 100644 index 0000000..d3a67b5 --- /dev/null +++ b/examples/old/dever/text/config/setting.json @@ -0,0 +1,10 @@ +{ + "database": { + "default": { + "type": "sqlite", + "path": "data/db/app.db", + "max_connections": 1, + "max_page_size": 100 + } + } +} diff --git a/examples/old/dever/text/data/.gitkeep b/examples/old/dever/text/data/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/examples/old/dever/text/data/.gitkeep @@ -0,0 +1 @@ + diff --git a/examples/old/dever/text/module/main.dever b/examples/old/dever/text/module/main.dever new file mode 100644 index 0000000..aabfc3c --- /dev/null +++ b/examples/old/dever/text/module/main.dever @@ -0,0 +1,29 @@ +# 文本下标按 Unicode 标量；解析失败和越界保留 null，不自动改成零或空字符串。 + +main() ( + cleaned: Text, + character: Text?, + slice: Text?, + pieces: List<Text>, + joined: Text, + parsed: Int?, + invalid: Int?, + exact: Decimal?, + rounded: Decimal, + utf8: Text, + invalid_utf8: dever.bytes.DecodeResult, + invalid_byte: dever.bytes.BufferResult +) pure { + cleaned = text.lower(text.trim(" Dever ")) + character = text.at("a中😀", 2) + slice = text.slice("a中😀", 1, 3) + pieces = text.split("a,,b", ",") + joined = text.join(pieces, "|") + parsed = int.parse("42") + invalid = int.parse("forty-two") + exact = decimal.parse("1.25") + rounded = decimal.round(2.345, 2) + utf8 = dever.bytes.to_text(dever.bytes.from_text("你好")) + invalid_utf8 = result(dever.bytes.to_text(dever.bytes.from_ints([255]))) + invalid_byte = result(dever.bytes.from_ints([256])) +} diff --git a/examples/old/markdown/config/setting.json b/examples/old/markdown/config/setting.json new file mode 100644 index 0000000..d3a67b5 --- /dev/null +++ b/examples/old/markdown/config/setting.json @@ -0,0 +1,10 @@ +{ + "database": { + "default": { + "type": "sqlite", + "path": "data/db/app.db", + "max_connections": 1, + "max_page_size": 100 + } + } +} diff --git a/examples/old/markdown/data/.gitkeep b/examples/old/markdown/data/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/examples/old/markdown/data/.gitkeep @@ -0,0 +1 @@ + diff --git a/examples/old/markdown/module/main.dever.md b/examples/old/markdown/module/main.dever.md new file mode 100644 index 0000000..e7cc5bf --- /dev/null +++ b/examples/old/markdown/module/main.dever.md @@ -0,0 +1,150 @@ +# 购物车结算 + +计算购物车中所有商品的合计金额、优惠金额和最终应付金额。 + +- 包：`main` +- 公开类型： + - `Product` + - `OrderSummary` +- 公开方法：无 +- 使用：无 + +## 商品 + +保存商品名称、单价和购买数量。 + +- 类型：`Product` +- 字段： + - `name: Text`：商品名称 + - `price: Decimal`：商品单价 + - `quantity: Int >= 1`：购买数量，最少为 1 + +```typescript dever +public type Product { + name: Text + price: Decimal + quantity: Int >= 1 +} +``` + +## 订单结算结果 + +保存参与结算的商品以及各项金额。 + +- 类型：`OrderSummary` +- 字段： + - `items: List<Product>`：参与结算的商品 + - `subtotal: Decimal`：优惠前的商品合计 + - `discount: Decimal`：优惠金额 + - `payable: Decimal`：最终应付金额 + +```typescript dever +public type OrderSummary { + items: List<Product> + subtotal: Decimal + discount: Decimal + payable: Decimal +} +``` + +## 计算单项商品金额 + +使用商品单价乘以购买数量。 + +- 函数：`line_total` +- 输入： + - `product: Product`：需要计算金额的商品 +- 输出： + - `total: Decimal`：该商品的总金额 + +```typescript dever +line_total(product: Product) (total: Decimal) { + total = product.price * product.quantity +} +``` + +## 汇总商品金额 + +计算商品列表中所有商品的金额总和。 + +- 函数：`sum_products` +- 输入： + - `products: List<Product>`：需要汇总的商品列表 +- 输出： + - `subtotal: Decimal`：所有商品的合计金额 + +```typescript dever +sum_products(products: List<Product>) (subtotal: Decimal) { + subtotal = sum(line_total, products) +} +``` + +## 结算订单 + +根据商品列表和会员应付比例计算订单结果。 + +- 函数：`checkout` +- 输入： + - `products: List<Product>`：需要结算的商品列表 + - `member_rate: Decimal`：会员实际应付比例，例如 0.90 +- 输出： + - `summary: OrderSummary`：订单结算结果 + +```typescript dever +checkout(products: List<Product>, member_rate: Decimal) (summary: OrderSummary) { + subtotal = sum_products(products) + payable = decimal.round(subtotal * member_rate, 2) + discount = subtotal - payable + summary = OrderSummary { + items = products + subtotal = subtotal + discount = discount + payable = payable + } +} +``` + +## 运行购物车示例 + +创建商品列表，并以九折会员价格完成结算。 + +- 函数：`main` +- 输入：无 +- 输出： + - `customer: Text`：客户名称 + - `summary: OrderSummary`：客户的订单结算结果 + +```typescript dever +main() (customer: Text, summary: OrderSummary) { + customer = "小明" + products = [ + Product { + name = "笔记本" + price = 12.50 + quantity = 3 + }, + Product { + name = "签字笔" + price = 5.00 + quantity = 2 + } + ] + summary = checkout(products, 0.90) +} +``` + +### 执行 + +```bash +deverc check examples/old/markdown --entry app.main +deverc fmt examples/old/markdown --check +deverc run examples/old/markdown app.main +``` + +结算结果： + +| 项目 | 金额 | +| --- | ---: | +| 商品合计 | 47.50 | +| 优惠金额 | 4.75 | +| 应付金额 | 42.75 | diff --git a/library/decimal.dever b/library/decimal.dever new file mode 100644 index 0000000..237f0c3 --- /dev/null +++ b/library/decimal.dever @@ -0,0 +1,15 @@ +public round(value: Decimal, places: Int) (result: Decimal) { + result = dever.system.decimal_round(value, places) +} + +public parse(value: Text) (result: Decimal?) { + result = dever.system.decimal_parse(value) +} + +public to_text(value: Decimal) (result: Text) { + result = dever.system.decimal_to_text(value) +} + +public from_int(value: Int) (result: Decimal) { + result = dever.system.decimal_from_int(value) +} diff --git a/library/dever/api.dever b/library/dever/api.dever new file mode 100644 index 0000000..e556b8f --- /dev/null +++ b/library/dever/api.dever @@ -0,0 +1,84 @@ +public serve() () { + dever.system.api_serve() +} + +# Upload is a compiler-owned affine POST input. upload_filename, upload_content_type +# and upload_size borrow its metadata; close_upload consumes its temporary file. +public close_upload(upload: Upload) () { + dever.system.upload_close(upload) +} + +public type Error { + error Invalid + error Unauthorized + error Forbidden + error NotFound + error Conflict + error TooManyRequests +} + +public type SameSite { + Strict + Lax + None +} + +public type CookieOptions { + path: Text + domain: Text? + same_site: SameSite + secure: Bool + http_only: Bool + max_age: Int? +} + +public secure_cookie_options() (options: CookieOptions) { + options = CookieOptions { + path = "/" + domain = null + same_site = SameSite.Lax + secure = true + http_only = true + max_age = null + } +} + +public request_id() (id: Text) { + id = dever.system.api_request_id() +} + +public method() (value: Text) { + value = dever.system.api_method() +} + +public path() (value: Text) { + value = dever.system.api_path() +} + +public client_address() (value: Text?) { + value = dever.system.api_client_address() +} + +public header(name: Text) (value: Text?) { + value = dever.system.api_header(name) +} + +public cookie(name: Text) (value: Text?) { + value = dever.system.api_cookie(name) +} + +public secret_cookie(name: Text) (value: Secret?) { + value = dever.system.api_secret_cookie(name) +} + +public set_header(name: Text, value: Text) () { + dever.system.api_response_header(name, value) +} + +public set_cookie(name: Text, value: Text, options: CookieOptions) () { + dever.system.api_response_cookie(name, value, options) +} + +public set_secret_cookie(name: Text, value: Secret, options: CookieOptions) () { + dever.system.api_response_secret_cookie(name, value, options) +} diff --git a/library/dever/auth.dever b/library/dever/auth.dever new file mode 100644 index 0000000..ef813aa --- /dev/null +++ b/library/dever/auth.dever @@ -0,0 +1,68 @@ +public type Claims { + subject: Text + session: Text + tenant: Text? + site: Text +} + +public type Permission { + key: Text + component: Text + domain: Text + site: Text + action: Text + method: Text +} + +public issue(subject: Text, session: Text, tenant: Text?) (token: Secret) { + token = dever.system.auth_issue(subject, session, tenant) +} + +public issue_cookie(subject: Text, session: Text, tenant: Text?) () { + dever.system.auth_issue_cookie(subject, session, tenant) +} + +public clear_cookie() () { + dever.system.auth_clear_cookie() +} + +public id() (value: Text) { + value = dever.system.auth_id() +} + +public session() (value: Text) { + value = dever.system.auth_session() +} + +public user_id() (value: Int?) { + value = dever.system.auth_user_id() +} + +public tenant_id() (value: Int?) { + value = dever.system.auth_tenant_id() +} + +public permissions() (values: List<Permission>) { + values = dever.system.auth_permissions() +} + +public save_role( + id: Text, + name: Text, + all_permissions: Bool, + permission_keys: List<Text> +) () { + dever.system.auth_save_role(id, name, all_permissions, permission_keys) +} + +public grant_role(user_id: Int, role_id: Text) () { + dever.system.auth_grant_role(user_id, role_id) +} + +public revoke_role(user_id: Int, role_id: Text) () { + dever.system.auth_revoke_role(user_id, role_id) +} + +public disable_role(role_id: Text) () { + dever.system.auth_disable_role(role_id) +} diff --git a/library/dever/bytes.dever b/library/dever/bytes.dever new file mode 100644 index 0000000..d2a07b5 --- /dev/null +++ b/library/dever/bytes.dever @@ -0,0 +1,53 @@ +public type DecodeResult { + Decoded(text: Text) + error Failed(message: Text) +} + +public type BufferResult { + Ready(bytes: Bytes) + error Failed(message: Text) +} + +public from_text(text: Text) (bytes: Bytes) { + bytes = dever.system.bytes_from_text(text) +} + +public to_text(bytes: Bytes) (text: Text) { + text = decoded(dever.system.bytes_to_text(bytes)) +} + +decoded(result: DecodeResult.Decoded(value)) (text: Text) { + text = value +} + +decoded(result: DecodeResult.Failed(message)) (text: Text) { + fail(DecodeResult.Failed(message)) +} + +public from_ints(values: List<Int>) (bytes: Bytes) { + bytes = ready(dever.system.bytes_from_ints(values)) +} + +public length(bytes: Bytes) (length: Int) { + length = dever.system.bytes_length(bytes) +} + +public at(bytes: Bytes, index: Int) (value: Int?) { + value = dever.system.bytes_at(bytes, index) +} + +public slice(bytes: Bytes, start: Int, end: Int) (result: Bytes) { + result = ready(dever.system.bytes_slice(bytes, start, end)) +} + +ready(result: BufferResult.Ready(bytes)) (value: Bytes) { + value = bytes +} + +ready(result: BufferResult.Failed(message)) (value: Bytes) { + fail(BufferResult.Failed(message)) +} + +public concat(left: Bytes, right: Bytes) (bytes: Bytes) { + bytes = dever.system.bytes_concat(left, right) +} diff --git a/library/dever/crypto.dever b/library/dever/crypto.dever new file mode 100644 index 0000000..c3381c8 --- /dev/null +++ b/library/dever/crypto.dever @@ -0,0 +1,62 @@ +public type TokenResult { + Ready(value: Secret) + error Failed(message: Text) +} + +public type HashResult { + Ready(value: Text) + error Failed(message: Text) +} + +public type VerifyResult { + Ready(value: Bool) + error Failed(message: Text) +} + +public token(bytes: Int) (value: Secret) { + value = token_value(dever.system.crypto_token(bytes)) +} + +token_value(result: TokenResult.Ready(value)) (secret: Secret) { + secret = value +} + +token_value(result: TokenResult.Failed(message)) (secret: Secret) { + fail(TokenResult.Failed(message)) +} + +public password_hash(password: Secret) (value: Text) { + value = hash_value(dever.system.crypto_password_hash(password)) +} + +hash_value(result: HashResult.Ready(value)) (hash: Text) { + hash = value +} + +hash_value(result: HashResult.Failed(message)) (hash: Text) { + fail(HashResult.Failed(message)) +} + +public password_verify(password: Secret, hash: Text) (value: Bool) { + value = verify_value(dever.system.crypto_password_verify(password, hash)) +} + +verify_value(result: VerifyResult.Ready(value)) (valid: Bool) { + valid = value +} + +verify_value(result: VerifyResult.Failed(message)) (valid: Bool) { + fail(VerifyResult.Failed(message)) +} + +public sha256(bytes: Bytes) (value: Bytes) pure { + value = dever.system.crypto_sha256(bytes) +} + +public hmac_sha256(key: Secret, bytes: Bytes) (value: Bytes) pure { + value = dever.system.crypto_hmac_sha256(key, bytes) +} + +public constant_time_eq(left: Bytes, right: Bytes) (equal: Bool) pure { + equal = dever.system.crypto_constant_time_eq(left, right) +} diff --git a/library/dever/database.dever b/library/dever/database.dever new file mode 100644 index 0000000..565488d --- /dev/null +++ b/library/dever/database.dever @@ -0,0 +1,13 @@ +# Database failures retain their category across services and transactions. +public type Error { + error Pool(message: Text) + error PoolExhausted(message: Text) + error Connection(message: Text) + error Timeout(message: Text) + error Cancelled(message: Text) + error Database(message: Text) + error Constraint(message: Text) + error NotFound(message: Text) + error InvalidData(message: Text) + error Migration(message: Text) +} diff --git a/library/dever/http.dever b/library/dever/http.dever new file mode 100644 index 0000000..36fa1ac --- /dev/null +++ b/library/dever/http.dever @@ -0,0 +1,262 @@ +public type PoolLimits { + idle_ms: Int + chunk_bytes: Int + read_ms: Int +} + +public type StreamRequest { + method: Text + target: Text + headers: List<Header> + body: AsyncStream<dever.io.ReadEvent> +} + +public type StreamResponse { + status: Int + headers: List<Header> + body: AsyncStream<dever.io.ReadEvent> +} + +public type ClientResult { + Ready(client: dever.system.HttpClient) + error Failed(message: Text) +} + +public type StreamResponseResult { + Read(response: StreamResponse) + error Failed(message: Text) +} + +public default_pool_limits() (result: PoolLimits) { + result = PoolLimits { + idle_ms = 10000 + chunk_bytes = 65536 + read_ms = 60000 + } +} + +public client( + origin: Text, + tls: dever.system.ClientTls, + limits: Limits, + pool: PoolLimits +) (client: dever.system.HttpClient) { + client = ready_client(dever.system.http_client(origin, tls, limits, pool)) +} + +ready_client(result: ClientResult.Ready(client)) (value: dever.system.HttpClient) { + value = client +} + +ready_client(result: ClientResult.Failed(message)) (value: dever.system.HttpClient) { + fail(ClientResult.Failed(message)) +} + +public request(client: dever.system.HttpClient, request: Request) ( + response: Response +) { + response = read_response(dever.system.http_request(client, request)) +} + +public open(client: dever.system.HttpClient, request: Request) ( + response: StreamResponse +) { + response = read_stream_response(dever.system.http_open(client, request)) +} + +public open_stream(client: dever.system.HttpClient, request: StreamRequest) ( + response: StreamResponse +) { + response = read_stream_response(dever.system.http_open_stream(client, request)) +} + +read_stream_response(result: StreamResponseResult.Read(response)) ( + value: StreamResponse +) { + value = response +} + +read_stream_response(result: StreamResponseResult.Failed(message)) ( + value: StreamResponse +) { + fail(StreamResponseResult.Failed(message)) +} + +public close_client(client: dever.system.HttpClient) () { + closed(dever.system.http_close_client(client)) +} + +public serve_tls( + route: handler(request: Request) (response: Response), + listener: dever.system.Listener, + limits: Limits, + tls: dever.system.ServerTls +) () { + dever.system.http_serve_tls(route, listener, limits, tls) +} + +public serve_live_tls( + route: handler(request: Request, reply: dever.system.HttpReply) (), + listener: dever.system.Listener, + limits: Limits, + live: LiveLimits, + tls: dever.system.ServerTls +) () { + dever.system.http_serve_live_tls(route, listener, limits, live, tls) +} + +public type Header { + name: Text + value: Bytes +} + +public type Request { + method: Text + target: Text + headers: List<Header> + body: Bytes +} + +public type Response { + status: Int + headers: List<Header> + body: Bytes +} + +public type Limits { + header_bytes: Int + body_bytes: Int + timeout_ms: Int + connections: Int + http2: Http2Limits? +} + +# streams 为每连接并发流数；窗口限制协议层未消费的接收正文。 +public type Http2Limits { + streams: Int + stream_window_bytes: Int + connection_window_bytes: Int +} + +public default_http2_limits() (result: Http2Limits) { + result = Http2Limits { + streams = 16 + stream_window_bytes = 65535 + connection_window_bytes = 262144 + } +} + +public type LiveLimits { + chunk_bytes: Int + idle_ms: Int + heartbeat_ms: Int +} + +public default_live_limits() (result: LiveLimits) { + result = LiveLimits { + chunk_bytes = 65536 + idle_ms = 60000 + heartbeat_ms = 15000 + } +} + +public type ResponseResult { + Read(response: Response) + error Failed(message: Text) +} + +public default_limits() (result: Limits) { + result = Limits { + header_bytes = 8192 + body_bytes = 1048576 + timeout_ms = 5000 + connections = 32 + http2 = null + } +} + +public header(name: Text, value: Text) (result: Header) { + result = Header { + name = text.lower(name) + value = dever.bytes.from_text(value) + } +} + +# 返回第一个匹配项的原始值；重复头保留在 headers 列表中。 +public header_value(headers: List<Header>, name: Text) (value: Bytes?) { + value = found_header(find(named_header, headers, text.lower(name))) +} + +named_header(header: Header, name: Text) (matches: Bool) { + matches = text.lower(header.name) == name +} + +found_header(header: null) (value: Bytes?) { + value = null +} + +found_header(header: Header) (value: Bytes?) { + value = header.value +} + +# serve 是长驻任务，调用者用 run/stop 管理生命周期；故障沿任务作用域传播。 +public serve( + route: handler(request: Request) (response: Response), + listener: dever.system.Listener, + limits: Limits +) () { + dever.system.http_serve(route, listener, limits) +} + +# 单次请求，无连接池或重试；address 为数字 IP，Host 由请求头指定。 +public send(address: Text, port: Int, request: Request, limits: Limits) ( + response: Response +) { + response = read_response(dever.system.http_send(address, port, request, limits)) +} + +read_response(result: ResponseResult.Read(response)) (value: Response) { + value = response +} + +read_response(result: ResponseResult.Failed(message)) (value: Response) { + fail(ResponseResult.Failed(message)) +} + +# handler 可发送完整正文、逐块写入、SSE，或升级 WebSocket；返回即结束会话。 +public serve_live( + route: handler(request: Request, reply: dever.system.HttpReply) (), + listener: dever.system.Listener, + limits: Limits, + live: LiveLimits +) () { + dever.system.http_serve_live(route, listener, limits, live) +} + +public respond(reply: dever.system.HttpReply, response: Response) () { + wrote(dever.system.http_respond(reply, response)) +} + +public start(reply: dever.system.HttpReply, status: Int, headers: List<Header>) () { + wrote(dever.system.http_start(reply, status, headers)) +} + +public write(reply: dever.system.HttpReply, bytes: Bytes) () { + wrote(dever.system.http_write(reply, bytes)) +} + +public finish(reply: dever.system.HttpReply) () { + closed(dever.system.http_finish(reply)) +} + +wrote(result: dever.io.WriteResult.Done) () {} + +wrote(result: dever.io.WriteResult.Failed(message)) () { + fail(dever.io.WriteResult.Failed(message)) +} + +closed(result: dever.io.CloseResult.Closed) () {} + +closed(result: dever.io.CloseResult.Failed(message)) () { + fail(dever.io.CloseResult.Failed(message)) +} diff --git a/library/dever/id.dever b/library/dever/id.dever new file mode 100644 index 0000000..71d6396 --- /dev/null +++ b/library/dever/id.dever @@ -0,0 +1,7 @@ +public from_text(text: Text) (id: Id) { + id = dever.system.id_from_text(text) +} + +public to_text(id: Id) (text: Text) { + text = dever.system.id_to_text(id) +} diff --git a/library/dever/io.dever b/library/dever/io.dever new file mode 100644 index 0000000..92a9bfa --- /dev/null +++ b/library/dever/io.dever @@ -0,0 +1,108 @@ +public type OpenResult { + Opened(file: dever.system.File) + error Failed(message: Text) +} + +public type ReadState { + Read(bytes: Bytes) + End +} + +public type ReadResult { + Done(state: ReadState) + error Failed(message: Text) +} + +type SystemReadResult { + Read(bytes: Bytes) + End + error Failed(message: Text) +} + +public type WriteResult { + Done + error Failed(message: Text) +} + +public type CloseResult { + Closed + error Failed(message: Text) +} + +public type ReadEvent { + Chunk(bytes: Bytes) + error Failed(message: Text) +} + +public type ReadStreamResult { + Streaming(stream: Stream<ReadEvent>) + error Failed(message: Text) +} + +public println(text: Text) () { + dever.system.stdout_write(text + "\n") +} + +public open(path: Text) (file: dever.system.File) { + file = opened(dever.system.file_open(path)) +} + +public create(path: Text) (file: dever.system.File) { + file = opened(dever.system.file_create(path)) +} + +opened(result: OpenResult.Opened(file)) (value: dever.system.File) { + value = file +} + +opened(result: OpenResult.Failed(message)) (value: dever.system.File) { + fail(OpenResult.Failed(message)) +} + +public read(file: dever.system.File, limit: Int) (state: ReadState) { + state = read_state(dever.system.file_read(file, limit)) +} + +read_state(result: SystemReadResult.Read(bytes)) (state: ReadState) { + state = ReadState.Read(bytes) +} + +read_state(result: SystemReadResult.End) (state: ReadState) { + state = ReadState.End +} + +read_state(result: SystemReadResult.Failed(message)) (state: ReadState) { + fail(ReadResult.Failed(message)) +} + +public write(file: dever.system.File, bytes: Bytes) () { + wrote(dever.system.file_write(file, bytes)) +} + +wrote(result: WriteResult.Done) () {} + +wrote(result: WriteResult.Failed(message)) () { + fail(WriteResult.Failed(message)) +} + +public close(file: dever.system.File) () { + closed(dever.system.file_close(file)) +} + +closed(result: CloseResult.Closed) () {} + +closed(result: CloseResult.Failed(message)) () { + fail(CloseResult.Failed(message)) +} + +public chunks(file: dever.system.File, limit: Int) (stream: Stream<ReadEvent>) { + stream = read_stream(dever.system.file_chunks(file, limit)) +} + +read_stream(result: ReadStreamResult.Streaming(stream)) (value: Stream<ReadEvent>) { + value = stream +} + +read_stream(result: ReadStreamResult.Failed(message)) (value: Stream<ReadEvent>) { + fail(ReadStreamResult.Failed(message)) +} diff --git a/library/dever/job.dever b/library/dever/job.dever new file mode 100644 index 0000000..6008b41 --- /dev/null +++ b/library/dever/job.dever @@ -0,0 +1,3 @@ +public serve() () { + dever.system.job_serve() +} diff --git a/library/dever/json.dever b/library/dever/json.dever new file mode 100644 index 0000000..fadd903 --- /dev/null +++ b/library/dever/json.dever @@ -0,0 +1,76 @@ +# JSON values and capture result choices live in dever.json.value. +public parse(source: Text) (document: dever.json.value.Document) { + document = dever.json.parse.parse(source) +} + +public stringify(document: dever.json.value.Document) (text: Text) { + text = dever.json.write.stringify(document) +} + +public node(document: dever.json.value.Document, id: Int) (result: dever.json.value.Value?) { + result = get(document.nodes, id) +} + +public property(document: dever.json.value.Document, id: Int, name: Text) (result: Int?) { + result = object_property(node(document, id), name) +} + +object_property(value: dever.json.value.Value.Object(members), name: Text) ( + result: Int? +) { + result = get(members, name) +} + +object_property(value: null, name: Text) (result: Int?) { + result = null +} + +object_property(value: other, name: Text) (result: Int?) { + result = null +} + +type ElementState { + target: Int + index: Int + found: Int? +} + +public element(document: dever.json.value.Document, id: Int, index: Int) (result: Int?) { + result = array_element(node(document, id), index) +} + +array_element(value: dever.json.value.Value.Array(children), index: Int) (result: Int?) { + state = reduce_until( + element_at, + children, + ElementState { + target = index + index = 0 + found = null + } + ) + result = state.found +} + +array_element(value: null, index: Int) (result: Int?) { + result = null +} + +array_element(value: other, index: Int) (result: Int?) { + result = null +} + +element_at(id: Int, state: ElementState) (next: ElementState, stop: Bool) { + next = selected(state.index == state.target, id, state) + next.index = state.index + 1 + stop = state.index >= state.target +} + +selected(matches: true, id: Int, state: ElementState) (next: ElementState) { + next = state + next.found = id +} + +selected(matches: false, id: Int, state: ElementState) (next: ElementState) { + next = state +} diff --git a/library/dever/json/number.dever b/library/dever/json/number.dever new file mode 100644 index 0000000..f8485a5 --- /dev/null +++ b/library/dever/json/number.dever @@ -0,0 +1,149 @@ +type Phase { + Start + Sign + Zero + Integer + Point + Fraction + Exponent + ExponentSign + ExponentDigits + Invalid +} + +public valid(raw: Text) (result: Bool) { + phase = reduce(advance, text.split(raw, ""), Phase.Start) + result = complete(phase) +} + +advance(character: Text, phase: Phase.Start) (result: Phase) { + result = start(character) +} + +advance(character: Text, phase: Phase.Sign) (result: Phase) { + result = integer_start(character) +} + +advance(character: Text, phase: Phase.Zero) (result: Phase) { + result = after_integer(character) +} + +advance(character: Text, phase: Phase.Integer) (result: Phase) { + result = integer_digit(text.contains("0123456789", character), character) +} + +advance(character: Text, phase: Phase.Point) (result: Phase) { + result = digit(text.contains("0123456789", character), Phase.Fraction) +} + +advance(character: Text, phase: Phase.Fraction) (result: Phase) { + result = fraction_digit(text.contains("0123456789", character), character) +} + +advance(character: Text, phase: Phase.Exponent) (result: Phase) { + result = exponent_start(character) +} + +advance(character: Text, phase: Phase.ExponentSign) (result: Phase) { + result = digit(text.contains("0123456789", character), Phase.ExponentDigits) +} + +advance(character: Text, phase: Phase.ExponentDigits) (result: Phase) { + result = digit(text.contains("0123456789", character), Phase.ExponentDigits) +} + +advance(character: Text, phase: Phase.Invalid) (result: Phase) { + result = Phase.Invalid +} + +start(character: "-") (result: Phase) { + result = Phase.Sign +} + +start(character: other) (result: Phase) { + result = integer_start(character) +} + +integer_start(character: "0") (result: Phase) { + result = Phase.Zero +} + +integer_start(character: other) (result: Phase) { + result = digit(text.contains("123456789", character), Phase.Integer) +} + +digit(valid: true, phase: Phase) (result: Phase) { + result = phase +} + +digit(valid: false, phase: Phase) (result: Phase) { + result = Phase.Invalid +} + +integer_digit(valid: true, character: Text) (result: Phase) { + result = Phase.Integer +} + +integer_digit(valid: false, character: Text) (result: Phase) { + result = after_integer(character) +} + +after_integer(character: ".") (result: Phase) { + result = Phase.Point +} + +after_integer(character: other) (result: Phase) { + result = exponent(character) +} + +fraction_digit(valid: true, character: Text) (result: Phase) { + result = Phase.Fraction +} + +fraction_digit(valid: false, character: Text) (result: Phase) { + result = exponent(character) +} + +exponent(character: "e") (result: Phase) { + result = Phase.Exponent +} + +exponent(character: "E") (result: Phase) { + result = Phase.Exponent +} + +exponent(character: other) (result: Phase) { + result = Phase.Invalid +} + +exponent_start(character: "+") (result: Phase) { + result = Phase.ExponentSign +} + +exponent_start(character: "-") (result: Phase) { + result = Phase.ExponentSign +} + +exponent_start(character: other) (result: Phase) { + result = digit(text.contains("0123456789", character), Phase.ExponentDigits) +} + +complete(phase: Phase.Zero) (result: Bool) { + result = true +} + +complete(phase: Phase.Integer) (result: Bool) { + result = true +} + +complete(phase: Phase.Fraction) (result: Bool) { + result = true +} + +complete(phase: Phase.ExponentDigits) (result: Bool) { + result = true +} + +complete(phase: other) (result: Bool) { + result = false +} diff --git a/library/dever/json/parse.dever b/library/dever/json/parse.dever new file mode 100644 index 0000000..77ff659 --- /dev/null +++ b/library/dever/json/parse.dever @@ -0,0 +1,388 @@ +type Expect { + RootValue + RootDone + ArrayFirst + ArrayValue + ArrayNext + ObjectFirst + ObjectKey + ObjectColon + ObjectValue + ObjectNext +} + +type Frame { + expect: Expect + children: List<Int> + members: Map<Text, Int> + key: Text +} + +type State { + stack: Map<Int, Frame> + depth: Int + nodes: Map<Int, dever.json.value.Value> + next_id: Int + root: Int? + failure: Text + position: Int +} + +public parse(source: Text) (document: dever.json.value.Document) { + document = scanned(result(dever.json.scan.scan(source)), length(source)) +} + +scanned(result: dever.json.scan.Result.Scanned(tokens), end: Int) ( + document: dever.json.value.Document +) recover("scanner failures become public JSON parse failures") { + state = reduce_until( + token, + tokens, + State { + stack = { + 0 = frame(Expect.RootValue) + } + depth = 0 + nodes = {} + next_id = 0 + root = null + failure = "" + position = 0 + } + ) + document = finish(state.failure == "", state, end) +} + +scanned(result: dever.json.scan.Result.Failed(position, message), end: Int) ( + document: dever.json.value.Document +) recover("scanner failures become public JSON parse failures") { + fail(dever.json.value.ParseResult.Failed(position, message)) +} + +frame(expect: Expect) (result: Frame) { + result = Frame { + expect = expect + children = [] + members = {} + key = "" + } +} + +finish(valid: false, state: State, end: Int) (document: dever.json.value.Document) { + fail(dever.json.value.ParseResult.Failed(state.position, state.failure)) +} + +finish(valid: true, state: State, end: Int) (document: dever.json.value.Document) { + document = balanced(state.depth == 0, state, end) +} + +balanced(complete: false, state: State, end: Int) (document: dever.json.value.Document) { + fail(dever.json.value.ParseResult.Failed(end, "unterminated JSON container")) +} + +balanced(complete: true, state: State, end: Int) (document: dever.json.value.Document) { + document = root(state.root, state.nodes, end) +} + +root(id: null, nodes: Map<Int, dever.json.value.Value>, end: Int) ( + document: dever.json.value.Document +) { + fail(dever.json.value.ParseResult.Failed(end, "expected a JSON value")) +} + +root(id: Int, nodes: Map<Int, dever.json.value.Value>, end: Int) ( + document: dever.json.value.Document +) { + document = dever.json.value.Document { + root = id + nodes = nodes + } +} + +token(token: dever.json.scan.Token, state: State) (next: State, stop: Bool) { + positioned = state + positioned.position = token.position + next = current(get(state.stack, state.depth), token.kind, positioned) + stop = next.failure != "" +} + +current(frame: null, kind: dever.json.scan.Kind, state: State) (next: State) { + next = invalid_state("invalid JSON parser stack", state) +} + +current(frame: Frame, kind: dever.json.scan.Kind, state: State) (next: State) { + next = expected(frame.expect, kind, frame, state) +} + +expected( + expect: Expect.RootValue, + kind: dever.json.scan.Kind, + frame: Frame, + state: State +) (next: State) { + next = value(kind, state) +} + +expected( + expect: Expect.RootDone, + kind: dever.json.scan.Kind, + frame: Frame, + state: State +) (next: State) { + next = invalid_state("unexpected content after the JSON value", state) +} + +expected( + expect: Expect.ArrayFirst, + kind: dever.json.scan.Kind, + frame: Frame, + state: State +) (next: State) { + next = array_first(kind, frame, state) +} + +expected( + expect: Expect.ArrayValue, + kind: dever.json.scan.Kind, + frame: Frame, + state: State +) (next: State) { + next = value(kind, state) +} + +expected( + expect: Expect.ArrayNext, + kind: dever.json.scan.Kind, + frame: Frame, + state: State +) (next: State) { + next = array_next(kind, frame, state) +} + +expected( + expect: Expect.ObjectFirst, + kind: dever.json.scan.Kind, + frame: Frame, + state: State +) (next: State) { + next = object_first(kind, frame, state) +} + +expected( + expect: Expect.ObjectKey, + kind: dever.json.scan.Kind, + frame: Frame, + state: State +) (next: State) { + next = object_key(kind, frame, state) +} + +expected( + expect: Expect.ObjectColon, + kind: dever.json.scan.Kind, + frame: Frame, + state: State +) (next: State) { + next = object_colon(kind, frame, state) +} + +expected( + expect: Expect.ObjectValue, + kind: dever.json.scan.Kind, + frame: Frame, + state: State +) (next: State) { + next = value(kind, state) +} + +expected( + expect: Expect.ObjectNext, + kind: dever.json.scan.Kind, + frame: Frame, + state: State +) (next: State) { + next = object_next(kind, frame, state) +} + +value(kind: dever.json.scan.Kind.Scalar(value), state: State) (next: State) { + next = commit(value, state) +} + +value(kind: dever.json.scan.Kind.ArrayOpen, state: State) (next: State) { + next = open(state.depth < dever.json.value.maximum_depth(), Expect.ArrayFirst, state) +} + +value(kind: dever.json.scan.Kind.ObjectOpen, state: State) (next: State) { + next = open(state.depth < dever.json.value.maximum_depth(), Expect.ObjectFirst, state) +} + +value(kind: other, state: State) (next: State) { + next = invalid_state("expected a JSON value", state) +} + +open(allowed: false, expect: Expect, state: State) (next: State) { + next = invalid_state("JSON nesting exceeds 256 containers", state) +} + +open(allowed: true, expect: Expect, state: State) (next: State) { + next = state + next.depth = state.depth + 1 + next.stack = put(state.stack, next.depth, frame(expect)) +} + +array_first(kind: dever.json.scan.Kind.ArrayClose, frame: Frame, state: State) ( + next: State +) { + next = close(dever.json.value.Value.Array(frame.children), state) +} + +array_first(kind: other, frame: Frame, state: State) (next: State) { + next = value(kind, state) +} + +array_next(kind: dever.json.scan.Kind.ArrayClose, frame: Frame, state: State) ( + next: State +) { + next = close(dever.json.value.Value.Array(frame.children), state) +} + +array_next(kind: dever.json.scan.Kind.Comma, frame: Frame, state: State) (next: State) { + next = expecting(Expect.ArrayValue, frame, state) +} + +array_next(kind: other, frame: Frame, state: State) (next: State) { + next = invalid_state("expected ',' or ']' after an array element", state) +} + +object_first(kind: dever.json.scan.Kind.ObjectClose, frame: Frame, state: State) ( + next: State +) { + next = close(dever.json.value.Value.Object(frame.members), state) +} + +object_first(kind: other, frame: Frame, state: State) (next: State) { + next = object_key(kind, frame, state) +} + +object_key(kind: dever.json.scan.Kind.Scalar(value), frame: Frame, state: State) ( + next: State +) { + next = key_value(value, frame, state) +} + +object_key(kind: other, frame: Frame, state: State) (next: State) { + next = invalid_state("expected a quoted JSON object name", state) +} + +key_value(value: dever.json.value.Value.String(key), frame: Frame, state: State) ( + next: State +) { + next = unique_key(get(frame.members, key) == null, key, frame, state) +} + +key_value(value: other, frame: Frame, state: State) (next: State) { + next = invalid_state("expected a quoted JSON object name", state) +} + +unique_key(unique: false, key: Text, frame: Frame, state: State) (next: State) { + next = invalid_state("duplicate JSON object name: " + key, state) +} + +unique_key(unique: true, key: Text, frame: Frame, state: State) (next: State) { + keyed = frame + keyed.key = key + next = expecting(Expect.ObjectColon, keyed, state) +} + +object_colon(kind: dever.json.scan.Kind.Colon, frame: Frame, state: State) (next: State) { + next = expecting(Expect.ObjectValue, frame, state) +} + +object_colon(kind: other, frame: Frame, state: State) (next: State) { + next = invalid_state("expected ':' after a JSON object name", state) +} + +object_next(kind: dever.json.scan.Kind.ObjectClose, frame: Frame, state: State) ( + next: State +) { + next = close(dever.json.value.Value.Object(frame.members), state) +} + +object_next(kind: dever.json.scan.Kind.Comma, frame: Frame, state: State) (next: State) { + next = expecting(Expect.ObjectKey, frame, state) +} + +object_next(kind: other, frame: Frame, state: State) (next: State) { + next = invalid_state("expected ',' or '}' after an object member", state) +} + +expecting(expect: Expect, frame: Frame, state: State) (next: State) { + updated = frame + updated.expect = expect + next = state + next.stack = put(state.stack, state.depth, updated) +} + +close(value: dever.json.value.Value, state: State) (next: State) { + parent = state + parent.stack = remove(state.stack, state.depth) + parent.depth = state.depth - 1 + next = commit(value, parent) +} + +commit(value: dever.json.value.Value, state: State) (next: State) { + inserted = state + inserted.nodes = put(state.nodes, state.next_id, value) + inserted.next_id = state.next_id + 1 + next = attach(get(state.stack, state.depth), state.next_id, inserted) +} + +attach(frame: null, id: Int, state: State) (next: State) { + next = invalid_state("invalid JSON parser parent", state) +} + +attach(frame: Frame, id: Int, state: State) (next: State) { + next = attach_expected(frame.expect, frame, id, state) +} + +attach_expected(expect: Expect.RootValue, frame: Frame, id: Int, state: State) ( + next: State +) { + next = expecting(Expect.RootDone, frame, state) + next.root = id +} + +attach_expected(expect: Expect.ArrayFirst, frame: Frame, id: Int, state: State) ( + next: State +) { + next = append_child(frame, id, state) +} + +attach_expected(expect: Expect.ArrayValue, frame: Frame, id: Int, state: State) ( + next: State +) { + next = append_child(frame, id, state) +} + +attach_expected(expect: Expect.ObjectValue, frame: Frame, id: Int, state: State) ( + next: State +) { + updated = frame + updated.members = put(frame.members, frame.key, id) + next = expecting(Expect.ObjectNext, updated, state) +} + +attach_expected(expect: other, frame: Frame, id: Int, state: State) (next: State) { + next = invalid_state("unexpected JSON value", state) +} + +append_child(frame: Frame, id: Int, state: State) (next: State) { + updated = frame + updated.children = append(frame.children, id) + next = expecting(Expect.ArrayNext, updated, state) +} + +invalid_state(message: Text, state: State) (next: State) { + next = state + next.failure = message +} diff --git a/library/dever/json/scan.dever b/library/dever/json/scan.dever new file mode 100644 index 0000000..0196e32 --- /dev/null +++ b/library/dever/json/scan.dever @@ -0,0 +1,442 @@ +public type Kind { + Scalar(value: dever.json.value.Value) + ObjectOpen + ObjectClose + ArrayOpen + ArrayClose + Colon + Comma +} + +public type Token { + kind: Kind + position: Int +} + +public type Result { + Scanned(tokens: List<Token>) + error Failed(position: Int, message: Text) +} + +type Mode { + Normal + Word + String + Escape + Unicode + LowSlash + LowU +} + +type State { + tokens: List<Token> + mode: Mode + buffer: Text + position: Int + start: Int + unicode: Int + digits: Int + high: Int + failure: Text + error_position: Int +} + +public scan(source: Text) (tokens: List<Token>) { + state = reduce_until( + character, + text.split(source + " ", ""), + State { + tokens = [] + mode = Mode.Normal + buffer = "" + position = 0 + start = 0 + unicode = 0 + digits = 0 + high = 0 + failure = "" + error_position = 0 + } + ) + tokens = finish(state.failure == "", state, length(source)) +} + +finish(valid: false, state: State, end: Int) (tokens: List<Token>) { + fail(Result.Failed(state.error_position, state.failure)) +} + +finish(valid: true, state: State, end: Int) (tokens: List<Token>) { + tokens = finish_mode(state.mode, state.tokens, end) +} + +finish_mode(mode: Mode.Normal, tokens: List<Token>, end: Int) (result: List<Token>) { + result = tokens +} + +finish_mode(mode: other, tokens: List<Token>, end: Int) (result: List<Token>) { + fail(Result.Failed(end, "unterminated JSON string or escape")) +} + +character(value: Text, state: State) (next: State, stop: Bool) { + next = consume(state.mode, value, state) + next.position = state.position + 1 + stop = next.failure != "" +} + +consume(mode: Mode.Normal, value: Text, state: State) (next: State) { + next = normal(value, state) +} + +consume(mode: Mode.Word, value: Text, state: State) (next: State) { + next = word(text.contains(" \t\r\n{}[]:,\"", value), value, state) +} + +consume(mode: Mode.String, value: Text, state: State) (next: State) { + next = string(value, state) +} + +consume(mode: Mode.Escape, value: Text, state: State) (next: State) { + next = escape(value, state) +} + +consume(mode: Mode.Unicode, value: Text, state: State) (next: State) { + next = unicode_digit(text.index_of("0123456789abcdef", text.lower(value)), state) +} + +consume(mode: Mode.LowSlash, value: Text, state: State) (next: State) { + next = low_slash(value == "\\", state) +} + +consume(mode: Mode.LowU, value: Text, state: State) (next: State) { + next = low_u(value == "u", state) +} + +normal(value: "{", state: State) (next: State) { + next = emit(Kind.ObjectOpen, state.position, state) +} + +normal(value: "}", state: State) (next: State) { + next = emit(Kind.ObjectClose, state.position, state) +} + +normal(value: "[", state: State) (next: State) { + next = emit(Kind.ArrayOpen, state.position, state) +} + +normal(value: "]", state: State) (next: State) { + next = emit(Kind.ArrayClose, state.position, state) +} + +normal(value: ":", state: State) (next: State) { + next = emit(Kind.Colon, state.position, state) +} + +normal(value: ",", state: State) (next: State) { + next = emit(Kind.Comma, state.position, state) +} + +normal(value: "\"", state: State) (next: State) { + next = state + next.mode = Mode.String + next.start = state.position + next.buffer = "" +} + +normal(value: other, state: State) (next: State) { + next = whitespace(text.contains(" \t\r\n", value), value, state) +} + +whitespace(space: true, value: Text, state: State) (next: State) { + next = state +} + +whitespace(space: false, value: Text, state: State) (next: State) { + next = state + next.mode = Mode.Word + next.start = state.position + next.buffer = value +} + +word(ended: false, value: Text, state: State) (next: State) { + next = state + next.buffer = state.buffer + value +} + +word(ended: true, value: Text, state: State) (next: State) { + finished = atom(state.buffer, state) + next = after_word(finished.failure == "", value, finished) +} + +after_word(valid: true, value: Text, state: State) (next: State) { + next = normal(value, state) +} + +after_word(valid: false, value: Text, state: State) (next: State) { + next = state +} + +atom(raw: "true", state: State) (next: State) { + next = scalar(dever.json.value.Value.Boolean(true), state) +} + +atom(raw: "false", state: State) (next: State) { + next = scalar(dever.json.value.Value.Boolean(false), state) +} + +atom(raw: "null", state: State) (next: State) { + next = scalar(dever.json.value.Value.Null, state) +} + +atom(raw: other, state: State) (next: State) { + next = number(dever.json.number.valid(raw), raw, state) +} + +number(valid: true, raw: Text, state: State) (next: State) { + next = scalar(dever.json.value.Value.Number(raw), state) +} + +number(valid: false, raw: Text, state: State) (next: State) { + next = invalid_state(state.start, "invalid JSON literal or number", state) +} + +scalar(value: dever.json.value.Value, state: State) (next: State) { + next = emit(Kind.Scalar(value), state.start, state) + next.mode = Mode.Normal +} + +emit(kind: Kind, position: Int, state: State) (next: State) { + next = state + next.tokens = append( + state.tokens, + Token { + kind = kind + position = position + } + ) +} + +invalid_state(position: Int, message: Text, state: State) (next: State) { + next = state + next.failure = message + next.error_position = position +} + +string(value: "\"", state: State) (next: State) { + next = scalar(dever.json.value.Value.String(state.buffer), state) +} + +string(value: "\\", state: State) (next: State) { + next = state + next.mode = Mode.Escape +} + +string(value: other, state: State) (next: State) { + next = string_character(text.codepoint(value), value, state) +} + +string_character(code: Int, value: Text, state: State) (next: State) { + next = string_control(code < 32, value, state) +} + +string_character(code: null, value: Text, state: State) (next: State) { + next = invalid_state(state.position, "invalid Unicode character", state) +} + +string_control(control: true, value: Text, state: State) (next: State) { + next = invalid_state(state.position, "unescaped control character in JSON string", state) +} + +string_control(control: false, value: Text, state: State) (next: State) { + next = decoded(value, state) +} + +decoded(value: Text, state: State) (next: State) { + next = state + next.buffer = state.buffer + value + next.mode = Mode.String +} + +escape(value: "\"", state: State) (next: State) { + next = decoded("\"", state) +} + +escape(value: "\\", state: State) (next: State) { + next = decoded("\\", state) +} + +escape(value: "/", state: State) (next: State) { + next = decoded("/", state) +} + +escape(value: "b", state: State) (next: State) { + next = decoded("\u{8}", state) +} + +escape(value: "f", state: State) (next: State) { + next = decoded("\u{c}", state) +} + +escape(value: "n", state: State) (next: State) { + next = decoded("\n", state) +} + +escape(value: "r", state: State) (next: State) { + next = decoded("\r", state) +} + +escape(value: "t", state: State) (next: State) { + next = decoded("\t", state) +} + +escape(value: "u", state: State) (next: State) { + next = state + next.mode = Mode.Unicode + next.unicode = 0 + next.digits = 0 +} + +escape(value: other, state: State) (next: State) { + next = invalid_state(state.position, "invalid JSON string escape", state) +} + +unicode_digit(digit: null, state: State) (next: State) { + next = invalid_state(state.position, "expected four hexadecimal escape digits", state) +} + +unicode_digit(digit: Int, state: State) (next: State) { + advanced = state + advanced.unicode = state.unicode * 16 + digit + advanced.digits = state.digits + 1 + next = unicode_ready(advanced.digits == 4, advanced) +} + +unicode_ready(ready: false, state: State) (next: State) { + next = state +} + +unicode_ready(ready: true, state: State) (next: State) { + next = unicode_unit(state.high == 0, state.unicode, state) +} + +unicode_unit(first: true, code: Int, state: State) (next: State) { + next = first_unit(code, state) +} + +unicode_unit(first: false, code: Int, state: State) (next: State) { + next = second_unit(code >= 56320 and code <= 57343, code, state) +} + +first_unit(code: Int >= 55296 and <= 56319, state: State) (next: State) { + next = state + next.high = code + next.mode = Mode.LowSlash +} + +first_unit(code: Int >= 56320 and <= 57343, state: State) (next: State) { + next = invalid_state(state.position, "unpaired low Unicode surrogate", state) +} + +first_unit(code: other, state: State) (next: State) { + next = scalar_character(text.from_codepoint(code), state) +} + +second_unit(valid: false, code: Int, state: State) (next: State) { + next = invalid_state(state.position, "expected a low Unicode surrogate", state) +} + +second_unit(valid: true, code: Int, state: State) (next: State) { + next = scalar_character( + text.from_codepoint(65536 + (state.high - 55296) * 1024 + code - 56320), + state + ) +} + +scalar_character(value: Text, state: State) (next: State) { + next = decoded(value, state) + next.high = 0 +} + +scalar_character(value: null, state: State) (next: State) { + next = invalid_state(state.position, "invalid Unicode scalar", state) +} + +low_slash(valid: true, state: State) (next: State) { + next = state + next.mode = Mode.LowU +} + +low_slash(valid: false, state: State) (next: State) { + next = invalid_state(state.position, "expected a low Unicode surrogate escape", state) +} + +low_u(valid: true, state: State) (next: State) { + next = escape("u", state) +} + +low_u(valid: false, state: State) (next: State) { + next = invalid_state(state.position, "expected a low Unicode surrogate escape", state) +} + +public quote(value: Text) (result: Text) { + result = "\"" + text.join(each(quoted_character, text.split(value, "")), "") + "\"" +} + +quoted_character(value: "\"") (result: Text) { + result = "\\\"" +} + +quoted_character(value: "\\") (result: Text) { + result = "\\\\" +} + +quoted_character(value: "\u{8}") (result: Text) { + result = "\\b" +} + +quoted_character(value: "\u{c}") (result: Text) { + result = "\\f" +} + +quoted_character(value: "\n") (result: Text) { + result = "\\n" +} + +quoted_character(value: "\r") (result: Text) { + result = "\\r" +} + +quoted_character(value: "\t") (result: Text) { + result = "\\t" +} + +quoted_character(value: other) (result: Text) { + result = quoted_code(text.codepoint(value), value) +} + +quoted_code(code: Int, value: Text) (result: Text) { + result = quote_control(code < 32, code, value) +} + +quoted_code(code: null, value: Text) (result: Text) { + result = value +} + +quote_control(control: false, code: Int, value: Text) (result: Text) { + result = value +} + +quote_control(control: true, code: Int, value: Text) (result: Text) { + result = "\\u00" + hex_digit(code // 16) + hex_digit(code % 16) +} + +hex_digit(value: Int) (result: Text) { + result = hex_character(text.at("0123456789abcdef", value)) +} + +hex_character(value: Text) (result: Text) { + result = value +} + +hex_character(value: null) (result: Text) { + result = "" +} diff --git a/library/dever/json/value.dever b/library/dever/json/value.dever new file mode 100644 index 0000000..a7f0d8c --- /dev/null +++ b/library/dever/json/value.dever @@ -0,0 +1,29 @@ +# Nodes use consecutive postorder IDs starting at zero. Children precede their parent. +public type Value { + Null + Boolean(value: Bool) + Number(raw: Text) + String(text: Text) + Array(children: List<Int>) + Object(members: Map<Text, Int>) +} + +public type Document { + root: Int + nodes: Map<Int, Value> +} + +# Parse positions are zero-based Unicode scalar offsets, including the EOF position. +public type ParseResult { + Parsed(document: Document) + error Failed(position: Int, message: Text) +} + +public type WriteResult { + Written(text: Text) + error Failed(node: Int, message: Text) +} + +public maximum_depth() (result: Int) { + result = 256 +} diff --git a/library/dever/json/write.dever b/library/dever/json/write.dever new file mode 100644 index 0000000..76832b8 --- /dev/null +++ b/library/dever/json/write.dever @@ -0,0 +1,269 @@ +type Encoded { + text: Text + depth: Int + failure: Text +} + +type State { + document: dever.json.value.Document + encoded: Map<Int, Encoded> + index: Int + failure: Text +} + +type Children { + encoded: Map<Int, Encoded> + parent: Int + pieces: List<Text> + depth: Int + size: Int + failure: Text +} + +# The output budget also bounds expansion of user-constructed shared child graphs. +maximum_size() (result: Int) { + result = 16777216 +} + +public stringify(document: dever.json.value.Document) (text: Text) { + state = reduce_until( + node, + entries(document.nodes), + State { + document = document + encoded = {} + index = 0 + failure = "" + } + ) + text = finish(state.failure == "", state) +} + +finish(valid: false, state: State) (text: Text) { + fail(dever.json.value.WriteResult.Failed(state.index, state.failure)) +} + +finish(valid: true, state: State) (text: Text) { + text = root(get(state.encoded, state.document.root), state.document.root) +} + +root(value: null, id: Int) (text: Text) { + fail( + dever.json.value.WriteResult.Failed( + id, + "JSON root does not refer to a document node" + ) + ) +} + +root(value: Encoded, id: Int) (text: Text) { + text = value.text +} + +# Iteration supplies a count; lookup uses IDs, so Map insertion order is immaterial. +node(entry: MapEntry<Int, dever.json.value.Value>, state: State) ( + next: State, + stop: Bool +) { + encoded = document_node(get(state.document.nodes, state.index), state) + next = store(encoded.failure == "", encoded, state) + stop = next.failure != "" +} + +document_node(value: null, state: State) (result: Encoded) { + result = failed("JSON node IDs must be consecutive from zero") +} + +document_node(value: dever.json.value.Value, state: State) (result: Encoded) { + encoded = encode(value, state) + result = bounded(length(encoded.text) <= maximum_size(), encoded) +} + +store(valid: false, encoded: Encoded, state: State) (next: State) { + next = state + next.failure = encoded.failure +} + +store(valid: true, encoded: Encoded, state: State) (next: State) { + next = state + next.encoded = put(state.encoded, state.index, encoded) + next.index = state.index + 1 +} + +bounded(valid: true, encoded: Encoded) (result: Encoded) { + result = encoded +} + +bounded(valid: false, encoded: Encoded) (result: Encoded) { + result = failed("JSON output exceeds 16777216 Unicode scalars") +} + +encoded(text: Text) (result: Encoded) { + result = Encoded { + text = text + depth = 0 + failure = "" + } +} + +failed(message: Text) (result: Encoded) { + result = Encoded { + text = "" + depth = 0 + failure = message + } +} + +encode(value: dever.json.value.Value.Null, state: State) (result: Encoded) { + result = encoded("null") +} + +encode(value: dever.json.value.Value.Boolean(boolean), state: State) (result: Encoded) { + result = encoded(boolean_text(boolean)) +} + +encode(value: dever.json.value.Value.Number(raw), state: State) (result: Encoded) { + result = number(dever.json.number.valid(raw), raw) +} + +encode(value: dever.json.value.Value.String(string), state: State) (result: Encoded) { + result = encoded(dever.json.scan.quote(string)) +} + +encode(value: dever.json.value.Value.Array(children), state: State) (result: Encoded) { + built = reduce_until(array_child, children, new_children(state)) + result = container(built.failure == "", "[", "]", built) +} + +encode(value: dever.json.value.Value.Object(members), state: State) (result: Encoded) { + built = reduce_until(object_child, entries(members), new_children(state)) + result = container(built.failure == "", "{", "}", built) +} + +boolean_text(value: true) (result: Text) { + result = "true" +} + +boolean_text(value: false) (result: Text) { + result = "false" +} + +number(valid: true, raw: Text) (result: Encoded) { + result = encoded(raw) +} + +number(valid: false, raw: Text) (result: Encoded) { + result = failed("invalid JSON number spelling") +} + +new_children(state: State) (result: Children) { + result = Children { + encoded = state.encoded + parent = state.index + pieces = [] + depth = 1 + size = 2 + failure = "" + } +} + +array_child(id: Int, state: Children) (next: Children, stop: Bool) { + next = child(id >= 0 and id < state.parent, id, "", state) + stop = next.failure != "" +} + +object_child(member: MapEntry<Text, Int>, state: Children) (next: Children, stop: Bool) { + next = child( + member.value >= 0 and member.value < state.parent, + member.value, + dever.json.scan.quote(member.key) + ":", + state + ) + stop = next.failure != "" +} + +child(valid: false, id: Int, prefix: Text, state: Children) (next: Children) { + next = child_failure("JSON child references must precede their parent", state) +} + +child(valid: true, id: Int, prefix: Text, state: Children) (next: Children) { + next = child_value(get(state.encoded, id), prefix, state) +} + +child_value(value: null, prefix: Text, state: Children) (next: Children) { + next = child_failure("JSON child node is missing", state) +} + +child_value(value: Encoded, prefix: Text, state: Children) (next: Children) { + size = state.size + length(prefix) + length(value.text) + separator_size( + length(state.pieces) > 0 + ) + next = child_size(size <= maximum_size(), size, value, prefix, state) +} + +separator_size(needed: true) (result: Int) { + result = 1 +} + +separator_size(needed: false) (result: Int) { + result = 0 +} + +child_size(valid: false, size: Int, value: Encoded, prefix: Text, state: Children) ( + next: Children +) { + next = child_failure("JSON output exceeds 16777216 Unicode scalars", state) +} + +child_size(valid: true, size: Int, value: Encoded, prefix: Text, state: Children) ( + next: Children +) { + next = state + next.pieces = append(state.pieces, prefix + value.text) + next.depth = maximum(state.depth, value.depth + 1) + next.size = size +} + +maximum(left: Int, right: Int) (result: Int) { + result = larger(left >= right, left, right) +} + +larger(left_larger: true, left: Int, right: Int) (result: Int) { + result = left +} + +larger(left_larger: false, left: Int, right: Int) (result: Int) { + result = right +} + +child_failure(message: Text, state: Children) (next: Children) { + next = state + next.failure = message +} + +container(valid: false, open: Text, close: Text, state: Children) (result: Encoded) { + result = failed(state.failure) +} + +container(valid: true, open: Text, close: Text, state: Children) (result: Encoded) { + result = container_depth( + state.depth <= dever.json.value.maximum_depth(), + open, + close, + state + ) +} + +container_depth(valid: false, open: Text, close: Text, state: Children) ( + result: Encoded +) { + result = failed("JSON nesting exceeds 256 containers") +} + +container_depth(valid: true, open: Text, close: Text, state: Children) (result: Encoded) { + result = Encoded { + text = open + text.join(state.pieces, ",") + close + depth = state.depth + failure = "" + } +} diff --git a/library/dever/log.dever b/library/dever/log.dever new file mode 100644 index 0000000..5984bdb --- /dev/null +++ b/library/dever/log.dever @@ -0,0 +1,15 @@ +public debug(message: Text, fields: Map<Text, Text>) () { + dever.system.log_debug(message, fields) +} + +public info(message: Text, fields: Map<Text, Text>) () { + dever.system.log_info(message, fields) +} + +public warn(message: Text, fields: Map<Text, Text>) () { + dever.system.log_warn(message, fields) +} + +public error(message: Text, fields: Map<Text, Text>) () { + dever.system.log_error(message, fields) +} diff --git a/library/dever/net.dever b/library/dever/net.dever new file mode 100644 index 0000000..8cb82c5 --- /dev/null +++ b/library/dever/net.dever @@ -0,0 +1,134 @@ +public type ConnectResult { + Connected(socket: dever.system.Socket) + error Failed(message: Text) +} + +public type ListenResult { + Listening(listener: dever.system.Listener) + error Failed(message: Text) +} + +public type PortResult { + Bound(port: Int) + error Failed(message: Text) +} + +public type ReadStreamResult { + Streaming(stream: AsyncStream<dever.io.ReadEvent>) + error Failed(message: Text) +} + +type SystemReadResult { + Read(bytes: Bytes) + End + error Failed(message: Text) +} + +public connect(host: Text, port: Int) (socket: dever.system.Socket) { + socket = connected(dever.system.tcp_connect(host, port)) +} + +# 有界连接使用数值 IP，避免把 DNS 解析误称为受此超时控制。 +public connect_timeout(address: Text, port: Int, milliseconds: Int) (socket: dever.system.Socket) { + socket = connected(dever.system.tcp_connect_timeout(address, port, milliseconds)) +} + +public accept(listener: dever.system.Listener) (socket: dever.system.Socket) { + socket = connected(dever.system.tcp_accept(listener)) +} + +connected(result: ConnectResult.Connected(socket)) (value: dever.system.Socket) { + value = socket +} + +connected(result: ConnectResult.Failed(message)) (value: dever.system.Socket) { + fail(ConnectResult.Failed(message)) +} + +public listen(host: Text, port: Int) (listener: dever.system.Listener) { + listener = listening(dever.system.tcp_listen(host, port)) +} + +listening(result: ListenResult.Listening(listener)) (value: dever.system.Listener) { + value = listener +} + +listening(result: ListenResult.Failed(message)) (value: dever.system.Listener) { + fail(ListenResult.Failed(message)) +} + +public port(listener: dever.system.Listener) (port: Int) { + port = bound(dever.system.tcp_port(listener)) +} + +bound(result: PortResult.Bound(port)) (value: Int) { + value = port +} + +bound(result: PortResult.Failed(message)) (value: Int) { + fail(PortResult.Failed(message)) +} + +public read(socket: dever.system.Socket, limit: Int) (state: dever.io.ReadState) { + state = read_state(dever.system.tcp_read(socket, limit)) +} + +read_state(result: SystemReadResult.Read(bytes)) (state: dever.io.ReadState) { + state = dever.io.ReadState.Read(bytes) +} + +read_state(result: SystemReadResult.End) (state: dever.io.ReadState) { + state = dever.io.ReadState.End +} + +read_state(result: SystemReadResult.Failed(message)) (state: dever.io.ReadState) { + fail(dever.io.ReadResult.Failed(message)) +} + +public write(socket: dever.system.Socket, bytes: Bytes) () { + wrote(dever.system.tcp_write(socket, bytes)) +} + +public timeout(socket: dever.system.Socket, milliseconds: Int) () { + wrote(dever.system.tcp_timeout(socket, milliseconds)) +} + +wrote(result: dever.io.WriteResult.Done) () {} + +wrote(result: dever.io.WriteResult.Failed(message)) () { + fail(dever.io.WriteResult.Failed(message)) +} + +public close(socket: dever.system.Socket) () { + closed(dever.system.tcp_close(socket)) +} + +public close_listener(listener: dever.system.Listener) () { + closed(dever.system.tcp_close_listener(listener)) +} + +closed(result: dever.io.CloseResult.Closed) () {} + +closed(result: dever.io.CloseResult.Failed(message)) () { + fail(dever.io.CloseResult.Failed(message)) +} + +public chunks(socket: dever.system.Socket, limit: Int) (stream: AsyncStream<dever.io.ReadEvent>) { + stream = read_stream(dever.system.tcp_chunks(socket, limit)) +} + +read_stream(result: ReadStreamResult.Streaming(stream)) ( + value: AsyncStream<dever.io.ReadEvent> +) { + value = stream +} + +read_stream(result: ReadStreamResult.Failed(message)) ( + value: AsyncStream<dever.io.ReadEvent> +) { + fail(ReadStreamResult.Failed(message)) +} + +public connections(listener: dever.system.Listener) (result: AsyncStream<ConnectResult>) { + result = dever.system.tcp_connections(listener) +} diff --git a/library/dever/process.dever b/library/dever/process.dever new file mode 100644 index 0000000..d878471 --- /dev/null +++ b/library/dever/process.dever @@ -0,0 +1,26 @@ +public type Output { + code: Int + stdout: Bytes + stderr: Bytes +} + +public type Error { + error Failed(message: Text) +} + +public type ArgumentsResult { + Read(values: List<Text>) + error Failed(message: Text) +} + +public arguments() (values: List<Text>) { + values = arguments_value(dever.system.process_arguments()) +} + +arguments_value(result: ArgumentsResult.Read(values)) (value: List<Text>) { + value = values +} + +arguments_value(result: ArgumentsResult.Failed(message)) (value: List<Text>) { + fail(ArgumentsResult.Failed(message)) +} diff --git a/library/dever/site.dever b/library/dever/site.dever new file mode 100644 index 0000000..7c59ea8 --- /dev/null +++ b/library/dever/site.dever @@ -0,0 +1,3 @@ +public key() (value: Text) { + value = dever.system.site_key() +} diff --git a/library/dever/sse.dever b/library/dever/sse.dever new file mode 100644 index 0000000..2a79c18 --- /dev/null +++ b/library/dever/sse.dever @@ -0,0 +1,21 @@ +public type Event { + event: Text + data: Text + id: Text? + retry_ms: Int? +} + +# 自动输出 Content-Type、默认 no-cache 和空闲心跳；重连位置由 Request 头读取。 +public start(reply: dever.system.HttpReply, headers: List<dever.http.Header>) () { + wrote(dever.system.sse_start(reply, headers)) +} + +public send(reply: dever.system.HttpReply, event: Event) () { + wrote(dever.system.sse_send(reply, event)) +} + +wrote(result: dever.io.WriteResult.Done) () {} + +wrote(result: dever.io.WriteResult.Failed(message)) () { + fail(dever.io.WriteResult.Failed(message)) +} diff --git a/library/dever/storage.dever b/library/dever/storage.dever new file mode 100644 index 0000000..6bb0535 --- /dev/null +++ b/library/dever/storage.dever @@ -0,0 +1,16 @@ +public type PutResult { + Stored(key: Uuid) + error Failed(message: Text) +} + +public put(upload: Upload) (key: Uuid) { + key = stored(dever.system.upload_store(upload)) +} + +stored(result: PutResult.Stored(key)) (value: Uuid) { + value = key +} + +stored(result: PutResult.Failed(message)) (value: Uuid) { + fail(PutResult.Failed(message)) +} diff --git a/library/dever/task.dever b/library/dever/task.dever new file mode 100644 index 0000000..7783ce5 --- /dev/null +++ b/library/dever/task.dever @@ -0,0 +1,14 @@ +# 第一个 tick 在一个周期后；落后的周期跳过，不积压待处理 tick。 +public ticks(millis: Int) (values: AsyncStream<Int>) { + values = dever.system.task_ticks(millis) +} + +public sleep(millis: Int) () { + slept(dever.system.task_sleep(millis)) +} + +slept(result: dever.time.SleepResult.Done) () {} + +slept(result: dever.time.SleepResult.Failed(message)) () { + fail(dever.time.SleepResult.Failed(message)) +} diff --git a/library/dever/time.dever b/library/dever/time.dever new file mode 100644 index 0000000..b95c55d --- /dev/null +++ b/library/dever/time.dever @@ -0,0 +1,144 @@ +public type ClockResult { + Read(value: Int) + error Failed(message: Text) +} + +public type SleepResult { + Done + error Failed(message: Text) +} + +public unix_millis() (value: Int) { + value = clock_value(dever.system.time_unix_millis()) +} + +public monotonic_nanos() (value: Int) { + value = clock_value(dever.system.time_monotonic_nanos()) +} + +clock_value(result: ClockResult.Read(value)) (clock: Int) { + clock = value +} + +clock_value(result: ClockResult.Failed(message)) (clock: Int) { + fail(ClockResult.Failed(message)) +} + +public sleep(millis: Int) () { + slept(dever.system.time_sleep(millis)) +} + +slept(result: SleepResult.Done) () {} + +slept(result: SleepResult.Failed(message)) () { + fail(SleepResult.Failed(message)) +} + +public type DateTimeResult { + Read(value: DateTime) + error Failed(message: Text) +} + +datetime_value(result: DateTimeResult.Read(value)) (output: DateTime) pure { + output = value +} + +datetime_value(result: DateTimeResult.Failed(message)) (output: DateTime) pure { + fail(DateTimeResult.Failed(message)) +} + +public type DateResult { + Read(value: Date) + error Failed(message: Text) +} + +date_value(result: DateResult.Read(value)) (output: Date) pure { + output = value +} + +date_value(result: DateResult.Failed(message)) (output: Date) pure { + fail(DateResult.Failed(message)) +} + +public type TimeResult { + Read(value: Time) + error Failed(message: Text) +} + +time_value(result: TimeResult.Read(value)) (output: Time) pure { + output = value +} + +time_value(result: TimeResult.Failed(message)) (output: Time) pure { + fail(TimeResult.Failed(message)) +} + +public type DurationResult { + Read(value: Duration) + error Failed(message: Text) +} + +duration_value(result: DurationResult.Read(value)) (output: Duration) pure { + output = value +} + +duration_value(result: DurationResult.Failed(message)) (output: Duration) pure { + fail(DurationResult.Failed(message)) +} + +public type TextResult { + Read(value: Text) + error Failed(message: Text) +} + +text_value(result: TextResult.Read(value)) (output: Text) pure { + output = value +} + +text_value(result: TextResult.Failed(message)) (output: Text) pure { + fail(TextResult.Failed(message)) +} + +public now() (output: DateTime) { + output = datetime_value(dever.system.time_now()) +} + +public parse_datetime(text: Text) (output: DateTime) pure { + output = datetime_value(dever.system.time_parse_datetime(text)) +} + +public format_datetime(value: DateTime) (output: Text) pure { + output = text_value(dever.system.time_format_datetime(value)) +} + +public parse_date(text: Text) (output: Date) pure { + output = date_value(dever.system.time_parse_date(text)) +} + +public format_date(value: Date) (output: Text) pure { + output = text_value(dever.system.time_format_date(value)) +} + +public parse_time(text: Text) (output: Time) pure { + output = time_value(dever.system.time_parse_time(text)) +} + +public format_time(value: Time) (output: Text) pure { + output = text_value(dever.system.time_format_time(value)) +} + +public add(value: DateTime, duration: Duration) (output: DateTime) pure { + output = datetime_value(dever.system.time_add(value, duration)) +} + +public subtract(value: DateTime, duration: Duration) (output: DateTime) pure { + output = datetime_value(dever.system.time_subtract(value, duration)) +} + +public difference(later: DateTime, earlier: DateTime) (output: Duration) pure { + output = duration_value(dever.system.time_difference(later, earlier)) +} + +public duration(millis: Int) (output: Duration) pure { + output = dever.system.time_duration(millis) +} diff --git a/library/dever/tls.dever b/library/dever/tls.dever new file mode 100644 index 0000000..100489c --- /dev/null +++ b/library/dever/tls.dever @@ -0,0 +1,39 @@ +public type ClientResult { + Ready(config: dever.system.ClientTls) + error Failed(message: Text) +} + +public type ServerResult { + Ready(config: dever.system.ServerTls) + error Failed(message: Text) +} + +# 内置公共 CA；校验证书链、有效期和连接目标名称。 +public system() (config: dever.system.ClientTls) { + config = dever.system.tls_system() +} + +# 仅信任给定 PEM CA；不提供跳过校验的模式。 +public client(ca_pem: Bytes) (config: dever.system.ClientTls) { + config = client_config(dever.system.tls_client(ca_pem)) +} + +client_config(result: ClientResult.Ready(config)) (value: dever.system.ClientTls) { + value = config +} + +client_config(result: ClientResult.Failed(message)) (value: dever.system.ClientTls) { + fail(ClientResult.Failed(message)) +} + +public server(cert_pem: Bytes, key_pem: Bytes) (config: dever.system.ServerTls) { + config = server_config(dever.system.tls_server(cert_pem, key_pem)) +} + +server_config(result: ServerResult.Ready(config)) (value: dever.system.ServerTls) { + value = config +} + +server_config(result: ServerResult.Failed(message)) (value: dever.system.ServerTls) { + fail(ServerResult.Failed(message)) +} diff --git a/library/dever/websocket.dever b/library/dever/websocket.dever new file mode 100644 index 0000000..4bf5ae8 --- /dev/null +++ b/library/dever/websocket.dever @@ -0,0 +1,111 @@ +public open(url: Text, tls: dever.system.ClientTls, limits: Limits) ( + socket: dever.system.WebSocket +) { + socket = connected(dever.system.ws_open(url, tls, limits)) +} + +public type Limits { + message_bytes: Int + idle_ms: Int + write_ms: Int +} + +public type Message { + Text(value: Text) + Binary(value: Bytes) + Ping(value: Bytes) + Pong(value: Bytes) +} + +public type ConnectResult { + Connected(socket: dever.system.WebSocket) + error Failed(message: Text) +} + +public type ReceiveState { + Read(message: Message) + End +} + +public type ReceiveResult { + Done(state: ReceiveState) + error Failed(message: Text) +} + +type SystemReceiveResult { + Read(message: Message) + End + error Failed(message: Text) +} + +public type MessageEvent { + Read(message: Message) + error Failed(message: Text) +} + +public default_limits() (result: Limits) { + result = Limits { + message_bytes = 1048576 + idle_ms = 60000 + write_ms = 5000 + } +} + +public accept(reply: dever.system.HttpReply, limits: Limits) (socket: dever.system.WebSocket) { + socket = connected(dever.system.ws_accept(reply, limits)) +} + +public connect(address: Text, port: Int, target: Text, limits: Limits) ( + socket: dever.system.WebSocket +) { + socket = connected(dever.system.ws_connect(address, port, target, limits)) +} + +connected(result: ConnectResult.Connected(socket)) (value: dever.system.WebSocket) { + value = socket +} + +connected(result: ConnectResult.Failed(message)) (value: dever.system.WebSocket) { + fail(ConnectResult.Failed(message)) +} + +public send(socket: dever.system.WebSocket, message: Message) () { + wrote(dever.system.ws_send(socket, message)) +} + +wrote(result: dever.io.WriteResult.Done) () {} + +wrote(result: dever.io.WriteResult.Failed(message)) () { + fail(dever.io.WriteResult.Failed(message)) +} + +public receive(socket: dever.system.WebSocket) (state: ReceiveState) { + state = receive_state(dever.system.ws_receive(socket)) +} + +receive_state(result: SystemReceiveResult.Read(message)) (state: ReceiveState) { + state = ReceiveState.Read(message) +} + +receive_state(result: SystemReceiveResult.End) (state: ReceiveState) { + state = ReceiveState.End +} + +receive_state(result: SystemReceiveResult.Failed(message)) (state: ReceiveState) { + fail(ReceiveResult.Failed(message)) +} + +# 拉取时处理协议控制帧；自然关闭结束序列，协议故障产生最后一个 Failed。 +public messages(socket: dever.system.WebSocket) (result: AsyncStream<MessageEvent>) { + result = dever.system.ws_messages(socket) +} + +public close(socket: dever.system.WebSocket, code: Int, reason: Text) () { + closed(dever.system.ws_close(socket, code, reason)) +} + +closed(result: dever.io.CloseResult.Closed) () {} + +closed(result: dever.io.CloseResult.Failed(message)) () { + fail(dever.io.CloseResult.Failed(message)) +} diff --git a/library/float.dever b/library/float.dever new file mode 100644 index 0000000..de25ddc --- /dev/null +++ b/library/float.dever @@ -0,0 +1,11 @@ +public from_int(value: Int) (result: Float) { + result = dever.system.float_from_int(value) +} + +public parse(value: Text) (result: Float?) { + result = dever.system.float_parse(value) +} + +public to_text(value: Float) (result: Text) { + result = dever.system.float_to_text(value) +} diff --git a/library/int.dever b/library/int.dever new file mode 100644 index 0000000..f65fe76 --- /dev/null +++ b/library/int.dever @@ -0,0 +1,7 @@ +public parse(value: Text) (result: Int?) { + result = dever.system.int_parse(value) +} + +public to_text(value: Int) (result: Text) { + result = dever.system.int_to_text(value) +} diff --git a/library/math.dever b/library/math.dever new file mode 100644 index 0000000..e89b959 --- /dev/null +++ b/library/math.dever @@ -0,0 +1,31 @@ +public sqrt(value: Float) (result: Float) { + result = dever.system.float_sqrt(value) +} + +public sin(value: Float) (result: Float) { + result = dever.system.float_sin(value) +} + +public cos(value: Float) (result: Float) { + result = dever.system.float_cos(value) +} + +public log(value: Float) (result: Float) { + result = dever.system.float_log(value) +} + +public pow(value: Float, exponent: Float) (result: Float) { + result = dever.system.float_pow(value, exponent) +} + +public is_nan(value: Float) (result: Bool) { + result = dever.system.float_is_nan(value) +} + +public is_finite(value: Float) (result: Bool) { + result = dever.system.float_is_finite(value) +} + +public is_infinite(value: Float) (result: Bool) { + result = dever.system.float_is_infinite(value) +} diff --git a/library/text.dever b/library/text.dever new file mode 100644 index 0000000..8cc1f2f --- /dev/null +++ b/library/text.dever @@ -0,0 +1,89 @@ +public trim(value: Text) (result: Text) { + result = dever.system.text_trim(value) +} + +public lower(value: Text) (result: Text) { + result = dever.system.text_lower(value) +} + +public upper(value: Text) (result: Text) { + result = dever.system.text_upper(value) +} + +public codepoint(value: Text) (result: Int?) { + result = dever.system.text_codepoint(value) +} + +public from_codepoint(value: Int) (result: Text?) { + result = dever.system.text_from_codepoint(value) +} + +public at(value: Text, index: Int) (result: Text?) { + result = dever.system.text_at(value, index) +} + +public slice(value: Text, start: Int, end: Int) (result: Text?) { + result = dever.system.text_slice(value, start, end) +} + +public index_of(value: Text, pattern: Text) (result: Int?) { + result = dever.system.text_index_of(value, pattern) +} + +public contains(value: Text, pattern: Text) (result: Bool) { + result = dever.system.text_contains(value, pattern) +} + +public starts_with(value: Text, prefix: Text) (result: Bool) { + result = dever.system.text_starts_with(value, prefix) +} + +public ends_with(value: Text, suffix: Text) (result: Bool) { + result = dever.system.text_ends_with(value, suffix) +} + +public split(value: Text, separator: Text) (result: List<Text>) { + result = dever.system.text_split(value, separator) +} + +public replace(value: Text, pattern: Text, replacement: Text) (result: Text) { + result = dever.system.text_replace(value, pattern, replacement) +} + +public is_empty(value: Text) (result: Bool) { + result = length(value) == 0 +} + +type JoinState { + text: Text + separator: Text + first: Bool +} + +public join(values: List<Text>, separator: Text) (result: Text) { + state = reduce( + join_piece, + values, + JoinState { + text = "" + separator = separator + first = true + } + ) + result = state.text +} + +join_piece(value: Text, state: JoinState) (result: JoinState) { + result = join_next(state.first, value, state) +} + +join_next(first: true, value: Text, state: JoinState) (result: JoinState) { + result = state + result.text = value + result.first = false +} + +join_next(first: false, value: Text, state: JoinState) (result: JoinState) { + result = state + result.text = state.text + state.separator + value +} diff --git a/library/uuid.dever b/library/uuid.dever new file mode 100644 index 0000000..3f1d00f --- /dev/null +++ b/library/uuid.dever @@ -0,0 +1,7 @@ +public parse(value: Text) (result: Uuid?) { + result = dever.system.uuid_parse(value) +} + +public to_text(value: Uuid) (result: Text) { + result = dever.system.uuid_to_text(value) +} diff --git a/sdk/README.md b/sdk/README.md new file mode 100644 index 0000000..0edc1a6 --- /dev/null +++ b/sdk/README.md @@ -0,0 +1,27 @@ +# External Worker SDKs + +Compiler authors: the private [native release workflow](native-release.md) builds and signs Dever runtime packs. It is separate from these external-language Worker SDKs and from application configuration. + +These are stdio Worker libraries for Python, JavaScript (Node), and Go. The compiler remains the sole owner of Port schemas. Each checked external Adapter exposes `ExternalWorkerContract::sdk_manifest()`, including the exact identity and recursive setting, input, output and declared-error types. The managed Python and JavaScript runners package this manifest and bind each Port operation to a same-named function exported by the Adapter entry; JavaScript accepts ESM named exports or CommonJS `module.exports` properties. Application code does not maintain a second registration table. Python uses `Worker.from_manifest(manifest, handlers)`, JavaScript uses `Worker.fromManifest(manifest, handlers)`, and Go uses `NewFromManifest(manifest)` followed by `RegisterHandlers(handlers)`. These paths decode inputs and validate results and declared business-error payloads from the same compiler-generated contract. The SDK checks the identity during `hello` and never adopts a schema supplied by the supervisor. + +The lower-level `Contract`/`Worker` registration API remains available for protocol fixtures. The setting decoder runs before `ready`; it receives only the `hello.setting` value. Handlers return JSON-compatible values or a declared `BusinessError`. Python handlers are async; JavaScript handlers receive an `AbortSignal`; Go handlers receive a `context.Context`. A cooperative cancellation receives one `dever.cancelled` protocol error and discards its handler result. If a handler ignores cancellation for one second, the Worker fails and admits no further calls. The supervisor owns the Worker process and its stdin lifetime; an in-process Go `Serve` caller must close or unblock its own `io.Reader` when returning. + +The three `example` entries use the same fixture identity and contract to demonstrate the lower-level API. `fixture-schema-v1` and `test/external-sdk/typed_contract.json` are test data, not production Port hashes. Worker stdout is reserved for 4-byte big-endian length-prefixed UTF-8 JSON frames. Diagnostics go to stderr. The libraries use no external packages and do not read environment variables or `PATH`. A managed Worker entry reads only its explicitly packaged manifest path. Managed Python packs require fixed `-I -S -B` flags; the runner imports its packaged SDK before exposing Adapter source and dependencies on `sys.path`. + +Run the opt-in host SDK check with explicit absolute runtime paths, for example: + +```text +/usr/bin/python3 -B test/external-sdk/check.py --python /usr/bin/python3 --node /absolute/path/to/node --go /absolute/path/to/go +``` + +Omitted runtime paths report `unavailable`; the script never discovers language runtimes through `PATH`. It builds the Go fixtures in a temporary offline cache. This check exercises both the lower-level and compiler-manifest-shaped typed paths on host interpreters. Managed runtime packaging and execution on a machine without host language runtimes require separate acceptance. + +Managed dependencies use `dever-lock-v5`. Python extras and selected wheel METADATA, npm nested instances/peers/optional/bundled dependencies, and authenticated Go module evidence are part of the exact lock. Native Python wheels install into the packaged interpreter prefix and must satisfy its signed tags, extension ABI and loader closure. + +Cross builds select the application platform explicitly: `dever lib update <project> --target linux-aarch64`, followed by `dever build <project> --target linux-aarch64 --output <new-file>`. Worker source and Port contracts stay unchanged. Locks, interpreters, native dependencies and sandbox assets must match that target. A Go runtime pack additionally declares `build.host` for its compiler/linker/analyzer; `build.goos/goarch` and standard-library object headers describe the output architecture. The tools run with a cleared environment and must have the correct target baked in. Missing target assets are errors, with no host runtime fallback. `run` and ordinary tests remain host operations. + +Only explicit Lib preparation may build Python sdists or run npm registry installation hooks. The shared build owner uses signed compiler/header/sysroot/frontend packs in a Linux sandbox with no network, immutable inputs and owned writable scratch space. Python PEP 517 build requirements have their own exact lock; each hook has a fresh process, and the parent retains the metadata snapshot used to validate the final wheel. npm uses its packaged lifecycle runner for `preinstall`, `install` and `postinstall`, including dependency bins and node-gyp. Missing tools do not fall back to host `PATH`. + +Build receipts bind sources, target runtimes, tools, frontend, dependency graph and final output hashes. Worker preparation replays those outputs offline without running installation hooks. Every npm `.node` output is classified by target/loader checks and an isolated Node load probe; only admitted addons enter the immutable Worker tree, including trees used by threads or child processes. Rejected platform/ABI variants remain in the locked archive. Runtime native dependencies must come from that tree or the signed OS library set, never the compiler sysroot. These rules do not change the Port protocol or require an application-side registration layer. + +A generated addon that fails admission fails the build; a pure JavaScript fallback is not evidence of successful native compilation. Build subprocesses have a five-minute deadline and separate 16 MiB stdout/stderr limits. Scratch usage is checked every 250 ms against 512 MiB and 65,536 entries, with owned-process cleanup on failure. This periodic check is not a kernel disk quota and may observe overshoot between checks. The final Worker installation remains limited to 256 MiB and 65,536 entries. diff --git a/sdk/apparmor/dever-sandbox b/sdk/apparmor/dever-sandbox new file mode 100644 index 0000000..cee6db7 --- /dev/null +++ b/sdk/apparmor/dever-sandbox @@ -0,0 +1,45 @@ +# Based on AppArmor 4.0 extras/bwrap-userns-restrict (GPL-2.0-only). +# Only signed static helpers published by the root-owned Dever installer attach. +# The exec transition is stacked under no_new_privs; child capabilities are denied. +# On restricted hosts, kernel.apparmor_restrict_unprivileged_unconfined=1 is +# mandatory BEFORE installation/loading: attachment paths do not constrain an +# explicit change_profile by an unconfined caller. Never enable this policy on +# such a host with that flag unset. Rollback removes this policy before restoring +# the previous strict flag. Dever never changes host sysctls automatically. +abi <abi/4.0>, + +@{DEVER_SANDBOX_HASH} = [0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f] + +profile dever-bwrap /opt/dever/sandbox/@{DEVER_SANDBOX_HASH}/bwrap flags=(attach_disconnected,mediate_deleted) { + allow capability, + allow file rwlkm /{**,}, + allow network, + allow unix, + allow ptrace, + allow signal, + allow mqueue, + allow io_uring, + allow userns, + allow mount, + allow umount, + allow pivot_root, + allow dbus, + allow px /** -> dever-bwrap//&dever-bwrap-child, +} + +profile dever-bwrap-child flags=(attach_disconnected,mediate_deleted) { + allow file rwlkm /{**,}, + allow network, + allow unix, + allow ptrace, + allow signal, + allow mqueue, + allow io_uring, + allow userns, + allow mount, + allow umount, + allow pivot_root, + allow dbus, + allow pix /** -> &dever-bwrap-child, + audit deny capability, +} diff --git a/sdk/go/component.go b/sdk/go/component.go new file mode 100644 index 0000000..add4724 --- /dev/null +++ b/sdk/go/component.go @@ -0,0 +1,699 @@ +// Package component implements the Dever external Worker process protocol. +// The compiler owns schema and typed Port codecs; callers bind its identity here. +package component + +import ( + "bytes" + "context" + "encoding/binary" + "encoding/json" + "errors" + "io" + "math" + "regexp" + "sort" + "strconv" + "strings" + "time" + "unicode/utf8" +) + +const ( + Version = "dever-component-1" + maxBytes = 16 * 1024 * 1024 + maxDepth = 64 + maxElements = 65536 + cancelGrace = time.Second +) + +var ( + decimalPattern = regexp.MustCompile(`^[+-]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][+-]?[0-9]+)?$`) + uuidPattern = regexp.MustCompile(`(?i)^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$`) + timePattern = regexp.MustCompile(`^[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]{3})?$`) + datetimePattern = regexp.MustCompile(`^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]{1,3})?(?:Z|[+-][0-9]{2}:[0-9]{2})$`) +) + +type Contract struct { + Port, Schema, Adapter string + Operations, Capabilities, Errors []string +} + +type TypeSchema struct { + Type string `json:"type"` + Name string `json:"name,omitempty"` + Value *TypeSchema `json:"value,omitempty"` + Fields map[string]TypeSchema `json:"fields,omitempty"` +} + +type Manifest struct { + Port string `json:"port"` + Schema string `json:"schema"` + Adapter string `json:"adapter"` + Operations []string `json:"operations"` + Capabilities []string `json:"capabilities"` + Setting *TypeSchema `json:"setting"` + Inputs map[string]TypeSchema `json:"inputs"` + Outputs map[string]TypeSchema `json:"outputs"` + Errors map[string]TypeSchema `json:"errors"` +} + +type BusinessError struct { + Identity string + Payload any +} + +func (e BusinessError) Error() string { return e.Identity } + +type handler[S any] func(context.Context, any, S) (any, error) + +type Worker[S any] struct { + contract Contract + decodeSetting func(any) (S, error) + handlers map[string]handler[S] + inputTypes map[string]TypeSchema + outputTypes map[string]TypeSchema + errorTypes map[string]TypeSchema +} + +func NewFromManifest(manifest Manifest) (*Worker[any], error) { + if len(manifest.Inputs) != len(manifest.Operations) || len(manifest.Outputs) != len(manifest.Operations) { + return nil, errors.New("incomplete typed Worker contract") + } + for _, operation := range manifest.Operations { + if _, ok := manifest.Inputs[operation]; !ok { + return nil, errors.New("missing operation input contract") + } + if _, ok := manifest.Outputs[operation]; !ok { + return nil, errors.New("missing operation output contract") + } + } + identities := make([]string, 0, len(manifest.Errors)) + for identity := range manifest.Errors { + identities = append(identities, identity) + } + sort.Strings(identities) + worker, err := New(Contract{ + Port: manifest.Port, Schema: manifest.Schema, Adapter: manifest.Adapter, + Operations: manifest.Operations, Capabilities: manifest.Capabilities, Errors: identities, + }, func(value any) (any, error) { + if manifest.Setting == nil { + if value != nil { + return nil, errors.New("unexpected setting") + } + return nil, nil + } + return decodeTyped(value, *manifest.Setting) + }) + if err != nil { + return nil, err + } + worker.inputTypes, worker.outputTypes, worker.errorTypes = manifest.Inputs, manifest.Outputs, manifest.Errors + return worker, nil +} + +// ServeManifest binds the checked contract to compile-time selected handlers. +func ServeManifest(manifestBytes []byte, handlers map[string]func(context.Context, any, any) (any, error), input io.Reader, output io.Writer) error { + var manifest Manifest + if err := json.Unmarshal(manifestBytes, &manifest); err != nil { + return err + } + worker, err := NewFromManifest(manifest) + if err != nil { + return err + } + if err := worker.RegisterHandlers(handlers); err != nil { + return err + } + return worker.Serve(input, output) +} + +func (w *Worker[S]) RegisterHandler(operation string, run func(context.Context, any, S) (any, error)) error { + schema, ok := w.inputTypes[operation] + if !ok || run == nil { + return errors.New("operation is not in a typed Worker contract") + } + if w.handlers[operation] != nil { + return errors.New("duplicate component operation") + } + w.handlers[operation] = func(ctx context.Context, raw any, setting S) (any, error) { + payload, err := decodeTyped(raw, schema) + if err != nil { + return nil, errors.New("invalid component payload") + } + result, err := run(ctx, payload, setting) + if err != nil { + var business BusinessError + if errors.As(err, &business) { + typeSchema, declared := w.errorTypes[business.Identity] + if !declared { + return nil, errors.New("undeclared business error identity") + } + business.Payload, err = decodeTyped(business.Payload, typeSchema) + if err != nil { + return nil, errors.New("invalid business error payload") + } + return nil, business + } + return nil, err + } + return decodeTyped(result, w.outputTypes[operation]) + } + return nil +} + +func (w *Worker[S]) RegisterHandlers(handlers map[string]func(context.Context, any, S) (any, error)) error { + if len(handlers) != len(w.contract.Operations) { + return errors.New("component operations are not fully registered") + } + for _, operation := range w.contract.Operations { + if err := w.RegisterHandler(operation, handlers[operation]); err != nil { + return err + } + } + return nil +} + +func decodeTyped(value any, schema TypeSchema) (any, error) { + switch schema.Type { + case "nullable": + if value == nil { + return nil, nil + } + if schema.Value == nil { + return nil, errors.New("invalid nullable contract") + } + return decodeTyped(value, *schema.Value) + case "list": + values, ok := value.([]any) + if !ok || schema.Value == nil { + return nil, errors.New("expected list") + } + result := make([]any, len(values)) + for index, item := range values { + decoded, err := decodeTyped(item, *schema.Value) + if err != nil { + return nil, err + } + result[index] = decoded + } + return result, nil + case "record": + object, ok := value.(map[string]any) + if !ok { + return nil, errors.New("expected record") + } + for name := range object { + if _, ok := schema.Fields[name]; !ok { + return nil, errors.New("unknown record field") + } + } + result := make(map[string]any, len(schema.Fields)) + for name, field := range schema.Fields { + entry, exists := object[name] + if !exists && field.Type != "nullable" { + return nil, errors.New("missing record field") + } + decoded, err := decodeTyped(entry, field) + if err != nil { + return nil, err + } + result[name] = decoded + } + return result, nil + case "int64", "duration", "model_id": + switch number := value.(type) { + case json.Number: + if _, err := number.Int64(); err == nil { + return number, nil + } + case int64: + return number, nil + case int: + return int64(number), nil + } + case "float64": + switch number := value.(type) { + case json.Number: + parsed, err := number.Float64() + if err == nil && !math.IsInf(parsed, 0) && !math.IsNaN(parsed) { + return number, nil + } + case float64: + if !math.IsInf(number, 0) && !math.IsNaN(number) { + return number, nil + } + } + case "bool": + if boolean, ok := value.(bool); ok { + return boolean, nil + } + case "json": + return value, nil + case "text", "id", "secret", "decimal", "uuid", "datetime", "date", "time": + if text, ok := value.(string); ok { + switch schema.Type { + case "decimal": + if !decimalPattern.MatchString(text) { + return nil, errors.New("invalid decimal") + } + case "uuid": + if !uuidPattern.MatchString(text) { + return nil, errors.New("invalid UUID") + } + case "date": + if _, err := time.Parse("2006-01-02", text); err != nil { + return nil, err + } + case "time": + if !timePattern.MatchString(text) { + return nil, errors.New("invalid Time") + } + layout := "15:04:05" + if len(text) == 12 { + layout = "15:04:05.000" + } + if _, err := time.Parse(layout, text); err != nil { + return nil, err + } + case "datetime": + if !datetimePattern.MatchString(text) { + return nil, errors.New("invalid DateTime") + } + parsed, err := time.Parse(time.RFC3339Nano, text) + if err != nil || parsed.Nanosecond()%1_000_000 != 0 { + return nil, errors.New("invalid DateTime") + } + } + return text, nil + } + } + return nil, errors.New("invalid typed Worker value") +} + +func New[S any](contract Contract, decodeSetting func(any) (S, error)) (*Worker[S], error) { + if contract.Port == "" || contract.Schema == "" || contract.Adapter == "" || len(contract.Operations) == 0 || decodeSetting == nil || !unique(contract.Operations) || !unique(contract.Capabilities) || !unique(contract.Errors) { + return nil, errors.New("incomplete component contract") + } + if contract.Capabilities == nil { + contract.Capabilities = []string{} + } + if contract.Errors == nil { + contract.Errors = []string{} + } + contract.Operations = append([]string(nil), contract.Operations...) + contract.Capabilities = append([]string{}, contract.Capabilities...) + contract.Errors = append([]string{}, contract.Errors...) + return &Worker[S]{contract: contract, decodeSetting: decodeSetting, handlers: make(map[string]handler[S])}, nil +} + +func unique(values []string) bool { + seen := make(map[string]bool, len(values)) + for _, value := range values { + if value == "" || seen[value] { + return false + } + seen[value] = true + } + return true +} + +// RegisterTyped binds a compiler-declared operation to its application codec. +func RegisterTyped[S, P any](worker *Worker[S], operation string, decode func(any) (P, error), run func(context.Context, P, S) (any, error)) error { + if !contains(worker.contract.Operations, operation) || worker.handlers[operation] != nil || decode == nil || run == nil { + return errors.New("unknown or duplicate component operation") + } + worker.handlers[operation] = func(ctx context.Context, raw any, setting S) (any, error) { + payload, err := decode(raw) + if err != nil { + return nil, errors.New("invalid component payload") + } + return run(ctx, payload, setting) + } + return nil +} + +func contains(values []string, target string) bool { + for _, value := range values { + if value == target { + return true + } + } + return false +} + +type event struct { + message map[string]any + err error +} + +type result struct { + id int64 + payload any + err error +} + +func (w *Worker[S]) Serve(input io.Reader, output io.Writer) error { + if len(w.handlers) != len(w.contract.Operations) { + return errors.New("component operations are not fully registered") + } + messages := make(chan event, 1) + done := make(chan struct{}) + defer close(done) + go func() { + for { + message, err := readFrame(input) + select { + case messages <- event{message, err}: + case <-done: + return + } + if err != nil { + return + } + } + }() + first := <-messages + if first.err != nil { + return first.err + } + setting, err := w.handshake(first.message, output) + if err != nil { + return err + } + results := make(chan result, 1) + var cancel context.CancelFunc + var active int64 + var activeDone chan struct{} + defer func() { + if active != 0 { + _ = stopHandler(cancel, activeDone) + } + }() + nextID := int64(1) + lastCancelled := false + for { + select { + case received := <-messages: + if received.err != nil { + return received.err + } + message := received.message + kind, ok := message["kind"].(string) + if !ok { + return errors.New("invalid component kind") + } + switch kind { + case "health": + if !exact(message, "kind", "id") || requestID(message["id"]) != 0 { + return errors.New("invalid component health") + } + if err := writeFrame(output, message); err != nil { + return err + } + case "call": + if !exact(message, "kind", "id", "operation", "payload") { + return errors.New("invalid component call") + } + id := requestID(message["id"]) + operation, ok := message["operation"].(string) + run := w.handlers[operation] + if id != nextID || active != 0 || !ok || run == nil { + return errors.New("invalid component call") + } + nextID++ + ctx, stop := context.WithCancel(context.Background()) + cancel, active = stop, id + activeDone = make(chan struct{}) + payload := message["payload"] + go func(finished chan struct{}) { + defer close(finished) + value, err := run(ctx, payload, setting) + select { + case results <- result{id, value, err}: + case <-ctx.Done(): + case <-done: + } + }(activeDone) + case "cancel": + if !exact(message, "kind", "id") { + return errors.New("invalid component cancel") + } + id := requestID(message["id"]) + if id <= 0 { + return errors.New("invalid component cancel") + } + if id == active { + if err := stopHandler(cancel, activeDone); err != nil { + active = 0 + activeDone = nil + return err + } + active = 0 + activeDone = nil + lastCancelled = true + if err := writeFrame(output, map[string]any{"kind": "error", "id": id, "error": "dever.cancelled", "payload": nil}); err != nil { + return err + } + } else if id != nextID-1 || lastCancelled { + return errors.New("unknown component cancel id") + } + case "shutdown": + if !exact(message, "kind") { + return errors.New("invalid component shutdown") + } + if active != 0 { + if err := stopHandler(cancel, activeDone); err != nil { + active = 0 + activeDone = nil + return err + } + active = 0 + activeDone = nil + } + return writeFrame(output, map[string]any{"kind": "shutdown"}) + default: + return errors.New("unknown component message") + } + case completed := <-results: + if completed.id != active { + continue + } + cancel() + active = 0 + activeDone = nil + lastCancelled = false + if completed.err == nil { + if err := writeFrame(output, map[string]any{"kind": "result", "id": completed.id, "payload": completed.payload}); err != nil { + return err + } + continue + } + var business BusinessError + if !errors.As(completed.err, &business) || !contains(w.contract.Errors, business.Identity) { + return errors.New("component handler failed") + } + if err := writeFrame(output, map[string]any{"kind": "error", "id": completed.id, "error": business.Identity, "payload": business.Payload}); err != nil { + return err + } + } + } +} + +func stopHandler(cancel context.CancelFunc, finished <-chan struct{}) error { + cancel() + timer := time.NewTimer(cancelGrace) + defer timer.Stop() + select { + case <-finished: + return nil + case <-timer.C: + return errors.New("component handler did not stop after cancellation") + } +} + +func (w *Worker[S]) handshake(hello map[string]any, output io.Writer) (S, error) { + var zero S + if !exact(hello, "kind", "version", "port", "schema", "adapter", "capabilities", "operations", "setting") || hello["kind"] != "hello" || hello["version"] != Version || hello["port"] != w.contract.Port || hello["schema"] != w.contract.Schema || hello["adapter"] != w.contract.Adapter || !sameNames(hello["capabilities"], w.contract.Capabilities) || !sameNames(hello["operations"], w.contract.Operations) { + return zero, errors.New("component handshake does not match registered contract") + } + setting, err := w.decodeSetting(hello["setting"]) + if err != nil { + return zero, errors.New("invalid component setting") + } + if err := writeFrame(output, map[string]any{"kind": "ready", "version": Version, "port": w.contract.Port, "schema": w.contract.Schema, "adapter": w.contract.Adapter, "capabilities": w.contract.Capabilities, "operations": w.contract.Operations}); err != nil { + return zero, err + } + return setting, nil +} + +func sameNames(raw any, expected []string) bool { + values, ok := raw.([]any) + if !ok || len(values) != len(expected) { + return false + } + for i, value := range values { + if value != expected[i] { + return false + } + } + return true +} + +func exact(message map[string]any, names ...string) bool { + if len(message) != len(names) { + return false + } + for _, name := range names { + if _, ok := message[name]; !ok { + return false + } + } + return true +} + +func requestID(value any) int64 { + number, ok := value.(json.Number) + if !ok { + return -1 + } + id, err := number.Int64() + if err != nil || id < 0 { + return -1 + } + return id +} + +func readFrame(input io.Reader) (map[string]any, error) { + var header [4]byte + if _, err := io.ReadFull(input, header[:]); err != nil { + return nil, errors.New("incomplete component frame") + } + size := binary.BigEndian.Uint32(header[:]) + if size == 0 || size > maxBytes { + return nil, errors.New("component frame exceeds byte limit") + } + body := make([]byte, size) + if _, err := io.ReadFull(input, body); err != nil { + return nil, errors.New("incomplete component frame") + } + return parseJSON(body) +} + +func writeFrame(output io.Writer, message map[string]any) error { + body, err := json.Marshal(message) + if err != nil { + return errors.New("invalid handler response") + } + if len(body) == 0 || len(body) > maxBytes { + return errors.New("component frame exceeds byte limit") + } + if _, err := parseJSON(body); err != nil { + return err + } + var header [4]byte + binary.BigEndian.PutUint32(header[:], uint32(len(body))) + if err := writeAll(output, header[:]); err != nil { + return err + } + return writeAll(output, body) +} + +func writeAll(output io.Writer, data []byte) error { + for len(data) > 0 { + count, err := output.Write(data) + if err != nil { + return err + } + if count == 0 { + return io.ErrShortWrite + } + data = data[count:] + } + return nil +} + +func parseJSON(body []byte) (map[string]any, error) { + if !utf8.Valid(body) { + return nil, errors.New("component frame is not UTF-8") + } + decoder := json.NewDecoder(bytes.NewReader(body)) + decoder.UseNumber() + remaining := maxElements + value, err := parseValue(decoder, 0, &remaining) + if err != nil { + return nil, err + } + if _, err := decoder.Token(); err != io.EOF { + return nil, errors.New("invalid component JSON") + } + object, ok := value.(map[string]any) + if !ok { + return nil, errors.New("component message must be an object") + } + return object, nil +} + +func parseValue(decoder *json.Decoder, depth int, remaining *int) (any, error) { + if depth > maxDepth || *remaining <= 0 { + return nil, errors.New("wire JSON exceeds limit") + } + *remaining-- + token, err := decoder.Token() + if err != nil { + return nil, errors.New("invalid component JSON") + } + delim, ok := token.(json.Delim) + if !ok { + if number, isNumber := token.(json.Number); isNumber && strings.ContainsAny(string(number), ".eE") { + value, err := strconv.ParseFloat(string(number), 64) + if err != nil || math.IsInf(value, 0) || math.IsNaN(value) { + return nil, errors.New("non-finite wire Float") + } + } + return token, nil + } + if depth == maxDepth { + return nil, errors.New("wire JSON exceeds depth limit") + } + switch delim { + case '{': + object := make(map[string]any) + for decoder.More() { + key, err := decoder.Token() + if err != nil { + return nil, errors.New("invalid component JSON") + } + name, ok := key.(string) + if !ok { + return nil, errors.New("invalid component JSON") + } + if _, exists := object[name]; exists { + return nil, errors.New("duplicate JSON field") + } + value, err := parseValue(decoder, depth+1, remaining) + if err != nil { + return nil, err + } + object[name] = value + } + if end, err := decoder.Token(); err != nil || end != json.Delim('}') { + return nil, errors.New("invalid component JSON") + } + return object, nil + case '[': + array := make([]any, 0) + for decoder.More() { + value, err := parseValue(decoder, depth+1, remaining) + if err != nil { + return nil, err + } + array = append(array, value) + } + if end, err := decoder.Token(); err != nil || end != json.Delim(']') { + return nil, errors.New("invalid component JSON") + } + return array, nil + default: + return nil, errors.New("invalid component JSON") + } +} diff --git a/sdk/go/example/main.go b/sdk/go/example/main.go new file mode 100644 index 0000000..4c81c5f --- /dev/null +++ b/sdk/go/example/main.go @@ -0,0 +1,102 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "time" + + component "dever-component" +) + +type payload struct { + Text string + DelayMS int64 +} + +func setting(raw any) (string, error) { + object, ok := raw.(map[string]any) + if !ok || len(object) != 1 { + return "", errors.New("invalid setting") + } + value, ok := object["prefix"].(string) + if !ok { + return "", errors.New("invalid setting") + } + return value, nil +} + +func decode(raw any) (payload, error) { + object, ok := raw.(map[string]any) + if !ok || len(object) != 2 { + return payload{}, errors.New("invalid payload") + } + text, ok := object["text"].(string) + if !ok { + return payload{}, errors.New("invalid payload") + } + delay, ok := object["delay_ms"].(interface{ Int64() (int64, error) }) + if !ok { + return payload{}, errors.New("invalid payload") + } + milliseconds, err := delay.Int64() + if err != nil || milliseconds < 0 || milliseconds > 10000 { + return payload{}, errors.New("invalid payload") + } + return payload{Text: text, DelayMS: milliseconds}, nil +} + +func main() { + worker, err := component.New(component.Contract{ + Port: "example.Text", Schema: "fixture-schema-v1", Adapter: "example.TextAdapter", + Operations: []string{"text.render", "text.fail", "number.echo", "text.stubborn"}, Errors: []string{"example.rejected"}, + }, setting) + if err == nil { + err = component.RegisterTyped(worker, "text.render", decode, func(ctx context.Context, value payload, prefix string) (any, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(time.Duration(value.DelayMS) * time.Millisecond): + return map[string]any{"value": prefix + value.Text}, nil + } + }) + } + if err == nil { + err = component.RegisterTyped(worker, "text.fail", decode, func(context.Context, payload, string) (any, error) { + return nil, component.BusinessError{Identity: "example.rejected", Payload: map[string]any{"reason": "rejected"}} + }) + } + if err == nil { + err = component.RegisterTyped(worker, "number.echo", func(raw any) (json.Number, error) { + object, ok := raw.(map[string]any) + if !ok || len(object) != 1 { + return "", errors.New("invalid number") + } + number, ok := object["number"].(json.Number) + if !ok { + return "", errors.New("invalid number") + } + if _, err := number.Int64(); err != nil { + return "", errors.New("invalid number") + } + return number, nil + }, func(_ context.Context, value json.Number, _ string) (any, error) { + return map[string]any{"number": value}, nil + }) + } + if err == nil { + err = component.RegisterTyped(worker, "text.stubborn", decode, func(_ context.Context, value payload, prefix string) (any, error) { + time.Sleep(5 * time.Second) + return map[string]any{"value": prefix + value.Text}, nil + }) + } + if err == nil { + err = worker.Serve(os.Stdin, os.Stdout) + } + if err != nil { + fmt.Fprintln(os.Stderr, "component worker:", err) + os.Exit(1) + } +} diff --git a/sdk/go/go.mod b/sdk/go/go.mod new file mode 100644 index 0000000..e4f2d0d --- /dev/null +++ b/sdk/go/go.mod @@ -0,0 +1,3 @@ +module dever-component + +go 1.21 diff --git a/sdk/javascript/dever_component.mjs b/sdk/javascript/dever_component.mjs new file mode 100644 index 0000000..b35c187 --- /dev/null +++ b/sdk/javascript/dever_component.mjs @@ -0,0 +1,368 @@ +const VERSION = 'dever-component-1'; +const MAX_BYTES = 16 * 1024 * 1024; +const MAX_DEPTH = 64; +const MAX_ELEMENTS = 65536; +const CANCEL_GRACE_MS = 1000; +const decoder = new TextDecoder('utf-8', { fatal: true }); + +export class ProtocolError extends Error {} + +export class BusinessError extends Error { + constructor(identity, payload = null) { + super(identity); + this.identity = identity; + this.payload = payload; + } +} + +// JSON.parse accepts duplicate object keys; this parser enforces the runtime's +// duplicate, depth and value budgets before returning ordinary JS values. +function parseJSON(text) { + let offset = 0; + let remaining = MAX_ELEMENTS; + const fail = () => { throw new ProtocolError('invalid component JSON'); }; + const space = () => { while (offset < text.length && /[ \t\r\n]/.test(text[offset])) offset++; }; + function string() { + const start = offset++; + while (offset < text.length) { + const char = text[offset++]; + if (char === '"') return JSON.parse(text.slice(start, offset)); + if (char === '\\') offset++; + else if (char.charCodeAt(0) < 32) fail(); + } + fail(); + } + function value(depth) { + if (depth > MAX_DEPTH || --remaining < 0) throw new ProtocolError('wire JSON exceeds limit'); + space(); + const char = text[offset]; + if (char === '"') return string(); + if (char === '[' || char === '{') { + if (depth === MAX_DEPTH) throw new ProtocolError('wire JSON exceeds depth limit'); + offset++; + const array = char === '['; + const result = array ? [] : {}; + const seen = new Set(); + space(); + if (text[offset] === (array ? ']' : '}')) { offset++; return result; } + for (;;) { + if (array) result.push(value(depth + 1)); + else { + if (text[offset] !== '"') fail(); + const key = string(); + if (seen.has(key)) throw new ProtocolError('duplicate JSON field'); + seen.add(key); + space(); + if (text[offset++] !== ':') fail(); + Object.defineProperty(result, key, { value: value(depth + 1), enumerable: true, writable: true, configurable: true }); + } + space(); + const end = array ? ']' : '}'; + if (text[offset] === end) { offset++; return result; } + if (text[offset++] !== ',') fail(); + space(); + } + } + const match = /^(?:true|false|null|-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?)/.exec(text.slice(offset)); + if (!match) fail(); + offset += match[0].length; + if (match[0] === 'true' || match[0] === 'false' || match[0] === 'null') return JSON.parse(match[0]); + if (match[0] === '-0') return -0; + if (/^-?(?:0|[1-9]\d*)$/.test(match[0])) { + const integer = BigInt(match[0]); + return integer <= BigInt(Number.MAX_SAFE_INTEGER) && integer >= BigInt(Number.MIN_SAFE_INTEGER) ? Number(integer) : integer; + } + const number = JSON.parse(match[0]); + if (!Number.isFinite(number)) throw new ProtocolError('invalid JSON number'); + return number; + } + const result = value(0); + space(); + if (offset !== text.length || result === null || Array.isArray(result) || typeof result !== 'object') fail(); + return result; +} + +function encodeJSON(value) { + let remaining = MAX_ELEMENTS; + function encode(current, depth) { + if (depth > MAX_DEPTH || --remaining < 0) throw new ProtocolError('wire JSON exceeds limit'); + if (current === null || typeof current === 'boolean') return String(current); + if (typeof current === 'string') return JSON.stringify(current); + if (typeof current === 'bigint') return current.toString(); + if (typeof current === 'number') { + if (!Number.isFinite(current)) throw new ProtocolError('invalid JSON number'); + return Object.is(current, -0) ? '-0' : String(current); + } + if (depth === MAX_DEPTH) throw new ProtocolError('wire JSON exceeds depth limit'); + if (Array.isArray(current)) return `[${current.map(item => encode(item, depth + 1)).join(',')}]`; + if (current && typeof current === 'object') { + return `{${Object.keys(current).map(key => `${JSON.stringify(key)}:${encode(current[key], depth + 1)}`).join(',')}}`; + } + throw new ProtocolError('invalid handler response'); + } + return encode(value, 0); +} + +function exact(message, names) { + const actual = Object.keys(message); + if (actual.length !== names.length || names.some(name => !Object.hasOwn(message, name))) { + throw new ProtocolError('component message has missing or extra fields'); + } +} + +function requestID(value) { + if (!Number.isSafeInteger(value) || value < 0) throw new ProtocolError('invalid request id'); + return value; +} + +function names(values) { + return Array.isArray(values) && values.every(value => typeof value === 'string' && value.length) && new Set(values).size === values.length; +} + +function typed(value, schema) { + switch (schema.type) { + case 'nullable': return value === null ? null : typed(value, schema.value); + case 'list': + if (!Array.isArray(value)) throw new ProtocolError('expected list'); + return value.map(item => typed(item, schema.value)); + case 'record': { + if (!value || typeof value !== 'object' || Array.isArray(value) || Object.keys(value).some(name => !Object.hasOwn(schema.fields, name))) throw new ProtocolError('invalid record fields'); + const result = {}; + for (const [name, field] of Object.entries(schema.fields)) { + if (!Object.hasOwn(value, name) && field.type !== 'nullable') throw new ProtocolError('missing required record field'); + Object.defineProperty(result, name, { value: Object.hasOwn(value, name) ? typed(value[name], field) : null, enumerable: true }); + } + return result; + } + case 'int64': case 'duration': case 'model_id': { + if ((typeof value !== 'bigint' && !Number.isSafeInteger(value)) || BigInt(value) < -(1n << 63n) || BigInt(value) >= (1n << 63n)) throw new ProtocolError('expected signed 64-bit integer'); + return value; + } + case 'float64': + if (typeof value !== 'number' || !Number.isFinite(value)) throw new ProtocolError('expected finite float'); + return value; + case 'bool': + if (typeof value !== 'boolean') throw new ProtocolError('expected boolean'); + return value; + case 'json': return value; + case 'text': case 'id': case 'secret': case 'decimal': case 'uuid': case 'datetime': case 'date': case 'time': + if (typeof value !== 'string') throw new ProtocolError('expected text'); + if (schema.type === 'decimal' && !/^[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?$/.test(value)) throw new ProtocolError('invalid decimal'); + if (schema.type === 'uuid' && !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(value)) throw new ProtocolError('invalid UUID'); + if (schema.type === 'date' && (!/^\d{4}-\d{2}-\d{2}$/.test(value) || Number.isNaN(Date.parse(`${value}T00:00:00Z`)))) throw new ProtocolError('invalid Date'); + if (schema.type === 'time' && !/^\d{2}:\d{2}:\d{2}(?:\.\d{3})?$/.test(value)) throw new ProtocolError('invalid Time'); + if (schema.type === 'datetime' && (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,3})?(?:Z|[+-]\d{2}:\d{2})$/.test(value) || Number.isNaN(Date.parse(value)))) throw new ProtocolError('invalid DateTime'); + return value; + default: throw new ProtocolError('unknown worker type'); + } +} + +async function stopHandler(current) { + current.controller.abort(); + let timeout; + try { + await Promise.race([ + current.settled, + new Promise((_, reject) => { + timeout = setTimeout(() => reject(new ProtocolError('component handler did not stop after cancellation')), CANCEL_GRACE_MS); + }), + ]); + } finally { + clearTimeout(timeout); + } +} + +class Frames { + constructor() { + this.buffer = Buffer.alloc(0); + this.queue = []; + this.queuedBytes = 0; + this.waiter = null; + this.failed = null; + process.stdin.on('data', chunk => this.receive(chunk)); + process.stdin.on('end', () => this.fail(new ProtocolError('incomplete component frame'))); + process.stdin.on('error', error => this.fail(error)); + } + + fail(error) { + this.failed = error; + if (this.waiter) { this.waiter.reject(error); this.waiter = null; } + } + + receive(chunk) { + if (this.failed) return; + this.buffer = Buffer.concat([this.buffer, chunk]); + try { + while (this.buffer.length >= 4) { + const size = this.buffer.readUInt32BE(0); + if (!size || size > MAX_BYTES) throw new ProtocolError('component frame exceeds byte limit'); + if (this.buffer.length < size + 4) break; + const message = parseJSON(decoder.decode(this.buffer.subarray(4, size + 4))); + this.buffer = this.buffer.subarray(size + 4); + if (this.waiter) { this.waiter.resolve(message); this.waiter = null; } + else { + if (this.queue.length === 4 || this.queuedBytes + size > MAX_BYTES) throw new ProtocolError('component input queue exceeds limit'); + this.queue.push({ message, size }); + this.queuedBytes += size; + } + } + } catch (error) { this.fail(error); } + } + + next() { + if (this.failed) return Promise.reject(this.failed); + if (this.queue.length) { + const next = this.queue.shift(); + this.queuedBytes -= next.size; + return Promise.resolve(next.message); + } + return new Promise((resolve, reject) => { this.waiter = { resolve, reject }; }); + } + + async send(message) { + const body = Buffer.from(encodeJSON(message)); + if (!body.length || body.length > MAX_BYTES) throw new ProtocolError('component frame exceeds byte limit'); + parseJSON(decoder.decode(body)); + const frame = Buffer.allocUnsafe(body.length + 4); + frame.writeUInt32BE(body.length); + body.copy(frame, 4); + await new Promise((resolve, reject) => process.stdout.write(frame, error => error ? reject(error) : resolve())); + } +} + +export class Worker { + constructor(contract, decodeSetting) { + if (!contract || ![contract.port, contract.schema, contract.adapter].every(value => typeof value === 'string' && value.length) || !names(contract.operations) || !contract.operations.length || !names(contract.capabilities ?? []) || !names(contract.errors ?? []) || typeof decodeSetting !== 'function') { + throw new Error('incomplete component contract'); + } + this.contract = Object.freeze({ + port: contract.port, schema: contract.schema, adapter: contract.adapter, + operations: Object.freeze([...contract.operations]), + capabilities: Object.freeze([...(contract.capabilities ?? [])]), + errors: Object.freeze([...(contract.errors ?? [])]), + }); + this.decodeSetting = decodeSetting; + this.handlers = new Map(); + this.outputTypes = null; + this.errorTypes = null; + } + + static fromManifest(manifest, handlers = null) { + const operations = manifest.operations; + if (!names(operations) || Object.keys(manifest.inputs).sort().join() !== [...operations].sort().join() || Object.keys(manifest.outputs).sort().join() !== [...operations].sort().join()) throw new Error('incomplete typed Worker contract'); + const worker = new Worker({ + port: manifest.port, schema: manifest.schema, adapter: manifest.adapter, + operations, capabilities: manifest.capabilities, errors: Object.keys(manifest.errors), + }, value => { + if (manifest.setting !== null) return typed(value, manifest.setting); + if (value !== null) throw new ProtocolError('unexpected setting'); + return null; + }); + worker.inputTypes = manifest.inputs; + worker.outputTypes = manifest.outputs; + worker.errorTypes = manifest.errors; + if (handlers !== null) { + if (Object.keys(handlers).sort().join() !== [...operations].sort().join()) throw new Error('component operations are not fully registered'); + for (const [operation, handler] of Object.entries(handlers)) worker.registerHandler(operation, handler); + } + return worker; + } + + registerHandler(operation, handler) { + if (!this.inputTypes || !Object.hasOwn(this.inputTypes, operation)) throw new Error('operation is not in a typed Worker contract'); + this.register(operation, value => typed(value, this.inputTypes[operation]), handler); + } + + register(operation, decodePayload, handler) { + if (!this.contract.operations.includes(operation) || this.handlers.has(operation) || typeof decodePayload !== 'function' || typeof handler !== 'function') throw new Error('unknown or duplicate operation'); + this.handlers.set(operation, { decodePayload, handler }); + } + + async serve() { + const contract = this.contract; + if (this.handlers.size !== contract.operations.length) throw new Error('component operations are not fully registered'); + const frames = new Frames(); + const hello = await frames.next(); + exact(hello, ['kind', 'version', 'port', 'schema', 'adapter', 'capabilities', 'operations', 'setting']); + if (hello.kind !== 'hello' || hello.version !== VERSION || hello.port !== contract.port || hello.schema !== contract.schema || hello.adapter !== contract.adapter || JSON.stringify(hello.capabilities) !== JSON.stringify(contract.capabilities) || JSON.stringify(hello.operations) !== JSON.stringify(contract.operations)) { + throw new ProtocolError('component handshake does not match registered contract'); + } + let setting; + try { setting = this.decodeSetting(hello.setting); } + catch { throw new ProtocolError('invalid component setting'); } + await frames.send({ kind: 'ready', version: VERSION, port: contract.port, schema: contract.schema, adapter: contract.adapter, capabilities: contract.capabilities, operations: contract.operations }); + + let active = null; + let nextID = 1; + let lastCancelled = false; + for (;;) { + const message = await frames.next(); + switch (message.kind) { + case 'health': + exact(message, ['kind', 'id']); + if (requestID(message.id) !== 0) throw new ProtocolError('invalid health id'); + await frames.send(message); + break; + case 'call': { + exact(message, ['kind', 'id', 'operation', 'payload']); + const id = requestID(message.id); + const operation = this.handlers.get(message.operation); + if (id !== nextID++ || active || !operation) throw new ProtocolError('invalid component call'); + let payload; + try { payload = operation.decodePayload(message.payload); } + catch { throw new ProtocolError('invalid component payload'); } + const current = { id, controller: new AbortController() }; + active = current; + current.settled = Promise.resolve().then(() => operation.handler(payload, setting, current.controller.signal)).then( + async result => { + if (active !== current) return; + active = null; + lastCancelled = false; + if (this.outputTypes) result = typed(result, this.outputTypes[message.operation]); + await frames.send({ kind: 'result', id, payload: result }); + }, + async error => { + if (active !== current) return; + active = null; + lastCancelled = false; + if (!(error instanceof BusinessError) || !contract.errors.includes(error.identity)) throw new ProtocolError('component handler failed'); + if (this.errorTypes) error.payload = typed(error.payload, this.errorTypes[error.identity]); + await frames.send({ kind: 'error', id, error: error.identity, payload: error.payload }); + }, + ).catch(error => { frames.fail(error); }); + break; + } + case 'cancel': + exact(message, ['kind', 'id']); + requestID(message.id); + if (active?.id === message.id) { + const current = active; + active = null; + await stopHandler(current); + lastCancelled = true; + await frames.send({ kind: 'error', id: message.id, error: 'dever.cancelled', payload: null }); + } else if (message.id !== nextID - 1 || lastCancelled) throw new ProtocolError('unknown component cancel id'); + break; + case 'shutdown': + exact(message, ['kind']); + if (active) { + const current = active; + active = null; + await stopHandler(current); + } + await frames.send({ kind: 'shutdown' }); + return; + default: + throw new ProtocolError('unknown component message'); + } + } + } +} + +export async function main(worker) { + try { await worker.serve(); } + catch (error) { + console.error(`component worker: ${error instanceof ProtocolError ? error.message : 'internal failure'}`); + process.exit(1); + } +} diff --git a/sdk/javascript/example.mjs b/sdk/javascript/example.mjs new file mode 100644 index 0000000..ac6c81b --- /dev/null +++ b/sdk/javascript/example.mjs @@ -0,0 +1,29 @@ +import { BusinessError, Worker, main } from './dever_component.mjs'; + +function setting(value) { + if (!value || typeof value !== 'object' || Array.isArray(value) || Object.keys(value).join() !== 'prefix' || typeof value.prefix !== 'string') throw new Error('invalid setting'); + return value.prefix; +} + +function payload(value) { + if (!value || typeof value !== 'object' || Array.isArray(value) || Object.keys(value).sort().join() !== 'delay_ms,text' || typeof value.text !== 'string' || !Number.isInteger(value.delay_ms) || value.delay_ms < 0 || value.delay_ms > 10000) throw new Error('invalid payload'); + return value; +} + +function number(value) { + if (!value || typeof value !== 'object' || Array.isArray(value) || Object.keys(value).join() !== 'number' || (typeof value.number !== 'bigint' && !Number.isSafeInteger(value.number))) throw new Error('invalid number'); + return value.number; +} + +const worker = new Worker({ port: 'example.Text', schema: 'fixture-schema-v1', adapter: 'example.TextAdapter', operations: ['text.render', 'text.fail', 'number.echo', 'text.stubborn'], errors: ['example.rejected'] }, setting); +worker.register('text.render', payload, async (value, prefix, signal) => { + await new Promise((resolve, reject) => { + const timer = setTimeout(resolve, value.delay_ms); + signal.addEventListener('abort', () => { clearTimeout(timer); reject(new Error('cancelled')); }, { once: true }); + }); + return { value: prefix + value.text }; +}); +worker.register('text.fail', payload, async () => { throw new BusinessError('example.rejected', { reason: 'rejected' }); }); +worker.register('number.echo', number, async value => ({ number: value })); +worker.register('text.stubborn', payload, async () => new Promise(() => {})); +await main(worker); diff --git a/sdk/native-release.md b/sdk/native-release.md new file mode 100644 index 0000000..1c6816b --- /dev/null +++ b/sdk/native-release.md @@ -0,0 +1,269 @@ +# Native release authoring + +This is the private compiler-author workflow. Application projects continue to use `dever run/build`; they do not need Cargo, a signing key or author settings. A Linux x86_64 compiler release can contain both native x86_64 and cross-compiled ARM64 application packs. The compiler and its LLVM libraries remain host binaries. + +Build the four runtime archives separately, then assemble and sign a new directory. The maker never builds source, downloads dependencies, searches `PATH`, reads configuration from environment variables, or modifies its inputs. It reuses the runtime manifest and signed release formats consumed by `deverd` and `MachineManager`. + +## Build inputs + +Build the compiler with `cargo build --offline --locked -p dever-cli --bin dever`; its author input becomes the installed `dever-core`. Build the stable machine launcher with `--bin dever-launcher` and the daemon with `--bin deverd`. Use `target/debug/dever-launcher` for the bootstrap launcher input, which the maker publishes as `bootstrap/dever`. The installed public command remains `dever`; `dever-launcher` is only an author build name. Preserve this distinction when preparing release inputs. + +Prepare the pinned Rust dependencies and compiler LLVM 18/LLD SDK explicitly. Use an explicitly selected Cargo/Rust toolchain for author builds. The root `runtime-pack` Cargo profile inherits the release optimization settings, uses one codegen unit and removes debug information while retaining archive symbols needed by LLD. It must not enable the bridge's `embedded` feature. + +All profiles include `runtime-api,runtime-external`; their dependencies enable `runtime-abi`. Database drivers vary: + +| Profile | Additional features | Saved archive | +| --- | --- | --- | +| base | none | `base.a` | +| sqlite | `runtime-sqlite` | `sqlite.a` | +| postgres | `runtime-postgres` | `postgres.a` | +| both | `runtime-sqlite,runtime-postgres` | `both.a` | + +For example, build base for Linux x86_64: + +```sh +cargo build --offline --locked -p dever-backend-bridge --lib \ + --profile runtime-pack --no-default-features \ + --features runtime-api,runtime-external \ + --target x86_64-unknown-linux-gnu --target-dir target/native-runtime-abi +``` + +Copy `target/native-runtime-abi/x86_64-unknown-linux-gnu/runtime-pack/libdever_backend_bridge.a` to a new `base.a` input, then repeat with each row's features and save its archive before the next build. Share this one Cargo target directory; do not create four dependency caches. Do not copy the both archive into all four inputs or strip a hard-linked fixture in place. Do not use `strip --strip-all` on link inputs. + +Prepare a matching compiler core, `libLLVM.so.18.1`, target CRT objects and genuine static libraries. Linux GNU link order is explicit: start objects, runtime, static libraries, end objects. Linker scripts and thin archives are rejected. Preserve the compiler version and the exact compiler/CRT/SDK versions and input hashes in build records; the maker does not infer feature content or compiler versions from filenames. + +## Author settings + +Application authors select `dever build <project-root> --target linux-aarch64 --output <new-file>`; omitted targets, `run`, and `test` use the host. This selects application output, not an ARM compiler installation. Missing target packs are errors. Product commands do not discover cross tools or download packs while building. + +For an additional application target, add `native_targets.linux-aarch64` to the author settings. Its value has the same `profiles`, `start`, `libraries`, and `end` fields as the host inputs below. Build all four archives with Rust target `aarch64-unknown-linux-gnu` and an explicitly prepared cross linker/sysroot. Use ARM CRT and static libraries, never host files or linker scripts. Both standalone object and archive member ELF headers are checked against the declared target before publishing and on cache access. + +The parallel maps `ecosystem_targets.linux-aarch64`, `sandbox_targets.linux-aarch64`, and `build_targets.linux-aarch64` use the corresponding host `runtimes`, `sandbox`, and `builds` shapes. Python/Node interpreters, native wheel/addon files, the sandbox guard, static bwrap, and its OS libraries must match the application target. Go is different: `build.host` identifies the host executable tools; `build.goos` and `build.goarch` identify the output and standard-library archives. The compiler/linker must produce that target with a cleared environment. The private fixture maker supports `go run test/go-managed/make_pack.go <go-binary> <new-pack> linux-aarch64`; it uses an author-only Go overlay to bake ARM defaults into x86 tools without changing the installed Go SDK. + +Prepare project Libs explicitly with `dever lib update <project-root> --target linux-aarch64` (or `lib add` with the same option). The resulting lock and cached resources bind the selected target. Offline `build` verifies that binding. Pure Python wheels/npm packages and Go cross compilation use this path; Python source distributions or npm lifecycle builds require already prepared matching target outputs and complete build receipts. An x86 tool must never silently produce a native extension labeled ARM. + +The opt-in ARM acceptance uses explicit inputs under `target/arm64-cross/`: four archives/CRT libraries, target ecosystem and sandbox packs, locked upstream dependencies, and private QEMU/Alpine kernel/initramfs files. `native_release::acceptance::cross` exports independent applications under `guest/apps/` after deleting its source and machine store. Run `python3 -B test/ecosystem-release/arm64/simulate.py` to rebuild only the owned guest images and execute Python, Node, Go and HTTP under the complete ARM kernel. It requires all success/failure/cleanup markers and writes serial logs plus a result JSON under `logs/`. The VM has no external network or system language installations. Its initramfs switches into a normal root mount so bubblewrap can use `pivot_root`. This is functional emulation, not physical ARM performance or a universal kernel compatibility result. + +Use a separate author root, not a Dever application's configuration. Place its explicit input files and private Ed25519 PKCS#8 key below that root. `config/setting.json` has this shape; replace every illustrative digest with the file's actual lowercase SHA-256: + +```json +{ + "format": "dever-native-release-input-v1", + "version": "0.1.0", + "target": "x86_64-unknown-linux-gnu", + "signing_key": "private/release.pk8", + "core": {"source": "inputs/dever-core", "sha256": "<sha256>"}, + "skill": [ + {"source": "inputs/skills/SKILL.md", "path": "SKILL.md", "sha256": "<sha256>"}, + {"source": "inputs/skills/references/development.md", "path": "references/development.md", "sha256": "<sha256>"} + ], + "core_libraries": [ + {"source": "inputs/libLLVM.so.18.1", "path": "libLLVM.so.18.1", "sha256": "<sha256>"}, + {"source": "inputs/libstdc++.so.6", "path": "libstdc++.so.6", "sha256": "<sha256>"} + ], + "profiles": { + "base": {"source": "inputs/base.a", "sha256": "<sha256>"}, + "sqlite": {"source": "inputs/sqlite.a", "sha256": "<sha256>"}, + "postgres": {"source": "inputs/postgres.a", "sha256": "<sha256>"}, + "both": {"source": "inputs/both.a", "sha256": "<sha256>"} + }, + "start": [ + {"source": "inputs/crt1.o", "path": "crt1.o", "sha256": "<sha256>"}, + {"source": "inputs/crti.o", "path": "crti.o", "sha256": "<sha256>"}, + {"source": "inputs/crtbeginT.o", "path": "crtbeginT.o", "sha256": "<sha256>"} + ], + "libraries": [ + {"source": "inputs/libc.a", "path": "libc.a", "sha256": "<sha256>"}, + {"source": "inputs/libm.a", "path": "libm.a", "sha256": "<sha256>"}, + {"source": "inputs/libmvec.a", "path": "libmvec.a", "sha256": "<sha256>"}, + {"source": "inputs/libgcc.a", "path": "libgcc.a", "sha256": "<sha256>"}, + {"source": "inputs/libgcc_eh.a", "path": "libgcc_eh.a", "sha256": "<sha256>"} + ], + "end": [ + {"source": "inputs/crtend.o", "path": "crtend.o", "sha256": "<sha256>"}, + {"source": "inputs/crtn.o", "path": "crtn.o", "sha256": "<sha256>"} + ] +} +``` + +`source` and `signing_key` are normalized relative paths below the author root, without symlinks. Link-input `path` is relative to the output's `runtime/<platform>` directory; reserved and duplicate destinations fail. The key must be owner-private on Unix. Unknown fields, missing profiles, invalid targets or digests fail explicitly. The key and author configuration are never included in the output. + +`core_libraries` must list the full dynamic dependency closure, not just the two illustrative entries above. These `path` values are plain SONAMEs published under `lib/`. Current Linux x86_64 inputs additionally require gcc_s, z, ffi, edit, zstd, tinfo, xml2, bsd, md, icuuc, icudata and lzma. The compiler declares inherited `RPATH=$ORIGIN/lib`; library search entries may only name that same private directory (`$ORIGIN` or upstream LLVM's `$ORIGIN/../lib`). Maker and signed native installation reject missing dependencies, mismatched SONAME/architecture, unused libraries and escaping search paths. The declared system baseline is GNU libc and its companion libraries; the current prepared compiler requires glibc 2.39. The pack must not replace that system ABI. This replaces the single `llvm` author field directly. + +## Versioned AI skill and public release assets + +`skill` is required. Copy the complete standalone `dever-main-skills` checkout into the author inputs, excluding `.git` and temporary files, and list every skill file as `{source,path,sha256}`. Paths are relative to its skill root; the maker publishes them under `skills/dever-language/` in the signed manifest. The two entries above illustrate the shape, not the complete file list. `SKILL.md` is mandatory; at most 256 files, 1 MiB per file and 8 MiB total are allowed. + +`dever skill path [project-root]` validates and returns the skill belonging to the selected compiler. `dever skill install <new-directory>` installs a small AI entry that resolves the active signed skill on each task. It refuses existing directories. `dever update` and `dever use` change the one active-version journal used by both core and skill; they do not rewrite every user's AI directory. Ordinary users can read the signed skill without the install lock or administrator permissions. + +Publish official versions to `shemic/dever-main` GitHub Releases with tag `v<version>`. `sdk/release-assets.py` converts the signed directory into three assets named `dever-<platform>.manifest.json`, `dever-<platform>.manifest.sig`, and `dever-<platform>.tar.gz`. The archive contains exactly the manifest's ordinary artifact files, with no directory headers or embedded manifest. The installer and explicit `dever install/update` download these assets, verify the independent pinned public key before payload extraction, and never use application configuration or environment variables for the official address. Project `run/build` remain offline. + +## Linux first installation + +Include the stable launcher and daemon in the same signed release by adding this author field: + +```json +"bootstrap": { + "launcher": {"source":"inputs/dever", "sha256":"<sha256>"}, + "daemon": {"source":"inputs/deverd", "sha256":"<sha256>"}, + "libraries": [ + {"source":"inputs/libgcc_s.so.1", "path":"libgcc_s.so.1", "sha256":"<sha256>"} + ] +} +``` + +Both Rust entry points must be built with their private `$ORIGIN/lib` RPATH. The maker verifies their separate ELF dependency closure and adds `bootstrap/dever`, `bootstrap/deverd`, `bootstrap/lib/*` and the fixed `bootstrap/deverd.service` to the existing signed manifest. Core-only releases may omit this field; first installation requires it. + +An administrator must independently authenticate the first installer executable, its private libraries, and the release public key. A key shipped inside a downloaded release is not a trust source. Place the following configuration under an absolute, root-owned installer directory, in `config/setting.json`: + +```json +{ + "release": "/prepared/release", + "system_root": "/owned/system-image", + "trusted_key": "/independently-provisioned/release.pub", + "activate_service": false +} +``` + +Run the independently trusted launcher with `--dever-install /absolute/installer-root`. It installs the signed compiler version, `/opt/dever/bin` bootstrap closure, `/usr/local/bin/dever` symlink, root-owned systemd unit and its enable link below `system_root`. The root daemon owns the shared machine store; ordinary projects and users share it through the existing authenticated service protocol. `install`, `update` and `use` retain their version-management contracts. + +Offline image installation never invokes systemctl. `activate_service: true` is accepted only for the real `/` system root and explicitly performs daemon reload, restart and readiness checking. This session's acceptance uses owned image roots and manually started owned daemons; it does not activate any global service. + +The initial key is written to a checked temporary file, synced, then published without replacing an existing key. A bounded journal records seven fixed destinations: bootstrap directory, sandbox helper store, AppArmor policy file, service unit, public entry, enable link and active version. File modes and child directories are persisted before publication. Interrupted uncommitted installs restore their recorded predecessors; committed installs finish cleanup without undoing publication. Rerun the explicit installer to recover. Normal launcher/version operations refuse an unfinished journal. A cleanup failure remains an explicit error with the journal retained. Existing service/policy ownership is checked against its previously signed template, allowing legitimate template upgrades while rejecting unrelated files. These ordering guarantees and interruption tests are not a claim of hardware power-loss testing. + +## Ecosystem inputs + +The same author configuration accepts an optional `runtimes` object with `pip`, `npm` and `go` entries. Each entry declares `name`, `version` and `files`: normalized archive `path`, author-root `source`, and exact `sha256`. Include `dever-runtime.json` in that list. Python additionally requires `python_markers` and nonempty `python_wheel_tags` matching the prepared interpreter; these are registry resolution metadata, not settings supplied by an application. + +Python packs must contain a relocatable interpreter, its standard library, native extensions and their non-OS dependencies, with fixed `-I -S -B` arguments. Node packs include the interpreter with its embedded standard library. Go build packs contain `compile`, `link`, the Dever source analyzer and every standard-library archive referenced by their `importcfg`; applications embed only the resulting Go Worker. Interpreter portability must be tested against an explicit OS baseline. A copied host executable alone does not establish a complete pack. + +The maker and Worker builder share descriptor, path and file-closure validation. Packs are deterministic gzip/tar files with sorted entries, fixed archive metadata and executable modes derived from the descriptor. Every runtime input is copied and rehashed; the private-key exclusion applies to these files too. The maker calculates the pack digest and emits `runtime/<ecosystem>/<platform>/{runtime.pack,manifest.json}`. Both files enter the existing top-level release signature. No new lock, installation or signature protocol is introduced. + +Node registry metadata may declare `npm_libc` as `glibc` or `musl`, matching the explicitly prepared interpreter and target. This is signed author metadata for package eligibility, never host probing. The pinned GNU Node fixture declares `glibc`. + +## Sandbox inputs + +External Workers, including `exec`, require the installed release's sandbox assets. Add a `sandbox` array to author settings using the same `{source, path, sha256}` entries as other leaf inputs. Its paths are `bin/bwrap`, `bin/guard`, and flat `lib/<name>` libraries, including the platform loader. Build `dever-sandbox-guard` from the current source with the explicit author toolchain. The bwrap input must support bind-fd and be a static target ELF with neither PT_INTERP nor DT_NEEDED; runtime executes it directly. The guard and language runtimes retain their complete private loader/library closure. The maker validates every asset before publishing `sandbox/<platform>/<path>` under the existing release signature. The asset budget is 128 files / 32 MiB. Inputs must not use RPATH/RUNPATH to escape the supplied bootstrap library set. + +The Linux backend requires user, mount, PID and network namespace permission, seccomp and `openat2`. An active host loader preload configuration is rejected. When `apparmor_restrict_unprivileged_userns=1`, a non-root launch selects only `/opt/dever/sandbox/<embedded-bwrap-sha256>/bwrap`. Every ancestor must be a real root-owned directory without group/other write permission; the regular executable must have no setuid/setgid or file capabilities, and its digest must equal the embedded verified asset. Missing or invalid helpers produce an installation error before spawning. Other launches execute the same static embedded bwrap directly; there is no PATH, environment, dynamic-loader or system-bwrap fallback. + +The signed bootstrap includes `bootstrap/dever-sandbox`, copied to `/etc/apparmor.d/dever-sandbox` by the same installation transaction. The source is [sdk/apparmor/dever-sandbox](apparmor/dever-sandbox), derived from the [AppArmor 4.0 bwrap restriction profile](https://gitlab.com/apparmor/apparmor/-/raw/apparmor-4.0/profiles/apparmor/profiles/extras/bwrap-userns-restrict). Its attachment permits only 64-lowercase-hex directories below the trusted store. bwrap creates namespaces under the parent policy and exec stacks the capability-denying child policy, including under no-new-privileges. Attachment paths do not restrict an explicit profile transition from an unconfined process: on hosts with user namespace restriction enabled, `kernel.apparmor_restrict_unprivileged_unconfined=1` is therefore mandatory before installation or loading. See [Ubuntu's explanation of strict profile changes](https://discourse.ubuntu.com/t/understanding-apparmor-user-namespace-restriction/58007). Non-root launches check this flag before selecting a helper; a missing, disabled or unreadable flag fails closed. The real-host installer checks the same prerequisite before any installation changes and again before publishing the policy file, because an independent system policy reload could load even an initially inactive file. Offline image preparation does not change or require the author's host settings; the deployed image must satisfy this prerequisite before its policy becomes loadable. + +The runtime also requires both named profiles in enforce mode using the read-only AppArmor policy filesystem; absent or inaccessible policy metadata is an explicit deployment error. No launch changes sysctl or loads policy. A container or restricted mount namespace must expose this policy metadata if it runs such non-root Workers. Hosts without the user namespace restriction do not require the strict-profile sysctl to exist. + +Administrator order is explicit: first review and approve the host-wide effect of strict profile transitions, record the original sysctl value, and establish `kernel.apparmor_restrict_unprivileged_unconfined=1`; then install the matching signed helper/policy; only then load with `apparmor_parser -r -K /etc/apparmor.d/dever-sandbox`. Dever never changes this global setting automatically. Review and offline validation use `apparmor_parser -Q -K -j 1 sdk/apparmor/dever-sandbox`, which does not load policy or write its cache. First-install rollback must unload this exact policy with `apparmor_parser -R -K /etc/apparmor.d/dever-sandbox` and remove the newly installed policy file from reload discovery before restoring the original strict-profile sysctl value. For an upgrade, restore the previously approved policy instead, retaining strict mode for any policy that requires it. Keep all helpers needed by deployed applications. Each host operation requires authorization; never disable the user namespace restriction. Real non-root, NNP/aa-exec (including immediate mode) and capability-denial acceptance must follow loading; offline parsing and root-only acceptance do not establish those results. + +The installer preserves at most 24 helper identities, each with its original signed manifest/signature, independently of compiler version directories. Unknown entries, incorrect signatures and digest mismatches fail without overwriting that store. The existing transaction's 128-entry/80-MiB tree budget also applies. Retained hashes support independently deployed applications after source or compiler-version removal. A new compiler version selecting different bwrap bytes requires the explicit trusted bootstrap installation before affected non-root Workers can start. Other OS backends and GPU device isolation are not currently supported. + +For approved profile-transition acceptance, `test/dever-sandbox-tests/apparmor_transition.c` is a standalone negative probe: compile it statically with the explicit author C toolchain, then invoke it as a nonzero UID through `aa-exec -p dever-bwrap`, immediate `aa-exec -i -p dever-bwrap`, and no-new-privileges variants. It never executes another program; it attempts only its own user/network namespaces, UID/GID maps and loopback capability operation. Record aa-exec rejection separately from a probe result. Exit 0 reports a denied operation, exit 1 reports unsafe access and exit 2 means unavailable setup. Creating the network namespace already proves a capability check succeeded, so the probe returns 1 even if a later loopback operation fails. Preserve the initial label and exact denial stage for each case. + +### Static Linux x86_64 author input + +The current private fixture uses upstream [bubblewrap v0.11.0](https://github.com/containers/bubblewrap/tree/v0.11.0), SHA-256 `cfeeb15fcc47d177d195f06fdf0847e93ee3aa6bf46f6ac0a141fa142759e2c3` for `https://codeload.github.com/containers/bubblewrap/tar.gz/refs/tags/v0.11.0`, and Ubuntu Noble's `libcap-dev_2.66-5ubuntu2.4_amd64.deb`, SHA-256 `07f2462867569a2119a2ad0f1593232663f2d1612b791c230d22a8d73a15abee` from the authenticated apt package index. Its download URL is `https://archive.ubuntu.com/ubuntu/pool/main/libc/libcap2/libcap-dev_2.66-5ubuntu2.4_amd64.deb`. Extract with `dpkg-deb -x` into the author directory; do not install packages on the target host. Retain upstream `COPYING`, the libcap package's `usr/share/doc/libcap-dev/copyright`, sources and the exact build recipe with author inputs. + +Below, `target/sandbox-author-inputs/` contains the extracted `bubblewrap-0.11.0/` and `libcap/`. Its source-local `config.h` contains only `#define PACKAGE_STRING "bubblewrap 0.11.0"` and `#define ENABLE_REQUIRE_USERNS 1`. Build with the explicitly selected author GNU C toolchain and static libc; no customer build tools are needed: + +```sh +/usr/bin/cc -static -O2 -D_GNU_SOURCE \ + -I target/sandbox-author-inputs/libcap/usr/include \ + -I target/sandbox-author-inputs/bubblewrap-0.11.0 \ + target/sandbox-author-inputs/bubblewrap-0.11.0/bubblewrap.c \ + target/sandbox-author-inputs/bubblewrap-0.11.0/bind-mount.c \ + target/sandbox-author-inputs/bubblewrap-0.11.0/network.c \ + target/sandbox-author-inputs/bubblewrap-0.11.0/utils.c \ + target/sandbox-author-inputs/libcap/usr/lib/x86_64-linux-gnu/libcap.a \ + -o target/sandbox-author-inputs/bwrap-static +readelf -l target/sandbox-author-inputs/bwrap-static +readelf -d target/sandbox-author-inputs/bwrap-static +sha256sum target/sandbox-author-inputs/bwrap-static +``` + +The 2026-10-02 private author build is 1,099,016 bytes with SHA-256 `c9e8bd758c2d01ca6668a842bad7b020bf700436e87b666cfaae3803be0be1b3`; readelf confirms no interpreter and no dynamic section. A different author compiler/libc may produce a different digest: sign the actual validated bytes and deploy their matching content-addressed helper. The current policy source SHA-256 is `a6d2ef490d12914e90a96272a197b8b78667403702b1324bae661a10b78125ec`; AppArmor parser 4.0.1 accepted it offline with the command above. These are input/parse results, not evidence that host policy has been loaded. + +Application file access is explicit in `config/setting.json`, for example `adapter.notification.delivery.files.uploads = {"path":"data/upload", "write":true}` with checked `allow file`. The Worker sees `/data/uploads`. Grant paths cannot write the executable cache or other execution inputs; no environment configuration is introduced. + +## Make and consume + +Build the private author executable once, then run it directly. The command optimizes existing compression and digest dependencies even in a development build, so complete language distributions do not spend minutes in unoptimized byte loops. These author/test options do not change application configuration or compile-service limits. + +```sh +cargo build --offline --locked -p dever-cli --example native-release \ + --config 'profile.dev.package.miniz_oxide.opt-level=3' \ + --config 'profile.dev.package.flate2.opt-level=3' \ + --config 'profile.dev.package.sha2.opt-level=3' +target/debug/examples/native-release <author-root> --output <new-release-directory> +``` + +The output parent must exist. A private sibling staging directory holds copied inputs until the existing runtime validator succeeds and the manifest is signed. Publication atomically requires an absent destination, including under concurrent makers. Failures remove only owned staging. Output artifacts are independent copies; subsequent source changes do not change a completed pack. + +The result contains `dever-core`, `lib/libLLVM.so.18.1`, `runtime/<platform>/manifest.json`, four `archives/<profile>.a` files, the configured link inputs, and top-level `manifest.json`/`manifest.sig`. The signed manifest covers the complete native closure. Place this version directory in the existing machine download store and use the established installer; trusted public-key provisioning remains explicit and separate. The maker does not trust a key merely because it accompanies a release, install a service, publish a catalog or update a global command. + +For private `dever` development, the same verified `runtime/` and `lib/` trees may be copied beside a matching private compiler in a new directory. An executable named `dever-core` still requires the managed installation layout and authenticated daemon. This is not a service-unavailable fallback. + +## Acceptance + +Default `native_release` tests use synthetic input and temporary keys. The explicit Linux acceptance additionally consumes the four optimized archives in `target/native-release-inputs/`, the prepared native SDK and the built author executable: + +Real ecosystem acceptance temporarily holds the signed release, installed version, compiler cache and Worker bundles simultaneously. Allow several GiB of free storage in addition to author inputs and Cargo artifacts. When the system temporary filesystem is too small, an optional author-only `target/native-release-inputs/config/setting.json` can select an existing executable filesystem: + +```json +{"machine_temporary_root":"/dev/shm"} +``` + +The path must be absolute; unknown configuration fields are rejected. Only the ecosystem machine directory and bootstrap image use this setting. Author inputs remain on the normal temporary filesystem so their immutable hard links stay on the source filesystem. With no configuration, the existing temporary-directory behavior is unchanged. No environment variable is read for this setting. A memory-backed filesystem consumes real memory: provide both sufficient free capacity and an explicit acceptance-process memory budget, and run these large acceptances serially. + +Before the no-host checks, build their test-only static init explicitly: + +```sh +cargo rustc --offline --locked -j1 -p dever-cli --example native-acceptance-init -- -C target-feature=+crt-static +``` + +```sh +cargo test --offline --locked -p dever-cli --test native_release +cargo test --offline --locked -p dever-cli --test native_release \ + acceptance::optimized_profiles_make_reproducible_signed_release_and_run_through_daemon \ + -- --exact --ignored --nocapture +cargo test --offline --locked -p dever-cli --test native_release \ + acceptance::signed_core_builds_without_host_compiler_libraries_or_tools \ + -- --exact --ignored --nocapture +``` + +The second command starts only a test-owned daemon and SQLite databases. It checks actual driver symbols, identical manifests/signatures for repeated packaging, signed installation, four profile executions through the daemon, and independent programs after source removal and daemon shutdown. PostgreSQL server behavior is a separate acceptance, not implied by linking the PostgreSQL profile. + +The third command requires root and namespace permission for a test-owned mount namespace root. It checks source and builds a SQLite program inside a root containing only the signed compiler closure, selected native archives and named GNU OS ABI libraries, then executes the program after removing its source. No host LLVM, C++ library, language tool, compiler SDK or package manager is visible there. The harness uses the explicitly prepared bwrap/loader assets to pivot into that root. + +The harness rejects a dynamically linked init. Inside the already pivoted namespace, it replaces bwrap's proc submounts with a complete private proc mount so the nested Worker can mount its own proc. It keeps the outer root fixture's existing capabilities; the product's inner namespace, capability removal and guard remain unchanged. It never binds the host proc into the final root. This private compiler test complements the managed daemon test; it does not install a machine service. + +The complete first-install acceptance additionally requires the built `dever` and `deverd` binaries: + +```sh +cargo test --offline --locked -p dever-cli --test native_release \ + acceptance::bootstrap::signed_bootstrap_installs_and_serves_two_projects_without_host_libraries \ + -- --exact --ignored --nocapture +``` + +It installs only into a new owned image, verifies stable entry and systemd assets, exercises a legitimate older signed unit upgrade and rejects tampering, then runs two projects through the installed daemon in the minimal OS root. Separate clients retain two real kernel UIDs to check shared-cache use and denied machine mutation. Finally the independently built programs run after source and machine-store removal. Neither the global service nor a public signing identity is involved. + +Identical prepared inputs and signing key produce identical payloads, manifests and signatures. This establishes deterministic packaging, not reproducible source builds across different machines, CRTs or Rust toolchains. Temporary test signatures and Linux execution do not establish public release, six-platform delivery, an OS sandbox or performance capacity. + +The explicit three-ecosystem acceptance uses fixed upstream archives and the retained Go build fixture listed with exact digests in `test/ecosystem-release/prepare.py`. Supply those inputs explicitly, then prepare a new directory: + +| Input | Explicit source | +| --- | --- | +| `downloads/cpython-3.12.14.tar.gz` | [python-build-standalone 20260901, Linux x86_64 install-only stripped](https://github.com/astral-sh/python-build-standalone/releases/download/20260901/cpython-3.12.14%2B20260901-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz) | +| `downloads/node-v24.15.0-linux-x64.tar.xz` | [Node v24.15.0 Linux x64](https://nodejs.org/dist/v24.15.0/node-v24.15.0-linux-x64.tar.xz), verified against the release's `SHASUMS256.txt` | +| `target/go-managed/runtime.pack` | Go 1.26.3 private host fixture with explicit `build.host=linux-x86_64`; the pinned archive preserves the original tools/stdlib and repacks the updated descriptor. New fixtures use `test/go-managed/make_pack.go`; this is not an upstream published pack. | + +The two download paths are below `target/ecosystem-release-inputs/`. No command here performs implicit downloads; the preparation script rejects different digests. + +```sh +/usr/bin/python3 test/ecosystem-release/prepare.py target/ecosystem-release-inputs/prepared +cargo test --offline --locked -p dever-cli --test native_release \ + --config 'profile.dev.package.miniz_oxide.opt-level=3' \ + --config 'profile.dev.package.flate2.opt-level=3' \ + --config 'profile.dev.package.sha2.opt-level=3' \ + acceptance::ecosystems::signed_ecosystems_build_and_run_without_system_language_installations \ + -- --exact --ignored --nocapture +``` + +This opt-in test requires Linux root and mount/PID/user namespace capability. It signs and installs an owned release, uses the managed daemon for each ecosystem's lock/check/run/build, then removes project sources and the entire machine store. The resulting programs run inside an owned mount namespace root containing only named Linux OS ABI libraries, a private proc mount and synthetic devices. Python checks bundle-local imports, standard-library native modules and the source-built simplejson C extension; Node loads the source-built bufferutil addon; Go retains its managed probe. The already prepared source-build locks and exact output artifacts come from `target/source-build/dependency` and `target/npm-build/dependency`; this acceptance never rebuilds hooks or downloads dependencies. It signs the exact runtime-pack bytes bound by those build receipts, so this test's author signature differs from the separate deterministic-maker evidence. No system Python, Node, Go, compiler, SDK or package manager is visible there. Other operating systems and a public release identity remain separate acceptance. diff --git a/sdk/native-release.rs b/sdk/native-release.rs new file mode 100644 index 0000000..115c923 --- /dev/null +++ b/sdk/native-release.rs @@ -0,0 +1,27 @@ +//! Private release-author entry point; never distributed as a user command. + +use std::path::Path; + +fn main() { + if let Err(error) = run() { + eprintln!("native-release: {error}"); + std::process::exit(1); + } +} + +fn run() -> Result<(), String> { + let arguments: Vec<_> = std::env::args_os().skip(1).collect(); + let [author_root, flag, output] = arguments.as_slice() else { + return Err("usage: native-release <author-root> --output <new-dir>".into()); + }; + if flag != "--output" { + return Err("usage: native-release <author-root> --output <new-dir>".into()); + } + let manifest = + dever_cli::toolchain::packaging::create(Path::new(author_root), Path::new(output))?; + println!( + "created Dever {} for {}", + manifest.version, manifest.platform + ); + Ok(()) +} diff --git a/sdk/python/dever_component.py b/sdk/python/dever_component.py new file mode 100644 index 0000000..4ec223a --- /dev/null +++ b/sdk/python/dever_component.py @@ -0,0 +1,404 @@ +"""Python Worker for the Dever Component Protocol. + +The compiler owns the Port schema. Pass its exact identity to Contract; this +module only transports and decodes values supplied by the application. +""" + +import asyncio +import json +import math +import re +import struct +import sys +import uuid +from contextlib import suppress +from dataclasses import dataclass +from typing import Any, Callable + + +VERSION = "dever-component-1" +MAX_BYTES = 16 * 1024 * 1024 +MAX_DEPTH = 64 +MAX_ELEMENTS = 65_536 +CANCEL_GRACE = 1 + + +class ProtocolError(Exception): + pass + + +class BusinessError(Exception): + def __init__(self, identity: str, payload: Any = None): + super().__init__(identity) + self.identity = identity + self.payload = payload + + +@dataclass(frozen=True) +class Contract: + port: str + schema: str + adapter: str + operations: tuple[str, ...] + capabilities: tuple[str, ...] = () + errors: tuple[str, ...] = () + + def __post_init__(self) -> None: + for name in ("operations", "capabilities", "errors"): + values = getattr(self, name) + if not isinstance(values, (list, tuple)): + raise ValueError("component contract entries must be lists") + object.__setattr__(self, name, tuple(values)) + if not all(isinstance(value, str) and value for value in (self.port, self.schema, self.adapter)) or not self.operations: + raise ValueError("incomplete component contract") + for names in (self.operations, self.capabilities, self.errors): + if any(not isinstance(name, str) or not name for name in names) or len(set(names)) != len(names): + raise ValueError("invalid or duplicate component contract entry") + + +def _typed(value: Any, schema: dict[str, Any]) -> Any: + kind = schema["type"] + if kind == "nullable": + return None if value is None else _typed(value, schema["value"]) + if kind == "list": + if not isinstance(value, list): + raise ValueError("expected list") + return [_typed(item, schema["value"]) for item in value] + if kind == "record": + fields = schema["fields"] + if not isinstance(value, dict) or not value.keys() <= fields.keys(): + raise ValueError("invalid record fields") + return {name: _typed(value[name], field) if name in value else + (None if field["type"] == "nullable" else _missing_field()) + for name, field in fields.items()} + if kind in ("int64", "duration", "model_id"): + if type(value) is not int or not -(2**63) <= value < 2**63: + raise ValueError("expected signed 64-bit integer") + elif kind == "float64": + if type(value) not in (int, float) or not math.isfinite(value): + raise ValueError("expected finite float") + elif kind == "bool": + if type(value) is not bool: + raise ValueError("expected boolean") + elif kind == "json": + _bounded(value, 0, [MAX_ELEMENTS]) + elif kind in ("text", "id", "secret", "decimal", "uuid", "datetime", "date", "time"): + if not isinstance(value, str): + raise ValueError("expected text") + if kind == "decimal" and not re.fullmatch(r"[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?", value): + raise ValueError("invalid decimal") + if kind == "uuid": + if not re.fullmatch(r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}", value): + raise ValueError("invalid UUID") + uuid.UUID(value) + if kind == "date": + import datetime + if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", value): + raise ValueError("invalid Date") + datetime.date.fromisoformat(value) + if kind == "time": + import datetime + if not re.fullmatch(r"\d{2}:\d{2}:\d{2}(?:\.\d{3})?", value): + raise ValueError("invalid Time") + datetime.time.fromisoformat(value) + if kind == "datetime": + import datetime + if not re.fullmatch(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,3})?(?:Z|[+-]\d{2}:\d{2})", value): + raise ValueError("invalid DateTime") + datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) + else: + raise ValueError("unknown worker type") + return value + + +def _missing_field() -> Any: + raise ValueError("missing required record field") + + +def _pairs(values: list[tuple[str, Any]]) -> dict[str, Any]: + result = {} + for key, value in values: + if key in result: + raise ProtocolError("duplicate JSON field") + result[key] = value + return result + + +def _reject_constant(_: str) -> None: + raise ProtocolError("invalid JSON number") + + +def _bounded(value: Any, depth: int, budget: list[int]) -> None: + if depth > MAX_DEPTH or (depth == MAX_DEPTH and isinstance(value, (dict, list))): + raise ProtocolError("wire JSON exceeds depth limit") + budget[0] -= 1 + if budget[0] < 0: + raise ProtocolError("wire JSON exceeds element limit") + if isinstance(value, dict): + for child in value.values(): + _bounded(child, depth + 1, budget) + elif isinstance(value, list): + for child in value: + _bounded(child, depth + 1, budget) + elif isinstance(value, float) and not math.isfinite(value): + raise ProtocolError("non-finite wire Float") + + +def _decode(body: bytes) -> dict[str, Any]: + try: + value = json.loads(body.decode("utf-8"), object_pairs_hook=_pairs, + parse_constant=_reject_constant) + except (UnicodeError, ValueError, RecursionError) as error: + raise ProtocolError("invalid component JSON") from error + _bounded(value, 0, [MAX_ELEMENTS]) + if not isinstance(value, dict): + raise ProtocolError("component message must be an object") + return value + + +def _fields(message: dict[str, Any], names: set[str]) -> None: + if message.keys() != names: + raise ProtocolError("component message has missing or extra fields") + + +def _id(value: Any) -> int: + if type(value) is not int or not 0 <= value <= 2**63 - 1: + raise ProtocolError("invalid request id") + return value + + +async def _read_frame(reader: asyncio.StreamReader) -> dict[str, Any]: + try: + size = struct.unpack(">I", await reader.readexactly(4))[0] + if not 0 < size <= MAX_BYTES: + raise ProtocolError("component frame exceeds byte limit") + return _decode(await reader.readexactly(size)) + except asyncio.IncompleteReadError as error: + raise ProtocolError("incomplete component frame") from error + + +def _write_frame(message: dict[str, Any]) -> None: + try: + body = json.dumps(message, ensure_ascii=False, allow_nan=False, separators=(",", ":")).encode("utf-8") + except (TypeError, ValueError) as error: + raise ProtocolError("invalid handler response") from error + _bounded(_decode(body), 0, [MAX_ELEMENTS]) + if not 0 < len(body) <= MAX_BYTES: + raise ProtocolError("component frame exceeds byte limit") + sys.__stdout__.buffer.write(struct.pack(">I", len(body)) + body) + sys.__stdout__.buffer.flush() + + +async def _stop_handler(task: asyncio.Task) -> None: + task.cancel() + completed, _ = await asyncio.wait({task}, timeout=CANCEL_GRACE) + if not completed: + raise ProtocolError("component handler did not stop after cancellation") + + +class Worker: + def __init__(self, contract: Contract, decode_setting: Callable[[Any], Any]): + if not callable(decode_setting): + raise ValueError("component setting decoder is required") + self.contract = contract + self.decode_setting = decode_setting + self.handlers: dict[str, tuple[Callable[[Any], Any], Callable[..., Any]]] = {} + self.output_types: dict[str, Any] = {} + self.error_types: dict[str, Any] = {} + + @classmethod + def from_manifest(cls, manifest: dict[str, Any], handlers: dict[str, Callable[..., Any]] | None = None) -> "Worker": + operations = tuple(manifest["operations"]) + inputs = manifest["inputs"] + outputs = manifest["outputs"] + errors = manifest["errors"] + if set(inputs) != set(operations) or set(outputs) != set(operations): + raise ValueError("incomplete typed Worker contract") + contract = Contract(manifest["port"], manifest["schema"], manifest["adapter"], + operations, tuple(manifest["capabilities"]), tuple(errors)) + setting = manifest["setting"] + worker = cls(contract, lambda value: _typed(value, setting) if setting is not None else + (None if value is None else _missing_field())) + worker.input_types = inputs + worker.output_types = outputs + worker.error_types = errors + if handlers is not None: + if set(handlers) != set(operations): + raise ValueError("component operations are not fully registered") + for operation, handler in handlers.items(): + worker.register_handler(operation, handler) + return worker + + def register_handler(self, operation: str, handler: Callable[..., Any]) -> None: + if not hasattr(self, "input_types") or operation not in self.input_types: + raise ValueError("operation is not in a typed Worker contract") + self.register(operation, lambda value: _typed(value, self.input_types[operation]), handler) + + def register(self, operation: str, decode_payload: Callable[[Any], Any], handler: Callable[..., Any]) -> None: + if operation not in self.contract.operations or operation in self.handlers: + raise ValueError("unknown or duplicate operation") + if not callable(decode_payload): + raise ValueError("component payload decoder is required") + if not asyncio.iscoroutinefunction(handler): + raise ValueError("component handler must be async") + self.handlers[operation] = (decode_payload, handler) + + async def serve(self) -> None: + if set(self.handlers) != set(self.contract.operations): + raise ValueError("component operations are not fully registered") + loop = asyncio.get_running_loop() + reader = asyncio.StreamReader() + transport, _ = await loop.connect_read_pipe( + lambda: asyncio.StreamReaderProtocol(reader), sys.stdin.buffer + ) + try: + await self._serve(reader) + finally: + transport.close() + + async def _serve(self, reader: asyncio.StreamReader) -> None: + hello = await _read_frame(reader) + _fields(hello, {"kind", "version", "port", "schema", "adapter", "capabilities", "operations", "setting"}) + expected = self.contract + if (hello["kind"], hello["version"], hello["port"], hello["schema"], hello["adapter"], hello["capabilities"], hello["operations"]) != ("hello", VERSION, expected.port, expected.schema, expected.adapter, list(expected.capabilities), list(expected.operations)): + raise ProtocolError("component handshake does not match registered contract") + try: + setting = self.decode_setting(hello["setting"]) + except Exception as error: + raise ProtocolError("invalid component setting") from error + _write_frame({"kind": "ready", "version": VERSION, "port": expected.port, "schema": expected.schema, "adapter": expected.adapter, "capabilities": list(expected.capabilities), "operations": list(expected.operations)}) + + messages: asyncio.Queue = asyncio.Queue(maxsize=4) + + async def read_messages() -> None: + while True: + try: + message = await _read_frame(reader) + except (OSError, ProtocolError) as error: + await messages.put(("fault", error)) + return + await messages.put(("frame", message)) + + receiver = asyncio.create_task(read_messages()) + try: + await self._dispatch(messages, setting) + finally: + # Cancel and join the pipe reader before Python finalizes stdio. + receiver.cancel() + with suppress(asyncio.CancelledError): + await receiver + + async def _dispatch(self, messages: asyncio.Queue, setting: Any) -> None: + loop = asyncio.get_running_loop() + active_id: int | None = None + active_task: asyncio.Task | None = None + active_operation: str | None = None + next_id = 1 + last_cancelled = False + completions: set[asyncio.Task] = set() + + def on_done(task: asyncio.Task) -> None: + try: + task.exception() + except asyncio.CancelledError: + pass + notification = loop.create_task(messages.put(("done", task))) + completions.add(notification) + notification.add_done_callback(completions.discard) + + try: + while True: + event, value = await messages.get() + if event == "fault": + raise value + if event == "done": + task = value + if task is active_task: + try: + payload = task.result() + if self.output_types: + payload = _typed(payload, self.output_types[active_operation]) + _write_frame({"kind": "result", "id": active_id, "payload": payload}) + except BusinessError as error: + if error.identity not in self.contract.errors: + raise ProtocolError("undeclared business error identity") from error + if self.error_types: + error.payload = _typed(error.payload, self.error_types[error.identity]) + _write_frame({"kind": "error", "id": active_id, "error": error.identity, "payload": error.payload}) + except asyncio.CancelledError: + pass + except Exception as error: + raise ProtocolError("component handler failed") from error + active_id, active_task, active_operation = None, None, None + last_cancelled = False + continue + message = value + kind = message.get("kind") + if kind == "health": + _fields(message, {"kind", "id"}) + if _id(message["id"]) != 0: + raise ProtocolError("invalid health id") + _write_frame(message) + elif kind == "call": + _fields(message, {"kind", "id", "operation", "payload"}) + request_id = _id(message["id"]) + operation = message["operation"] + if request_id != next_id or active_task is not None or not isinstance(operation, str) or operation not in self.handlers: + raise ProtocolError("invalid component call") + next_id += 1 + decoder, handler = self.handlers[operation] + try: + payload = decoder(message["payload"]) + except Exception as error: + raise ProtocolError("invalid component payload") from error + async def invoke() -> Any: + return await handler(payload, setting) + active_id = request_id + active_operation = operation + active_task = loop.create_task(invoke()) + active_task.add_done_callback(on_done) + elif kind == "cancel": + _fields(message, {"kind", "id"}) + request_id = _id(message["id"]) + if request_id == 0: + raise ProtocolError("invalid cancel id") + if active_id == request_id and active_task is not None: + stopped_task = active_task + active_id, active_task, active_operation = None, None, None + await _stop_handler(stopped_task) + last_cancelled = True + _write_frame({"kind": "error", "id": request_id, "error": "dever.cancelled", "payload": None}) + elif request_id != next_id - 1 or last_cancelled: + raise ProtocolError("unknown component cancel id") + elif kind == "shutdown": + _fields(message, {"kind"}) + if active_task is not None: + stopped_task = active_task + active_id, active_task = None, None + await _stop_handler(stopped_task) + _write_frame({"kind": "shutdown"}) + return + else: + raise ProtocolError("unknown component message") + + finally: + if active_task is not None: + active_task.remove_done_callback(on_done) + for notification in completions: + notification.cancel() + await asyncio.gather(*completions, return_exceptions=True) + + +def main(worker: Worker) -> None: + loop = asyncio.new_event_loop() + try: + asyncio.set_event_loop(loop) + loop.run_until_complete(worker.serve()) + except (ProtocolError, OSError, ValueError) as error: + print(f"component worker: {error}", file=sys.stderr) + raise SystemExit(1) from None + finally: + for task in asyncio.all_tasks(loop): + task.cancel() + loop.close() + asyncio.set_event_loop(None) diff --git a/sdk/python/example.py b/sdk/python/example.py new file mode 100644 index 0000000..199c104 --- /dev/null +++ b/sdk/python/example.py @@ -0,0 +1,55 @@ +"""Runnable protocol example with a fixed compiler-owned fixture schema.""" + +import asyncio +from dever_component import BusinessError, Contract, Worker, main + + +def setting(value): + if not isinstance(value, dict) or set(value) != {"prefix"} or not isinstance(value["prefix"], str): + raise ValueError("invalid setting") + return value["prefix"] + + +def payload(value): + if not isinstance(value, dict) or set(value) != {"text", "delay_ms"}: + raise ValueError("invalid payload") + if not isinstance(value["text"], str) or type(value["delay_ms"]) is not int or not 0 <= value["delay_ms"] <= 10_000: + raise ValueError("invalid payload") + return value + + +async def render(value, prefix): + await asyncio.sleep(value["delay_ms"] / 1000) + return {"value": prefix + value["text"]} + + +async def fail(value, prefix): + raise BusinessError("example.rejected", {"reason": "rejected"}) + + +def number(value): + if not isinstance(value, dict) or set(value) != {"number"} or type(value["number"]) is not int: + raise ValueError("invalid number") + return value["number"] + + +async def echo_number(value, prefix): + return {"number": value} + + +async def stubborn(value, prefix): + try: + await asyncio.sleep(5) + except asyncio.CancelledError: + await asyncio.sleep(5) + return {"value": prefix + value["text"]} + + +worker = Worker(Contract("example.Text", "fixture-schema-v1", "example.TextAdapter", ("text.render", "text.fail", "number.echo", "text.stubborn"), errors=("example.rejected",)), setting) +worker.register("text.render", payload, render) +worker.register("text.fail", payload, fail) +worker.register("number.echo", number, echo_number) +worker.register("text.stubborn", payload, stubborn) + +if __name__ == "__main__": + main(worker) diff --git a/sdk/release-assets.py b/sdk/release-assets.py new file mode 100644 index 0000000..0d406bf --- /dev/null +++ b/sdk/release-assets.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Turn a prepared signed release into the three official GitHub release assets.""" + +import argparse +import ctypes +import gzip +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import tarfile +import tempfile + + +def create(release, output): + release = release.absolute() + output = output.absolute() + if release.is_symlink() or not release.is_dir(): + raise ValueError("release must be a real prepared directory") + if output.exists() or output.is_symlink(): + raise ValueError("output already exists") + if not output.parent.is_dir(): + raise ValueError("output parent must exist") + manifest_bytes = (release / "manifest.json").read_bytes() + manifest = json.loads(manifest_bytes) + if manifest["format"] != "dever-release-v1" or manifest["platform"] not in ("linux-x86_64", "linux-aarch64"): + raise ValueError("unsupported release format or platform") + artifacts = manifest["artifacts"] + if not 1 <= len(artifacts) <= 4096 or sum(entry["bytes"] for entry in artifacts) > 2 * 1024**3: + raise ValueError("release exceeds the official asset limits") + names = set() + for artifact in artifacts: + name = artifact["path"] + if (not name or str(PurePosixPath(name)) != name or name.startswith("/") + or any(part in ("", ".", "..") for part in name.split("/")) + or any(character in name for character in ("\\", ":", "\0")) + or name in names or name in ("manifest.json", "manifest.sig")): + raise ValueError("invalid release artifact path") + names.add(name) + path = release / name + for parent in (path, *path.parents): + if parent.is_symlink(): + raise ValueError("release artifacts must not traverse symlinks") + if parent == release: + break + if not path.is_file() or path.stat().st_size != artifact["bytes"]: + raise ValueError("release artifact type/size mismatch") + with path.open("rb") as stream: + if hashlib.file_digest(stream, "sha256").hexdigest() != artifact["sha256"]: + raise ValueError("release artifact digest mismatch") + with tempfile.TemporaryDirectory(prefix=".dever-release-assets-", dir=output.parent) as scratch: + prepared = Path(scratch) / "assets" + prepared.mkdir() + stem = f"dever-{manifest['platform']}" + (prepared / f"{stem}.manifest.json").write_bytes(manifest_bytes) + (prepared / f"{stem}.manifest.sig").write_bytes((release / "manifest.sig").read_bytes()) + with (prepared / f"{stem}.tar.gz").open("xb") as compressed: + with gzip.GzipFile(filename="", fileobj=compressed, mode="wb", mtime=0) as stream: + with tarfile.open(fileobj=stream, mode="w|", format=tarfile.USTAR_FORMAT) as archive: + for artifact in sorted(artifacts, key=lambda entry: entry["path"]): + path = release / artifact["path"] + header = tarfile.TarInfo(artifact["path"]) + header.size = artifact["bytes"] + header.mode = 0o644 + with path.open("rb") as contents: + archive.addfile(header, contents) + publish(prepared, output) + return sorted(path.name for path in output.iterdir()) + + +def publish(prepared, output): + # Linux renameat2 publishes the entire owned tree without replacing even an + # empty directory created by another publisher. Failure cleans only staging. + libc = ctypes.CDLL("libc.so.6", use_errno=True) + rename = libc.renameat2 + rename.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint] + rename.restype = ctypes.c_int + if rename(-100, os.fsencode(prepared), -100, os.fsencode(output), 1) != 0: + error = ctypes.get_errno() + raise OSError(error, os.strerror(error), str(output)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("release", type=Path) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + try: + for name in create(args.release, args.output): + print(name) + except (OSError, ValueError, KeyError, TypeError, tarfile.TarError) as error: + parser.exit(1, f"cannot prepare release assets: {error}\n") + + +if __name__ == "__main__": + main() diff --git a/skills b/skills new file mode 160000 index 0000000..9fd78fc --- /dev/null +++ b/skills @@ -0,0 +1 @@ +Subproject commit 9fd78fc8205e717bde4f00e5d26d8331cdef7b8f diff --git a/test/backend_bridge.rs b/test/backend_bridge.rs new file mode 100644 index 0000000..38c1bd8 --- /dev/null +++ b/test/backend_bridge.rs @@ -0,0 +1,345 @@ +use dever_backend_bridge::{ + BinaryResource, LinkKind, Target, emit_object, emit_object_with_resources, link, +}; +use std::fs; +#[path = "dever-tests/tests/support/temp.rs"] +mod temp; + +const IR: &str = "define i64 @answer() { ret i64 42 }"; + +#[test] +fn binary_initializers_reject_invalid_declarations_and_preserve_code_limit() { + let resource = BinaryResource { + symbol: "payload", + bytes: &[1, 2], + }; + for declaration in [ + "@other = external constant [2 x i8]", + "@payload = external constant [3 x i8]", + "@payload = external constant [2 x i16]", + "@payload = external global [2 x i8]", + "@payload = constant [2 x i8] zeroinitializer", + "@payload = external thread_local constant [2 x i8]", + ] { + let ir = format!("{declaration}\ndefine ptr @read() {{ ret ptr @payload }}"); + assert!(emit_object_with_resources(&ir, Target::LinuxX86_64, &[resource]).is_err()); + } + let ir = "@payload = external constant [2 x i8]\ndefine ptr @read() { ret ptr @payload }"; + assert!( + emit_object_with_resources(ir, Target::LinuxX86_64, &[resource, resource]) + .unwrap_err() + .contains("duplicate") + ); + assert!( + emit_object_with_resources( + "@payload = external constant [2 x i8]", + Target::LinuxX86_64, + &[resource] + ) + .unwrap_err() + .contains("used external") + ); + for symbol in ["", "1bad", "with.dot", "bad\0name"] { + assert!( + emit_object_with_resources( + ir, + Target::LinuxX86_64, + &[BinaryResource { symbol, ..resource }] + ) + .unwrap_err() + .contains("symbol") + ); + } + assert!( + emit_object_with_resources(ir, Target::LinuxX86_64, &vec![resource; 4097]) + .unwrap_err() + .contains("4096") + ); + assert!( + emit_object(&" ".repeat(8 * 1024 * 1024 + 1), Target::LinuxX86_64) + .unwrap_err() + .contains("8388608") + ); + let empty = BinaryResource { + symbol: "empty", + bytes: &[], + }; + for target in Target::ALL { + assert!( + !emit_object_with_resources( + "@empty = external constant [0 x i8]\ndefine ptr @read() { ret ptr @empty }", + target, + &[empty] + ) + .unwrap() + .is_empty() + ); + } +} + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[test] +fn binary_payload_larger_than_text_limit_links_and_preserves_every_byte() { + let bytes = (0..9 * 1024 * 1024) + .map(|index| (index % 251) as u8) + .collect::<Vec<_>>(); + let ir = format!( + r#"@payload = external constant [{} x i8] +define void @_start() noreturn {{ + %written = call i64 asm sideeffect "syscall", "={{rax}},{{rax}},{{rdi}},{{rsi}},{{rdx}},~{{rcx}},~{{r11}},~{{memory}}"(i64 1, i64 1, ptr @payload, i64 {}) + %complete = icmp eq i64 %written, {} + %failed = select i1 %complete, i64 0, i64 1 + call void asm sideeffect "syscall", "{{rax}},{{rdi}},~{{rcx}},~{{r11}},~{{memory}}"(i64 60, i64 %failed) + unreachable +}}"#, + bytes.len(), + bytes.len(), + bytes.len() + ); + assert!(ir.len() < 1024); + let directory = temp::TemporaryDirectory::new(); + let object = directory.path().join("entry.o"); + fs::write( + &object, + emit_object_with_resources( + &ir, + Target::LinuxX86_64, + &[BinaryResource { + symbol: "payload", + bytes: &bytes, + }], + ) + .unwrap(), + ) + .unwrap(); + let executable = directory.path().join("program"); + link( + Target::LinuxX86_64, + LinkKind::Executable, + &[object], + &executable, + Some("_start"), + ) + .unwrap(); + let output = directory.path().join("bytes"); + let status = std::process::Command::new(executable) + .env_clear() + .stdout(fs::File::create(&output).unwrap()) + .status() + .unwrap(); + assert!(status.success()); + assert_eq!(fs::read(output).unwrap(), bytes); +} + +#[test] +fn six_targets_emit_real_architecture_specific_objects_and_link() { + let directory = temp::TemporaryDirectory::new(); + for (index, target) in Target::ALL.into_iter().enumerate() { + let bytes = emit_object(IR, target).unwrap(); + assert!(bytes.len() > 64); + match target { + Target::LinuxX86_64 | Target::LinuxAarch64 => { + assert_eq!(&bytes[..4], b"\x7fELF"); + let machine = u16::from_le_bytes(bytes[18..20].try_into().unwrap()); + assert_eq!( + machine, + if target == Target::LinuxX86_64 { + 62 + } else { + 183 + } + ); + } + Target::MacosX86_64 | Target::MacosAarch64 => { + assert_eq!(&bytes[..4], b"\xcf\xfa\xed\xfe"); + let cpu = u32::from_le_bytes(bytes[4..8].try_into().unwrap()); + assert_eq!( + cpu, + if target == Target::MacosX86_64 { + 0x1000007 + } else { + 0x100000c + } + ); + } + Target::WindowsX86_64 | Target::WindowsAarch64 => { + let machine = u16::from_le_bytes(bytes[..2].try_into().unwrap()); + assert_eq!( + machine, + if target == Target::WindowsX86_64 { + 0x8664 + } else { + 0xaa64 + } + ); + } + } + let object = directory.path().join(format!("{index}.o")); + let output = directory.path().join(format!("{index}.image")); + fs::write(&object, bytes).unwrap(); + link(target, LinkKind::Shared, &[object], &output, None).unwrap(); + assert!(fs::metadata(output).unwrap().len() > 128); + } +} + +#[test] +fn malformed_ir_target_mismatch_and_failed_link_do_not_poison_later_calls() { + assert!(emit_object("not LLVM", Target::LinuxX86_64).is_err()); + assert!(emit_object("define i64 @bad() { ret i32 3 }", Target::LinuxX86_64).is_err()); + assert!(emit_object("", Target::LinuxX86_64).is_err()); + assert!( + emit_object("\0", Target::LinuxX86_64) + .unwrap_err() + .contains("NUL") + ); + let invalid_ssa = "define i64 @bad(i1 %condition) {\nentry: br i1 %condition, label %left, label %right\nleft: %value = add i64 40, 2\nbr label %done\nright: br label %done\ndone: ret i64 %value\n}"; + assert!( + emit_object(invalid_ssa, Target::LinuxX86_64) + .unwrap_err() + .contains("does not dominate") + ); + assert!( + emit_object( + "target datalayout = \"e-p:32:32\"\ndefine i64 @answer() { ret i64 42 }", + Target::LinuxX86_64 + ) + .unwrap_err() + .contains("layout disagrees") + ); + assert!( + emit_object( + "target triple = \"aarch64-unknown-linux-gnu\"\n", + Target::LinuxX86_64 + ) + .unwrap_err() + .contains("disagrees") + ); + let directory = temp::TemporaryDirectory::new(); + let object = directory.path().join("bad.o"); + fs::write(&object, b"not object").unwrap(); + let output = directory.path().join("image"); + assert!( + link( + Target::LinuxX86_64, + LinkKind::Shared, + std::slice::from_ref(&object), + &output, + None + ) + .is_err() + ); + assert!(!output.exists()); + fs::write(&object, emit_object(IR, Target::LinuxX86_64).unwrap()).unwrap(); + link( + Target::LinuxX86_64, + LinkKind::Shared, + std::slice::from_ref(&object), + &output, + None, + ) + .unwrap(); + let original = fs::read(&output).unwrap(); + assert!( + link( + Target::LinuxX86_64, + LinkKind::Shared, + &[object], + &output, + None + ) + .is_err() + ); + assert_eq!(fs::read(output).unwrap(), original); +} + +#[test] +fn concurrent_links_cannot_overwrite_the_same_new_output() { + let directory = temp::TemporaryDirectory::new(); + let object = directory.path().join("entry.o"); + let output = directory.path().join("image"); + fs::write(&object, emit_object(IR, Target::LinuxX86_64).unwrap()).unwrap(); + std::thread::scope(|scope| { + let execute = || { + link( + Target::LinuxX86_64, + LinkKind::Shared, + std::slice::from_ref(&object), + &output, + None, + ) + }; + let left = scope.spawn(execute); + let right = scope.spawn(execute); + assert_eq!( + [left.join().unwrap(), right.join().unwrap()] + .iter() + .filter(|result| result.is_ok()) + .count(), + 1 + ); + }); + assert!(fs::metadata(output).unwrap().len() > 128); +} + +#[cfg(unix)] +#[test] +fn link_rejects_shared_directories_and_mutable_ancestors_before_claiming_output() { + use std::os::unix::fs::{DirBuilderExt, PermissionsExt}; + let directory = temp::TemporaryDirectory::new(); + let object = directory.path().join("entry.o"); + fs::write(&object, emit_object(IR, Target::LinuxX86_64).unwrap()).unwrap(); + let output = directory.path().join("image"); + fs::set_permissions(directory.path(), fs::Permissions::from_mode(0o755)).unwrap(); + assert!( + link( + Target::LinuxX86_64, + LinkKind::Shared, + std::slice::from_ref(&object), + &output, + None + ) + .unwrap_err() + .contains("private directory") + ); + assert!(!output.exists()); + + let leaf = directory.path().join("private"); + let mut builder = fs::DirBuilder::new(); + builder.mode(0o700).create(&leaf).unwrap(); + fs::set_permissions(directory.path(), fs::Permissions::from_mode(0o777)).unwrap(); + let output = leaf.join("image"); + let result = link( + Target::LinuxX86_64, + LinkKind::Shared, + std::slice::from_ref(&object), + &output, + None, + ); + fs::set_permissions(directory.path(), fs::Permissions::from_mode(0o700)).unwrap(); + assert!(result.unwrap_err().contains("publicly writable ancestor")); + assert!(!output.exists()); +} + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[test] +fn native_llvm_lld_program_executes_without_rustc_or_cargo() { + // Test-owned freestanding process: no runtime or system library dependency. + let ir = "define void @_start() noreturn {\ncall void asm sideeffect \"syscall\", \"{rax},{rdi},~{rcx},~{r11},~{memory}\"(i64 60, i64 42)\nunreachable\n}"; + let directory = temp::TemporaryDirectory::new(); + let object = directory.path().join("entry.o"); + fs::write(&object, emit_object(ir, Target::LinuxX86_64).unwrap()).unwrap(); + let output = directory.path().join("program"); + link( + Target::LinuxX86_64, + LinkKind::Executable, + &[object], + &output, + Some("_start"), + ) + .unwrap(); + let status = std::process::Command::new(output) + .env_clear() + .status() + .unwrap(); + assert_eq!(status.code(), Some(42)); +} diff --git a/test/ci/README.md b/test/ci/README.md new file mode 100644 index 0000000..51fc06d --- /dev/null +++ b/test/ci/README.md @@ -0,0 +1,92 @@ +# Linux 质量门 + +`run.py` 仅串行编排既有 Cargo、Python、SDK、PostgreSQL fixture 和 performance 运行器；没有下载工具、全局安装或环境变量产品配置。根 `config/setting.json` 保留原有字段，在既有 `performance` 对象内增加 `linux_ci`，与 peer 路径同级。顶层 `linux_ci` 不受支持，会被 runtime 严格配置解析拒绝： + +```json +{ + "performance": { + "network_peer": "/absolute/network_bench", + "live_peer": "/absolute/live_bench", + "linux_ci": { + "tools": { + "cargo": "/absolute/toolchain/bin/cargo", + "cargo_fmt": "/absolute/rustfmt/bin/cargo-fmt", + "cargo_clippy": "/absolute/clippy/bin/cargo-clippy", + "python": "/absolute/python3", + "node": "/absolute/node", + "go": "/absolute/go" + }, + "cargo_config": [ + "profile.dev.package.miniz_oxide.opt-level=3", + "profile.dev.package.flate2.opt-level=3", + "profile.dev.package.sha2.opt-level=3" + ], + "output": "/absolute/repository/target/linux-ci", + "command_timeout_seconds": 14400, + "remove_test_executables": true, + "cgroup_parent": "/sys/fs/cgroup/system.slice/dever-linux-ci.service", + "inputs": [], + "required_inputs": {}, + "author_outputs": [], + "author_network": false, + "soak": { + "artifacts": "/absolute/prepared-protocols", + "cms_manifest": "/absolute/prepared-cms/manifest.json", + "seconds": 1800, + "cms_concurrency": 16, + "http2_rate": 5000, + "rss_late_growth_mib": 8 + } + } + } +} +``` + +替换路径，勿直接复制示例运行。下文 CI 字段均位于 `performance.linux_ci`。`inputs` 列出固定原始作者输入及准备好的资产文件/目录，逐文件计 SHA-256；工具、协议产物、CMS manifest/可执行文件/配置和 `performance.network_peer/live_peer` 也自动计摘要，`linux_ci` 对象不作为 peer 路径解析。输出目录不得包含在 inputs 中。报告只存摘要，不复制数据库密码。 + +`required_inputs` 为 `author`、`native`、`postgres`、`sandbox`、`bounded-perf`、`soak` 各提供非空绝对路径列表，必须落在 `inputs` 声明范围内。缺路径记录 blocked，不算 pass。作者 fixture 的具体固定路径/准备命令仍以 `sdk/native-release.md`、`test/ecosystem-release/prepare.py`、`test/ecosystem-release/build_inputs.py` 和对应 ignored 原因文本为准；gate 不把现存文件冒充由当前源码重建的 provenance。 + +显式 LLVM Worker 验收读取 `target/native-release-inputs/component-fixture`：先用当前源码构建 `dever-tests` 的 `component-fixture` bin，再将程序复制到该准备目录，记录源码/程序摘要，并把固定副本列入 `inputs` 和 `required_inputs.native`。不能将 `target/debug/component-fixture` 当作不可变输入：Cargo 在 default/all-feature 组合间会重新链接这个输出。普通 Worker 测试仍使用 Cargo 为各自组合生成的 `CARGO_BIN_EXE_component-fixture`，不改质量门的输入变更拒绝规则。 + +`author_outputs` 精确声明 author 会制作的 pack、下载结果和 receipt 路径。author 阶段只将其余原始输入当作不可变输入，另外记录完整输出摘要；后续验收按准备后完整输入摘要绑定。不要把整个作者输入根排除掉。显式 `author_network: true` 才能执行已分类的固定官方下载/源码构建。新 ignored 必须人工更新 `ignored.json`；inventory 核对源码声明及实际 all-feature libtest 名单，执行时一律使用完整 `--exact` 名称。内部 probe 记录 parent-only，只由所属父用例运行。 + +资源准备归外部托管者：自有空父启用 memory/cpu，限制总 CPU<=2、memory<=4GiB、swap=0；runner/Cargo/负载客户端放 `control` 子组，自有 PostgreSQL 可放兄弟子组，64/128MiB 服务子组由现有 performance owner 创建。gate 只读检查现有父限制与自身归属，不创建/修改主机策略。作者工具所需固定 PATH/动态库路径可由外部托管者显式准备，不写入产品配置。 + +```sh +python3 -B test/ci/run.py plan +python3 -B test/ci/run.py run --stage static +python3 -B test/ci/run.py run --stage default +python3 -B test/ci/run.py run --stage all-features +python3 -B test/ci/run.py run --stage features +python3 -B test/ci/run.py run --stage inventory +python3 -B test/ci/run.py run --stage author +python3 -B test/ci/run.py run --stage python +python3 -B test/ci/run.py run --stage sdk +python3 -B test/ci/run.py run --stage native +python3 -B test/ci/run.py run --stage postgres +python3 -B test/ci/run.py run --stage sandbox +python3 -B test/ci/run.py run --stage bounded-perf +``` + +以上命令由同一受限托管进程串行执行。若 author 改变输入，应在作者准备完成后再跑正式静态/default/all-feature门；较早输入版本的证据会显示过期。每阶段写入新的独占时间目录，开始即记录 running；缺输入记录 blocked，失败命令保留 exit/count/日志与原始异常。`summary` 与 inventory 消费者只接受最新一次尝试的当前源码/输入证据，不能越过较新的失败或中断回用旧 pass；所有必选阶段通过才返回0。源码摘要覆盖未提交文件，不只看 Git HEAD。 + +default/all-features 由实际 Cargo metadata 注册 target 和已解析 feature 形成串行矩阵，覆盖 lib/bin/integration/doctest；default 显式保留 workspace 解析的 package features，最小 feature 阶段另按单 package 验证。只有显式启用 `remove_test_executables` 时才删除 Cargo 本次返回、`profile.test=true` 且位于 target 下 `deps/` 的测试可执行文件，失败阶段同样清理；不删除作者工具、静态 init、依赖 archive 或整个 target。删除对象在日志 JSON 中可审计，重跑 Cargo 可恢复。inventory 按 package/target/name 三元组核对完整性，跨 target 同名用例不能代替缺失项。 + +Python 阶段包含 `test/ci`、`test/performance`、`test/ecosystem-release` 三个目录；后两者必须已配置 `performance.network_peer/live_peer`，真实自有 loopback检查由主代理执行。三 SDK 路径全部必需，三个明确成功标志全部出现才通过。 + +PostgreSQL 只接受显式 loopback端口和恰好一个 `{case}` 库名模板，拒绝 query 覆盖 host/dbname 等。增加 `performance.linux_ci.owned_postgres: {"data_directory":"/absolute/owned-cluster","port":51234}`，由托管者创建所需基库并在 finally 停止该自有实例；gate复用现有用例的 schema/tenant 清理，不替用户启动或连接既有业务数据库。 + +长测复用原 runner，逐阶段分别执行： + +```sh +for stage in cms-64 cms-128 http2-64 http2-128 recovery-64 recovery-128 live-64 live-128; do + python3 -B test/ci/run.py run --stage "$stage" || exit "$?" +done +python3 -B test/ci/run.py summary +``` + +每个 CMS 源码/h2c/TLS/恢复协议/TCP/WS/SSE 子场景至少30分钟，服务在整个子场景保持同一进程。HTTP/2选代表 JSON 路径；其他body路径仍由短合同测试覆盖。live/recovery 使用60×30秒循环，受现有1小时生命周期限制；总默认矩阵约9小时加准备。HTTP/2要求零错误、成功>=99%计划请求；CMS逐回复验证实际发布列表。全部要求OOM=0、断连FD恢复。RSS记录原始采样和四段中位数，最后两段中位数增量默认不得超过8MiB；这是本轮可配置回归预算，不把 allocator保留直接等同泄漏。失效报告和失败负载保留，校准降速后须完整重跑。 + +CMS读取时长支持0.1..3600秒；长测客户端用固定2048桶统计延迟，不随请求数保存无限样本。分位数为对数桶上界，包含请求与结果核对时间。报告记录实际时长及统计方式，不与此前精确短测分位数直接混算。 + +CMS先关闭控制客户端，再记录FD基线；断连后按FD目标及重复数核对所有非数据库资源。负载前已确定的SQLite主文件/WAL句柄各不得超过配置的`max_connections`，同进程SHM句柄不得超过1。`descriptors_before`、`descriptors_after`和`pooled_descriptor_limits`保存实际依据；总`fd_delta_after_disconnect`保留为观测值，不把正常池扩容当成连接泄漏。每个源码目录的`report.json`在失败时也保存已完成的发布/读取数据、资源采样、预算及错误，总报告收集该失败case。HTTP/2/live无数据库池，继续使用原来的完整FD回落门。 diff --git a/test/ci/acceptance.py b/test/ci/acceptance.py new file mode 100644 index 0000000..c4a073b --- /dev/null +++ b/test/ci/acceptance.py @@ -0,0 +1,298 @@ +"""显式 ignored 验收与现有长测运行器的 Linux gate 接线。""" + +import json +from pathlib import Path +import re +from urllib.parse import parse_qsl, unquote, urlsplit + +from linux_gate import (ROOT, MissingInput, absolute_file, run_command, + target_command, workspace_targets, remove_executables, input_fingerprint) +from settings import write_private_json +from process import resource_trend +from cms import validate_read_descriptor_recovery + + +def execute(command, config, directory, result, *, single_harness=False): + record, content = run_command(command, directory, len(result["commands"]), + config.get("command_timeout_seconds", 14400), + single_harness=single_harness) + result["commands"].append(record) + return record, content + + +def classify_cases(names, manifest, target): + by_name = {} + for row in manifest: + key = (row["package"], row["target"], row["name"]) + if key in by_name: + raise RuntimeError(f"duplicate ignored classification: {key}") + by_name[key] = row + classified = [] + for name in names: + leaf = name.rsplit("::", 1)[-1] + key = (target["package"], target["target"], leaf) + if key not in by_name: + raise RuntimeError(f"unclassified ignored test: {name}") + classified.append({**by_name[key], "exact": name}) + if len({case["name"] for case in classified}) != len(classified): + raise RuntimeError("same ignored leaf appears in multiple modules; classify by a distinct source name") + return classified + + +def audit_classification(): + manifest = json.loads((ROOT / "test/ci/ignored.json").read_text()) + stages = {"probe", "author", "native", "postgres", "sandbox", "bounded-perf"} + if any(row["stage"] not in stages for row in manifest): + raise RuntimeError("ignored classification contains an unknown execution stage") + expected = {(row["source"], row["name"], row["reason"]) for row in manifest} + actual = set() + pattern = re.compile(r'#\[ignore(?:\s*=\s*"([^"]*)")?\]\s*(?:#\[[^\]]*\]\s*)*(?:pub\s+)?(?:async\s+)?fn\s+(\w+)') + for directory in ("test", "crates"): + for path in (ROOT / directory).rglob("*.rs"): + for match in pattern.finditer(path.read_text()): + actual.add((str(path.relative_to(ROOT)), match[2], match[1] or "")) + if actual != expected: + raise RuntimeError(f"ignored classification drift: new={sorted(actual - expected)}, removed={sorted(expected - actual)}") + return manifest + + +def test_executable(content, target): + artifacts = [] + for line in content.splitlines(): + if not line.startswith("{"): + continue + message = json.loads(line) + if (message.get("reason") == "compiler-artifact" and message.get("profile", {}).get("test") + and message.get("target", {}).get("name") == target["target"] and message.get("executable")): + artifacts.append(message["executable"]) + if len(set(artifacts)) != 1: + raise RuntimeError(f"expected one libtest executable for {target}") + return artifacts[0] + + +def inventory(config, directory, result): + manifest = audit_classification() + targets, target_directory = workspace_targets(config, directory, result, True) + cases = [] + for target in targets: + if target["selector"] == ["--doc"]: + continue + _, content = execute([*target_command(config, target), "--no-run", "--message-format=json"], config, directory, result) + executable = test_executable(content, target) + _, listing = execute([executable, "--ignored", "--list", "--format", "terse"], config, directory, result) + names = [line.removesuffix(": test") for line in listing.splitlines() if line.endswith(": test")] + cases.extend({**case, "target": target, "target_directory": str(target_directory)} + for case in classify_cases(names, manifest, target)) + result.setdefault("removed_test_executables", []).extend( + remove_executables(content, target_directory, config.get("remove_test_executables", False))) + write_private_json(directory / "inventory.json", cases, replace=(directory / "inventory.json").exists()) + missing = {(row["package"], row["target"], row["name"]) for row in manifest} - { + (case["target"]["package"], case["target"]["target"], case["name"]) for case in cases} + if missing: + raise RuntimeError(f"ignored source cases missing from Linux all-feature libtest inventory: {sorted(missing)}") + result["classified_cases"] = len(cases) + result["internal_probes"] = [{"target": case["target"], "exact": case["exact"], + "status": "parent-only", "reason": case["reason"]} + for case in cases if case["stage"] == "probe"] + + +def current_inventory(document, config, result): + paths = sorted((Path(config.get("output", ROOT / "target/linux-ci")) / "inventory").glob("*/result.json")) + if paths: + path = paths[-1] + record = json.loads(path.read_text()) + if (record.get("source"), record.get("inputs"), record.get("status")) == (result["source"], input_fingerprint(document, config, "inventory"), "passed"): + return json.loads((path.parent / "inventory.json").read_text()) + raise MissingInput("run inventory for the current source and input fingerprints first") + + +def validate_postgres(document): + setting = document.get("database", {}).get("postgres_test", {}) + url = urlsplit(setting.get("url", "")) + database = unquote(url.path.removeprefix("/")) + if (setting.get("type") != "postgres" or url.scheme not in ("postgres", "postgresql") + or url.hostname not in ("127.0.0.1", "::1") or url.port is None + or database.count("{case}") != 1 + or any(key.lower() in ("dbname", "host", "hostaddr", "port", "service") for key, _ in parse_qsl(url.query))): + raise MissingInput("database.postgres_test must select the explicitly owned loopback cluster and one {case} database template") + + +def require_inputs(stage, config): + required = config.get("required_inputs", {}).get(stage) + if not required: + raise MissingInput(f"performance.linux_ci.required_inputs.{stage} must declare the prepared author/fixture input paths") + declared = [Path(value).resolve() for value in config.get("inputs", [])] + for value in required: + path = Path(value) + if not path.is_absolute() or not path.exists(): + raise MissingInput(f"missing {stage} input: {value}") + if not any(path.resolve() == root or path.resolve().is_relative_to(root) for root in declared): + raise MissingInput(f"{stage} input is not fingerprinted by performance.linux_ci.inputs: {value}") + + +def ignored_stage(stage, document, config, directory, result): + require_inputs(stage, config) + if stage == "postgres": + validate_postgres(document) + if not config.get("owned_postgres"): + raise MissingInput("performance.linux_ci.owned_postgres must record the owned cluster data_directory and port") + owned = config["owned_postgres"] + if urlsplit(document["database"]["postgres_test"]["url"]).port != owned.get("port"): + raise MissingInput("PostgreSQL URL does not match performance.linux_ci.owned_postgres.port") + if not Path(owned.get("data_directory", "")).is_absolute(): + raise MissingInput("owned_postgres.data_directory must be absolute") + if stage == "author" and config.get("author_network") is not True: + raise MissingInput("author stage fetches fixed upstream inputs; explicitly set performance.linux_ci.author_network=true") + if stage == "author" and not config.get("author_outputs"): + raise MissingInput("declare performance.linux_ci.author_outputs separately from immutable raw author inputs") + cases = [case for case in current_inventory(document, config, result) if case["stage"] == stage] + if not cases: + raise RuntimeError(f"no classified cases for {stage}") + # 固定作者先准备工具，再执行真实源码包构建；其他用例按 target 复用链接产物。 + cases.sort(key=lambda case: (0 if case["name"].startswith("prepare_") else 1, + case["target"]["package"], case["target"]["target"], case["exact"])) + target_key = None + build_content = None + target_directory = ROOT / "target" + for case in cases: + target = case["target"] + target_directory = Path(case["target_directory"]) + result["target_directory"] = str(target_directory) + key = json.dumps(target, sort_keys=True) + if key != target_key: + if build_content is not None: + result.setdefault("removed_test_executables", []).extend( + remove_executables(build_content, target_directory, config.get("remove_test_executables", False))) + _, build_content = execute([*target_command(config, target), "--no-run", "--message-format=json"], config, directory, result) + executable = test_executable(build_content, target) + target_key = key + record, _ = execute([executable, "--ignored", "--exact", case["exact"], "--test-threads=1", "--nocapture"], + config, directory, result, single_harness=True) + if record.get("rust_tests") != {"passed": 1, "failed": 0, "ignored": 0}: + raise RuntimeError(f"exact ignored selection did not execute one case: {case['exact']}") + result.setdefault("cases", []).append({"exact": case["exact"], "target": target, "status": "passed"}) + result.setdefault("removed_test_executables", []).extend( + remove_executables(build_content, target_directory, config.get("remove_test_executables", False))) + + +def validate_budget(config): + parent = Path(config.get("cgroup_parent", "")) + if not parent.is_absolute() or not (parent / "cgroup.controllers").is_file(): + raise MissingInput("performance.linux_ci.cgroup_parent must select the owned empty cgroup v2 parent") + if (parent / "cgroup.procs").read_text().strip(): + raise MissingInput("CI runner must be in a control child, not the cgroup parent") + if not {"cpu", "memory"} <= set((parent / "cgroup.subtree_control").read_text().split()): + raise MissingInput("owned parent requires enabled memory/cpu child controllers") + quota, period = (parent / "cpu.max").read_text().split() + memory = (parent / "memory.max").read_text().strip() + if quota == "max" or int(quota) / int(period) > 2 or memory == "max" or int(memory) > 4 * 1024 ** 3: + raise MissingInput("owned parent must bound total CPU <=2 and memory <=4GiB") + if (parent / "memory.swap.max").read_text().strip() != "0": + raise MissingInput("owned parent must disable swap") + membership = Path("/proc/self/cgroup").read_text().strip().split("::", 1)[1] + current = Path("/sys/fs/cgroup") / membership.lstrip("/") + if current == parent or not current.is_relative_to(parent): + raise MissingInput("CI runner is outside the configured owned parent control subgroup") + return parent + + +def validate_soak(report, suite, seconds, growth_limit, memory_mib): + if report.get("status") not in ("passed", "complete"): + raise RuntimeError("soak runner did not complete") + cases = list(report["cases"].values()) if suite == "cms" else report["cases"] + expected = 3 if suite == "live" else 2 + if len(cases) != expected: + raise RuntimeError(f"expected {expected} {suite} cases, received {len(cases)}") + if suite == "cms": + coverage = set(report["cases"]) + required = {"dever", "md"} + else: + coverage = {case["protocol" if suite == "live" else "transport"] for case in cases} + required = {"tcp", "ws", "sse"} if suite == "live" else {"http", "https"} + if any(case["suite"] != suite or (suite != "live" and case["implementation"] != "dever") for case in cases): + raise RuntimeError("soak report contains a different suite or implementation") + if coverage != required: + raise RuntimeError("soak report does not cover every required source or transport") + for case in cases: + budget = case.get("cgroup") if suite == "cms" else case.get("budget_after_stop") + if budget is None or any(budget["events"][key] for key in ("oom", "oom_kill")): + raise RuntimeError("missing memory-budget evidence or nonzero OOM") + if budget["max"] != str(memory_mib * 1024 ** 2): + raise RuntimeError("soak used a different memory limit from the selected stage") + if suite == "cms": + load = case["concurrent_reads"] + if load["elapsed_seconds"] < seconds or load["errors"] or load["succeeded"] == 0: + raise RuntimeError("CMS duration, business load or FD recovery failed") + validate_read_descriptor_recovery(load) + trend = load["trend"] + elif suite == "live": + if (case["result"]["elapsed_ms"] < seconds * 1000 or case["result"]["errors"] + or any(case["fd_delta_after_cycles"])): + raise RuntimeError("live duration, protocol or FD recovery failed") + samples = [phase["last"] for phase in case["phases"] if phase["phase"] == "disconnected"] + trend = resource_trend(samples) + else: + loads = [case["load"]] if suite == "http2" else [cycle["load"] for cycle in case["cycles"]] + disconnected = [case["disconnected"]] if suite == "http2" else [cycle["disconnected"] for cycle in case["cycles"]] + if any(sample["fd_delta"] != 0 for sample in disconnected): + raise RuntimeError("HTTP/2 did not restore file descriptors after disconnect") + if sum(load["elapsed_seconds"] for load in loads) < seconds: + raise RuntimeError("HTTP/2 measured load is shorter than the required duration") + for load in loads: + if load["errors"] or load["succeeded"] <= 0 or load["succeeded"] < .99 * load["scheduled"]: + raise RuntimeError("HTTP/2 did not achieve 99% scheduled successful replies with zero errors") + trend = resource_trend(case["raw"]) + if trend is None or trend["rss_late_growth_bytes"] > growth_limit: + raise RuntimeError("late RSS growth exceeds the configured budget; inspect raw resource samples") + case["gate_resource_trend"] = trend + + +def soak_stage(stage, config, directory, result): + suite, memory = stage.rsplit("-", 1) + suite = {"recovery": "http2-recovery"}.get(suite, suite) + require_inputs("soak", config) + parent = validate_budget(config) + settings = config.get("soak", {}) + seconds = settings.get("seconds", 1800) + if not 1800 <= seconds <= 2700: + raise ValueError("performance.linux_ci.soak.seconds must be 1800..2700") + python = str(absolute_file(config["tools"].get("python"), "performance.linux_ci.tools.python")) + output = directory / "measurement" + if suite == "cms": + manifest = str(absolute_file(settings.get("cms_manifest"), "performance.linux_ci.soak.cms_manifest")) + command = [python, "-B", "test/performance/cms.py", "run", "--manifest", manifest, + "--concurrency", str(settings.get("cms_concurrency", 16)), "--read-seconds", str(seconds)] + else: + artifacts = Path(settings.get("artifacts", "")) + if not artifacts.is_absolute() or not (artifacts / "manifest.json").is_file(): + raise MissingInput("performance.linux_ci.soak.artifacts requires prepared protocol artifacts") + command = [python, "-B", "test/performance/run.py", "run", "--artifacts", str(artifacts), + "--suite", suite, "--implementations", "dever", "--repeats", "1", "--interval", "0.1"] + if suite == "http2": + command += ["--duration", str(seconds), "--warmup", "5", "--paths", "/json", + "--rates", str(settings.get("http2_rate", 5000)), "--http2-connections", "4"] + else: + if seconds % 30: + raise ValueError("cyclic soak seconds must be divisible by 30") + command += ["--duration", "30", "--cycles", str(seconds // 30)] + command += (["--connection-tiers", "32"] if suite == "live" else + ["--recovery-rate", str(settings.get("http2_rate", 5000)), + "--recovery-path", "/json", "--recovery-connections", "4"]) + if "server_cpu" in settings: + command += ["--server-cpu", str(settings["server_cpu"]), "--client-cpus", str(settings["client_cpu"])] + command += ["--output", str(output), "--memory-mib", memory, "--cgroup-parent", str(parent)] + execute(command, config, directory, result) + report = json.loads((output / "report.json").read_text()) + validate_soak(report, suite, seconds, settings.get("rss_late_growth_mib", 8) * 1024 ** 2, int(memory)) + write_private_json(directory / "validated-report.json", report) + result["measurement"] = str(output / "report.json") + result["coverage"] = "CMS dever/md; HTTP/2 h2c/TLS JSON; recovery h2c/TLS; live TCP/WS/SSE, each selected stage only" + + +def run_stage(stage, document, config, directory, result): + if stage == "inventory": + inventory(config, directory, result) + elif stage in ("author", "native", "postgres", "sandbox", "bounded-perf"): + ignored_stage(stage, document, config, directory, result) + else: + soak_stage(stage, config, directory, result) diff --git a/test/ci/ignored.json b/test/ci/ignored.json new file mode 100644 index 0000000..545dc39 --- /dev/null +++ b/test/ci/ignored.json @@ -0,0 +1,1738 @@ +[ + { + "source": "test/dever-sandbox-tests/isolation.rs", + "name": "unprivileged_launcher_probe", + "reason": "internal non-root launcher probe, invoked by enforces_unprivileged_namespace", + "stage": "probe", + "package": "dever-sandbox", + "target": "isolation" + }, + { + "source": "test/dever-sandbox-tests/trusted_helper.rs", + "name": "rejects_untrusted_helper_paths_bytes_modes_owners_and_capabilities", + "reason": "requires root and the prepared static author bwrap; touches only owned target files", + "stage": "sandbox", + "package": "dever-sandbox", + "target": "dever_sandbox" + }, + { + "source": "test/native_runtime_link.rs", + "name": "canonical_c_header_links_and_executes_the_real_runtime_archive", + "reason": "build runtime-abi-only staticlib in target/native-runtime-abi before running", + "stage": "native", + "package": "dever-backend-bridge", + "target": "native_runtime_link" + }, + { + "source": "test/native_runtime_link.rs", + "name": "llvm_object_calls_the_versioned_runtime_abi_without_rust_layouts", + "reason": "build runtime-abi-only staticlib in target/native-runtime-abi before running", + "stage": "native", + "package": "dever-backend-bridge", + "target": "native_runtime_link" + }, + { + "source": "test/native_runtime_link.rs", + "name": "protocol_poll_rejects_invalid_output_before_advancing_the_operation", + "reason": "build runtime-abi-only staticlib in target/native-runtime-abi before running", + "stage": "native", + "package": "dever-backend-bridge", + "target": "native_runtime_link" + }, + { + "source": "test/native_runtime_link.rs", + "name": "logger_flush_is_a_stable_native_allocation_barrier", + "reason": "build runtime-abi-only staticlib in target/native-runtime-abi before running", + "stage": "native", + "package": "dever-backend-bridge", + "target": "native_runtime_link" + }, + { + "source": "test/native_runtime_link.rs", + "name": "wire_codecs_validate_slots_before_callback_and_release_partial_values", + "reason": "build runtime-abi-only staticlib in target/native-runtime-abi before running", + "stage": "native", + "package": "dever-backend-bridge", + "target": "native_runtime_link" + }, + { + "source": "test/native_runtime_link.rs", + "name": "database_abi_preserves_typed_owners_and_rejects_invalid_slots_without_io", + "reason": "build a runtime-sqlite or runtime-postgres staticlib in target/native-runtime-abi before running", + "stage": "native", + "package": "dever-backend-bridge", + "target": "native_runtime_link" + }, + { + "source": "test/llvm_managed_native.rs", + "name": "unicode_text_nullable_results_and_aliases_execute_and_release", + "reason": "build runtime-abi-only staticlib in target/native-runtime-abi before running", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_managed_native" + }, + { + "source": "test/llvm_managed_native.rs", + "name": "decimal_numbers_parsing_rendering_and_ieee_intrinsics_execute", + "reason": "build runtime-abi-only staticlib in target/native-runtime-abi before running", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_managed_native" + }, + { + "source": "test/llvm_managed_native.rs", + "name": "nested_records_choices_nullable_and_field_overwrites_keep_value_semantics", + "reason": "build runtime-abi-only staticlib in target/native-runtime-abi before running", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_managed_native" + }, + { + "source": "test/llvm_managed_native.rs", + "name": "list_map_cow_order_and_supported_map_keys_execute", + "reason": "build runtime-abi-only staticlib in target/native-runtime-abi before running", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_managed_native" + }, + { + "source": "test/llvm_managed_native.rs", + "name": "static_handler_traversals_and_bytes_execute_without_interpreting_values", + "reason": "build runtime-abi-only staticlib in target/native-runtime-abi before running", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_managed_native" + }, + { + "source": "test/llvm_managed_native.rs", + "name": "nullable_elements_flatten_and_unit_handlers_and_owned_reduce_state_release", + "reason": "build runtime-abi-only staticlib in target/native-runtime-abi before running", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_managed_native" + }, + { + "source": "test/llvm_managed_native.rs", + "name": "traversal_failure_cleans_managed_arguments_results_and_error_paths", + "reason": "build runtime-abi-only staticlib in target/native-runtime-abi before running", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_managed_native" + }, + { + "source": "test/llvm_managed_native.rs", + "name": "dynamic_map_duplicate_key_preserves_runtime_diagnostic_and_releases_it", + "reason": "build runtime-abi-only staticlib in target/native-runtime-abi before running", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_managed_native" + }, + { + "source": "test/llvm_managed_native.rs", + "name": "root_owned_outputs_are_transferred_and_released_by_the_caller", + "reason": "build runtime-abi-only staticlib in target/native-runtime-abi before running", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_managed_native" + }, + { + "source": "test/llvm_result_native.rs", + "name": "direct_failure_and_single_success_capture", + "reason": "requires explicit target/native-runtime-abi staticlib", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_result_native" + }, + { + "source": "test/llvm_result_native.rs", + "name": "zero_and_multiple_outputs_capture_in_order", + "reason": "requires explicit target/native-runtime-abi staticlib", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_result_native" + }, + { + "source": "test/llvm_result_native.rs", + "name": "wrapped_and_explicit_outer_failures_keep_nominal_identity", + "reason": "requires explicit target/native-runtime-abi staticlib", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_result_native" + }, + { + "source": "test/llvm_result_native.rs", + "name": "static_handler_failures_release_cursors_context_and_state", + "reason": "requires explicit target/native-runtime-abi staticlib", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_result_native" + }, + { + "source": "test/llvm_result_native.rs", + "name": "root_failure_retains_business_code_owned_payload_and_call_chain", + "reason": "requires explicit target/native-runtime-abi staticlib", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_result_native" + }, + { + "source": "test/llvm_result_native.rs", + "name": "nested_owned_failure_payload_keeps_cow_aliases_independent", + "reason": "requires explicit target/native-runtime-abi staticlib", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_result_native" + }, + { + "source": "test/llvm_result_native.rs", + "name": "captured_success_transfers_owned_root_output", + "reason": "requires explicit target/native-runtime-abi staticlib", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_result_native" + }, + { + "source": "test/llvm_result_native.rs", + "name": "capture_does_not_consume_dynamic_numeric_faults", + "reason": "requires explicit target/native-runtime-abi staticlib", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_result_native" + }, + { + "source": "test/llvm_result_native.rs", + "name": "capture_does_not_consume_original_runtime_abi_faults", + "reason": "requires explicit target/native-runtime-abi staticlib", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_result_native" + }, + { + "source": "test/llvm_result_native.rs", + "name": "bytes_results_decode_unicode_and_keep_aliases", + "reason": "requires explicit target/native-runtime-abi staticlib", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_result_native" + }, + { + "source": "test/llvm_result_native.rs", + "name": "bytes_failures_keep_original_messages", + "reason": "requires explicit target/native-runtime-abi staticlib", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_result_native" + }, + { + "source": "test/llvm_api_security_cases.rs", + "name": "private_rest_isolates_tenants_roles_identity_and_transactions", + "reason": "requires explicit runtime-api/sqlite archive; uses only owned loopback and two owned tenant files", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_api_source" + }, + { + "source": "test/llvm_api_security_cases.rs", + "name": "postgres_private_rest_uses_the_same_identity_permission_and_transaction_contract", + "reason": "requires explicit PostgreSQL fixture in config/setting.json; owns control and two tenant databases", + "stage": "postgres", + "package": "dever-backend-bridge", + "target": "llvm_api_source" + }, + { + "source": "test/llvm_api_security_cases.rs", + "name": "disabled_component_cannot_be_bypassed_by_tenant_owner", + "reason": "requires explicit runtime-api/sqlite archive; owns all listener, component and tenant state", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_api_source" + }, + { + "source": "test/llvm_database_source.rs", + "name": "sqlite_crud_defaults_owned_rows_and_repeat_entry", + "reason": "requires explicitly prepared database runtime archive; owned SQLite only", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_database_source" + }, + { + "source": "test/llvm_database_source.rs", + "name": "sqlite_bulk_upsert_and_static_predicates", + "reason": "requires explicitly prepared database runtime archive; owned SQLite only", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_database_source" + }, + { + "source": "test/llvm_database_source.rs", + "name": "sqlite_nullable_equality_keeps_present_and_absent_binds", + "reason": "requires explicitly prepared database runtime archive; owned SQLite only", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_database_source" + }, + { + "source": "test/llvm_database_source.rs", + "name": "sqlite_cursor_ties_nullable_next_and_offset_page", + "reason": "requires explicitly prepared database runtime archive; owned SQLite only", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_database_source" + }, + { + "source": "test/llvm_database_source.rs", + "name": "sqlite_persistent_scalars_choices_and_private_fields_round_trip", + "reason": "requires explicitly prepared database runtime archive; owned SQLite only", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_database_source" + }, + { + "source": "test/llvm_database_source.rs", + "name": "sqlite_nested_transactions_share_the_one_connection", + "reason": "requires explicitly prepared database runtime archive; owned SQLite only", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_database_source" + }, + { + "source": "test/llvm_database_source.rs", + "name": "sqlite_row_stream_full_early_stop_and_close_release_connections", + "reason": "requires explicitly prepared database runtime archive; owned SQLite only", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_database_source" + }, + { + "source": "test/llvm_database_source.rs", + "name": "sqlite_static_sql_typed_optional_list_and_record_results", + "reason": "requires explicitly prepared database runtime archive; owned SQLite only", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_database_source" + }, + { + "source": "test/llvm_database_source.rs", + "name": "sqlite_captured_nested_database_faults_keep_exact_categories", + "reason": "requires explicitly prepared database runtime archive; owned SQLite only", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_database_source" + }, + { + "source": "test/llvm_database_source.rs", + "name": "sqlite_task_wait_keeps_database_failure_identity", + "reason": "requires explicitly prepared database runtime archive; owned SQLite only", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_database_source" + }, + { + "source": "test/llvm_database_source.rs", + "name": "sqlite_numeric_fault_rolls_back_before_the_next_entry", + "reason": "requires explicitly prepared database runtime archive; owned SQLite only", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_database_source" + }, + { + "source": "test/llvm_database_source.rs", + "name": "sqlite_command_encoding_failure_rolls_back_before_retry", + "reason": "requires explicitly prepared database runtime archive; owned SQLite only", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_database_source" + }, + { + "source": "test/llvm_database_source.rs", + "name": "sqlite_uncaught_constraint_rolls_back_before_the_next_command", + "reason": "requires explicitly prepared database runtime archive; owned SQLite only", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_database_source" + }, + { + "source": "test/llvm_database_source.rs", + "name": "sqlite_sequential_database_handler_inherits_transaction_context", + "reason": "requires explicitly prepared database runtime archive; owned SQLite only", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_database_source" + }, + { + "source": "test/llvm_database_source.rs", + "name": "sqlite_seed_history_is_reused_without_overwriting_business_rows", + "reason": "requires explicitly prepared database runtime archive; owned SQLite only", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_database_source" + }, + { + "source": "test/llvm_database_source.rs", + "name": "sqlite_empty_create_many_retains_valid_generated_defaults", + "reason": "requires explicitly prepared database runtime archive; owned SQLite only", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_database_source" + }, + { + "source": "test/llvm_database_source.rs", + "name": "sqlite_related_cycles_join_batch_and_unloaded_values", + "reason": "requires explicitly prepared database runtime archive; owned SQLite only", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_database_source" + }, + { + "source": "test/llvm_database_source.rs", + "name": "sqlite_schema_upgrade_preserves_rows_and_replays_migration_and_seed_once", + "reason": "requires explicitly prepared database runtime archive; owned SQLite only", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_database_source" + }, + { + "source": "test/llvm_database_source.rs", + "name": "sqlite_cancelled_row_stream_drains_before_single_connection_fallback", + "reason": "requires explicitly prepared database runtime archive; owned SQLite only", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_database_source" + }, + { + "source": "test/llvm_database_source.rs", + "name": "postgres_checked_models_transactions_streams_and_schema_upgrade", + "reason": "requires configured isolated database.postgres_test and an explicit database runtime archive", + "stage": "postgres", + "package": "dever-backend-bridge", + "target": "llvm_database_source" + }, + { + "source": "test/llvm_database_source.rs", + "name": "postgres_command_encoding_failure_rolls_back_before_retry", + "reason": "requires configured isolated database.postgres_test and an explicit database runtime archive", + "stage": "postgres", + "package": "dever-backend-bridge", + "target": "llvm_database_source" + }, + { + "source": "test/llvm_database_source.rs", + "name": "postgres_uncaught_constraint_rolls_back_before_the_next_command", + "reason": "requires configured isolated database.postgres_test and an explicit database runtime archive", + "stage": "postgres", + "package": "dever-backend-bridge", + "target": "llvm_database_source" + }, + { + "source": "test/llvm_stream_network_source.rs", + "name": "nullable_memory_streams_early_stop_and_alias_close", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_stream_network_source" + }, + { + "source": "test/llvm_stream_network_source.rs", + "name": "channel_stream_backpressure_with_suspending_handler", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_stream_network_source" + }, + { + "source": "test/llvm_stream_network_source.rs", + "name": "ticks_suspend_then_early_stop_closes_aliases", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_stream_network_source" + }, + { + "source": "test/llvm_stream_network_source.rs", + "name": "parallel_stream_and_memory_workers_preserve_all_values", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_stream_network_source" + }, + { + "source": "test/llvm_stream_network_source.rs", + "name": "parallel_stream_preserves_typed_owned_worker_failure", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_stream_network_source" + }, + { + "source": "test/llvm_stream_network_source.rs", + "name": "parallel_failure_interrupts_idle_producer_and_preserves_channel_alias", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_stream_network_source" + }, + { + "source": "test/llvm_stream_network_source.rs", + "name": "synchronous_parallel_memory_dispatch_preserves_failure_and_ownership", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_stream_network_source" + }, + { + "source": "test/llvm_stream_network_source.rs", + "name": "async_parallel_dispatch_supports_non_suspending_handlers", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_stream_network_source" + }, + { + "source": "test/llvm_stream_network_source.rs", + "name": "synchronous_file_stream_parallel_failure_releases_workers_and_rows", + "reason": "requires explicit runtime-abi archive and owned temporary file", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_stream_network_source" + }, + { + "source": "test/llvm_stream_network_source.rs", + "name": "timeout_preserves_owned_multi_output_and_some_null", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_stream_network_source" + }, + { + "source": "test/llvm_stream_network_source.rs", + "name": "race_preserves_owned_multi_output_and_drains_losers", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_stream_network_source" + }, + { + "source": "test/llvm_stream_network_source.rs", + "name": "zero_output_timeout_and_race_preserve_presence_and_cleanup", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_stream_network_source" + }, + { + "source": "test/llvm_stream_network_source.rs", + "name": "timeout_and_race_propagate_business_failure_without_fallback", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_stream_network_source" + }, + { + "source": "test/llvm_stream_network_source.rs", + "name": "tcp_read_write_eof_and_shared_resource_close", + "reason": "requires explicit runtime-abi archive and owned loopback sockets", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_stream_network_source" + }, + { + "source": "test/llvm_stream_network_source.rs", + "name": "tcp_chunks_collects_exact_bytes_across_short_reads_and_eof", + "reason": "requires explicit runtime-abi archive and owned loopback sockets", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_stream_network_source" + }, + { + "source": "test/llvm_stream_network_source.rs", + "name": "invalid_network_inputs_preserve_original_failure_messages", + "reason": "requires explicit runtime-abi archive and owned loopback sockets", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_stream_network_source" + }, + { + "source": "test/llvm_stream_network_source.rs", + "name": "chunks_and_connections_emit_one_terminal_failure_before_end", + "reason": "requires explicit runtime-abi archive and owned loopback sockets", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_stream_network_source" + }, + { + "source": "test/llvm_stream_network_source.rs", + "name": "timeout_drains_pending_descendant_socket_before_fallback_returns", + "reason": "requires explicit runtime-abi archive and owned loopback sockets", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_stream_network_source" + }, + { + "source": "test/llvm_stream_network_source.rs", + "name": "zero_ticks_is_an_original_uncaptured_runtime_fault", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_stream_network_source" + }, + { + "source": "test/llvm_stream_network_source.rs", + "name": "zero_task_timeout_is_an_original_uncaptured_runtime_fault", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_stream_network_source" + }, + { + "source": "test/llvm_stream_network_source.rs", + "name": "timeout_and_race_keep_numeric_fault_kind_and_origin", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_stream_network_source" + }, + { + "source": "test/llvm_job_test_source.rs", + "name": "pure_suite_reports_source_failures_without_loading_deployment_settings", + "reason": "requires explicit runtime archive; bounded native ownership checks", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_job_test_source" + }, + { + "source": "test/llvm_job_test_source.rs", + "name": "jobs_use_case_fakes_clock_retry_cron_and_atomic_transactions", + "reason": "requires explicit runtime archive; isolated SQLite and no production Worker", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_job_test_source" + }, + { + "source": "test/llvm_job_test_source.rs", + "name": "suite_keeps_blocking_and_async_fakes_separate", + "reason": "requires explicit runtime archive; case-local Port effects and no external Worker", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_job_test_source" + }, + { + "source": "test/llvm_job_test_source.rs", + "name": "job_dispatch_blocks_incompatible_schema_payload_and_policy", + "reason": "requires explicit runtime archive; inspect only this suite's persisted queue", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_job_test_source" + }, + { + "source": "test/llvm_job_test_source.rs", + "name": "command_enqueues_system_jobs_and_releases_repeated_application_roots", + "reason": "requires explicit runtime archive; CMD enters System scope without running worker", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_job_test_source" + }, + { + "source": "test/llvm_job_test_source.rs", + "name": "invalid_worker_policy_fails_before_creating_database", + "reason": "requires explicit runtime archive; invalid worker policy must fail before schema I/O", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_job_test_source" + }, + { + "source": "test/llvm_job_test_source.rs", + "name": "timed_out_job_drains_children_and_rolls_back_before_retry", + "reason": "requires explicit runtime archive; bounded task cancellation inside owned SQLite transactions", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_job_test_source" + }, + { + "source": "test/llvm_async_source.rs", + "name": "named_async_call_preserves_nested_managed_output", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_async_source" + }, + { + "source": "test/llvm_async_source.rs", + "name": "cloned_channel_fields_survive_their_source_scope", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_async_source" + }, + { + "source": "test/llvm_async_source.rs", + "name": "task_wait_and_capacity_one_group_preserve_order", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_async_source" + }, + { + "source": "test/llvm_async_source.rs", + "name": "nullable_channel_backpressure_close_and_drain", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_async_source" + }, + { + "source": "test/llvm_async_source.rs", + "name": "task_business_failure_preserves_owned_payload", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_async_source" + }, + { + "source": "test/llvm_async_source.rs", + "name": "stop_task_group_and_nested_descendants_release_frames", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_async_source" + }, + { + "source": "test/llvm_async_source.rs", + "name": "blocking_and_parallel_return_checked_values", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_async_source" + }, + { + "source": "test/llvm_async_source.rs", + "name": "capture_then_refail_preserves_owned_history", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_async_source" + }, + { + "source": "test/llvm_async_source.rs", + "name": "runtime_fault_is_not_captured_as_business_failure", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_async_source" + }, + { + "source": "test/llvm_async_source.rs", + "name": "blocking_sync_call_propagates_typed_runtime_fault", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_async_source" + }, + { + "source": "test/llvm_async_source.rs", + "name": "blocking_abi_fault_keeps_original_buffer_and_is_not_captured", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_async_source" + }, + { + "source": "test/llvm_async_source.rs", + "name": "async_static_handlers_preserve_traversal_state", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_async_source" + }, + { + "source": "test/llvm_async_source.rs", + "name": "synchronous_task_multi_output_and_group_unit", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_async_source" + }, + { + "source": "test/llvm_async_source.rs", + "name": "blocking_and_parallel_preserve_owned_inputs_and_outputs", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_async_source" + }, + { + "source": "test/llvm_async_source.rs", + "name": "blocking_and_parallel_preserve_typed_business_failure", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_async_source" + }, + { + "source": "test/llvm_async_source.rs", + "name": "negative_task_sleep_keeps_runtime_message", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_async_source" + }, + { + "source": "test/llvm_async_source.rs", + "name": "synchronous_channel_only_root_releases_aliases", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_async_source" + }, + { + "source": "test/llvm_async_source.rs", + "name": "default_failure_retains_origin_callers_and_owned_payload", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_async_source" + }, + { + "source": "test/llvm_external_source.rs", + "name": "worker_success_and_declared_failure_survive_repeated_roots", + "reason": "requires explicit runtime-external archive and existing fixture Worker", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_external_source" + }, + { + "source": "test/llvm_external_source.rs", + "name": "typed_nested_values_and_multiple_outputs_round_trip", + "reason": "requires explicit runtime-external archive and existing fixture Worker", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_external_source" + }, + { + "source": "test/llvm_external_source.rs", + "name": "malformed_output_and_business_error_payloads_are_runtime_faults", + "reason": "requires explicit runtime-external archive and existing fixture Worker", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_external_source" + }, + { + "source": "test/llvm_external_source.rs", + "name": "handshake_and_setting_failures_do_not_leak_a_worker", + "reason": "requires explicit runtime-external archive and existing fixture Worker", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_external_source" + }, + { + "source": "test/llvm_external_source.rs", + "name": "cancelling_a_call_preserves_the_shared_worker", + "reason": "requires explicit runtime-external archive and existing fixture Worker", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_external_source" + }, + { + "source": "test/llvm_external_source.rs", + "name": "independent_executables_use_verified_embedded_exec_and_managed_launches", + "reason": "requires explicit runtime-external archive and existing fixture Worker", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_external_source" + }, + { + "source": "test/llvm_job_security_cases.rs", + "name": "queued_users_recheck_roles_sessions_catalog_and_tenant_components", + "reason": "requires explicit runtime archive; owned loopback API/worker and two tenant SQLite databases", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_api_source" + }, + { + "source": "test/llvm_async_native.rs", + "name": "switched_resume_coroutine_suspends_resumes_and_destroys", + "reason": "build runtime-abi-only staticlib in target/native-runtime-abi before running", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_async_native" + }, + { + "source": "test/llvm_async_native.rs", + "name": "async_operations_preserve_scope_and_backpressure", + "reason": "build runtime-abi-only staticlib in target/native-runtime-abi before running", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_async_native" + }, + { + "source": "test/llvm_resource_native.rs", + "name": "read_eof_and_alias_close_preserve_business_error_identity", + "reason": "requires the explicitly prepared runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_resource_native" + }, + { + "source": "test/llvm_resource_native.rs", + "name": "stream_aliases_share_progress_and_early_close", + "reason": "requires the explicitly prepared runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_resource_native" + }, + { + "source": "test/llvm_resource_native.rs", + "name": "terminal_read_failure_is_one_event_then_end", + "reason": "requires the explicitly prepared runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_resource_native" + }, + { + "source": "test/llvm_resource_native.rs", + "name": "reduce_until_retains_the_shared_unvisited_cursor", + "reason": "requires the explicitly prepared runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_resource_native" + }, + { + "source": "test/llvm_resource_native.rs", + "name": "owned_file_output_and_stream_release_do_not_leak", + "reason": "requires the explicitly prepared runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_resource_native" + }, + { + "source": "test/llvm_resource_native.rs", + "name": "create_new_and_write_never_overwrite_existing_content", + "reason": "requires the explicitly prepared runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_resource_native" + }, + { + "source": "test/llvm_resource_native.rs", + "name": "runtime_fault_unwinds_owned_file_and_partly_read_stream", + "reason": "requires the explicitly prepared runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_resource_native" + }, + { + "source": "test/llvm_resource_native.rs", + "name": "each_propagates_owned_business_failure_and_releases_its_cursor", + "reason": "requires the explicitly prepared runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_resource_native" + }, + { + "source": "test/llvm_resource_native.rs", + "name": "resource_c_abi_rejects_invalid_outputs_before_io_or_cursor_progress", + "reason": "requires the explicitly prepared runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_resource_native" + }, + { + "source": "test/llvm_api_source.rs", + "name": "runtime_rejects_missing_authentication_before_serving", + "reason": "requires explicit runtime-api archive; invalid descriptor must not bind a listener", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_api_source" + }, + { + "source": "test/llvm_api_source.rs", + "name": "failed_listener_start_drains_application_resources", + "reason": "requires explicit runtime-api archive; occupied listener belongs to this test only", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_api_source" + }, + { + "source": "test/llvm_api_source.rs", + "name": "log_only_cmd_uses_configured_owner_and_flushes_before_return", + "reason": "requires explicit runtime-api archive; reads only owned setting.json and captured stderr", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_api_source" + }, + { + "source": "test/llvm_api_source.rs", + "name": "mixed_http_application_dispatches_cmd_without_binding_http", + "reason": "requires explicit runtime-api archive; does not start a listener", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_api_source" + }, + { + "source": "test/llvm_api_source.rs", + "name": "public_http_inputs_context_errors_and_repeated_roots_release_owners", + "reason": "requires explicit runtime-api archive; starts only test-owned loopback services", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_api_source" + }, + { + "source": "test/llvm_api_source.rs", + "name": "upload_adapter_store_and_failed_input_release_affine_owners", + "reason": "requires explicit runtime-api archive; owns listener, upload files and storage outputs", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_api_source" + }, + { + "source": "test/llvm_http_system_source.rs", + "name": "single_http_request_preserves_headers_body_and_status", + "reason": "requires explicit runtime-abi archive and owned loopback", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_http_system_source" + }, + { + "source": "test/llvm_http_system_source.rs", + "name": "pooled_http_requests_and_stream_upload_download", + "reason": "requires explicit runtime-abi archive and owned loopback", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_http_system_source" + }, + { + "source": "test/llvm_http_system_source.rs", + "name": "pooled_http2_requests_and_stream_upload_download", + "reason": "requires explicit runtime-abi archive and owned loopback", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_http_system_source" + }, + { + "source": "test/llvm_http_system_source.rs", + "name": "tls_verifies_local_certificate_and_streams_http", + "reason": "requires explicit runtime-abi archive and owned loopback", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_http_system_source" + }, + { + "source": "test/llvm_http_system_source.rs", + "name": "tls_negotiates_http2_and_streams_upload_download", + "reason": "requires explicit runtime-abi archive and owned loopback", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_http_system_source" + }, + { + "source": "test/llvm_http_system_source.rs", + "name": "live_reply_respond_preserves_response", + "reason": "requires explicit runtime-abi archive and owned loopback", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_http_system_source" + }, + { + "source": "test/llvm_http_system_source.rs", + "name": "live_reply_start_write_finish_preserves_chunks", + "reason": "requires explicit runtime-abi archive and owned loopback", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_http_system_source" + }, + { + "source": "test/llvm_http_system_source.rs", + "name": "sse_preserves_nullable_metadata_and_multiline_data", + "reason": "requires explicit runtime-abi archive and owned loopback", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_http_system_source" + }, + { + "source": "test/llvm_http_system_source.rs", + "name": "owned_handler_failure_is_a_500_then_service_continues", + "reason": "requires explicit runtime-abi archive and owned loopback", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_http_system_source" + }, + { + "source": "test/llvm_http_system_source.rs", + "name": "static_suspending_handler_clones_owned_context_before_initial_suspend", + "reason": "requires explicit runtime-abi archive and owned loopback", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_http_system_source" + }, + { + "source": "test/llvm_http_system_source.rs", + "name": "websocket_connect_echoes_text_binary_and_natural_end", + "reason": "requires explicit runtime-abi archive and owned loopback", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_http_system_source" + }, + { + "source": "test/llvm_http_system_source.rs", + "name": "websocket_url_open_echoes_text_binary_and_natural_end", + "reason": "requires explicit runtime-abi archive and owned loopback", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_http_system_source" + }, + { + "source": "test/llvm_http_system_source.rs", + "name": "secure_websocket_uses_local_tls_trust", + "reason": "requires explicit runtime-abi archive and owned loopback", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_http_system_source" + }, + { + "source": "test/llvm_http_system_source.rs", + "name": "websocket_message_stream_keeps_all_typed_variants_and_closes_aliases", + "reason": "requires explicit runtime-abi archive and owned loopback", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_http_system_source" + }, + { + "source": "test/llvm_http_system_source.rs", + "name": "calendar_duration_and_clocks_keep_runtime_semantics", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_http_system_source" + }, + { + "source": "test/llvm_http_system_source.rs", + "name": "uuid_parse_null_and_canonical_text", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_http_system_source" + }, + { + "source": "test/llvm_http_system_source.rs", + "name": "crypto_secret_alias_sha256_hmac_and_comparison", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_http_system_source" + }, + { + "source": "test/llvm_http_system_source.rs", + "name": "recoverable_system_errors_keep_the_original_messages", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_http_system_source" + }, + { + "source": "test/llvm_http_system_source.rs", + "name": "password_hash_verify_uses_the_existing_blocking_boundary", + "reason": "requires explicit runtime-abi archive and bounded crypto workers", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_http_system_source" + }, + { + "source": "test/llvm_http_system_source.rs", + "name": "process_arguments_exclude_executable_and_keep_unicode", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_http_system_source" + }, + { + "source": "test/llvm_http_system_source.rs", + "name": "stdout_and_structured_logs_are_flushed_before_native_return", + "reason": "requires explicit runtime-abi archive", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_http_system_source" + }, + { + "source": "test/llvm_http_system_source.rs", + "name": "http_client_matches_independent_chunked_wire_and_repeated_headers", + "reason": "requires explicit runtime-abi archive and independent owned loopback peer", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_http_system_source" + }, + { + "source": "test/llvm_http_system_source.rs", + "name": "invalid_server_limits_are_an_uncaptured_original_runtime_fault", + "reason": "requires explicit runtime-abi archive and owned loopback listener", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_http_system_source" + }, + { + "source": "test/llvm_application_source.rs", + "name": "cmd_owned_record_list_unicode_and_missing_nullable", + "reason": "requires explicitly prepared runtime-only archive; no database or service", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_application_source" + }, + { + "source": "test/llvm_application_source.rs", + "name": "cmd_scalar_codecs_preserve_existing_wire_formats", + "reason": "requires explicitly prepared runtime-only archive; no database or service", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_application_source" + }, + { + "source": "test/llvm_application_source.rs", + "name": "cmd_owned_inputs_survive_the_initial_suspend", + "reason": "requires explicitly prepared runtime-only archive; no database or service", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_application_source" + }, + { + "source": "test/llvm_application_source.rs", + "name": "cmd_input_errors_release_partially_decoded_owners", + "reason": "requires explicitly prepared runtime-only archive; no database or service", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_application_source" + }, + { + "source": "test/llvm_application_source.rs", + "name": "cmd_argument_errors_do_not_run_app", + "reason": "requires explicitly prepared runtime-only archive; no database or service", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_application_source" + }, + { + "source": "test/llvm_application_source.rs", + "name": "port_selects_only_the_configured_adapter_and_releases_settings", + "reason": "requires explicitly prepared runtime-only archive; no database or service", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_application_source" + }, + { + "source": "test/llvm_application_source.rs", + "name": "selected_adapter_invalid_setting_fails_closed", + "reason": "requires explicitly prepared runtime-only archive; no database or service", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_application_source" + }, + { + "source": "test/llvm_application_source.rs", + "name": "cmd_keeps_typed_business_faults_and_explicit_recovery", + "reason": "requires explicitly prepared runtime-only archive; no database or service", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_application_source" + }, + { + "source": "test/llvm_application_source.rs", + "name": "cmd_numeric_fault_is_not_reclassified_as_input_or_business_error", + "reason": "requires explicitly prepared runtime-only archive; no database or service", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_application_source" + }, + { + "source": "test/llvm_application_source.rs", + "name": "cmd_preserves_native_input_diagnostics_and_validation_order", + "reason": "requires explicitly prepared runtime-only archive; no database or service", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_application_source" + }, + { + "source": "test/llvm_application_source.rs", + "name": "selected_setting_diagnostic_is_redacted_after_partial_secret_decode", + "reason": "requires explicitly prepared runtime-only archive; no database or service", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_application_source" + }, + { + "source": "test/llvm_application_source.rs", + "name": "cmd_top_level_nullable_missing_null_and_owned_present", + "reason": "requires explicitly prepared runtime-only archive; no database or service", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_application_source" + }, + { + "source": "test/llvm_application_source.rs", + "name": "single_unconfigured_adapter_needs_no_setting_file", + "reason": "requires explicitly prepared runtime-only archive; no database or service", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_application_source" + }, + { + "source": "test/llvm_application_source.rs", + "name": "task_reads_selected_setting_within_the_application_scope", + "reason": "requires explicitly prepared runtime-only archive; no database or service", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_application_source" + }, + { + "source": "test/llvm_application_source.rs", + "name": "later_adapter_initialization_failure_releases_earlier_settings_and_gate", + "reason": "requires explicitly prepared runtime-only archive; no database or service", + "stage": "native", + "package": "dever-backend-bridge", + "target": "llvm_application_source" + }, + { + "source": "test/dever-sandbox-tests/isolation.rs", + "name": "enforces_files_network_process_and_threads", + "reason": "requires explicitly prepared target/sandbox-inputs assets; owns only test processes", + "stage": "sandbox", + "package": "dever-sandbox", + "target": "isolation" + }, + { + "source": "test/dever-sandbox-tests/isolation.rs", + "name": "pins_grants_against_directory_replacement", + "reason": "requires explicitly prepared sandbox assets; mutates only owned fixture paths", + "stage": "sandbox", + "package": "dever-sandbox", + "target": "isolation" + }, + { + "source": "test/dever-sandbox-tests/isolation.rs", + "name": "pinned_grant_probe", + "reason": "internal sandbox descriptor-pinning probe", + "stage": "probe", + "package": "dever-sandbox", + "target": "isolation" + }, + { + "source": "test/dever-sandbox-tests/isolation.rs", + "name": "rejects_writable_aliases_of_executable_inputs", + "reason": "requires explicitly prepared sandbox assets", + "stage": "sandbox", + "package": "dever-sandbox", + "target": "isolation" + }, + { + "source": "test/dever-sandbox-tests/isolation.rs", + "name": "prepared_python_and_node_run_without_host_files_or_process_creation", + "reason": "requires explicitly prepared signed-release Python/Node inputs and sandbox assets", + "stage": "sandbox", + "package": "dever-sandbox", + "target": "isolation" + }, + { + "source": "test/dever-sandbox-tests/isolation.rs", + "name": "enforces_unprivileged_namespace", + "reason": "requires explicitly prepared assets and host permission for unprivileged user namespaces", + "stage": "sandbox", + "package": "dever-sandbox", + "target": "isolation" + }, + { + "source": "test/dever-sandbox-tests/isolation.rs", + "name": "restricted_probe", + "reason": "internal sandbox probe, invoked only inside an owned namespace", + "stage": "probe", + "package": "dever-sandbox", + "target": "isolation" + }, + { + "source": "test/dever-sandbox-tests/isolation.rs", + "name": "terminates_detached_descendants_on_kill", + "reason": "requires explicitly prepared assets; kills only this test's namespace", + "stage": "sandbox", + "package": "dever-sandbox", + "target": "isolation" + }, + { + "source": "test/dever-sandbox-tests/isolation.rs", + "name": "tree_probe", + "reason": "internal sandbox process-tree probe", + "stage": "probe", + "package": "dever-sandbox", + "target": "isolation" + }, + { + "source": "test/dever-sandbox-tests/isolation.rs", + "name": "descendant_probe", + "reason": "internal sandbox descendant probe", + "stage": "probe", + "package": "dever-sandbox", + "target": "isolation" + }, + { + "source": "test/dever-sandbox-tests/isolation.rs", + "name": "granted_probe", + "reason": "internal sandbox probe, invoked only inside an owned namespace", + "stage": "probe", + "package": "dever-sandbox", + "target": "isolation" + }, + { + "source": "test/dever-sandbox-tests/isolation.rs", + "name": "rejects_missing_dependency_before_host_loader_can_supply_it", + "reason": "requires explicitly prepared target/sandbox-inputs assets", + "stage": "sandbox", + "package": "dever-sandbox", + "target": "isolation" + }, + { + "source": "test/dever-sandbox-tests/isolation.rs", + "name": "kernel_rejects_namespace_ptrace_and_truncated_ioctl_requests", + "reason": "requires explicit assets and the prepared C syscall probe", + "stage": "sandbox", + "package": "dever-sandbox", + "target": "isolation" + }, + { + "source": "test/dever-cli-tests/llvm_cli.rs", + "name": "default_llvm_runs_builds_moves_and_reuses_cache_without_host_toolchain", + "reason": "requires explicitly prepared full runtime archive and Linux CRT files; serial offline CLI execution", + "stage": "native", + "package": "dever-cli", + "target": "llvm_cli" + }, + { + "source": "test/dever-cli-tests/llvm_cli.rs", + "name": "unsigned_llvm_run_and_build_reject_exec_workers", + "reason": "requires explicitly prepared runtime archive and component-fixture; no downloads or language SDK", + "stage": "native", + "package": "dever-cli", + "target": "llvm_cli" + }, + { + "source": "test/dever-cli-tests/llvm_cli.rs", + "name": "llvm_test_preserves_case_isolation_fakes_and_failure_continuation", + "reason": "requires explicitly prepared full runtime archive; isolated SQLite and case-local fake processes", + "stage": "native", + "package": "dever-cli", + "target": "llvm_cli" + }, + { + "source": "test/dever-cli-tests/packages.rs", + "name": "installed_managed_package_worker_guard", + "reason": "run only from the signed private fixture", + "stage": "probe", + "package": "dever-cli", + "target": "packages" + }, + { + "source": "test/dever-cli-tests/packages.rs", + "name": "installed_package_passes_check_test_run_build_and_standalone_execution", + "reason": "requires explicitly prepared native runtime archive and owned daemon", + "stage": "native", + "package": "dever-cli", + "target": "packages" + }, + { + "source": "test/dever-cli-tests/shared_toolchain.rs", + "name": "deverd_uses_kernel_credentials_for_other_machine_users", + "reason": "requires root; drops only owned test clients to an unprivileged uid", + "stage": "native", + "package": "dever-cli", + "target": "shared_toolchain" + }, + { + "source": "test/dever-cli-tests/external_workers.rs", + "name": "compiles_go_worker_with_locked_tools_and_runs_without_source_or_go", + "reason": "requires explicit Go runtime pack, official UUID proof/archive and sandbox assets", + "stage": "native", + "package": "dever-cli", + "target": "external_workers" + }, + { + "source": "test/dever-cli-tests/external_workers.rs", + "name": "runs_official_native_python_wheel_in_sandbox", + "reason": "requires the explicit official native-wheel and complete Python runtime fixture at target/python-managed", + "stage": "native", + "package": "dever-cli", + "target": "external_workers" + }, + { + "source": "test/dever-cli-tests/external_workers.rs", + "name": "runs_official_python_source_build_in_sandbox", + "reason": "requires explicitly prepared official PEP517 source build at target/source-build", + "stage": "native", + "package": "dever-cli", + "target": "external_workers" + }, + { + "source": "test/dever-cli-tests/external_workers.rs", + "name": "runs_javascript_runner_from_explicit_fixture_runtime_path", + "reason": "requires the explicit host Node fixture path", + "stage": "native", + "package": "dever-cli", + "target": "external_workers" + }, + { + "source": "test/dever-cli-tests/external_workers.rs", + "name": "runs_python_runner_from_explicit_fixture_runtime_path", + "reason": "requires the explicit host Python fixture path", + "stage": "native", + "package": "dever-cli", + "target": "external_workers" + }, + { + "source": "test/dever-cli-tests/external_workers.rs", + "name": "binds_commonjs_exports_without_a_registration_table", + "reason": "requires the explicit host Node fixture path", + "stage": "native", + "package": "dever-cli", + "target": "external_workers" + }, + { + "source": "test/dever-cli-tests/external_workers.rs", + "name": "binds_esm_exports_without_a_registration_table", + "reason": "requires the explicit host Node fixture path", + "stage": "native", + "package": "dever-cli", + "target": "external_workers" + }, + { + "source": "test/dever-cli-tests/external_workers.rs", + "name": "imports_packaged_sdk_before_adapter_modules", + "reason": "requires the explicit host Python fixture path", + "stage": "native", + "package": "dever-cli", + "target": "external_workers" + }, + { + "source": "test/dever-cli-tests/external_workers.rs", + "name": "compiled_managed_worker_runs_after_source_and_pack_are_removed", + "reason": "requires the explicit host Node fixture and native Rust compiler", + "stage": "native", + "package": "dever-cli", + "target": "external_workers" + }, + { + "source": "test/dever-tests/tests/postgres_api.rs", + "name": "postgres_cms_http_isolates_two_tenant_databases", + "reason": "requires an isolated PostgreSQL instance configured in config/setting.json", + "stage": "postgres", + "package": "dever-tests", + "target": "postgres_api" + }, + { + "source": "test/dever-tests/tests/native_cache.rs", + "name": "actual_cli_checks_baseline_on_check_run_and_cached_build", + "reason": "requires a freshly built local dever-cli binary", + "stage": "native", + "package": "dever-tests", + "target": "native_cache" + }, + { + "source": "test/dever-tests/tests/postgres_orm.rs", + "name": "postgres_orm_uses_the_configured_isolated_database", + "reason": "requires an isolated database configured in config/setting.json", + "stage": "postgres", + "package": "dever-tests", + "target": "postgres_orm" + }, + { + "source": "test/dever-tests/tests/external_resources.rs", + "name": "packaged_exec_runs_without_source_worker_or_inherited_environment", + "reason": "requires explicitly prepared sandbox assets and the private Rust bootstrap compiler", + "stage": "native", + "package": "dever-tests", + "target": "external_resources" + }, + { + "source": "test/dever-tests/tests/performance.rs", + "name": "allocation_sensitive_source_scenarios", + "reason": "explicit bounded record/list/JSON/large-map benchmark", + "stage": "bounded-perf", + "package": "dever-tests", + "target": "performance" + }, + { + "source": "test/dever-tests/tests/performance.rs", + "name": "native_hot_paths_stay_within_the_matching_runtime_baseline", + "reason": "explicit bounded native hot-path benchmark; never part of default checks", + "stage": "bounded-perf", + "package": "dever-tests", + "target": "performance" + }, + { + "source": "test/dever-tests/tests/authorization_postgres.rs", + "name": "postgres_authorization_preserves_site_scoped_roles_and_catalog", + "reason": "requires an isolated database configured in config/setting.json", + "stage": "postgres", + "package": "dever-tests", + "target": "authorization_postgres" + }, + { + "source": "test/dever-tests/tests/concurrency.rs", + "name": "parallel_task_granularity_measurement", + "reason": "explicit bounded local scheduling benchmark", + "stage": "bounded-perf", + "package": "dever-tests", + "target": "concurrency" + }, + { + "source": "test/dever-cli-tests/external_workers/npm.rs", + "name": "runs_official_npm_addon_from_offline_resources_and_threads", + "reason": "requires the explicit official native npm build fixture at target/npm-build", + "stage": "native", + "package": "dever-cli", + "target": "external_workers" + }, + { + "source": "test/dever-cli-tests/packages/managed_compile.rs", + "name": "trusted_compilation_cache_requires_sources_and_revalidates_signed_pack", + "reason": "requires explicitly prepared native archive; starts an owned daemon", + "stage": "native", + "package": "dever-cli", + "target": "packages" + }, + { + "source": "test/dever-cli-tests/packages/managed_compile.rs", + "name": "different_machine_users_share_compilation_without_private_cache", + "reason": "requires root and prepared archive; only owned clients change UID", + "stage": "native", + "package": "dever-cli", + "target": "packages" + }, + { + "source": "test/dever-cli-tests/packages/managed_compile.rs", + "name": "cancellation_disconnect_and_version_lease_reap_worker_and_staging", + "reason": "requires prepared archive, explicit author cc/LLVM inputs; supervises an owned signed sleeper", + "stage": "native", + "package": "dever-cli", + "target": "packages" + }, + { + "source": "test/dever-cli-tests/native_release/bootstrap.rs", + "name": "bootstrap_installer_rejects_untrusted_inputs_before_publication", + "reason": "root-only installer trust rejection; never touches the actual machine installation", + "stage": "native", + "package": "dever-cli", + "target": "native_release" + }, + { + "source": "test/dever-cli-tests/native_release/bootstrap.rs", + "name": "bootstrap_recovery_is_idempotent_and_rejects_unrecorded_targets", + "reason": "root-only crash-recovery fixture; all journal targets are below an owned image root", + "stage": "native", + "package": "dever-cli", + "target": "native_release" + }, + { + "source": "test/dever-cli-tests/native_release/ecosystem_acceptance.rs", + "name": "signed_ecosystems_build_and_run_without_system_language_installations", + "reason": "requires prepared pinned ecosystem inputs and root for owned namespace; no existing services", + "stage": "native", + "package": "dever-cli", + "target": "native_release" + }, + { + "source": "test/dever-cli-tests/native_release/acceptance.rs", + "name": "signed_core_builds_without_host_compiler_libraries_or_tools", + "reason": "requires prepared optimized archives and root for an owned GNU ABI root", + "stage": "native", + "package": "dever-cli", + "target": "native_release" + }, + { + "source": "test/dever-cli-tests/native_release/acceptance.rs", + "name": "optimized_profiles_make_reproducible_signed_release_and_run_through_daemon", + "reason": "requires four optimized archives and component-fixture; starts only an owned daemon, SQLite and exec Worker fixtures", + "stage": "native", + "package": "dever-cli", + "target": "native_release" + }, + { + "source": "test/dever-cli-tests/native_release/bootstrap_acceptance.rs", + "name": "signed_bootstrap_installs_and_serves_two_projects_without_host_libraries", + "reason": "requires optimized author archives and Linux root; installs only into an owned image root", + "stage": "native", + "package": "dever-cli", + "target": "native_release" + }, + { + "source": "test/dever-cli-tests/external_libs/sumdb.rs", + "name": "prepare_official_go_dependency_fixture", + "reason": "author operation: explicitly fetches official Go proxy/sumdb; requires target/go-managed/config/setting.json and runtime.pack", + "stage": "author", + "package": "dever-cli", + "target": "external_libs" + }, + { + "source": "test/dever-cli-tests/external_libs/build.rs", + "name": "prepare_source_build_tool_fixture", + "reason": "explicit author operation: assembles previously hashed native-build-inputs; no network or host execution", + "stage": "author", + "package": "dever-cli", + "target": "external_libs" + }, + { + "source": "test/dever-cli-tests/external_libs/build.rs", + "name": "pep517_hooks_keep_requirements_cli_and_metadata_boundaries", + "reason": "isolated offline PEP517 contracts; requires explicit target/source-build tool/runtime fixture", + "stage": "native", + "package": "dever-cli", + "target": "external_libs" + }, + { + "source": "test/dever-cli-tests/external_libs/build.rs", + "name": "builds_official_python_source_with_isolated_pep517", + "reason": "explicit official PyPI source build: downloads fixed source/build requirements and executes signed managed build tools", + "stage": "author", + "package": "dever-cli", + "target": "external_libs" + }, + { + "source": "test/dever-cli-tests/external_libs/npm_build.rs", + "name": "prepare_npm_build_tool_fixture", + "reason": "explicit author operation: assembles hashed npm/compiler/header inputs; no network or host execution", + "stage": "author", + "package": "dever-cli", + "target": "external_libs" + }, + { + "source": "test/dever-cli-tests/external_libs/npm_build.rs", + "name": "builds_official_npm_addon_with_locked_lifecycle", + "reason": "explicit official npm native source build; network only in registry resolver, never in lifecycle sandbox", + "stage": "author", + "package": "dever-cli", + "target": "external_libs" + }, + { + "source": "test/dever-cli-tests/external_libs/npm_build.rs", + "name": "npm_lifecycle_preserves_optional_failures_bins_and_offline_remove", + "reason": "explicit managed npm hooks; uses local registry and real signed author runtime/tools", + "stage": "native", + "package": "dever-cli", + "target": "external_libs" + }, + { + "source": "test/dever-cli-tests/external_libs/npm_build.rs", + "name": "npm_build_scratch_budget_kills_owned_hook_and_cleans_stage", + "reason": "explicit managed npm hook: sparse scratch overflow and child cleanup; no network", + "stage": "native", + "package": "dever-cli", + "target": "external_libs" + }, + { + "source": "test/dever-cli-tests/external_libs/npm_build.rs", + "name": "npm_build_output_hard_links_stay_inside_the_exported_tree", + "reason": "explicit managed npm hook: internal output aliases are flattened; aliases outside output are rejected", + "stage": "native", + "package": "dever-cli", + "target": "external_libs" + }, + { + "source": "test/dever-cli-tests/external_libs/wheel.rs", + "name": "prepare_official_native_python_fixture", + "reason": "author operation: explicitly fetches PyPI simplejson; requires target/python-managed/config/setting.json and runtime.pack", + "stage": "author", + "package": "dever-cli", + "target": "external_libs" + } +] diff --git a/test/ci/run.py b/test/ci/run.py new file mode 100644 index 0000000..25eed69 --- /dev/null +++ b/test/ci/run.py @@ -0,0 +1,372 @@ +"""Linux 质量门：串行复用现有检查，按源码和输入摘要保存阶段证据。""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import signal +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[2] +sys.path.append(str(ROOT / "test/performance")) +sys.modules["linux_gate"] = sys.modules[__name__] +from settings import configured_peer, write_private_json + +STAGES = ("static", "default", "all-features", "features", "python", "sdk", + "inventory", "author", "native", "postgres", "sandbox", "bounded-perf", + "cms-64", "cms-128", "http2-64", "http2-128", "recovery-64", "recovery-128", + "live-64", "live-128") +SOURCE_ROOTS = ("crates", "test", "sdk", "library", "examples", ".cargo") + + +class MissingInput(Exception): + pass + + +def digest_file(path): + with path.open("rb") as source: + return hashlib.file_digest(source, "sha256").hexdigest() + + +def source_fingerprint(root=ROOT): + paths = [root / name for name in ("Cargo.toml", "Cargo.lock")] + for name in SOURCE_ROOTS: + directory = root / name + if directory.exists(): + paths.extend(path for path in directory.rglob("*") if path.is_file() + and not {"__pycache__", "target", ".git"}.intersection(path.parts)) + digest = hashlib.sha256() + for path in sorted(paths): + digest.update(str(path.relative_to(root)).encode() + b"\0") + digest.update(digest_file(path).encode() + b"\0") + return digest.hexdigest() + + +def absolute_file(value, name): + path = Path(value or "") + if not path.is_absolute() or not path.is_file(): + raise MissingInput(f"{name} requires an existing absolute file path") + return path + + +def load_config(): + path = ROOT / "config/setting.json" + if not path.is_file(): + raise MissingInput("create config/setting.json with performance.linux_ci (see test/ci/README.md)") + document = json.loads(path.read_text()) + performance = document.get("performance") if isinstance(document, dict) else None + config = performance.get("linux_ci") if isinstance(performance, dict) else None + if not isinstance(config, dict): + raise MissingInput("config/setting.json must define performance.linux_ci") + return document, config + + +def input_fingerprint(document, config, stage=None): + inputs = {} + for name, value in config.get("tools", {}).items(): + path = Path(value) + if not path.is_absolute(): + raise ValueError("performance.linux_ci.tools must use absolute paths") + inputs[f"tool:{name}:{value}"] = digest_file(path) if path.is_file() else "missing" + paths = set(config.get("inputs", [])) + paths.update(str(configured_peer(name, ROOT / "config/setting.json")) + for name in ("network_peer", "live_peer") if name in document.get("performance", {})) + soak = config.get("soak", {}) + if soak.get("artifacts"): + paths.add(soak["artifacts"]) + if soak.get("cms_manifest"): + manifest = Path(soak["cms_manifest"]) + paths.add(str(manifest)) + if manifest.is_file(): + for case in json.loads(manifest.read_text()).get("cases", {}).values(): + paths.update(case[key] for key in ("executable", "config")) + for value in sorted(paths): + path = Path(value) + if not path.is_absolute(): + raise ValueError("performance.linux_ci.inputs must use absolute paths") + if stage == "inventory": + continue # 清单依赖当前源码和工具，不消费作者资产。 + outputs = [Path(value) for value in config.get("author_outputs", [])] if stage == "author" else [] + if any(path == output or path.is_relative_to(output) for output in outputs): + continue + if path.is_dir(): + for child in sorted(path.rglob("*")): + if child.is_file() and not any(child == output or child.is_relative_to(output) for output in outputs): + inputs[str(child)] = digest_file(child) + else: + inputs[value] = digest_file(path) if path.is_file() else "missing" + # 保存摘要而非数据库密码/令牌；配置任何变化都使旧证据失效。 + return hashlib.sha256(json.dumps([document, inputs], sort_keys=True).encode()).hexdigest() + + +def command_plan(stage, config): + tools = config.get("tools", {}) + cargo = str(absolute_file(tools.get("cargo"), "performance.linux_ci.tools.cargo")) + common = ["--offline", "--locked", "-j1", *cargo_config(config)] + test = [cargo, "test", *common, "--workspace"] + if stage == "static": + fmt = str(absolute_file(tools.get("cargo_fmt"), "performance.linux_ci.tools.cargo_fmt")) + clippy = str(absolute_file(tools.get("cargo_clippy"), "performance.linux_ci.tools.cargo_clippy")) + return [[fmt, "fmt", "--all", "--check"], + *[[clippy, "clippy", *common, "--workspace", "--all-targets", *features, + "--", "-D", "warnings"] for features in ([], ["--all-features"])]] + if stage in ("default", "all-features"): + return [[*test, *(["--all-features"] if stage == "all-features" else []), + "--", "--test-threads=1"]] + if stage == "features": + profiles = { + "dever-runtime": ["", "auth-crypto", "crypto", "wire", "external", "api", "database", "sqlite", "postgres", "sqlite,postgres"], + "dever-backend-bridge": ["", "embedded", "runtime-abi", "runtime-database", "runtime-api", "runtime-external", "runtime-sqlite", "runtime-postgres", "runtime-sqlite,runtime-postgres"], + } + return [[cargo, "check", *common, "-p", package, "--lib", "--no-default-features", + *(["--features", features] if features else [])] + for package, combinations in profiles.items() for features in combinations] + [ + [cargo, "tree", "--offline", "--locked", "-p", "dever-cli", "--edges", "features"]] + python = str(absolute_file(tools.get("python"), "performance.linux_ci.tools.python")) + if stage == "python": + return [[python, "-B", "-m", "unittest", "discover", "-s", directory, "-p", "test_*.py", "-v"] + for directory in ("test/ci", "test/performance", "test/ecosystem-release")] + if stage == "sdk": + return [[python, "-B", "test/external-sdk/check.py", "--python", python, + "--node", str(absolute_file(tools.get("node"), "performance.linux_ci.tools.node")), + "--go", str(absolute_file(tools.get("go"), "performance.linux_ci.tools.go"))]] + raise ValueError(f"stage has a dedicated runner: {stage}") + + +def cargo_config(config): + return [value for override in config.get("cargo_config", []) for value in ("--config", override)] + + +def workspace_targets(config, directory, result, all_features): + cargo = str(absolute_file(config["tools"].get("cargo"), "performance.linux_ci.tools.cargo")) + command = [cargo, "metadata", "--offline", "--locked", "--format-version", "1", *cargo_config(config)] + if all_features: + command.append("--all-features") + record, content = run_command(command, directory, len(result["commands"]), 120) + result["commands"].append(record) + metadata = next(json.loads(line) for line in content.splitlines() if line.startswith('{"packages":')) + active = {node["id"]: set(node["features"]) for node in metadata["resolve"]["nodes"]} + targets = [] + result["target_directory"] = metadata["target_directory"] + for package in metadata["packages"]: + if package["id"] not in metadata["workspace_members"]: + continue + for target in package["targets"]: + required = set(target.get("required-features", [])) + if not required <= active[package["id"]]: + continue + kind = target["kind"][0] + if target["test"] and kind in ("lib", "rlib", "staticlib", "bin", "test"): + selector = ["--lib"] if kind in ("lib", "rlib", "staticlib") else ["--" + kind, target["name"]] + targets.append({"package": package["name"], "target": target["name"], "selector": selector, + "features": sorted(active[package["id"]])}) + if target["doctest"] and kind in ("lib", "rlib", "staticlib"): + targets.append({"package": package["name"], "target": target["name"], "selector": ["--doc"], + "features": sorted(active[package["id"]])}) + return targets, Path(metadata["target_directory"]) + + +def target_command(config, target, all_features=True): + features = (["--all-features"] if all_features else + ["--features", ",".join(target["features"])] if target.get("features") else []) + return [config["tools"]["cargo"], "test", "--offline", "--locked", "-j1", *cargo_config(config), + "-p", target["package"], *target["selector"], *features] + + +def remove_executables(content, target_directory, enabled): + if not enabled: + return [] + removed = [] + for line in content.splitlines(): + if not line.startswith("{"): + continue + try: + message = json.loads(line) + except json.JSONDecodeError: + continue + if (message.get("reason") == "compiler-artifact" and message.get("profile", {}).get("test") + and message.get("executable")): + path = Path(message["executable"]).resolve() + if not path.is_relative_to(target_directory.resolve()) or path.parent.name != "deps": + raise RuntimeError(f"refusing to remove a non-test output: {path}") + if path.exists(): + path.unlink() + removed.append(str(path)) + return removed + + +def cleanup_test_executables(directory, config, result): + if not config.get("remove_test_executables", False) or "target_directory" not in result: + return + for command_file in sorted(directory.glob("*.command.json")): + command = json.loads(command_file.read_text()) + arguments = command["command"] + if arguments[0] != config["tools"]["cargo"] or "--message-format=json" not in arguments: + continue + log = Path(command["log"]) + result.setdefault("removed_test_executables", []).extend( + remove_executables(log.read_text(errors="replace"), Path(result["target_directory"]), True)) + + +def run_workspace(stage, config, directory, result): + all_features = stage == "all-features" + targets, target_directory = workspace_targets(config, directory, result, all_features) + result["targets"] = targets + passed = 0 + for target in targets: + command = [*target_command(config, target, all_features), "--message-format=json", "--", "--test-threads=1"] + print(f"{stage}: {target['package']} {' '.join(target['selector'])}", flush=True) + record, content = run_command(command, directory, len(result["commands"]), config.get("command_timeout_seconds", 14400)) + result["commands"].append(record) + passed += record.get("rust_tests", {}).get("passed", 0) + result.setdefault("removed_test_executables", []).extend( + remove_executables(content, target_directory, config.get("remove_test_executables", False))) + if not targets or not passed: + raise RuntimeError("workspace target matrix executed no Rust tests") + result["passed_tests"] = passed + + +def run_command(command, directory, number, timeout, *, single_harness=False): + path = directory / f"{number:03d}.log" + started = time.time() + record = {"command": command, "log": str(path), "started": started} + write_private_json(directory / f"{number:03d}.command.json", record) + failure = None + child = None + try: + with path.open("x") as output: + child = subprocess.Popen(command, cwd=ROOT, stdout=output, stderr=subprocess.STDOUT, + start_new_session=True) + try: + child.wait(timeout=timeout) + except BaseException: + try: + # Python 测量器以 KeyboardInterrupt 执行其 finally，回收独立进程组和 cgroup。 + os.killpg(child.pid, signal.SIGINT) + child.wait(timeout=10) + except subprocess.TimeoutExpired: + os.killpg(child.pid, signal.SIGKILL) + child.wait(timeout=5) + except ProcessLookupError: + child.wait() + raise + except BaseException as error: + failure = error + record["error"] = str(error) or type(error).__name__ + record["exit_code"] = child.returncode if child is not None else None + record["elapsed_seconds"] = time.time() - started + content = path.read_text(errors="replace") + counts = re.findall(r"test result: (?:ok|FAILED)\. (\d+) passed; (\d+) failed; (\d+) ignored", content) + if single_harness: + # 直接运行一个 libtest 时，最后一条是父 harness；前面的可能是其子进程探针。 + counts = counts[-1:] + if counts: + record["rust_tests"] = {name: sum(int(row[index]) for row in counts) + for index, name in enumerate(("passed", "failed", "ignored"))} + record["python_tests"] = sum(map(int, re.findall(r"Ran (\d+) tests? in", content))) + write_private_json(directory / f"{number:03d}.command.json", record, replace=True) + if failure is not None: + raise failure + if record["exit_code"]: + raise RuntimeError(f"command exited {record['exit_code']}: {path}") + if re.search(r"OK \(skipped=|: unavailable", content): + raise MissingInput(f"command skipped required verification: {path}") + return record, content + + +def check_output(stage, record, content): + if stage in ("default", "all-features") and record.get("rust_tests", {}).get("passed", 0) == 0: + raise RuntimeError("workspace command did not execute any Rust tests") + if stage == "python" and record["python_tests"] == 0: + raise RuntimeError("Python discovery executed no tests") + if stage == "sdk" and any(f"{language}: handshake, health, result, business error, cancel, shutdown and rejection passed" + not in content for language in ("Python", "JavaScript", "Go")): + raise RuntimeError("all three SDK acceptance results are required") + if stage == "features" and "dever-core feature \"reference\"" in content: + raise RuntimeError("production CLI depends on the reference feature") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=("plan", "run", "summary")) + parser.add_argument("--stage", choices=STAGES) + args = parser.parse_args() + document, config = load_config() + output = Path(config.get("output", ROOT / "target/linux-ci")) + if not output.is_absolute(): + raise ValueError("performance.linux_ci.output must be absolute") + source = source_fingerprint() + if args.action == "summary": + results = {} + for stage in STAGES: + candidates = sorted((output / stage).glob("*/result.json")) + record = json.loads(candidates[-1].read_text()) if candidates else {} + stage_inputs = input_fingerprint(document, config, stage) + current = (record.get("source") == source and record.get("inputs") == stage_inputs + and (stage != "author" or record.get("outputs") == input_fingerprint(document, config))) + results[stage] = record["status"] if current else "missing-current-evidence" + print(json.dumps(results, indent=2)) + return 0 if all(status == "passed" for status in results.values()) else 1 + if args.action == "plan": + inputs = input_fingerprint(document, config, args.stage) + print(json.dumps({"stages": STAGES, "source": source, "inputs": inputs, + "commands": command_plan(args.stage, config) if args.stage in STAGES[:6] else None}, indent=2)) + return 0 + if args.stage is None: + parser.error("run requires --stage (all stages are serialized by the caller)") + directory = output / args.stage / f"{time.time_ns()}" + directory.mkdir(parents=True, exist_ok=False) + result = {"stage": args.stage, "source": source, "inputs": None, "status": "running", "commands": []} + write_private_json(directory / "result.json", result) + try: + try: + inputs = input_fingerprint(document, config, args.stage) + except (ValueError, OSError) as error: + raise MissingInput(str(error)) from error + result["inputs"] = inputs + write_private_json(directory / "result.json", result, replace=True) + from acceptance import validate_budget + validate_budget(config) + if args.stage in ("default", "all-features"): + run_workspace(args.stage, config, directory, result) + elif args.stage in STAGES[:6]: + for number, command in enumerate(command_plan(args.stage, config)): + print(f"{args.stage}: {' '.join(command)}", flush=True) + record, content = run_command(command, directory, number, config.get("command_timeout_seconds", 14400)) + result["commands"].append(record) + check_output(args.stage, record, content) + else: + from acceptance import run_stage + run_stage(args.stage, document, config, directory, result) + current_document, current_config = load_config() + final_inputs = input_fingerprint(current_document, current_config, args.stage) + if args.stage == "author": + result["outputs"] = input_fingerprint(current_document, current_config) + if source_fingerprint() != source or final_inputs != inputs: + raise RuntimeError("source or declared inputs changed while the stage ran; evidence is stale") + result["status"] = "passed" + except MissingInput as error: + result.update(status="blocked", error=str(error)) + except (Exception, KeyboardInterrupt) as error: + result.update(status="failed", error=str(error) or type(error).__name__) + finally: + try: + cleanup_test_executables(directory, config, result) + except Exception as error: + result.update(status="failed", cleanup_error=str(error)) + result["commands"] = [json.loads(path.read_text()) for path in sorted(directory.glob("*.command.json"))] + write_private_json(directory / "result.json", result, replace=True) + print(json.dumps(result, indent=2)) + return 0 if result["status"] == "passed" else 1 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except (MissingInput, ValueError, OSError) as error: + print(f"blocked: {error}", file=sys.stderr) + sys.exit(2) diff --git a/test/ci/test_driver.py b/test/ci/test_driver.py new file mode 100644 index 0000000..9b70dbe --- /dev/null +++ b/test/ci/test_driver.py @@ -0,0 +1,41 @@ +"""验证共享分配账本：线程缓存正常析构，真实泄漏和错误结果必须拒绝。""" + +from pathlib import Path +import subprocess +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[2] +DRIVER = ROOT / "test/native-runtime-abi/managed-driver.c" +FIXTURE = ROOT / "test/native-runtime-abi/driver-fixture.c" +WRAPPED = ("malloc", "calloc", "realloc", "free", "posix_memalign", "realpath") + + +class DriverTests(unittest.TestCase): + def run_fixture(self, scenario): + with tempfile.TemporaryDirectory() as temporary: + executable = Path(temporary) / "ledger" + subprocess.run([ + "/usr/bin/cc", "-std=c11", "-Wall", "-Wextra", "-Werror", "-O2", + f"-DDEVER_LEDGER_SCENARIO={scenario}", str(DRIVER), str(FIXTURE), + *[f"-Wl,--wrap={name}" for name in WRAPPED], "-pthread", "-B/usr/bin/", + "-o", str(executable), + ], check=True, env={}, capture_output=True, text=True, timeout=20) + return subprocess.run([str(executable)], env={}, capture_output=True, + text=True, timeout=5) + + def test_late_thread_cache_is_released_before_counting(self): + result = self.run_fixture(0) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_owned_leaks_and_changed_results_fail(self): + for scenario, message in ((1, "allocation imbalance: 1"), + (2, "changed its result on repeated execution: 7")): + with self.subTest(scenario=scenario): + result = self.run_fixture(scenario) + self.assertNotEqual(result.returncode, 0) + self.assertIn(message, result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/ci/test_gate.py b/test/ci/test_gate.py new file mode 100644 index 0000000..47d4980 --- /dev/null +++ b/test/ci/test_gate.py @@ -0,0 +1,317 @@ +"""质量门防假通过合同；不运行 Cargo、数据库或真实压力测试。""" + +import importlib.util +import contextlib +import io +import json +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location("linux_gate", Path(__file__).with_name("run.py")) +gate = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = gate +spec.loader.exec_module(gate) +import acceptance +from acceptance import classify_cases, validate_postgres, validate_soak + + +class GateTests(unittest.TestCase): + def test_ci_config_uses_the_shared_performance_section(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + (root / "config").mkdir() + setting = root / "config/setting.json" + config = {"output": str(root / "results"), "inputs": []} + document = {"performance": {"linux_ci": config}} + with patch.object(gate, "ROOT", root): + with self.assertRaisesRegex(gate.MissingInput, r"performance\.linux_ci"): + gate.load_config() + setting.write_text(json.dumps(document)) + self.assertEqual(gate.load_config(), (document, config)) + before = gate.input_fingerprint(document, config) + config["command_timeout_seconds"] = 120 + self.assertNotEqual(before, gate.input_fingerprint(document, config)) + for invalid in ({"linux_ci": config}, {"performance": {}}, + {"performance": []}, {"performance": {"linux_ci": []}}): + setting.write_text(json.dumps(invalid)) + with self.subTest(document=invalid), self.assertRaisesRegex( + gate.MissingInput, r"performance\.linux_ci"): + gate.load_config() + + def test_performance_peer_fingerprints_resolve_project_relative_paths(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + (root / "config").mkdir() + (root / "peers").mkdir() + peer = root / "peers/network" + peer.write_text("original peer") + live_peer = root / "peers/live" + live_peer.write_text("original live peer") + document = {"performance": {"network_peer": "peers/network", "live_peer": str(live_peer), + "linux_ci": {"inputs": []}}} + (root / "config/setting.json").write_text(json.dumps(document)) + with patch.object(gate, "ROOT", root): + before = gate.input_fingerprint(document, {}) + peer.write_text("changed peer") + self.assertNotEqual(before, gate.input_fingerprint(document, {})) + before = gate.input_fingerprint(document, {}) + live_peer.write_text("changed live peer") + self.assertNotEqual(before, gate.input_fingerprint(document, {})) + for config in ({"inputs": ["peers/network"]}, {"tools": {"cargo": "peers/network"}}): + with self.subTest(config=config), self.assertRaisesRegex(ValueError, "absolute paths"): + gate.input_fingerprint(document, config) + + def test_default_targets_preserve_workspace_resolved_features(self): + target = {"package": "dever-backend-bridge", "selector": ["--test", "backend_bridge"], + "features": ["embedded", "runtime-api"]} + command = gate.target_command({"tools": {"cargo": "/explicit/cargo"}}, target, False) + self.assertEqual(command[-2:], ["--features", "embedded,runtime-api"]) + self.assertNotIn("--all-features", command) + + def test_sdk_missing_and_partial_execution_fail(self): + config = {"tools": {"cargo": sys.executable, "python": sys.executable}} + with self.assertRaises(gate.MissingInput): + gate.command_plan("sdk", config) + with self.assertRaises(RuntimeError): + gate.check_output("sdk", {}, "Python: handshake passed") + with self.assertRaises(RuntimeError): + gate.check_output("python", {"python_tests": 0}, "OK") + + def test_classification_rejects_unknown_duplicate_and_wrong_target(self): + row = {"package": "sandbox", "target": "isolation", "name": "restricted_probe", "stage": "probe"} + target = {"package": "sandbox", "target": "isolation"} + self.assertEqual(classify_cases(["restricted_probe"], [row], target)[0]["stage"], "probe") + for names, rows, owner in ((["unknown"], [row], target), (["restricted_probe"], [row, row], target), + (["restricted_probe"], [row], {**target, "target": "other"}), + (["a::restricted_probe", "b::restricted_probe"], [row], target)): + with self.assertRaises(RuntimeError): + classify_cases(names, rows, owner) + + def test_postgres_requires_owned_loopback_case_template_without_override(self): + def document(url): + return {"database": {"postgres_test": {"type": "postgres", "url": url}}} + validate_postgres(document("postgresql://fixture@127.0.0.1:51234/dever_{case}")) + for url in ("postgresql://fixture@db.example:5432/dever_{case}", + "postgresql://fixture@127.0.0.1:5432/fixed", "postgresql://fixture@127.0.0.1/dever_{case}", + "postgresql://fixture@127.0.0.1:5432/dever_{case}?dbname=production"): + with self.assertRaises(gate.MissingInput): + validate_postgres(document(url)) + + def test_failed_command_keeps_exit_and_counts(self): + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + with self.assertRaises(RuntimeError): + gate.run_command([sys.executable, "-c", "print('test result: FAILED. 2 passed; 1 failed; 3 ignored'); raise SystemExit(7)"], directory, 0, 5) + record = json.loads((directory / "000.command.json").read_text()) + self.assertEqual(record["exit_code"], 7) + self.assertEqual(record["rust_tests"], {"passed": 2, "failed": 1, "ignored": 3}) + + def test_single_harness_counts_parent_instead_of_nested_probe(self): + child = "test result: ok. 1 passed; 0 failed; 0 ignored" + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + for number, (single, parent_passed, parent_ignored, exit_code) in enumerate( + ((False, 1, 0, 0), (True, 1, 0, 0), (True, 0, 0, 0), + (True, 0, 1, 0), (True, 0, 0, 7))): + parent_failed = int(exit_code != 0) + parent = (f"test result: {'FAILED' if parent_failed else 'ok'}. " + f"{parent_passed} passed; {parent_failed} failed; {parent_ignored} ignored") + command = [sys.executable, "-c", + f"print({child!r}); print({parent!r}); raise SystemExit({exit_code})"] + with self.subTest(single=single, parent=parent): + if exit_code: + with self.assertRaises(RuntimeError): + gate.run_command(command, directory, number, 5, single_harness=single) + else: + gate.run_command(command, directory, number, 5, single_harness=single) + record = json.loads((directory / f"{number:03d}.command.json").read_text()) + self.assertEqual(record["exit_code"], exit_code) + self.assertEqual(record["rust_tests"], { + "passed": parent_passed + int(not single), + "failed": parent_failed, "ignored": parent_ignored}) + + def test_exact_ignored_acceptance_requires_one_successful_parent(self): + target = {"package": "sandbox", "target": "isolation"} + case = {"name": "parent", "exact": "parent", "stage": "sandbox", + "target": target, "target_directory": "/unused/target"} + for counts in ({"passed": 1, "failed": 0, "ignored": 0}, + {"passed": 0, "failed": 0, "ignored": 0}, + {"passed": 0, "failed": 0, "ignored": 1}, + {"passed": 1, "failed": 1, "ignored": 0}, + {"passed": 2, "failed": 0, "ignored": 0}, None): + with (self.subTest(counts=counts), + patch.object(acceptance, "require_inputs"), + patch.object(acceptance, "current_inventory", return_value=[case]), + patch.object(acceptance, "target_command", return_value=["cargo"]), + patch.object(acceptance, "test_executable", return_value="/unused/test"), + patch.object(acceptance, "remove_executables", return_value=[]), + patch.object(acceptance, "execute", side_effect=[({}, "build"), + ({"rust_tests": counts}, "test")]) as execute): + result = {"commands": []} + expected = counts == {"passed": 1, "failed": 0, "ignored": 0} + with contextlib.nullcontext() if expected else self.assertRaisesRegex(RuntimeError, "one case"): + acceptance.ignored_stage("sandbox", {}, {}, Path("/unused"), result) + self.assertTrue(execute.call_args.kwargs["single_harness"]) + self.assertEqual(len(result.get("cases", [])), int(expected)) + + def test_source_and_author_input_fingerprints_detect_real_changes(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + for name in ("Cargo.toml", "Cargo.lock"): + (root / name).write_text(name) + (root / "test").mkdir() + before = gate.source_fingerprint(root) + (root / "test/new.py").write_text("changed") + self.assertNotEqual(before, gate.source_fingerprint(root)) + output = root / "generated.pack" + output.write_text("old") + config = {"inputs": [str(root)], "author_outputs": [str(output)]} + author_before = gate.input_fingerprint({}, config, "author") + native_before = gate.input_fingerprint({}, config, "native") + output.write_text("new") + self.assertEqual(author_before, gate.input_fingerprint({}, config, "author")) + self.assertNotEqual(native_before, gate.input_fingerprint({}, config, "native")) + (root / "Cargo.lock").write_text("changed immutable input") + self.assertNotEqual(author_before, gate.input_fingerprint({}, config, "author")) + + def test_cleanup_cannot_delete_author_binary(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + executable = root / "native-acceptance-init" + executable.write_text("owned author fixture") + message = json.dumps({"reason": "compiler-artifact", "profile": {"test": True}, "executable": str(executable)}) + with self.assertRaises(RuntimeError): + gate.remove_executables(message, root, True) + self.assertTrue(executable.exists()) + + def test_failed_workspace_command_still_cleans_its_test_executable(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + executable = root / "target/debug/deps/test-owned" + executable.parent.mkdir(parents=True) + executable.write_text("rebuildable executable") + config = {"output": str(root / "results"), "remove_test_executables": True, + "tools": {"cargo": sys.executable}} + + def fail_workspace(stage, config, directory, result): + result["target_directory"] = str(root / "target") + message = json.dumps({"reason": "compiler-artifact", "profile": {"test": True}, + "executable": str(executable)}) + gate.run_command([sys.executable, "-c", f"print({message!r}); raise SystemExit(7)", + "--message-format=json"], + directory, 0, 5) + + with (patch.object(gate, "load_config", return_value=({}, config)), + patch.object(gate, "source_fingerprint", return_value="source"), + patch.object(gate, "run_workspace", side_effect=fail_workspace), + patch.object(acceptance, "validate_budget"), + patch.object(sys, "argv", ["run.py", "run", "--stage", "default"]), + contextlib.redirect_stdout(io.StringIO())): + self.assertEqual(gate.main(), 1) + self.assertFalse(executable.exists()) + record = json.loads(next((root / "results/default").glob("*/result.json")).read_text()) + self.assertEqual(record["status"], "failed") + self.assertIn("command exited 7", record["error"]) + self.assertEqual(record["removed_test_executables"], [str(executable)]) + + def test_missing_input_is_saved_as_blocked_stage_evidence(self): + with tempfile.TemporaryDirectory() as temporary: + config = {"output": temporary} + with (patch.object(gate, "load_config", return_value=({}, config)), + patch.object(gate, "source_fingerprint", return_value="source"), + patch.object(gate, "input_fingerprint", side_effect=FileNotFoundError("missing peer")), + patch.object(sys, "argv", ["run.py", "run", "--stage", "python"]), + contextlib.redirect_stdout(io.StringIO())): + self.assertEqual(gate.main(), 1) + record = json.loads(next((Path(temporary) / "python").glob("*/result.json")).read_text()) + self.assertEqual(record["status"], "blocked") + self.assertIn("missing peer", record["error"]) + + def test_inventory_never_reuses_pass_after_a_newer_failure_or_interruption(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + config = {"output": temporary} + fingerprint = gate.input_fingerprint({}, config, "inventory") + for index, status in enumerate(("passed", "running")): + directory = root / "inventory" / str(index) + directory.mkdir(parents=True) + (directory / "result.json").write_text(json.dumps({"source": "source", "inputs": fingerprint, + "status": status})) + (directory / "inventory.json").write_text("[]") + with self.assertRaises(gate.MissingInput): + acceptance.current_inventory({}, config, {"source": "source"}) + output = io.StringIO() + with (patch.object(gate, "load_config", return_value=({}, config)), + patch.object(gate, "source_fingerprint", return_value="source"), + patch.object(gate, "STAGES", ("inventory",)), + patch.object(sys, "argv", ["run.py", "summary"]), + contextlib.redirect_stdout(output)): + self.assertEqual(gate.main(), 1) + self.assertEqual(json.loads(output.getvalue()), {"inventory": "running"}) + (root / "inventory/1/result.json").write_text(json.dumps({ + "source": "source", "inputs": fingerprint, "status": "failed"})) + with self.assertRaises(gate.MissingInput): + acceptance.current_inventory({}, config, {"source": "source"}) + + def test_inventory_detects_a_missing_owner_even_when_leaf_names_match(self): + row = {"package": "package", "target": "first", "name": "same_name", "stage": "native"} + target = {"package": "package", "target": "first", "selector": ["--lib"]} + artifact = json.dumps({"reason": "compiler-artifact", "profile": {"test": True}, + "target": {"name": "first"}, "executable": "/owned/test"}) + with (tempfile.TemporaryDirectory() as temporary, + patch.object(acceptance, "audit_classification", return_value=[row, {**row, "target": "second"}]), + patch.object(acceptance, "workspace_targets", return_value=([target], Path(temporary))), + patch.object(acceptance, "execute", side_effect=[({}, artifact), ({}, "same_name: test\n")])): + with self.assertRaisesRegex(RuntimeError, "second"): + acceptance.inventory({"tools": {"cargo": "/cargo"}}, Path(temporary), {"commands": []}) + + def test_soak_rejects_short_load_errors_drops_missing_budget_and_fd_leak(self): + samples = [{"rss_bytes": 100, "fds": 3, "elapsed_seconds": index * 600} for index in range(4)] + case = {"suite": "http2", "implementation": "dever", "transport": "http", + "budget_after_stop": {"events": {"oom": 0, "oom_kill": 0}, "max": str(64 * 1024 ** 2)}, + "load": {"elapsed_seconds": 1800, "scheduled": 1000, "succeeded": 1000, "errors": 0}, + "disconnected": {"fd_delta": 0}, "raw": samples} + report = {"status": "complete", "cases": [case, {**case, "transport": "https"}]} + validate_soak(report, "http2", 1800, 1024, 64) + for field, value in (("elapsed_seconds", 1799), ("errors", 1), ("succeeded", 989)): + invalid = json.loads(json.dumps(report)) + invalid["cases"][0]["load"][field] = value + with self.assertRaises(RuntimeError): + validate_soak(invalid, "http2", 1800, 1024, 64) + for field, value in (("transport", "https"), ("implementation", "hyper"), + ("disconnected", {"fd_delta": 1}), + ("budget_after_stop", {"events": {"oom": 0, "oom_kill": 0}, "max": "max"})): + invalid = json.loads(json.dumps(report)) + invalid["cases"][0][field] = value + with self.assertRaises(RuntimeError): + validate_soak(invalid, "http2", 1800, 1024, 64) + case["budget_after_stop"] = None + with self.assertRaises(RuntimeError): + validate_soak(report, "http2", 1800, 1024, 64) + + def test_cms_soak_checks_descriptor_identity_and_pool_bound_not_total_count(self): + load = {"elapsed_seconds": 1800, "succeeded": 1000, "errors": 0, + "fd_delta_after_disconnect": 1, + "descriptors_before": {"3": "socket:[listener]", "4": "/fixture/platform.db"}, + "descriptors_after": {"3": "socket:[listener]", "4": "/fixture/platform.db", + "5": "/fixture/platform.db"}, + "pooled_descriptor_limits": {"/fixture/platform.db": 4}, + "trend": {"rss_late_growth_bytes": 0}} + case = {"cgroup": {"events": {"oom": 0, "oom_kill": 0}, "max": str(64 * 1024 ** 2)}, + "concurrent_reads": load} + report = {"status": "passed", "cases": {"dever": case, "md": case}} + validate_soak(report, "cms", 1800, 1024, 64) + for descriptors in ({**load["descriptors_after"], "3": "socket:[replacement]"}, + {**load["descriptors_after"], "9": "socket:[leaked]"}, + {**load["descriptors_after"], **{str(n): "/fixture/platform.db" for n in range(10, 15)}}): + invalid = json.loads(json.dumps(report)) + invalid["cases"]["md"]["concurrent_reads"]["descriptors_after"] = descriptors + with self.subTest(descriptors=descriptors), self.assertRaises(RuntimeError): + validate_soak(invalid, "cms", 1800, 1024, 64) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/cms/README.md b/test/cms/README.md new file mode 100644 index 0000000..0dba12b --- /dev/null +++ b/test/cms/README.md @@ -0,0 +1,13 @@ +# CMS 业务验收 + +显式运行，使用随机 loopback 端口、真实登录及核心角色授权、临时 SQLite 和正常 Job Worker；不访问部署数据库，不启动压测。 + +```sh +python3 test/cms/acceptance.py --compiler /absolute/path/to/dever --output /new/report-directory +``` + +运行器复制 plain CMS，并只在副本追加已有 `test/dever-tests/fixtures/postgres_api` 的真实账号/成员建立 CMD。输出目录保存无配置/凭据的 `cms-app`，可用 `--executable /absolute/path/to/cms-app` 跳过后续重复构建；该程序必须包含同一个 `fixture_member` CMD，`--project` 提供匹配的源项目配置。输入源码、二进制和部署配置均不修改。临时密钥、密码和数据库随工作目录清理。 + +覆盖分类、上传与文件清理、关联文章、修订快照、过期/并发版本冲突、作者和角色权限、发布快照、定时/同刻任务、重复执行、失败回滚/重试、排队后撤权和会话撤销/到期。仅在自有 SQLite 加临时拒绝 INSERT 的 trigger 注入晚期故障；重放只重置已完成队列行的执行状态，保留原身份、租户、权限、schema 与 payload；会话到期通过把自有平台库的 admin 会话截止时间设为过去验证，不伪造身份。所有普通业务和授权经真实 API/CMD。不会将数据库回滚等同于已提交存储文件回滚。 + +每个 HTTP 请求、构建和进程均有时限；退出时停止并回收自有进程。此验收是有界业务正确性检查，不代表 PostgreSQL、浏览器 HTTPS、长期负载或性能测试。双源码等价由现有编译器 CMS 合同检查负责，不在此复制 Markdown 解析器。 diff --git a/test/cms/acceptance.py b/test/cms/acceptance.py new file mode 100644 index 0000000..cf2114d --- /dev/null +++ b/test/cms/acceptance.py @@ -0,0 +1,476 @@ +"""自有 SQLite/HTTP CMS 业务验收；不启动压测，不使用部署数据库。""" + +import argparse +from concurrent.futures import ThreadPoolExecutor +from contextlib import closing, contextmanager +from datetime import datetime, timezone +import http.client +import json +from pathlib import Path +import secrets +import shutil +import sqlite3 +import sys +import tempfile +from threading import Barrier +import time + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "test/performance")) +from cms import (CmsClient, available_port, catalog_key, copy_cms_project, + decode_response, execute, isolated_settings, login_client, + provision_identity, require) +from process import Process +from settings import write_private_json + +ARTICLE = "/news/article/admin/manage" +REVISION = "/content/revision/admin/manage/create" +CATEGORY = "/content/category/admin/manage" +ASSET = "/media/asset/admin/manage/upload" +SCHEDULE = "/publishing/schedule/admin/manage/schedule" +ROLES = "/user/authorization/admin/manage" +BUSINESS_TABLES = ("article", "revision", "publication", "schedule", "operation", "asset", "category") + + +def append_member_fixture(project): + """复用真实凭据/成员关系 fixture；仅追加到本运行器复制的 plain 项目。""" + fixtures = ROOT / "test/dever-tests/fixtures/postgres_api" + for target, name in (("user/account/app.dever", "account_app.dever"), + ("user/membership/app.dever", "membership_app.dever"), + ("user/account/api.dever", "account_api.dever")): + path = project / "module" / target + require(path.is_file(), "compiler mode requires the plain CMS project") + with path.open("a") as destination: + destination.write("\n" + (fixtures / name).read_text()) + + +def sql_read(database, sql, parameters=()): + with closing(sqlite3.connect(database.as_uri() + "?mode=ro", uri=True, timeout=5)) as connection: + connection.row_factory = sqlite3.Row + return [dict(row) for row in connection.execute(sql, parameters)] + + +def snapshot(database): + return {table: sql_read(database, f'SELECT * FROM "{table}" ORDER BY id') + for table in BUSINESS_TABLES} + + +def fixture_write(database, sql, parameters=()): + # 此入口只接收自有临时数据库。所有正常业务创建和授权仍走真实 API/CMD。 + with closing(sqlite3.connect(database, timeout=5)) as connection: + with connection: + connection.execute(sql, parameters) + + +@contextmanager +def reject_inserts(database, table): + require(table in BUSINESS_TABLES, "unsupported failure injection table") + fixture_write(database, f'''CREATE TRIGGER cms_reject_insert BEFORE INSERT ON "{table}" + BEGIN SELECT RAISE(ABORT, 'owned CMS acceptance failure'); END''') + try: + yield + finally: + fixture_write(database, "DROP TRIGGER cms_reject_insert") + + +def wait_for(check, process, label, timeout=15): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + require(process.child.poll() is None, f"{label}: child exited; see {process.directory}") + value = check() + if value: + return value + time.sleep(0.05) + raise TimeoutError(f"{label}: timed out after {timeout}s") + + +def ready(process, client): + def probe(): + try: + client.request("GET", ARTICLE + "/list", expected=401) + return True + except (OSError, http.client.HTTPException): + client.close() + return False + wait_for(probe, process, "API startup", timeout=20) + + +def multipart(client, content, *, expected=200, field="file"): + boundary = "cms-" + secrets.token_hex(12) + body = (f'--{boundary}\r\nContent-Disposition: form-data; name="{field}"; ' + 'filename="article.txt"\r\nContent-Type: text/plain\r\n\r\n').encode() + body += content + f"\r\n--{boundary}--\r\n".encode() + client.connection.request("POST", ASSET, body, { + "Content-Type": f"multipart/form-data; boundary={boundary}", + "Origin": client.origin, "Cookie": client.cookies.get("admin", ""), + }) + response = client.connection.getresponse() + reply = response.read() + require(response.status == expected, + f"upload: HTTP {response.status}, expected {expected}: {reply!r}") + if expected != 200: + require(json.loads(reply)["code"] != 0, "upload rejection returned success") + return None + return decode_response(reply, response.status) + + +class Acceptance: + def __init__(self, executable, directory, settings, identity, password, member, member_password): + self.executable = executable + self.directory = directory + self.settings = settings + self.identity = identity + self.password = password + self.member_identity = member + self.member_password = member_password + self.port = settings["http"]["port"] + self.owner = CmsClient(self.port) + self.member = CmsClient(self.port) + self.anonymous = CmsClient(self.port) + self.database = directory / "data/tenants" / f"tenant_{identity['tenant_id']}.db" + self.worker_number = 0 + self.checks = [] + + def close(self): + for client in (self.owner, self.member, self.anonymous): + client.close() + + def passed(self, name): + self.checks.append(name) + print(f"PASS {name}", flush=True) + + def assert_unchanged(self, before, label): + require(snapshot(self.database) == before, f"{label}: business rows changed") + + def create(self, client, slug, **fields): + return client.request("POST", ARTICLE + "/create", { + "slug": slug, "title": f"Title {slug}", "body": "Draft body", **fields, + }, site="admin") + + def edit(self, client, article, *, title="Edited title", expected=200): + revision = client.request("POST", REVISION, { + "selected_article_id": article["id"], "title": title, "body": "Edited body", + "expected_version": article["version"], "category_id": article.get("category_id"), + "asset_id": article.get("asset_id"), + }, site="admin", expected=expected) + if expected != 200: + return None + current = client.request("GET", ARTICLE + "/detail", {"id": article["id"]}, site="admin") + require(revision["article_id"] == current["id"] and revision["version"] == current["version"] + and revision["title"] == current["title"] and revision["body"] == current["body"], + "revision and current article disagree") + return current + + def schedule(self, client, article, milliseconds): + instant = datetime.fromtimestamp(milliseconds / 1000, timezone.utc) + return client.request("POST", SCHEDULE, { + "article_id": article["id"], + "publish_at": instant.isoformat(timespec="milliseconds").replace("+00:00", "Z"), + }, site="admin") + + def job(self, schedule): + rows = sql_read(self.database, + "SELECT * FROM _dever_jobs WHERE target=? ORDER BY id", + ("publishing.schedule.job.publish",)) + matches = [row for row in rows if json.loads(row["payload"])["schedule_id"] == schedule["id"]] + require(len(matches) == 1, f"schedule {schedule['id']} must own exactly one Job: {rows}") + return matches[0] + + @contextmanager + def worker(self): + self.worker_number += 1 + settings = {**self.settings, "runtime": {**self.settings["runtime"], "mode": "worker"}} + write_private_json(self.directory / "config/setting.json", settings, replace=True) + try: + with Process([self.executable], self.directory / f"worker-{self.worker_number}") as process: + yield process + require(process.child.returncode == 0, f"Worker did not shut down cleanly: {process.directory}") + finally: + write_private_json(self.directory / "config/setting.json", self.settings, replace=True) + + def authorization(self): + login_client(self.owner, self.identity, self.password, "cms") + login_client(self.member, self.member_identity, self.member_password, "cms") + before = snapshot(self.database) + self.anonymous.request("GET", ARTICLE + "/list", expected=401) + self.member.request("GET", ARTICLE + "/list", site="admin", expected=403) + for client, status in ((self.anonymous, 401), (self.member, 403)): + client.request("POST", ARTICLE + "/create", { + "slug": "must-not-create", "title": "Rejected", "body": "Rejected", + }, site="admin", expected=status) + self.assert_unchanged(before, "unauthenticated and ungranted writes") + catalog = self.owner.request("GET", ROLES + "/permissions", site="admin") + selections = (("news", "article", "create", "POST"), + ("news", "article", "list", "GET"), + ("news", "article", "detail", "GET"), + ("news", "article", "publish", "POST"), + ("news", "article", "remove", "DELETE"), + ("content", "revision", "create", "POST"), + ("publishing", "schedule", "schedule", "POST")) + keys = [catalog_key(catalog, component, domain, "admin", action, method) + for component, domain, action, method in selections] + self.owner.request("POST", ROLES + "/save", { + "id": "cms-editor", "name": "CMS editor", "all_permissions": False, + "permission_keys": keys, + }, site="admin") + self.owner.request("POST", ROLES + "/grant", { + "user_id": self.member_identity["user_id"], "role_id": "cms-editor", + }, site="admin") + self.member.request("GET", ARTICLE + "/list", {"size": 20}, site="admin") + self.passed("real login, ungranted denial and explicit member role") + + def editorial(self): + category = self.owner.request("POST", CATEGORY + "/create", {"name": " CMS News "}, site="admin") + upload = multipart(self.owner, b"CMS attachment\n") + stored = self.directory / "data/upload" / upload["storage_key"] + require(stored.read_bytes() == b"CMS attachment\n", "uploaded bytes changed") + files = sorted(path.name for path in stored.parent.iterdir()) + rows = snapshot(self.database) + multipart(self.owner, b"x" * 2049, expected=400) + multipart(self.owner, b"bad field", field="unknown", expected=400) + require(sorted(path.name for path in stored.parent.iterdir()) == files, + "rejected upload persisted a file") + require(not list((self.directory / "data/tmp").iterdir()), "upload temporary files leaked") + self.assert_unchanged(rows, "rejected uploads") + article = self.create(self.owner, "linked-article", category_id=category["id"], asset_id=upload["id"]) + require(article["category_id"] == category["id"] and article["asset_id"] == upload["id"], + "article lost category/asset references") + before = snapshot(self.database) + self.edit(self.member, article, expected=403) + self.member.request("POST", ARTICLE + "/publish", {"article_id": article["id"]}, + site="admin", expected=403) + self.member.request("DELETE", ARTICLE + "/remove", {"id": article["id"]}, + site="admin", expected=403) + self.member.request("POST", SCHEDULE, { + "article_id": article["id"], + "publish_at": datetime.fromtimestamp(time.time() + 30, timezone.utc).isoformat(timespec="milliseconds"), + }, site="admin", expected=403) + self.assert_unchanged(before, "non-author mutation") + edited = self.edit(self.owner, article) + require(edited["version"] == article["version"] + 1, "edit did not advance version") + before = snapshot(self.database) + self.edit(self.owner, article, expected=409) + self.assert_unchanged(before, "stale edit") + self.passed("multipart cleanup, attachment references, author and stale-version rejection") + + # 失败发生在文章写入之后，必须连同版本/修订一起回滚。 + with reject_inserts(self.database, "revision"): + self.edit(self.owner, edited, title="Must roll back", expected=500) + self.assert_unchanged(before, "revision insert failure") + self.passed("HTTP edit failure rolls back article and revision") + winner = self.concurrent_edit(edited) + before = snapshot(self.database) + with reject_inserts(self.database, "publication"): + self.owner.request("POST", ARTICLE + "/publish", {"article_id": article["id"]}, + site="admin", expected=500) + self.assert_unchanged(before, "publication insert failure") + published = self.owner.request("POST", ARTICLE + "/publish", {"article_id": article["id"]}, site="admin") + require(published["title"] == winner["title"] and published["body"] == winner["body"], + "publication is not the winning stored revision") + before = snapshot(self.database) + self.owner.request("POST", ARTICLE + "/publish", {"article_id": article["id"]}, site="admin", expected=409) + self.assert_unchanged(before, "repeated manual publish") + self.owner.request("DELETE", CATEGORY + "/disable", {"id": category["id"]}, site="admin") + before = snapshot(self.database) + self.owner.request("POST", ARTICLE + "/create", { + "slug": "disabled-category", "title": "Rejected", "body": "Rejected", "category_id": category["id"], + }, site="admin", expected=409) + self.assert_unchanged(before, "disabled category") + self.passed("publish snapshot, late-failure rollback, repeat conflict and disabled category") + + def concurrent_edit(self, article): + barrier = Barrier(2, timeout=5) + def compete(index): + client = CmsClient(self.port) + client.cookies = self.owner.cookies.copy() + try: + barrier.wait() + payload = {"selected_article_id": article["id"], "title": f"Concurrent {index}", + "body": "Winning body", "expected_version": article["version"], + "category_id": article["category_id"], "asset_id": article["asset_id"]} + client.connection.request("POST", REVISION, json.dumps(payload), { + "Content-Type": "application/json", "Origin": client.origin, + "Cookie": client.cookies["admin"], + }) + response = client.connection.getresponse() + body = response.read() + document = json.loads(body) + require(set(document) == {"code", "message", "data"}, "invalid race envelope") + require((response.status, document["code"]) in ((200, 0), (409, 409)), + f"unexpected concurrent edit: {response.status} {body!r}") + return response.status, document["data"] + finally: + client.close() + prior = len(sql_read(self.database, "SELECT id FROM revision WHERE article_id=?", (article["id"],))) + with ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(compete, (1, 2))) + require(sorted(status for status, _ in results) == [200, 409], "concurrent edit must have one winner") + winner = next(value for status, value in results if status == 200) + require(winner["version"] == article["version"] + 1, "concurrent version advanced twice") + revisions = sql_read(self.database, "SELECT * FROM revision WHERE article_id=? ORDER BY version", (article["id"],)) + require(len(revisions) == prior + 1 and revisions[-1]["title"] == winner["title"], + "concurrent edit persisted duplicate or mismatched revisions") + self.passed("two simultaneous edits produce one version and one revision") + return winner + + def scheduled_publish(self): + articles = [self.create(self.owner, f"scheduled-{index}") for index in (1, 2)] + due = int(time.time() * 1000) + 3000 + schedules = [self.schedule(self.owner, article, due) for article in articles] + jobs = [self.job(schedule) for schedule in schedules] + require(len({job["id"] for job in jobs}) == 2 and all(job["run_at"] == due for job in jobs), + "same-time schedules merged or enqueue_at lost the due time") + before = snapshot(self.database) + with self.worker() as worker: + require(time.time() * 1000 < due - 500, "fixture did not leave a pre-due observation window") + while time.time() * 1000 < due - 300: + require(worker.child.poll() is None, "worker stopped before due time") + require(all(self.job(value)["state"] == "pending" for value in schedules), "Job ran early") + self.assert_unchanged(before, "not-yet-due schedule") + time.sleep(0.05) + wait_for(lambda: all(self.job(value)["state"] == "succeeded" for value in schedules), worker, + "same-time scheduled publications") + for article in articles: + require(len(sql_read(self.database, "SELECT id FROM publication WHERE slug=?", (article["slug"],))) == 1, + "scheduled article was not published exactly once") + self.passed("real future enqueue_at and independent same-time publications") + before = snapshot(self.database) + for schedule in schedules: + # 重放真实已完成任务；保留原身份/租户/权限/指纹/payload，不伪造执行身份。 + fixture_write(self.database, "UPDATE _dever_jobs SET state='pending',attempt=0,run_at=?," + "claim_token='',lease_until=0,error='' WHERE id=? AND state='succeeded'", + (int(time.time() * 1000), self.job(schedule)["id"])) + with self.worker() as worker: + wait_for(lambda: all(self.job(value)["state"] == "succeeded" for value in schedules), worker, + "completed task replay") + self.assert_unchanged(before, "completed task replay") + self.passed("completed Job replay adds no publication or audit") + + def retry_and_revocation(self): + article = self.create(self.owner, "retry-publish") + schedule = self.schedule(self.owner, article, int(time.time() * 1000) + 500) + before = snapshot(self.database) + with reject_inserts(self.database, "publication"): + with self.worker() as worker: + wait_for(lambda: self.job(schedule)["attempt"] == 1 + and self.job(schedule)["state"] == "pending", worker, "first failed Job attempt") + failed = self.job(schedule) + require(failed["error"] == "handler_failed", "fixture did not reach business failure") + self.assert_unchanged(before, "failed Job transaction") + with self.worker() as worker: + wait_for(lambda: self.job(schedule)["state"] == "succeeded", worker, "automatic delayed retry") + require(self.job(schedule)["attempt"] == 2, "retry did not use exactly one new claim") + self.passed("Job late failure rolls back publication/audit/status and retries successfully") + + article = self.create(self.member, "revoked-publish") + schedule = self.schedule(self.member, article, int(time.time() * 1000) + 500) + before = snapshot(self.database) + self.owner.request("DELETE", ROLES + "/revoke", { + "user_id": self.member_identity["user_id"], "role_id": "cms-editor", + }, site="admin") + self.member.request("GET", ARTICLE + "/list", site="admin", expected=403) + with self.worker() as worker: + wait_for(lambda: self.job(schedule)["state"] == "blocked", worker, "revoked queued permission") + require(self.job(schedule)["error"] == "identity_rejected", "Job blocked for the wrong reason") + self.assert_unchanged(before, "permission revoked before Job execution") + self.passed("immediate role revocation blocks queued Job before business writes") + + def session_lifecycle(self): + before = snapshot(self.database) + logged_out = self.owner.request("POST", "/user/account/admin/session/logout", {}, site="admin") + require(logged_out["revoked"], "logout did not revoke the session") + # 保留旧 Cookie 再请求，证明服务端撤销而非仅清除浏览器 Cookie。 + self.owner.request("GET", ARTICLE + "/list", site="admin", expected=401) + login_client(self.owner, self.identity, self.password, "cms") + self.owner.request("GET", ARTICLE + "/list", {"size": 20}, site="admin") + control = self.directory / "data/db/platform.db" + expired = int(time.time() * 1000) - 1 + fixture_write(control, "UPDATE session SET expires_at=? WHERE user_id=? AND site='admin'", + (expired, self.identity["user_id"])) + self.owner.request("GET", ARTICLE + "/list", site="admin", expected=401) + self.owner.request("GET", "/news/article/front/browse/list", {"size": 100}, site="front") + self.assert_unchanged(before, "revoked and expired sessions") + self.passed("logout invalidates the old cookie; expired admin session preserves front isolation") + + def run(self): + with Process([self.executable], self.directory / "api") as process: + ready(process, self.anonymous) + self.authorization() + self.editorial() + self.scheduled_publish() + self.retry_and_revocation() + self.session_lifecycle() + require(process.child.returncode == 0, f"API did not shut down cleanly: {process.directory}") + + +def run(args, directory, report): + project = directory / "project" + copy_cms_project(args.project, project) + # 忽略输入工程的任何数据库地址；本验收只生成自有 SQLite 配置。 + settings = json.loads((project / "config/setting.json").read_text()) + settings["database"] = {"default": {"type": "sqlite", "path": "data/db/platform.db", + "tenant_directory": "data/tenants", "max_connections": 4, + "max_page_size": 100}} + settings = isolated_settings(settings, available_port()) + settings["runtime"] = {"mode": "api", "shutdown_ms": 3000} + settings["job"] = {"workers": 1, "poll_ms": 50, "lease_ms": 60000, + "retry_base_ms": 2000, "retry_max_ms": 2000} + settings["http"]["upload"] = {"file_bytes": 2048} + write_private_json(project / "config/setting.json", settings, replace=True) + if args.compiler: + append_member_fixture(project) + execute([args.compiler, "check", project], project, "check") + execute([args.compiler, "build", project, "--output", project / "cms-app"], project, "build", timeout=300) + else: + shutil.copy2(args.executable, project / "cms-app") + executable = project / "cms-app" + # 二进制不含部署配置或fixture凭据，可复用以免调试断言时反复编译。 + shutil.copy2(executable, args.output / "cms-app") + report["executable_bytes"] = executable.stat().st_size + identity, password = provision_identity(executable, project, "cms") + member_password = secrets.token_urlsafe(24) + member = execute([executable, "user.account.fixture_member", json.dumps({ + "selected_tenant_id": identity["tenant_id"], "email": "member@example.invalid", + "display_name": "CMS member", "password": member_password, + })], project, "member", envelope=True) + acceptance = Acceptance(executable, project, settings, identity, password, member, member_password) + report["passed"] = acceptance.checks + try: + acceptance.run() + return {"passed": acceptance.checks, "database": "owned SQLite", "load_test": False} + finally: + acceptance.close() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument("--compiler", type=Path, help="explicit Dever compiler; builds an owned plain source copy") + source.add_argument("--executable", type=Path, help="prepared CMS executable including fixture_member CMD") + parser.add_argument("--project", type=Path, default=ROOT / "examples/cms/dever") + parser.add_argument("--output", type=Path, required=True, help="new directory for credential-free report/logs") + args = parser.parse_args() + args.project = args.project.resolve(strict=True) + for name in ("compiler", "executable"): + if getattr(args, name): + setattr(args, name, getattr(args, name).resolve(strict=True)) + args.output.mkdir(parents=True, exist_ok=False) + report = {"success": False} + with tempfile.TemporaryDirectory(prefix="dever-cms-acceptance-") as temporary: + directory = Path(temporary) + try: + report.update(run(args, directory, report), success=True) + except BaseException as error: + report["error"] = f"{type(error).__name__}: {error}" + raise + finally: + for log in directory.rglob("*.log"): + destination = args.output / log.relative_to(directory) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(log, destination) + write_private_json(args.output / "report.json", report) + print(json.dumps(report, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/test/dever-cli-tests/build_arguments.rs b/test/dever-cli-tests/build_arguments.rs new file mode 100644 index 0000000..2107e82 --- /dev/null +++ b/test/dever-cli-tests/build_arguments.rs @@ -0,0 +1,98 @@ +use super::{Action, BuildTarget, OsString, arguments_for}; + +#[test] +fn build_target_is_explicit_or_defaults_to_host() { + for options in [ + vec!["--output", "program", "--target", "linux-aarch64"], + vec!["--target", "linux-aarch64", "--output", "program"], + vec!["--output", "program"], + ] { + let explicit = options.contains(&"--target"); + let arguments: Vec<_> = [vec!["build", "project"], options] + .concat() + .into_iter() + .map(OsString::from) + .collect(); + let (root, action) = arguments_for(&arguments).unwrap(); + assert_eq!(root.to_str(), Some("project")); + let Action::Build { output, target } = action else { + panic!("expected build"); + }; + assert_eq!(output.to_str(), Some("program")); + assert_eq!( + target, + if explicit { + BuildTarget::LinuxAarch64 + } else { + BuildTarget::host().unwrap() + } + ); + } +} + +#[test] +fn build_rejects_missing_unknown_duplicate_and_misplaced_options() { + for options in [ + vec![], + vec!["--output"], + vec!["--target"], + vec!["--target", "linux-aarch64"], + vec!["--output", "program", "--target", "arm64"], + vec![ + "--output", + "program", + "--target", + "linux-aarch64", + "--target", + "linux-aarch64", + ], + vec!["--output", "program", "--output", "second"], + vec!["--output", "--target", "linux-aarch64"], + vec!["--output", "program", "--unknown"], + vec!["--output", "program", "extra"], + vec!["--output", ""], + ] { + let arguments: Vec<_> = [vec!["build", "project"], options] + .concat() + .into_iter() + .map(OsString::from) + .collect(); + assert!(arguments_for(&arguments).is_err(), "accepted {arguments:?}"); + } + for command in ["run", "test", "check"] { + let arguments = [command, "project", "--target", "linux-aarch64"].map(OsString::from); + assert!(arguments_for(&arguments).is_err()); + } +} + +#[test] +fn lib_target_keeps_specs_and_rejects_option_drift() { + let arguments = [ + "lib", + "add", + "project", + "pip:demo", + "--target", + "linux-aarch64", + ] + .map(OsString::from); + let (_, action) = arguments_for(&arguments).unwrap(); + let Action::Lib { specs, target, .. } = action else { + panic!("expected lib"); + }; + assert_eq!(specs, ["pip:demo"]); + assert_eq!(target, BuildTarget::LinuxAarch64); + for options in [ + vec!["--target"], + vec!["--target", "arm64"], + vec!["--unknown"], + vec!["--target", "linux-aarch64", "--target", "linux-aarch64"], + ] { + let arguments: Vec<_> = [vec!["lib", "update", "project"], options] + .concat() + .into_iter() + .map(OsString::from) + .collect(); + assert!(arguments_for(&arguments).is_err()); + } +} diff --git a/test/dever-cli-tests/cms_project.rs b/test/dever-cli-tests/cms_project.rs new file mode 100644 index 0000000..c40277f --- /dev/null +++ b/test/dever-cli-tests/cms_project.rs @@ -0,0 +1,451 @@ +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output, Stdio}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +#[path = "../dever-tests/tests/support/process.rs"] +pub mod process; + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "support/llvm_pack.rs"] +pub mod llvm_pack; + +static NEXT_DIRECTORY: AtomicU64 = AtomicU64::new(0); + +struct TemporaryProject(PathBuf); + +impl TemporaryProject { + fn new() -> Self { + let suffix = NEXT_DIRECTORY.fetch_add(1, Ordering::Relaxed); + let root = + std::env::temp_dir().join(format!("dever-cms-project-{}-{suffix}", std::process::id())); + fs::create_dir(&root).unwrap(); + Self(root) + } + + fn copy_from(source: &Path) -> Self { + let project = Self::new(); + copy_directory(&source.join("config"), &project.path().join("config")); + copy_directory(&source.join("module"), &project.path().join("module")); + let tests = source.join("test"); + if tests.exists() { + copy_directory(&tests, &project.path().join("test")); + } + fs::create_dir(project.path().join("data")).unwrap(); + project + } + + fn package(source: &Path, executable: &Path) -> Self { + let project = Self::new(); + copy_directory(&source.join("config"), &project.path().join("config")); + fs::copy(executable, project.path().join("cms-app")).unwrap(); + project + } + + fn path(&self) -> &Path { + &self.0 + } + + fn block_deployment_database(&self) { + // Package metadata needs valid JSON. A directory as the configured + // database proves tests use their own SQLite configuration instead. + fs::create_dir(self.path().join("data/deployment.db")).unwrap(); + fs::write( + self.path().join("config/setting.json"), + r#"{"database":{"default":{"type":"sqlite","path":"data/deployment.db"}}}"#, + ) + .unwrap(); + } +} + +impl Drop for TemporaryProject { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +fn copy_directory(source: &Path, target: &Path) { + fs::create_dir(target).unwrap(); + for entry in fs::read_dir(source).unwrap() { + let entry = entry.unwrap(); + let source = entry.path(); + let target = target.join(entry.file_name()); + if entry.file_type().unwrap().is_dir() { + copy_directory(&source, &target); + } else { + fs::copy(source, target).unwrap(); + } + } +} + +fn dever(project: &Path, action: &str, arguments: &[&str]) -> Output { + dever_with_pid(project, action, arguments).1 +} + +fn dever_with_pid(project: &Path, action: &str, arguments: &[&str]) -> (u32, Output) { + output_with_pid( + Command::new(env!("CARGO_BIN_EXE_dever")) + .arg(action) + .arg(project) + .args(arguments), + ) +} + +fn output_with_pid(command: &mut Command) -> (u32, Output) { + let capture = TemporaryProject::new(); + let stdout = capture.path().join("stdout"); + let stderr = capture.path().join("stderr"); + let mut child = command + .env_clear() + .stdout(Stdio::from(fs::File::create(&stdout).unwrap())) + .stderr(Stdio::from(fs::File::create(&stderr).unwrap())) + .spawn() + .unwrap(); + let pid = child.id(); + let status = process::wait(&mut child, Duration::from_secs(180)).unwrap(); + ( + pid, + Output { + status, + stdout: fs::read(stdout).unwrap(), + stderr: fs::read(stderr).unwrap(), + }, + ) +} + +fn assert_test_artifacts_removed(pid: u32) { + let test_prefix = format!("dever-application-test-{pid}-"); + let build_prefix = format!("dever-build-{pid}-"); + let leftovers = fs::read_dir(std::env::temp_dir()) + .unwrap() + .filter_map(Result::ok) + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .filter(|name| name.starts_with(&test_prefix) || name.starts_with(&build_prefix)) + .collect::<Vec<_>>(); + assert!( + leftovers.is_empty(), + "temporary test artifacts remain: {leftovers:?}" + ); +} + +fn assert_success(output: &Output) { + assert!( + output.status.success(), + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[cfg(unix)] +#[test] +fn clean_removes_only_stale_owned_dever_artifacts() { + use std::os::unix::fs::{MetadataExt, symlink}; + use std::time::{SystemTime, UNIX_EPOCH}; + + let project = TemporaryProject::new(); + let pid = u64::from(std::process::id()) + 1_000_000; + let suffix = NEXT_DIRECTORY.fetch_add(1, Ordering::Relaxed); + let stale = std::env::temp_dir().join(format!("dever-build-{pid}-{suffix}")); + fs::create_dir(&stale).unwrap(); + let owner = fs::metadata(&stale).unwrap().uid(); + fs::write( + stale.join(".dever-owner"), + format!( + "dever-build-owner-v1\ncreated=1\npid={pid}\nowner={owner}\ncompiler={}\n", + env!("CARGO_PKG_VERSION"), + ), + ) + .unwrap(); + fs::write(stale.join("program"), b"stale").unwrap(); + + let fresh_pid = pid + 1; + let fresh = std::env::temp_dir().join(format!("dever-build-{fresh_pid}-{suffix}")); + fs::create_dir(&fresh).unwrap(); + let created = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(); + fs::write( + fresh.join(".dever-owner"), + format!( + "dever-build-owner-v1\ncreated={created}\npid={fresh_pid}\nowner={owner}\ncompiler={}\n", + env!("CARGO_PKG_VERSION"), + ), + ).unwrap(); + let unknown = std::env::temp_dir().join(format!("dever-build-{}-{suffix}", pid + 2)); + fs::create_dir(&unknown).unwrap(); + let link = std::env::temp_dir().join(format!("dever-build-{}-{suffix}", pid + 3)); + symlink(&unknown, &link).unwrap(); + + let mut active_process = Command::new("/bin/sleep").arg("60").spawn().unwrap(); + let active_pid = u64::from(active_process.id()); + let active = std::env::temp_dir().join(format!("dever-build-{active_pid}-{suffix}")); + fs::create_dir(&active).unwrap(); + fs::write( + active.join(".dever-owner"), + format!( + "dever-build-owner-v1\ncreated=1\npid={active_pid}\nowner={owner}\ncompiler={}\n", + env!("CARGO_PKG_VERSION"), + ), + ) + .unwrap(); + fs::write(active.join("program"), b"active").unwrap(); + + let legacy = project.path().join(format!(".dever-run-{pid}")); + fs::write(&legacy, b"stale legacy").unwrap(); + let legacy_file = fs::OpenOptions::new().write(true).open(&legacy).unwrap(); + legacy_file + .set_times( + fs::FileTimes::new().set_modified(UNIX_EPOCH + std::time::Duration::from_secs(1)), + ) + .unwrap(); + let fresh_legacy = project.path().join(format!(".dever-run-{}", pid + 1)); + fs::write(&fresh_legacy, b"fresh legacy").unwrap(); + let active_legacy = project.path().join(format!(".dever-run-{active_pid}")); + fs::write(&active_legacy, b"active legacy").unwrap(); + let active_legacy_file = fs::OpenOptions::new() + .write(true) + .open(&active_legacy) + .unwrap(); + active_legacy_file + .set_times( + fs::FileTimes::new().set_modified(UNIX_EPOCH + std::time::Duration::from_secs(1)), + ) + .unwrap(); + let explicit = project.path().join("application"); + fs::write(&explicit, b"explicit output").unwrap(); + + let output = dever(project.path(), "clean", &[]); + active_process.kill().unwrap(); + active_process.wait().unwrap(); + assert_success(&output); + assert!( + String::from_utf8_lossy(&output.stdout).starts_with("cleaned 1 directories, 3 files, ") + ); + assert!(!stale.exists()); + assert!(fresh.exists()); + assert!(unknown.exists()); + assert!(link.symlink_metadata().is_ok()); + assert!(active.exists()); + assert!(!legacy.exists()); + assert!(fresh_legacy.exists()); + assert!(active_legacy.exists()); + assert!(explicit.exists()); + + fs::remove_dir_all(fresh).unwrap(); + fs::remove_file(link).unwrap(); + fs::remove_dir_all(unknown).unwrap(); + fs::remove_dir_all(active).unwrap(); +} + +#[test] +fn cms_project_checks_runs_and_builds_from_an_isolated_copy() { + let workspace = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + for format in ["dever", "md"] { + let project = TemporaryProject::copy_from(&workspace.join("examples/cms").join(format)); + let settings_path = project.path().join("config/setting.json"); + let production_settings = fs::read(&settings_path).unwrap(); + project.block_deployment_database(); + for _ in 0..2 { + let (pid, output) = dever_with_pid(project.path(), "test", &[]); + assert_cms_tests(&output); + assert_test_artifacts_removed(pid); + } + fs::write(&settings_path, production_settings).unwrap(); + fs::write(project.path().join("test/ignored.dever"), "not valid Dever").unwrap(); + + assert_success(&dever(project.path(), "check", &[])); + for _ in 0..2 { + assert_health_stdout(&dever( + project.path(), + "run", + &["--", "system.health.ping", "{}"], + )); + } + + let executable = build(project.path(), "cms-app"); + let package = TemporaryProject::package(project.path(), &executable); + assert!(!package.path().join("module").exists()); + for _ in 0..2 { + let output = output_with_pid( + Command::new(package.path().join("cms-app")) + .args(["system.health.ping", "{}"]) + .current_dir(package.path()), + ) + .1; + assert_health_stdout(&output); + } + assert!(fs::read_dir(project.path()).unwrap().all(|entry| { + !entry + .unwrap() + .file_name() + .to_string_lossy() + .starts_with(".dever-run-") + })); + } +} + +#[test] +fn project_without_tests_reports_zero_and_succeeds() { + let project = TemporaryProject::new(); + fs::create_dir(project.path().join("module")).unwrap(); + fs::write(project.path().join("module/main.dever"), "main() () {}").unwrap(); + + let (pid, output) = dever_with_pid(project.path(), "test", &[]); + assert_success(&output); + assert_eq!( + String::from_utf8_lossy(&output.stdout), + "running 0 tests\n\ntest result: ok. 0 passed; 0 failed; 0 not run\n" + ); + assert_test_artifacts_removed(pid); +} + +#[test] +fn mixed_suite_initializes_database_only_for_database_cases() { + let workspace = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let project = TemporaryProject::copy_from(&workspace.join("examples/cms/dever")); + project.block_deployment_database(); + let health = project.path().join("test/system/health"); + fs::create_dir_all(&health).unwrap(); + fs::write( + health.join("ping.dever"), + "ping() () { assert_eq(\"ready\", \"ready\") }\n", + ) + .unwrap(); + + let (pid, output) = dever_with_pid(project.path(), "test", &[]); + assert_success(&output); + assert_eq!( + String::from_utf8_lossy(&output.stdout), + "running 6 tests\ntest content/category/lifecycle ... ok\ntest news/article/publish ... ok\ntest system/health/ping ... ok\ntest user/account/bootstrap ... ok\ntest user/account/validation ... ok\ntest user/session/lifecycle ... ok\n\ntest result: ok. 6 passed; 0 failed; 0 not run\n" + ); + assert_test_artifacts_removed(pid); +} + +#[test] +fn failed_test_reports_captured_output_and_does_not_stop_later_tests() { + let project = TemporaryProject::new(); + fs::create_dir(project.path().join("module")).unwrap(); + fs::create_dir_all(project.path().join("test/sample/case")).unwrap(); + fs::write(project.path().join("module/main.dever"), "main() () {}").unwrap(); + fs::write( + project.path().join("test/sample/case/fails.dever"), + r#"fails() () { + blocking(dever.io.println("failed output")) + assert_eq(1, 2) +} +"#, + ) + .unwrap(); + fs::write( + project.path().join("test/sample/case/passes.dever"), + r#"passes() () { + blocking(dever.io.println("hidden successful output")) + assert(true) +} +"#, + ) + .unwrap(); + + let (pid, output) = dever_with_pid(project.path(), "test", &[]); + assert!(!output.status.success()); + assert!(output.stderr.is_empty()); + let stdout = String::from_utf8_lossy(&output.stdout); + let failed = stdout.find("test sample/case/fails ... FAILED").unwrap(); + let passed = stdout.find("test sample/case/passes ... ok").unwrap(); + assert!(failed < passed, "{stdout}"); + assert!( + stdout.contains("---- stdout ----\nfailed output\n"), + "{stdout}" + ); + assert!(stdout.contains("---- stderr ----"), "{stdout}"); + assert!( + stdout.contains("assert_eq failed: actual = 1, expected = 2"), + "{stdout}" + ); + assert!(!stdout.contains("hidden successful output"), "{stdout}"); + assert!( + stdout.ends_with("test result: FAILED. 1 passed; 1 failed; 0 not run\n"), + "{stdout}" + ); + assert_test_artifacts_removed(pid); +} + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[test] +fn suite_compilation_failure_marks_every_discovered_case_failed() { + let project = TemporaryProject::new(); + fs::create_dir(project.path().join("module")).unwrap(); + fs::create_dir_all(project.path().join("test/sample/case")).unwrap(); + fs::write(project.path().join("module/main.dever"), "main() () {}").unwrap(); + fs::write( + project.path().join("test/sample/case/first.dever"), + "first() () { assert(true) }\n", + ) + .unwrap(); + fs::write( + project.path().join("test/sample/case/second.dever"), + "second() () { assert(true) }\n", + ) + .unwrap(); + + let toolchain = TemporaryProject::new(); + let compiler = llvm_pack::compiler( + Path::new(env!("CARGO_BIN_EXE_dever")), + toolchain.path(), + "dever", + ); + let (pid, output) = output_with_pid(Command::new(compiler).arg("test").arg(project.path())); + assert!(!output.status.success()); + assert!(output.stderr.is_empty()); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + stdout.contains("test sample/case/first ... FAILED"), + "{stdout}" + ); + assert!( + stdout.contains("test sample/case/second ... FAILED"), + "{stdout}" + ); + assert_eq!( + stdout.matches("cannot inspect runtime pack").count(), + 2, + "{stdout}" + ); + assert!( + stdout.ends_with("test result: FAILED. 0 passed; 2 failed; 0 not run\n"), + "{stdout}" + ); + assert_test_artifacts_removed(pid); + assert!(!toolchain.path().join("cache").exists()); +} + +fn build(project: &Path, name: &str) -> PathBuf { + let executable = project.join(name); + let build = dever( + project, + "build", + &["--output", executable.to_str().unwrap()], + ); + assert_success(&build); + assert!(executable.is_file()); + executable +} + +fn assert_health_stdout(output: &Output) { + assert_success(output); + let body: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(body["code"], 0); + assert_eq!(body["message"], "ok"); + assert_eq!(body["data"]["status"], "ready"); +} + +fn assert_cms_tests(output: &Output) { + assert_success(output); + assert_eq!( + String::from_utf8_lossy(&output.stdout), + "running 5 tests\ntest content/category/lifecycle ... ok\ntest news/article/publish ... ok\ntest user/account/bootstrap ... ok\ntest user/account/validation ... ok\ntest user/session/lifecycle ... ok\n\ntest result: ok. 5 passed; 0 failed; 0 not run\n" + ); +} diff --git a/test/dever-cli-tests/external_libs.rs b/test/dever-cli-tests/external_libs.rs new file mode 100644 index 0000000..2b344e6 --- /dev/null +++ b/test/dever-cli-tests/external_libs.rs @@ -0,0 +1,1916 @@ +use std::collections::BTreeMap; +use std::fs; +use std::io::{Cursor, Write}; +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; + +use base64::Engine; +use dever_cli::libs::{ + ArtifactStore, Ecosystem, FixtureArtifactStore, FixtureRegistry, LibSpec, LockFile, + RegistryResolver, RegistryRuntime, RegistryTransport, RuntimePack, SdkFixture, prepare, + prepare_resources, provider_manifest, +}; +use pep508_rs::MarkerEnvironment; +use sha2::{Digest, Sha256, Sha512}; + +#[path = "../dever-tests/tests/support/temp.rs"] +mod temp; + +use temp::TemporaryDirectory; + +#[path = "external_libs/build.rs"] +mod build_tests; +#[path = "support/native_elf.rs"] +#[allow( + dead_code, + reason = "This test crate uses only the shared static ELF fixture" +)] +mod native_elf; +#[path = "external_libs/npm_build.rs"] +mod npm_build_tests; +#[path = "external_libs/npm.rs"] +mod npm_tests; +#[path = "external_libs/sumdb.rs"] +mod sumdb_tests; +#[path = "support/wheel.rs"] +mod wheel_fixture; +#[path = "external_libs/wheel.rs"] +mod wheel_tests; + +fn publish_fixture_artifacts(output: &std::path::Path, lock: &LockFile, store: &dyn ArtifactStore) { + let archives = output.join("artifacts"); + fs::create_dir_all(&archives).unwrap(); + for artifact in lock + .libs + .iter() + .flat_map(|lib| &lib.artifacts) + .chain(dever_cli::libs::build::npm_outputs(lock).map(|receipt| &receipt.output)) + { + let bytes = store + .verify_exact(&artifact.sha256, artifact.bytes, &artifact.target) + .unwrap(); + let path = archives.join(&artifact.sha256); + match fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&path) + { + Ok(mut file) => file.write_all(&bytes).unwrap(), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { + assert_eq!(fs::read(&path).unwrap(), bytes, "fixture archive drift"); + } + Err(error) => panic!("cannot publish fixture archive: {error}"), + } + } + lock.write_atomic(output).unwrap(); + assert_eq!( + LockFile::decode(&fs::read(output.join("dever.lock")).unwrap()).unwrap(), + *lock + ); +} + +fn doctor(lock: &LockFile) -> Result<(), String> { + dever_cli::libs::doctor_with_go_verifier(lock, &sumdb_tests::verifier()) +} + +fn verify_locked_artifacts( + lock: &LockFile, + store: &dyn ArtifactStore, + target: &str, +) -> Result<Vec<dever_cli::libs::VerifiedArtifact>, String> { + dever_cli::libs::verify_locked_artifacts_with_go_verifier( + lock, + store, + target, + &sumdb_tests::verifier(), + ) +} + +fn exec_program() -> dever_core::hir::Program { + let mut sources = dever_core::source::SourceMap::default(); + sources.add( + "notification/mail/port.dever", + "read() (value: Int) fails app.ReadResult\n", + ); + sources.add("notification/mail/app.dever", "type ReadResult { error Unavailable(message: Text) }\nread() (value: Int) { value = port.read() }\n"); + sources.add( + "notification/mail/adapter/first.dever", + "external exec \"first-worker\" {}", + ); + sources.add( + "notification/mail/adapter/second.dever", + "external exec \"second-worker\" {}", + ); + dever_core::check(&sources).unwrap_or_else(|errors| panic!("{errors:?}")) +} + +#[test] +fn exec_preparation_embeds_every_candidate_with_its_current_bytes() { + let root = TemporaryDirectory::new(); + let program = exec_program(); + let entries = program.external_worker_entries(); + assert_eq!(entries.len(), 2); + for entry in &entries { + let path = root.path().join(entry); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + let mut bytes = native_elf::static_executable(); + bytes.extend_from_slice(entry.as_bytes()); + fs::write(&path, bytes).unwrap(); + } + assert!( + prepare_resources( + root.path(), + &program, + dever_cli::toolchain::BuildTarget::host().unwrap() + ) + .unwrap_err() + .contains("signed managed Dever release") + ); + let prepare = || { + let input = entries + .iter() + .map(|entry| { + let bytes = fs::read(root.path().join(entry)).unwrap(); + dever_core::native::EmbeddedResource { + path: entry.clone(), + sha256: digest(&bytes), + bytes, + executable: true, + } + }) + .collect::<Vec<_>>(); + dever_cli::workers::prepare(root.path(), &program, &input).unwrap() + }; + let resources = prepare(); + assert_eq!(resources.len(), entries.len() * 2); + for entry in &entries { + let manifest = resources + .iter() + .find(|resource| resource.path == format!("{entry}.dever-worker.json")) + .unwrap(); + let manifest: serde_json::Value = serde_json::from_slice(&manifest.bytes).unwrap(); + let worker = resources + .iter() + .find(|resource| resource.path == manifest["executable"].as_str().unwrap()) + .unwrap(); + assert_eq!(worker.bytes, fs::read(root.path().join(entry)).unwrap()); + assert!(worker.executable); + } + let mut changed_binary = native_elf::static_executable(); + changed_binary.extend_from_slice(b"changed"); + fs::write(root.path().join(&entries[0]), &changed_binary).unwrap(); + let changed = prepare(); + assert!( + changed + .iter() + .any(|resource| resource.bytes == changed_binary) + ); + assert!( + !resources + .iter() + .any(|resource| resource.bytes == changed_binary) + ); + fs::remove_file(root.path().join(&entries[1])).unwrap(); + assert!( + prepare_resources( + root.path(), + &program, + dever_cli::toolchain::BuildTarget::host().unwrap() + ) + .unwrap_err() + .contains("cannot inspect") + ); +} + +#[test] +fn exec_worker_preparation_rejects_host_elf_for_arm_target() { + use dever_cli::toolchain::BuildTarget; + let root = TemporaryDirectory::new(); + let program = exec_program(); + let mut arm = native_elf::static_executable(); + arm[18..20].copy_from_slice(&183u16.to_le_bytes()); + let mut resources = program + .external_worker_entries() + .into_iter() + .map(|path| dever_core::native::EmbeddedResource { + path, + sha256: digest(&arm), + bytes: arm.clone(), + executable: true, + }) + .collect::<Vec<_>>(); + let output = dever_cli::workers::prepare_for_target( + root.path(), + &program, + &resources, + BuildTarget::LinuxAarch64, + &[], + ) + .unwrap(); + assert_eq!( + output.iter().filter(|file| file.executable).count(), + resources.len() + ); + resources[0].bytes[18..20].copy_from_slice(&62u16.to_le_bytes()); + resources[0].sha256 = digest(&resources[0].bytes); + assert!( + dever_cli::workers::prepare_for_target( + root.path(), + &program, + &resources, + BuildTarget::LinuxAarch64, + &[] + ) + .unwrap_err() + .contains("differs from target linux-aarch64") + ); +} + +#[cfg(unix)] +#[test] +fn exec_preparation_rejects_project_worker_symlinks() { + let root = TemporaryDirectory::new(); + let outside = TemporaryDirectory::new(); + let program = exec_program(); + for entry in program.external_worker_entries() { + let path = root.path().join(entry); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(outside.path().join("worker"), b"outside").unwrap(); + std::os::unix::fs::symlink(outside.path().join("worker"), path).unwrap(); + } + assert!( + prepare_resources( + root.path(), + &program, + dever_cli::toolchain::BuildTarget::host().unwrap() + ) + .unwrap_err() + .contains("symbolic link") + ); +} + +static NEXT: AtomicU64 = AtomicU64::new(0); + +fn project() -> PathBuf { + let path = std::env::temp_dir().join(format!( + "dever-lib-test-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir(&path).unwrap(); + path +} + +struct LocalRegistry(BTreeMap<(Ecosystem, String), Vec<u8>>); + +impl RegistryTransport for LocalRegistry { + fn get_optional( + &self, + ecosystem: Ecosystem, + path: &str, + _limit: usize, + ) -> Result<Option<Vec<u8>>, String> { + Ok(self.0.get(&(ecosystem, path.into())).cloned()) + } + fn get(&self, ecosystem: Ecosystem, path: &str, _limit: usize) -> Result<Vec<u8>, String> { + self.0 + .get(&(ecosystem, path.into())) + .cloned() + .ok_or_else(|| format!("local registry has no {path}")) + } +} + +fn digest(bytes: &[u8]) -> String { + Sha256::digest(bytes) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +fn zip_file(path: &str) -> Vec<u8> { + let mut writer = zip::ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file(path, zip::write::SimpleFileOptions::default()) + .unwrap(); + writer.write_all(b"package bytes").unwrap(); + writer.finish().unwrap().into_inner() +} + +fn python_markers() -> MarkerEnvironment { + wheel_fixture::markers() +} + +fn pure_wheel(name: &str, version: &str, requires: &[&str], extras: &[&str]) -> Vec<u8> { + wheel_fixture::pack( + name, + version, + "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + &wheel_fixture::metadata(name, version, requires, extras), + &[(&format!("{name}/__init__.py"), b"package bytes".to_vec())], + ) +} + +fn registry_runtime(ecosystem: Ecosystem) -> RegistryRuntime { + RegistryRuntime { + pack: RuntimePack { + name: ecosystem.as_str().into(), + version: "3.12.0".into(), + sha256: digest(b"signed runtime"), + }, + target: "linux-x86_64".into(), + python_markers: (ecosystem == Ecosystem::Pip).then(python_markers), + python_wheel_tags: if ecosystem == Ecosystem::Pip { + vec!["py3-none-any".into()] + } else { + Vec::new() + }, + npm_libc: (ecosystem == Ecosystem::Npm).then(|| "glibc".into()), + } +} + +fn local_registry() -> LocalRegistry { + let py_a = pure_wheel("a", "1.0.0", &["b >=1.0.0,<2.0.0"], &[]); + let py_b = pure_wheel("b", "1.1.0", &[], &[]); + let npm_a = npm_tests::archive( + "root", + "1.0.0", + serde_json::json!({"dependencies":{"child":"^1.0.0"}}), + &[], + ); + let npm_b = npm_tests::archive("child", "1.2.0", serde_json::json!({}), &[]); + let go_a = zip_file("example.com/a@v1.0.0/a.go"); + let go_b = zip_file("example.com/b@v1.2.0/b.go"); + let mut files = BTreeMap::new(); + for (name, version, bytes, requires) in [ + ("a", "1.0.0", &py_a, vec!["b >=1.0.0,<2.0.0"]), + ("b", "1.1.0", &py_b, vec![]), + ] { + files.insert( + (Ecosystem::Pip, format!("/pypi/{name}/json")), + serde_json::to_vec(&serde_json::json!({"releases": {(version): []}})).unwrap(), + ); + files.insert((Ecosystem::Pip, format!("/pypi/{name}/{version}/json")), serde_json::to_vec(&serde_json::json!({ + "info": {"requires_dist": requires}, + "urls": [{"packagetype": "bdist_wheel", "filename": format!("{name}-{version}-py3-none-any.whl"), + "url": format!("https://mock/{name}.whl"), "digests": {"sha256": digest(bytes)}}] + })).unwrap()); + files.insert( + (Ecosystem::Pip, format!("https://mock/{name}.whl")), + bytes.clone(), + ); + } + for (name, version, bytes, dependencies) in [ + ( + "root", + "1.0.0", + &npm_a, + serde_json::json!({"child": "^1.0.0"}), + ), + ("child", "1.2.0", &npm_b, serde_json::json!({})), + ] { + files.insert( + (Ecosystem::Npm, format!("/{name}")), + serde_json::to_vec(&serde_json::json!({"versions": {(version): {}}})).unwrap(), + ); + files.insert((Ecosystem::Npm, format!("/{name}/{version}")), serde_json::to_vec(&serde_json::json!({ + "dependencies": dependencies, + "dist": {"tarball": format!("https://mock/{name}.tgz"), + "integrity": format!("sha512-{}", base64::engine::general_purpose::STANDARD.encode(Sha512::digest(bytes)))} + })).unwrap()); + files.insert( + (Ecosystem::Npm, format!("https://mock/{name}.tgz")), + bytes.clone(), + ); + } + files.insert( + (Ecosystem::Go, "/example.com/a/@v/v1.0.0.mod".into()), + b"module example.com/a\nrequire example.com/b v1.2.0\n".to_vec(), + ); + files.insert( + (Ecosystem::Go, "/example.com/b/@v/v1.2.0.mod".into()), + b"module example.com/b\n".to_vec(), + ); + files.insert((Ecosystem::Go, "/example.com/a/@v/v1.0.0.zip".into()), go_a); + files.insert((Ecosystem::Go, "/example.com/b/@v/v1.2.0.zip".into()), go_b); + let mut registry = LocalRegistry(files); + sumdb_tests::seal(&mut registry); + registry +} + +fn local_resolver<'a>( + registry: &'a LocalRegistry, + cache: &'a FixtureArtifactStore, +) -> RegistryResolver<'a> { + RegistryResolver { + build: None, + transport: registry, + store: cache, + runtimes: [Ecosystem::Pip, Ecosystem::Npm, Ecosystem::Go] + .into_iter() + .map(|ecosystem| (ecosystem.clone(), registry_runtime(ecosystem))) + .collect(), + go_sumdb: dever_cli::libs::sumdb::ChecksumDatabase::new(sumdb_tests::verifier(), None) + .unwrap(), + } +} + +#[test] +fn real_registry_protocols_resolve_transitive_closure_and_cache_exact_archives() { + let registry = local_registry(); + let cache = FixtureArtifactStore::default(); + let resolver = local_resolver(®istry, &cache); + let roots = ["pip:a@1.0.0", "npm:root@1.0.0", "go:example.com/a@1.0.0"] + .into_iter() + .map(str::parse) + .collect::<Result<Vec<LibSpec>, _>>() + .unwrap(); + let lock = dever_cli::libs::LibResolver::resolve(&resolver, &roots).unwrap(); + assert_eq!(lock.libs.len(), 6); + assert_eq!( + dever_cli::libs::LibResolver::resolve(&local_resolver(®istry, &cache), &roots) + .unwrap() + .encode() + .unwrap(), + lock.encode().unwrap() + ); + doctor(&lock).unwrap(); + assert_eq!( + verify_locked_artifacts(&lock, &cache, "linux-x86_64") + .unwrap() + .len(), + 6 + ); + assert!( + verify_locked_artifacts(&lock, &FixtureArtifactStore::default(), "linux-x86_64") + .unwrap_err() + .contains("missing") + ); +} + +#[test] +fn cross_target_resolver_locks_all_three_ecosystems_to_arm_without_host_tools() { + let registry = local_registry(); + let cache = FixtureArtifactStore::default(); + let mut resolver = local_resolver(®istry, &cache); + for runtime in resolver.runtimes.values_mut() { + runtime.target = "linux-aarch64".into(); + if let Some(markers) = &runtime.python_markers { + let mut markers = serde_json::to_value(markers).unwrap(); + markers["platform_machine"] = serde_json::json!("aarch64"); + runtime.python_markers = Some(serde_json::from_value(markers).unwrap()); + } + } + let roots = ["pip:a@1.0.0", "npm:root@1.0.0", "go:example.com/a@1.0.0"] + .into_iter() + .map(str::parse) + .collect::<Result<Vec<LibSpec>, _>>() + .unwrap(); + let lock = dever_cli::libs::LibResolver::resolve(&resolver, &roots).unwrap(); + assert_eq!(lock.libs.len(), 6); + assert!( + lock.libs + .iter() + .flat_map(|lib| &lib.artifacts) + .all(|artifact| artifact.target == "linux-aarch64") + ); + assert_eq!( + verify_locked_artifacts(&lock, &cache, "linux-aarch64") + .unwrap() + .len(), + 6 + ); + assert!( + verify_locked_artifacts(&lock, &cache, "linux-x86_64") + .unwrap_err() + .contains("not linux-x86_64") + ); +} + +#[test] +fn real_registry_rejects_conflicts_hash_drift_and_archive_escape() { + let registry = local_registry(); + let cache = FixtureArtifactStore::default(); + let resolver = local_resolver(®istry, &cache); + let conflicting = [ + "pip:a@1.0.0".parse().unwrap(), + "pip:a@2.0.0".parse().unwrap(), + ]; + assert!( + dever_cli::libs::LibResolver::resolve(&resolver, &conflicting) + .unwrap_err() + .contains("no pip version satisfies") + ); + + let mut registry = local_registry(); + registry.0.insert( + (Ecosystem::Pip, "https://mock/a.whl".into()), + zip_file("changed.py"), + ); + let resolver = local_resolver(®istry, &cache); + assert!( + dever_cli::libs::LibResolver::resolve(&resolver, &["pip:a@1.0.0".parse().unwrap()]) + .unwrap_err() + .contains("SHA-256 mismatch") + ); + + let mut registry = local_registry(); + registry.0.insert( + (Ecosystem::Go, "/example.com/a/@v/v1.0.0.zip".into()), + zip_file("../escape"), + ); + let resolver = local_resolver(®istry, &cache); + assert!( + dever_cli::libs::LibResolver::resolve( + &resolver, + &["go:example.com/a@1.0.0".parse().unwrap()] + ) + .unwrap_err() + .contains("escapes its package") + ); +} + +#[test] +fn python_names_and_unconstrained_requirements_follow_pep_rules() { + let canonical: LibSpec = "pip:Requests_FOO@1.0.0".parse().unwrap(); + assert_eq!(canonical.key(), "pip:requests-foo@1.0.0"); + assert_eq!(canonical, "pip:requests.foo@1.0.0".parse().unwrap()); + + let mut registry = local_registry(); + let key = (Ecosystem::Pip, "/pypi/a/1.0.0/json".into()); + let mut release: serde_json::Value = + serde_json::from_slice(registry.0.get(&key).unwrap()).unwrap(); + release["info"]["requires_dist"] = serde_json::json!(["b"]); + let archive = pure_wheel("a", "1.0.0", &["b"], &[]); + release["urls"][0]["digests"]["sha256"] = serde_json::json!(digest(&archive)); + registry + .0 + .insert((Ecosystem::Pip, "https://mock/a.whl".into()), archive); + registry + .0 + .insert(key.clone(), serde_json::to_vec(&release).unwrap()); + let cache = FixtureArtifactStore::default(); + let resolver = local_resolver(®istry, &cache); + assert_eq!( + dever_cli::libs::LibResolver::resolve(&resolver, &["pip:a@1.0.0".parse().unwrap()]) + .unwrap() + .libs + .len(), + 2 + ); + + release["info"]["requires_dist"] = serde_json::json!({"invalid": true}); + registry + .0 + .insert(key, serde_json::to_vec(&release).unwrap()); + let resolver = local_resolver(®istry, &cache); + // Even malformed release-level upload metadata cannot replace the selected + // wheel's validated METADATA. + assert_eq!( + dever_cli::libs::LibResolver::resolve(&resolver, &["pip:a@1.0.0".parse().unwrap()]) + .unwrap() + .libs + .len(), + 2 + ); +} + +#[test] +fn registry_transport_failure_never_selects_an_older_candidate() { + let mut registry = local_registry(); + registry.0.insert( + (Ecosystem::Pip, "/pypi/b/json".into()), + serde_json::to_vec(&serde_json::json!({"releases": {"1.2.0": [], "1.1.0": []}})).unwrap(), + ); + let cache = FixtureArtifactStore::default(); + let resolver = local_resolver(®istry, &cache); + let error = dever_cli::libs::LibResolver::resolve(&resolver, &["pip:a@1.0.0".parse().unwrap()]) + .unwrap_err(); + assert!(error.contains("local registry has no /pypi/b/1.2.0/json")); +} + +fn python_release( + registry: &mut LocalRegistry, + name: &str, + version: &str, + extras: &[&str], + requires: &[&str], +) { + let index = registry + .0 + .entry((Ecosystem::Pip, format!("/pypi/{name}/json"))) + .or_insert_with(|| br#"{"releases":{}}"#.to_vec()); + let mut document: serde_json::Value = serde_json::from_slice(index).unwrap(); + document["releases"][version] = serde_json::json!([]); + *index = serde_json::to_vec(&document).unwrap(); + let wheel = pure_wheel(name, version, requires, extras); + let url = format!("https://mock/{name}-{version}.whl"); + registry.0.insert((Ecosystem::Pip, format!("/pypi/{name}/{version}/json")), + serde_json::to_vec(&serde_json::json!({ + "info": {"provides_extra": extras, "requires_dist": requires}, + "urls": [{"packagetype": "bdist_wheel", "filename": format!("{name}-{version}-py3-none-any.whl"), + "url": url, "digests": {"sha256": digest(&wheel)}}] + })).unwrap()); + registry.0.insert((Ecosystem::Pip, url), wheel); +} + +fn python_lock(registry: &LocalRegistry, roots: &[&str]) -> Result<LockFile, String> { + let cache = FixtureArtifactStore::default(); + let resolver = local_resolver(registry, &cache); + dever_cli::libs::LibResolver::resolve( + &resolver, + &roots + .iter() + .map(|root| root.parse().unwrap()) + .collect::<Vec<_>>(), + ) +} + +fn dependency_keys(lock: &LockFile, key: &str) -> Vec<String> { + lock.libs + .iter() + .find(|lib| lib.spec.key() == key) + .unwrap() + .dependencies + .iter() + .map(|dependency| dependency.spec.key()) + .collect() +} + +#[test] +fn python_extras_are_canonical_exact_identities() { + let spec: LibSpec = "pip:Demo_Pkg[Foo_Bar,z,foo.bar,Z]@1.0.0".parse().unwrap(); + assert_eq!(spec.key(), "pip:demo-pkg[foo-bar,z]@1.0.0"); + assert_eq!(spec.distribution_name(), "demo-pkg"); + assert_eq!(spec.key().parse::<LibSpec>().unwrap(), spec); + for invalid in [ + "pip:a[]@1.0.0", + "pip:a[x,]@1.0.0", + "pip:a[,x]@1.0.0", + "pip:a[x,,y]@1.0.0", + "pip:[x]@1.0.0", + "pip:@1.0.0", + "pip:a[x@y]@1.0.0", + "pip:a[x][y]@1.0.0", + "pip:a[x@1.0.0", + "npm:a[x]@1.0.0", + "go:example.com/a[x]@1.0.0", + ] { + assert!(invalid.parse::<LibSpec>().is_err(), "accepted {invalid}"); + } + let mut registry = LocalRegistry(BTreeMap::new()); + python_release(&mut registry, "a", "1.0.0", &["x", "y"], &[]); + let lock = python_lock(®istry, &["pip:a[y,x]@1.0.0"]).unwrap(); + let encoded = lock.encode().unwrap(); + assert_eq!(LockFile::decode(&encoded).unwrap(), lock); + let invalid = String::from_utf8(encoded) + .unwrap() + .replace("a[x,y]", "a[y,x]"); + assert!( + LockFile::decode(invalid.as_bytes()) + .unwrap_err() + .contains("not canonical") + ); + python_release(&mut registry, "a", "1.0.0", &[""], &[]); + assert!( + python_lock(®istry, &["pip:a@1.0.0"]) + .unwrap_err() + .contains("Provides-Extra") + ); +} + +#[test] +fn python_extras_expand_transitively_after_selection_without_leaking_into_base() { + let mut registry = LocalRegistry(BTreeMap::new()); + python_release(&mut registry, "root", "1.0.0", &[], &["a[x]", "z"]); + python_release( + &mut registry, + "a", + "1.0.0", + &["x", "y"], + &[ + "c; extra == 'x'", + "d; extra == 'y'", + "absent; extra == 'x' and sys_platform == 'win32'", + ], + ); + python_release(&mut registry, "z", "1.0.0", &[], &["a[y]"]); + python_release(&mut registry, "c", "1.0.0", &[], &[]); + python_release(&mut registry, "d", "1.0.0", &[], &[]); + let lock = python_lock(®istry, &["pip:root@1.0.0", "pip:a@1.0.0"]).unwrap(); + doctor(&lock).unwrap(); + assert_eq!(dependency_keys(&lock, "pip:a@1.0.0"), Vec::<String>::new()); + assert_eq!(dependency_keys(&lock, "pip:a[x]@1.0.0"), ["pip:c@1.0.0"]); + assert_eq!(dependency_keys(&lock, "pip:a[y]@1.0.0"), ["pip:d@1.0.0"]); + assert_eq!(dependency_keys(&lock, "pip:z@1.0.0"), ["pip:a[y]@1.0.0"]); + let variants = lock + .libs + .iter() + .filter(|lib| lib.spec.distribution_name() == "a") + .collect::<Vec<_>>(); + assert_eq!(variants.len(), 3); + assert!( + variants + .iter() + .all(|lib| lib.artifacts == variants[0].artifacts) + ); + assert_eq!( + python_lock(®istry, &["pip:a@1.0.0"]).unwrap().libs.len(), + 1 + ); +} + +#[test] +fn python_extra_conflicts_backtrack_without_retaining_failed_edges() { + let mut registry = LocalRegistry(BTreeMap::new()); + python_release( + &mut registry, + "root", + "1.0.0", + &[], + &["b >=1,<3", "c ==1.0.0", "z"], + ); + python_release( + &mut registry, + "b", + "2.0.0", + &["x"], + &["c >=2; extra == 'x'"], + ); + python_release(&mut registry, "b", "1.0.0", &["x"], &["d; extra == 'x'"]); + python_release(&mut registry, "c", "1.0.0", &[], &[]); + python_release(&mut registry, "d", "1.0.0", &[], &[]); + python_release(&mut registry, "z", "1.0.0", &[], &["b[x]"]); + let lock = python_lock(®istry, &["pip:root@1.0.0"]).unwrap(); + assert_eq!(dependency_keys(&lock, "pip:b[x]@1.0.0"), ["pip:d@1.0.0"]); + assert!(!lock.libs.iter().any(|lib| lib.spec.version == "2.0.0")); + doctor(&lock).unwrap(); + python_release(&mut registry, "b", "2.0.0", &[], &[]); + assert!( + python_lock(®istry, &["pip:root@1.0.0"]) + .unwrap() + .libs + .iter() + .any(|lib| lib.spec.key() == "pip:b[x]@1.0.0") + ); + assert!( + python_lock(®istry, &["pip:b[missing]@1.0.0"]) + .unwrap_err() + .contains("does not provide requested extra") + ); + assert!( + python_lock(®istry, &["pip:b[x]@1.0.0", "pip:b@2.0.0"]) + .unwrap_err() + .contains("no pip version satisfies") + ); +} + +#[test] +fn python_self_extra_activation_is_not_a_distribution_cycle() { + let mut registry = LocalRegistry(BTreeMap::new()); + python_release( + &mut registry, + "a", + "1.0.0", + &["x"], + &["a[x]", "b; extra == 'x'"], + ); + python_release(&mut registry, "b", "1.0.0", &[], &[]); + let lock = python_lock(®istry, &["pip:a@1.0.0"]).unwrap(); + doctor(&lock).unwrap(); + assert_eq!(dependency_keys(&lock, "pip:a@1.0.0"), ["pip:a[x]@1.0.0"]); + assert_eq!(dependency_keys(&lock, "pip:a[x]@1.0.0"), ["pip:b@1.0.0"]); + python_release( + &mut registry, + "a", + "1.0.0", + &["x", "y"], + &[ + "a[y]; extra == 'x'", + "a[x]; extra == 'y'", + "b; extra == 'y'", + ], + ); + let lock = python_lock(®istry, &["pip:a[x]@1.0.0"]).unwrap(); + doctor(&lock).unwrap(); + assert_eq!(lock.libs.len(), 3); + python_release(&mut registry, "b", "1.0.0", &[], &["a[x]"]); + assert!( + doctor(&python_lock(®istry, &["pip:a[x]@1.0.0"]).unwrap()) + .unwrap_err() + .contains("cyclic locked lib") + ); +} + +#[test] +fn python_extra_workers_keep_exact_roots_and_reject_inconsistent_locks() { + use dever_cli::libs::{LockedWorker, resolve_workers}; + let mut registry = LocalRegistry(BTreeMap::new()); + python_release(&mut registry, "a", "1.0.0", &["x"], &["b; extra == 'x'"]); + python_release(&mut registry, "b", "1.0.0", &[], &[]); + python_release(&mut registry, "b", "2.0.0", &[], &[]); + let cache = FixtureArtifactStore::default(); + let resolver = local_resolver(®istry, &cache); + let worker = |adapter: &str, roots: &[&str]| LockedWorker { + port: "sample.worker".into(), + adapter: adapter.into(), + ecosystem: "pip".into(), + runtime: Some(registry_runtime(Ecosystem::Pip).pack), + entry: format!("worker/{adapter}.py"), + schema: "checked-port".into(), + capabilities: vec![], + operations: vec!["read".into()], + libs: roots.iter().map(|root| root.parse().unwrap()).collect(), + }; + let lock = resolve_workers( + &resolver, + &[], + vec![ + worker("plain", &["pip:a@1.0.0"]), + worker("extra", &["pip:a[x]@1.0.0"]), + worker("old", &["pip:b@1.0.0"]), + ], + ) + .unwrap(); + assert_eq!(dependency_keys(&lock, "pip:a@1.0.0"), Vec::<String>::new()); + assert_eq!(dependency_keys(&lock, "pip:a[x]@1.0.0"), ["pip:b@2.0.0"]); + assert_eq!( + lock.workers + .iter() + .find(|worker| worker.adapter == "extra") + .unwrap() + .libs[0] + .key(), + "pip:a[x]@1.0.0" + ); + assert_eq!(LockFile::decode(&lock.encode().unwrap()).unwrap(), lock); + let root = TemporaryDirectory::new(); + lock.write_atomic(root.path()).unwrap(); + let report = prepare(root.path(), &["pip:a[x]@1.0.0".into()]) + .unwrap() + .unwrap(); + assert_eq!(report.libs, 4); + assert_eq!(report.artifacts, 3); + assert_eq!( + report.artifact_bytes, + (pure_wheel("a", "1.0.0", &["b; extra == 'x'"], &["x"]).len() + + pure_wheel("b", "1.0.0", &[], &[]).len() + + pure_wheel("b", "2.0.0", &[], &[]).len()) as u64 + ); + assert!( + prepare(root.path(), &["pip:a[y]@1.0.0".into()]) + .unwrap_err() + .contains("does not contain declared") + ); + let mut resources = verify_locked_artifacts(&lock, &cache, "linux-x86_64") + .unwrap() + .into_iter() + .map(|artifact| dever_core::native::EmbeddedResource { + path: artifact.path, + sha256: artifact.sha256, + bytes: artifact.bytes, + executable: false, + }) + .collect::<Vec<_>>(); + dever_cli::libs::prune_expanded_resources( + &lock, + &lock.workers, + &["pip:a@1.0.0".parse().unwrap()], + &mut resources, + ) + .unwrap(); + let archive = &lock + .libs + .iter() + .find(|lib| lib.spec.name == "a") + .unwrap() + .artifacts[0] + .path; + assert!(resources.iter().any(|resource| &resource.path == archive)); + let mut conflict = lock.clone(); + conflict + .workers + .iter_mut() + .find(|worker| worker.adapter == "plain") + .unwrap() + .libs + .extend([ + "pip:a[x]@1.0.0".parse().unwrap(), + "pip:b@1.0.0".parse().unwrap(), + ]); + assert!( + doctor(&conflict) + .unwrap_err() + .contains("conflicting versions") + ); + let mut drift = lock.clone(); + drift + .libs + .iter_mut() + .find(|lib| lib.spec.name == "a[x]") + .unwrap() + .artifacts[0] + .sha256 = "00".repeat(32); + assert!( + doctor(&drift) + .unwrap_err() + .contains("inconsistent distribution artifacts") + ); + let mut missing = lock; + missing.libs.retain(|lib| lib.spec.name != "a[x]"); + assert!(doctor(&missing).unwrap_err().contains("references missing")); + // Folding all Workers' variants into base names would invent an a/b cycle. + python_release(&mut registry, "a", "1.0.0", &["x"], &["b; extra == 'x'"]); + python_release(&mut registry, "b", "2.0.0", &["y"], &["a; extra == 'y'"]); + let resolver = local_resolver(®istry, &cache); + let independent = resolve_workers( + &resolver, + &[], + vec![ + worker("first", &["pip:a[x]@1.0.0"]), + worker("second", &["pip:b[y]@2.0.0"]), + ], + ) + .unwrap(); + doctor(&independent).unwrap(); +} + +#[test] +fn go_proxy_rejects_module_identity_and_unavailable_managed_toolchain() { + let cache = FixtureArtifactStore::default(); + let mut registry = local_registry(); + registry.0.insert( + (Ecosystem::Go, "/example.com/b/@v/v1.2.0.mod".into()), + b"module wrong.example/b\n".to_vec(), + ); + sumdb_tests::seal(&mut registry); + let resolver = local_resolver(®istry, &cache); + assert!( + dever_cli::libs::LibResolver::resolve( + &resolver, + &["go:example.com/a@1.0.0".parse().unwrap()] + ) + .unwrap_err() + .contains("declaration does not match") + ); + + let mut registry = local_registry(); + registry.0.insert( + (Ecosystem::Go, "/example.com/a/@v/v1.0.0.mod".into()), + b"module example.com/a\ntoolchain go99.0.0\n".to_vec(), + ); + sumdb_tests::seal(&mut registry); + let resolver = local_resolver(®istry, &cache); + assert!( + dever_cli::libs::LibResolver::resolve( + &resolver, + &["go:example.com/a@1.0.0".parse().unwrap()] + ) + .unwrap_err() + .contains("requires toolchain") + ); + + let mut registry = local_registry(); + registry.0.insert( + (Ecosystem::Go, "/example.com/a/@v/v1.0.0.zip".into()), + zip_file("other.example/a@v1.0.0/a.go"), + ); + let resolver = local_resolver(®istry, &cache); + assert!( + dever_cli::libs::LibResolver::resolve( + &resolver, + &["go:example.com/a@1.0.0".parse().unwrap()] + ) + .unwrap_err() + .contains("mismatched module root") + ); +} + +#[test] +fn http_registry_uses_an_explicit_loopback_origin_without_host_proxy() { + use std::io::{Read, Write}; + use std::net::TcpListener; + + let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let origin = format!("http://{}", listener.local_addr().unwrap()); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = [0u8; 1024]; + let received = stream.read(&mut request).unwrap(); + assert!( + std::str::from_utf8(&request[..received]) + .unwrap() + .starts_with("GET /pypi/a/json HTTP/1.1") + ); + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}") + .unwrap(); + }); + let registry = dever_cli::libs::HttpRegistry::new( + [(Ecosystem::Pip, origin.clone())], + [(Ecosystem::Pip, vec![origin])], + ); + assert_eq!( + registry.get(Ecosystem::Pip, "/pypi/a/json", 16).unwrap(), + b"{}" + ); + assert!( + registry + .get(Ecosystem::Pip, "https://other.example/a.whl", 16) + .unwrap_err() + .contains("untrusted origin") + ); + server.join().unwrap(); +} + +fn loopback_go_proxy( + respond: impl Fn(&str, &str) -> (u16, Option<String>, Vec<u8>) + Send + 'static, +) -> ( + String, + std::sync::mpsc::Sender<()>, + std::thread::JoinHandle<Vec<String>>, +) { + use std::io::{Read, Write}; + use std::net::TcpListener; + + let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + listener.set_nonblocking(true).unwrap(); + let origin = format!("http://{}", listener.local_addr().unwrap()); + let (done, finished) = std::sync::mpsc::channel(); + let server_origin = origin.clone(); + let server = std::thread::spawn(move || { + let mut requests = Vec::new(); + while finished.try_recv().is_err() { + let (mut stream, _) = match listener.accept() { + Ok(connection) => connection, + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + std::thread::sleep(std::time::Duration::from_millis(2)); + continue; + } + Err(error) => panic!("loopback proxy accept failed: {error}"), + }; + let mut request = [0u8; 2048]; + let count = stream.read(&mut request).unwrap(); + let line = std::str::from_utf8(&request[..count]) + .unwrap() + .lines() + .next() + .unwrap(); + let path = line.split_whitespace().nth(1).unwrap().to_owned(); + requests.push(path.clone()); + let (status, location, body) = respond(&server_origin, &path); + let location = location + .map(|value| format!("Location: {value}\r\n")) + .unwrap_or_default(); + let header = format!( + "HTTP/1.1 {status} Test\r\n{location}Content-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + stream.write_all(header.as_bytes()).unwrap(); + stream.write_all(&body).unwrap(); + } + requests + }); + (origin, done, server) +} + +fn loopback_go_registry(origin: &str) -> dever_cli::libs::HttpRegistry { + dever_cli::libs::HttpRegistry::new( + [(Ecosystem::Go, origin.to_owned())], + [(Ecosystem::Go, vec![origin.to_owned()])], + ) + .with_go_archive_mirror(&format!("{origin}/proxy-golang-org-prod/")) + .unwrap() +} + +#[test] +fn go_archive_redirect_resolves_and_locks_the_mirror_bytes() { + let archive = zip_file("example.com/a@v1.0.0/a.go"); + let mirror_bytes = archive.clone(); + let (origin, done, server) = loopback_go_proxy(move |origin, path| match path { + "/example.com/a/@v/v1.0.0.mod" => (200, None, b"module example.com/a\n".to_vec()), + "/example.com/a/@v/v1.0.0.zip" => ( + 302, + Some(format!( + "{origin}/proxy-golang-org-prod/module.zip?X-Goog-Signature=fixture" + )), + Vec::new(), + ), + "/proxy-golang-org-prod/module.zip?X-Goog-Signature=fixture" => { + (200, None, mirror_bytes.clone()) + } + _ => panic!("unexpected Go proxy request {path}"), + }); + let registry = loopback_go_registry(&origin); + let mut checksums = LocalRegistry(BTreeMap::from([ + ( + (Ecosystem::Go, "/example.com/a/@v/v1.0.0.mod".into()), + b"module example.com/a\n".to_vec(), + ), + ( + (Ecosystem::Go, "/example.com/a/@v/v1.0.0.zip".into()), + archive.clone(), + ), + ])); + sumdb_tests::seal(&mut checksums); + let transport = sumdb_tests::Transport { + registry: ®istry, + checksums: &checksums, + }; + let cache = FixtureArtifactStore::default(); + let resolver = RegistryResolver { + build: None, + transport: &transport, + store: &cache, + runtimes: [(Ecosystem::Go, registry_runtime(Ecosystem::Go))].into(), + go_sumdb: dever_cli::libs::sumdb::ChecksumDatabase::new(sumdb_tests::verifier(), None) + .unwrap(), + }; + let lock = dever_cli::libs::LibResolver::resolve( + &resolver, + &["go:example.com/a@1.0.0".parse().unwrap()], + ) + .unwrap(); + assert_eq!(lock.libs[0].artifacts[0].sha256, digest(&archive)); + assert_eq!( + cache.verify(&digest(&archive), "linux-x86_64").unwrap(), + archive + ); + done.send(()).unwrap(); + assert_eq!( + server.join().unwrap(), + [ + "/example.com/a/@v/v1.0.0.mod", + "/example.com/a/@v/v1.0.0.zip", + "/proxy-golang-org-prod/module.zip?X-Goog-Signature=fixture" + ] + ); + + let (origin, done, server) = loopback_go_proxy(|origin, path| match path { + "/example.com/a/@v/v1.0.0.mod" => (200, None, b"module example.com/a\n".to_vec()), + "/example.com/a/@v/v1.0.0.zip" => ( + 302, + Some(format!("{origin}/proxy-golang-org-prod/module.zip")), + Vec::new(), + ), + "/proxy-golang-org-prod/module.zip" => (200, None, zip_file("wrong.example/a@v1.0.0/a.go")), + _ => panic!("unexpected Go proxy request {path}"), + }); + let registry = loopback_go_registry(&origin); + let transport = sumdb_tests::Transport { + registry: ®istry, + checksums: &checksums, + }; + let resolver = RegistryResolver { + build: None, + transport: &transport, + store: &cache, + runtimes: [(Ecosystem::Go, registry_runtime(Ecosystem::Go))].into(), + go_sumdb: dever_cli::libs::sumdb::ChecksumDatabase::new(sumdb_tests::verifier(), None) + .unwrap(), + }; + assert!( + dever_cli::libs::LibResolver::resolve( + &resolver, + &["go:example.com/a@1.0.0".parse().unwrap()] + ) + .unwrap_err() + .contains("mismatched module root") + ); + done.send(()).unwrap(); + assert_eq!(server.join().unwrap().len(), 3); +} + +#[test] +fn go_archive_redirect_rejects_untrusted_locations_and_loops_before_request() { + for location in [ + "https://evil.example/proxy-golang-org-prod/archive.zip", + "/proxy-golang-org-prod/archive.zip", + "http://127.0.0.1:1/proxy-golang-org-prod/archive.zip", + "http://user@127.0.0.1/proxy-golang-org-prod/archive.zip", + "http://127.0.0.1/proxy-golang-org-prod/archive.zip#fragment", + ] { + let location = location.to_owned(); + let (origin, done, server) = + loopback_go_proxy(move |_, _| (302, Some(location.clone()), Vec::new())); + let registry = loopback_go_registry(&origin); + assert!( + registry + .get(Ecosystem::Go, "/example.com/a/@v/v1.0.0.zip", 1024) + .is_err() + ); + done.send(()).unwrap(); + assert_eq!(server.join().unwrap().len(), 1); + } + + let (origin, done, server) = loopback_go_proxy(|origin, _| { + let host = origin.strip_prefix("http://").unwrap(); + ( + 302, + Some(format!("http://@{host}/proxy-golang-org-prod/archive.zip")), + Vec::new(), + ) + }); + let registry = loopback_go_registry(&origin); + let error = registry + .get(Ecosystem::Go, "/example.com/a/@v/v1.0.0.zip", 1024) + .unwrap_err(); + assert!(error.contains("trusted mirror bucket"), "{error}"); + done.send(()).unwrap(); + assert_eq!(server.join().unwrap().len(), 1); + + let (origin, done, server) = loopback_go_proxy(|_, _| (302, None, Vec::new())); + let registry = loopback_go_registry(&origin); + assert!( + registry + .get(Ecosystem::Go, "/example.com/a/@v/v1.0.0.zip", 1024) + .unwrap_err() + .contains("lacks Location") + ); + done.send(()).unwrap(); + assert_eq!(server.join().unwrap().len(), 1); + + let (origin, done, server) = loopback_go_proxy(|origin, _| { + ( + 302, + Some(format!("{origin}/proxy-golang-org-prod/archive.zip")), + Vec::new(), + ) + }); + let registry = loopback_go_registry(&origin); + assert!( + registry + .get(Ecosystem::Go, "/example.com/a/@v/v1.0.0.zip", 1024) + .unwrap_err() + .contains("loop") + ); + done.send(()).unwrap(); + assert_eq!(server.join().unwrap().len(), 2); + + let (origin, done, server) = loopback_go_proxy(|origin, path| { + let target = match path { + "/example.com/a/@v/v1.0.0.zip" => "first.zip", + "/proxy-golang-org-prod/first.zip" => "second.zip", + "/proxy-golang-org-prod/second.zip" => "third.zip", + _ => panic!("unexpected redirect request {path}"), + }; + ( + 302, + Some(format!("{origin}/proxy-golang-org-prod/{target}")), + Vec::new(), + ) + }); + let registry = loopback_go_registry(&origin); + assert!( + registry + .get(Ecosystem::Go, "/example.com/a/@v/v1.0.0.zip", 1024) + .unwrap_err() + .contains("redirect is not allowed") + ); + done.send(()).unwrap(); + assert_eq!(server.join().unwrap().len(), 3); + + let (origin, done, server) = loopback_go_proxy(|origin, path| match path { + "/example.com/a/@v/v1.0.0.zip" => ( + 302, + Some(format!( + "{origin}/proxy-golang-org-prod/module.zip?X-Goog-Signature=private" + )), + Vec::new(), + ), + "/proxy-golang-org-prod/module.zip?X-Goog-Signature=private" => (503, None, Vec::new()), + _ => panic!("unexpected Go proxy request {path}"), + }); + let registry = loopback_go_registry(&origin); + let error = registry + .get(Ecosystem::Go, "/example.com/a/@v/v1.0.0.zip", 1024) + .unwrap_err(); + assert!(!error.contains("X-Goog-Signature")); + assert!(!error.contains("private")); + done.send(()).unwrap(); + assert_eq!(server.join().unwrap().len(), 2); +} + +#[test] +fn only_go_zip_endpoints_can_follow_the_fixed_mirror() { + for (ecosystem, path) in [ + (Ecosystem::Go, "/example.com/a/@v/v1.0.0.mod"), + (Ecosystem::Pip, "/pypi/a/json"), + (Ecosystem::Npm, "/a"), + ] { + let (origin, done, server) = loopback_go_proxy(|origin, _| { + ( + 302, + Some(format!("{origin}/proxy-golang-org-prod/archive.zip")), + Vec::new(), + ) + }); + let registry = dever_cli::libs::HttpRegistry::new( + [ + (Ecosystem::Go, origin.clone()), + (Ecosystem::Pip, origin.clone()), + (Ecosystem::Npm, origin.clone()), + ], + [ + (Ecosystem::Go, vec![origin.clone()]), + (Ecosystem::Pip, vec![origin.clone()]), + (Ecosystem::Npm, vec![origin.clone()]), + ], + ) + .with_go_archive_mirror(&format!("{origin}/proxy-golang-org-prod/")) + .unwrap(); + assert!( + registry + .get(ecosystem, path, 1024) + .unwrap_err() + .contains("redirect is not allowed") + ); + done.send(()).unwrap(); + assert_eq!(server.join().unwrap().len(), 1); + } +} + +#[test] +fn fixture_lock_is_offline_deterministic_and_cache_is_content_addressed() { + let first: LibSpec = "pip:fixture@1.0.0".parse().unwrap(); + let second: LibSpec = "npm:fixture@1.0.0".parse().unwrap(); + let requested = [first, second]; + let lock = FixtureRegistry::builtin().resolve(&requested).unwrap(); + let bytes = lock.encode().unwrap(); + assert_eq!(LockFile::decode(&bytes).unwrap().encode().unwrap(), bytes); + doctor(&lock).unwrap(); + + let cache = FixtureArtifactStore::default(); + let digest = cache.publish(b"sdk fixture bytes", "linux-x86_64").unwrap(); + assert!(cache.has(&digest, "linux-x86_64").unwrap()); + assert!(!cache.has(&digest, "windows-x86_64").unwrap()); + assert_eq!( + cache.verify(&digest, "linux-x86_64").unwrap(), + b"sdk fixture bytes" + ); + assert!(cache.verify(&digest, "windows-x86_64").is_err()); + + let sdk = SdkFixture { + ecosystem: Ecosystem::Pip, + schema: "fixture-schema-pip-v1".into(), + }; + sdk.handshake("fixture-schema-pip-v1").unwrap(); + assert!(sdk.handshake("fixture-schema-pip-v2").is_err()); + + let root = project(); + fs::write(root.join("dever.lock"), bytes).unwrap(); + let decoded = LockFile::decode(&fs::read(root.join("dever.lock")).unwrap()).unwrap(); + assert_eq!(decoded.libs.len(), 2); + let _ = fs::remove_dir_all(root); +} + +#[test] +fn scoped_npm_names_are_checked() { + assert!("npm:@dever/sdk@1.0.0".parse::<LibSpec>().is_ok()); + assert!("npm:@dever bad/sdk@1.0.0".parse::<LibSpec>().is_err()); + assert!("npm:@dever/SDK@1.0.0".parse::<LibSpec>().is_ok()); + assert!("npm:@dever/..@1.0.0".parse::<LibSpec>().is_err()); + assert!("go:example.com/../module@1.0.0".parse::<LibSpec>().is_err()); +} + +#[test] +fn project_preparation_requires_a_canonical_lock_and_reports_sizes() { + let root = project(); + fs::create_dir_all(root.join("config")).unwrap(); + fs::write( + root.join("config/setting.json"), + r#"{"lib":["pip:fixture@1.0.0"]}"#, + ) + .unwrap(); + assert!(prepare(&root, &[]).is_err()); + let lock = FixtureRegistry::builtin() + .resolve(&["pip:fixture@1.0.0".parse().unwrap()]) + .unwrap(); + lock.write_atomic(&root).unwrap(); + let report = prepare(&root, &[]).unwrap().unwrap(); + assert_eq!(report.libs, 1); + assert_eq!(report.artifacts, 1); + assert!(report.artifact_bytes > 0); + let _ = fs::remove_dir_all(root); +} + +#[test] +fn lib_commands_preserve_project_settings_in_one_private_file() { + let root = TemporaryDirectory::new(); + fs::create_dir(root.path().join("config")).unwrap(); + let path = root.path().join("config/setting.json"); + fs::write( + &path, + r#"{"log":{"level":"warn"},"http":{"host":"127.0.0.1","port":9080}}"#, + ) + .unwrap(); + assert!(prepare(root.path(), &[]).unwrap().is_none()); + let requested = vec!["pip:fixture@1.0.0".to_owned()]; + dever_cli::libs::execute("add", root.path(), &requested).unwrap(); + let document: serde_json::Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + assert_eq!(document["lib"], serde_json::json!(requested)); + assert_eq!(document["log"]["level"], "warn"); + assert_eq!(document["http"]["port"], 9080); + assert!(!root.path().join("config/lib.json").exists()); + assert_eq!( + dever_runtime::config::Settings::load_project(root.path()) + .unwrap() + .external_libs(), + requested + ); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!( + fs::metadata(&path).unwrap().permissions().mode() & 0o777, + 0o600 + ); + } + dever_cli::libs::execute("remove", root.path(), &requested).unwrap(); + let document: serde_json::Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + assert!(document.get("lib").is_none()); + assert_eq!(document["log"]["level"], "warn"); + assert!(prepare(root.path(), &[]).unwrap().is_none()); +} + +#[test] +fn lib_remove_prunes_the_existing_lock_without_resolving_again() { + let root = TemporaryDirectory::new(); + let requested = [ + "pip:fixture@1.0.0".to_owned(), + "npm:fixture@1.0.0".to_owned(), + ]; + dever_cli::libs::execute("add", root.path(), &requested).unwrap(); + dever_cli::libs::execute("remove", root.path(), &requested[..1]).unwrap(); + let lock = LockFile::decode(&fs::read(root.path().join("dever.lock")).unwrap()).unwrap(); + assert_eq!( + lock.libs + .iter() + .map(|lib| lib.spec.key()) + .collect::<Vec<_>>(), + vec![requested[1].clone()] + ); + assert_eq!( + dever_cli::libs::execute("list", root.path(), &[]).unwrap(), + requested[1] + ); +} + +#[test] +fn unmanaged_cli_cannot_resolve_real_packages_or_change_settings() { + let root = TemporaryDirectory::new(); + let requested = vec!["pip:requests@2.31.0".to_owned()]; + let error = dever_cli::libs::execute("add", root.path(), &requested).unwrap_err(); + assert!(error.contains("signed managed Dever release")); + assert!(!root.path().join("dever.lock").exists()); + assert!(!root.path().join("config/setting.json").exists()); +} + +#[test] +fn malformed_or_stale_lib_configuration_is_never_ignored() { + let root = TemporaryDirectory::new(); + fs::create_dir(root.path().join("config")).unwrap(); + for invalid in [ + r#"{"lib":null}"#, + r#"{"lib":[1]}"#, + r#"{"lib":["pip:fixture@latest"]}"#, + r#"{"lib":[],"lib":[]}"#, + ] { + fs::write(root.path().join("config/setting.json"), invalid).unwrap(); + assert!(prepare(root.path(), &[]).is_err(), "accepted {invalid}"); + } + fs::write(root.path().join("config/setting.json"), "{}").unwrap(); + fs::write(root.path().join("config/lib.json"), "{}").unwrap(); + assert!( + prepare(root.path(), &[]) + .unwrap_err() + .contains("not supported") + ); +} + +#[cfg(unix)] +#[test] +fn lib_commands_do_not_follow_configuration_symlinks() { + let root = TemporaryDirectory::new(); + let outside = TemporaryDirectory::new(); + fs::write(outside.path().join("setting.json"), "{}").unwrap(); + std::os::unix::fs::symlink(outside.path(), root.path().join("config")).unwrap(); + let error = + dever_cli::libs::execute("add", root.path(), &["pip:fixture@1.0.0".into()]).unwrap_err(); + assert!(error.contains("symbolic link")); + assert_eq!( + fs::read_to_string(outside.path().join("setting.json")).unwrap(), + "{}" + ); + assert!(!root.path().join("dever.lock").exists()); +} + +#[test] +fn lock_binds_the_checked_worker_contract_and_rejects_schema_drift() { + let root = TemporaryDirectory::new(); + let domain = root.path().join("module/notification/mail"); + fs::create_dir_all(&domain).unwrap(); + fs::write( + domain.join("port.dever"), + "read() (value: Int) fails app.ReadResult\n", + ) + .unwrap(); + fs::write(domain.join("app.dever"), "type ReadResult { error Unavailable(message: Text) }\nread() (value: Int) { value = port.read() }\n").unwrap(); + let adapter = + "setting { endpoint: Text }\nexternal exec \"worker\" { lib \"pip:fixture@1.0.0\" }\n"; + fs::write(domain.join("adapter.dever"), adapter).unwrap(); + fn checked(root: &std::path::Path) -> dever_core::hir::Program { + let sources = dever_core::source::SourceMap::load(&root.join("module")).unwrap(); + dever_core::check(&sources).unwrap_or_else(|errors| panic!("{errors:?}")) + } + let program = checked(root.path()); + assert!( + dever_cli::libs::prepare_program( + root.path(), + &program, + dever_cli::toolchain::BuildTarget::host().unwrap() + ) + .is_err() + ); + dever_cli::libs::execute("add", root.path(), &[]).unwrap(); + let lock = LockFile::decode(&fs::read(root.path().join("dever.lock")).unwrap()).unwrap(); + assert_eq!(lock.workers.len(), 1); + assert_eq!( + lock.workers[0].schema, + program.external_worker_contracts()[0].schema + ); + assert_ne!(lock.workers[0].schema, lock.libs[0].schema); + assert!( + dever_cli::libs::prepare_program( + root.path(), + &program, + dever_cli::toolchain::BuildTarget::host().unwrap() + ) + .is_ok() + ); + fs::write( + domain.join("adapter.dever"), + adapter.replace("endpoint: Text", "endpoint: Int"), + ) + .unwrap(); + let changed = checked(root.path()); + assert!( + dever_cli::libs::prepare_program( + root.path(), + &changed, + dever_cli::toolchain::BuildTarget::host().unwrap() + ) + .unwrap_err() + .contains("binding changed") + ); + assert!(dever_cli::libs::execute("doctor", root.path(), &[]).is_err()); + dever_cli::libs::execute("update", root.path(), &[]).unwrap(); + assert!( + dever_cli::libs::prepare_program( + root.path(), + &changed, + dever_cli::toolchain::BuildTarget::host().unwrap() + ) + .is_ok() + ); + assert!( + dever_cli::libs::execute("remove", root.path(), &["pip:fixture@1.0.0".into()]) + .unwrap_err() + .contains("declared by an external Adapter") + ); +} + +#[test] +fn independent_worker_environments_can_lock_different_library_versions() { + use dever_cli::libs::{ + FixtureArtifact, FixturePackage, LockedWorker, RuntimePack, resolve_workers, + }; + let first: LibSpec = "pip:library@1.0.0".parse().unwrap(); + let second: LibSpec = "pip:library@2.0.0".parse().unwrap(); + let mut registry = FixtureRegistry::default(); + for spec in [&first, &second] { + registry + .register(FixturePackage { + spec: spec.clone(), + dependencies: Vec::new(), + runtime: RuntimePack { + name: "fixture-python".into(), + version: spec.version.clone(), + sha256: "00".repeat(32), + }, + artifacts: vec![FixtureArtifact { + target: "host".into(), + path: format!("{}/library", spec.version), + bytes: vec![1], + }], + schema: "package-metadata".into(), + }) + .unwrap(); + } + let worker = |adapter: &str, libs| LockedWorker { + port: "notification.mail".into(), + adapter: adapter.into(), + ecosystem: "exec".into(), + runtime: None, + entry: format!("worker/{adapter}"), + schema: "checked-port".into(), + operations: vec!["read".into()], + capabilities: Vec::new(), + libs, + }; + let lock = resolve_workers( + ®istry, + &[], + vec![ + worker("first", vec![first.clone()]), + worker("second", vec![second.clone()]), + ], + ) + .unwrap(); + assert_eq!(lock.libs.len(), 2); + assert_eq!(lock.workers.len(), 2); + doctor(&lock).unwrap(); + let error = resolve_workers( + ®istry, + &[], + vec![worker("conflict", vec![first.clone(), second.clone()])], + ) + .unwrap_err(); + assert!(error.contains("version conflict")); + let mut invalid = lock; + invalid.workers[0].libs = vec![first, second]; + assert!( + doctor(&invalid) + .unwrap_err() + .contains("conflicting versions") + ); +} + +#[test] +fn managed_worker_without_libs_still_locks_its_runtime() { + use dever_cli::libs::LockedWorker; + + let mut lock = LockFile::new(Vec::new()).unwrap(); + lock.workers.push(LockedWorker { + port: "notification.mail".into(), + adapter: "python".into(), + ecosystem: "pip".into(), + runtime: Some(registry_runtime(Ecosystem::Pip).pack), + entry: "worker/entry.py".into(), + schema: "checked-port".into(), + operations: vec!["read".into()], + capabilities: Vec::new(), + libs: Vec::new(), + }); + + let encoded = lock.encode().unwrap(); + let decoded = LockFile::decode(&encoded).unwrap(); + doctor(&decoded).unwrap(); + assert_eq!(decoded.workers[0].runtime, lock.workers[0].runtime); + + lock.workers[0].runtime = None; + assert!(lock.encode().unwrap_err().contains("runtime pack identity")); +} + +#[test] +fn expanded_worker_resources_keep_archives_needed_by_exec_or_declarations() { + use dever_cli::libs::{ + LockedArtifact, LockedDependency, LockedLib, LockedWorker, prune_expanded_resources, + }; + use dever_core::native::EmbeddedResource; + + let shared: LibSpec = "pip:shared@1.0.0".parse().unwrap(); + let dependency: LibSpec = "pip:dependency@1.0.0".parse().unwrap(); + let managed_only: LibSpec = "pip:managed-only@1.0.0".parse().unwrap(); + let unrelated: LibSpec = "npm:unrelated@1.0.0".parse().unwrap(); + let target = dever_cli::toolchain::platform_identity(); + let path = + |spec: &LibSpec| format!("lib/{}/{}/archive.whl", spec.ecosystem.as_str(), spec.name); + let locked = |spec: &LibSpec, dependencies: Vec<LibSpec>| LockedLib { + build: None, + spec: spec.clone(), + dependencies: dependencies + .into_iter() + .map(|spec| LockedDependency { spec }) + .collect(), + runtime: registry_runtime(spec.ecosystem.clone()).pack, + artifacts: vec![LockedArtifact { + target: target.clone(), + path: path(spec), + bytes: 1, + sha256: digest(b"x"), + }], + schema: "checked".into(), + }; + let mut lock = LockFile::new(vec![ + locked(&shared, vec![dependency.clone()]), + locked(&dependency, vec![]), + locked(&managed_only, vec![]), + locked(&unrelated, vec![]), + ]) + .unwrap(); + lock.npm + .push(dever_cli::libs::npm::Environment::single(unrelated.clone())); + let worker = |ecosystem: &str, adapter: &str, libs: Vec<LibSpec>| LockedWorker { + port: "notification.mail".into(), + adapter: adapter.into(), + ecosystem: ecosystem.into(), + runtime: (ecosystem != "exec").then(|| registry_runtime(ecosystem.parse().unwrap()).pack), + entry: format!("worker/{adapter}"), + schema: "checked".into(), + operations: vec!["send".into()], + capabilities: vec![], + libs, + }; + let managed = worker("pip", "managed", vec![shared.clone(), managed_only.clone()]); + let exec = worker("exec", "exec", vec![shared.clone()]); + let npm = worker("npm", "empty", vec![]); + let runtime_path = |ecosystem: &str| format!("lib/runtime/{ecosystem}/{target}/runtime.pack"); + let paths = [ + path(&shared), + path(&dependency), + path(&managed_only), + path(&unrelated), + runtime_path("pip"), + runtime_path("npm"), + "other/resource".into(), + ]; + let resources = || { + paths + .iter() + .map(|path| EmbeddedResource { + path: path.clone(), + bytes: vec![1], + sha256: digest(&[1]), + executable: false, + }) + .collect::<Vec<_>>() + }; + + let mut with_exec = resources(); + prune_expanded_resources( + &lock, + &[managed.clone(), exec, npm.clone()], + &[], + &mut with_exec, + ) + .unwrap(); + let kept = with_exec + .iter() + .map(|resource| resource.path.as_str()) + .collect::<std::collections::BTreeSet<_>>(); + assert!(kept.contains(path(&shared).as_str())); + assert!(kept.contains(path(&dependency).as_str())); + assert!(kept.contains(runtime_path("pip").as_str())); + assert!(!kept.contains(path(&managed_only).as_str())); + assert!(!kept.contains(runtime_path("npm").as_str())); + assert!(kept.contains(path(&unrelated).as_str())); + assert!(kept.contains("other/resource")); + + let mut managed_only_resources = resources(); + prune_expanded_resources( + &lock, + &[managed.clone(), npm.clone()], + std::slice::from_ref(&unrelated), + &mut managed_only_resources, + ) + .unwrap(); + let kept = managed_only_resources + .iter() + .map(|resource| resource.path.as_str()) + .collect::<std::collections::BTreeSet<_>>(); + assert!(!kept.contains(path(&shared).as_str())); + assert!(!kept.contains(path(&dependency).as_str())); + assert!(!kept.contains(runtime_path("pip").as_str())); + assert!(kept.contains(path(&unrelated).as_str())); + + let mut declared_resources = resources(); + prune_expanded_resources( + &lock, + &[managed, npm], + std::slice::from_ref(&managed_only), + &mut declared_resources, + ) + .unwrap(); + let kept = declared_resources + .iter() + .map(|resource| resource.path.as_str()) + .collect::<std::collections::BTreeSet<_>>(); + assert!(kept.contains(path(&managed_only).as_str())); + assert!(kept.contains(runtime_path("pip").as_str())); + assert!(!kept.contains(path(&shared).as_str())); +} + +#[test] +fn embedding_rejects_extra_artifacts_and_drifted_provider_metadata() { + let root = TemporaryDirectory::new(); + let original = FixtureRegistry::builtin() + .resolve(&[ + "pip:fixture@1.0.0".parse().unwrap(), + "npm:fixture@1.0.0".parse().unwrap(), + ]) + .unwrap(); + original.write_atomic(root.path()).unwrap(); + assert_eq!( + dever_cli::libs::embedded_resources(root.path()) + .unwrap() + .len(), + 2 + ); + for field in ["runtime", "schema", "artifact", "dependency"] { + let mut changed = original.clone(); + match field { + "runtime" => changed.libs[0].runtime.sha256 = "11".repeat(32), + "schema" => changed.libs[0].schema = "other-schema".into(), + "artifact" => { + let mut extra = changed.libs[0].artifacts[0].clone(); + extra.path = "extra/resource".into(); + changed.libs[0].artifacts.push(extra); + } + "dependency" => { + let dependency = changed.libs[1].spec.clone(); + changed.libs[0] + .dependencies + .push(dever_cli::libs::LockedDependency { spec: dependency }); + } + _ => unreachable!(), + } + changed.write_atomic(root.path()).unwrap(); + assert!( + dever_cli::libs::embedded_resources(root.path()) + .unwrap_err() + .contains("metadata"), + "accepted {field} drift" + ); + } +} + +#[test] +fn provider_boundary_requires_a_signed_runtime_without_path_fallback() { + let spec: LibSpec = "pip:requests@2.31.0".parse().unwrap(); + let error = provider_manifest(Ecosystem::Pip).blocked_message(&spec); + assert!(error.contains("managed-cpython")); + assert!(error.contains("requires a signed")); + assert!(!error.contains("install the managed")); + assert!(error.contains("never use PATH")); + let registry = FixtureRegistry::builtin(); + assert!(registry.resolve(std::slice::from_ref(&spec)).is_err()); +} + +#[test] +fn locked_package_dependencies_and_artifacts_are_verified_before_worker_use() { + let root = project(); + let root_spec: LibSpec = "pip:fixture@1.0.0".parse().unwrap(); + let lock = FixtureRegistry::builtin() + .resolve(std::slice::from_ref(&root_spec)) + .unwrap(); + let cache = FixtureArtifactStore::default(); + let bytes = b"dever fixture pip:fixture@1.0.0\n"; + let digest = cache.publish(bytes, "host").unwrap(); + assert_eq!(digest, lock.libs[0].artifacts[0].sha256); + let verified = verify_locked_artifacts(&lock, &cache, "linux-x86_64").unwrap(); + assert_eq!(verified.len(), 1); + assert_eq!(verified[0].lib, root_spec); + assert_eq!(verified[0].bytes, bytes); + let _ = fs::remove_dir_all(root); +} + +#[test] +fn doctor_rejects_missing_and_cyclic_transitive_dependencies() { + let mut registry = FixtureRegistry::default(); + let a: LibSpec = "pip:a@1.0.0".parse().unwrap(); + let b: LibSpec = "pip:b@1.0.0".parse().unwrap(); + let runtime = dever_cli::libs::RuntimePack { + name: "fixture".into(), + version: "1.0.0".into(), + sha256: "00".repeat(32), + }; + let artifact = || dever_cli::libs::FixtureArtifact { + target: "host".into(), + path: "worker.bin".into(), + bytes: vec![1], + }; + registry + .register(dever_cli::libs::FixturePackage { + spec: a.clone(), + dependencies: vec![b.clone()], + runtime: runtime.clone(), + artifacts: vec![artifact()], + schema: "a".into(), + }) + .unwrap(); + registry + .register(dever_cli::libs::FixturePackage { + spec: b.clone(), + dependencies: vec![a.clone()], + runtime, + artifacts: vec![artifact()], + schema: "b".into(), + }) + .unwrap(); + let lock = registry.resolve(&[a]).unwrap(); + assert!( + doctor(&lock) + .unwrap_err() + .contains("cyclic locked lib dependency") + ); +} diff --git a/test/dever-cli-tests/external_libs/build.rs b/test/dever-cli-tests/external_libs/build.rs new file mode 100644 index 0000000..4916d4c --- /dev/null +++ b/test/dever-cli-tests/external_libs/build.rs @@ -0,0 +1,468 @@ +use super::*; +use dever_cli::libs::build::{Inputs, Session, pack}; +use serde::{Deserialize, Serialize}; + +#[path = "../../dever-tests/tests/support/sandbox.rs"] +pub(super) mod sandbox; + +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct Settings { + runtime: RegistryRuntime, + runtime_file: String, + build: pack::Descriptor, + build_file: String, +} + +struct AuthorInputs { + root: PathBuf, + settings: Settings, +} + +#[test] +fn signed_build_pack_binds_runtime_layout_tools_and_archive_digest() { + let runtime = registry_runtime(Ecosystem::Pip); + let mut descriptor = pack::Descriptor { + format: "dever-registry-build-v1".into(), + ecosystem: Ecosystem::Pip, + target: runtime.target.clone(), + runtime_sha256: runtime.pack.sha256.clone(), + python_runtime_sha256: None, + sha256: "0".repeat(64), + }; + descriptor.validate(&runtime, None).unwrap(); + let manifest = pack::Manifest { + format: "dever-lib-build-v1".into(), + ecosystem: Ecosystem::Pip, + target: runtime.target.clone(), + runtime_sha256: runtime.pack.sha256.clone(), + python_runtime_sha256: None, + executables: ["sh", "cc", "c++", "as", "ld", "ar", "ranlib", "make"] + .into_iter() + .map(|name| format!("rootfs/usr/bin/{name}")) + .collect(), + }; + let mut files = manifest + .executables + .iter() + .map(|path| (path.clone(), b"#!/bin/sh\nexit 0\n".to_vec())) + .collect::<Vec<_>>(); + files.push(( + "dever-build.json".into(), + serde_json::to_vec(&manifest).unwrap(), + )); + manifest.validate(&files, &descriptor).unwrap(); + let mut changed = descriptor.clone(); + changed.runtime_sha256 = "f".repeat(64); + assert!(changed.validate(&runtime, None).is_err()); + assert!(manifest.validate(&files, &changed).is_err()); + let mut changed = files.clone(); + changed.push(("rootfs/etc/ld.so.preload".into(), b"injected".to_vec())); + assert!(manifest.validate(&changed, &descriptor).is_err()); + let mut changed = manifest.clone(); + changed.executables.push("rootfs/usr/bin/absent".into()); + assert!(changed.validate(&files, &descriptor).is_err()); + let mut archive = tar::Builder::new(flate2::write::GzEncoder::new( + Vec::new(), + flate2::Compression::default(), + )); + for (path, bytes) in files { + let mut header = tar::Header::new_gnu(); + header.set_size(bytes.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + archive + .append_data(&mut header, path, bytes.as_slice()) + .unwrap(); + } + let bytes = archive.into_inner().unwrap().finish().unwrap(); + descriptor.sha256 = digest(&bytes); + pack::unpack(&bytes, &descriptor).unwrap(); + let mut changed = bytes; + changed[0] ^= 1; + assert!(pack::unpack(&changed, &descriptor).is_err()); +} + +impl Inputs for AuthorInputs { + fn diagnostic(&self, output: &[u8]) { + eprintln!("{}", String::from_utf8_lossy(output)); + } + + fn runtime(&self, ecosystem: Ecosystem) -> Result<(RegistryRuntime, Vec<u8>), String> { + if ecosystem != Ecosystem::Pip { + return Err("fixture only supplies Python".into()); + } + let bytes = fs::read(self.root.join(&self.settings.runtime_file)) + .map_err(|error| error.to_string())?; + if digest(&bytes) != self.settings.runtime.pack.sha256 { + return Err("author runtime drift".into()); + } + Ok((self.settings.runtime.clone(), bytes)) + } + fn tools(&self, ecosystem: Ecosystem) -> Result<(pack::Descriptor, Vec<u8>), String> { + if ecosystem != Ecosystem::Pip { + return Err("fixture only supplies Python".into()); + } + Ok(( + self.settings.build.clone(), + fs::read(self.root.join(&self.settings.build_file)) + .map_err(|error| error.to_string())?, + )) + } + fn assets(&self) -> Result<Vec<(String, Vec<u8>)>, String> { + let workspace = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../.."); + Ok(sandbox::assets(&workspace) + .into_iter() + .map(|asset| { + let _mode = asset.executable; + ( + asset.path.trim_start_matches("sandbox/").into(), + asset.bytes, + ) + }) + .collect()) + } +} + +#[test] +#[ignore = "explicit author operation: assembles previously hashed native-build-inputs; no network or host execution"] +fn prepare_source_build_tool_fixture() { + let workspace = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../.."); + let runtime: RegistryRuntime = serde_json::from_value( + serde_json::from_slice::<serde_json::Value>( + &fs::read(workspace.join("target/python-managed/config/setting.json")).unwrap(), + ) + .unwrap()["runtime"] + .clone(), + ) + .unwrap(); + let bytes = fs::read(workspace.join("target/python-managed/runtime.pack")).unwrap(); + assert_eq!(digest(&bytes), runtime.pack.sha256); + drop(bytes); + let root = workspace.join("target/source-build"); + let descriptor = prepare_tool_pack(Ecosystem::Pip, &runtime, None, &root); + fs::write( + root.join("config/setting.json"), + serde_json::to_vec_pretty(&Settings { + runtime, + runtime_file: "../python-managed/runtime.pack".into(), + build: descriptor, + build_file: "build.pack".into(), + }) + .unwrap(), + ) + .unwrap(); +} + +pub(super) fn prepare_tool_pack( + ecosystem: Ecosystem, + runtime: &RegistryRuntime, + python: Option<&RegistryRuntime>, + root: &std::path::Path, +) -> pack::Descriptor { + let workspace = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../.."); + let inputs = workspace.join("target/native-build-inputs"); + let document: serde_json::Value = + serde_json::from_slice(&fs::read(inputs.join("inputs.json")).unwrap()).unwrap(); + let mut descriptor = pack::Descriptor { + format: "dever-registry-build-v1".into(), + ecosystem: ecosystem.clone(), + target: runtime.target.clone(), + runtime_sha256: runtime.pack.sha256.clone(), + python_runtime_sha256: python.map(|runtime| runtime.pack.sha256.clone()), + sha256: "0".repeat(64), + }; + descriptor.validate(runtime, python).unwrap(); + let manifest = pack::Manifest { + format: "dever-lib-build-v1".into(), + ecosystem: ecosystem.clone(), + target: runtime.target.clone(), + runtime_sha256: runtime.pack.sha256.clone(), + python_runtime_sha256: descriptor.python_runtime_sha256.clone(), + executables: serde_json::from_value(document[ecosystem.as_str()]["executables"].clone()) + .unwrap(), + }; + let mut files: Vec<(String, Vec<u8>)> = vec![( + "dever-build.json".into(), + serde_json::to_vec(&manifest).unwrap(), + )]; + for input in document[ecosystem.as_str()]["files"].as_array().unwrap() { + let source = inputs.join(input["source"].as_str().unwrap()); + assert!(fs::symlink_metadata(&source).unwrap().is_file()); + let bytes = fs::read(&source).unwrap(); + assert_eq!(digest(&bytes), input["sha256"].as_str().unwrap()); + files.push((input["path"].as_str().unwrap().into(), bytes)); + } + files.sort_by(|left, right| left.0.cmp(&right.0)); + manifest.validate(&files, &descriptor).unwrap(); + let assets = sandbox::assets(&workspace) + .into_iter() + .map(|asset| { + ( + asset.path.trim_start_matches("sandbox/").into(), + asset.bytes, + ) + }) + .collect::<Vec<_>>(); + pack::validate_tools(&manifest, &files, &assets).unwrap(); + fs::create_dir_all(root.join("config")).unwrap(); + let encoder = flate2::GzBuilder::new() + .mtime(0) + .operating_system(255) + .write( + fs::File::create(root.join("build.pack")).unwrap(), + flate2::Compression::default(), + ); + let mut archive = tar::Builder::new(encoder); + for (path, bytes) in files { + let mut header = tar::Header::new_gnu(); + header.set_size(bytes.len() as u64); + header.set_mode(if manifest.executables.contains(&path) { + 0o755 + } else { + 0o644 + }); + header.set_uid(0); + header.set_gid(0); + header.set_mtime(0); + header.set_cksum(); + archive + .append_data(&mut header, path, bytes.as_slice()) + .unwrap(); + } + archive.into_inner().unwrap().finish().unwrap(); + descriptor.sha256 = digest(&fs::read(root.join("build.pack")).unwrap()); + descriptor +} + +struct SourceOnly(dever_cli::libs::HttpRegistry); + +#[test] +fn incompatible_sdist_candidate_backtracks_before_loading_build_tools() { + let mut registry = LocalRegistry(BTreeMap::new()); + python_release(&mut registry, "app", "1.0.0", &[], &["backend>=1"]); + python_release(&mut registry, "backend", "1.0.0", &[], &[]); + python_release(&mut registry, "backend", "2.0.0", &[], &[]); + let path = (Ecosystem::Pip, "/pypi/backend/2.0.0/json".into()); + let mut release: serde_json::Value = serde_json::from_slice(®istry.0[&path]).unwrap(); + release["urls"] = serde_json::json!([{"packagetype":"sdist","requires_python":">=3.13","filename":"backend-2.0.0.tar.gz","url":"must-not-fetch","digests":{"sha256":"0".repeat(64)}}]); + registry + .0 + .insert(path, serde_json::to_vec(&release).unwrap()); + let lock = python_lock(®istry, &["pip:app@1.0.0"]).unwrap(); + assert!( + lock.libs + .iter() + .any(|lib| lib.spec.key() == "pip:backend@1.0.0") + ); +} + +fn backend_fixture(mutate: bool) -> LocalRegistry { + let mut registry = LocalRegistry(BTreeMap::new()); + python_release(&mut registry, "buildhelper", "1.0.0", &[], &[]); + registry.0.insert( + (Ecosystem::Pip, "/pypi/sample/json".into()), + serde_json::to_vec(&serde_json::json!({"releases":{"1.0.0":[]}})).unwrap(), + ); + let helper = wheel_fixture::pack( + "buildhelper", + "1.0.0", + "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + &wheel_fixture::metadata("buildhelper", "1.0.0", &[], &[]), + &[ + ( + "buildhelper.py", + b"def cli():\n print('isolated-cli')\n".to_vec(), + ), + ( + "buildhelper-1.0.0.dist-info/entry_points.txt", + b"[console_scripts]\nbuild-probe = buildhelper:cli\n".to_vec(), + ), + ], + ); + let key = (Ecosystem::Pip, "/pypi/buildhelper/1.0.0/json".into()); + let mut release: serde_json::Value = serde_json::from_slice(®istry.0[&key]).unwrap(); + let url = release["urls"][0]["url"].as_str().unwrap().to_owned(); + release["urls"][0]["digests"]["sha256"] = serde_json::json!(digest(&helper)); + registry + .0 + .insert(key, serde_json::to_vec(&release).unwrap()); + registry.0.insert((Ecosystem::Pip, url), helper); + let original = wheel_fixture::metadata("sample", "1.0.0", &[], &[]); + let changed = format!("{original}Summary: changed by wheel hook\n"); + let wheel = wheel_fixture::pack( + "sample", + "1.0.0", + "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + if mutate { &changed } else { &original }, + &[], + ); + let backend = format!( + r#"import os, shutil, subprocess +from pathlib import Path +def get_requires_for_build_wheel(config_settings=None): + try: + import buildhelper + return [] + except ImportError: + return ['buildhelper==1.0.0'] +def prepare_metadata_for_build_wheel(directory, config_settings=None): + assert subprocess.check_output(['build-probe'], text=True).strip() == 'isolated-cli' + metadata = Path(directory) / 'sample-1.0.0.dist-info' + metadata.mkdir() + shutil.copy('original-metadata', metadata / 'METADATA') + os.chdir('/') + return metadata.name +def build_wheel(directory, config_settings=None, metadata_directory=None): + assert Path.cwd().name == 'source', 'previous hook leaked its cwd' + if {mutate}: + shutil.copy('changed-metadata', Path(metadata_directory) / 'METADATA') + name = 'sample-1.0.0-py3-none-any.whl' + shutil.copy('result.whl', Path(directory) / name) + return name +"#, + mutate = if mutate { "True" } else { "False" } + ); + let sourcefiles = vec![ + ( + "sample-1.0.0/pyproject.toml", + b"[build-system]\nrequires = []\nbuild-backend = 'backend'\nbackend-path = ['.']\n" + .to_vec(), + ), + ("sample-1.0.0/backend.py", backend.into_bytes()), + ("sample-1.0.0/original-metadata", original.into_bytes()), + ("sample-1.0.0/changed-metadata", changed.into_bytes()), + ("sample-1.0.0/result.whl", wheel), + ]; + let mut tar = tar::Builder::new(flate2::write::GzEncoder::new( + Vec::new(), + flate2::Compression::default(), + )); + for (name, bytes) in sourcefiles { + let mut header = tar::Header::new_gnu(); + header.set_size(bytes.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + tar.append_data(&mut header, name, bytes.as_slice()) + .unwrap(); + } + let source = tar.into_inner().unwrap().finish().unwrap(); + registry.0.insert((Ecosystem::Pip,"/pypi/sample/1.0.0/json".into()),serde_json::to_vec(&serde_json::json!({"info":{},"urls":[{"packagetype":"sdist","filename":"sample-1.0.0.tar.gz","url":"sample-source","digests":{"sha256":digest(&source)}}]})).unwrap()); + registry + .0 + .insert((Ecosystem::Pip, "sample-source".into()), source); + registry +} + +#[test] +#[ignore = "isolated offline PEP517 contracts; requires explicit target/source-build tool/runtime fixture"] +fn pep517_hooks_keep_requirements_cli_and_metadata_boundaries() { + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../target/source-build"); + let settings: Settings = + serde_json::from_slice(&fs::read(root.join("config/setting.json")).unwrap()).unwrap(); + let inputs = AuthorInputs { root, settings }; + for mutate in [false, true] { + let session = Session::new(&inputs); + let transport = backend_fixture(mutate); + let store = FixtureArtifactStore::default(); + let resolver = RegistryResolver { + transport: &transport, + store: &store, + runtimes: BTreeMap::from([(Ecosystem::Pip, inputs.settings.runtime.clone())]), + go_sumdb: dever_cli::libs::sumdb::ChecksumDatabase::official(None).unwrap(), + build: Some(&session), + }; + let result = dever_cli::libs::LibResolver::resolve( + &resolver, + &["pip:sample@1.0.0".parse().unwrap()], + ); + if mutate { + assert!(result.unwrap_err().contains("metadata differs")); + continue; + } + let lock = result.unwrap(); + dever_cli::libs::doctor(&lock).unwrap(); + assert_eq!(lock.builds[0].dynamic_requires, vec!["buildhelper==1.0.0"]); + for mutation in 0..3 { + let mut altered = lock.clone(); + match mutation { + 0 => altered.builds[0].tools.sha256 = "not-a-hash".into(), + 1 => altered.builds[0].dynamic_requires = vec!["buildhelper>=2".into()], + _ => *altered.builds[0].dependencies = LockFile::new(vec![]).unwrap(), + } + altered.libs[0].build = Some(altered.builds[0].identity().unwrap()); + assert!( + dever_cli::libs::doctor(&altered).is_err(), + "accepted contradictory receipt {mutation}" + ); + } + } +} +impl RegistryTransport for SourceOnly { + fn get(&self, ecosystem: Ecosystem, path: &str, limit: usize) -> Result<Vec<u8>, String> { + let bytes = self.0.get(ecosystem, path, limit)?; + // Explicit author acceptance forces this one package's official sdist. + // Backend requirements still use their ordinary authenticated wheels. + if path == "/pypi/simplejson/3.20.1/json" { + let mut document: serde_json::Value = + serde_json::from_slice(&bytes).map_err(|error| error.to_string())?; + document["urls"] + .as_array_mut() + .ok_or("official release lacks files")? + .retain(|file| file["packagetype"] == "sdist"); + serde_json::to_vec(&document).map_err(|error| error.to_string()) + } else { + Ok(bytes) + } + } +} + +#[test] +#[ignore = "explicit official PyPI source build: downloads fixed source/build requirements and executes signed managed build tools"] +fn builds_official_python_source_with_isolated_pep517() { + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../target/source-build"); + let settings: Settings = + serde_json::from_slice(&fs::read(root.join("config/setting.json")).unwrap()).unwrap(); + let inputs = AuthorInputs { + root: root.clone(), + settings, + }; + let session = Session::new(&inputs); + let transport = SourceOnly(dever_cli::libs::HttpRegistry::official()); + let store = FixtureArtifactStore::default(); + let resolver = RegistryResolver { + transport: &transport, + store: &store, + runtimes: BTreeMap::from([(Ecosystem::Pip, inputs.settings.runtime.clone())]), + go_sumdb: dever_cli::libs::sumdb::ChecksumDatabase::official(None).unwrap(), + build: Some(&session), + }; + let requested = vec!["pip:simplejson@3.20.1".parse().unwrap()]; + let lock = dever_cli::libs::LibResolver::resolve(&resolver, &requested).unwrap(); + dever_cli::libs::doctor(&lock).unwrap(); + assert_eq!(lock.builds.len(), 1); + assert!(!lock.builds[0].dependencies.libs.is_empty()); + let artifact = &lock + .libs + .iter() + .find(|lib| lib.spec.name == "simplejson") + .unwrap() + .artifacts[0]; + let wheel = store + .verify_exact(&artifact.sha256, artifact.bytes, &artifact.target) + .unwrap(); + let mut archive = zip::ZipArchive::new(Cursor::new(wheel)).unwrap(); + assert!((0..archive.len()).any(|index| { + let file = archive.by_index(index).unwrap(); + file.name().contains("_speedups.") && file.name().ends_with(".so") + })); + assert_eq!( + lock, + dever_cli::libs::LibResolver::resolve(&resolver, &requested).unwrap(), + "same session must replay one exact built artifact" + ); + publish_fixture_artifacts(&root.join("dependency"), &lock, &store); + let mut altered = lock.clone(); + altered.builds[0].frontend = "0".repeat(64); + assert!(dever_cli::libs::doctor(&altered).is_err()); +} diff --git a/test/dever-cli-tests/external_libs/npm.rs b/test/dever-cli-tests/external_libs/npm.rs new file mode 100644 index 0000000..7535d30 --- /dev/null +++ b/test/dever-cli-tests/external_libs/npm.rs @@ -0,0 +1,691 @@ +use super::*; +use dever_cli::libs::{LibResolver, LockedWorker, npm, resolve_workers}; + +pub(super) fn archive( + name: &str, + version: &str, + mut manifest: serde_json::Value, + extra: &[(&str, Vec<u8>)], +) -> Vec<u8> { + manifest["name"] = name.into(); + manifest["version"] = version.into(); + let encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + let mut writer = tar::Builder::new(encoder); + for (name, bytes) in [ + ( + "package/package.json", + serde_json::to_vec(&manifest).unwrap(), + ), + ( + "package/index.js", + format!("module.exports = '{name}/{version}';\n").into_bytes(), + ), + ] + .into_iter() + .chain(extra.iter().map(|(name, bytes)| (*name, bytes.clone()))) + { + let mut header = tar::Header::new_gnu(); + header.set_size(bytes.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + writer + .append_data(&mut header, name, bytes.as_slice()) + .unwrap(); + } + writer.into_inner().unwrap().finish().unwrap() +} + +pub(super) fn release( + registry: &mut LocalRegistry, + name: &str, + version: &str, + manifest: serde_json::Value, + extra: &[(&str, Vec<u8>)], +) { + let archive = archive(name, version, manifest.clone(), extra); + let index_path = (Ecosystem::Npm, format!("/{name}")); + let mut index = registry + .0 + .get(&index_path) + .map(|bytes| serde_json::from_slice::<serde_json::Value>(bytes).unwrap()) + .unwrap_or(serde_json::json!({"versions":{}})); + index["versions"][version] = serde_json::json!({}); + registry + .0 + .insert(index_path, serde_json::to_vec(&index).unwrap()); + let mut document = manifest; + document["name"] = name.into(); + document["version"] = version.into(); + let url = format!("https://mock/{name}/{version}.tgz"); + document["dist"] = serde_json::json!({"tarball":url,"integrity":format!("sha512-{}", base64::engine::general_purpose::STANDARD.encode(Sha512::digest(&archive)))}); + registry.0.insert( + (Ecosystem::Npm, format!("/{name}/{version}")), + serde_json::to_vec(&document).unwrap(), + ); + registry.0.insert((Ecosystem::Npm, url), archive); +} + +fn resolved(registry: &LocalRegistry, requested: &[&str]) -> (LockFile, FixtureArtifactStore) { + let store = FixtureArtifactStore::default(); + let lock = local_resolver(registry, &store) + .resolve( + &requested + .iter() + .map(|spec| spec.parse().unwrap()) + .collect::<Vec<_>>(), + ) + .unwrap(); + doctor(&lock).unwrap(); + assert_eq!(LockFile::decode(&lock.encode().unwrap()).unwrap(), lock); + (lock, store) +} + +fn files(lock: &LockFile, store: &FixtureArtifactStore, index: usize) -> BTreeMap<String, Vec<u8>> { + let archives = lock + .libs + .iter() + .map(|lib| { + ( + lib.spec.clone(), + store + .verify_exact( + &lib.artifacts[0].sha256, + lib.artifacts[0].bytes, + &lib.artifacts[0].target, + ) + .unwrap(), + ) + }) + .collect::<BTreeMap<_, _>>(); + npm::files(&lock.npm[index], lock, |lib| { + Ok(archives[&lib.spec].as_slice()) + }) + .unwrap() + .into_iter() + .collect() +} + +#[test] +fn nested_versions_and_shared_packages_keep_distinct_peer_contexts() { + let mut registry = LocalRegistry(BTreeMap::new()); + for (name, dependencies) in [ + ("left", serde_json::json!({"plugin":"1.0.0","host":"1.0.0"})), + ( + "right", + serde_json::json!({"plugin":"1.0.0","host":"2.0.0"}), + ), + ] { + release( + &mut registry, + name, + "1.0.0", + serde_json::json!({"dependencies":dependencies}), + &[], + ); + } + release( + &mut registry, + "plugin", + "1.0.0", + serde_json::json!({"peerDependencies":{"host":">=1.0.0 <3.0.0"}}), + &[], + ); + for version in ["1.0.0", "2.0.0"] { + release(&mut registry, "host", version, serde_json::json!({}), &[]); + } + let (lock, store) = resolved(®istry, &["npm:left@1.0.0", "npm:right@1.0.0"]); + assert_eq!(lock.libs.len(), 5); + let files = files(&lock, &store, 0); + for (side, version) in [("left", "1.0.0"), ("right", "2.0.0")] { + let node = lock.npm[0] + .instances + .iter() + .find(|node| node.path == format!("node_modules/{side}/node_modules/plugin")) + .unwrap(); + assert_eq!( + node.edges[0].target.as_deref(), + Some(format!("node_modules/{side}/node_modules/host").as_str()) + ); + assert!( + String::from_utf8( + files[&format!("node_modules/{side}/node_modules/host/index.js")].clone() + ) + .unwrap() + .contains(version) + ); + } +} + +#[test] +fn optional_dependencies_override_regular_edges_and_record_target_or_absence() { + let mut registry = LocalRegistry(BTreeMap::new()); + release( + &mut registry, + "root", + "1.0.0", + serde_json::json!({"dependencies":{"child":"1.0.0"},"optionalDependencies":{"child":"2.0.0","missing":"*","darwin":"*","musl":"*"}}), + &[], + ); + release(&mut registry, "child", "2.0.0", serde_json::json!({}), &[]); + release( + &mut registry, + "darwin", + "1.0.0", + serde_json::json!({"os":["darwin"]}), + &[], + ); + release( + &mut registry, + "musl", + "1.0.0", + serde_json::json!({"os":["linux"],"libc":["musl"]}), + &[], + ); + let (lock, store) = resolved(®istry, &["npm:root@1.0.0"]); + assert_eq!(lock.libs.len(), 2); + let root = lock.npm[0] + .instances + .iter() + .find(|node| node.spec.name == "root") + .unwrap(); + assert_eq!( + root.edges + .iter() + .find(|edge| edge.name == "child") + .unwrap() + .kind, + npm::Kind::Optional + ); + for name in ["darwin", "musl"] { + assert_eq!( + root.edges + .iter() + .find(|edge| edge.name == name) + .unwrap() + .omission, + Some(npm::Omission::Target) + ); + } + assert_eq!( + root.edges + .iter() + .find(|edge| edge.name == "missing") + .unwrap() + .omission, + Some(npm::Omission::Unavailable) + ); + assert!( + !files(&lock, &store, 0) + .keys() + .any(|path| path.contains("darwin") || path.contains("musl")) + ); +} + +#[test] +fn optional_integrity_and_protocol_failures_are_never_omissions() { + let mut registry = LocalRegistry(BTreeMap::new()); + release( + &mut registry, + "root", + "1.0.0", + serde_json::json!({"optionalDependencies":{"child":"*"}}), + &[], + ); + release(&mut registry, "child", "1.0.0", serde_json::json!({}), &[]); + registry.0.insert( + (Ecosystem::Npm, "https://mock/child/1.0.0.tgz".into()), + b"tampered".to_vec(), + ); + let store = FixtureArtifactStore::default(); + assert!( + local_resolver(®istry, &store) + .resolve(&["npm:root@1.0.0".parse().unwrap()]) + .unwrap_err() + .contains("integrity mismatch") + ); + release( + &mut registry, + "child", + "1.0.0", + serde_json::json!({"dependencies":{"other":"not-a-semver"}}), + &[], + ); + assert!( + local_resolver(®istry, &store) + .resolve(&["npm:root@1.0.0".parse().unwrap()]) + .unwrap_err() + .contains("invalid npm range") + ); +} + +#[test] +fn required_peers_auto_install_and_backtrack_but_optional_peers_do_not() { + let mut registry = LocalRegistry(BTreeMap::new()); + release( + &mut registry, + "root", + "1.0.0", + serde_json::json!({"dependencies":{"plugin":"*","host":"1.0.0"}}), + &[], + ); + release( + &mut registry, + "plugin", + "2.0.0", + serde_json::json!({"peerDependencies":{"host":"2.0.0"}}), + &[], + ); + release( + &mut registry, + "plugin", + "1.0.0", + serde_json::json!({"peerDependencies":{"host":"1.0.0","absent":"*"},"peerDependenciesMeta":{"absent":{"optional":true}}}), + &[], + ); + for version in ["1.0.0", "2.0.0"] { + release(&mut registry, "host", version, serde_json::json!({}), &[]); + } + let (lock, _) = resolved(®istry, &["npm:root@1.0.0"]); + let plugin = lock.npm[0] + .instances + .iter() + .find(|node| node.spec.name == "plugin") + .unwrap(); + assert_eq!(plugin.spec.version, "1.0.0"); + assert_eq!( + plugin + .edges + .iter() + .find(|edge| edge.name == "absent") + .unwrap() + .omission, + Some(npm::Omission::OptionalPeerAbsent) + ); + let (lock, _) = resolved(®istry, &["npm:plugin@1.0.0"]); + assert!( + lock.npm[0] + .instances + .iter() + .any(|node| node.path == "node_modules/host") + ); + let store = FixtureArtifactStore::default(); + assert!( + local_resolver(®istry, &store) + .resolve(&[ + "npm:plugin@2.0.0".parse().unwrap(), + "npm:host@1.0.0".parse().unwrap() + ]) + .is_err() + ); +} + +#[test] +fn bundled_transitive_packages_are_bound_to_parent_and_never_downloaded() { + let mut registry = LocalRegistry(BTreeMap::new()); + let extra = vec![ + ( + "package/node_modules/b/package.json", + br#"{"name":"b","version":"1.0.0","dependencies":{"c":"2.0.0"}}"#.to_vec(), + ), + ( + "package/node_modules/b/index.js", + b"module.exports = require('c');".to_vec(), + ), + ( + "package/node_modules/c/package.json", + br#"{"name":"c","version":"2.0.0"}"#.to_vec(), + ), + ( + "package/node_modules/c/index.js", + b"module.exports = 42;".to_vec(), + ), + ]; + release( + &mut registry, + "root", + "1.0.0", + serde_json::json!({"dependencies":{"b":"1.0.0"},"bundleDependencies":["b"]}), + &extra, + ); + let (lock, store) = resolved(®istry, &["npm:root@1.0.0"]); + assert_eq!(lock.libs.len(), 1); + assert_eq!(lock.npm[0].instances.len(), 3); + assert_eq!( + files(&lock, &store, 0)["node_modules/root/node_modules/c/index.js"], + b"module.exports = 42;" + ); + release( + &mut registry, + "root", + "1.0.0", + serde_json::json!({}), + &extra, + ); + assert!( + local_resolver(®istry, &store) + .resolve(&["npm:root@1.0.0".parse().unwrap()]) + .unwrap_err() + .contains("undeclared") + ); +} + +#[test] +fn worker_root_sets_keep_separate_npm_environments_and_detect_lock_tampering() { + let mut registry = LocalRegistry(BTreeMap::new()); + release( + &mut registry, + "plugin", + "1.0.0", + serde_json::json!({"peerDependencies":{"host":"*"}}), + &[], + ); + for version in ["1.0.0", "2.0.0"] { + release(&mut registry, "host", version, serde_json::json!({}), &[]); + } + let store = FixtureArtifactStore::default(); + let worker = |adapter: &str, version: &str| LockedWorker { + port: "sample.worker".into(), + adapter: adapter.into(), + ecosystem: "npm".into(), + runtime: Some(registry_runtime(Ecosystem::Npm).pack), + entry: format!("worker/{adapter}.mjs"), + schema: "checked-port".into(), + capabilities: vec![], + operations: vec!["read".into()], + libs: vec![ + "npm:plugin@1.0.0".parse().unwrap(), + format!("npm:host@{version}").parse().unwrap(), + ], + }; + let lock = resolve_workers( + &local_resolver(®istry, &store), + &[], + vec![worker("first", "1.0.0"), worker("second", "2.0.0")], + ) + .unwrap(); + assert_eq!(lock.npm.len(), 2); + for index in 0..2 { + assert!(files(&lock, &store, index).contains_key("node_modules/plugin/index.js")); + } + let mut changed = lock.clone(); + changed.npm[0].instances[0].path = "../escape".into(); + assert!(doctor(&changed).is_err()); + let mut changed = lock.clone(); + changed.npm[0] + .instances + .iter_mut() + .find(|node| node.spec.name == "plugin") + .unwrap() + .edges[0] + .target = Some("node_modules/missing".into()); + assert!(doctor(&changed).is_err()); + let mut changed = lock; + changed.npm[0] + .instances + .iter_mut() + .find(|node| node.spec.name == "plugin") + .unwrap() + .edges + .clear(); + let archives = changed + .libs + .iter() + .map(|lib| { + ( + lib.spec.clone(), + store + .verify_exact( + &lib.artifacts[0].sha256, + lib.artifacts[0].bytes, + &lib.artifacts[0].target, + ) + .unwrap(), + ) + }) + .collect::<BTreeMap<_, _>>(); + assert!( + npm::files(&changed.npm[0], &changed, |lib| Ok( + archives[&lib.spec].as_slice() + )) + .unwrap_err() + .contains("differs from package.json") + ); +} + +#[test] +fn bundled_instances_cannot_switch_to_another_parents_same_version_package() { + let mut registry = LocalRegistry(BTreeMap::new()); + for name in ["a", "c"] { + release( + &mut registry, + name, + "1.0.0", + serde_json::json!({"dependencies":{"b":"1.0.0"},"bundleDependencies":["b"]}), + &[ + ( + "package/node_modules/b/package.json", + br#"{"name":"b","version":"1.0.0"}"#.to_vec(), + ), + ( + "package/node_modules/b/index.js", + format!("module.exports = '{name}';").into_bytes(), + ), + ], + ); + } + release(&mut registry, "b", "1.0.0", serde_json::json!({}), &[]); + let (original, store) = resolved(®istry, &["npm:a@1.0.0", "npm:c@1.0.0", "npm:b@1.0.0"]); + let archives = original + .libs + .iter() + .map(|lib| { + ( + lib.spec.clone(), + store + .verify_exact( + &lib.artifacts[0].sha256, + lib.artifacts[0].bytes, + &lib.artifacts[0].target, + ) + .unwrap(), + ) + }) + .collect::<BTreeMap<_, _>>(); + for (archive, prefix) in [ + ("npm:c@1.0.0", "package/node_modules/b/"), + ("npm:b@1.0.0", "package/"), + ] { + let mut changed = original.clone(); + changed.npm[0] + .instances + .iter_mut() + .find(|node| node.path == "node_modules/a/node_modules/b") + .unwrap() + .source = npm::Source { + archive: archive.parse().unwrap(), + prefix: prefix.into(), + }; + assert!( + doctor(&changed) + .unwrap_err() + .contains("parent archive installation") + ); + assert!( + npm::files(&changed.npm[0], &changed, |lib| Ok( + archives[&lib.spec].as_slice() + )) + .unwrap_err() + .contains("parent archive installation") + ); + // Removing the locked layout cannot hide the original archive binding. + for node in &mut changed.npm[0].instances { + node.bundled.clear(); + } + assert!( + npm::files(&changed.npm[0], &changed, |lib| Ok( + archives[&lib.spec].as_slice() + )) + .is_err() + ); + } +} + +#[test] +fn production_and_optional_dependencies_override_same_named_peers() { + let mut registry = LocalRegistry(BTreeMap::new()); + release(&mut registry, "host", "2.0.0", serde_json::json!({}), &[]); + release( + &mut registry, + "plugin", + "1.0.0", + serde_json::json!({"dependencies":{"host":"2.0.0"},"peerDependencies":{"host":"1.0.0"}}), + &[], + ); + let (lock, _) = resolved(®istry, &["npm:plugin@1.0.0", "npm:host@2.0.0"]); + let plugin = lock.npm[0] + .instances + .iter() + .find(|node| node.spec.name == "plugin") + .unwrap(); + assert_eq!(plugin.edges.len(), 1); + assert_eq!(plugin.edges[0].kind, npm::Kind::Dependency); + assert_eq!( + plugin.edges[0].target.as_deref(), + Some("node_modules/plugin/node_modules/host") + ); + release( + &mut registry, + "plugin", + "1.0.0", + serde_json::json!({"optionalDependencies":{"absent":"*"},"peerDependencies":{"absent":"1.0.0"}}), + &[], + ); + let (lock, _) = resolved(®istry, &["npm:plugin@1.0.0"]); + assert_eq!(lock.npm[0].instances[0].edges.len(), 1); + assert_eq!(lock.npm[0].instances[0].edges[0].kind, npm::Kind::Optional); + assert_eq!( + lock.npm[0].instances[0].edges[0].omission, + Some(npm::Omission::Unavailable) + ); +} + +#[test] +fn removing_a_root_prunes_its_entire_dependency_cycle() { + for bundled in [false, true] { + let mut registry = LocalRegistry(BTreeMap::new()); + release(&mut registry, "z", "1.0.0", serde_json::json!({}), &[]); + let b = serde_json::json!({"name":"b","version":"1.0.0","dependencies":{"a":"1.0.0"}}); + let mut a = serde_json::json!({"dependencies":{"b":"1.0.0"}}); + let extra = if bundled { + a["bundleDependencies"] = serde_json::json!(["b"]); + vec![( + "package/node_modules/b/package.json", + serde_json::to_vec(&b).unwrap(), + )] + } else { + release(&mut registry, "b", "1.0.0", b, &[]); + vec![] + }; + release(&mut registry, "a", "1.0.0", a, &extra); + let (lock, _) = resolved(®istry, &["npm:a@1.0.0", "npm:z@1.0.0"]); + let root = TemporaryDirectory::new(); + fs::create_dir(root.path().join("config")).unwrap(); + fs::write( + root.path().join("config/setting.json"), + br#"{"lib":["npm:a@1.0.0","npm:z@1.0.0"]}"#, + ) + .unwrap(); + lock.write_atomic(root.path()).unwrap(); + dever_cli::libs::execute("remove", root.path(), &["npm:a@1.0.0".into()]).unwrap(); + let retained = + LockFile::decode(&fs::read(root.path().join("dever.lock")).unwrap()).unwrap(); + assert_eq!(retained.libs.len(), 1); + assert_eq!(retained.libs[0].spec.name, "z"); + assert_eq!(retained.npm[0].instances.len(), 1); + doctor(&retained).unwrap(); + } +} + +#[test] +fn removing_a_dependency_still_required_by_a_remaining_root_is_rejected() { + let mut registry = LocalRegistry(BTreeMap::new()); + release( + &mut registry, + "a", + "1.0.0", + serde_json::json!({"dependencies":{"b":"1.0.0"}}), + &[], + ); + release(&mut registry, "b", "1.0.0", serde_json::json!({}), &[]); + let (lock, _) = resolved(®istry, &["npm:a@1.0.0", "npm:b@1.0.0"]); + let root = TemporaryDirectory::new(); + fs::create_dir(root.path().join("config")).unwrap(); + let setting = br#"{"lib":["npm:a@1.0.0","npm:b@1.0.0"]}"#; + fs::write(root.path().join("config/setting.json"), setting).unwrap(); + lock.write_atomic(root.path()).unwrap(); + let before = fs::read(root.path().join("dever.lock")).unwrap(); + assert!( + dever_cli::libs::execute("remove", root.path(), &["npm:b@1.0.0".into()]) + .unwrap_err() + .contains("required by") + ); + assert_eq!(fs::read(root.path().join("dever.lock")).unwrap(), before); + assert_eq!( + fs::read(root.path().join("config/setting.json")).unwrap(), + setting + ); +} + +#[test] +fn repeated_archive_instances_are_bounded_before_materialization_clones() { + let mut registry = LocalRegistry(BTreeMap::new()); + release( + &mut registry, + "leaf", + "1.0.0", + serde_json::json!({}), + &[("package/payload.js", vec![b'x'; 2 * 1024 * 1024])], + ); + let mut roots = Vec::new(); + for number in 0..129 { + let name = format!("root-{number}"); + release( + &mut registry, + &name, + "1.0.0", + serde_json::json!({"dependencies":{"leaf":"1.0.0"}}), + &[], + ); + roots.push(format!("npm:{name}@1.0.0")); + } + let (lock, store) = resolved( + ®istry, + &roots.iter().map(String::as_str).collect::<Vec<_>>(), + ); + assert_eq!(lock.npm[0].instances.len(), 258); + let archives = lock + .libs + .iter() + .map(|lib| { + ( + lib.spec.clone(), + store + .verify_exact( + &lib.artifacts[0].sha256, + lib.artifacts[0].bytes, + &lib.artifacts[0].target, + ) + .unwrap(), + ) + }) + .collect::<BTreeMap<_, _>>(); + assert!( + npm::files( + &lock.npm[0], + &lock, + |lib| Ok(archives[&lib.spec].as_slice()) + ) + .unwrap_err() + .contains("tree file/byte budget") + ); +} diff --git a/test/dever-cli-tests/external_libs/npm_build.rs b/test/dever-cli-tests/external_libs/npm_build.rs new file mode 100644 index 0000000..862d442 --- /dev/null +++ b/test/dever-cli-tests/external_libs/npm_build.rs @@ -0,0 +1,590 @@ +use super::*; +use dever_cli::libs::build::{Inputs, NativeRejection, Receipt, Session, pack}; +use dever_cli::libs::{LibResolver, npm}; +use serde::{Deserialize, Serialize}; + +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct Settings { + runtime: RegistryRuntime, + runtime_file: String, + python: RegistryRuntime, + python_file: String, + build: pack::Descriptor, + build_file: String, +} + +struct AuthorInputs { + root: PathBuf, + settings: Settings, +} + +#[test] +fn npm_lifecycle_output_preserves_locked_identity_and_instance_ownership() { + let environment = npm::Environment::single("npm:app@1.0.0".parse().unwrap()); + let files = vec![ + ( + "node_modules/app/package.json".into(), + br#"{"name":"app","version":"1.0.0"}"#.to_vec(), + ), + ( + "node_modules/app/generated.js".into(), + b"module.exports=42;".to_vec(), + ), + ]; + npm::validate_built_files(&environment, &files).unwrap(); + let mut changed = files.clone(); + changed[0].1 = + br#"{"name":"app","version":"1.0.0","dependencies":{"injected":"1.0.0"}}"#.to_vec(); + assert!(npm::validate_built_files(&environment, &changed).is_err()); + let mut changed = files.clone(); + changed.push(("node_modules/injected/index.js".into(), vec![])); + assert!(npm::validate_built_files(&environment, &changed).is_err()); + let mut changed = files.clone(); + changed.push(("node_modules/app/../escape".into(), vec![])); + assert!(npm::validate_built_files(&environment, &changed).is_err()); + assert!(npm::validate_built_files(&environment, &files[1..]).is_err()); +} + +#[test] +fn npm_registry_install_hooks_do_not_run_publishing_lifecycle() { + let metadata = |script: &str| { + vec![( + "node_modules/app/package.json".into(), + serde_json::to_vec(&serde_json::json!({"scripts":{script:"node command.js"}})).unwrap(), + )] + }; + for script in ["preinstall", "install", "postinstall"] { + assert!(npm::requires_build(&metadata(script)).unwrap()); + } + for script in ["prepare", "prepack", "prepublish"] { + assert!(!npm::requires_build(&metadata(script)).unwrap()); + } + assert!( + npm::requires_build(&[("node_modules/app/binding.gyp".into(), b"{}".to_vec())]).unwrap() + ); +} + +fn locked_installation_fixture() -> (LockFile, Vec<u8>) { + let mut registry = LocalRegistry(BTreeMap::new()); + npm_tests::release( + &mut registry, + "app", + "1.0.0", + serde_json::json!({}), + &[( + "package/prebuilds/foreign/addon.node", + b"foreign-object".to_vec(), + )], + ); + npm_tests::release(&mut registry, "other", "1.0.0", serde_json::json!({}), &[]); + let store = FixtureArtifactStore::default(); + let mut lock = local_resolver(®istry, &store) + .resolve(&[ + "npm:app@1.0.0".parse().unwrap(), + "npm:other@1.0.0".parse().unwrap(), + ]) + .unwrap(); + let sources = lock + .libs + .iter() + .flat_map(|lib| lib.artifacts.clone()) + .collect(); + let archives = lock + .libs + .iter() + .map(|lib| { + ( + lib.spec.clone(), + store + .verify_exact( + &lib.artifacts[0].sha256, + lib.artifacts[0].bytes, + &lib.artifacts[0].target, + ) + .unwrap(), + ) + }) + .collect::<BTreeMap<_, _>>(); + let files = npm::build_files( + &lock.npm[0], + &lock, + |lib| Ok(archives[&lib.spec].as_slice()), + ) + .unwrap(); + let mut writer = tar::Builder::new(flate2::write::GzEncoder::new( + Vec::new(), + flate2::Compression::default(), + )); + for (path, bytes) in files { + let mut header = tar::Header::new_gnu(); + header.set_size(bytes.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + writer + .append_data(&mut header, path, bytes.as_slice()) + .unwrap(); + } + let bytes = writer.into_inner().unwrap().finish().unwrap(); + let runtime = registry_runtime(Ecosystem::Npm); + let python = registry_runtime(Ecosystem::Pip); + let hash = digest(&bytes); + let receipt = Receipt { + ecosystem: Ecosystem::Npm, + roots: lock.npm[0].roots.clone(), + sources, + graph: Some(lock.npm[0].clone()), + runtime: runtime.pack.clone(), + auxiliary_runtime: Some(python.pack.clone()), + native_entries: Default::default(), + native_rejections: BTreeMap::from([( + "node_modules/app/prebuilds/foreign/addon.node".into(), + NativeRejection::Format, + )]), + python_markers: None, + tools: pack::Descriptor { + format: "dever-registry-build-v1".into(), + ecosystem: Ecosystem::Npm, + target: runtime.target.clone(), + runtime_sha256: runtime.pack.sha256.clone(), + python_runtime_sha256: Some(python.pack.sha256), + sha256: "a".repeat(64), + }, + frontend: digest( + &[ + include_bytes!("../../../crates/dever-cli/src/libs/build/npm.js").as_slice(), + include_bytes!("../../../crates/dever-cli/src/libs/build/npm/probe.cjs").as_slice(), + ] + .concat(), + ), + python: None, + dynamic_requires: vec![], + dependencies: Box::new(LockFile::new(vec![]).unwrap()), + config_settings: BTreeMap::new(), + output: dever_cli::libs::LockedArtifact { + target: runtime.target, + path: format!("build/npm/{hash}/installation.tgz"), + bytes: bytes.len() as u64, + sha256: hash, + }, + }; + lock.npm[0].build = Some(receipt.identity().unwrap()); + lock.builds.push(receipt); + doctor(&lock).unwrap(); + (lock, bytes) +} + +#[test] +fn npm_expanded_outputs_are_pruned_only_when_no_raw_consumer_remains() { + use dever_cli::libs::{LockedWorker, prune_expanded_resources}; + use dever_core::native::EmbeddedResource; + + let (mut lock, bytes) = locked_installation_fixture(); + let roots = lock.npm[0].roots.clone(); + let other = vec!["npm:other@1.0.0".parse().unwrap()]; + lock.npm = npm::retain(&lock, &[roots.clone(), other.clone()]).unwrap(); + let output = &lock.builds[0].output; + let managed = LockedWorker { + port: "sample.worker".into(), + adapter: "default".into(), + ecosystem: "npm".into(), + runtime: Some(registry_runtime(Ecosystem::Npm).pack), + entry: "worker/main.cjs".into(), + schema: "checked".into(), + capabilities: vec![], + operations: vec!["probe".into()], + libs: roots.clone(), + }; + let resource = EmbeddedResource { + path: output.path.clone(), + bytes, + sha256: output.sha256.clone(), + executable: false, + }; + let mut resources = vec![resource.clone()]; + prune_expanded_resources(&lock, std::slice::from_ref(&managed), &[], &mut resources).unwrap(); + assert!( + resources.is_empty(), + "expanded installation archive leaked into program" + ); + + // A projected environment shares its original build output with the full + // Worker. Retention follows current consumers, not the receipt's old roots. + for declared in [roots, other] { + let mut resources = vec![resource.clone()]; + prune_expanded_resources( + &lock, + std::slice::from_ref(&managed), + &declared, + &mut resources, + ) + .unwrap(); + assert_eq!(resources.len(), 1); + } + let mut exec = managed.clone(); + exec.ecosystem = "exec".into(); + exec.runtime = None; + let mut resources = vec![resource]; + prune_expanded_resources(&lock, &[managed, exec], &[], &mut resources).unwrap(); + assert_eq!(resources.len(), 1); +} + +#[test] +fn npm_native_admission_accounts_every_addon_and_prunes_unsupported_objects() { + let (lock, bytes) = locked_installation_fixture(); + let files = dever_cli::libs::build::npm_installation(&lock, &lock.npm[0], &bytes).unwrap(); + assert!(files.iter().all(|(path, _, _)| !path.ends_with(".node"))); + let mut missing = lock.clone(); + missing.builds[0].native_rejections.clear(); + missing.npm[0].build = Some(missing.builds[0].identity().unwrap()); + assert!( + dever_cli::libs::build::npm_installation(&missing, &missing.npm[0], &bytes) + .unwrap_err() + .contains("exact addon files") + ); + let mut corrupted = bytes.clone(); + corrupted[10] ^= 1; + assert!(dever_cli::libs::build::npm_installation(&lock, &lock.npm[0], &corrupted).is_err()); + for change in ["tools", "graph", "runtime"] { + let mut changed = lock.clone(); + match change { + "tools" => changed.builds[0].tools.sha256 = "not-a-hash".into(), + "graph" => changed.builds[0].graph.as_mut().unwrap().instances.clear(), + _ => changed.builds[0].runtime.sha256 = "b".repeat(64), + } + changed.npm[0].build = Some(changed.builds[0].identity().unwrap()); + assert!( + doctor(&changed).is_err(), + "accepted internally inconsistent {change}" + ); + } +} + +#[test] +fn npm_locked_output_projects_retained_roots_without_rerunning_hooks() { + let (lock, bytes) = locked_installation_fixture(); + let project = TemporaryDirectory::new(); + fs::create_dir(project.path().join("config")).unwrap(); + fs::write( + project.path().join("config/setting.json"), + br#"{"lib":["npm:app@1.0.0","npm:other@1.0.0"]}"#, + ) + .unwrap(); + lock.write_atomic(project.path()).unwrap(); + dever_cli::libs::execute("remove", project.path(), &["npm:app@1.0.0".into()]).unwrap(); + let retained = LockFile::decode(&fs::read(project.path().join("dever.lock")).unwrap()).unwrap(); + doctor(&retained).unwrap(); + let files = + dever_cli::libs::build::npm_installation(&retained, &retained.npm[0], &bytes).unwrap(); + assert_eq!(retained.builds, lock.builds); + assert!( + files + .iter() + .all(|(path, _, _)| path.starts_with("node_modules/other/")) + ); +} + +impl AuthorInputs { + fn load() -> Self { + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../target/npm-build"); + let settings = + serde_json::from_slice(&fs::read(root.join("config/setting.json")).unwrap()).unwrap(); + Self { root, settings } + } +} + +impl Inputs for AuthorInputs { + fn runtime(&self, ecosystem: Ecosystem) -> Result<(RegistryRuntime, Vec<u8>), String> { + let (runtime, path) = match ecosystem { + Ecosystem::Pip => (&self.settings.python, &self.settings.python_file), + Ecosystem::Npm => (&self.settings.runtime, &self.settings.runtime_file), + _ => return Err("fixture only supplies Node and auxiliary Python".into()), + }; + let bytes = fs::read(self.root.join(path)).map_err(|error| error.to_string())?; + if digest(&bytes) != runtime.pack.sha256 { + return Err("author runtime drift".into()); + } + Ok((runtime.clone(), bytes)) + } + fn tools(&self, ecosystem: Ecosystem) -> Result<(pack::Descriptor, Vec<u8>), String> { + if ecosystem != Ecosystem::Npm { + return Err("fixture only supplies npm tools".into()); + } + Ok(( + self.settings.build.clone(), + fs::read(self.root.join(&self.settings.build_file)) + .map_err(|error| error.to_string())?, + )) + } + fn assets(&self) -> Result<Vec<(String, Vec<u8>)>, String> { + let workspace = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../.."); + Ok(build_tests::sandbox::assets(&workspace) + .into_iter() + .map(|asset| { + ( + asset.path.trim_start_matches("sandbox/").into(), + asset.bytes, + ) + }) + .collect()) + } + fn diagnostic(&self, output: &[u8]) { + eprintln!("{}", String::from_utf8_lossy(output)); + } +} + +#[test] +#[ignore = "explicit author operation: assembles hashed npm/compiler/header inputs; no network or host execution"] +fn prepare_npm_build_tool_fixture() { + let workspace = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../.."); + let runtime = |name: &str| { + let root = workspace.join(format!("target/{name}-managed")); + let config: serde_json::Value = + serde_json::from_slice(&fs::read(root.join("config/setting.json")).unwrap()).unwrap(); + let runtime: RegistryRuntime = serde_json::from_value(config["runtime"].clone()).unwrap(); + assert_eq!( + digest(&fs::read(root.join("runtime.pack")).unwrap()), + runtime.pack.sha256 + ); + runtime + }; + let node = runtime("npm"); + let python = runtime("python"); + let root = workspace.join("target/npm-build"); + let build = build_tests::prepare_tool_pack(Ecosystem::Npm, &node, Some(&python), &root); + let settings = Settings { + runtime: node, + runtime_file: "../npm-managed/runtime.pack".into(), + python, + python_file: "../python-managed/runtime.pack".into(), + build, + build_file: "build.pack".into(), + }; + fs::write( + root.join("config/setting.json"), + serde_json::to_vec_pretty(&settings).unwrap(), + ) + .unwrap(); +} + +fn resolver<'a>( + inputs: &AuthorInputs, + session: &'a Session<'a>, + transport: &'a dyn RegistryTransport, + store: &'a FixtureArtifactStore, +) -> RegistryResolver<'a> { + RegistryResolver { + transport, + store, + runtimes: BTreeMap::from([(Ecosystem::Npm, inputs.settings.runtime.clone())]), + go_sumdb: dever_cli::libs::sumdb::ChecksumDatabase::official(None).unwrap(), + build: Some(session), + } +} + +#[test] +#[ignore = "explicit official npm native source build; network only in registry resolver, never in lifecycle sandbox"] +fn builds_official_npm_addon_with_locked_lifecycle() { + let inputs = AuthorInputs::load(); + let session = Session::new(&inputs); + let transport = dever_cli::libs::HttpRegistry::official(); + let store = FixtureArtifactStore::default(); + let resolver = resolver(&inputs, &session, &transport, &store); + let roots = vec!["npm:bufferutil@4.0.9".parse().unwrap()]; + let lock = resolver.resolve(&roots).unwrap(); + doctor(&lock).unwrap(); + assert_eq!(lock.builds.len(), 1); + let receipt = &lock.builds[0]; + assert!( + receipt + .native_entries + .contains("node_modules/bufferutil/build/Release/bufferutil.node") + ); + let bytes = store + .verify_exact( + &receipt.output.sha256, + receipt.output.bytes, + &receipt.output.target, + ) + .unwrap(); + let files = dever_cli::libs::build::npm_installation(&lock, &lock.npm[0], &bytes).unwrap(); + assert!(files.iter().any( + |(path, bytes, _)| path.ends_with("/build/Release/bufferutil.node") + && bytes.starts_with(b"\x7fELF") + )); + assert_eq!(lock, resolver.resolve(&roots).unwrap()); + publish_fixture_artifacts(&inputs.root.join("dependency"), &lock, &store); +} + +#[test] +#[ignore = "explicit managed npm hooks; uses local registry and real signed author runtime/tools"] +fn npm_lifecycle_preserves_optional_failures_bins_and_offline_remove() { + let inputs = AuthorInputs::load(); + let session = Session::new(&inputs); + let mut transport = LocalRegistry(BTreeMap::new()); + npm_tests::release(&mut transport, "helper", "1.0.0", serde_json::json!({"bin":{"build-probe":"cli.js"}}), &[("package/cli.js", b"#!/usr/bin/env node\nrequire('fs').writeFileSync('generated.js','module.exports = 42;');\n".to_vec())]); + npm_tests::release( + &mut transport, + "broken", + "1.0.0", + serde_json::json!({"dependencies":{"cycle":"1.0.0"},"scripts":{"install":"node -e 'process.exit(7)'"}}), + &[], + ); + npm_tests::release( + &mut transport, + "cycle", + "1.0.0", + serde_json::json!({"dependencies":{"broken":"1.0.0"}}), + &[], + ); + npm_tests::release( + &mut transport, + "app", + "1.0.0", + serde_json::json!({"dependencies":{"helper":"1.0.0"},"optionalDependencies":{"broken":"1.0.0"},"scripts":{"postinstall":"build-probe"}}), + &[], + ); + npm_tests::release(&mut transport, "other", "1.0.0", serde_json::json!({}), &[]); + let store = FixtureArtifactStore::default(); + let resolver = resolver(&inputs, &session, &transport, &store); + let lock = resolver + .resolve(&[ + "npm:app@1.0.0".parse().unwrap(), + "npm:other@1.0.0".parse().unwrap(), + ]) + .unwrap(); + doctor(&lock).unwrap(); + assert!( + !lock + .libs + .iter() + .any(|lib| matches!(lib.spec.name.as_str(), "broken" | "cycle")) + ); + assert!( + lock.npm[0] + .instances + .iter() + .flat_map(|node| &node.edges) + .any(|edge| edge.omission == Some(npm::Omission::BuildFailure)) + ); + let receipt = &lock.builds[0]; + let bytes = store + .verify_exact( + &receipt.output.sha256, + receipt.output.bytes, + &receipt.output.target, + ) + .unwrap(); + let files = dever_cli::libs::build::npm_installation(&lock, &lock.npm[0], &bytes).unwrap(); + assert!( + files + .iter() + .any(|(path, bytes, _)| path == "node_modules/app/generated.js" + && bytes == b"module.exports = 42;") + ); + assert!( + resolver + .resolve(&["npm:broken@1.0.0".parse().unwrap()]) + .unwrap_err() + .contains("failed") + ); + let project = TemporaryDirectory::new(); + fs::create_dir(project.path().join("config")).unwrap(); + fs::write( + project.path().join("config/setting.json"), + br#"{"lib":["npm:app@1.0.0","npm:other@1.0.0"]}"#, + ) + .unwrap(); + lock.write_atomic(project.path()).unwrap(); + dever_cli::libs::execute("remove", project.path(), &["npm:app@1.0.0".into()]).unwrap(); + let retained = LockFile::decode(&fs::read(project.path().join("dever.lock")).unwrap()).unwrap(); + doctor(&retained).unwrap(); + let files = + dever_cli::libs::build::npm_installation(&retained, &retained.npm[0], &bytes).unwrap(); + assert!( + files + .iter() + .all(|(path, _, _)| path.starts_with("node_modules/other/")) + ); +} + +#[test] +#[ignore = "explicit managed npm hook: sparse scratch overflow and child cleanup; no network"] +fn npm_build_scratch_budget_kills_owned_hook_and_cleans_stage() { + let inputs = AuthorInputs::load(); + let session = Session::new(&inputs); + let mut transport = LocalRegistry(BTreeMap::new()); + npm_tests::release(&mut transport,"oversized","1.0.0",serde_json::json!({"scripts":{"install":"node fill.js"}}),&[("package/fill.js",b"const fs=require('fs');fs.writeFileSync('large',Buffer.alloc(1));fs.truncateSync('large',600*1024*1024);setInterval(()=>{},1000);".to_vec())]); + let store = FixtureArtifactStore::default(); + let resolver = resolver(&inputs, &session, &transport, &store); + let owned_stages = || { + fs::read_dir(std::env::temp_dir()) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .filter(|name| name.starts_with(&format!("dever-lib-build-{}-", std::process::id()))) + .collect::<std::collections::BTreeSet<_>>() + }; + let before = owned_stages(); + let error = resolver + .resolve(&["npm:oversized@1.0.0".parse().unwrap()]) + .unwrap_err(); + assert!(error.contains("scratch budget"), "{error}"); + assert_eq!( + owned_stages(), + before, + "failed hook leaked its owned staging tree" + ); +} + +#[test] +#[ignore = "explicit managed npm hook: internal output aliases are flattened; aliases outside output are rejected"] +fn npm_build_output_hard_links_stay_inside_the_exported_tree() { + let inputs = AuthorInputs::load(); + let session = Session::new(&inputs); + for (name, source, accepted) in [ + ( + "insidealias", + "const fs=require('fs');fs.writeFileSync('original.txt','same-bytes');fs.linkSync('original.txt','alias.txt');", + true, + ), + ( + "outsidealias", + "const fs=require('fs');fs.writeFileSync('/data/build/outside.txt','outside');fs.linkSync('/data/build/outside.txt','alias.txt');", + false, + ), + ] { + let mut transport = LocalRegistry(BTreeMap::new()); + npm_tests::release( + &mut transport, + name, + "1.0.0", + serde_json::json!({"scripts":{"install":"node aliases.js"}}), + &[("package/aliases.js", source.as_bytes().to_vec())], + ); + let store = FixtureArtifactStore::default(); + let result = + resolver(&inputs, &session, &transport, &store) + .resolve(&[format!("npm:{name}@1.0.0").parse().unwrap()]); + if accepted { + let lock = result.unwrap(); + doctor(&lock).unwrap(); + let receipt = &lock.builds[0]; + let bytes = store + .verify_exact( + &receipt.output.sha256, + receipt.output.bytes, + &receipt.output.target, + ) + .unwrap(); + let files = + dever_cli::libs::build::npm_installation(&lock, &lock.npm[0], &bytes).unwrap(); + for leaf in ["original.txt", "alias.txt"] { + assert!(files.iter().any(|(path, bytes, _)| path + == &format!("node_modules/{name}/{leaf}") + && bytes == b"same-bytes")); + } + } else { + assert!(result.unwrap_err().contains("hard-link alias outside")); + } + } +} diff --git a/test/dever-cli-tests/external_libs/sumdb.rs b/test/dever-cli-tests/external_libs/sumdb.rs new file mode 100644 index 0000000..1302066 --- /dev/null +++ b/test/dever-cli-tests/external_libs/sumdb.rs @@ -0,0 +1,754 @@ +use super::*; +use dever_cli::libs::sumdb::{self, ChecksumDatabase, Evidence, Verifier}; +use ring::signature::{Ed25519KeyPair, KeyPair}; + +fn key() -> Ed25519KeyPair { + Ed25519KeyPair::from_seed_unchecked(&[47; 32]).unwrap() +} + +pub(super) fn verifier() -> Verifier { + let mut encoded = vec![1]; + encoded.extend(key().public_key().as_ref()); + let mut hash = Sha256::new(); + hash.update(b"dever-sumdb-test\n"); + hash.update(&encoded); + let id = hash.finalize(); + Verifier::new(&format!( + "dever-sumdb-test+{}+{}", + id[..4] + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::<String>(), + base64::engine::general_purpose::STANDARD.encode(encoded) + )) + .unwrap() +} + +fn signed(size: usize, root: [u8; 32]) -> String { + let text = format!( + "go.sum database tree\n{size}\n{}\n", + base64::engine::general_purpose::STANDARD.encode(root) + ); + let mut hash = Sha256::new(); + hash.update(b"dever-sumdb-test\n"); + hash.update([1]); + hash.update(key().public_key().as_ref()); + let mut signature = hash.finalize()[..4].to_vec(); + signature.extend(key().sign(text.as_bytes()).as_ref()); + format!( + "{text}\n— dever-sumdb-test {}\n", + base64::engine::general_purpose::STANDARD.encode(signature) + ) +} + +fn leaf(record: &[u8]) -> [u8; 32] { + let mut hash = Sha256::new(); + hash.update([0]); + hash.update(record); + hash.finalize().into() +} + +fn root(hashes: &[[u8; 32]]) -> [u8; 32] { + if hashes.is_empty() { + return Sha256::digest([]).into(); + } + if hashes.len() == 1 { + return hashes[0]; + } + let split = hashes.len().next_power_of_two() / 2; + let mut hash = Sha256::new(); + hash.update([1]); + hash.update(root(&hashes[..split])); + hash.update(root(&hashes[split..])); + hash.finalize().into() +} + +fn unescape(path: &str) -> String { + let mut chars = path.chars(); + let mut output = String::new(); + while let Some(ch) = chars.next() { + output.push(if ch == '!' { + chars.next().unwrap().to_ascii_uppercase() + } else { + ch + }); + } + output +} + +pub(super) fn seal(registry: &mut LocalRegistry) { + registry + .0 + .retain(|(_, path), _| !path.starts_with("https://sum.golang.org/")); + let records = registry + .0 + .iter() + .filter_map(|((ecosystem, path), bytes)| { + if *ecosystem != Ecosystem::Go || !path.ends_with(".mod") { + return None; + } + let (module, version) = path.strip_prefix('/')?.split_once("/@v/v")?; + let version = version.strip_suffix(".mod")?; + let module = unescape(module); + let archive = registry.0.get(&( + Ecosystem::Go, + format!("{}zip", path.strip_suffix("mod").unwrap()), + ))?; + let record = format!( + "{module} v{version} {}\n{module} v{version}/go.mod {}\n", + sumdb::zip_hash(&module, version, archive).unwrap(), + sumdb::mod_hash(bytes) + ); + Some(( + format!( + "/lookup/{}@v{version}", + path.strip_prefix('/') + .unwrap() + .split("/@v/") + .next() + .unwrap() + ), + record, + )) + }) + .collect::<Vec<_>>(); + let hashes = records + .iter() + .map(|(_, record)| leaf(record.as_bytes())) + .collect::<Vec<_>>(); + let checkpoint = signed(records.len(), root(&hashes)); + registry.0.insert( + (Ecosystem::Go, "https://sum.golang.org/latest".into()), + checkpoint.as_bytes().to_vec(), + ); + for (id, (path, record)) in records.iter().enumerate() { + registry.0.insert( + (Ecosystem::Go, format!("https://sum.golang.org{path}")), + format!("{id}\n{record}\n{checkpoint}").into_bytes(), + ); + } + // Fixtures deliberately remain below one tile. Larger trees use official + // captured vectors, not a second production tile-address implementation. + assert!(hashes.len() < 256); + if !hashes.is_empty() { + registry.0.insert( + ( + Ecosystem::Go, + format!("https://sum.golang.org/tile/8/0/000.p/{}", hashes.len()), + ), + hashes.iter().flatten().copied().collect(), + ); + } +} + +pub(super) struct Transport<'a> { + pub registry: &'a dyn RegistryTransport, + pub checksums: &'a LocalRegistry, +} +impl RegistryTransport for Transport<'_> { + fn get(&self, ecosystem: Ecosystem, path: &str, limit: usize) -> Result<Vec<u8>, String> { + if path.starts_with("https://sum.golang.org/") { + self.checksums.get(ecosystem, path, limit) + } else { + self.registry.get(ecosystem, path, limit) + } + } +} + +fn resolved( + registry: &LocalRegistry, + previous: Option<String>, + roots: &[&str], +) -> Result<LockFile, String> { + let cache = FixtureArtifactStore::default(); + let mut resolver = local_resolver(registry, &cache); + resolver.go_sumdb = ChecksumDatabase::new(verifier(), previous)?; + dever_cli::libs::LibResolver::resolve( + &resolver, + &roots + .iter() + .map(|root| root.parse().unwrap()) + .collect::<Vec<_>>(), + ) +} + +#[test] +fn authenticated_modules_replay_offline_and_reject_content_or_proof_drift() { + let registry = local_registry(); + let lock = resolved(®istry, None, &["go:example.com/a@1.0.0"]).unwrap(); + let evidence = &lock.go_sumdb[0]; + let checked = evidence.verify(&verifier()).unwrap(); + let archive = ®istry.0[&(Ecosystem::Go, "/example.com/a/@v/v1.0.0.zip".into())]; + checked + .verify_zip("example.com/a", "1.0.0", archive) + .unwrap(); + assert!( + evidence + .verify(&Verifier::official()) + .unwrap_err() + .contains("trusted signature") + ); + let mut changed = evidence.clone(); + changed.records[0].go_mod.push_str("// changed\n"); + assert!(changed.verify(&verifier()).unwrap_err().contains("go.mod")); + let mut changed = evidence.clone(); + changed.records[0].module = "wrong.example/a".into(); + assert!( + changed + .verify(&verifier()) + .unwrap_err() + .contains("different module/version") + ); + let mut changed = evidence.clone(); + changed.records[0].inclusion[0] = base64::engine::general_purpose::STANDARD.encode([0; 32]); + assert!( + changed + .verify(&verifier()) + .unwrap_err() + .contains("inclusion proof") + ); + let mut changed = evidence.clone(); + changed.checkpoint = changed + .checkpoint + .replace("go.sum database tree", "go.sum database free"); + assert!( + changed + .verify(&verifier()) + .unwrap_err() + .contains("signature verification") + ); + assert!( + checked + .verify_zip( + "example.com/a", + "1.0.0", + &zip_file("example.com/a@v1.0.0/changed.go") + ) + .unwrap_err() + .contains("sumdb h1") + ); +} + +#[test] +fn checkpoint_updates_require_consistency_and_never_move_backwards() { + let mut first = local_registry(); + first + .0 + .remove(&(Ecosystem::Go, "/example.com/b/@v/v1.2.0.mod".into())); + first.0.insert( + (Ecosystem::Go, "/example.com/a/@v/v1.0.0.mod".into()), + b"module example.com/a\n".to_vec(), + ); + seal(&mut first); + let old = resolved(&first, None, &["go:example.com/a@1.0.0"]).unwrap(); + let checkpoint = old.go_sumdb[0].verify(&verifier()).unwrap().checkpoint; + let mut next = first; + next.0.insert( + (Ecosystem::Go, "/example.com/z/@v/v1.0.0.mod".into()), + b"module example.com/z\n".to_vec(), + ); + next.0.insert( + (Ecosystem::Go, "/example.com/z/@v/v1.0.0.zip".into()), + zip_file("example.com/z@v1.0.0/z.go"), + ); + seal(&mut next); + let new = resolved(&next, Some(checkpoint.clone()), &["go:example.com/z@1.0.0"]).unwrap(); + let current = new.go_sumdb[0].verify(&verifier()).unwrap().checkpoint; + assert!(resolved(&next, Some(current.clone()), &["go:example.com/z@1.0.0"]).is_ok()); + next.0.insert( + (Ecosystem::Go, "https://sum.golang.org/latest".into()), + checkpoint.into_bytes(), + ); + assert!( + resolved(&next, Some(current), &["go:example.com/a@1.0.0"]) + .unwrap_err() + .contains("rollback") + ); + let mut fork = new.go_sumdb[0].clone(); + fork.consistency[0] = base64::engine::general_purpose::STANDARD.encode([0; 32]); + assert!( + fork.verify(&verifier()) + .unwrap_err() + .contains("conflicting history") + ); +} + +#[test] +fn sumdb_verifier_key_and_upstream_signed_note_vectors() { + let note = "go.sum database tree\n63512409\nF7eHRPqN/cqPtqti14NKdGGaI1Giw8coWP3KxihFZbg=\n\n— sum.golang.org Az3gri1OBHKT87Xj8UXU8zjoHGy/X+7Nl94f8TZkELVaasolmGApUsWqwVG9vE6hgqYb/ph/w2vdkLIVRH7bMB/FPQ4=\n"; + let evidence = Evidence { + previous: None, + checkpoint: note.into(), + consistency: vec![], + records: vec![], + }; + evidence.verify(&Verifier::official()).unwrap(); + assert!(evidence.verify(&verifier()).is_err()); + assert!(Verifier::new(&sumdb::OFFICIAL_VERIFIER.replace("033de0ae", "033de0af")).is_err()); +} + +#[test] +fn official_large_tree_inclusion_and_consistency_vectors_verify_offline() { + // Captured from sum.golang.org/lookup/github.com/google/uuid@v1.6.0 + // and its immutable height-8 tiles on 2026-10-02. Hashes were assembled + // independently of this Rust implementation. Both notes retain Google's + // original Ed25519 signatures; no fixture signer can mint these vectors. + let evidence: Evidence = serde_json::from_str(include_str!("sumdb_uuid.json")).unwrap(); + let trusted = Verifier::official(); + let verified = evidence.verify(&trusted).unwrap(); + assert!(verified.contains("github.com/google/uuid", "1.6.0")); + assert_eq!( + sumdb::mod_hash(b"module github.com/google/uuid\n"), + "h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=" + ); + for index in 0..evidence.records[0].inclusion.len() { + let mut changed = evidence.clone(); + changed.records[0].inclusion[index] = + base64::engine::general_purpose::STANDARD.encode([0; 32]); + assert!( + changed.verify(&trusted).is_err(), + "accepted corrupt inclusion hash {index}" + ); + } + for index in 0..evidence.consistency.len() { + let mut changed = evidence.clone(); + changed.consistency[index] = base64::engine::general_purpose::STANDARD.encode([0; 32]); + assert!( + changed.verify(&trusted).is_err(), + "accepted corrupt consistency hash {index}" + ); + } +} + +#[test] +fn go_h1_ignores_zip_order_and_compression_and_preserves_module_case() { + let archive = |reverse, compression| { + let mut writer = zip::ZipWriter::new(Cursor::new(Vec::new())); + let mut files = vec![ + ("a.go", b"package a\n".as_slice()), + ("LICENSE", b"fixture license\n".as_slice()), + ]; + if reverse { + files.reverse(); + } + for (name, content) in files { + writer + .start_file( + format!("example.com/Mixed@v1.0.0/{name}"), + zip::write::SimpleFileOptions::default().compression_method(compression), + ) + .unwrap(); + writer.write_all(content).unwrap(); + } + writer.finish().unwrap().into_inner() + }; + let first = archive(false, zip::CompressionMethod::Stored); + let second = archive(true, zip::CompressionMethod::Deflated); + assert_ne!(digest(&first), digest(&second)); + assert_eq!( + sumdb::zip_hash("example.com/Mixed", "1.0.0", &first).unwrap(), + sumdb::zip_hash("example.com/Mixed", "1.0.0", &second).unwrap() + ); + let mut registry = LocalRegistry(BTreeMap::from([ + ( + (Ecosystem::Go, "/example.com/!mixed/@v/v1.0.0.mod".into()), + b"module example.com/Mixed\n".to_vec(), + ), + ( + (Ecosystem::Go, "/example.com/!mixed/@v/v1.0.0.zip".into()), + first, + ), + ])); + seal(&mut registry); + let lock = resolved(®istry, None, &["go:example.com/Mixed@1.0.0"]).unwrap(); + assert_eq!(lock.libs[0].spec.name, "example.com/Mixed"); + assert!( + lock.go_sumdb[0] + .verify(&verifier()) + .unwrap() + .contains("example.com/Mixed", "1.0.0") + ); +} + +#[test] +fn every_go_mod_consulted_by_mvs_is_authenticated_even_when_not_selected() { + let mut registry = LocalRegistry(BTreeMap::new()); + for (name, version, source) in [ + ( + "a", + "1.0.0", + "module example.com/a\nrequire (\nexample.com/b v1.0.0\nexample.com/c v1.0.0\n)\n", + ), + ("b", "1.0.0", "module example.com/b\n"), + ("b", "2.0.0", "module example.com/b\n"), + ( + "c", + "1.0.0", + "module example.com/c\nrequire example.com/b v2.0.0\n", + ), + ] { + registry.0.insert( + ( + Ecosystem::Go, + format!("/example.com/{name}/@v/v{version}.mod"), + ), + source.as_bytes().to_vec(), + ); + registry.0.insert( + ( + Ecosystem::Go, + format!("/example.com/{name}/@v/v{version}.zip"), + ), + zip_file(&format!("example.com/{name}@v{version}/{name}.go")), + ); + } + seal(&mut registry); + let lock = resolved(®istry, None, &["go:example.com/a@1.0.0"]).unwrap(); + assert!( + !lock + .libs + .iter() + .any(|lib| lib.spec.key() == "go:example.com/b@1.0.0") + ); + assert!( + lock.go_sumdb[0] + .records + .iter() + .any(|record| record.module == "example.com/b" && record.version == "1.0.0") + ); + registry + .0 + .get_mut(&(Ecosystem::Go, "/example.com/b/@v/v1.0.0.mod".into())) + .unwrap() + .extend(b"// modified discarded metadata\n"); + let cache = FixtureArtifactStore::default(); + let resolver = local_resolver(®istry, &cache); + assert!( + dever_cli::libs::LibResolver::resolve( + &resolver, + &["go:example.com/a@1.0.0".parse().unwrap()] + ) + .unwrap_err() + .contains("go.mod does not match") + ); + for ((_, path), bytes) in ®istry.0 { + if path.ends_with(".zip") { + assert!(!cache.has(&digest(bytes), "linux-x86_64").unwrap()); + } + } +} + +#[test] +fn go_hashzip_includes_empty_directory_entries() { + let archive = |directory| { + let mut writer = zip::ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file( + "example.com/a@v1.0.0/a.go", + zip::write::SimpleFileOptions::default(), + ) + .unwrap(); + writer.write_all(b"package a\n").unwrap(); + if directory { + writer + .add_directory( + "example.com/a@v1.0.0/empty/", + zip::write::SimpleFileOptions::default(), + ) + .unwrap(); + } + writer.finish().unwrap().into_inner() + }; + // Independent Hash1 vectors: sorted "sha256(content) filename\n" records, + // including the directory's trailing slash and SHA-256 of empty content. + assert_eq!( + sumdb::zip_hash("example.com/a", "1.0.0", &archive(false)).unwrap(), + "h1:VEUcYKYLbKBYq0eA/VEJ3mMQBzi4jjWpusjuduGJcBQ=" + ); + assert_eq!( + sumdb::zip_hash("example.com/a", "1.0.0", &archive(true)).unwrap(), + "h1:y9ZWKWCmVsKnLxdFkIVoxr4MyXyHgBUvp1hgBzZ+IfE=" + ); +} + +#[test] +fn project_updates_and_removing_last_lib_preserve_official_checkpoint_anchor() { + let root = TemporaryDirectory::new(); + let evidence: Evidence = serde_json::from_str(include_str!("sumdb_uuid.json")).unwrap(); + let mut lock = LockFile::new(Vec::new()).unwrap(); + lock.go_sumdb.push(evidence.clone()); + lock.write_atomic(root.path()).unwrap(); + let spec = vec!["pip:fixture@1.0.0".into()]; + for command in ["add", "update", "remove"] { + dever_cli::libs::execute(command, root.path(), &spec).unwrap(); + let lock = LockFile::decode(&fs::read(root.path().join("dever.lock")).unwrap()).unwrap(); + assert_eq!(lock.go_sumdb, vec![evidence.clone()]); + let current = sumdb::verify_chain(&lock.go_sumdb, &Verifier::official()).unwrap(); + let previous = evidence.previous.clone().unwrap(); + // A subsequent online resolution cannot accept the older official tree. + let mut registry = local_registry(); + registry.0.insert( + (Ecosystem::Go, "https://sum.golang.org/latest".into()), + previous.into_bytes(), + ); + let cache = FixtureArtifactStore::default(); + let mut resolver = local_resolver(®istry, &cache); + resolver.go_sumdb = ChecksumDatabase::official(Some(current.checkpoint)).unwrap(); + assert!( + dever_cli::libs::LibResolver::resolve( + &resolver, + &["go:example.com/a@1.0.0".parse().unwrap()] + ) + .unwrap_err() + .contains("rollback") + ); + } +} + +#[test] +fn lock_publication_rejects_offline_byte_and_element_overflow_before_replacement() { + let root = TemporaryDirectory::new(); + let original = LockFile::new(Vec::new()).unwrap(); + original.write_atomic(root.path()).unwrap(); + let path = root.path().join("dever.lock"); + let before = fs::read(&path).unwrap(); + let receipt = Evidence { + previous: None, + checkpoint: "x".into(), + consistency: Vec::new(), + records: Vec::new(), + }; + for overflow in [true, false] { + let mut lock = original.clone(); + let mut receipt = receipt.clone(); + if overflow { + receipt.checkpoint = "x".repeat(dever_runtime::wire::MAX_BYTES); + } else { + receipt.consistency = vec!["x".into(); dever_runtime::wire::MAX_ELEMENTS]; + } + lock.go_sumdb.push(receipt); + assert!(lock.encode().unwrap_err().contains("invalid dever.lock")); + assert!(lock.write_atomic(root.path()).is_err()); + assert_eq!(fs::read(&path).unwrap(), before); + assert_eq!(fs::read_dir(root.path()).unwrap().count(), 1); + } +} + +#[test] +fn truncated_surplus_and_out_of_bounds_sumdb_proofs_are_rejected() { + let original: Evidence = serde_json::from_str(include_str!("sumdb_uuid.json")).unwrap(); + let trusted = Verifier::official(); + let mut changed = original.clone(); + changed.records[0].inclusion.pop(); + assert!(changed.verify(&trusted).is_err()); + let mut changed = original.clone(); + changed.records[0] + .inclusion + .push(base64::engine::general_purpose::STANDARD.encode([0; 32])); + assert!(changed.verify(&trusted).is_err()); + for id in ["66321505", "09223372036854775807", "9223372036854775808"] { + let mut changed = original.clone(); + let (_, suffix) = changed.records[0].lookup.split_once('\n').unwrap(); + changed.records[0].lookup = format!("{id}\n{suffix}"); + assert!(changed.verify(&trusted).is_err()); + } + let mut changed = original.clone(); + changed.checkpoint.pop(); + assert!(changed.verify(&trusted).is_err()); + let mut disconnected = original.clone(); + disconnected.previous = None; + disconnected.consistency.clear(); + assert!( + sumdb::verify_chain(&[original, disconnected], &trusted) + .unwrap_err() + .contains("not connected") + ); +} + +#[cfg(unix)] +#[test] +fn lib_and_package_mutations_share_a_stable_nonblocking_project_lock() { + use std::os::unix::fs::OpenOptionsExt; + let root = TemporaryDirectory::new(); + let requested = vec!["pip:fixture@1.0.0".into()]; + dever_cli::libs::execute("add", root.path(), &requested).unwrap(); + let lock_path = root.path().join("dever.lock"); + let config_path = root.path().join("config/setting.json"); + let before_lock = fs::read(&lock_path).unwrap(); + let before_config = fs::read(&config_path).unwrap(); + let guard = fs::OpenOptions::new() + .read(true) + .write(true) + .mode(0o600) + .open(root.path().join(".dever-project.lock")) + .unwrap(); + fs2::FileExt::try_lock_exclusive(&guard).unwrap(); + for command in ["add", "update", "remove"] { + assert!( + dever_cli::libs::execute(command, root.path(), &requested) + .unwrap_err() + .contains("mutation is busy") + ); + assert!( + dever_cli::packages::execute(command, root.path(), &[]) + .unwrap_err() + .contains("mutation is busy") + ); + struct UnusedTransport; + impl dever_cli::packages::PackageTransport for UnusedTransport { + fn get(&self, _: &str, _: usize) -> Result<Vec<u8>, String> { + panic!("busy mutation must not contact a registry") + } + } + assert!( + dever_cli::packages::execute_with( + command, + root.path(), + &[], + &UnusedTransport, + &FixtureArtifactStore::default() + ) + .unwrap_err() + .contains("mutation is busy") + ); + } + assert_eq!(fs::read(&lock_path).unwrap(), before_lock); + assert_eq!(fs::read(&config_path).unwrap(), before_config); + drop(guard); + dever_cli::libs::execute("update", root.path(), &requested).unwrap(); + assert_eq!(fs::read(&lock_path).unwrap(), before_lock); +} + +#[cfg(unix)] +#[test] +fn project_mutation_rejects_linked_or_unprotected_lock_paths() { + use std::os::unix::fs::{PermissionsExt, symlink}; + let root = TemporaryDirectory::new(); + let outside = TemporaryDirectory::new(); + let target = outside.path().join("untouched"); + fs::write(&target, "keep").unwrap(); + let path = root.path().join(".dever-project.lock"); + symlink(&target, &path).unwrap(); + let requested = vec!["pip:fixture@1.0.0".into()]; + assert!( + dever_cli::libs::execute("add", root.path(), &requested) + .unwrap_err() + .contains("without symlinks") + ); + assert_eq!(fs::read(&target).unwrap(), b"keep"); + fs::remove_file(&path).unwrap(); + fs::hard_link(&target, &path).unwrap(); + assert!( + dever_cli::libs::execute("add", root.path(), &requested) + .unwrap_err() + .contains("private and unlinked") + ); + fs::remove_file(&path).unwrap(); + fs::write(&path, "").unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o666)).unwrap(); + assert!( + dever_cli::libs::execute("add", root.path(), &requested) + .unwrap_err() + .contains("private and unlinked") + ); + fs::set_permissions(root.path(), fs::Permissions::from_mode(0o777)).unwrap(); + assert!( + dever_cli::libs::execute("add", root.path(), &requested) + .unwrap_err() + .contains("not group/other writable") + ); +} + +#[test] +fn independent_go_workers_merge_connected_offline_evidence() { + use dever_cli::libs::{LockedWorker, resolve_workers}; + let registry = local_registry(); + let cache = FixtureArtifactStore::default(); + let resolver = local_resolver(®istry, &cache); + let worker = |adapter: &str, spec: &str| LockedWorker { + port: "sample.worker".into(), + adapter: adapter.into(), + ecosystem: "go".into(), + runtime: Some(registry_runtime(Ecosystem::Go).pack), + entry: format!("worker/{adapter}.go"), + schema: "checked-port".into(), + capabilities: vec![], + operations: vec!["read".into()], + libs: vec![spec.parse().unwrap()], + }; + let lock = resolve_workers( + &resolver, + &[], + vec![ + worker("first", "go:example.com/a@1.0.0"), + worker("second", "go:example.com/b@1.2.0"), + ], + ) + .unwrap(); + assert_eq!(lock.go_sumdb.len(), 2); + assert_eq!( + lock.go_sumdb[1].previous.as_ref(), + Some(&lock.go_sumdb[0].verify(&verifier()).unwrap().checkpoint) + ); + let decoded = LockFile::decode(&lock.encode().unwrap()).unwrap(); + let verified = sumdb::verify_chain(&decoded.go_sumdb, &verifier()).unwrap(); + assert!(verified.contains("example.com/a", "1.0.0")); + assert!(verified.contains("example.com/b", "1.2.0")); + verify_locked_artifacts(&decoded, &cache, "linux-x86_64").unwrap(); +} + +#[test] +#[ignore = "author operation: explicitly fetches official Go proxy/sumdb; requires target/go-managed/config/setting.json and runtime.pack"] +fn prepare_official_go_dependency_fixture() { + #[derive(serde::Deserialize)] + #[serde(deny_unknown_fields)] + struct Settings { + runtime: RegistryRuntime, + } + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../target/go-managed"); + let settings: Settings = serde_json::from_slice( + &fs::read(root.join("config/setting.json")) + .expect("prepare the explicit fixture settings first"), + ) + .unwrap(); + let pack = + fs::read(root.join("runtime.pack")).expect("prepare the explicit Go runtime pack first"); + assert_eq!( + settings.runtime.pack.sha256, + digest(&pack), + "configured runtime does not match runtime.pack" + ); + assert_eq!( + settings.runtime.target, + dever_cli::toolchain::platform_identity() + ); + let output = root.join("dependency"); + let lock_path = output.join("dever.lock"); + let previous = if lock_path.exists() { + let lock = LockFile::decode(&fs::read(&lock_path).unwrap()).unwrap(); + dever_cli::libs::doctor(&lock).unwrap(); + let verified = sumdb::verify_chain(&lock.go_sumdb, &Verifier::official()).unwrap(); + (!verified.checkpoint.is_empty()).then_some(verified.checkpoint) + } else { + None + }; + let target = settings.runtime.target.clone(); + let store = FixtureArtifactStore::default(); + let transport = dever_cli::libs::HttpRegistry::official(); + let resolver = RegistryResolver { + build: None, + transport: &transport, + store: &store, + runtimes: BTreeMap::from([(Ecosystem::Go, settings.runtime)]), + go_sumdb: ChecksumDatabase::official(previous).unwrap(), + }; + let lock = dever_cli::libs::LibResolver::resolve( + &resolver, + &["go:github.com/google/uuid@1.6.0".parse().unwrap()], + ) + .unwrap(); + // The same official verifier and offline artifact checks protect normal builds. + dever_cli::libs::doctor(&lock).unwrap(); + dever_cli::libs::verify_locked_artifacts(&lock, &store, &target).unwrap(); + publish_fixture_artifacts(&output, &lock, &store); +} diff --git a/test/dever-cli-tests/external_libs/sumdb_uuid.json b/test/dever-cli-tests/external_libs/sumdb_uuid.json new file mode 100644 index 0000000..f1894cf --- /dev/null +++ b/test/dever-cli-tests/external_libs/sumdb_uuid.json @@ -0,0 +1,68 @@ +{ + "previous": "go.sum database tree\n63512409\nF7eHRPqN/cqPtqti14NKdGGaI1Giw8coWP3KxihFZbg=\n\n— sum.golang.org Az3gri1OBHKT87Xj8UXU8zjoHGy/X+7Nl94f8TZkELVaasolmGApUsWqwVG9vE6hgqYb/ph/w2vdkLIVRH7bMB/FPQ4=\n", + "checkpoint": "go.sum database tree\n66321505\nnL/ZK4zIq2TWUhMi/cDJQRq6hXj67qFBbyEqxks3epY=\n\n— sum.golang.org Az3gruvaP9rIy6foXnttmnViaXM55mqUcP66AeP7N5r+tpFo2ll40wDr7sN0guXk6rXp0fL63fe30pchMCTqmbRo+Ak=\n", + "consistency": [ + "3OyjS9XS6r41pxaFWGnW/WSmX7b1ik3nupquDNipdZ8=", + "zX5IsB02+IutT35ka/MwY7gFiPSAlihfnt2cu6smUjw=", + "Qc0CqO89A+/onGz1u4ZsDoMOfRIgPZpayRGTETOv8kk=", + "JhdrRO/y8Iu+MDeUupFG0uM4+odAADMjVH0mOgLOA/c=", + "lOgywJvTdrBWol0uphM6ZulvAebwJGNlekm8MIwiwKM=", + "FRf0N51Fw+8eSLHEa/e/AsnDway0EkKo4jI0E1Ne7dk=", + "Ak0gyC+RS6/ahVBCBuSBDA1wFN9mgq3acIbYm9uJHQ4=", + "FFJ8WrKTTQgCjTuAl5CukBdjDaQ7opSdHBpNSolRYWw=", + "rdNcd+MUjUxzlX/nlVGeQP5BkkZyTT89sn/yfg0AXQ0=", + "gJzOh5y5rP1BILUrFJw+aGjod840PErDnjy4T0BoUE4=", + "7I8BPI+ADxeJmu19mQxO+xt6bNhg3Ueq1j2dIfOppwA=", + "urhcUF9QOyTRAYRjJjsRHjHdwIc+BxfiQ6a+NuYeC6E=", + "p9eGEUHXeKLFnrIPB7VXHHMuwtaBnvHG4CYwgC0wq+w=", + "Gcry0ScaNfFmQujPMT95RQ3rbzKDSynQsL546C8CkfU=", + "dIpym+JRT/+z9/tlhpuIRyv+uoWGqgND/MHw0f1MtqA=", + "cQrKLJIBbPoWDnRnbGfAEMfpsau0h+Aw3aqDOJ5LAVU=", + "C46761Ui7jL+lf4vRXBlveoxuFS6tykAd6mMEfNttPE=", + "45Kz4HcZ9iZ84ySMHCy21d4vd2MvibvfNMzYm0hKYRs=", + "4JZTJ+4VUtLMTEs11jn4GmCxoz56Lm5HDwkYmkmAWlw=", + "2hJV4h1jZ2YujjyXUw+7YYCwLXGYZy+ondhRUgxKmXI=", + "lrn/kRfgYdYSDxgPdSQ8/GnZw6u/joKu9++AR+Saa5M=", + "hnm9+kaHQdhHWrAcyrvY5fbv8QB+GwVjRMFeobtTg8A=", + "rPFmKL/vD6Edsk/0DmpvDa/nMOqWgjfpPGRjBUL9yKs=", + "2IzbyezmwwGZZPJ276ejAQZiAnar+RcjAUU3MEMtbIA=", + "k3kfr9AECmkV+UA3/D6EMqaBDr67aubL5hXGsvJ+Nng=", + "/WnQtddEqG03mbcU1XID1iws1nlKHVkJcaREWknCOiE=", + "xJ20xXM4QItt5vxFOtolZ5l18p7ej1+RimNkgWmoRFM=" + ], + "records": [{ + "module": "github.com/google/uuid", + "version": "1.6.0", + "go_mod": "module github.com/google/uuid\n", + "lookup": "22152757\ngithub.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=\ngithub.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=\n\ngo.sum database tree\n66321505\nnL/ZK4zIq2TWUhMi/cDJQRq6hXj67qFBbyEqxks3epY=\n\n— sum.golang.org Az3gruvaP9rIy6foXnttmnViaXM55mqUcP66AeP7N5r+tpFo2ll40wDr7sN0guXk6rXp0fL63fe30pchMCTqmbRo+Ak=\n", + "inclusion": [ + "vYU6vRfhcMyd5/U04n+AYg52mm3BQrjwyN+27saiSnU=", + "syWiyNbj6gybIdLKoAFd7xF3qNltS7OhQjtp4y8rCFg=", + "Zqpa9zJsOg/sI/mv0UkXO9d7gvPHLLjMAKnZCPzeiNM=", + "tGbcXa1AQlYBIm5gM2KDy8JvdAFDizj4lUop18Eehxs=", + "/w9FG1Sae/omKxZdOvj719sT1rCow3tlP8VNo0fDQtE=", + "dR6Bk+gh/hyJx7bu65HfhQdfuvT/OdVv/G7RUsOc26Y=", + "fRC86/2mWWtFWNPz/Vou6f7hdyvRLHFPo9KaOlVOVmc=", + "jJS2b3XVP2sst4CS5srpnF34agHpiluU63h6UQyFPDc=", + "cOL2+zAXTEOaAiyvmCrL3Ned6soAg08CJJJ2VW3+a18=", + "In5NzPUzIAZfV6bnUL11Bkm4Aqoit/ZG7gJ8nxOdB2w=", + "r+8IH0m7Ig3BS+uwCVVG1CZT0I/Rro3bbSUVYcSrN2Q=", + "wOgTamt00bdokE1Nbv3aHYu2um06V1xdLFEmOBjt0GI=", + "21woobbbCvogV/ZXACdAozfVAFjRv5xMID/iLcryHzw=", + "5+o2gn9/zVod+MYr4UuoeMzuz68vt/hA5+9H4YZaVME=", + "040B+RrLQeFCJhPU69kzoKr4BTqhSwdrnYEVhknZ+I0=", + "ioI6UByPBEr2lEZB0dI6pCXgcNss6UkKxm+TYAADujg=", + "HOJ0dLK4hnl0tFeu1tmT7TRrNKRIbi/T8KI/2s4DXDs=", + "BBcHtB4mS19jHff/LQTNI/+WCVBZ1Up52gNqyoln71U=", + "Oz/OlF1Lj7wkqqE15p324ZSX1XHcnLzDSMKNtUx6rwY=", + "ofpmTAcvFvNzlwjZnsrMqeG4iIBCP2lt5wIusvZ2gG4=", + "IRpVS0aap6n0qb02jyOy6ikkOPxdFIQtWwckaGF2MAQ=", + "liRusDUG3TanNZLGkfjb58ppX/6d40M7X2s3wptjxMw=", + "ROjUzjXHDN0Yo/I6EeqHhpszFgIs7FCABVeQniliPTk=", + "NTjIv8CJqIIGrOHhNQWSP9FV6XEPL6uGA4lmMfNKGJg=", + "bwqUQG2wJXGxo1QLA2CFb3chH17xvlqXe6eQy43p0HY=", + "bqa+4xGvzGcI/PYne2fXfUvdULgy8o3zQmD13q/gu1Y=" + ], + "consistency": [] + }] +} diff --git a/test/dever-cli-tests/external_libs/wheel.rs b/test/dever-cli-tests/external_libs/wheel.rs new file mode 100644 index 0000000..c53ca7e --- /dev/null +++ b/test/dever-cli-tests/external_libs/wheel.rs @@ -0,0 +1,531 @@ +use super::*; +use dever_cli::workers::python_wheel::Wheel; + +fn validate_native( + target: &str, + suffixes: &[String], + interpreter: &str, + files: &BTreeMap<String, &[u8]>, +) -> Result<(), String> { + let wheel_files = files + .keys() + .filter(|path| !path.starts_with("runtime/") && !path.starts_with("sandbox/")) + .cloned() + .collect(); + dever_cli::workers::python_wheel::validate_native( + target, + suffixes, + interpreter, + files, + &wheel_files, + ) +} + +fn spec() -> LibSpec { + "pip:sample@1.0.0".parse().unwrap() +} + +#[test] +fn wheel_console_scripts_use_the_same_private_prefix_and_record_generated_files() { + let bytes=wheel_fixture::pack("sample","1.0.0","Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + &wheel_fixture::metadata("sample","1.0.0",&[],&[]),&[("sample-1.0.0.dist-info/entry_points.txt",b"[console_scripts]\nsample-tool = sample.cli:main\n[gui_scripts]\nsample-gui = sample.gui:App.run [desktop]\n".to_vec())]); + let installed = Wheel::parse(&spec(), None, &tags(), &bytes) + .unwrap() + .install("runtime/bin/python3", "3.12") + .unwrap(); + let script = installed + .iter() + .find(|file| file.path == "runtime/bin/sample-tool") + .unwrap(); + assert!(script.executable); + assert!( + script + .bytes + .starts_with(b"#!/worker/runtime/bin/python3 -I\n") + ); + assert!(String::from_utf8_lossy(&script.bytes).contains("sample.cli")); + let record = installed + .iter() + .find(|file| file.path.ends_with(".dist-info/RECORD")) + .unwrap(); + assert!(String::from_utf8_lossy(&record.bytes).contains("../../../bin/sample-tool,sha256=")); + let collision = wheel_fixture::pack( + "sample", + "1.0.0", + "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + &wheel_fixture::metadata("sample", "1.0.0", &[], &[]), + &[ + ( + "sample-1.0.0.dist-info/entry_points.txt", + b"[console_scripts]\ntool = sample:main\n".to_vec(), + ), + ("sample-1.0.0.data/scripts/tool", b"#!python\n".to_vec()), + ], + ); + assert!( + Wheel::parse(&spec(), None, &tags(), &collision) + .unwrap() + .install("runtime/bin/python3", "3.12") + .is_err() + ); +} +fn tags() -> Vec<String> { + vec!["py3-none-any".into()] +} +fn valid() -> Vec<u8> { + pure_wheel("sample", "1.0.0", &[], &[]) +} +fn unpack(bytes: &[u8]) -> Vec<(String, Vec<u8>)> { + use std::io::Read; + let mut archive = zip::ZipArchive::new(Cursor::new(bytes)).unwrap(); + (0..archive.len()) + .map(|index| { + let mut file = archive.by_index(index).unwrap(); + let name = file.name().to_owned(); + let mut bytes = vec![]; + file.read_to_end(&mut bytes).unwrap(); + (name, bytes) + }) + .collect() +} + +#[test] +fn wheel_identity_and_all_compressed_filename_tags_are_bound_to_metadata() { + Wheel::parse( + &spec(), + Some("sample-1.0.0-py3-none-any.whl"), + &tags(), + &valid(), + ) + .unwrap(); + for filename in [ + "other-1.0.0-py3-none-any.whl", + "sample-2.0.0-py3-none-any.whl", + "sample-1.0.0-py2.py3-none-any.whl", + "sample-1.0.0-1-py3-none-any.whl", + ] { + assert!( + Wheel::parse(&spec(), Some(filename), &tags(), &valid()).is_err(), + "{filename}" + ); + } + let compressed = wheel_fixture::pack( + "sample", + "1.0.0", + "Wheel-Version: 1.0\nRoot-Is-Purelib: false\nTag: py2-none-any\nTag: py3-none-any\n", + &wheel_fixture::metadata("sample", "1.0.0", &[], &[]), + &[], + ); + Wheel::parse( + &spec(), + Some("sample-1.0.0-py2.py3-none-any.whl"), + &tags(), + &compressed, + ) + .unwrap(); + assert!( + Wheel::parse( + &spec(), + None, + &["cp312-cp312-linux_x86_64".into()], + &valid() + ) + .is_err() + ); + let forged_metadata = wheel_fixture::pack( + "sample", + "1.0.0", + "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + &wheel_fixture::metadata("other", "1.0.0", &[], &[]), + &[], + ); + assert!(Wheel::parse(&spec(), None, &tags(), &forged_metadata).is_err()); + let hidden_binary = wheel_fixture::pack( + "sample", + "1.0.0", + "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + &wheel_fixture::metadata("sample", "1.0.0", &[], &[]), + &[("sample/native.so", elf(62, None, None))], + ); + assert!(Wheel::parse(&spec(), None, &tags(), &hidden_binary).is_err()); +} + +#[test] +#[ignore = "author operation: explicitly fetches PyPI simplejson; requires target/python-managed/config/setting.json and runtime.pack"] +fn prepare_official_native_python_fixture() { + #[derive(serde::Deserialize)] + #[serde(deny_unknown_fields)] + struct Settings { + runtime: RegistryRuntime, + } + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../target/python-managed"); + let settings: Settings = + serde_json::from_slice(&fs::read(root.join("config/setting.json")).unwrap()).unwrap(); + let pack = fs::read(root.join("runtime.pack")).unwrap(); + assert_eq!(digest(&pack), settings.runtime.pack.sha256); + assert_eq!( + settings.runtime.target, + dever_cli::toolchain::platform_identity() + ); + let target = settings.runtime.target.clone(); + let store = FixtureArtifactStore::default(); + let transport = dever_cli::libs::HttpRegistry::official(); + let resolver = RegistryResolver { + build: None, + transport: &transport, + store: &store, + runtimes: BTreeMap::from([(Ecosystem::Pip, settings.runtime)]), + go_sumdb: dever_cli::libs::sumdb::ChecksumDatabase::official(None).unwrap(), + }; + let lock = dever_cli::libs::LibResolver::resolve( + &resolver, + &["pip:simplejson@3.20.1".parse().unwrap()], + ) + .unwrap(); + dever_cli::libs::doctor(&lock).unwrap(); + dever_cli::libs::verify_locked_artifacts(&lock, &store, &target).unwrap(); + let artifact = &lock.libs[0].artifacts[0]; + let wheel = store + .verify_exact(&artifact.sha256, artifact.bytes, &target) + .unwrap(); + assert!( + unpack(&wheel) + .iter() + .any(|(path, bytes)| path.contains("_speedups.") + && path.ends_with(".so") + && bytes.starts_with(b"\x7fELF")), + "fixture must use native speedups, not the optional pure-Python fallback" + ); + publish_fixture_artifacts(&root.join("dependency"), &lock, &store); +} + +#[test] +fn wheel_record_requires_exact_files_hashes_sizes_and_unique_csv_paths() { + for mutation in 0..6 { + let mut files = unpack(&valid()); + match mutation { + 0 => files + .iter_mut() + .find(|(path, _)| path.ends_with("__init__.py")) + .unwrap() + .1 + .push(b'!'), + 1 => files.push(("sample/unlisted.py".into(), vec![b'x'])), + 2 => { + files.retain(|(path, _)| !path.ends_with("__init__.py")); + } + _ => { + let record = &mut files + .iter_mut() + .find(|(path, _)| path.ends_with("/RECORD")) + .unwrap() + .1; + let mut content = String::from_utf8(record.clone()).unwrap(); + match mutation { + 3 => content = content.replace("sha256=", "sha1="), + 4 => content.push_str("sample-1.0.0.dist-info/RECORD,,\n"), + 5 => content.push_str("../outside.py,sha256=abc,1\n"), + _ => unreachable!(), + } + *record = content.into_bytes(); + } + } + assert!( + Wheel::parse(&spec(), None, &tags(), &wheel_fixture::zip(&files)).is_err(), + "mutation {mutation}" + ); + } +} + +#[test] +fn wheel_installs_all_standard_schemes_and_rewrites_installed_record() { + let bytes = wheel_fixture::pack( + "sample", + "1.0.0", + "Wheel-Version: 1.0\nRoot-Is-Purelib: false\nTag: py3-none-any\n", + &wheel_fixture::metadata("sample", "1.0.0", &[], &[]), + &[ + ("sample-1.0.0.data/purelib/plain.py", b"plain".to_vec()), + ( + "sample-1.0.0.data/platlib/platform.py", + b"platform".to_vec(), + ), + ( + "sample-1.0.0.data/scripts/tool", + b"#!python\nprint('ok')\n".to_vec(), + ), + ("sample-1.0.0.data/headers/public.h", b"header".to_vec()), + ( + "sample-1.0.0.data/data/share/sample/schema.json", + b"{}".to_vec(), + ), + ("sample/name,with\"quotes.py", b"quoted".to_vec()), + ], + ); + let files = Wheel::parse(&spec(), None, &tags(), &bytes) + .unwrap() + .install("runtime/bin/python3", "3.12") + .unwrap(); + for path in [ + "runtime/lib/python3.12/site-packages/plain.py", + "runtime/lib/python3.12/site-packages/platform.py", + "runtime/include/python3.12/sample/public.h", + "runtime/share/sample/schema.json", + "runtime/lib/python3.12/site-packages/sample/name,with\"quotes.py", + ] { + assert!(files.iter().any(|file| file.path == path), "{path}"); + } + let script = files + .iter() + .find(|file| file.path == "runtime/bin/tool") + .unwrap(); + assert!(script.executable); + assert!( + script + .bytes + .starts_with(b"#!/worker/runtime/bin/python3 -I\n") + ); + let record = files + .iter() + .find(|file| file.path.ends_with("/RECORD")) + .unwrap(); + let rows = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(record.bytes.as_slice()) + .records() + .collect::<Result<Vec<_>, _>>() + .unwrap(); + let installed_script = rows + .iter() + .find(|row| &row[0] == "../../../bin/tool") + .unwrap(); + assert_eq!( + installed_script[1].to_owned(), + format!( + "sha256={}", + base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(Sha256::digest(&script.bytes)) + ) + ); +} + +#[test] +fn wheel_rejects_scheme_collisions_and_uses_artifact_dependencies() { + let bytes = wheel_fixture::pack( + "sample", + "1.0.0", + "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + &wheel_fixture::metadata("sample", "1.0.0", &[], &[]), + &[ + ("same.py", vec![b'a']), + ("sample-1.0.0.data/platlib/same.py", vec![b'b']), + ], + ); + assert!( + Wheel::parse(&spec(), None, &tags(), &bytes) + .unwrap() + .install("runtime/bin/python3", "3.12") + .is_err() + ); + let mut registry = local_registry(); + let release = registry + .0 + .get_mut(&(Ecosystem::Pip, "/pypi/a/1.0.0/json".into())) + .unwrap(); + let mut document: serde_json::Value = serde_json::from_slice(release).unwrap(); + document["info"]["requires_dist"] = serde_json::json!([]); + *release = serde_json::to_vec(&document).unwrap(); + let lock = python_lock(®istry, &["pip:a@1.0.0"]).unwrap(); + assert_eq!(dependency_keys(&lock, "pip:a@1.0.0"), ["pip:b@1.1.0"]); +} + +#[test] +fn solver_backtracks_on_selected_wheel_metadata_and_its_actual_extras() { + let mut registry = LocalRegistry(BTreeMap::new()); + python_release( + &mut registry, + "root", + "1.0.0", + &[], + &["a[x] >=1", "b ==1.0.0"], + ); + python_release( + &mut registry, + "a", + "2.0.0", + &["x"], + &["b ==2.0.0; extra == 'x'"], + ); + python_release( + &mut registry, + "a", + "1.0.0", + &["x"], + &["b ==1.0.0; extra == 'x'"], + ); + python_release(&mut registry, "b", "2.0.0", &[], &[]); + python_release(&mut registry, "b", "1.0.0", &[], &[]); + for version in ["1.0.0", "2.0.0"] { + let json = registry + .0 + .get_mut(&(Ecosystem::Pip, format!("/pypi/a/{version}/json"))) + .unwrap(); + let mut release: serde_json::Value = serde_json::from_slice(json).unwrap(); + release["info"] = serde_json::json!({"requires_dist":null,"provides_extra":null}); + *json = serde_json::to_vec(&release).unwrap(); + } + let lock = python_lock(®istry, &["pip:root@1.0.0"]).unwrap(); + assert!( + lock.libs + .iter() + .any(|lib| lib.spec.key() == "pip:a[x]@1.0.0") + ); + assert!(!lock.libs.iter().any(|lib| lib.spec.version == "2.0.0")); + assert_eq!(dependency_keys(&lock, "pip:a[x]@1.0.0"), ["pip:b@1.0.0"]); +} + +/// ELF64 protocol fixture with PT_LOAD and optional PT_DYNAMIC/DT_STRTAB. +/// It is parsed only; no executable bytes are launched by this test. +fn elf(machine: u16, needed: Option<&str>, rpath: Option<&str>) -> Vec<u8> { + let mut bytes = vec![0u8; 1024]; + bytes[..7].copy_from_slice(b"\x7fELF\x02\x01\x01"); + bytes[16..18].copy_from_slice(&3u16.to_le_bytes()); + bytes[18..20].copy_from_slice(&machine.to_le_bytes()); + bytes[20..24].copy_from_slice(&1u32.to_le_bytes()); + bytes[32..40].copy_from_slice(&64u64.to_le_bytes()); + bytes[52..54].copy_from_slice(&64u16.to_le_bytes()); + bytes[54..56].copy_from_slice(&56u16.to_le_bytes()); + bytes[56..58].copy_from_slice(&2u16.to_le_bytes()); + bytes[64..68].copy_from_slice(&1u32.to_le_bytes()); + bytes[96..104].copy_from_slice(&1024u64.to_le_bytes()); + bytes[104..112].copy_from_slice(&1024u64.to_le_bytes()); + bytes[120..124].copy_from_slice(&2u32.to_le_bytes()); + bytes[128..136].copy_from_slice(&176u64.to_le_bytes()); + bytes[136..144].copy_from_slice(&176u64.to_le_bytes()); + let mut strings = vec![0]; + let mut dynamic = vec![]; + for (tag, value) in [(1u64, needed), (15u64, rpath)] { + if let Some(value) = value { + dynamic.push((tag, strings.len() as u64)); + strings.extend(value.bytes()); + strings.push(0); + } + } + dynamic.extend([(5, 512), (10, strings.len() as u64), (0, 0)]); + bytes[152..160].copy_from_slice(&((dynamic.len() * 16) as u64).to_le_bytes()); + bytes[160..168].copy_from_slice(&((dynamic.len() * 16) as u64).to_le_bytes()); + for (index, (tag, value)) in dynamic.into_iter().enumerate() { + bytes[176 + index * 16..184 + index * 16].copy_from_slice(&tag.to_le_bytes()); + bytes[184 + index * 16..192 + index * 16].copy_from_slice(&value.to_le_bytes()); + } + bytes[512..512 + strings.len()].copy_from_slice(&strings); + bytes +} + +#[test] +fn native_wheel_checks_target_suffix_and_closed_library_search() { + let runtime = elf(62, None, None); + let library = elf(62, None, None); + let native = elf(62, Some("libsample.so.1"), Some("$ORIGIN/../sample.libs")); + let suffixes = vec![ + ".cpython-312-x86_64-linux-gnu.so".into(), + ".abi3.so".into(), + ".so".into(), + ]; + let mut files = BTreeMap::from([ + ("runtime/bin/python3".into(), runtime.as_slice()), + ( + "site-packages/sample/_native.cpython-312-x86_64-linux-gnu.so".into(), + native.as_slice(), + ), + ( + "site-packages/sample.libs/libsample.so.1".into(), + library.as_slice(), + ), + ]); + validate_native("linux-x86_64", &suffixes, "runtime/bin/python3", &files).unwrap(); + files.remove("site-packages/sample.libs/libsample.so.1"); + assert!( + validate_native("linux-x86_64", &suffixes, "runtime/bin/python3", &files) + .unwrap_err() + .contains("packaged dependency closure") + ); + files.insert("sandbox/lib/libsample.so.1".into(), library.as_slice()); + validate_native("linux-x86_64", &suffixes, "runtime/bin/python3", &files).unwrap(); + let old = files + .remove("site-packages/sample/_native.cpython-312-x86_64-linux-gnu.so") + .unwrap(); + files.insert( + "site-packages/sample/_native.cpython-311-x86_64-linux-gnu.so".into(), + old, + ); + assert!( + validate_native("linux-x86_64", &suffixes, "runtime/bin/python3", &files) + .unwrap_err() + .contains("suffix") + ); + for (machine, rpath, message) in [ + (183, "$ORIGIN", "target"), + (62, "/usr/lib", "$ORIGIN"), + (62, "$ORIGIN/../../../..", "escapes"), + ] { + let rejected = elf(machine, None, Some(rpath)); + let files = BTreeMap::from([ + ("runtime/bin/python3".into(), runtime.as_slice()), + ("site-packages/sample/native.so".into(), rejected.as_slice()), + ]); + assert!( + validate_native("linux-x86_64", &suffixes, "runtime/bin/python3", &files) + .unwrap_err() + .contains(message) + ); + } +} + +#[test] +fn native_runpath_outranks_ancestor_rpath_and_helpers_keep_import_context() { + let runtime = elf(62, None, Some("$ORIGIN/../lib")); + let shared = elf(62, None, None); + let wrong_target = elf(183, None, None); + let mut extension = elf( + 62, + Some("libsample.so.1"), + Some("$ORIGIN/../../runtime/alternate"), + ); + // Replace DT_RPATH with DT_RUNPATH in the independently encoded ELF vector. + for offset in (176..256).step_by(16) { + if u64::from_le_bytes(extension[offset..offset + 8].try_into().unwrap()) == 15 { + extension[offset..offset + 8].copy_from_slice(&29u64.to_le_bytes()); + } + } + let suffixes = vec![".cpython-312-x86_64-linux-gnu.so".into()]; + let files = BTreeMap::from([ + ("runtime/bin/python3".into(), runtime.as_slice()), + ("runtime/lib/libsample.so.1".into(), shared.as_slice()), + ( + "runtime/alternate/libsample.so.1".into(), + wrong_target.as_slice(), + ), + ( + "site-packages/pkg/mod.cpython-312-x86_64-linux-gnu.so".into(), + extension.as_slice(), + ), + ]); + assert!( + validate_native("linux-x86_64", &suffixes, "runtime/bin/python3", &files) + .unwrap_err() + .contains("target") + ); + + let extension = elf(62, Some("liba.so.1"), Some("$ORIGIN/../vendor")); + let helper = elf(62, Some("libb.so.1"), None); + let files = BTreeMap::from([ + ("runtime/bin/python3".into(), runtime.as_slice()), + ( + "site-packages/pkg/mod.cpython-312-x86_64-linux-gnu.so".into(), + extension.as_slice(), + ), + ("site-packages/vendor/liba.so.1".into(), helper.as_slice()), + ("site-packages/vendor/libb.so.1".into(), shared.as_slice()), + ]); + validate_native("linux-x86_64", &suffixes, "runtime/bin/python3", &files).unwrap(); +} diff --git a/test/dever-cli-tests/external_workers.rs b/test/dever-cli-tests/external_workers.rs new file mode 100644 index 0000000..44ef638 --- /dev/null +++ b/test/dever-cli-tests/external_workers.rs @@ -0,0 +1,1299 @@ +#[path = "support/native_elf.rs"] +#[allow( + dead_code, + reason = "This test crate uses only the shared static ELF fixture" +)] +mod native_elf; +#[path = "../dever-tests/tests/support/sandbox.rs"] +mod sandbox; +#[path = "../dever-tests/tests/support/temp.rs"] +mod temp; +#[path = "support/wheel.rs"] +mod wheel_fixture; + +use std::fs; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::mpsc; +use std::thread; +use std::time::{Duration, Instant}; + +use dever_cli::libs::{LibSpec, LockFile, LockedArtifact, LockedLib, LockedWorker, RuntimePack}; +use dever_cli::workers; +use dever_core::native::EmbeddedResource; +use dever_core::source::SourceMap; +use flate2::Compression; +use flate2::write::GzEncoder; +use serde_json::json; +use sha2::{Digest, Sha256}; +use temp::TemporaryDirectory; + +#[path = "external_workers/command.rs"] +mod command; + +#[cfg(unix)] +struct OwnedProcessGroup(std::process::Child); + +#[cfg(unix)] +impl OwnedProcessGroup { + fn stop(&mut self) { + if self.0.try_wait().ok().flatten().is_none() + && let Some(pid) = rustix::process::Pid::from_raw(self.0.id() as i32) + { + let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL); + } + let _ = self.0.wait(); + } +} + +#[cfg(unix)] +impl Drop for OwnedProcessGroup { + fn drop(&mut self) { + self.stop(); + } +} + +fn sha(bytes: &[u8]) -> String { + Sha256::digest(bytes) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +fn archive(files: &[(&str, &[u8])]) -> Vec<u8> { + let encoder = GzEncoder::new(Vec::new(), Compression::none()); + let mut tar = tar::Builder::new(encoder); + for (name, bytes) in files { + let mut header = tar::Header::new_gnu(); + header.set_size(bytes.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + tar.append_data(&mut header, name, *bytes).unwrap(); + } + tar.into_inner().unwrap().finish().unwrap() +} + +fn wheel(metadata: &str) -> Vec<u8> { + wheel_fixture::pack( + "fixture", + "1.0.0", + &format!("{metadata}Tag: py3-none-any\n"), + &wheel_fixture::metadata("fixture", "1.0.0", &[], &["extra"]), + &[("fixture/__init__.py", b"value = 1\n".to_vec())], + ) +} + +fn resource(path: String, bytes: Vec<u8>) -> EmbeddedResource { + EmbeddedResource { + sha256: sha(&bytes), + path, + bytes, + executable: false, + } +} + +fn prepare_sandboxed_worker( + root: &Path, + program: &dever_core::hir::Program, + mut resources: Vec<EmbeddedResource>, +) -> Vec<EmbeddedResource> { + let workspace = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + for asset in sandbox::assets(&workspace) { + let mut file = resource(asset.path, asset.bytes); + file.executable = asset.executable; + resources.push(file); + } + let mut generated = workers::prepare(root, program, &resources).unwrap(); + generated.extend( + resources + .into_iter() + .filter(|resource| resource.path.starts_with("sandbox/")), + ); + generated +} + +fn frame(value: serde_json::Value) -> Vec<u8> { + let body = serde_json::to_vec(&value).unwrap(); + let mut bytes = (body.len() as u32).to_be_bytes().to_vec(); + bytes.extend(body); + bytes +} + +fn response(stream: &mut impl Read) -> Option<serde_json::Value> { + let mut length = [0; 4]; + stream.read_exact(&mut length).ok()?; + let mut body = vec![0; u32::from_be_bytes(length) as usize]; + stream.read_exact(&mut body).ok()?; + serde_json::from_slice(&body).ok() +} + +fn fixture( + root: &Path, + ecosystem: &str, + binary: &[u8], + package_metadata: Option<&str>, +) -> (dever_core::hir::Program, Vec<EmbeddedResource>) { + let entry = match ecosystem { + "npm" => "worker/main.mjs", + "pip" => "worker/main.py", + _ => panic!("unsupported fixture ecosystem"), + }; + fixture_entry(root, ecosystem, binary, package_metadata, entry) +} + +fn fixture_entry( + root: &Path, + ecosystem: &str, + binary: &[u8], + package_metadata: Option<&str>, + entry: &str, +) -> (dever_core::hir::Program, Vec<EmbeddedResource>) { + let (program, resources, _) = + fixture_entry_with_sources(root, ecosystem, binary, package_metadata, entry); + (program, resources) +} + +fn fixture_entry_with_sources( + root: &Path, + ecosystem: &str, + binary: &[u8], + package_metadata: Option<&str>, + entry: &str, +) -> (dever_core::hir::Program, Vec<EmbeddedResource>, SourceMap) { + fixture_for_target( + root, + ecosystem, + binary, + package_metadata, + entry, + dever_cli::toolchain::BuildTarget::host().unwrap(), + ) +} + +fn fixture_for_target( + root: &Path, + ecosystem: &str, + binary: &[u8], + package_metadata: Option<&str>, + entry: &str, + target: dever_cli::toolchain::BuildTarget, +) -> (dever_core::hir::Program, Vec<EmbeddedResource>, SourceMap) { + let (executable, source) = match ecosystem { + "npm" if entry.ends_with(".cjs") => ( + "bin/node", + "module.exports = { send: ({ value }) => ({ receipt: value + 1 }) };\n", + ), + "npm" => ( + "bin/node", + "export const send = ({ value }) => ({ receipt: value + 1 });\n", + ), + "pip" => ( + "bin/python3", + "async def send(payload, setting):\n return {'receipt': payload['value'] + 1}\n", + ), + _ => panic!("unsupported fixture ecosystem"), + }; + let library = if package_metadata.is_some() { + "lib \"fixture@1.0.0\"" + } else { + "" + }; + let sources = worker_sources(ecosystem, entry, library); + let program = dever_core::check(&sources).unwrap_or_else(|errors| panic!("{errors:?}")); + let contract = program.external_worker_contracts().remove(0); + let entry_path = root.join(&contract.entry); + fs::create_dir_all(entry_path.parent().unwrap()).unwrap(); + fs::write(entry_path, source).unwrap(); + let target = target.platform(); + let arguments = if ecosystem == "pip" { + vec!["-I", "-S", "-B"] + } else { + vec![] + }; + let mut metadata = json!({"format":"dever-worker-runtime-v1","ecosystem":ecosystem,"target":target, + "executable":executable,"arguments":arguments}); + if ecosystem == "pip" { + metadata["python_wheel_tags"] = json!(["py3-none-any"]); + let architecture = target.strip_prefix("linux-").unwrap(); + metadata["python_extension_suffixes"] = json!([ + format!(".cpython-312-{architecture}-linux-gnu.so"), + ".abi3.so", + ".so" + ]); + let mut markers = serde_json::to_value(wheel_fixture::markers()).unwrap(); + markers["platform_machine"] = json!(architecture); + metadata["python_markers"] = markers; + } + let pack = archive(&[ + ( + "dever-runtime.json", + &serde_json::to_vec(&metadata).unwrap(), + ), + (executable, binary), + ]); + let runtime = RuntimePack { + name: format!("fixture-{ecosystem}"), + version: "1".into(), + sha256: sha(&pack), + }; + let mut archives = Vec::new(); + let mut libraries = Vec::new(); + if let Some(package_metadata) = package_metadata { + let spec: LibSpec = format!("{ecosystem}:fixture@1.0.0").parse().unwrap(); + let (bytes, extension) = match ecosystem { + "npm" => ( + archive(&[ + ("package/package.json", package_metadata.as_bytes()), + ("package/index.js", b"export const value = 1;"), + ]), + "tgz", + ), + "pip" => (wheel(package_metadata), "whl"), + _ => panic!("unsupported fixture package"), + }; + let path = format!( + "lib/{ecosystem}/{}/1.0.0/archive.{extension}", + sha(b"fixture") + ); + libraries.push(LockedLib { + build: None, + spec: spec.clone(), + dependencies: vec![], + runtime: runtime.clone(), + artifacts: vec![LockedArtifact { + target: target.into(), + path: path.clone(), + bytes: bytes.len() as u64, + sha256: sha(&bytes), + }], + schema: contract.schema.clone(), + }); + archives.push(resource(path, bytes)); + } + let mut lock = LockFile::new(libraries).unwrap(); + if ecosystem == "npm" && package_metadata.is_some() { + lock.npm.push(dever_cli::libs::npm::Environment::single( + lock.libs[0].spec.clone(), + )); + } + lock.workers.push(LockedWorker { + port: contract.port, + adapter: contract.adapter, + ecosystem: contract.ecosystem, + runtime: Some(runtime), + entry: contract.entry, + schema: contract.schema, + capabilities: contract.capabilities, + operations: contract.operations, + libs: contract + .libs + .iter() + .map(|value| value.parse().unwrap()) + .collect(), + }); + lock.write_atomic(root).unwrap(); + archives.push(resource( + format!("lib/runtime/{ecosystem}/{target}/runtime.pack"), + pack, + )); + (program, archives, sources) +} + +fn worker_sources(ecosystem: &str, entry: &str, libraries: &str) -> SourceMap { + let mut sources = SourceMap::default(); + sources.add( + "mail/send/port.dever", + "send(value: Int) (receipt: Int) fails app.SendResult", + ); + sources.add("mail/send/app.dever", "type SendResult { error Unavailable(message: Text) }\nsend(value: Int) (receipt: Int) { receipt = port.send(value) }"); + sources.add("mail/send/api.dever", "cmd send = app.send"); + sources.add( + "mail/send/adapter.dever", + format!("external {ecosystem} \"{entry}\" {{ {libraries} }}"), + ); + sources +} + +#[test] +fn packages_checked_javascript_worker_without_host_resolution() { + let temp = TemporaryDirectory::new(); + let (program, resources) = fixture(temp.path(), "npm", &native_elf::static_executable(), None); + let generated = workers::prepare(temp.path(), &program, &resources).unwrap(); + let entry = &program.external_worker_contracts()[0].entry; + let launch = generated + .iter() + .find(|resource| resource.path == format!("{entry}.dever-worker.json")) + .unwrap(); + let manifest: serde_json::Value = serde_json::from_slice(&launch.bytes).unwrap(); + let executable = manifest["executable"].as_str().unwrap(); + let runner = manifest["arguments"].as_array().unwrap().last().unwrap()["resource"] + .as_str() + .unwrap(); + assert!( + generated + .iter() + .any(|resource| resource.path == executable && resource.executable) + ); + assert!(generated.iter().any(|resource| resource.path == runner)); + assert_eq!(manifest["entry"], entry.as_str()); + assert_eq!(manifest["ecosystem"], "npm"); + assert!( + generated + .iter() + .any(|resource| resource.path.ends_with("/contract.json")) + ); + + assert!( + workers::prepare(temp.path(), &program, &[]) + .unwrap_err() + .contains("runtime pack is missing") + ); +} + +#[test] +fn cross_target_interpreters_and_dependency_archives_remain_offline_and_target_bound() { + use dever_cli::toolchain::BuildTarget; + let target = BuildTarget::LinuxAarch64; + let mut arm = native_elf::static_executable(); + arm[18..20].copy_from_slice(&183u16.to_le_bytes()); + for (ecosystem, entry, metadata) in [ + ( + "pip", + "worker/main.py", + "Wheel-Version: 1.0\nRoot-Is-Purelib: true\n", + ), + ( + "npm", + "worker/main.mjs", + r#"{"name":"fixture","version":"1.0.0"}"#, + ), + ] { + let temp = TemporaryDirectory::new(); + let (program, resources, _) = + fixture_for_target(temp.path(), ecosystem, &arm, Some(metadata), entry, target); + let generated = + workers::prepare_for_target(temp.path(), &program, &resources, target, &[]).unwrap(); + let binary = generated + .iter() + .find(|file| file.executable && file.path.contains("/runtime/bin/")) + .unwrap(); + assert_eq!( + goblin::elf::Elf::parse(&binary.bytes) + .unwrap() + .header + .e_machine, + 183 + ); + assert!( + generated + .iter() + .any(|file| file.path.contains(if ecosystem == "pip" { + "site-packages/fixture/__init__.py" + } else { + "node_modules/fixture/index.js" + })) + ); + assert!( + workers::prepare_for_target( + temp.path(), + &program, + &resources, + BuildTarget::LinuxX86_64, + &[] + ) + .unwrap_err() + .contains("runtime pack is missing") + ); + + let mut lock = + LockFile::decode(&fs::read(temp.path().join("dever.lock")).unwrap()).unwrap(); + lock.libs[0].artifacts[0].target = "linux-x86_64".into(); + lock.write_atomic(temp.path()).unwrap(); + assert!( + workers::prepare_for_target(temp.path(), &program, &resources, target, &[]) + .unwrap_err() + .contains("archive for linux-aarch64 is missing") + ); + } +} + +#[test] +fn cross_target_runtime_rejects_host_binary_even_with_matching_pack_hash() { + use dever_cli::toolchain::BuildTarget; + let mut host = native_elf::static_executable(); + host[18..20].copy_from_slice(&62u16.to_le_bytes()); + for (ecosystem, entry) in [("pip", "worker/main.py"), ("npm", "worker/main.mjs")] { + let temp = TemporaryDirectory::new(); + let (program, resources, _) = fixture_for_target( + temp.path(), + ecosystem, + &host, + None, + entry, + BuildTarget::LinuxAarch64, + ); + assert!( + workers::prepare_for_target( + temp.path(), + &program, + &resources, + BuildTarget::LinuxAarch64, + &[] + ) + .unwrap_err() + .contains("differs from target linux-aarch64") + ); + } +} + +#[cfg(target_os = "linux")] +#[test] +#[ignore = "requires explicit Go runtime pack, official UUID proof/archive and sandbox assets"] +fn compiles_go_worker_with_locked_tools_and_runs_without_source_or_go() { + use std::os::unix::fs::PermissionsExt; + + let temp = TemporaryDirectory::new(); + let mut sources = SourceMap::default(); + sources.add( + "mail/send/port.dever", + "send(value: Int) (receipt: Int) fails app.SendResult", + ); + sources.add("mail/send/app.dever", "type SendResult { error Unavailable(message: Text) }\nsend(value: Int) (receipt: Int) { receipt = port.send(value) }"); + sources.add("mail/send/api.dever", "cmd send = app.send"); + sources.add( + "mail/send/adapter.dever", + "external go \"worker/main.go\" { lib \"github.com/google/uuid@1.6.0\" }", + ); + let program = dever_core::check(&sources).unwrap(); + let contract = program.external_worker_contracts().remove(0); + let entry = temp.path().join(&contract.entry); + fs::create_dir_all(entry.parent().unwrap()).unwrap(); + fs::write(&entry, r#"package main +import ( + "context" + "encoding/json" + _ "embed" + "github.com/google/uuid" +) +//go:embed note.txt +var note string +func Send(_ context.Context, payload any, _ any) (any, error) { + value := payload.(map[string]any)["value"].(json.Number) + number, err := value.Int64() + if err != nil { return nil, err } + parsed, err := uuid.Parse("00000000-0000-0000-0000-000000000001") + if err != nil { return nil, err } + return map[string]any{"receipt": number + int64(parsed[15]) + TargetBonus() + int64(len(note)) - 1}, nil +} +"#).unwrap(); + fs::write(entry.with_file_name("note.txt"), "x").unwrap(); + fs::write( + entry.with_file_name("target_linux.go"), + "//go:build linux\npackage main\nfunc TargetBonus() int64 { return 0 }\n", + ) + .unwrap(); + fs::write( + entry.with_file_name("target_windows.go"), + "//go:build windows\npackage main\nfunc TargetBonus() int64 { return 100 }\n", + ) + .unwrap(); + let workspace = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let prepared = workspace.join("target/go-managed"); + let pack = fs::read(prepared.join("runtime.pack")).unwrap(); + let target = dever_cli::toolchain::platform_identity(); + let mut lock = + LockFile::decode(&fs::read(prepared.join("dependency/dever.lock")).unwrap()).unwrap(); + assert_eq!(lock.libs.len(), 1); + let dependency = &mut lock.libs[0]; + assert_eq!(dependency.spec.key(), "go:github.com/google/uuid@1.6.0"); + assert_eq!(dependency.runtime.sha256, sha(&pack)); + dependency.schema = contract.schema.clone(); + let spec = dependency.spec.clone(); + let runtime = dependency.runtime.clone(); + let mut resources = vec![resource( + format!("lib/runtime/go/{target}/runtime.pack"), + pack, + )]; + for artifact in &dependency.artifacts { + let bytes = fs::read(prepared.join("dependency/artifacts").join(&artifact.sha256)).unwrap(); + assert_eq!(sha(&bytes), artifact.sha256); + resources.push(resource(artifact.path.clone(), bytes)); + } + for asset in sandbox::assets(&workspace) { + let mut file = resource(asset.path, asset.bytes); + file.executable = asset.executable; + resources.push(file); + } + lock.workers.push(LockedWorker { + port: contract.port.clone(), + adapter: contract.adapter.clone(), + ecosystem: "go".into(), + runtime: Some(runtime), + entry: contract.entry.clone(), + schema: contract.schema.clone(), + capabilities: contract.capabilities.clone(), + operations: contract.operations.clone(), + libs: vec![spec], + }); + lock.write_atomic(temp.path()).unwrap(); + let generated = workers::prepare(temp.path(), &program, &resources).unwrap(); + assert!( + !generated + .iter() + .any(|resource| resource.path.contains("/source/") + || resource.path.contains("/runtime/")) + ); + let binary = generated + .iter() + .find(|resource| resource.path.ends_with("/worker") && resource.executable) + .unwrap(); + let rebuilt = workers::prepare(temp.path(), &program, &resources).unwrap(); + let identities = |files: &[EmbeddedResource]| { + files + .iter() + .map(|file| (file.path.clone(), file.sha256.clone(), file.executable)) + .collect::<Vec<_>>() + }; + assert_eq!( + identities(&generated), + identities(&rebuilt), + "fresh Go build directories must produce identical Worker resources" + ); + let working_directory = temp + .path() + .join(binary.path.strip_suffix("/worker").unwrap()); + for file in generated.iter().chain( + resources + .iter() + .filter(|file| file.path.starts_with("sandbox/")), + ) { + let path = temp.path().join(&file.path); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(&path, &file.bytes).unwrap(); + if file.executable { + fs::set_permissions(path, fs::Permissions::from_mode(0o700)).unwrap(); + } + } + fs::remove_dir_all(entry.parent().unwrap()).unwrap(); + let child = dever_sandbox::Launch { + assets: &temp.path().join("sandbox"), + worker: &working_directory, + executable: &temp.path().join(&binary.path), + arguments: &[], + working_directory: &working_directory, + capabilities: dever_sandbox::Capabilities::default(), + grants: &[], + } + .command() + .unwrap() + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + assert_fixture_protocol(child, &contract); +} + +#[test] +fn packages_zero_lib_python_worker_with_checked_manifest() { + let temp = TemporaryDirectory::new(); + let (program, resources) = fixture(temp.path(), "pip", &native_elf::static_executable(), None); + let generated = workers::prepare(temp.path(), &program, &resources).unwrap(); + let contract = program.external_worker_contracts().remove(0); + let launch = generated + .iter() + .find(|resource| resource.path == format!("{}.dever-worker.json", contract.entry)) + .unwrap(); + let manifest: serde_json::Value = serde_json::from_slice(&launch.bytes).unwrap(); + assert_eq!(manifest["ecosystem"], "pip"); + assert!( + manifest["executable"] + .as_str() + .unwrap() + .ends_with("/runtime/bin/python3") + ); + assert!( + generated + .iter() + .any(|resource| resource.path.ends_with("/dever_component.py")) + ); + assert!( + generated + .iter() + .any(|resource| resource.path.ends_with("/runner.py")) + ); +} + +#[test] +fn installs_locked_npm_package_and_rejects_lifecycle_hooks() { + let pure = TemporaryDirectory::new(); + let (program, resources) = fixture( + pure.path(), + "npm", + &native_elf::static_executable(), + Some(r#"{"name":"fixture","version":"1.0.0","type":"module"}"#), + ); + let generated = workers::prepare(pure.path(), &program, &resources).unwrap(); + assert!(generated.iter().any(|resource| { + resource + .path + .ends_with("/node_modules/fixture/package.json") + })); + assert!( + generated + .iter() + .any(|resource| resource.path.ends_with("/node_modules/fixture/index.js")) + ); + + let hooked = TemporaryDirectory::new(); + let (program, resources) = fixture( + hooked.path(), + "npm", + &native_elf::static_executable(), + Some(r#"{"name":"fixture","version":"1.0.0","scripts":{"install":"node install.js"}}"#), + ); + assert!( + workers::prepare(hooked.path(), &program, &resources) + .unwrap_err() + .contains("lifecycle scripts") + ); + + let collision = TemporaryDirectory::new(); + let (program, mut resources) = fixture( + collision.path(), + "npm", + &native_elf::static_executable(), + Some(r#"{"name":"fixture","version":"1.0.0"}"#), + ); + let archive_resource = resources + .iter_mut() + .find(|resource| resource.path.ends_with("archive.tgz")) + .unwrap(); + *archive_resource = resource( + archive_resource.path.clone(), + archive(&[ + ( + "package/package.json", + br#"{"name":"fixture","version":"1.0.0"}"#, + ), + ("package/overlap", b"file"), + ("package/overlap/child.js", b"file"), + ]), + ); + let mut lock = + LockFile::decode(&fs::read(collision.path().join("dever.lock")).unwrap()).unwrap(); + lock.libs[0].artifacts[0].sha256 = archive_resource.sha256.clone(); + lock.libs[0].artifacts[0].bytes = archive_resource.bytes.len() as u64; + lock.write_atomic(collision.path()).unwrap(); + assert!( + workers::prepare(collision.path(), &program, &resources) + .unwrap_err() + .contains("file/directory conflict") + ); +} + +#[test] +fn installs_purelib_and_platlib_python_wheels() { + let pure = TemporaryDirectory::new(); + let (program, resources) = fixture( + pure.path(), + "pip", + &native_elf::static_executable(), + Some("Wheel-Version: 1.0\nRoot-Is-Purelib: true\n"), + ); + let generated = workers::prepare(pure.path(), &program, &resources).unwrap(); + assert!(generated.iter().any(|resource| { + resource + .path + .ends_with("/site-packages/fixture/__init__.py") + })); + + let platform = TemporaryDirectory::new(); + let (program, resources) = fixture( + platform.path(), + "pip", + &native_elf::static_executable(), + Some("Wheel-Version: 1.0\nRoot-Is-Purelib: false\n"), + ); + assert!( + workers::prepare(platform.path(), &program, &resources) + .unwrap() + .iter() + .any(|resource| resource + .path + .ends_with("/site-packages/fixture/__init__.py")) + ); +} + +#[test] +#[ignore = "requires the explicit official native-wheel and complete Python runtime fixture at target/python-managed"] +fn runs_official_native_python_wheel_in_sandbox() { + run_native_python_fixture("python-managed"); +} + +#[test] +#[ignore = "requires explicitly prepared official PEP517 source build at target/source-build"] +fn runs_official_python_source_build_in_sandbox() { + run_native_python_fixture("source-build"); +} + +fn run_native_python_fixture(fixture: &str) { + let temp = TemporaryDirectory::new(); + let sources = worker_sources( + "pip", + "worker/main.py", + "lib \"simplejson@3.20.1\"\nlib \"prefixfixture@1.0.0\"", + ); + let program = dever_core::check(&sources).unwrap(); + let contract = program.external_worker_contracts().remove(0); + let entry = temp.path().join(&contract.entry); + fs::create_dir_all(entry.parent().unwrap()).unwrap(); + fs::write(&entry, "import simplejson\nimport prefixfixture\nfrom simplejson import _speedups\nassert _speedups.__file__.endswith('.so')\nassert prefixfixture.VALUE == 'prefix-ok'\nasync def send(payload, setting):\n value = simplejson.loads(simplejson.dumps(payload))\n return {'receipt': value['value'] + 1}\n").unwrap(); + let workspace = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let prepared = workspace.join("target").join(fixture); + let config: serde_json::Value = + serde_json::from_slice(&fs::read(prepared.join("config/setting.json")).unwrap()).unwrap(); + let runtime_file = config + .get("runtime_file") + .and_then(serde_json::Value::as_str) + .unwrap_or("runtime.pack"); + let pack = fs::read(prepared.join(runtime_file)).unwrap(); + let mut lock = + LockFile::decode(&fs::read(prepared.join("dependency/dever.lock")).unwrap()).unwrap(); + assert_eq!(lock.libs[0].spec.key(), "pip:simplejson@3.20.1"); + assert_eq!(lock.libs[0].runtime.sha256, sha(&pack)); + let runtime = lock.libs[0].runtime.clone(); + let mut resources = vec![resource( + format!( + "lib/runtime/pip/{}/runtime.pack", + dever_cli::toolchain::platform_identity() + ), + pack, + )]; + for library in &mut lock.libs { + library.schema = contract.schema.clone(); + for artifact in &library.artifacts { + let bytes = + fs::read(prepared.join("dependency/artifacts").join(&artifact.sha256)).unwrap(); + assert_eq!(sha(&bytes), artifact.sha256); + resources.push(resource(artifact.path.clone(), bytes)); + } + } + let probe = wheel_fixture::pack("prefixfixture", "1.0.0", + "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + &wheel_fixture::metadata("prefixfixture", "1.0.0", &[], &[]), &[ + ("prefixfixture/__init__.py", b"import sys, sysconfig\nfrom pathlib import Path\nassert sysconfig.get_path('data') == sys.prefix\nVALUE = (Path(sys.prefix) / 'share/prefixfixture/message.txt').read_text()\n".to_vec()), + ("prefixfixture-1.0.0.data/data/share/prefixfixture/message.txt", b"prefix-ok".to_vec()), + ("prefixfixture-1.0.0.data/scripts/prefix-probe", b"#!python\nimport prefixfixture\nprint(prefixfixture.VALUE)\n".to_vec()), + ]); + let probe_path = format!("lib/pip/{}/1.0.0/archive.whl", sha(b"prefixfixture")); + lock.libs.push(LockedLib { + build: None, + spec: "pip:prefixfixture@1.0.0".parse().unwrap(), + dependencies: vec![], + runtime: runtime.clone(), + artifacts: vec![LockedArtifact { + target: dever_cli::toolchain::platform_identity(), + path: probe_path.clone(), + bytes: probe.len() as u64, + sha256: sha(&probe), + }], + schema: contract.schema.clone(), + }); + resources.push(resource(probe_path, probe)); + lock.workers.push(LockedWorker { + port: contract.port.clone(), + adapter: contract.adapter.clone(), + ecosystem: "pip".into(), + runtime: Some(runtime), + entry: contract.entry.clone(), + schema: contract.schema.clone(), + capabilities: contract.capabilities.clone(), + operations: contract.operations.clone(), + libs: contract + .libs + .iter() + .map(|spec| spec.parse().unwrap()) + .collect(), + }); + lock.write_atomic(temp.path()).unwrap(); + let generated = prepare_sandboxed_worker(temp.path(), &program, resources); + assert!( + generated + .iter() + .any(|resource| resource.path.contains("_speedups.") + && resource.bytes.starts_with(b"\x7fELF")) + ); + let (deployment, launch) = run_packaged_protocol(&temp, &contract, "pip", &generated); + let output = dever_sandbox::Launch { + assets: &deployment.join("sandbox"), + worker: &launch.working_directory, + executable: &launch.working_directory.join("runtime/bin/prefix-probe"), + arguments: &[], + working_directory: &launch.working_directory, + capabilities: dever_sandbox::Capabilities::default(), + grants: &[], + } + .command() + .unwrap() + .output() + .unwrap(); + assert!( + output.status.success(), + "standalone wheel script failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(output.stdout, b"prefix-ok\n"); +} + +fn run_packaged_protocol( + temp: &TemporaryDirectory, + contract: &dever_core::hir::ExternalWorkerContract, + ecosystem: &str, + generated: &[EmbeddedResource], +) -> (PathBuf, dever_runtime::external::WorkerLaunch) { + let borrowed = generated + .iter() + .map(|resource| dever_runtime::external::Resource { + path: &resource.path, + bytes: &resource.bytes, + sha256: &resource.sha256, + executable: resource.executable, + }) + .collect::<Vec<_>>(); + let deployment = dever_runtime::external::extract( + temp.path(), + &dever_runtime::external::bundle_digest(&borrowed), + &borrowed, + ) + .unwrap(); + let launch = dever_runtime::external::verified_worker_launch( + &deployment, + &borrowed, + &contract.entry, + ecosystem, + ) + .unwrap(); + let child = dever_sandbox::Launch { + assets: &deployment.join("sandbox"), + worker: &launch.working_directory, + executable: &launch.executable, + arguments: &launch.arguments, + working_directory: &launch.working_directory, + capabilities: dever_sandbox::Capabilities::default(), + grants: &[], + } + .command() + .unwrap() + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + assert_fixture_protocol(child, contract); + (deployment, launch) +} + +#[path = "external_workers/npm.rs"] +mod npm_native; + +#[test] +fn installs_one_wheel_for_base_and_extra_variants() { + let temp = TemporaryDirectory::new(); + let (_, resources) = fixture_entry( + temp.path(), + "pip", + &native_elf::static_executable(), + Some("Wheel-Version: 1.0\nRoot-Is-Purelib: true\n"), + "worker/main.py", + ); + let sources = worker_sources( + "pip", + "worker/main.py", + "lib \"fixture@1.0.0\"\nlib \"fixture[extra]@1.0.0\"", + ); + let program = dever_core::check(&sources).unwrap_or_else(|errors| panic!("{errors:?}")); + let contract = program.external_worker_contracts().remove(0); + let mut lock = LockFile::decode(&fs::read(temp.path().join("dever.lock")).unwrap()).unwrap(); + let mut extra = lock.libs[0].clone(); + extra.spec = "pip:fixture[extra]@1.0.0".parse().unwrap(); + lock.libs.push(extra); + lock.workers[0].schema = contract.schema.clone(); + for library in &mut lock.libs { + library.schema = contract.schema.clone(); + } + lock.workers[0].libs = contract + .libs + .iter() + .map(|spec| spec.parse().unwrap()) + .collect(); + lock.write_atomic(temp.path()).unwrap(); + let generated = workers::prepare(temp.path(), &program, &resources).unwrap(); + assert_eq!( + generated + .iter() + .filter(|resource| resource + .path + .ends_with("/site-packages/fixture/__init__.py")) + .count(), + 1 + ); + + lock.libs[1].artifacts[0].sha256 = "0".repeat(64); + lock.write_atomic(temp.path()).unwrap(); + assert!(workers::prepare(temp.path(), &program, &resources).is_err()); +} + +#[test] +fn rejects_runtime_identity_and_adapter_source_links() { + let temp = TemporaryDirectory::new(); + let (program, mut resources) = + fixture(temp.path(), "npm", &native_elf::static_executable(), None); + let target = dever_cli::toolchain::platform_identity(); + let invalid = json!({"format":"dever-worker-runtime-v1","ecosystem":"pip","target":target, + "executable":"bin/node","arguments":[]}); + let pack = archive(&[ + ("dever-runtime.json", &serde_json::to_vec(&invalid).unwrap()), + ("bin/node", &native_elf::static_executable()), + ]); + resources[0] = resource(resources[0].path.clone(), pack); + let mut lock = LockFile::decode(&fs::read(temp.path().join("dever.lock")).unwrap()).unwrap(); + lock.workers[0].runtime.as_mut().unwrap().sha256 = resources[0].sha256.clone(); + lock.write_atomic(temp.path()).unwrap(); + assert!( + workers::prepare(temp.path(), &program, &resources) + .unwrap_err() + .contains("runtime pack identity") + ); + + #[cfg(unix)] + { + use std::os::unix::fs::symlink; + let linked = TemporaryDirectory::new(); + let (linked_program, linked_resources) = + fixture(linked.path(), "npm", &native_elf::static_executable(), None); + let entry = &linked_program.external_worker_contracts()[0].entry; + symlink( + linked.path().join(entry), + linked.path().join(entry).with_file_name("helper.mjs"), + ) + .unwrap(); + assert!( + workers::prepare(linked.path(), &linked_program, &linked_resources) + .unwrap_err() + .contains("symbolic link") + ); + } +} + +#[test] +fn rejects_python_runtime_without_isolation_flags() { + let temp = TemporaryDirectory::new(); + let (program, mut resources) = + fixture(temp.path(), "pip", &native_elf::static_executable(), None); + let target = dever_cli::toolchain::platform_identity(); + let metadata = json!({"format":"dever-worker-runtime-v1","ecosystem":"pip","target":target, + "executable":"bin/python3","arguments":[]}); + let pack = archive(&[ + ( + "dever-runtime.json", + &serde_json::to_vec(&metadata).unwrap(), + ), + ("bin/python3", &native_elf::static_executable()), + ]); + resources[0] = resource(resources[0].path.clone(), pack); + let mut lock = LockFile::decode(&fs::read(temp.path().join("dever.lock")).unwrap()).unwrap(); + lock.workers[0].runtime.as_mut().unwrap().sha256 = resources[0].sha256.clone(); + lock.write_atomic(temp.path()).unwrap(); + assert!( + workers::prepare(temp.path(), &program, &resources) + .unwrap_err() + .contains("-I -S -B") + ); +} + +#[test] +#[ignore = "requires the explicit host Node fixture path"] +fn runs_javascript_runner_from_explicit_fixture_runtime_path() { + run_fixture_runtime("npm", "/root/.nvm/versions/node/v24.15.0/bin/node"); +} + +#[test] +#[ignore = "requires the explicit host Python fixture path"] +fn runs_python_runner_from_explicit_fixture_runtime_path() { + run_fixture_runtime("pip", "/usr/bin/python3.12"); +} + +#[test] +#[ignore = "requires the explicit host Node fixture path"] +fn binds_commonjs_exports_without_a_registration_table() { + let temp = TemporaryDirectory::new(); + let (program, resources) = fixture_entry( + temp.path(), + "npm", + &native_elf::static_executable(), + None, + "worker/main.cjs", + ); + execute_generated_fixture( + &temp, + &program, + &resources, + Some("/root/.nvm/versions/node/v24.15.0/bin/node"), + ); +} + +#[test] +#[ignore = "requires the explicit host Node fixture path"] +fn binds_esm_exports_without_a_registration_table() { + let temp = TemporaryDirectory::new(); + let (program, resources) = fixture(temp.path(), "npm", &native_elf::static_executable(), None); + execute_generated_fixture( + &temp, + &program, + &resources, + Some("/root/.nvm/versions/node/v24.15.0/bin/node"), + ); +} + +#[test] +#[ignore = "requires the explicit host Python fixture path"] +fn imports_packaged_sdk_before_adapter_modules() { + let temp = TemporaryDirectory::new(); + let (program, resources) = fixture(temp.path(), "pip", &native_elf::static_executable(), None); + let entry = &program.external_worker_contracts()[0].entry; + fs::write( + temp.path().join(entry).with_file_name("dever_component.py"), + "raise RuntimeError('shadowed SDK')\n", + ) + .unwrap(); + execute_generated_fixture(&temp, &program, &resources, Some("/usr/bin/python3.12")); +} + +#[cfg(unix)] +#[test] +#[ignore = "requires the explicit host Node fixture and native Rust compiler"] +fn compiled_managed_worker_runs_after_source_and_pack_are_removed() { + use std::os::unix::process::CommandExt; + + let temp = TemporaryDirectory::new(); + let binary = fs::read("/root/.nvm/versions/node/v24.15.0/bin/node").unwrap(); + let (program, resources, sources) = + fixture_entry_with_sources(temp.path(), "npm", &binary, None, "worker/main.mjs"); + let source_entry = program.external_worker_contracts()[0].entry.clone(); + let raw_pack_path = temp.path().join("runtime.pack"); + fs::write(&raw_pack_path, &resources[0].bytes).unwrap(); + let raw = resource(resources[0].path.clone(), fs::read(&raw_pack_path).unwrap()); + let packaged = prepare_sandboxed_worker(temp.path(), &program, vec![raw]); + let native = dever_core::native::compile_project_with_resources( + &program, + &sources, + std::ffi::OsStr::new("/root/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/bin/rustc"), + dever_runtime::config::RuntimeProfile::default(), + &packaged, + ) + .unwrap(); + let executable = temp.path().join("program"); + native.save(&executable).unwrap(); + drop(native); + drop(sources); + drop(program); + drop(resources); + fs::remove_file(temp.path().join(source_entry)).unwrap(); + fs::remove_file(temp.path().join("dever.lock")).unwrap(); + fs::remove_file(raw_pack_path).unwrap(); + fs::create_dir(temp.path().join("config")).unwrap(); + fs::write( + temp.path().join("config/setting.json"), + r#"{"adapter":{"mail.send":{"use":"default"}}}"#, + ) + .unwrap(); + + let child = Command::new(&executable) + .process_group(0) + .current_dir(temp.path()) + .env_clear() + .args(["mail.send.send", "{\"value\":7}"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let mut owned = OwnedProcessGroup(child); + let deadline = Instant::now() + Duration::from_secs(15); + let status = loop { + if let Some(status) = owned.0.try_wait().unwrap() { + break Some(status); + } + if Instant::now() >= deadline { + break None; + } + thread::sleep(Duration::from_millis(10)); + }; + if status.is_none() { + owned.stop(); + } + let mut stdout = Vec::new(); + let mut stderr = String::new(); + owned + .0 + .stdout + .take() + .unwrap() + .read_to_end(&mut stdout) + .unwrap(); + owned + .0 + .stderr + .take() + .unwrap() + .read_to_string(&mut stderr) + .unwrap(); + assert!( + status.is_some_and(|status| status.success()), + "native managed Worker failed or timed out: {stderr}" + ); + let response: serde_json::Value = serde_json::from_slice(&stdout).unwrap_or_else(|error| { + panic!( + "invalid CMD response: {error}; stdout={:?}; stderr={stderr}", + String::from_utf8_lossy(&stdout) + ) + }); + assert_eq!(response["code"], 0, "{response}; stderr={stderr}"); + assert_eq!(response["data"], 8, "{response}; stderr={stderr}"); +} + +fn run_fixture_runtime(ecosystem: &str, binary_path: &str) { + let temp = TemporaryDirectory::new(); + let binary = fs::read(binary_path).unwrap(); + let (program, resources) = fixture(temp.path(), ecosystem, &binary, None); + execute_generated_fixture(&temp, &program, &resources, None); +} + +fn execute_generated_fixture( + temp: &TemporaryDirectory, + program: &dever_core::hir::Program, + resources: &[EmbeddedResource], + host_runtime: Option<&str>, +) { + let generated = workers::prepare(temp.path(), program, resources).unwrap(); + let entry = &program.external_worker_contracts()[0].entry; + let launch = generated + .iter() + .find(|resource| resource.path == format!("{entry}.dever-worker.json")) + .unwrap(); + let manifest: serde_json::Value = serde_json::from_slice(&launch.bytes).unwrap(); + let executable = manifest["executable"].as_str().unwrap(); + let runner = manifest["arguments"].as_array().unwrap().last().unwrap()["resource"] + .as_str() + .unwrap(); + + for resource in &generated { + let path = temp.path().join(&resource.path); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(&path, &resource.bytes).unwrap(); + #[cfg(unix)] + if resource.executable { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(path, fs::Permissions::from_mode(0o755)).unwrap(); + } + } + let contract = program.external_worker_contracts().remove(0); + let runtime_path = host_runtime + .map(PathBuf::from) + .unwrap_or_else(|| temp.path().join(executable)); + let mut command = Command::new(runtime_path); + for flag in manifest["arguments"] + .as_array() + .unwrap() + .iter() + .take(manifest["arguments"].as_array().unwrap().len() - 1) + { + command.arg(flag["literal"].as_str().unwrap()); + } + let child = command + .arg(temp.path().join(runner)) + .arg("--dever-component") + .current_dir( + temp.path() + .join(manifest["working_directory"].as_str().unwrap()), + ) + .env_clear() + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + assert_fixture_protocol(child, &contract); +} + +fn assert_fixture_protocol( + mut child: std::process::Child, + contract: &dever_core::hir::ExternalWorkerContract, +) { + let hello = json!({"kind":"hello","version":"dever-component-1","port":contract.port, + "adapter":contract.adapter,"schema":contract.schema,"operations":contract.operations, + "capabilities":contract.capabilities,"setting":null}); + let (sender, receiver) = mpsc::channel(); + let mut stdout = child.stdout.take().unwrap(); + let reader = thread::spawn(move || { + while let Some(message) = response(&mut stdout) { + if sender.send(message).is_err() { + break; + } + } + }); + let exchange = (|| -> Result<Vec<serde_json::Value>, String> { + let mut messages = Vec::new(); + for request in [ + hello, + json!({"kind":"call","id":1,"operation":"send","payload":{"value":7}}), + json!({"kind":"shutdown"}), + ] { + child + .stdin + .as_mut() + .unwrap() + .write_all(&frame(request)) + .map_err(|error| error.to_string())?; + let message = receiver + .recv_timeout(Duration::from_secs(10)) + .map_err(|error| error.to_string())?; + messages.push(message); + } + Ok(messages) + })(); + drop(child.stdin.take()); + if exchange.is_err() { + let _ = child.kill(); + } + let deadline = Instant::now() + Duration::from_secs(10); + let status = loop { + if let Some(status) = child.try_wait().unwrap() { + break status; + } + if Instant::now() >= deadline { + child.kill().unwrap(); + child.wait().unwrap(); + panic!("managed fixture Worker did not stop"); + } + thread::sleep(Duration::from_millis(10)); + }; + let mut stderr = String::new(); + reader.join().unwrap(); + child + .stderr + .take() + .unwrap() + .read_to_string(&mut stderr) + .unwrap(); + assert!(status.success(), "{stderr}"); + let messages = exchange.unwrap_or_else(|error| panic!("{error}; stderr={stderr}")); + assert_eq!(messages.len(), 3, "{messages:?}; stderr={stderr}"); + assert_eq!(messages[0]["kind"], "ready"); + assert_eq!( + messages[1], + json!({"kind":"result","id":1,"payload":{"receipt":8}}) + ); + assert_eq!(messages[2], json!({"kind":"shutdown"})); +} diff --git a/test/dever-cli-tests/external_workers/command.rs b/test/dever-cli-tests/external_workers/command.rs new file mode 100644 index 0000000..b83200d --- /dev/null +++ b/test/dever-cli-tests/external_workers/command.rs @@ -0,0 +1,204 @@ +use super::*; +use dever_cli::toolchain::BuildTarget; + +fn command_fixture(root: &Path, binary: &[u8]) -> dever_core::hir::Program { + let mut sources = SourceMap::default(); + sources.add( + "media/tool/port.dever", + "execute(args: List<Text>) (output: dever.process.Output) fails dever.process.Error", + ); + sources.add( + "media/tool/adapter.dever", + "external command \"bin/tool\" {}\n", + ); + sources.add("media/tool/app.dever", "execute() (code: Int) {\n output = port.execute([\"a b\", \"$(literal)\"])\n code = output.code\n}\n"); + sources.add("media/tool/api.dever", "cmd execute = app.execute\n"); + let program = dever_core::check(&sources).unwrap_or_else(|errors| panic!("{errors:?}")); + let entry = &program.external_command_contracts()[0].entry; + fs::create_dir_all(root.join(entry).parent().unwrap()).unwrap(); + fs::write(root.join(entry), binary).unwrap(); + program +} + +fn command_manifest(resources: &[EmbeddedResource]) -> serde_json::Value { + let manifest = resources + .iter() + .find(|resource| resource.path.ends_with(".dever-command.json")) + .unwrap(); + serde_json::from_slice(&manifest.bytes).unwrap() +} + +#[test] +fn command_preparation_needs_no_ecosystem_lock_and_keeps_target_identity() { + for (target, machine) in [ + (BuildTarget::LinuxX86_64, 62u16), + (BuildTarget::LinuxAarch64, 183u16), + ] { + let temp = TemporaryDirectory::new(); + let mut binary = native_elf::static_executable(); + binary[18..20].copy_from_slice(&machine.to_le_bytes()); + let program = command_fixture(temp.path(), &binary); + assert!(program.external_worker_contracts().is_empty()); + assert!( + dever_cli::libs::prepare_program(temp.path(), &program, target) + .unwrap() + .is_none() + ); + let resources = + workers::prepare_for_target(temp.path(), &program, &[], target, &[]).unwrap(); + let manifest = command_manifest(&resources); + assert_eq!(manifest["format"], "dever-command-launch-v1"); + assert_eq!(manifest["arguments"], json!([])); + assert_eq!(manifest["ecosystem"], "command"); + let executable = resources + .iter() + .find(|resource| Some(resource.path.as_str()) == manifest["executable"].as_str()) + .unwrap(); + assert!(executable.executable); + assert_eq!(executable.bytes, binary); + let other = if target == BuildTarget::LinuxX86_64 { + BuildTarget::LinuxAarch64 + } else { + BuildTarget::LinuxX86_64 + }; + assert!( + workers::prepare_for_target(temp.path(), &program, &[], other, &[]) + .unwrap_err() + .contains("differs from target") + ); + assert!(!temp.path().join("dever.lock").exists()); + } +} + +#[test] +fn command_preparation_checks_recursive_library_closure_and_loader() { + let temp = TemporaryDirectory::new(); + let program = command_fixture(temp.path(), &native_elf::compiler(&["libfirst.so"])); + assert!( + workers::prepare(temp.path(), &program, &[]) + .unwrap_err() + .contains("libfirst.so") + ); + let lib = temp.path().join("module/media/tool/bin/lib"); + fs::create_dir_all(&lib).unwrap(); + fs::write( + lib.join("libfirst.so"), + native_elf::library("libfirst.so", &["libsecond.so"]), + ) + .unwrap(); + assert!( + workers::prepare(temp.path(), &program, &[]) + .unwrap_err() + .contains("libsecond.so") + ); + fs::write( + lib.join("libsecond.so"), + native_elf::library("libwrong.so", &[]), + ) + .unwrap(); + assert!( + workers::prepare(temp.path(), &program, &[]) + .unwrap_err() + .contains("SONAME") + ); + fs::write( + lib.join("libsecond.so"), + native_elf::library("libsecond.so", &[]), + ) + .unwrap(); + assert!( + workers::prepare(temp.path(), &program, &[]) + .unwrap_err() + .contains("signed target GNU loader") + ); + let loader_name = dever_sandbox::loader_name(); + let mut loader = resource( + format!("sandbox/lib/{loader_name}"), + native_elf::static_executable(), + ); + loader.executable = true; + let resources = workers::prepare(temp.path(), &program, &[loader]).unwrap(); + let manifest = command_manifest(&resources); + assert!( + manifest["executable"] + .as_str() + .unwrap() + .ends_with("/loader") + ); + assert_eq!(manifest["arguments"][0]["literal"], "--inhibit-cache"); + for name in ["libfirst.so", "libsecond.so"] { + assert!(resources.iter().any( + |resource| resource.path.ends_with(&format!("/lib/{name}")) && !resource.executable + )); + } +} + +#[test] +fn command_rejects_host_library_paths_and_signed_library_shadowing() { + let temp = TemporaryDirectory::new(); + let program = command_fixture( + temp.path(), + &native_elf::elf(None, &["libfirst.so"], Some("/usr/lib")), + ); + assert!( + workers::prepare(temp.path(), &program, &[]) + .unwrap_err() + .contains("$ORIGIN") + ); + let entry = &program.external_command_contracts()[0].entry; + for path in ["$ORIGIN/../data/grant", "${ORIGIN}/../lib"] { + fs::write( + temp.path().join(entry), + native_elf::elf(None, &[], Some(path)), + ) + .unwrap(); + assert!( + workers::prepare(temp.path(), &program, &[]) + .unwrap_err() + .contains("escapes its packaged tree") + ); + } + fs::write( + temp.path().join(entry), + native_elf::compiler(&["libfirst.so"]), + ) + .unwrap(); + let lib = temp.path().join("module/media/tool/bin/lib"); + fs::create_dir_all(&lib).unwrap(); + fs::write( + lib.join("libfirst.so"), + native_elf::library("libfirst.so", &[]), + ) + .unwrap(); + let signed = resource( + "sandbox/lib/libfirst.so".into(), + native_elf::library("libfirst.so", &[]), + ); + assert!( + workers::prepare(temp.path(), &program, &[signed]) + .unwrap_err() + .contains("overrides a signed sandbox library") + ); +} + +#[cfg(unix)] +#[test] +fn command_rejects_symlinks_and_library_subdirectories() { + let temp = TemporaryDirectory::new(); + let program = command_fixture(temp.path(), &native_elf::static_executable()); + let lib = temp.path().join("module/media/tool/bin/lib"); + fs::create_dir_all(&lib).unwrap(); + std::os::unix::fs::symlink("../tool", lib.join("libalias.so")).unwrap(); + assert!( + workers::prepare(temp.path(), &program, &[]) + .unwrap_err() + .contains("regular library files") + ); + fs::remove_file(lib.join("libalias.so")).unwrap(); + fs::create_dir(lib.join("hidden")).unwrap(); + assert!( + workers::prepare(temp.path(), &program, &[]) + .unwrap_err() + .contains("regular library files") + ); +} diff --git a/test/dever-cli-tests/external_workers/npm.rs b/test/dever-cli-tests/external_workers/npm.rs new file mode 100644 index 0000000..d0c6591 --- /dev/null +++ b/test/dever-cli-tests/external_workers/npm.rs @@ -0,0 +1,119 @@ +use super::*; + +#[test] +#[ignore = "requires the explicit official native npm build fixture at target/npm-build"] +fn runs_official_npm_addon_from_offline_resources_and_threads() { + let temp = TemporaryDirectory::new(); + let program = dever_core::check(&worker_sources( + "npm", + "worker/main.cjs", + "lib \"bufferutil@4.0.9\"", + )) + .unwrap(); + let contract = program.external_worker_contracts().remove(0); + let workspace = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let prepared = workspace.join("target/npm-build"); + let config: serde_json::Value = + serde_json::from_slice(&fs::read(prepared.join("config/setting.json")).unwrap()).unwrap(); + let pack = fs::read(prepared.join(config["runtime_file"].as_str().unwrap())).unwrap(); + let mut lock = + LockFile::decode(&fs::read(prepared.join("dependency/dever.lock")).unwrap()).unwrap(); + let runtime = lock.libs[0].runtime.clone(); + assert_eq!(sha(&pack), runtime.sha256); + let rejected = lock.builds[0] + .native_rejections + .keys() + .cloned() + .collect::<Vec<_>>(); + assert!( + !rejected.is_empty(), + "official multi-platform prebuilds must be classified" + ); + let entry = temp.path().join(&contract.entry); + fs::create_dir_all(entry.parent().unwrap()).unwrap(); + let adapter = r#"const addon = require('bufferutil/build/Release/bufferutil.node'); +const {Worker} = require('node:worker_threads'); +const path = require('node:path'); +const rejected = __REJECTED__.map(name => path.join(__dirname, '..', name)); +exports.send = async payload => { + await new Promise((resolve, reject) => { + const worker = new Worker(`const {parentPort,workerData}=require('node:worker_threads'); const fs=require('node:fs'); parentPort.postMessage(workerData.every(path=>!fs.existsSync(path)));`, {eval:true,workerData:rejected}); + worker.once('message', ok => ok ? resolve() : reject(new Error('unverified addon was installed'))); + worker.once('error', reject); + }); + const input=Buffer.from([payload.value]), output=Buffer.alloc(1), mask=Buffer.from([1,2,3,4]); + addon.mask(input,mask,output,0,1); addon.unmask(output,mask); + return {receipt:output[0]+1}; +}; +"#.replace("__REJECTED__",&serde_json::to_string(&rejected).unwrap()); + fs::write(&entry, adapter).unwrap(); + let mut resources = vec![resource( + format!( + "lib/runtime/npm/{}/runtime.pack", + dever_cli::toolchain::platform_identity() + ), + pack, + )]; + for library in &mut lock.libs { + library.schema = contract.schema.clone(); + } + for artifact in lock + .libs + .iter() + .flat_map(|lib| &lib.artifacts) + .chain(lock.builds.iter().map(|receipt| &receipt.output)) + { + let bytes = fs::read(prepared.join("dependency/artifacts").join(&artifact.sha256)).unwrap(); + assert_eq!(sha(&bytes), artifact.sha256); + resources.push(resource(artifact.path.clone(), bytes)); + } + for asset in sandbox::assets(&workspace) { + let mut file = resource(asset.path, asset.bytes); + file.executable = asset.executable; + resources.push(file); + } + lock.workers.push(LockedWorker { + port: contract.port.clone(), + adapter: contract.adapter.clone(), + ecosystem: "npm".into(), + runtime: Some(runtime), + entry: contract.entry.clone(), + schema: contract.schema.clone(), + capabilities: contract.capabilities.clone(), + operations: contract.operations.clone(), + libs: contract + .libs + .iter() + .map(|spec| spec.parse().unwrap()) + .collect(), + }); + lock.write_atomic(temp.path()).unwrap(); + let mut generated = workers::prepare(temp.path(), &program, &resources).unwrap(); + let again = workers::prepare(temp.path(), &program, &resources).unwrap(); + assert_eq!( + generated + .iter() + .map(|file| (&file.path, &file.sha256)) + .collect::<Vec<_>>(), + again + .iter() + .map(|file| (&file.path, &file.sha256)) + .collect::<Vec<_>>() + ); + assert!( + generated + .iter() + .any(|file| file.path.ends_with("/build/Release/bufferutil.node") + && file.bytes.starts_with(b"\x7fELF")) + ); + for rejected in &rejected { + assert!(!generated.iter().any(|file| file.path.ends_with(rejected))); + } + generated.extend( + resources + .into_iter() + .filter(|file| file.path.starts_with("sandbox/")), + ); + fs::remove_file(entry).unwrap(); + run_packaged_protocol(&temp, &contract, "npm", &generated); +} diff --git a/test/dever-cli-tests/llvm_cli.rs b/test/dever-cli-tests/llvm_cli.rs new file mode 100644 index 0000000..29ca4f8 --- /dev/null +++ b/test/dever-cli-tests/llvm_cli.rs @@ -0,0 +1,642 @@ +//! Actual default CLI boundary. Native cases consume an explicitly prepared +//! runtime archive; ordinary negative checks never start a compiler subprocess. +#![cfg(all(target_os = "linux", target_arch = "x86_64"))] + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output, Stdio}; +use std::time::Duration; + +use serde_json::{Value, json}; + +#[path = "support/llvm_pack.rs"] +mod pack; +#[path = "../dever-tests/tests/support/process.rs"] +mod process; +#[path = "../dever-tests/tests/support/temp.rs"] +mod temp; +use temp::TemporaryDirectory; + +const APP: &str = r#"type Outcome { error Denied(message: Text, labels: List<Text?>) } +echo(name: Text) (value: Text) { dever.task.sleep(1) + value = name } +divide(divisor: Int) (value: Int) { value = 42 // divisor } +deny() (value: Text) { fail(Outcome.Denied("owned denial", ["nested label", null])) } +"#; + +fn write(root: &Path, relative: &str, source: &str) { + let path = root.join(relative); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, source).unwrap(); +} + +fn project(root: &Path) { + write(root, "module/sample/echo/app.dever", APP); + write( + root, + "module/sample/echo/api.dever", + "cmd echo = app.echo\ncmd divide = app.divide\ncmd deny = app.deny", + ); + write(root, "config/setting.json", "{}"); +} + +fn compiler(root: &Path, name: &str) -> PathBuf { + pack::compiler(Path::new(env!("CARGO_BIN_EXE_dever")), root, name) +} + +fn output(command: &mut Command) -> Output { + let capture = TemporaryDirectory::new(); + let stdout = capture.path().join("stdout"); + let stderr = capture.path().join("stderr"); + command + .stdout(Stdio::from(fs::File::create(&stdout).unwrap())) + .stderr(Stdio::from(fs::File::create(&stderr).unwrap())); + let status = process::status(command, Duration::from_secs(180)).unwrap(); + Output { + status, + stdout: fs::read(stdout).unwrap(), + stderr: fs::read(stderr).unwrap(), + } +} + +fn cli(compiler: &Path, project: &Path, action: &str, args: &[&str]) -> Output { + output( + Command::new(compiler) + .env_clear() + .arg(action) + .arg(project) + .args(args), + ) +} + +fn success(output: &Output) { + assert!( + output.status.success(), + "stdout: {}\nstderr: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +fn failure(output: &Output, expected: &str) { + assert!(!output.status.success(), "unexpected success"); + let message = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!( + message.contains(expected), + "expected {expected:?}: {message}" + ); +} + +#[test] +fn missing_pack_fails_without_host_tools_and_check_and_empty_test_need_no_pack() { + let toolchain = TemporaryDirectory::new(); + let compiler = compiler(toolchain.path(), "dever"); + let help = output(Command::new(&compiler).env_clear().arg("--help")); + success(&help); + assert!(String::from_utf8_lossy(&help.stdout).contains("dever run <project-root>")); + let project_dir = TemporaryDirectory::new(); + project(project_dir.path()); + success(&cli(&compiler, project_dir.path(), "check", &[])); + let tests = cli(&compiler, project_dir.path(), "test", &[]); + success(&tests); + assert!(String::from_utf8_lossy(&tests.stdout).contains("0 passed; 0 failed")); + failure( + &cli( + &compiler, + project_dir.path(), + "run", + &["--", "sample.echo.echo", "{\"name\":\"Ada\"}"], + ), + "runtime pack", + ); + assert!(!toolchain.path().join("cache").exists()); +} + +#[test] +fn cross_build_requires_the_selected_pack_even_when_host_pack_exists() { + let root = TemporaryDirectory::new(); + let compiler = compiler(root.path(), "dever"); + pack::manifest(root.path(), false); + let project_dir = TemporaryDirectory::new(); + project(project_dir.path()); + let executable = root.path().join("arm-application"); + failure( + &cli( + &compiler, + project_dir.path(), + "build", + &[ + "--target", + "linux-aarch64", + "--output", + executable.to_str().unwrap(), + ], + ), + "runtime/linux-aarch64", + ); + assert!(!executable.exists()); + assert!(!root.path().join("cache").exists()); +} + +#[test] +fn managed_cores_never_fall_back_to_private_packs_or_cache() { + let root = TemporaryDirectory::new(); + let project_dir = TemporaryDirectory::new(); + project(project_dir.path()); + for (directory, diagnostic) in [ + ( + root.path().join("standalone"), + "managed compiler must be installed", + ), + ( + root.path().join("versions/0.1.0"), + "cannot inspect deverd state directory", + ), + ] { + let compiler = compiler(&directory, "dever-core"); + failure(&cli(&compiler, project_dir.path(), "run", &[]), diagnostic); + assert!(!directory.join("cache").exists()); + } + let installed = root.path().join("versions/0.1.0/dever-core"); + let worker = output( + Command::new(installed) + .env_clear() + .current_dir(project_dir.path()) + .arg("--dever-compile-worker"), + ); + failure(&worker, "service-owned native cache staging directory"); + assert!(!project_dir.path().join("program").exists()); +} + +#[test] +fn pack_contract_rejects_drift_duplicate_paths_and_undeclared_link_inputs() { + let root = TemporaryDirectory::new(); + let compiler = compiler(root.path(), "dever"); + let original = pack::manifest(root.path(), false); + let project_dir = TemporaryDirectory::new(); + project(project_dir.path()); + let mutations: &[(&str, Value, &str)] = &[ + ("/format", json!("future"), "does not match"), + ("/compiler_version", json!("99.0.0"), "does not match"), + ("/abi", json!(99), "does not match"), + ( + "/target", + json!("aarch64-unknown-linux-gnu"), + "does not match", + ), + ("/profiles/base", Value::Null, "missing the required 'base'"), + ( + "/profiles/base/runtime", + json!("missing.a"), + "undeclared or duplicate", + ), + ( + "/profiles/base/libraries/0", + json!("runtime.a"), + "undeclared or duplicate", + ), + ( + "/files/0/path", + json!("../runtime.a"), + "normalized relative", + ), + ( + "/files/0/path", + json!("nested//runtime.a"), + "normalized relative", + ), + ( + "/files/0/path", + json!("C:\\runtime.a"), + "normalized relative", + ), + ( + "/files/1/path", + json!("runtime.a"), + "duplicate runtime file", + ), + ("/files/0/bytes", json!(9), "SHA-256 verification"), + ( + "/files/0/sha256", + json!("0".repeat(64)), + "SHA-256 verification", + ), + ]; + for (pointer, value, expected) in mutations { + let mut manifest = original.clone(); + *manifest.pointer_mut(pointer).unwrap() = value.clone(); + pack::write_manifest(root.path(), &manifest); + failure(&cli(&compiler, project_dir.path(), "run", &[]), expected); + assert!(!root.path().join("cache").exists()); + } + let text = serde_json::to_string(&original).unwrap(); + fs::write( + pack::pack_root(root.path()).join("manifest.json"), + text.replacen("\"abi\":1", "\"abi\":1,\"abi\":1", 1), + ) + .unwrap(); + failure( + &cli(&compiler, project_dir.path(), "run", &[]), + "duplicate field", + ); +} + +#[test] +fn pack_rejects_symlinks_and_disguised_linker_scripts() { + use std::os::unix::fs::symlink; + let root = TemporaryDirectory::new(); + let compiler = compiler(root.path(), "dever"); + let mut manifest = pack::manifest(root.path(), false); + let project_dir = TemporaryDirectory::new(); + project(project_dir.path()); + let runtime = pack::pack_root(root.path()).join("runtime.a"); + let outside = root.path().join("outside.a"); + fs::rename(&runtime, &outside).unwrap(); + symlink(&outside, &runtime).unwrap(); + failure( + &cli(&compiler, project_dir.path(), "run", &[]), + "must be a real file", + ); + fs::remove_file(&runtime).unwrap(); + for content in [b"INPUT(/usr/lib/forbidden.a)".as_slice(), b"!<thin>\n"] { + fs::write(&runtime, content).unwrap(); + manifest["files"][0]["bytes"] = json!(content.len()); + manifest["files"][0]["sha256"] = json!(pack::digest(&runtime)); + pack::write_manifest(root.path(), &manifest); + failure( + &cli(&compiler, project_dir.path(), "run", &[]), + "linker scripts and thin archives are forbidden", + ); + } + assert!(!root.path().join("cache").exists()); +} + +fn cached_programs(root: &Path) -> Vec<PathBuf> { + let mut programs = fs::read_dir(root.join("cache/native")) + .unwrap() + .map(|entry| entry.unwrap().path().join("program")) + .filter(|path| path.is_file()) + .collect::<Vec<_>>(); + programs.sort(); + programs +} + +#[test] +#[ignore = "requires explicitly prepared full runtime archive and Linux CRT files; serial offline CLI execution"] +fn default_llvm_runs_builds_moves_and_reuses_cache_without_host_toolchain() { + let root = TemporaryDirectory::new(); + let compiler = compiler(root.path(), "dever"); + pack::manifest(root.path(), true); + let project_dir = TemporaryDirectory::new(); + project(project_dir.path()); + let args = ["--", "sample.echo.echo", "{\"name\":\"你好 LLVM\"}"]; + let marker = root.path().join("forbidden-tool-was-started"); + for name in ["rustc", "cargo", "cc"] { + use std::os::unix::fs::PermissionsExt; + let path = root.path().join(name); + fs::write( + &path, + format!( + "#!/bin/sh\nprintf forbidden > '{}'\nexit 99\n", + marker.display() + ), + ) + .unwrap(); + fs::set_permissions(path, fs::Permissions::from_mode(0o700)).unwrap(); + } + let run = output( + Command::new(&compiler) + .env_clear() + .env("PATH", root.path()) + .env("RUSTC", root.path().join("rustc")) + .env("CARGO", root.path().join("cargo")) + .env("CC", root.path().join("cc")) + .arg("run") + .arg(project_dir.path()) + .args(args), + ); + success(&run); + assert!( + !marker.exists(), + "LLVM compilation must not start a host compiler" + ); + let response: Value = serde_json::from_slice(&run.stdout).unwrap(); + assert_eq!(response["data"], "你好 LLVM"); + let cached = cached_programs(root.path()); + assert_eq!(cached.len(), 1); + let modified = fs::metadata(&cached[0]).unwrap().modified().unwrap(); + let binary = project_dir.path().join("standalone"); + success(&cli( + &compiler, + project_dir.path(), + "build", + &["--output", binary.to_str().unwrap()], + )); + assert_eq!( + fs::metadata(&cached[0]).unwrap().modified().unwrap(), + modified + ); + let first_digest = pack::digest(&binary); + failure( + &cli( + &compiler, + project_dir.path(), + "build", + &["--output", binary.to_str().unwrap()], + ), + "cannot save executable", + ); + assert_eq!(pack::digest(&binary), first_digest); + let moved = TemporaryDirectory::new(); + let relocated = moved.path().join("application"); + fs::rename(&binary, &relocated).unwrap(); + write(moved.path(), "config/setting.json", "{}"); + success(&output( + Command::new(&relocated).env_clear().args(&args[1..]), + )); + { + use std::os::fd::OwnedFd; + use std::os::unix::net::UnixStream; + let (reader, writer) = UnixStream::pair().unwrap(); + drop(reader); + let stderr = moved.path().join("closed-stdout-error"); + let status = process::status( + Command::new(&relocated) + .env_clear() + .args(&args[1..]) + .stdout(Stdio::from(OwnedFd::from(writer))) + .stderr(Stdio::from(fs::File::create(&stderr).unwrap())), + Duration::from_secs(30), + ) + .unwrap(); + assert_eq!( + status.code(), + Some(1), + "closed stdout must return a checked failure, not SIGPIPE" + ); + let error = fs::read_to_string(stderr).unwrap(); + assert!( + error.contains("Broken pipe") && error.contains(".dever"), + "{error}" + ); + } + let denied = cli( + &compiler, + project_dir.path(), + "run", + &["--", "sample.echo.deny", "{}"], + ); + failure(&denied, "owned denial"); + assert!(String::from_utf8_lossy(&denied.stderr).contains("nested label")); + assert!(String::from_utf8_lossy(&denied.stderr).contains("app.dever")); + failure( + &cli( + &compiler, + project_dir.path(), + "run", + &["--", "sample.echo.echo", "{}"], + ), + "name", + ); + let failure_output = cli( + &compiler, + project_dir.path(), + "run", + &["--", "sample.echo.divide", "{\"divisor\":0}"], + ); + failure(&failure_output, "division"); + assert!(String::from_utf8_lossy(&failure_output.stderr).contains("app.dever")); + // Corrupt only the fixture-owned cache copy, never a hard-linked runtime. + fs::write(&cached[0], b"corrupt cached executable").unwrap(); + failure( + &cli(&compiler, project_dir.path(), "run", &args), + "incomplete or modified", + ); + assert_eq!(fs::read(&cached[0]).unwrap(), b"corrupt cached executable"); + write( + project_dir.path(), + "module/sample/echo/app.dever", + &APP.replace("value = name", "value = name + \"!\""), + ); + let changed = cli(&compiler, project_dir.path(), "run", &args); + success(&changed); + assert_eq!( + serde_json::from_slice::<Value>(&changed.stdout).unwrap()["data"], + "你好 LLVM!" + ); + assert_eq!(cached_programs(root.path()).len(), 2); + + // Even a byte-level manifest change has its own exact pack identity. + let manifest = pack::pack_root(root.path()).join("manifest.json"); + let mut bytes = fs::read(&manifest).unwrap(); + bytes.push(b'\n'); + fs::write(&manifest, bytes).unwrap(); + success(&cli(&compiler, project_dir.path(), "run", &args)); + assert_eq!(cached_programs(root.path()).len(), 3); + + let markdown = TemporaryDirectory::new(); + write(markdown.path(), "config/setting.json", "{}"); + write( + markdown.path(), + "module/system/health/app.dever.md", + include_str!("../../examples/cms/md/module/system/health/app.dever.md"), + ); + write( + markdown.path(), + "module/system/health/api.dever.md", + include_str!("../../examples/cms/md/module/system/health/api.dever.md"), + ); + let response = cli( + &compiler, + markdown.path(), + "run", + &["--", "system.health.ping", "{}"], + ); + success(&response); + assert_eq!( + serde_json::from_slice::<Value>(&response.stdout).unwrap()["data"]["status"], + "ready" + ); + assert_eq!(cached_programs(root.path()).len(), 4); + let object = pack::pack_root(root.path()).join("crt1.o"); + let mut corrupt = fs::read(&object).unwrap(); + corrupt[16] ^= 1; + // Break this fixture's hard link before mutation; the host CRT is immutable. + fs::remove_file(&object).unwrap(); + fs::write(&object, corrupt).unwrap(); + failure( + &cli(&compiler, project_dir.path(), "run", &args), + "SHA-256 verification", + ); + assert_eq!(cached_programs(root.path()).len(), 4); +} + +#[test] +#[ignore = "requires explicitly prepared runtime archive and component-fixture; no downloads or language SDK"] +fn unsigned_llvm_run_and_build_reject_exec_workers() { + let root = TemporaryDirectory::new(); + let compiler = compiler(root.path(), "dever"); + pack::manifest(root.path(), true); + let project_dir = TemporaryDirectory::new(); + let directory = project_dir.path(); + write( + directory, + "module/notification/delivery/app.dever", + "read() (value: Int) { value = port.read() }", + ); + write( + directory, + "module/notification/delivery/port.dever", + "read() (value: Int) fails dever.time.SleepResult", + ); + write( + directory, + "module/notification/delivery/adapter.dever", + "setting { mode: Text }\nexternal exec \"worker\" {}", + ); + write( + directory, + "module/notification/delivery/api.dever", + "cmd read = app.read", + ); + let settings = r#"{"adapter":{"notification.delivery":{"setting":{"mode":"embedded"}}}}"#; + write(directory, "config/setting.json", settings); + let fixture = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../target/native-release-inputs/component-fixture"); + assert!( + fixture.is_file(), + "prepare the immutable component-fixture in target/native-release-inputs" + ); + fs::copy( + fixture, + directory.join("module/notification/delivery/worker"), + ) + .unwrap(); + let args = ["--", "notification.delivery.read", "{}"]; + let run = cli(&compiler, directory, "run", &args); + let binary = directory.join("packaged"); + let build = cli( + &compiler, + directory, + "build", + &["--output", binary.to_str().unwrap()], + ); + for rejected in [run, build] { + assert!(!rejected.status.success()); + assert!(rejected.stdout.is_empty()); + assert_eq!( + String::from_utf8(rejected.stderr).unwrap().trim(), + "real Lib providers require a signed managed Dever release" + ); + } + assert!(!binary.exists()); + assert!(!root.path().join("cache").exists()); +} + +#[test] +#[ignore = "requires explicitly prepared full runtime archive; isolated SQLite and case-local fake processes"] +fn llvm_test_preserves_case_isolation_fakes_and_failure_continuation() { + let root = TemporaryDirectory::new(); + let compiler = compiler(root.path(), "dever"); + pack::manifest(root.path(), true); + let project_dir = TemporaryDirectory::new(); + let directory = project_dir.path(); + write( + directory, + "module/sample/account/model.dever", + "type Account { name: Text }", + ); + write( + directory, + "module/sample/account/app.dever", + "create(name: Text) (count: Int) { row = model.create({ name = name })\n count = model.count() }", + ); + write( + directory, + "module/notification/mail/app.dever", + "read() (value: Int) { value = port.read() }", + ); + write( + directory, + "module/notification/mail/port.dever", + "read() (value: Int) fails dever.time.SleepResult", + ); + write( + directory, + "module/notification/mail/adapter.dever", + "external exec \"worker/must-not-start\" {}", + ); + write( + directory, + "config/setting.json", + r#"{"database":{"default":{"type":"sqlite","path":"deployment.db"}}}"#, + ); + // The CLI reads project dependency metadata from valid JSON, but test + // execution must use its generated database, never this unusable directory. + fs::create_dir(directory.join("deployment.db")).unwrap(); + for name in ["first", "second"] { + write( + directory, + &format!("test/sample/account/{name}.dever"), + &format!("{name}() () {{ assert_eq(app.create(\"Ada\"), 1) }}"), + ); + } + write( + directory, + "test/notification/mail/blocking.dever", + "blocking() () { dever.time.sleep(1)\n assert_eq(app.read(), 11) }\nport.read() (value: Int) { value = 11 }", + ); + write( + directory, + "test/notification/mail/suspending.dever", + "suspending() () { assert_eq(app.read(), 23) }\nport.read() (value: Int) { dever.task.sleep(1)\n value = 23 }", + ); + write( + directory, + "test/sample/account/failure.dever", + "failure() () { assert_eq(1, 2) }", + ); + write( + directory, + "test/sample/account/business.dever", + "type Outcome { error Denied(message: Text) }\nbusiness() () { fail(Outcome.Denied(\"test business denial\")) }", + ); + let result = cli(&compiler, directory, "test", &[]); + failure(&result, "4 passed; 2 failed; 0 not run"); + let report = String::from_utf8_lossy(&result.stdout); + assert!( + report.contains("test sample/account/second ... ok"), + "{report}" + ); + assert!(report.contains("failure.dever"), "{report}"); + assert!( + report.contains("assert_eq failed: actual = 1, expected = 2"), + "{report}" + ); + assert!(report.contains("test business denial"), "{report}"); + assert!(!directory.join("data").exists()); + assert_eq!(cached_programs(root.path()).len(), 1); + let suite = &cached_programs(root.path())[0]; + for args in [ + vec![], + vec![""], + vec!["-1"], + vec!["+0"], + vec![" 0"], + vec!["0 "], + vec!["18446744073709551616"], + vec!["6"], + vec!["0", "extra"], + ] { + let rejected = output(Command::new(suite).env_clear().args(args)); + failure(&rejected, "test program requires a valid case index"); + assert_eq!( + rejected.stderr, + b"test program requires a valid case index\n" + ); + } +} diff --git a/test/dever-cli-tests/native_release.rs b/test/dever-cli-tests/native_release.rs new file mode 100644 index 0000000..7125a28 --- /dev/null +++ b/test/dever-cli-tests/native_release.rs @@ -0,0 +1,542 @@ +//! Authoring contracts use tiny synthetic payloads; real link/install checks are opt-in. +#![cfg(all( + target_os = "linux", + any(target_arch = "x86_64", target_arch = "aarch64") +))] + +use std::collections::BTreeMap; +use std::fs; +use std::os::unix::fs::{MetadataExt, PermissionsExt, symlink}; +use std::path::{Path, PathBuf}; + +use dever_cli::toolchain::{BuildTarget, packaging, platform_identity, runtime_pack, sha256_file}; +use ring::rand::SystemRandom; +use ring::signature::{ED25519, Ed25519KeyPair, KeyPair, UnparsedPublicKey}; +use serde_json::{Value, json}; + +#[path = "../dever-tests/tests/support/temp.rs"] +mod temp; +use temp::TemporaryDirectory; + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "native_release/acceptance.rs"] +mod acceptance; + +#[path = "native_release/ecosystems.rs"] +mod ecosystems; + +#[cfg(target_os = "linux")] +#[path = "native_release/bootstrap.rs"] +mod bootstrap; + +#[path = "support/native_elf.rs"] +mod native_elf; +#[path = "support/native_objects.rs"] +mod native_objects; + +#[path = "native_release/targets.rs"] +mod targets; + +#[test] +fn sandbox_bootstrap_requires_static_elf_and_complete_guard_libraries() { + let loader = format!("lib/{}", dever_sandbox::loader_name()); + let mut files = vec![ + ("bin/bwrap".into(), native_elf::static_executable()), + ( + "bin/guard".into(), + native_elf::elf(None, &["libc.so.6"], None), + ), + ( + loader, + native_elf::library(dever_sandbox::loader_name(), &[]), + ), + ( + "lib/libc.so.6".into(), + native_elf::library("libc.so.6", &[]), + ), + ]; + dever_sandbox::validate_assets(&files).unwrap(); + files[0].1 = native_elf::elf(None, &[], None); + assert!( + dever_sandbox::validate_assets(&files) + .unwrap_err() + .contains("static target ELF") + ); + files[0].1 = native_elf::library("bwrap", &["libc.so.6"]); + assert!( + dever_sandbox::validate_assets(&files) + .unwrap_err() + .contains("static target ELF") + ); + files[0].1 = native_elf::static_executable(); + files.pop(); + assert!( + dever_sandbox::validate_assets(&files) + .unwrap_err() + .contains("sandbox dependency 'lib/libc.so.6' is missing") + ); +} + +struct Author { + directory: TemporaryDirectory, + root: PathBuf, + settings: Value, + public_key: Vec<u8>, +} + +impl Author { + fn new() -> Self { + let directory = TemporaryDirectory::new(); + let root = directory.path().join("author"); + fs::create_dir_all(root.join("config")).unwrap(); + fs::create_dir(root.join("inputs")).unwrap(); + let key = Ed25519KeyPair::generate_pkcs8(&SystemRandom::new()).unwrap(); + fs::write(root.join("signing.pk8"), key.as_ref()).unwrap(); + fs::set_permissions(root.join("signing.pk8"), fs::Permissions::from_mode(0o600)).unwrap(); + let public_key = Ed25519KeyPair::from_pkcs8(key.as_ref()) + .unwrap() + .public_key() + .as_ref() + .to_vec(); + let input = |name: &str, bytes: &[u8]| { + let source = format!("inputs/{name}"); + fs::write(root.join(&source), bytes).unwrap(); + json!({ "source": source, "sha256": sha256_file(&root.join(source)).unwrap() }) + }; + let core = input( + "core", + &native_elf::compiler(&["libLLVM.so.18.1", "libc.so.6"]), + ); + let mut llvm = input( + "llvm", + &native_elf::library("libLLVM.so.18.1", &["libc.so.6"]), + ); + llvm["path"] = json!("libLLVM.so.18.1"); + let mut profiles = json!({}); + for name in ["base", "sqlite", "postgres", "both"] { + profiles[name] = input( + name, + &native_objects::archive(BuildTarget::host().unwrap(), name), + ); + } + let link_input = |name: &str, bytes: &[u8]| { + let mut value = input(name, bytes); + value["path"] = json!(name); + value + }; + let start = link_input( + "start.o", + &native_objects::object(BuildTarget::host().unwrap()), + ); + let end = link_input( + "end.o", + &native_objects::object(BuildTarget::host().unwrap()), + ); + let library = link_input("system.a", b"!<arch>\n"); + let skill = link_input( + "SKILL.md", + b"---\nname: dever-language\ndescription: Develop Dever applications.\n---\n", + ); + let settings = json!({ + "format": "dever-native-release-input-v1", + "version": env!("CARGO_PKG_VERSION"), + "target": format!("{}-unknown-linux-gnu", std::env::consts::ARCH), + "signing_key": "signing.pk8", "core": core, "core_libraries": [llvm], + "skill": [skill], + "profiles": profiles, "start": [start], "libraries": [library], "end": [end], + }); + Self { + directory, + root, + settings, + public_key, + } + } + + fn save(&self) { + fs::write( + self.root.join("config/setting.json"), + serde_json::to_vec(&self.settings).unwrap(), + ) + .unwrap(); + } + + fn output(&self, name: &str) -> PathBuf { + self.directory.path().join(name) + } + + fn machine(&self) -> dever_cli::toolchain::Layout { + let layout = dever_cli::toolchain::Layout::new(self.output("machine")); + layout.initialize().unwrap(); + let key_path = layout.state().join("trusted-release-key"); + fs::write( + &key_path, + self.public_key + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::<String>(), + ) + .unwrap(); + fs::set_permissions(key_path, fs::Permissions::from_mode(0o644)).unwrap(); + layout + } + + fn reject(&self) -> String { + self.save(); + let output = self.output("rejected"); + let error = packaging::create(&self.root, &output).unwrap_err(); + assert!(!output.exists()); + assert_eq!( + fs::read_dir(self.directory.path()).unwrap().count(), + 1, + "staging leaked: {error}" + ); + assert!( + !error.contains("signing.pk8"), + "private key location leaked: {error}" + ); + error + } +} + +fn contents(directory: &Path) -> BTreeMap<PathBuf, Vec<u8>> { + fn visit(root: &Path, directory: &Path, files: &mut BTreeMap<PathBuf, Vec<u8>>) { + for entry in fs::read_dir(directory).unwrap() { + let entry = entry.unwrap(); + let path = entry.path(); + if entry.file_type().unwrap().is_dir() { + visit(root, &path, files); + } else { + files.insert( + path.strip_prefix(root).unwrap().to_owned(), + fs::read(path).unwrap(), + ); + } + } + } + let mut files = BTreeMap::new(); + visit(directory, directory, &mut files); + files +} + +fn assert_signed(author: &Author, release: &Path) { + let encoded = fs::read_to_string(release.join("manifest.sig")).unwrap(); + let (pairs, remainder) = encoded.trim().as_bytes().as_chunks::<2>(); + assert!(remainder.is_empty()); + let signature = pairs + .iter() + .map(|pair| u8::from_str_radix(std::str::from_utf8(pair).unwrap(), 16).unwrap()) + .collect::<Vec<_>>(); + UnparsedPublicKey::new(&ED25519, &author.public_key) + .verify( + &fs::read(release.join("manifest.json")).unwrap(), + &signature, + ) + .unwrap(); +} + +#[test] +fn release_skill_rejects_missing_entry_and_oversized_files() { + let mut author = Author::new(); + author.settings["skill"] = json!([]); + author.save(); + assert!(packaging::create(&author.root, &author.output("missing")).is_err()); + assert!(!author.output("missing").exists()); + let mut author = Author::new(); + fs::write( + author.root.join("inputs/SKILL.md"), + vec![b'x'; 1024 * 1024 + 1], + ) + .unwrap(); + author.settings["skill"][0]["sha256"] = + json!(sha256_file(&author.root.join("inputs/SKILL.md")).unwrap()); + author.save(); + assert!( + packaging::create(&author.root, &author.output("large")) + .unwrap_err() + .contains("limit") + ); +} + +#[test] +fn identical_inputs_produce_identical_signed_independent_releases() { + let author = Author::new(); + author.save(); + let first = author.output("first"); + let second = author.output("second"); + let manifest = packaging::create(&author.root, &first).unwrap(); + assert_eq!(packaging::create(&author.root, &second).unwrap(), manifest); + let before = contents(&first); + assert_eq!(contents(&second), before); + assert_eq!(before.len(), manifest.artifacts.len() + 2); + assert!( + manifest + .artifacts + .windows(2) + .all(|pair| pair[0].path < pair[1].path) + ); + for artifact in &manifest.artifacts { + assert_eq!( + sha256_file(&first.join(&artifact.path)).unwrap(), + artifact.sha256 + ); + assert_eq!(fs::metadata(first.join(&artifact.path)).unwrap().nlink(), 1); + } + assert_signed(&author, &first); + let runtime = runtime_pack::validate( + &first, + env!("CARGO_PKG_VERSION"), + BuildTarget::host().unwrap(), + ) + .unwrap(); + assert_eq!(runtime.len() + 3, manifest.artifacts.len()); + let metadata: Value = serde_json::from_slice( + &before[&PathBuf::from(format!("runtime/{}/manifest.json", platform_identity()))], + ) + .unwrap(); + for profile in ["base", "sqlite", "postgres", "both"] { + assert_eq!( + metadata["profiles"][profile]["runtime"], + format!("archives/{profile}.a") + ); + } + fs::write(author.root.join("inputs/base"), b"changed source").unwrap(); + assert_eq!(contents(&first), before); + assert_eq!(contents(&second), before); +} + +#[test] +fn compiler_libraries_must_form_a_closed_target_and_search_path() { + let invalid_libraries = [ + ( + native_elf::library("libLLVM.so.18.1", &["libz.so.1"]), + "not supplied", + ), + ( + native_elf::library("different.so", &["libc.so.6"]), + "SONAME", + ), + ( + native_elf::elf(Some("libLLVM.so.18.1"), &[], Some("/usr/lib")), + "search path", + ), + ( + native_elf::elf(Some("libLLVM.so.18.1"), &[], Some("$ORIGIN/../../lib")), + "search path", + ), + ( + native_elf::library("libLLVM.so.18.1", &["/usr/lib/libz.so.1"]), + "plain library", + ), + ]; + for (bytes, diagnostic) in invalid_libraries { + let mut author = Author::new(); + fs::write(author.root.join("inputs/llvm"), bytes).unwrap(); + author.settings["core_libraries"][0]["sha256"] = + json!(sha256_file(&author.root.join("inputs/llvm")).unwrap()); + assert!(author.reject().contains(diagnostic)); + } + let mut author = Author::new(); + let mut bytes = native_elf::compiler(&["libLLVM.so.18.1"]); + bytes[18..20].copy_from_slice(&0u16.to_le_bytes()); + fs::write(author.root.join("inputs/core"), bytes).unwrap(); + author.settings["core"]["sha256"] = + json!(sha256_file(&author.root.join("inputs/core")).unwrap()); + assert!(author.reject().contains("incompatible target")); + let mut author = Author::new(); + author.settings["core_libraries"] = json!([]); + assert!(author.reject().contains("not supplied")); + let mut author = Author::new(); + let bytes = native_elf::elf(None, &["libLLVM.so.18.1"], Some("/usr/lib")); + fs::write(author.root.join("inputs/core"), bytes).unwrap(); + author.settings["core"]["sha256"] = + json!(sha256_file(&author.root.join("inputs/core")).unwrap()); + assert!(author.reject().contains("inherited RPATH")); +} + +#[test] +fn installation_rechecks_the_signed_native_compiler_closure_before_health_execution() { + let author = Author::new(); + author.save(); + let layout = author.machine(); + let release = layout.downloads().join(env!("CARGO_PKG_VERSION")); + let mut manifest = packaging::create(&author.root, &release).unwrap(); + manifest + .artifacts + .retain(|artifact| artifact.path != "lib/libLLVM.so.18.1"); + fs::remove_file(release.join("lib/libLLVM.so.18.1")).unwrap(); + let encoded = serde_json::to_vec(&manifest).unwrap(); + let key = + Ed25519KeyPair::from_pkcs8(&fs::read(author.root.join("signing.pk8")).unwrap()).unwrap(); + fs::write(release.join("manifest.json"), &encoded).unwrap(); + fs::write( + release.join("manifest.sig"), + key.sign(&encoded) + .as_ref() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::<String>(), + ) + .unwrap(); + let error = dever_cli::toolchain::MachineManager::new(layout.clone()) + .install_requested(env!("CARGO_PKG_VERSION")) + .unwrap_err(); + assert!(error.contains("not supplied"), "{error}"); + assert!(!layout.versions().join(env!("CARGO_PKG_VERSION")).exists()); +} + +#[test] +fn native_core_resolution_rejects_unsigned_loader_search_entries() { + for entry in [ + "lib/libc.so.6", + "lib/glibc-hwcaps/x86-64-v3/libLLVM.so.18.1", + ] { + let author = Author::new(); + author.save(); + let layout = author.machine(); + let version = dever_cli::toolchain::Version::parse(env!("CARGO_PKG_VERSION")).unwrap(); + let release = layout.versions().join(version.as_str()); + packaging::create(&author.root, &release).unwrap(); + let extra = release.join(entry); + fs::create_dir_all(extra.parent().unwrap()).unwrap(); + fs::write(extra, b"unsigned loader input").unwrap(); + let error = dever_cli::toolchain::MachineManager::new(layout) + .resolve_core(&version) + .unwrap_err(); + assert!( + error.contains("unsigned file, directory or link"), + "{error}" + ); + } +} + +#[test] +fn rejects_invalid_configuration_and_rolls_back_partial_payloads() { + let changes = [ + ("/format", json!("unknown")), + ("/target", json!("x86_64-pc-windows-msvc")), + ("/version", json!("1.0")), + ("/unexpected", json!(true)), + ("/core/unexpected", json!(true)), + ("/core/source", json!("../outside")), + ("/core/source", json!("/etc/passwd")), + ("/core/source", json!("inputs/./core")), + ("/core/source", json!("inputs/missing")), + ("/profiles/base/sha256", json!("0".repeat(64))), + ("/profiles/base/sha256", json!("A".repeat(64))), + ("/profiles/postgres", Value::Null), + ("/start", json!([])), + ("/libraries", json!([])), + ("/end", json!([])), + ("/end/0/path", json!("start.o")), + ("/end/0/path", json!("archives/base.a")), + ("/end/0/path", json!("manifest.json")), + ("/end/0/path", json!("../../escape.o")), + ("/end/0/path", json!("end.txt")), + ]; + for (pointer, value) in changes { + let mut author = Author::new(); + if let Some(slot) = author.settings.pointer_mut(pointer) { + *slot = value; + } else if pointer == "/unexpected" { + author.settings["unexpected"] = value; + } else { + author.settings["core"]["unexpected"] = value; + } + author.reject(); + } + let mut author = Author::new(); + author.settings["profiles"] + .as_object_mut() + .unwrap() + .remove("sqlite"); + author.reject(); + for bytes in [b"!<thin>\n".as_slice(), b"linker script".as_slice()] { + let mut author = Author::new(); + fs::write(author.root.join("inputs/base"), bytes).unwrap(); + author.settings["profiles"]["base"]["sha256"] = + json!(sha256_file(&author.root.join("inputs/base")).unwrap()); + assert!(author.reject().contains("self-contained archive")); + } +} + +#[test] +fn private_key_and_symlink_inputs_never_enter_the_release() { + let author = Author::new(); + fs::set_permissions( + author.root.join("signing.pk8"), + fs::Permissions::from_mode(0o644), + ) + .unwrap(); + assert!(author.reject().contains("permissions")); + let mut author = Author::new(); + symlink("signing.pk8", author.root.join("key-link")).unwrap(); + author.settings["signing_key"] = json!("key-link"); + author.reject(); + let mut author = Author::new(); + symlink("inputs/core", author.root.join("payload-link")).unwrap(); + author.settings["core"]["source"] = json!("payload-link"); + author.reject(); + let mut author = Author::new(); + symlink("inputs", author.root.join("directory-link")).unwrap(); + author.settings["core"]["source"] = json!("directory-link/core"); + author.reject(); + let mut author = Author::new(); + fs::copy( + author.root.join("signing.pk8"), + author.root.join("inputs/key-copy"), + ) + .unwrap(); + author.settings["core"] = json!({ "source":"inputs/key-copy", "sha256":sha256_file(&author.root.join("inputs/key-copy")).unwrap() }); + assert!(author.reject().contains("cannot be included")); +} + +#[test] +fn existing_outputs_are_preserved_and_concurrent_creation_has_one_winner() { + let author = Author::new(); + author.save(); + let existing = author.output("existing"); + fs::create_dir(&existing).unwrap(); + fs::write(existing.join("keep"), b"original").unwrap(); + let before = contents(&existing); + assert!(packaging::create(&author.root, &existing).is_err()); + assert_eq!(contents(&existing), before); + let empty = author.output("empty"); + fs::create_dir(&empty).unwrap(); + assert!(packaging::create(&author.root, &empty).is_err()); + assert!(empty.is_dir()); + let link = author.output("link"); + symlink(&existing, &link).unwrap(); + assert!(packaging::create(&author.root, &link).is_err()); + assert_eq!(contents(&existing), before); + let output = author.output("concurrent"); + let outcomes = std::thread::scope(|scope| { + let first = scope.spawn(|| packaging::create(&author.root, &output)); + let second = scope.spawn(|| packaging::create(&author.root, &output)); + [first.join().unwrap(), second.join().unwrap()] + }); + assert_eq!(outcomes.iter().filter(|result| result.is_ok()).count(), 1); + runtime_pack::validate( + &output, + env!("CARGO_PKG_VERSION"), + BuildTarget::host().unwrap(), + ) + .unwrap(); + assert_eq!(fs::read_dir(author.directory.path()).unwrap().count(), 5); +} + +#[test] +fn duplicate_fields_and_invalid_private_keys_fail_before_publication() { + let author = Author::new(); + author.save(); + let config = author.root.join("config/setting.json"); + let original = fs::read_to_string(&config).unwrap(); + let duplicate = original.replacen('{', "{\"signing_key\":\"signing.pk8\",", 1); + fs::write(&config, duplicate).unwrap(); + let error = packaging::create(&author.root, &author.output("duplicate")).unwrap_err(); + assert!(error.contains("invalid author configuration")); + assert!(!error.contains("signing.pk8")); + assert_eq!(fs::read_dir(author.directory.path()).unwrap().count(), 1); + fs::write(author.root.join("signing.pk8"), b"invalid private key").unwrap(); + assert!(author.reject().contains("PKCS#8")); +} diff --git a/test/dever-cli-tests/native_release/acceptance.rs b/test/dever-cli-tests/native_release/acceptance.rs new file mode 100644 index 0000000..7e7e517 --- /dev/null +++ b/test/dever-cli-tests/native_release/acceptance.rs @@ -0,0 +1,567 @@ +//! Explicit Linux acceptance using four separately built optimized archives. +use std::fs; +use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output, Stdio}; +use std::time::Duration; + +use dever_cli::toolchain::{Layout, MachineManager, Version, cache_status, packaging, sha256_file}; +use ring::rand::SystemRandom; +use ring::signature::{Ed25519KeyPair, KeyPair}; +use serde::Deserialize; +use serde_json::{Value, json}; + +#[path = "../support/daemon.rs"] +mod daemon; +#[path = "../support/llvm_inputs.rs"] +mod inputs; +#[path = "../../dever-tests/tests/support/process.rs"] +mod process; +#[path = "../../dever-tests/tests/support/sandbox.rs"] +mod sandbox; +use super::temp::TemporaryDirectory; + +#[path = "ecosystem_acceptance.rs"] +mod ecosystems; + +#[path = "bootstrap_acceptance.rs"] +mod bootstrap; + +#[path = "cross_acceptance.rs"] +mod cross; + +#[path = "command_acceptance.rs"] +mod command_acceptance; + +const PROFILES: [&str; 4] = ["base", "sqlite", "postgres", "both"]; + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct AcceptanceSettings { + machine_temporary_root: Option<PathBuf>, +} + +fn machine_temporary_root(workspace: &Path) -> Result<Option<PathBuf>, String> { + let path = workspace.join("target/native-release-inputs/config/setting.json"); + let bytes = match fs::read(&path) { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(format!("cannot read acceptance settings: {error}")), + }; + let settings: AcceptanceSettings = serde_json::from_slice(&bytes) + .map_err(|error| format!("invalid acceptance settings: {error}"))?; + if let Some(root) = &settings.machine_temporary_root + && (!root.is_absolute() + || root + .components() + .any(|part| matches!(part, std::path::Component::ParentDir))) + { + return Err("machine_temporary_root must be an absolute path without '..'".into()); + } + Ok(settings.machine_temporary_root) +} + +fn machine_temporary_directory(workspace: &Path) -> TemporaryDirectory { + match machine_temporary_root(workspace).expect("valid native release acceptance settings") { + Some(root) => TemporaryDirectory::new_in(&root), + None => TemporaryDirectory::new(), + } +} + +#[test] +fn acceptance_machine_directory_uses_only_explicit_author_settings() { + let workspace = TemporaryDirectory::new(); + assert_eq!(machine_temporary_root(workspace.path()).unwrap(), None); + let config = "target/native-release-inputs/config/setting.json"; + write( + workspace.path(), + config, + br#"{"machine_temporary_root":"relative"}"#, + ); + assert!( + machine_temporary_root(workspace.path()) + .unwrap_err() + .contains("absolute path") + ); + write( + workspace.path(), + config, + br#"{"machine_temporary_root":"/tmp","unknown":true}"#, + ); + assert!( + machine_temporary_root(workspace.path()) + .unwrap_err() + .contains("unknown field") + ); + let selected = TemporaryDirectory::new(); + write( + workspace.path(), + config, + serde_json::to_vec(&json!({"machine_temporary_root":selected.path()})).unwrap(), + ); + let directory = machine_temporary_directory(workspace.path()); + assert_eq!(directory.path().parent(), Some(selected.path())); + assert!(directory.path().is_dir()); + let path = directory.path().to_owned(); + drop(directory); + assert!(!path.exists()); + assert!(selected.path().is_dir()); +} + +fn write(root: &Path, relative: &str, bytes: impl AsRef<[u8]>) { + let path = root.join(relative); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, bytes).unwrap(); +} + +fn input(author: &Path, source: &Path, name: &str) -> Value { + let relative = format!("inputs/{name}"); + let path = author.join(&relative); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + // This is immutable test input staging, never a published release artifact. + fs::hard_link(source, &path).unwrap(); + json!({"source":relative,"sha256":sha256_file(&path).unwrap()}) +} + +fn author_inputs(author: &Path, workspace: &Path) -> Ed25519KeyPair { + let document = Ed25519KeyPair::generate_pkcs8(&SystemRandom::new()).unwrap(); + let key = Ed25519KeyPair::from_pkcs8(document.as_ref()).unwrap(); + use std::io::Write; + fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(author.join("signing.pk8")) + .unwrap() + .write_all(document.as_ref()) + .unwrap(); + let mut profiles = serde_json::Map::new(); + for profile in PROFILES { + let name = format!("{profile}.a"); + profiles.insert( + profile.into(), + input( + author, + &workspace.join("target/native-release-inputs").join(&name), + &name, + ), + ); + } + let compiler = input(author, Path::new(env!("CARGO_BIN_EXE_dever")), "dever"); + let core_libraries = inputs::core_libraries() + .into_iter() + .map(|(name, source)| { + let mut value = input(author, &source, name); + value["path"] = json!(name); + value + }) + .collect::<Vec<_>>(); + let mut links = serde_json::Map::new(); + for (name, source) in inputs::native_inputs() + .into_iter() + .filter(|(name, _)| *name != "runtime.a") + { + let mut entry = input(author, &source, name); + entry["path"] = json!(name); + links.insert(name.into(), entry); + } + let ordered = |names: &[&str]| { + names + .iter() + .map(|name| links[*name].clone()) + .collect::<Vec<_>>() + }; + let sandbox = sandbox::assets(workspace).into_iter().map(|asset| { + let relative = format!("inputs/{}", asset.path); + write(author, &relative, &asset.bytes); + fs::set_permissions(author.join(&relative), fs::Permissions::from_mode(if asset.executable { 0o755 } else { 0o644 })).unwrap(); + json!({"source":relative, "path":asset.path.strip_prefix("sandbox/").unwrap(), "sha256":sha256_file(&author.join(&relative)).unwrap()}) + }).collect::<Vec<_>>(); + write( + author, + "inputs/SKILL.md", + b"---\nname: dever-language\ndescription: Dever acceptance skill.\n---\n", + ); + let skill = json!({"source":"inputs/SKILL.md", "path":"SKILL.md", "sha256":sha256_file(&author.join("inputs/SKILL.md")).unwrap()}); + write( + author, + "config/setting.json", + serde_json::to_vec(&json!({ + "format":"dever-native-release-input-v1", "version":env!("CARGO_PKG_VERSION"), + "target":"x86_64-unknown-linux-gnu", "signing_key":"signing.pk8", + "core":compiler, "core_libraries":core_libraries, "profiles":profiles, + "skill":[skill], + "start":ordered(&["crt1.o","crti.o","crtbeginT.o"]), + "libraries":ordered(&["libc.a","libm.a","libmvec.a","libgcc.a","libgcc_eh.a"]), + "end":ordered(&["crtend.o","crtn.o"]), "sandbox":sandbox + })) + .unwrap(), + ); + key +} + +fn command(command: &mut Command) -> Output { + command_with_timeout(command, Duration::from_secs(190)) +} + +fn isolated_os_command(root: &Path) -> Command { + let mut command = isolated_os_namespace(root); + command.args(["--", "/fixture-proc-init"]); + command +} + +fn isolated_os_namespace(root: &Path) -> Command { + let assets = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../target/sandbox-inputs/assets") + .canonicalize() + .expect("prepare the explicit Linux isolation assets"); + let init = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../target/debug/examples/native-acceptance-init") + .canonicalize() + .expect("build the explicit static native-acceptance-init example"); + let bytes = fs::read(&init).unwrap(); + let executable = goblin::elf::Elf::parse(&bytes).unwrap(); + assert!(executable.interpreter.is_none() && executable.libraries.is_empty()); + // A plain chroot cannot create a nested user namespace. Use a real mount + // namespace root so the program can retain its normal Worker sandbox. + dever_sandbox::validate_bwrap(&fs::read(assets.join("bin/bwrap")).unwrap()).unwrap(); + let mut command = Command::new(assets.join("bin/bwrap")); + command + .env_clear() + .args([ + "--unshare-all", + "--die-with-parent", + "--new-session", + "--bind", + ]) + .arg(root) + .args(["/", "--proc", "/proc", "--dev", "/dev", "--chdir", "/"]) + .arg("--ro-bind") + .arg(init) + .arg("/fixture-proc-init"); + command +} + +fn command_with_timeout(command: &mut Command, timeout: Duration) -> Output { + let capture = TemporaryDirectory::new(); + let stdout = capture.path().join("stdout"); + let stderr = capture.path().join("stderr"); + let status = process::status( + command + .env_clear() + .stdout(Stdio::from(fs::File::create(&stdout).unwrap())) + .stderr(Stdio::from(fs::File::create(&stderr).unwrap())), + timeout, + ) + .unwrap(); + let output = Output { + status, + stdout: fs::read(stdout).unwrap(), + stderr: fs::read(stderr).unwrap(), + }; + assert!( + output.status.success(), + "{:?}: {}\n{}", + command, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + output +} + +fn application(root: &Path, profile: &str) { + let sqlite = json!({"type":"sqlite","path":"data/owned.sqlite","max_connections":1}); + let postgres = + json!({"type":"postgres","url":"postgres://fixture@127.0.0.1:1/unused", "tls":"disabled"}); + let settings = match profile { + "base" => json!({}), + "sqlite" => json!({"database":{"default":sqlite}}), + "postgres" => json!({"database":{"default":postgres}}), + "both" => json!({"database":{"default":sqlite,"unused":postgres}}), + _ => unreachable!(), + }; + write( + root, + "config/setting.json", + serde_json::to_vec(&settings).unwrap(), + ); + write( + root, + "module/sample/value/api.dever", + "cmd exercise = app.exercise\n", + ); + let app = if matches!(profile, "sqlite" | "both") { + write( + root, + "module/sample/value/model.dever", + "type Value { name: Text(1, 32) }\n", + ); + "exercise() (okay: Bool) { saved = model.create({ name = \"pack\" })\n count = model.count()\n removed = model.delete(saved.id)\n okay = count == 1 and removed == 1 }\n" + } else { + "exercise() (okay: Bool) { dever.task.sleep(1)\n okay = true }\n" + }; + write(root, "module/sample/value/app.dever", app); +} + +fn assert_response(output: Output) { + assert_response_data(output, json!(true)); +} + +fn assert_response_data(output: Output, expected: Value) { + let value: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(value["code"], 0); + assert_eq!(value["data"], expected); +} + +fn exec_application(root: &Path, workspace: &Path) { + for (name, source) in [ + ("app", "read() (value: Int) { value = port.read() }"), + ("port", "read() (value: Int) fails dever.time.SleepResult"), + ( + "adapter", + "setting { mode: Text }\nexternal exec \"worker\" {}", + ), + ("api", "cmd read = app.read"), + ] { + write( + root, + &format!("module/notification/delivery/{name}.dever"), + source, + ); + } + write( + root, + "config/setting.json", + br#"{"adapter":{"notification.delivery":{"setting":{"mode":"embedded"}}}}"#, + ); + let fixture = workspace.join("target/native-release-inputs/component-fixture"); + assert!( + fixture.is_file(), + "prepare the immutable component-fixture in target/native-release-inputs" + ); + fs::copy(fixture, root.join("module/notification/delivery/worker")).unwrap(); +} + +fn install_release(layout: &Layout, version: &Version, key: &Ed25519KeyPair) { + let public_key = key + .public_key() + .as_ref() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::<String>(); + fs::write(layout.state().join("trusted-release-key"), public_key).unwrap(); + fs::set_permissions( + layout.state().join("trusted-release-key"), + fs::Permissions::from_mode(0o644), + ) + .unwrap(); + MachineManager::new(layout.clone()) + .install_requested(version.as_str()) + .unwrap(); +} + +fn archive_report(workspace: &Path) -> Value { + let mut profiles = serde_json::Map::new(); + for profile in PROFILES { + let path = workspace + .join("target/native-release-inputs") + .join(format!("{profile}.a")); + let output = command( + Command::new("/usr/bin/nm") + .args(["--defined-only", "--demangle"]) + .arg(&path), + ); + let symbols = String::from_utf8(output.stdout).unwrap(); + assert_eq!( + symbols.contains(" sqlite3_open"), + matches!(profile, "sqlite" | "both"), + "{profile} SQLite driver" + ); + assert_eq!( + symbols.contains("tokio_postgres"), + matches!(profile, "postgres" | "both"), + "{profile} PostgreSQL driver" + ); + assert!(!symbols.contains("LLVMContextCreate")); + profiles.insert(profile.into(), json!({"bytes":fs::metadata(&path).unwrap().len(),"sha256":sha256_file(&path).unwrap()})); + } + json!(profiles) +} + +#[test] +#[ignore = "requires prepared optimized archives and root for an owned GNU ABI root"] +fn signed_core_builds_without_host_compiler_libraries_or_tools() { + assert_eq!( + rustix::process::geteuid().as_raw(), + 0, + "explicit owned namespace acceptance requires root" + ); + let workspace = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let author = TemporaryDirectory::new(); + author_inputs(author.path(), &workspace); + let release = machine_temporary_directory(&workspace); + let payload = release.path().join("release"); + let manifest = packaging::create(author.path(), &payload).unwrap(); + let isolated = machine_temporary_directory(&workspace); + ecosystems::minimal_os(isolated.path()); + let compiler_root = isolated.path().join("compiler"); + for artifact in &manifest.artifacts { + let relative = if artifact.path == "dever-core" { + "dever" + } else { + &artifact.path + }; + let destination = compiler_root.join(relative); + fs::create_dir_all(destination.parent().unwrap()).unwrap(); + // Immutable signed inputs; no host directory is mounted into the root. + fs::hard_link(payload.join(&artifact.path), destination).unwrap(); + } + application(&isolated.path().join("project"), "sqlite"); + let invoke = |arguments: &[&str]| command(isolated_os_command(isolated.path()).args(arguments)); + invoke(&["/compiler/dever", "check", "/project"]); + invoke(&[ + "/compiler/dever", + "build", + "/project", + "--output", + "/project/program", + ]); + fs::remove_dir_all(isolated.path().join("project/module")).unwrap(); + assert_response(invoke(&["/project/program", "sample.value.exercise", "{}"])); + write(&workspace, "target/closure-core-loader-acceptance.json", serde_json::to_vec_pretty(&json!({ + "platform":"linux-x86_64", "os_baseline":"GNU libc 2.39 and its named companion libraries", + "core_libraries":manifest.artifacts.iter().filter(|artifact| artifact.path.starts_with("lib/")).collect::<Vec<_>>(), + "checks":{"signed_payload":true,"check_in_minimal_root":true,"sqlite_build_in_minimal_root":true,"source_removed_execution":true}, + "host_compiler_libraries":false,"host_language_tools":false + })).unwrap()); +} + +#[test] +#[ignore = "requires four optimized archives and component-fixture; starts only an owned daemon, SQLite and exec Worker fixtures"] +fn optimized_profiles_make_reproducible_signed_release_and_run_through_daemon() { + let workspace = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let profiles = archive_report(&workspace); + let author = TemporaryDirectory::new(); + let key = author_inputs(author.path(), &workspace); + let machine = machine_temporary_directory(&workspace); + let layout = Layout::new(machine.path()); + layout.initialize().unwrap(); + let first = machine.path().join("first"); + let expected = packaging::create(author.path(), &first).unwrap(); + let manifest = fs::read(first.join("manifest.json")).unwrap(); + let signature = fs::read(first.join("manifest.sig")).unwrap(); + // Remove only this test's first completed copy before making the second. + fs::remove_dir_all(&first).unwrap(); + let version = Version::parse(env!("CARGO_PKG_VERSION")).unwrap(); + let release = layout.downloads().join(version.as_str()); + command( + Command::new(workspace.join("target/debug/examples/native-release")) + .arg(author.path()) + .arg("--output") + .arg(&release), + ); + assert_eq!( + serde_json::from_slice::<dever_cli::toolchain::ReleaseManifest>( + &fs::read(release.join("manifest.json")).unwrap() + ) + .unwrap(), + expected + ); + assert_eq!(fs::read(release.join("manifest.json")).unwrap(), manifest); + assert_eq!(fs::read(release.join("manifest.sig")).unwrap(), signature); + install_release(&layout, &version, &key); + fs::remove_dir_all(&release).unwrap(); + drop(author); + let compiler = layout.versions().join(version.as_str()).join("dever-core"); + let mut daemon = daemon::TestDaemon::start(&layout); + let projects = TemporaryDirectory::new(); + let mut binaries = serde_json::Map::new(); + let mut programs: Vec<(PathBuf, PathBuf)> = Vec::new(); + for profile in PROFILES { + let project = projects.path().join(profile); + application(&project, profile); + command(Command::new(&compiler).arg("check").arg(&project)); + assert_response(command( + Command::new(&compiler).arg("run").arg(&project).args([ + "--", + "sample.value.exercise", + "{}", + ]), + )); + let executable = project.join("program"); + command( + Command::new(&compiler) + .arg("build") + .arg(&project) + .arg("--output") + .arg(&executable), + ); + binaries.insert(profile.into(), json!({"bytes":fs::metadata(&executable).unwrap().len(),"sha256":sha256_file(&executable).unwrap()})); + programs.push((project, executable)); + } + assert_eq!(cache_status(&layout).unwrap().entries, 4); + let exec_project = TemporaryDirectory::new(); + let exec_source = exec_project.path().to_owned(); + exec_application(&exec_source, &workspace); + assert_response_data( + command(Command::new(&compiler).arg("run").arg(&exec_source).args([ + "--", + "notification.delivery.read", + "{}", + ])), + json!(7), + ); + let standalone = TemporaryDirectory::new(); + let exec_program = standalone.path().join("application"); + fs::create_dir(standalone.path().join("config")).unwrap(); + fs::copy( + exec_source.join("config/setting.json"), + standalone.path().join("config/setting.json"), + ) + .unwrap(); + command( + Command::new(&compiler) + .arg("build") + .arg(&exec_source) + .arg("--output") + .arg(&exec_program), + ); + assert_eq!(cache_status(&layout).unwrap().entries, 5); + drop(exec_project); + assert!(!exec_source.exists()); + daemon.stop(); + assert_response_data( + command( + Command::new(&exec_program) + .current_dir(standalone.path()) + .args(["notification.delivery.read", "{}"]), + ), + json!(7), + ); + assert!(!standalone.path().join("module").exists()); + let exec = json!({ + "bytes": fs::metadata(&exec_program).unwrap().len(), + "sha256": sha256_file(&exec_program).unwrap(), + "managed_run": true, "standalone_run": true, + "source_project_removed": true, "response": 7 + }); + for (project, executable) in programs { + fs::remove_dir_all(project.join("module")).unwrap(); + assert_response(command( + Command::new(executable).args(["sample.value.exercise", "{}"]), + )); + } + let report = json!({ + "format":"dever-native-release-acceptance-v1", "platform":"linux-x86_64", + "version":version.as_str(), "profiles":profiles,"programs":binaries,"exec":exec, + "manifest_sha256":format!("{:x}", sha2::Sha256::digest(&manifest)), + "checks":{"pack_determinism":true,"signed_install":true,"managed_profiles":4,"standalone_profiles":4,"sqlite_crud":true,"signed_exec_worker":true}, + "postgres_server":"not_run", "cross_host_source_reproducibility":"not_run" + }); + use sha2::Digest; + write( + &workspace, + "target/closure-native-release-acceptance.json", + serde_json::to_vec_pretty(&report).unwrap(), + ); +} diff --git a/test/dever-cli-tests/native_release/bootstrap.rs b/test/dever-cli-tests/native_release/bootstrap.rs new file mode 100644 index 0000000..5b33c7b --- /dev/null +++ b/test/dever-cli-tests/native_release/bootstrap.rs @@ -0,0 +1,410 @@ +//! Bootstrap package validation and recovery use only owned fixture directories. +use super::*; +use dever_cli::toolchain::bootstrap; +use sha2::{Digest, Sha256}; + +#[test] +fn bootstrap_service_recovery_orders_actions_around_file_restoration() { + use bootstrap::RecoveryAction::{Reload, Restart, Stop}; + for active in [false, true] { + for loaded in [false, true] { + assert_eq!( + bootstrap::recovery_actions(false, active, loaded), + (vec![], vec![]) + ); + } + } + assert_eq!( + bootstrap::recovery_actions(true, false, false), + (vec![], vec![Reload]) + ); + assert_eq!( + bootstrap::recovery_actions(true, false, true), + (vec![Stop], vec![Reload]) + ); + assert_eq!( + bootstrap::recovery_actions(true, true, true), + (vec![Stop], vec![Reload, Restart]) + ); + assert_eq!( + bootstrap::recovery_actions(true, true, false), + (vec![], vec![Reload, Restart]) + ); +} + +fn add_bootstrap(author: &mut Author) { + let input = |name: &str, bytes: &[u8]| { + let relative = format!("inputs/{name}"); + fs::write(author.root.join(&relative), bytes).unwrap(); + json!({"source":relative,"sha256":sha256_file(&author.root.join(&relative)).unwrap()}) + }; + let launcher = input( + "dever", + &native_elf::compiler(&["libgcc_s.so.1", "libc.so.6"]), + ); + let daemon = input( + "deverd", + &native_elf::compiler(&["libgcc_s.so.1", "libc.so.6"]), + ); + let mut gcc = input("gcc", &native_elf::library("libgcc_s.so.1", &["libc.so.6"])); + gcc["path"] = json!("libgcc_s.so.1"); + author.settings["bootstrap"] = json!({"launcher":launcher,"daemon":daemon,"libraries":[gcc]}); +} + +#[test] +fn bootstrap_payload_has_a_separate_signed_complete_loader_closure() { + let mut author = Author::new(); + add_bootstrap(&mut author); + author.save(); + let release = author.output("complete"); + let manifest = packaging::create(&author.root, &release).unwrap(); + assert_signed(&author, &release); + for path in [ + "bootstrap/dever", + "bootstrap/deverd", + "bootstrap/lib/libgcc_s.so.1", + "bootstrap/deverd.service", + "bootstrap/dever-sandbox", + ] { + let artifact = manifest + .artifacts + .iter() + .find(|artifact| artifact.path == path) + .unwrap(); + assert_eq!(artifact.sha256, sha256_file(&release.join(path)).unwrap()); + } + let unit = fs::read_to_string(release.join("bootstrap/deverd.service")).unwrap(); + assert!(unit.contains("ExecStart=/opt/dever/bin/deverd --root /opt/dever")); + assert!(unit.contains("RuntimeDirectory=dever")); + assert!(!unit.contains("Environment")); + assert!(!unit.contains("PATH")); + assert_eq!( + fs::read_to_string(release.join("bootstrap/dever-sandbox")).unwrap(), + dever_sandbox::APPARMOR_PROFILE + ); +} + +#[test] +fn bootstrap_authoring_rejects_missing_libraries_wrong_sonames_and_search_paths() { + let mut author = Author::new(); + add_bootstrap(&mut author); + author.settings["bootstrap"]["libraries"] = json!([]); + assert!(author.reject().contains("libgcc_s.so.1")); + + let mut author = Author::new(); + add_bootstrap(&mut author); + fs::write( + author.root.join("inputs/gcc"), + native_elf::library("libwrong.so", &["libc.so.6"]), + ) + .unwrap(); + author.settings["bootstrap"]["libraries"][0]["sha256"] = + json!(sha256_file(&author.root.join("inputs/gcc")).unwrap()); + assert!(author.reject().contains("SONAME")); + + let mut author = Author::new(); + add_bootstrap(&mut author); + fs::write( + author.root.join("inputs/dever"), + native_elf::elf(None, &["libgcc_s.so.1"], Some("/usr/lib")), + ) + .unwrap(); + author.settings["bootstrap"]["launcher"]["sha256"] = + json!(sha256_file(&author.root.join("inputs/dever")).unwrap()); + assert!(author.reject().contains("RPATH")); +} + +#[test] +fn signed_installed_versions_keep_their_original_bootstrap_service_template() { + use dever_cli::toolchain::{Layout, MachineManager, Version}; + let mut author = Author::new(); + add_bootstrap(&mut author); + author.save(); + let machine = TemporaryDirectory::new(); + let layout = Layout::new(machine.path().join("machine")); + layout.initialize().unwrap(); + let version = Version::parse(env!("CARGO_PKG_VERSION")).unwrap(); + let installed = layout.versions().join(version.as_str()); + let mut manifest = packaging::create(&author.root, &installed).unwrap(); + let service = installed.join("bootstrap/deverd.service"); + let current = fs::read_to_string(&service).unwrap(); + let old = current.replace("RestartSec=1", "RestartSec=2"); + assert_ne!(current, old); + fs::write(&service, old).unwrap(); + let artifact = manifest + .artifacts + .iter_mut() + .find(|artifact| artifact.path == "bootstrap/deverd.service") + .unwrap(); + artifact.bytes = fs::metadata(&service).unwrap().len(); + artifact.sha256 = sha256_file(&service).unwrap(); + let encoded = serde_json::to_vec(&manifest).unwrap(); + let key = + Ed25519KeyPair::from_pkcs8(&fs::read(author.root.join("signing.pk8")).unwrap()).unwrap(); + fs::write(installed.join("manifest.json"), &encoded).unwrap(); + fs::write( + installed.join("manifest.sig"), + key.sign(&encoded) + .as_ref() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::<String>(), + ) + .unwrap(); + fs::write( + layout.state().join("trusted-release-key"), + author + .public_key + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::<String>(), + ) + .unwrap(); + let manager = MachineManager::new(layout); + assert_eq!( + manager.resolve_core(&version).unwrap(), + installed.join("dever-core") + ); + fs::write(service, b"unsigned modification").unwrap(); + assert!( + manager + .resolve_core(&version) + .unwrap_err() + .contains("integrity verification") + ); +} + +#[test] +#[ignore = "root-only installer trust rejection; never touches the actual machine installation"] +fn bootstrap_installer_rejects_untrusted_inputs_before_publication() { + assert_eq!(rustix::process::geteuid().as_raw(), 0); + assert!( + bootstrap::install(Path::new(".")) + .unwrap_err() + .contains("absolute paths") + ); + assert!( + bootstrap::recover(Path::new(".")) + .unwrap_err() + .contains("absolute paths") + ); + let mut author = Author::new(); + add_bootstrap(&mut author); + author.save(); + let package = author.output("release"); + packaging::create(&author.root, &package).unwrap(); + let installer = TemporaryDirectory::new(); + let system = TemporaryDirectory::new(); + let key = installer.path().join("trusted.pub"); + fs::write( + &key, + author + .public_key + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::<String>(), + ) + .unwrap(); + fs::create_dir(installer.path().join("config")).unwrap(); + let save = |key: &Path, activate: bool| { + fs::write(installer.path().join("config/setting.json"), serde_json::to_vec(&json!({"release":package,"system_root":system.path(),"trusted_key":key,"activate_service":activate})).unwrap()).unwrap() + }; + save(&key, true); + assert!( + bootstrap::install(installer.path()) + .unwrap_err() + .contains("actual system root") + ); + let inside = package.join("trusted.pub"); + fs::copy(&key, &inside).unwrap(); + save(&inside, false); + assert!( + bootstrap::install(installer.path()) + .unwrap_err() + .contains("independently") + ); + save(&key, false); + fs::set_permissions(&key, fs::Permissions::from_mode(0o666)).unwrap(); + assert!( + bootstrap::install(installer.path()) + .unwrap_err() + .contains("root-owned") + ); + fs::set_permissions(&key, fs::Permissions::from_mode(0o644)).unwrap(); + let signature = fs::read(package.join("manifest.sig")).unwrap(); + fs::write(package.join("manifest.sig"), "00".repeat(64)).unwrap(); + assert!( + bootstrap::install(installer.path()) + .unwrap_err() + .contains("signature verification") + ); + fs::write(package.join("manifest.sig"), signature).unwrap(); + // The key is pinned before the intentionally non-executable synthetic core + // fails its health check. Empty/partial temporary files model interrupted + // writes without injecting product failpoints or touching real services. + let state = system.path().join("opt/dever/state"); + let temporary_key = state.join(".trusted-release-key.next"); + let pinned_key = state.join("trusted-release-key"); + for partial in ["", "abc"] { + fs::write(&temporary_key, partial).unwrap(); + let error = bootstrap::install(installer.path()).unwrap_err(); + assert!(!error.contains("temporary release key"), "{error}"); + assert_eq!( + fs::read_to_string(&pinned_key).unwrap().trim(), + fs::read_to_string(&key).unwrap() + ); + assert!(!temporary_key.exists()); + fs::remove_file(&pinned_key).unwrap(); + } + fs::write(&pinned_key, "11".repeat(32)).unwrap(); + assert!( + bootstrap::install(installer.path()) + .unwrap_err() + .contains("pinned release key") + ); + assert_eq!(fs::read_to_string(&pinned_key).unwrap(), "11".repeat(32)); + fs::remove_file(&pinned_key).unwrap(); + fs::write(package.join("bootstrap/lib/libgcc_s.so.1"), b"tampered").unwrap(); + assert!( + bootstrap::install(installer.path()) + .unwrap_err() + .contains("integrity verification") + ); + fs::remove_file(package.join("bootstrap/lib/libgcc_s.so.1")).unwrap(); + assert!(bootstrap::install(installer.path()).is_err()); + assert!(!system.path().join("usr/local/bin/dever").exists()); + assert!( + !system + .path() + .join("opt/dever/state/active-version") + .exists() + ); +} + +// Independent construction of a one-node journal identity lets recovery tests +// simulate interruption at publication boundaries without product failpoints. +fn node_hash(kind: &str, bytes: &[u8]) -> String { + let mut hash = Sha256::new(); + hash.update(0u64.to_le_bytes()); + hash.update( + if kind == "directory" { + 0o755u32 + } else { + 0o644u32 + } + .to_le_bytes(), + ); + hash.update(kind.as_bytes()); + if kind == "file" { + hash.update((bytes.len() as u64).to_le_bytes()); + hash.update(bytes); + } + format!("{:x}", hash.finalize()) +} + +#[test] +#[ignore = "root-only crash-recovery fixture; all journal targets are below an owned image root"] +fn bootstrap_recovery_is_idempotent_and_rejects_unrecorded_targets() { + assert_eq!(rustix::process::geteuid().as_raw(), 0); + let system = TemporaryDirectory::new(); + let root = system.path(); + let layout = dever_cli::toolchain::Layout::new(root.join("opt/dever")); + layout.initialize().unwrap(); + for directory in [ + "usr/local/bin", + "etc/systemd/system/multi-user.target.wants", + "etc/apparmor.d", + ] { + fs::create_dir_all(root.join(directory)).unwrap(); + } + let service = root.join("etc/systemd/system/deverd.service"); + fs::write(&service, b"new service").unwrap(); + let entry = root.join("usr/local/bin/dever"); + symlink("/opt/dever/bin/dever", &entry).unwrap(); + let enable_stage = root + .join("etc/systemd/system/multi-user.target.wants/.dever-bootstrap-deverd.service.next"); + symlink("../deverd.service", &enable_stage).unwrap(); + let active_stage = layout.state().join(".dever-bootstrap-active-version.next"); + fs::write(&active_stage, b"new active").unwrap(); + let sentinel = root.join("etc/systemd/system/unrelated.service"); + fs::write(&sentinel, b"keep").unwrap(); + let link_hash = + |target: &str| format!("{:x}", Sha256::digest(format!("link\0{target}").as_bytes())); + let journal = json!({"format":"dever-bootstrap-transaction-v1","phase":"prepared","service_was_active":null,"service_start_requested":false,"changes":[ + {"target":"sandbox","before":null,"after":node_hash("directory",b"")}, + {"target":"apparmor","before":null,"after":node_hash("file",b"new profile")}, + {"target":"bin","before":node_hash("directory",b""),"after":node_hash("directory",b"")}, + {"target":"service","before":null,"after":node_hash("file",b"new service")}, + {"target":"entry","before":null,"after":link_hash("/opt/dever/bin/dever")}, + {"target":"enable","before":null,"after":link_hash("../deverd.service")}, + {"target":"active","before":null,"after":node_hash("file",b"new active")} + ]}); + fs::create_dir(root.join("opt/dever/.dever-bootstrap-sandbox.next")).unwrap(); + fs::write( + root.join("etc/apparmor.d/.dever-bootstrap-dever-sandbox.next"), + b"new profile", + ) + .unwrap(); + let path = layout.state().join("bootstrap-install.json"); + fs::write(&path, serde_json::to_vec(&journal).unwrap()).unwrap(); + let mut invalid = journal.clone(); + invalid["changes"][0]["target"] = json!("../../unrelated"); + fs::write(&path, serde_json::to_vec(&invalid).unwrap()).unwrap(); + assert!(bootstrap::recover(root).is_err()); + assert_eq!(fs::read(&sentinel).unwrap(), b"keep"); + fs::write(&path, serde_json::to_vec(&journal).unwrap()).unwrap(); + fs::write(&service, b"not this transaction").unwrap(); + assert!( + bootstrap::recover(root) + .unwrap_err() + .contains("outside this transaction") + ); + assert_eq!(fs::read(&service).unwrap(), b"not this transaction"); + fs::write(&service, b"new service").unwrap(); + bootstrap::recover(root).unwrap(); + bootstrap::recover(root).unwrap(); + assert!(!service.exists()); + assert!(fs::symlink_metadata(&entry).is_err()); + assert!(!active_stage.exists()); + assert!(fs::symlink_metadata(&enable_stage).is_err()); + assert_eq!(fs::read(&sentinel).unwrap(), b"keep"); + assert!(!path.exists()); + + // A committed publication cleans only its old staging copies; it must not + // undo the new active version, including after a failed cleanup retry. + fs::write(&service, b"new service").unwrap(); + symlink("/opt/dever/bin/dever", &entry).unwrap(); + fs::create_dir(root.join("opt/dever/sandbox")).unwrap(); + fs::write(root.join("etc/apparmor.d/dever-sandbox"), b"new profile").unwrap(); + symlink( + "../deverd.service", + root.join("etc/systemd/system/multi-user.target.wants/deverd.service"), + ) + .unwrap(); + fs::write(layout.state().join("active-version"), b"new active").unwrap(); + fs::write(&active_stage, b"old active").unwrap(); + let mut committed = journal; + committed["phase"] = json!("committed"); + fs::write(&path, serde_json::to_vec(&committed).unwrap()).unwrap(); + fs::write(&service, b"unexpected modification").unwrap(); + assert!( + bootstrap::recover(root) + .unwrap_err() + .contains("committed bootstrap destination changed") + ); + assert!(path.exists()); + assert_eq!( + fs::read(layout.state().join("active-version")).unwrap(), + b"new active" + ); + fs::write(&service, b"new service").unwrap(); + bootstrap::recover(root).unwrap(); + bootstrap::recover(root).unwrap(); + assert_eq!( + fs::read(layout.state().join("active-version")).unwrap(), + b"new active" + ); + assert!(!active_stage.exists()); + assert!(!path.exists()); + assert_eq!(fs::read(sentinel).unwrap(), b"keep"); +} diff --git a/test/dever-cli-tests/native_release/bootstrap_acceptance.rs b/test/dever-cli-tests/native_release/bootstrap_acceptance.rs new file mode 100644 index 0000000..93c9ad0 --- /dev/null +++ b/test/dever-cli-tests/native_release/bootstrap_acceptance.rs @@ -0,0 +1,476 @@ +//! Real bootstrap installation and execution; no global service is changed. +use super::*; +use std::os::unix::fs::{MetadataExt, chown}; +use std::os::unix::process::CommandExt; +use std::process::Child; + +struct Daemon(Child); +impl Drop for Daemon { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +fn wait_socket(daemon: &mut Daemon, socket: &Path) { + for _ in 0..200 { + assert!( + daemon.0.try_wait().unwrap().is_none(), + "installed daemon exited before becoming ready" + ); + if socket.exists() { + return; + } + std::thread::sleep(Duration::from_millis(20)); + } + panic!("installed daemon did not create its socket"); +} + +/// Clients share the daemon's PID namespace so SO_PEERCRED can identify both +/// peers. Separate bwrap invocations would hide the daemon PID from clients. +struct IsolatedDaemon { + process: Daemon, + namespace_pid: u32, + namespace: std::os::fd::OwnedFd, + root: PathBuf, + stopped: bool, +} + +impl IsolatedDaemon { + fn start(root: &Path) -> Self { + let information = TemporaryDirectory::new(); + let path = information.path().join("namespace.json"); + let output = fs::File::create(&path).unwrap(); + let mut process = Daemon( + isolated_os_namespace(root) + .args([ + "--info-fd", + "1", + "--", + "/fixture-proc-init", + "/opt/dever/bin/deverd", + "--root", + "/opt/dever", + ]) + .stdout(Stdio::from(output)) + .stderr(Stdio::inherit()) + .spawn() + .unwrap(), + ); + wait_socket(&mut process, &root.join("run/dever/deverd.sock")); + assert!(fs::metadata(&path).unwrap().len() <= 4096); + let info: Value = serde_json::from_slice(&fs::read(path).unwrap()).unwrap(); + let namespace_pid = u32::try_from(info["child-pid"].as_u64().unwrap()).unwrap(); + assert_ne!(namespace_pid, 0); + let namespace = rustix::process::pidfd_open( + rustix::process::Pid::from_raw(i32::try_from(namespace_pid).unwrap()).unwrap(), + rustix::process::PidfdFlags::empty(), + ) + .unwrap(); + Self { + process, + namespace_pid, + namespace, + root: root.to_owned(), + stopped: false, + } + } + + fn client(&mut self) -> Command { + assert!(self.process.0.try_wait().unwrap().is_none()); + let target = fs::metadata(format!("/proc/{}/root", self.namespace_pid)).unwrap(); + let expected = fs::metadata(&self.root).unwrap(); + assert_eq!( + (target.dev(), target.ino()), + (expected.dev(), expected.ino()) + ); + let mut command = Command::new("/usr/bin/nsenter"); + command.env_clear().args([ + "--target", + &self.namespace_pid.to_string(), + "--all", + "--root", + "--wdns=/", + "--", + ]); + command + } + + fn stop(&mut self) -> Result<(), String> { + if self.stopped { + return Ok(()); + } + // Target the pinned namespace init, not a reusable numeric PID. Its + // termination also kills its namespace descendants; waiting only for + // the outer bwrap process does not establish daemon lock release. + match rustix::process::pidfd_send_signal(&self.namespace, rustix::process::Signal::KILL) { + Ok(()) | Err(rustix::io::Errno::SRCH) => {} + Err(error) => return Err(format!("cannot stop owned namespace: {error}")), + } + let expected = fs::metadata(&self.root).map_err(|error| error.to_string())?; + let lock = fs::File::open(self.root.join("opt/dever/state/deverd.lock")) + .map_err(|error| error.to_string())?; + let deadline = std::time::Instant::now() + Duration::from_secs(10); + loop { + let wrapper_exited = self + .process + .0 + .try_wait() + .map_err(|error| error.to_string())? + .is_some(); + let namespace_exited = match fs::metadata(format!("/proc/{}/root", self.namespace_pid)) + { + Ok(current) => (current.dev(), current.ino()) != (expected.dev(), expected.ino()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => true, + Err(error) => return Err(format!("cannot inspect owned namespace exit: {error}")), + }; + if wrapper_exited && namespace_exited { + match fs2::FileExt::try_lock_exclusive(&lock) { + Ok(()) => { + self.stopped = true; + return Ok(()); + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {} + Err(error) => { + return Err(format!("cannot check owned daemon lock release: {error}")); + } + } + } + if std::time::Instant::now() >= deadline { + return Err("owned namespace or daemon lock remained live after 10 seconds".into()); + } + std::thread::sleep(Duration::from_millis(20)); + } + } +} + +impl Drop for IsolatedDaemon { + fn drop(&mut self) { + // Cleanup must not cause a second panic while unwinding a failed + // assertion. Normal stage transitions explicitly check stop(). + if let Err(error) = self.stop() { + eprintln!("bootstrap fixture cleanup failed: {error}"); + } + } +} + +fn add_inputs(author: &Path) { + let launcher = input( + author, + Path::new(env!("CARGO_BIN_EXE_dever-launcher")), + "bootstrap/dever", + ); + let daemon = input( + author, + Path::new(env!("CARGO_BIN_EXE_deverd")), + "bootstrap/deverd", + ); + let mut gcc = input( + author, + Path::new("/lib/x86_64-linux-gnu/libgcc_s.so.1"), + "bootstrap/libgcc_s.so.1", + ); + gcc["path"] = json!("libgcc_s.so.1"); + let path = author.join("config/setting.json"); + let mut settings: Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + settings["bootstrap"] = json!({"launcher":launcher,"daemon":daemon,"libraries":[gcc]}); + fs::write(path, serde_json::to_vec(&settings).unwrap()).unwrap(); +} + +#[test] +#[ignore = "requires optimized author archives and Linux root; installs only into an owned image root"] +fn signed_bootstrap_installs_and_serves_two_projects_without_host_libraries() { + assert_eq!(rustix::process::geteuid().as_raw(), 0); + let workspace = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let author = TemporaryDirectory::new(); + let key = author_inputs(author.path(), &workspace); + add_inputs(author.path()); + let downloaded = machine_temporary_directory(&workspace); + let release = downloaded.path().join("release"); + let manifest = packaging::create(author.path(), &release).unwrap(); + let image = machine_temporary_directory(&workspace); + ecosystems::minimal_os(image.path()); + let installer = TemporaryDirectory::new(); + let public = installer.path().join("release.pub"); + fs::write( + &public, + key.public_key() + .as_ref() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::<String>(), + ) + .unwrap(); + write(installer.path(), "config/setting.json", serde_json::to_vec(&json!({ + "release":release,"system_root":image.path(),"trusted_key":public,"activate_service":false + })).unwrap()); + // This author-built installer is the independently trusted first program; + // no executable from an unchecked downloaded directory starts as root. + command( + Command::new(env!("CARGO_BIN_EXE_dever-launcher")) + .arg("--dever-install") + .arg(installer.path()), + ); + let layout = Layout::new(image.path().join("opt/dever")); + let sandbox_artifact = manifest + .artifacts + .iter() + .find(|artifact| artifact.path == "sandbox/linux-x86_64/bin/bwrap") + .unwrap(); + let sandbox_store = layout.root().join("sandbox"); + let sandbox_helper = sandbox_store.join(&sandbox_artifact.sha256).join("bwrap"); + assert_eq!( + sha256_file(&sandbox_helper).unwrap(), + sandbox_artifact.sha256 + ); + assert_eq!( + fs::metadata(&sandbox_helper).unwrap().mode() & 0o7777, + 0o755 + ); + assert_eq!( + fs::read_to_string(image.path().join("etc/apparmor.d/dever-sandbox")).unwrap(), + dever_sandbox::APPARMOR_PROFILE + ); + let launcher = layout.bin().join("dever"); + let executable = layout.bin().join("deverd"); + assert_eq!( + fs::read_link(image.path().join("usr/local/bin/dever")).unwrap(), + Path::new("/opt/dever/bin/dever") + ); + assert_eq!( + fs::read_link( + image + .path() + .join("etc/systemd/system/multi-user.target.wants/deverd.service") + ) + .unwrap(), + Path::new("../deverd.service") + ); + let unit = fs::read_to_string(image.path().join("etc/systemd/system/deverd.service")).unwrap(); + assert!(unit.contains("ExecStart=/opt/dever/bin/deverd --root /opt/dever")); + assert!(unit.contains("User=root")); + for file in [ + &launcher, + &executable, + &layout.bin().join("lib/libgcc_s.so.1"), + &layout.state().join("trusted-release-key"), + &image.path().join("etc/systemd/system/deverd.service"), + ] { + let metadata = fs::metadata(file).unwrap(); + assert_eq!(metadata.uid(), 0); + assert_eq!(metadata.mode() & 0o022, 0); + } + let active = fs::read(layout.state().join("active-version")).unwrap(); + let launcher_hash = sha256_file(&launcher).unwrap(); + // Model an older, correctly signed bootstrap service template. Its exact + // signed bytes establish ownership; only the incoming payload must match + // the current installer's supported service template. + let unit_path = image.path().join("etc/systemd/system/deverd.service"); + let current_unit = fs::read_to_string(&unit_path).unwrap(); + let old_unit = current_unit.replace("RestartSec=1", "RestartSec=2"); + assert_ne!(old_unit, current_unit); + fs::write(layout.bin().join("deverd.service"), &old_unit).unwrap(); + let mut old_manifest: Value = + serde_json::from_slice(&fs::read(layout.bin().join("manifest.json")).unwrap()).unwrap(); + let service_artifact = old_manifest["artifacts"] + .as_array_mut() + .unwrap() + .iter_mut() + .find(|artifact| artifact["path"] == "bootstrap/deverd.service") + .unwrap(); + service_artifact["bytes"] = json!(old_unit.len()); + service_artifact["sha256"] = json!(sha256_file(&layout.bin().join("deverd.service")).unwrap()); + let encoded = serde_json::to_vec(&old_manifest).unwrap(); + fs::write( + layout.bin().join("manifest.sig"), + key.sign(&encoded) + .as_ref() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::<String>(), + ) + .unwrap(); + fs::write(layout.bin().join("manifest.json"), encoded).unwrap(); + fs::write(&unit_path, b"unrelated system unit").unwrap(); + let rejected = Command::new(env!("CARGO_BIN_EXE_dever-launcher")) + .env_clear() + .arg("--dever-install") + .arg(installer.path()) + .output() + .unwrap(); + assert!(!rejected.status.success()); + assert!(String::from_utf8_lossy(&rejected.stderr).contains("unrelated system service")); + assert_eq!( + fs::read(layout.state().join("active-version")).unwrap(), + active + ); + fs::write(&unit_path, old_unit).unwrap(); + // Retain a different, correctly signed static helper identity through an + // upgrade. ELF trailing padding keeps this executable's actual behavior. + let mut previous_helper = fs::read(&sandbox_helper).unwrap(); + previous_helper.push(0); + let previous_hash = dever_sandbox::validate_bwrap(&previous_helper).unwrap(); + let previous_root = sandbox_store.join(&previous_hash); + fs::create_dir(&previous_root).unwrap(); + fs::write(previous_root.join("bwrap"), &previous_helper).unwrap(); + fs::set_permissions( + previous_root.join("bwrap"), + fs::Permissions::from_mode(0o755), + ) + .unwrap(); + let mut previous_manifest = manifest.clone(); + let previous_artifact = previous_manifest + .artifacts + .iter_mut() + .find(|artifact| artifact.path == sandbox_artifact.path) + .unwrap(); + previous_artifact.sha256 = previous_hash.clone(); + previous_artifact.bytes = previous_helper.len() as u64; + let previous_encoded = serde_json::to_vec(&previous_manifest).unwrap(); + fs::write( + previous_root.join("manifest.sig"), + key.sign(&previous_encoded) + .as_ref() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::<String>(), + ) + .unwrap(); + fs::write(previous_root.join("manifest.json"), previous_encoded).unwrap(); + // Retry is supported, but a later damaged package cannot damage the + // published launcher or compiler selection. + command( + Command::new(env!("CARGO_BIN_EXE_dever-launcher")) + .arg("--dever-install") + .arg(installer.path()), + ); + assert_eq!(fs::read_to_string(&unit_path).unwrap(), current_unit); + assert_eq!( + sha256_file(&sandbox_helper).unwrap(), + sandbox_artifact.sha256 + ); + assert_eq!( + sha256_file(&previous_root.join("bwrap")).unwrap(), + previous_hash + ); + assert_eq!(fs::read_dir(&sandbox_store).unwrap().count(), 2); + let before_bad_install = fs::read(layout.state().join("active-version")).unwrap(); + fs::write( + release.join("bootstrap/lib/libgcc_s.so.1"), + b"changed after signing", + ) + .unwrap(); + let failed = Command::new(env!("CARGO_BIN_EXE_dever-launcher")) + .env_clear() + .arg("--dever-install") + .arg(installer.path()) + .output() + .unwrap(); + assert!(!failed.status.success()); + assert_eq!( + fs::read(layout.state().join("active-version")).unwrap(), + before_bad_install + ); + assert_eq!(sha256_file(&launcher).unwrap(), launcher_hash); + assert!(!active.is_empty()); + drop(downloaded); + drop(author); + drop(installer); + for name in ["first", "second"] { + application(&image.path().join(name), "base"); + } + let mut daemon = IsolatedDaemon::start(image.path()); + for name in ["first", "second"] { + command( + daemon + .client() + .args(["/usr/local/bin/dever", "check", &format!("/{name}")]), + ); + assert_response(command(daemon.client().args([ + "/usr/local/bin/dever", + "run", + &format!("/{name}"), + "--", + "sample.value.exercise", + "{}", + ]))); + command(daemon.client().args([ + "/usr/local/bin/dever", + "build", + &format!("/{name}"), + "--output", + &format!("/{name}/program"), + ])); + } + let status = command( + daemon + .client() + .args(["/usr/local/bin/dever", "cache", "status"]), + ); + assert_eq!( + serde_json::from_slice::<Value>(&status.stdout).unwrap()["entries"], + 1 + ); + daemon + .stop() + .expect("isolated daemon and namespace fully stopped"); + drop(daemon); + + // Namespace-root execution above proves the complete loader closure. These + // clients retain real distinct kernel UIDs, without a single-UID userns + // translating file ownership to overflow IDs. + fs::set_permissions(image.path(), fs::Permissions::from_mode(0o755)).unwrap(); + let mut daemon = Daemon( + Command::new(&executable) + .env_clear() + .args(["--root", layout.root().to_str().unwrap()]) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .spawn() + .unwrap(), + ); + wait_socket(&mut daemon, &layout.service_socket()); + for (name, uid) in [("first", 65533), ("second", 65534)] { + let project = image.path().join(name); + chown(&project, Some(uid), Some(uid)).unwrap(); + assert_response(command( + Command::new(&launcher) + .uid(uid) + .gid(uid) + .arg("run") + .arg(&project) + .args(["--", "sample.value.exercise", "{}"]), + )); + let denied = Command::new(&launcher) + .uid(uid) + .gid(uid) + .env_clear() + .args(["use", manifest.version.as_str()]) + .output() + .unwrap(); + assert!(!denied.status.success()); + assert_eq!( + fs::read(layout.state().join("active-version")).unwrap(), + before_bad_install + ); + } + assert_eq!(cache_status(&layout).unwrap().entries, 1); + drop(daemon); + for name in ["first", "second"] { + fs::remove_dir_all(image.path().join(name).join("module")).unwrap(); + } + fs::remove_dir_all(layout.root()).unwrap(); + for name in ["first", "second"] { + assert_response(command(isolated_os_command(image.path()).args([ + &format!("/{name}/program"), + "sample.value.exercise", + "{}", + ]))); + } + write(&workspace, "target/closure-linux-bootstrap-acceptance.json", serde_json::to_vec_pretty(&json!({ + "format":"dever-linux-bootstrap-acceptance-v1","platform":"linux-x86_64", + "bootstrap":manifest.artifacts.iter().filter(|artifact| artifact.path.starts_with("bootstrap/")).collect::<Vec<_>>(), + "checks":{"signed_first_install":true,"stable_public_entry":true,"root_owned_systemd_assets":true,"two_projects_in_minimal_os_root":true,"real_distinct_uids":true,"shared_compile_entries":1,"failed_install_preserves_old_entry":true,"source_and_machine_removed_execution":true}, + "global_service_actions":"not_run","public_release":"not_published" + })).unwrap()); +} diff --git a/test/dever-cli-tests/native_release/command_acceptance.rs b/test/dever-cli-tests/native_release/command_acceptance.rs new file mode 100644 index 0000000..71105aa --- /dev/null +++ b/test/dever-cli-tests/native_release/command_acceptance.rs @@ -0,0 +1,205 @@ +//! Owned signed CLI and relocated output; the tool is an ordinary dynamic ELF. +use super::*; +use dever_cli::libs::LockFile; +use dever_cli::packages::LockedPackage; +use dever_cli::toolchain::artifact_put; +use sha2::{Digest, Sha256}; +use std::io::{Cursor, Write}; + +fn project(root: &Path, workspace: &Path) { + write( + root, + "module/media/tool/port.dever", + "execute(args: List<Text>, stdin: Bytes?) (output: dever.process.Output) fails dever.process.Error\n", + ); + write( + root, + "module/media/tool/adapter.dever", + "external command \"bin/tool\" {}\n", + ); + write( + root, + "module/media/tool/app.dever", + r#"execute() (passed: Bool) { + input = dever.bytes.from_ints([0, 255, 128, 10]) + output = port.execute(["a b", "$(literal);*"], input) + passed = output.code == 23 and output.stdout == input and dever.bytes.to_text(output.stderr) == "ordinary command" +} +"#, + ); + write( + root, + "module/media/tool/api.dever", + "cmd execute = app.execute\n", + ); + write(root, "config/setting.json", "{}"); + let directory = root.join("module/media/tool/bin"); + fs::create_dir_all(directory.join("lib")).unwrap(); + command( + Command::new("/usr/bin/cc") + .arg("-B/usr/bin/") + .args(["-shared", "-fPIC", "-Wl,-soname,libcommand.so", "-o"]) + .arg(directory.join("lib/libcommand.so")) + .arg(workspace.join("test/dever-cli-tests/support/command_library.c")), + ); + command( + Command::new("/usr/bin/cc") + .arg("-B/usr/bin/") + .arg(workspace.join("test/dever-cli-tests/support/command_tool.c")) + .arg("-L") + .arg(directory.join("lib")) + .arg("-lcommand") + .arg("-o") + .arg(directory.join("tool")), + ); +} + +fn packaged_project(root: &Path, source: &Path, layout: &Layout) { + let manifest = br#"{"format":"dever-package-v1","name":"media","version":"1.0.0","dependencies":{},"lib":[]}"#; + let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new())); + zip.start_file( + "dever-package.json", + zip::write::SimpleFileOptions::default(), + ) + .unwrap(); + zip.write_all(manifest).unwrap(); + for file in [ + "port.dever", + "adapter.dever", + "app.dever", + "api.dever", + "bin/tool", + "bin/lib/libcommand.so", + ] { + let name = format!("module/media/tool/{file}"); + zip.start_file(&name, zip::write::SimpleFileOptions::default()) + .unwrap(); + zip.write_all(&fs::read(source.join(name)).unwrap()) + .unwrap(); + } + let archive = zip.finish().unwrap().into_inner(); + let receipt = artifact_put(layout, &archive).unwrap(); + fs::create_dir_all(root.join("module")).unwrap(); + write( + root, + "config/setting.json", + r#"{"package":{"registry":"https://packages.example.test","use":["media@1.0.0"]}}"#, + ); + let mut lock = LockFile::new(Vec::new()).unwrap(); + lock.packages.push(LockedPackage { + name: "media".into(), + version: "1.0.0".into(), + sha256: receipt.sha256, + bytes: archive.len() as u64, + dependencies: Vec::new(), + manifest_sha256: format!("{:x}", Sha256::digest(manifest)), + }); + lock.write_atomic(root).unwrap(); +} + +#[test] +#[ignore = "requires explicit current runtime archive, author C/LLVM inputs and owned daemon/sandbox"] +fn signed_cli_runs_and_packages_ordinary_command_with_private_library() { + let workspace = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let author = TemporaryDirectory::new(); + let key = author_inputs(author.path(), &workspace); + // This test validates the command boundary, not release-profile trimming. + // Use the explicitly rebuilt archive containing the current wire ABI. + let current = input( + author.path(), + &workspace.join("target/native-runtime-abi/debug/libdever_backend_bridge.a"), + "command-base.a", + ); + let settings_path = author.path().join("config/setting.json"); + let mut settings: Value = serde_json::from_slice(&fs::read(&settings_path).unwrap()).unwrap(); + settings["profiles"]["base"] = current; + fs::write(settings_path, serde_json::to_vec(&settings).unwrap()).unwrap(); + + let machine = machine_temporary_directory(&workspace); + let layout = Layout::new(machine.path()); + layout.initialize().unwrap(); + let version = Version::parse(env!("CARGO_PKG_VERSION")).unwrap(); + let release = layout.downloads().join(version.as_str()); + packaging::create(author.path(), &release).unwrap(); + install_release(&layout, &version, &key); + let compiler = layout.versions().join(version.as_str()).join("dever-core"); + let mut daemon = daemon::TestDaemon::start(&layout); + let source = TemporaryDirectory::new(); + project(source.path(), &workspace); + command(Command::new(&compiler).arg("check").arg(source.path())); + assert_response(command( + Command::new(&compiler).arg("run").arg(source.path()).args([ + "--", + "media.tool.execute", + "{}", + ]), + )); + assert!(!source.path().join("dever.lock").exists()); + + let deployment = TemporaryDirectory::new(); + let executable = deployment.path().join("program"); + command( + Command::new(&compiler) + .arg("build") + .arg(source.path()) + .arg("--output") + .arg(&executable), + ); + assert_eq!(cache_status(&layout).unwrap().entries, 1); + let package_source = TemporaryDirectory::new(); + packaged_project(package_source.path(), source.path(), &layout); + assert_response(command( + Command::new(&compiler) + .arg("run") + .arg(package_source.path()) + .args(["--", "media.tool.execute", "{}"]), + )); + let package_executable = deployment.path().join("package-program"); + command( + Command::new(&compiler) + .arg("build") + .arg(package_source.path()) + .arg("--output") + .arg(&package_executable), + ); + write(deployment.path(), "config/setting.json", "{}"); + daemon.stop(); + drop(source); + drop(package_source); + drop(author); + drop(machine); + assert_response(command( + Command::new(&executable) + .args(["media.tool.execute", "{}"]) + .current_dir(deployment.path()), + )); + assert_response(command( + Command::new(&package_executable) + .args(["media.tool.execute", "{}"]) + .current_dir(deployment.path()), + )); + let cache = deployment.path().join("data/cache/lib"); + let bundle = fs::read_dir(cache).unwrap().next().unwrap().unwrap().path(); + let commands = bundle.join("commands"); + let tree = fs::read_dir(commands) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + assert!(tree.join("lib/libcommand.so").is_file()); + fs::write(tree.join("lib/libcommand.so"), b"tampered").unwrap(); + let capture = deployment.path().join("tamper.stderr"); + let status = process::status( + Command::new(&executable) + .env_clear() + .current_dir(deployment.path()) + .args(["media.tool.execute", "{}"]) + .stdout(Stdio::null()) + .stderr(Stdio::from(fs::File::create(&capture).unwrap())), + Duration::from_secs(10), + ) + .unwrap(); + assert!(!status.success()); + assert!(fs::read_to_string(capture).unwrap().contains("resource")); +} diff --git a/test/dever-cli-tests/native_release/cross_acceptance.rs b/test/dever-cli-tests/native_release/cross_acceptance.rs new file mode 100644 index 0000000..f4017a2 --- /dev/null +++ b/test/dever-cli-tests/native_release/cross_acceptance.rs @@ -0,0 +1,291 @@ +//! Explicit x86 compiler -> ARM application acceptance; QEMU user mode does +//! not establish the namespace/seccomp contract of external Workers. +use super::*; +use dever_cli::toolchain::BuildTarget; +use dever_cli::toolchain::compilation::{CompileKind, CompileRequest}; +use dever_core::source::SourceMap; +use dever_runtime::config::Settings; +use std::io::{Read, Seek, SeekFrom, Write}; + +#[path = "cross_ecosystems.rs"] +mod ecosystems; + +fn add_arm_inputs(author: &Path, workspace: &Path) { + let prepared = workspace.join("target/arm64-cross"); + let mut profiles = serde_json::Map::new(); + for profile in PROFILES { + let name = format!("{profile}.a"); + profiles.insert( + profile.into(), + input( + author, + &prepared.join("archives").join(&name), + &format!("arm/{name}"), + ), + ); + } + let libc = prepared.join("tools/usr/aarch64-linux-gnu/lib"); + let gcc = prepared.join("tools/usr/lib/gcc-cross/aarch64-linux-gnu/13"); + let links = |root: &Path, names: &[&str]| -> Vec<Value> { + names + .iter() + .map(|name| { + let mut entry = input(author, &root.join(name), &format!("arm/{name}")); + entry["path"] = json!(name); + entry + }) + .collect() + }; + let start = [ + links(&libc, &["crt1.o", "crti.o"]), + links(&gcc, &["crtbeginT.o"]), + ] + .concat(); + let libraries = [ + links(&libc, &["libc.a", "libm-2.39.a", "libmvec.a"]), + links(&gcc, &["libgcc.a", "libgcc_eh.a"]), + ] + .concat(); + let end = [links(&gcc, &["crtend.o"]), links(&libc, &["crtn.o"])].concat(); + let path = author.join("config/setting.json"); + let mut settings: Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + settings["native_targets"] = json!({"linux-aarch64": {"profiles": profiles, "start":start, "libraries":libraries, "end":end}}); + fs::write(path, serde_json::to_vec(&settings).unwrap()).unwrap(); +} + +fn build(compiler: &Path, project: &Path, output: &Path, target: BuildTarget) { + command( + Command::new(compiler) + .arg("build") + .arg(project) + .args(["--target", target.platform(), "--output"]) + .arg(output), + ); + let bytes = fs::read(output).unwrap(); + let elf = goblin::elf::Elf::parse(&bytes).unwrap(); + assert!(elf.is_64 && elf.little_endian); + assert!(elf.interpreter.is_none() && elf.libraries.is_empty()); + assert_eq!( + elf.header.e_machine, + match target { + BuildTarget::LinuxX86_64 => goblin::elf::header::EM_X86_64, + BuildTarget::LinuxAarch64 => goblin::elf::header::EM_AARCH64, + } + ); +} + +fn request(project: &Path, target: BuildTarget) -> CompileRequest { + let sources = SourceMap::load_with_packages(&project.join("module"), None, &[]).unwrap(); + let bindings = Settings::load_project(project) + .unwrap() + .compilation_bindings(); + CompileRequest::new( + env!("CARGO_PKG_VERSION").into(), + CompileKind::Application, + &sources, + Some(bindings), + &[], + target, + ) + .unwrap() +} + +fn private_compiler( + installed: &Path, + destination: &Path, + manifest: &dever_cli::toolchain::ReleaseManifest, +) -> PathBuf { + for artifact in &manifest.artifacts { + if artifact.path != "dever-core" + && !artifact.path.starts_with("lib/") + && !artifact.path.starts_with("runtime/linux-aarch64/") + { + continue; + } + let relative = if artifact.path == "dever-core" { + "dever" + } else { + &artifact.path + }; + let output = destination.join(relative); + fs::create_dir_all(output.parent().unwrap()).unwrap(); + fs::hard_link(installed.join(&artifact.path), output).unwrap(); + } + destination.join("dever") +} + +fn reject_tampered_cache_hit(layout: &Layout, archive: &Path, request: &CompileRequest) { + let mut file = fs::OpenOptions::new() + .read(true) + .write(true) + .open(archive) + .unwrap(); + let mut original = [0; 1]; + file.read_exact(&mut original).unwrap(); + file.seek(SeekFrom::Start(0)).unwrap(); + file.write_all(&[original[0] ^ 1]).unwrap(); + file.sync_all().unwrap(); + let rejected = dever_cli::toolchain::compile(layout, request); + file.seek(SeekFrom::Start(0)).unwrap(); + file.write_all(&original).unwrap(); + file.sync_all().unwrap(); + assert!(rejected.unwrap_err().contains("integrity verification")); +} + +fn export_http_application(compiler: &Path, projects: &Path, workspace: &Path) -> Value { + let project = projects.join("http"); + for (path, source) in [ + ( + "user/account/model.dever", + "global type Account { name: Text(1, 32) }", + ), + ( + "platform/tenant/model.dever", + "global type Tenant { name: Text(1, 32) }", + ), + ( + "user/account/app.dever", + "type Identity { id: Text\nuser_id: user.account.model.id?\ntenant_id: platform.tenant.model.id? }\nverify(claims: dever.auth.Claims) (identity: Identity) { identity = Identity { id = claims.subject\nuser_id = null\ntenant_id = null } }", + ), + ( + "sample/http/app.dever", + "hello() (message: Text) { message = \"arm-http\" }", + ), + ("sample/http/api.dever", "public get hello = app.hello"), + ] { + write(&project, &format!("module/{path}"), source); + } + let settings = serde_json::to_vec(&json!({ + "http":{"host":"127.0.0.1","port":18080}, + "runtime":{"mode":"api","shutdown_ms":2000}, "log":{"level":"error"}, + "database":{"default":{"type":"sqlite","path":"data/http.sqlite","max_connections":1}}, + "auth":{"providers":{"session":{"verify":"user.account.verify","jwtSecret":"cross-acceptance-only-test-key-2026"}}}, + "sites":{"test":{"path":"","auth":"session"}} + })).unwrap(); + write(&project, "config/setting.json", &settings); + let program = project.join("application"); + build(compiler, &project, &program, BuildTarget::LinuxAarch64); + let exported = workspace.join("target/arm64-cross/guest/apps/http"); + write(&exported, "config/setting.json", settings); + fs::copy(&program, exported.join("application")).unwrap(); + json!({"sha256":sha256_file(&program).unwrap(), "profile":"sqlite", "url":"http://127.0.0.1:18080/sample/http/hello", "expected_data":"arm-http", "execution":"pending_guest_validation"}) +} + +#[test] +#[ignore = "requires prepared x86/ARM four-profile runtime archives and private QEMU; starts only an owned daemon"] +fn signed_host_compiler_builds_arm_profiles_and_runs_standalone_under_qemu() { + let workspace = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../..") + .canonicalize() + .unwrap(); + let emulator = workspace.join("target/arm64-cross/tools/usr/bin/qemu-aarch64-static"); + assert!( + emulator.is_file(), + "prepare the task-owned static QEMU emulator" + ); + let author = TemporaryDirectory::new(); + let key = author_inputs(author.path(), &workspace); + add_arm_inputs(author.path(), &workspace); + let machine = machine_temporary_directory(&workspace); + let layout = Layout::new(machine.path()); + layout.initialize().unwrap(); + let version = Version::parse(env!("CARGO_PKG_VERSION")).unwrap(); + let release = layout.downloads().join(version.as_str()); + let manifest = packaging::create(author.path(), &release).unwrap(); + assert_eq!(manifest.platform, BuildTarget::LinuxX86_64.platform()); + install_release(&layout, &version, &key); + fs::remove_dir_all(&release).unwrap(); + drop(author); + let compiler = layout.versions().join(version.as_str()).join("dever-core"); + let mut daemon = daemon::TestDaemon::start(&layout); + let projects = TemporaryDirectory::new(); + let standalone = TemporaryDirectory::new(); + let mut programs = Vec::new(); + let mut evidence = serde_json::Map::new(); + for profile in PROFILES { + let project = projects.path().join(profile); + application(&project, profile); + let destination = standalone.path().join(profile); + fs::create_dir_all(destination.join("config")).unwrap(); + fs::copy( + project.join("config/setting.json"), + destination.join("config/setting.json"), + ) + .unwrap(); + let executable = destination.join("application"); + build(&compiler, &project, &executable, BuildTarget::LinuxAarch64); + let before = cache_status(&layout).unwrap().entries; + let repeated = project.join("repeated"); + build(&compiler, &project, &repeated, BuildTarget::LinuxAarch64); + assert_eq!( + sha256_file(&executable).unwrap(), + sha256_file(&repeated).unwrap() + ); + assert_eq!(cache_status(&layout).unwrap().entries, before); + evidence.insert(profile.into(), json!({"sha256":sha256_file(&executable).unwrap(), "bytes":fs::metadata(&executable).unwrap().len(), "cache_hit":true})); + programs.push((destination, executable)); + } + assert_eq!(cache_status(&layout).unwrap().entries, 4); + let base_project = projects.path().join("base"); + let host = standalone.path().join("base/host-application"); + build(&compiler, &base_project, &host, BuildTarget::LinuxX86_64); + assert_response(command( + Command::new(&host).args(["sample.value.exercise", "{}"]), + )); + assert_eq!(cache_status(&layout).unwrap().entries, 5); + + let private = private_compiler( + compiler.parent().unwrap(), + &machine.path().join("private"), + &manifest, + ); + let private_directory = standalone.path().join("private-base"); + write(&private_directory, "config/setting.json", b"{}"); + let private_output = private_directory.join("application"); + build( + &private, + &base_project, + &private_output, + BuildTarget::LinuxAarch64, + ); + programs.push((private_directory, private_output)); + + // Use the same canonical request to prove integrity is checked before a hit. + let arm_request = request(&base_project, BuildTarget::LinuxAarch64); + assert_eq!( + dever_cli::toolchain::compile(&layout, &arm_request).unwrap(), + fs::read(&programs[0].1).unwrap() + ); + let archive = compiler + .parent() + .unwrap() + .join("runtime/linux-aarch64/archives/base.a"); + reject_tampered_cache_hit(&layout, &archive, &arm_request); + assert_eq!(cache_status(&layout).unwrap().entries, 5); + let http = export_http_application(&compiler, projects.path(), &workspace); + assert_eq!(cache_status(&layout).unwrap().entries, 6); + daemon.stop(); + drop(projects); + drop(machine); + + for (directory, executable) in programs { + assert!(!directory.join("module").exists()); + assert_response(command( + Command::new(&emulator) + .arg(executable) + .current_dir(directory) + .args(["sample.value.exercise", "{}"]), + )); + } + let report = json!({ + "format":"dever-cross-application-acceptance-v1", "host":"linux-x86_64", "target":"linux-aarch64", + "programs":evidence, "http":http, "qemu_sha256":sha256_file(&emulator).unwrap(), + "checks":{"signed_install":true,"profiles":4,"private_arm_build":true,"standalone_after_source_and_machine_removal":true,"sqlite_crud":true,"cache_entries":6,"cache_target_isolation":true,"tampered_target_cache_hit_rejected":true}, + "postgres_server":"not_run", "worker_sandbox":"requires_arm_kernel_system_emulation" + }); + write( + &workspace, + "target/arm64-cross/compile-acceptance.json", + serde_json::to_vec_pretty(&report).unwrap(), + ); +} diff --git a/test/dever-cli-tests/native_release/cross_ecosystems.rs b/test/dever-cli-tests/native_release/cross_ecosystems.rs new file mode 100644 index 0000000..ae11bcd --- /dev/null +++ b/test/dever-cli-tests/native_release/cross_ecosystems.rs @@ -0,0 +1,348 @@ +//! Produce real standalone ARM Worker applications for full-system acceptance. +use super::*; +use dever_cli::libs::{ + self, LibSpec, LockFile, LockedArtifact, LockedLib, LockedWorker, RegistryRuntime, +}; +use dever_cli::toolchain::{Artifact, ReleaseManifest}; +use dever_core::hir::ExternalWorkerContract; +use sha2::{Digest, Sha256}; +use std::collections::BTreeMap; + +const ECOSYSTEMS: [&str; 3] = ["pip", "npm", "go"]; +const TARGET: BuildTarget = BuildTarget::LinuxAarch64; + +fn digest(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) +} + +fn add_target_sandbox(author: &Path, workspace: &Path) { + let root = workspace.join("target/arm64-cross/sandbox"); + let mut files = Vec::new(); + let mut inputs = Vec::new(); + for directory in ["bin", "lib"] { + for entry in fs::read_dir(root.join(directory)).unwrap() { + let entry = entry.unwrap(); + assert!(entry.file_type().unwrap().is_file()); + let relative = format!("{directory}/{}", entry.file_name().to_str().unwrap()); + let bytes = fs::read(entry.path()).unwrap(); + let source = format!("inputs/arm-sandbox/{relative}"); + write(author, &source, &bytes); + inputs.push(json!({"source":source,"path":relative,"sha256":digest(&bytes)})); + files.push((relative, bytes)); + } + } + dever_sandbox::validate_assets_for_target(&files, TARGET.platform()).unwrap(); + let path = author.join("config/setting.json"); + let mut settings: Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + settings["sandbox_targets"] = json!({"linux-aarch64":inputs}); + fs::write(path, serde_json::to_vec(&settings).unwrap()).unwrap(); +} + +// Sign the exact prepared archives: their identities also bind each Lib lock. +// As in the native ecosystem fixture, this is an explicit private test author. +fn sign_target_runtimes( + release: &Path, + workspace: &Path, + key: &Ed25519KeyPair, + ecosystems: &[&str], +) -> BTreeMap<String, RegistryRuntime> { + let mut manifest: ReleaseManifest = + serde_json::from_slice(&fs::read(release.join("manifest.json")).unwrap()).unwrap(); + let mut runtimes = BTreeMap::new(); + for &ecosystem in ecosystems { + let prepared = workspace + .join("target/arm64-cross/ecosystems") + .join(ecosystem); + let settings: Value = + serde_json::from_slice(&fs::read(prepared.join("config/setting.json")).unwrap()) + .unwrap(); + let runtime: RegistryRuntime = serde_json::from_value(settings["runtime"].clone()).unwrap(); + assert_eq!(runtime.target, TARGET.platform()); + let pack = fs::read(prepared.join("runtime.pack")).unwrap(); + assert_eq!(digest(&pack), runtime.pack.sha256); + let prefix = format!("runtime/{ecosystem}/{}", TARGET.platform()); + write(release, &format!("{prefix}/runtime.pack"), pack); + write( + release, + &format!("{prefix}/manifest.json"), + serde_json::to_vec(&json!({"format":"dever-registry-runtime-v1","runtime":runtime})) + .unwrap(), + ); + for leaf in ["manifest.json", "runtime.pack"] { + let path = format!("{prefix}/{leaf}"); + assert!( + !manifest + .artifacts + .iter() + .any(|artifact| artifact.path == path) + ); + manifest.artifacts.push(Artifact { + bytes: fs::metadata(release.join(&path)).unwrap().len(), + sha256: sha256_file(&release.join(&path)).unwrap(), + path, + }); + } + runtimes.insert(ecosystem.to_owned(), runtime); + } + manifest + .artifacts + .sort_by(|left, right| left.path.cmp(&right.path)); + let encoded = serde_json::to_vec(&manifest).unwrap(); + fs::write(release.join("manifest.json"), &encoded).unwrap(); + fs::write( + release.join("manifest.sig"), + key.sign(&encoded) + .as_ref() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::<String>(), + ) + .unwrap(); + runtimes +} + +fn application(root: &Path, ecosystem: &str) -> ExternalWorkerContract { + let (extension, library, source) = match ecosystem { + "pip" => ( + "py", + "six@1.17.0", + include_str!("../../ecosystem-release/arm64/probe.py"), + ), + "npm" => ( + "mjs", + "is-number@7.0.0", + include_str!("../../ecosystem-release/arm64/probe.mjs"), + ), + "go" => ( + "go", + "github.com/google/uuid@1.6.0", + include_str!("../../ecosystem-release/arm64/probe.go"), + ), + _ => unreachable!(), + }; + write(root, "config/setting.json", "{}\n"); + write( + root, + "module/sample/worker/port.dever", + "probe() (okay: Bool) fails app.ProbeResult\nreject() (okay: Bool) fails app.ProbeResult\n", + ); + write( + root, + "module/sample/worker/app.dever", + "type ProbeResult { error Unavailable(message: Text) }\nprobe() (okay: Bool) { okay = port.probe() }\nreject() (okay: Bool) { okay = port.reject() }\n", + ); + write( + root, + "module/sample/worker/api.dever", + "cmd probe = app.probe\ncmd reject = app.reject\n", + ); + write( + root, + "module/sample/worker/adapter.dever", + format!("external {ecosystem} \"worker/probe.{extension}\" {{ lib \"{library}\" }}\n"), + ); + let sources = SourceMap::load(&root.join("module")).unwrap(); + let program = dever_core::check(&sources).unwrap_or_else(|errors| panic!("{errors:?}")); + let contract = program.external_worker_contracts().remove(0); + write(root, &contract.entry, source); + contract +} + +fn pure_package_lock( + workspace: &Path, + contract: &ExternalWorkerContract, + runtime: &RegistryRuntime, +) -> (LockFile, BTreeMap<String, Vec<u8>>) { + let spec: LibSpec = contract.libs[0].parse().unwrap(); + let (filename, extension) = match contract.ecosystem.as_str() { + "pip" => ("six-1.17.0-py2.py3-none-any.whl", "whl"), + "npm" => ("is-number-7.0.0.tgz", "tgz"), + _ => unreachable!(), + }; + let bytes = fs::read( + workspace + .join("target/arm64-cross/dependencies") + .join(filename), + ) + .unwrap(); + let sha256 = digest(&bytes); + let artifact = LockedArtifact { + target: TARGET.platform().into(), + path: format!( + "lib/{}/{}/{}/archive.{extension}", + contract.ecosystem, + digest(spec.distribution_name().as_bytes()), + spec.version + ), + bytes: bytes.len() as u64, + sha256: sha256.clone(), + }; + let mut lock = LockFile::new(vec![LockedLib { + spec: spec.clone(), + dependencies: vec![], + runtime: runtime.pack.clone(), + artifacts: vec![artifact], + schema: contract.schema.clone(), + build: None, + }]) + .unwrap(); + if contract.ecosystem == "npm" { + lock.npm.push(libs::npm::Environment::single(spec)); + } + (lock, BTreeMap::from([(sha256, bytes)])) +} + +fn go_package_lock( + workspace: &Path, + runtime: &RegistryRuntime, +) -> (LockFile, BTreeMap<String, Vec<u8>>) { + let prepared = workspace.join("target/go-managed/dependency"); + let mut lock = LockFile::decode(&fs::read(prepared.join("dever.lock")).unwrap()).unwrap(); + libs::doctor(&lock).unwrap(); + assert!(lock.builds.is_empty() && lock.workers.is_empty()); + assert_eq!(lock.libs.len(), 1); + assert_eq!(lock.libs[0].spec.key(), "go:github.com/google/uuid@1.6.0"); + let authenticated = + libs::sumdb::verify_chain(&lock.go_sumdb, &libs::sumdb::Verifier::official()).unwrap(); + let mut archives = BTreeMap::new(); + for library in &mut lock.libs { + // Go module ZIPs are source-only; keep the verified source and sumdb + // evidence unchanged while selecting the new runtime and build target. + library.runtime = runtime.pack.clone(); + for artifact in &mut library.artifacts { + let bytes = fs::read(prepared.join("artifacts").join(&artifact.sha256)).unwrap(); + assert_eq!(digest(&bytes), artifact.sha256); + assert_eq!(bytes.len() as u64, artifact.bytes); + authenticated + .verify_zip(&library.spec.name, &library.spec.version, &bytes) + .unwrap(); + artifact.target = TARGET.platform().into(); + archives.insert(artifact.sha256.clone(), bytes); + } + } + (lock, archives) +} + +fn prepare_locked_application( + project: &Path, + workspace: &Path, + layout: &Layout, + contract: ExternalWorkerContract, + runtime: &RegistryRuntime, +) -> LockFile { + let (mut lock, archives) = if contract.ecosystem == "go" { + go_package_lock(workspace, runtime) + } else { + pure_package_lock(workspace, &contract, runtime) + }; + for (sha256, bytes) in archives { + let receipt = dever_cli::toolchain::artifact_put(layout, &bytes).unwrap(); + assert_eq!(receipt.sha256, sha256); + assert_eq!(receipt.bytes, bytes.len() as u64); + } + lock.workers.push(LockedWorker { + libs: contract + .libs + .iter() + .map(|value| value.parse().unwrap()) + .collect(), + port: contract.port, + adapter: contract.adapter, + ecosystem: contract.ecosystem, + runtime: Some(runtime.pack.clone()), + entry: contract.entry, + schema: contract.schema, + capabilities: contract.capabilities, + operations: contract.operations, + }); + libs::doctor(&lock).unwrap(); + lock.write_atomic(project).unwrap(); + lock +} + +#[test] +#[ignore = "requires explicit ARM runtimes/dependencies/sandbox and native archives; builds with an owned host daemon and exports apps for ARM system QEMU"] +fn signed_arm_ecosystems_build_offline_standalone_apps_for_system_guest() { + build_guest_apps(&ECOSYSTEMS, "ecosystem-build-acceptance.json"); +} + +#[test] +#[ignore = "requires explicit ARM inputs; rebuilds only Python for SDK shutdown regression under the ARM guest"] +fn signed_arm_python_worker_rebuilds_with_current_sdk() { + build_guest_apps(&["pip"], "python-build-acceptance.json"); +} + +fn build_guest_apps(ecosystems: &[&str], report: &str) { + let workspace = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../..") + .canonicalize() + .unwrap(); + let destination = workspace.join("target/arm64-cross/guest/apps"); + fs::create_dir_all(&destination).unwrap(); + for &ecosystem in ecosystems { + assert!( + !destination.join(ecosystem).exists(), + "guest app already exists; preserve or remove the prior owned fixture explicitly" + ); + } + let author = TemporaryDirectory::new(); + let key = author_inputs(author.path(), &workspace); + add_arm_inputs(author.path(), &workspace); + add_target_sandbox(author.path(), &workspace); + let machine = machine_temporary_directory(&workspace); + let layout = Layout::new(machine.path()); + layout.initialize().unwrap(); + let version = Version::parse(env!("CARGO_PKG_VERSION")).unwrap(); + let release = layout.downloads().join(version.as_str()); + packaging::create(author.path(), &release).unwrap(); + let runtimes = sign_target_runtimes(&release, &workspace, &key, ecosystems); + install_release(&layout, &version, &key); + fs::remove_dir_all(&release).unwrap(); + drop(author); + let compiler = layout.versions().join(version.as_str()).join("dever-core"); + let mut daemon = daemon::TestDaemon::start(&layout); + let projects = TemporaryDirectory::new(); + let mut programs = serde_json::Map::new(); + for &ecosystem in ecosystems { + let project = projects.path().join(ecosystem); + let contract = application(&project, ecosystem); + let lock = prepare_locked_application( + &project, + &workspace, + &layout, + contract, + &runtimes[ecosystem], + ); + command( + Command::new(&compiler) + .args(["lib", "doctor"]) + .arg(&project) + .args(["--target", TARGET.platform()]), + ); + let output = destination.join(ecosystem); + fs::create_dir(&output).unwrap(); + write(&output, "config/setting.json", "{}\n"); + let executable = output.join("program"); + build(&compiler, &project, &executable, TARGET); + programs.insert(ecosystem.into(), json!({ + "path":format!("{ecosystem}/program"), "sha256":sha256_file(&executable).unwrap(), + "bytes":fs::metadata(&executable).unwrap().len(), "runtime":runtimes[ecosystem].pack, + "libs":lock.libs.iter().map(|lib| lib.spec.key()).collect::<Vec<_>>(), + "artifacts":lock.libs.iter().flat_map(|lib| &lib.artifacts).collect::<Vec<_>>() + })); + } + assert_eq!( + cache_status(&layout).unwrap().entries, + ecosystems.len() as u64 + ); + daemon.stop(); + drop(projects); + drop(machine); + write(&workspace, &format!("target/arm64-cross/{report}"), serde_json::to_vec_pretty(&json!({ + "format":"dever-cross-ecosystem-build-v1", "target":TARGET.platform(), "programs":programs, + "checks":{"signed_runtime_install":true,"offline_lib_doctor":ecosystems.len(),"actual_cli_arm_build":ecosystems.len(),"source_and_build_machine_removed":true}, + "guest_probe":{"arguments":["sample.worker.probe","{}"],"expected_data":true}, + "guest_failure":{"arguments":["sample.worker.reject","{}"],"expected":"runtime fault; next fresh probe still succeeds; no remaining Worker"}, + "guest_execution":"not_run", "sandbox_execution":"requires full ARM kernel guest" + })).unwrap()); +} diff --git a/test/dever-cli-tests/native_release/ecosystem_acceptance.rs b/test/dever-cli-tests/native_release/ecosystem_acceptance.rs new file mode 100644 index 0000000..e003ef9 --- /dev/null +++ b/test/dever-cli-tests/native_release/ecosystem_acceptance.rs @@ -0,0 +1,398 @@ +//! Explicit signed-release acceptance with no host language trees in the final root. +use super::*; +use std::os::unix::fs::{MetadataExt, chown}; + +const ECOSYSTEMS: [&str; 3] = ["pip", "npm", "go"]; +const STANDALONE_UID: u32 = 65534; + +fn nonroot_standalone(source: &Path, expected_program: &Value) -> Value { + let deployment = TemporaryDirectory::new_in(Path::new("/tmp")); + let program = deployment.path().join("program"); + let config = deployment.path().join("config"); + let settings = config.join("setting.json"); + fs::create_dir(&config).unwrap(); + fs::copy(source.join("program"), &program).unwrap(); + fs::copy(source.join("config/setting.json"), &settings).unwrap(); + let identity = json!({ + "bytes": fs::metadata(&program).unwrap().len(), + "sha256": sha256_file(&program).unwrap(), + }); + assert_eq!(&identity, expected_program); + for (path, mode) in [ + (program.as_path(), 0o700), + (config.as_path(), 0o700), + (settings.as_path(), 0o600), + (deployment.path(), 0o700), + ] { + fs::set_permissions(path, fs::Permissions::from_mode(mode)).unwrap(); + chown(path, Some(STANDALONE_UID), Some(STANDALONE_UID)).unwrap(); + } + // Only the standalone executable and private config cross this boundary. + // The actual non-root process must extract its own embedded Worker bundle. + assert!(!deployment.path().join("data").exists()); + let output = command( + Command::new("/usr/bin/setpriv") + .arg(format!("--reuid={STANDALONE_UID}")) + .arg(format!("--regid={STANDALONE_UID}")) + .args([ + "--clear-groups", + "--inh-caps=-all", + "--ambient-caps=-all", + "--", + ]) + .arg(&program) + .args(["sample.worker.probe", "{}"]) + .current_dir(deployment.path()) + .stdin(Stdio::null()), + ); + eprintln!( + "standalone UID {STANDALONE_UID} {}: {}\n{}", + source.display(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert_response(output); + assert_eq!( + fs::metadata(deployment.path().join("data/cache/lib")) + .unwrap() + .uid(), + STANDALONE_UID + ); + json!({"passed": true, "uid": STANDALONE_UID, "program": identity}) +} + +fn dependency_fixture(workspace: &Path, ecosystem: &str) -> PathBuf { + workspace.join(match ecosystem { + "pip" => "target/source-build", + "npm" => "target/npm-build", + _ => unreachable!(), + }) +} + +/// The fixtures were built against these exact pack bytes. The test author +/// signs those inputs, rather than rewriting receipts for a recompressed pack. +fn sign_build_runtimes( + release: &Path, + workspace: &Path, + key: &Ed25519KeyPair, +) -> dever_cli::toolchain::ReleaseManifest { + use dever_cli::libs::{LockFile, RegistryRuntime, RuntimePack}; + + let mut manifest: dever_cli::toolchain::ReleaseManifest = + serde_json::from_slice(&fs::read(release.join("manifest.json")).unwrap()).unwrap(); + for ecosystem in ["pip", "npm"] { + let fixture = dependency_fixture(workspace, ecosystem); + let settings: Value = + serde_json::from_slice(&fs::read(fixture.join("config/setting.json")).unwrap()) + .unwrap(); + let runtime: RegistryRuntime = serde_json::from_value(settings["runtime"].clone()).unwrap(); + assert_eq!(runtime.target, "linux-x86_64"); + let pack = fixture.join(settings["runtime_file"].as_str().unwrap()); + assert_eq!(sha256_file(&pack).unwrap(), runtime.pack.sha256); + let lock = + LockFile::decode(&fs::read(fixture.join("dependency/dever.lock")).unwrap()).unwrap(); + assert!( + !lock.builds.is_empty(), + "acceptance requires actual source build receipts" + ); + for receipt in &lock.builds { + assert_eq!(receipt.runtime, runtime.pack); + if let Some(python) = &receipt.auxiliary_runtime { + let python_config: Value = serde_json::from_slice( + &fs::read(dependency_fixture(workspace, "pip").join("config/setting.json")) + .unwrap(), + ) + .unwrap(); + assert_eq!( + *python, + serde_json::from_value::<RuntimePack>(python_config["runtime"]["pack"].clone()) + .unwrap() + ); + } + } + let prefix = format!("runtime/{ecosystem}/linux-x86_64"); + fs::copy(pack, release.join(&prefix).join("runtime.pack")).unwrap(); + fs::write( + release.join(&prefix).join("manifest.json"), + serde_json::to_vec(&json!({"format":"dever-registry-runtime-v1","runtime":runtime})) + .unwrap(), + ) + .unwrap(); + for suffix in ["runtime.pack", "manifest.json"] { + let name = format!("{prefix}/{suffix}"); + let artifact = manifest + .artifacts + .iter_mut() + .find(|artifact| artifact.path == name) + .unwrap(); + artifact.bytes = fs::metadata(release.join(&name)).unwrap().len(); + artifact.sha256 = sha256_file(&release.join(&name)).unwrap(); + } + } + let encoded = serde_json::to_vec(&manifest).unwrap(); + fs::write(release.join("manifest.json"), &encoded).unwrap(); + fs::write( + release.join("manifest.sig"), + key.sign(&encoded) + .as_ref() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::<String>(), + ) + .unwrap(); + manifest +} + +fn import_built_dependencies(project: &Path, fixture: &Path, layout: &Layout) { + use dever_cli::libs::{LockFile, LockedWorker, build}; + + let mut lock = + LockFile::decode(&fs::read(fixture.join("dependency/dever.lock")).unwrap()).unwrap(); + for artifact in lock + .libs + .iter() + .flat_map(|lib| &lib.artifacts) + .chain(build::npm_outputs(&lock).map(|receipt| &receipt.output)) + { + let bytes = fs::read(fixture.join("dependency/artifacts").join(&artifact.sha256)).unwrap(); + let stored = dever_cli::toolchain::artifact_put(layout, &bytes).unwrap(); + assert_eq!(stored.sha256, artifact.sha256); + assert_eq!(stored.bytes, artifact.bytes); + } + let sources = dever_core::source::SourceMap::load(&project.join("module")).unwrap(); + let program = dever_core::check(&sources).unwrap_or_else(|errors| panic!("{errors:?}")); + let contract = program.external_worker_contracts().remove(0); + let runtime = lock.libs[0].runtime.clone(); + assert!(lock.libs.iter().all(|lib| lib.runtime == runtime)); + assert!(lock.workers.is_empty()); + lock.workers.push(LockedWorker { + port: contract.port, + adapter: contract.adapter, + ecosystem: contract.ecosystem, + runtime: Some(runtime), + entry: contract.entry, + schema: contract.schema, + capabilities: contract.capabilities, + operations: contract.operations, + libs: contract + .libs + .iter() + .map(|spec| spec.parse().unwrap()) + .collect(), + }); + dever_cli::libs::doctor(&lock).unwrap(); + lock.write_atomic(project).unwrap(); +} + +fn ecosystem_inputs(author: &Path, workspace: &Path) { + let prepared = workspace.join("target/ecosystem-release-inputs/prepared"); + let fragment: Value = + serde_json::from_slice(&fs::read(prepared.join("config/setting.json")).unwrap()).unwrap(); + for ecosystem in ECOSYSTEMS { + for entry in fragment["runtimes"][ecosystem]["files"].as_array().unwrap() { + let relative = entry["source"].as_str().unwrap(); + let output = author.join(relative); + fs::create_dir_all(output.parent().unwrap()).unwrap(); + // Author staging only; the maker materializes its own immutable pack. + fs::hard_link(prepared.join(relative), output).unwrap(); + } + } + let config = author.join("config/setting.json"); + let mut settings: Value = serde_json::from_slice(&fs::read(&config).unwrap()).unwrap(); + settings["runtimes"] = fragment["runtimes"].clone(); + fs::write(config, serde_json::to_vec(&settings).unwrap()).unwrap(); +} + +fn application(root: &Path, ecosystem: &str) { + let (extension, source) = match ecosystem { + "pip" => ("py", include_str!("../../ecosystem-release/probe.py")), + "npm" => ("mjs", include_str!("../../ecosystem-release/probe.mjs")), + "go" => ("go", include_str!("../../ecosystem-release/probe.go")), + _ => unreachable!(), + }; + write(root, "config/setting.json", "{}\n"); + write( + root, + "module/sample/worker/port.dever", + "probe() (okay: Bool) fails app.ProbeResult\n", + ); + write( + root, + "module/sample/worker/app.dever", + "type ProbeResult { error Unavailable(message: Text) }\nprobe() (okay: Bool) { okay = port.probe() }\n", + ); + write( + root, + "module/sample/worker/api.dever", + "cmd probe = app.probe\n", + ); + write( + root, + "module/sample/worker/adapter.dever", + format!( + "external {ecosystem} \"worker/probe.{extension}\" {{\n{}\n}}\n", + match ecosystem { + "pip" => "lib \"simplejson@3.20.1\"", + "npm" => "lib \"bufferutil@4.0.9\"", + "go" => "", + _ => unreachable!(), + } + ), + ); + let sources = dever_core::source::SourceMap::load(&root.join("module")).unwrap(); + let program = dever_core::check(&sources).unwrap_or_else(|errors| panic!("{errors:?}")); + let entry = &program.external_worker_contracts()[0].entry; + write(root, entry, source); +} + +pub(super) fn minimal_os(root: &Path) { + // This root contains only the named Linux OS ABI libraries. No host language + // executable, standard library, PATH, package manager or source tree is visible. + for name in [ + "libc.so.6", + "libdl.so.2", + "libm.so.6", + "libpthread.so.0", + "librt.so.1", + "libutil.so.1", + "libresolv.so.2", + "ld-linux-x86-64.so.2", + ] { + let source = Path::new("/lib/x86_64-linux-gnu").join(name); + let output = root.join("lib/x86_64-linux-gnu").join(name); + fs::create_dir_all(output.parent().unwrap()).unwrap(); + fs::copy(source, output).unwrap(); + } + fs::create_dir(root.join("lib64")).unwrap(); + fs::copy( + "/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2", + root.join("lib64/ld-linux-x86-64.so.2"), + ) + .unwrap(); + fs::create_dir(root.join("tmp")).unwrap(); + fs::create_dir(root.join("proc")).unwrap(); + fs::create_dir(root.join("dev")).unwrap(); + // The outer namespace supplies only its own proc and synthetic device tree. + assert!(!root.join("usr").exists()); +} + +#[test] +#[ignore = "requires prepared pinned ecosystem inputs and root for owned namespace; no existing services"] +fn signed_ecosystems_build_and_run_without_system_language_installations() { + assert_eq!( + rustix::process::geteuid().as_raw(), + 0, + "explicit no-host acceptance requires the owned root namespace fixture" + ); + let workspace = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let projects = TemporaryDirectory::new(); + for ecosystem in ECOSYSTEMS { + application(&projects.path().join(ecosystem), ecosystem); + } + let author = TemporaryDirectory::new(); + let key = author_inputs(author.path(), &workspace); + ecosystem_inputs(author.path(), &workspace); + let machine = machine_temporary_directory(&workspace); + let layout = Layout::new(machine.path()); + layout.initialize().unwrap(); + let version = Version::parse(env!("CARGO_PKG_VERSION")).unwrap(); + let release = layout.downloads().join(version.as_str()); + command_with_timeout( + Command::new(workspace.join("target/debug/examples/native-release")) + .arg(author.path()) + .arg("--output") + .arg(&release), + // Debug gzip compresses complete language distributions; this is author + // preparation, independent of the service's bounded compile deadline. + Duration::from_secs(900), + ); + let signed = sign_build_runtimes(&release, &workspace, &key); + let mut runtimes = serde_json::Map::new(); + for ecosystem in ECOSYSTEMS { + let path = format!("runtime/{ecosystem}/linux-x86_64/runtime.pack"); + let artifact = signed + .artifacts + .iter() + .find(|artifact| artifact.path == path) + .unwrap(); + runtimes.insert( + ecosystem.into(), + json!({"bytes":artifact.bytes,"sha256":artifact.sha256}), + ); + } + install_release(&layout, &version, &key); + fs::remove_dir_all(&release).unwrap(); + drop(author); + let core = layout.versions().join(version.as_str()).join("dever-core"); + let mut daemon = daemon::TestDaemon::start(&layout); + let isolated = TemporaryDirectory::new(); + minimal_os(isolated.path()); + let mut programs = serde_json::Map::new(); + for ecosystem in ECOSYSTEMS { + let project = projects.path().join(ecosystem); + if ecosystem == "go" { + command(Command::new(&core).args(["lib", "add"]).arg(&project)); + } else { + import_built_dependencies( + &project, + &dependency_fixture(&workspace, ecosystem), + &layout, + ); + } + let lock = + dever_cli::libs::LockFile::decode(&fs::read(project.join("dever.lock")).unwrap()) + .unwrap(); + assert_eq!(lock.workers.len(), 1); + assert_eq!(lock.libs.is_empty(), ecosystem == "go"); + assert_eq!(lock.builds.is_empty(), ecosystem == "go"); + assert_eq!( + lock.workers[0].runtime.as_ref().unwrap().sha256, + runtimes[ecosystem]["sha256"].as_str().unwrap() + ); + command(Command::new(&core).arg("check").arg(&project)); + assert_response(command( + Command::new(&core) + .arg("run") + .arg(&project) + .args(["--", "sample.worker.probe", "{}"]), + )); + let destination = isolated.path().join(ecosystem); + fs::create_dir(&destination).unwrap(); + write(&destination, "config/setting.json", "{}\n"); + let executable = destination.join("program"); + command( + Command::new(&core) + .arg("build") + .arg(&project) + .arg("--output") + .arg(&executable), + ); + programs.insert(ecosystem.into(), json!({"bytes":fs::metadata(&executable).unwrap().len(),"sha256":sha256_file(&executable).unwrap()})); + } + assert_eq!(cache_status(&layout).unwrap().entries, 3); + daemon.stop(); + drop(machine); + drop(projects); + for ecosystem in ECOSYSTEMS { + assert_response(command( + isolated_os_command(isolated.path()) + .arg(format!("/{ecosystem}/program")) + .args(["sample.worker.probe", "{}"]), + )); + } + let mut standalone_nonroot = serde_json::Map::new(); + for ecosystem in ECOSYSTEMS { + standalone_nonroot.insert( + ecosystem.into(), + nonroot_standalone(&isolated.path().join(ecosystem), &programs[ecosystem]), + ); + } + write(&workspace, "target/closure-sandbox-ecosystem-release-acceptance.json", serde_json::to_vec_pretty(&json!({ + "format":"dever-ecosystem-release-acceptance-v1", "platform":"linux-x86_64", + "runtimes":runtimes,"programs":programs,"standalone_nonroot":standalone_nonroot, + "checks":{"signed_install":true,"locked_runtimes":3,"managed_check_run_build":3,"standalone_isolated_root":3,"standalone_nonroot":3,"shared_compile_entries":3,"python_stdlib_paths_in_bundle":true,"os_sandbox":true,"offline_build_receipt_replay":true}, + "os_baseline":"explicit Linux GNU OS libraries and named device nodes inside an owned root", + "runtime_pack_provenance":"test author signed the exact pip/npm packs bound to the completed source builds; maker determinism is verified separately", + "third_party_packages":{"pip":"simplejson@3.20.1 _speedups C extension","npm":"bufferutil@4.0.9 build/Release addon"}, "other_platforms":"not_run", "public_release":"not_published" + })).unwrap()); +} diff --git a/test/dever-cli-tests/native_release/ecosystems.rs b/test/dever-cli-tests/native_release/ecosystems.rs new file mode 100644 index 0000000..974fa90 --- /dev/null +++ b/test/dever-cli-tests/native_release/ecosystems.rs @@ -0,0 +1,429 @@ +//! Synthetic runtime closures exercise the production maker without host interpreters. + +use std::io::Read; + +use super::*; + +fn add_file(author: &mut Author, ecosystem: &str, path: &str, bytes: &[u8]) { + let source = format!("inputs/{ecosystem}/{path}"); + let file = author.root.join(&source); + fs::create_dir_all(file.parent().unwrap()).unwrap(); + fs::write(&file, bytes).unwrap(); + author.settings["runtimes"][ecosystem]["files"] + .as_array_mut() + .unwrap() + .push(json!({"source": source, "path": path, "sha256": sha256_file(&file).unwrap()})); +} + +fn set_manifest(author: &mut Author, ecosystem: &str, change: impl FnOnce(&mut Value)) { + let input = author.settings["runtimes"][ecosystem]["files"] + .as_array_mut() + .unwrap() + .iter_mut() + .find(|input| input["path"] == "dever-runtime.json") + .unwrap(); + let path = author.root.join(input["source"].as_str().unwrap()); + let mut manifest: Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + change(&mut manifest); + fs::write(&path, serde_json::to_vec(&manifest).unwrap()).unwrap(); + input["sha256"] = json!(sha256_file(&path).unwrap()); +} + +fn runtime_author() -> Author { + let mut author = Author::new(); + for (ecosystem, name, version) in [ + ("pip", "cpython", "3.12.0"), + ("npm", "node", "24.0.0"), + ("go", "go", "1.25.0"), + ] { + author.settings["runtimes"][ecosystem] = json!({"name":name,"version":version,"files":[]}); + let mut descriptor = json!({"format":"dever-worker-runtime-v1","ecosystem":ecosystem,"target":platform_identity()}); + if ecosystem == "go" { + let arch = if std::env::consts::ARCH == "x86_64" { + "amd64" + } else { + "arm64" + }; + descriptor["build"] = json!({"host":platform_identity(),"compiler":"bin/compile","linker":"bin/link","analyzer":"bin/analyze","stdlib_importcfg":"stdlib/importcfg","goos":"linux","goarch":arch,"go_version":format!("go version go1.25.0 linux/{arch}")}); + for path in ["bin/compile", "bin/link", "bin/analyze"] { + add_file( + &mut author, + ecosystem, + path, + &native_elf::static_executable(), + ); + } + let mut importcfg = String::new(); + for name in ["runtime", "context", "encoding/json"] { + let path = format!("stdlib/{name}.a"); + let object = format!("go object linux {arch} go1.25.0 fixture\n"); + let archive = format!( + "!<arch>\n{:<16}{:<12}{:<6}{:<6}{:<8}{:<10}`\n{object}", + "__.PKGDEF", + 0, + 0, + 0, + "644", + object.len() + ); + add_file(&mut author, ecosystem, &path, archive.as_bytes()); + importcfg.push_str(&format!("packagefile {name}={path}\n")); + } + add_file( + &mut author, + ecosystem, + "stdlib/importcfg", + importcfg.as_bytes(), + ); + } else { + descriptor["executable"] = json!(format!("bin/{name}")); + descriptor["arguments"] = if ecosystem == "pip" { + json!(["-I", "-S", "-B"]) + } else { + json!([]) + }; + add_file( + &mut author, + ecosystem, + &format!("bin/{name}"), + &native_elf::static_executable(), + ); + } + add_file( + &mut author, + ecosystem, + "dever-runtime.json", + &serde_json::to_vec(&descriptor).unwrap(), + ); + } + author.settings["runtimes"]["pip"]["python_markers"] = json!({ + "implementation_name":"cpython","implementation_version":"3.12.0","os_name":"posix", + "platform_machine":std::env::consts::ARCH,"platform_python_implementation":"CPython", + "platform_release":"","platform_system":"Linux","platform_version":"", + "python_full_version":"3.12.0","python_version":"3.12","sys_platform":"linux" + }); + author.settings["runtimes"]["pip"]["python_wheel_tags"] = json!(["py3-none-any"]); + let markers = author.settings["runtimes"]["pip"]["python_markers"].clone(); + set_manifest(&mut author, "pip", |manifest| { + manifest["python_markers"] = markers; + manifest["python_wheel_tags"] = json!(["py3-none-any"]); + manifest["python_extension_suffixes"] = json!([".abi3.so", ".so"]); + }); + author +} + +#[test] +fn ecosystem_archives_are_deterministic_and_signed_with_computed_identities() { + let mut author = runtime_author(); + author.save(); + let first = author.output("first"); + let manifest = packaging::create(&author.root, &first).unwrap(); + // Input order and filesystem mode are not archive metadata. + for ecosystem in ["pip", "npm", "go"] { + author.settings["runtimes"][ecosystem]["files"] + .as_array_mut() + .unwrap() + .reverse(); + } + fs::set_permissions( + author.root.join("inputs/npm/bin/node"), + fs::Permissions::from_mode(0o700), + ) + .unwrap(); + author.save(); + let second = author.output("second"); + assert_eq!(packaging::create(&author.root, &second).unwrap(), manifest); + assert_eq!(contents(&first), contents(&second)); + assert_signed(&author, &first); + for ecosystem in ["pip", "npm", "go"] { + let prefix = format!("runtime/{ecosystem}/{}", platform_identity()); + let metadata_path = format!("{prefix}/manifest.json"); + let pack_path = format!("{prefix}/runtime.pack"); + let metadata: Value = + serde_json::from_slice(&fs::read(first.join(&metadata_path)).unwrap()).unwrap(); + assert_eq!(metadata["format"], "dever-registry-runtime-v1"); + assert_eq!( + metadata["runtime"]["pack"]["sha256"], + sha256_file(&first.join(&pack_path)).unwrap() + ); + for path in [&metadata_path, &pack_path] { + let signed = manifest + .artifacts + .iter() + .find(|artifact| &artifact.path == path) + .unwrap(); + assert_eq!(signed.sha256, sha256_file(&first.join(path)).unwrap()); + } + let bytes = fs::read(first.join(&pack_path)).unwrap(); + let mut archive = tar::Archive::new(flate2::read::GzDecoder::new(bytes.as_slice())); + let mut names = Vec::new(); + for entry in archive.entries().unwrap() { + let mut entry = entry.unwrap(); + let path = entry.path().unwrap().to_str().unwrap().to_owned(); + assert_eq!(entry.header().mtime().unwrap(), 0); + assert_eq!(entry.header().uid().unwrap(), 0); + assert_eq!(entry.header().gid().unwrap(), 0); + assert_eq!( + entry.header().mode().unwrap(), + if path.starts_with("bin/") { + 0o755 + } else { + 0o644 + } + ); + let mut actual = Vec::new(); + entry.read_to_end(&mut actual).unwrap(); + assert_eq!( + actual, + fs::read(author.root.join(format!("inputs/{ecosystem}/{path}"))).unwrap() + ); + names.push(path); + } + assert!(names.windows(2).all(|pair| pair[0] < pair[1])); + } + let before = contents(&first); + fs::write(author.root.join("inputs/npm/bin/node"), b"changed").unwrap(); + assert_eq!(contents(&first), before); +} + +#[test] +fn rejects_incomplete_or_mismatched_runtime_descriptors() { + for ecosystem in ["pip", "npm", "go"] { + for field in ["format", "ecosystem", "target"] { + let mut author = runtime_author(); + set_manifest(&mut author, ecosystem, |manifest| { + manifest[field] = json!("wrong") + }); + assert!(author.reject().contains("identity")); + } + let mut author = runtime_author(); + set_manifest(&mut author, ecosystem, |manifest| { + manifest["unknown"] = json!(true) + }); + assert!(author.reject().contains("manifest")); + let mut author = runtime_author(); + author.settings["runtimes"][ecosystem]["files"] + .as_array_mut() + .unwrap() + .retain(|input| input["path"] != "dever-runtime.json"); + assert!(author.reject().contains("dever-runtime.json")); + } + for (ecosystem, field, value) in [ + ("pip", "arguments", json!(["-I", "-B"])), + ("npm", "executable", json!("bin/missing")), + ("npm", "arguments", json!(["source.js"])), + ("go", "executable", json!("bin/compile")), + ("go", "build", Value::Null), + ] { + let mut author = runtime_author(); + set_manifest(&mut author, ecosystem, |manifest| manifest[field] = value); + author.reject(); + } + for (field, value) in [ + ("goarch", "unknown"), + ("compiler", "bin/missing"), + ("stdlib_importcfg", "missing"), + ] { + let mut author = runtime_author(); + set_manifest(&mut author, "go", |manifest| { + manifest["build"][field] = json!(value) + }); + author.reject(); + } + let mut author = runtime_author(); + author.settings["runtimes"]["go"]["files"] + .as_array_mut() + .unwrap() + .retain(|input| input["path"] != "stdlib/context.a"); + assert!(author.reject().contains("archive")); +} + +#[test] +fn go_cross_pack_binds_host_tools_and_arm_standard_library_separately() { + for wrong_stdlib in [false, true] { + let mut author = runtime_author(); + set_manifest(&mut author, "go", |manifest| { + manifest["target"] = json!("linux-aarch64"); + manifest["build"]["goarch"] = json!("arm64"); + }); + for input in author.settings["runtimes"]["go"]["files"] + .as_array_mut() + .unwrap() + { + if !input["path"].as_str().unwrap().ends_with(".a") { + continue; + } + let path = author.root.join(input["source"].as_str().unwrap()); + let architecture = if wrong_stdlib { "amd64" } else { "arm64" }; + let object = format!("go object linux {architecture} go1.25.0 fixture\n"); + let archive = format!( + "!<arch>\n{:<16}{:<12}{:<6}{:<6}{:<8}{:<10}`\n{object}", + "__.PKGDEF", + 0, + 0, + 0, + "644", + object.len() + ); + fs::write(&path, archive).unwrap(); + input["sha256"] = json!(sha256_file(&path).unwrap()); + } + let input = author.settings["runtimes"] + .as_object_mut() + .unwrap() + .remove("go") + .unwrap(); + author.settings["ecosystem_targets"] = json!({"linux-aarch64":{"go":input}}); + author.save(); + let result = packaging::create(&author.root, &author.output("cross-go")); + if wrong_stdlib { + assert!( + result + .unwrap_err() + .contains("differs from its target or compiler version") + ); + } else { + let manifest = result.unwrap(); + assert!( + manifest + .artifacts + .iter() + .any(|file| file.path == "runtime/go/linux-aarch64/runtime.pack") + ); + } + } +} + +#[test] +fn sandbox_cross_assets_preserve_loader_mode_and_validate_deployment_target() { + let mut files = vec![ + ("bin/bwrap".into(), native_elf::static_executable()), + ("bin/guard".into(), native_elf::static_executable()), + ( + "lib/ld-linux-aarch64.so.1".into(), + native_elf::library("ld-linux-aarch64.so.1", &[]), + ), + ( + "lib/libc.so.6".into(), + native_elf::library("libc.so.6", &[]), + ), + ]; + for (_, bytes) in &mut files { + bytes[18..20].copy_from_slice(&183u16.to_le_bytes()); + } + dever_sandbox::validate_assets_for_target(&files, "linux-aarch64").unwrap(); + let mut author = Author::new(); + let inputs = files + .iter() + .map(|(path, bytes)| { + let source = format!("inputs/arm-sandbox/{path}"); + let file = author.root.join(&source); + fs::create_dir_all(file.parent().unwrap()).unwrap(); + fs::write(&file, bytes).unwrap(); + fs::set_permissions(&file, fs::Permissions::from_mode(0o600)).unwrap(); + json!({"source":source,"path":path,"sha256":sha256_file(&file).unwrap()}) + }) + .collect::<Vec<_>>(); + author.settings["sandbox_targets"] = json!({"linux-aarch64":inputs}); + author.save(); + let output = author.output("cross-sandbox"); + packaging::create(&author.root, &output).unwrap(); + for (path, mode) in [ + ("bin/bwrap", 0o755), + ("bin/guard", 0o755), + ("lib/ld-linux-aarch64.so.1", 0o755), + ("lib/libc.so.6", 0o644), + ] { + assert_eq!( + fs::metadata(output.join("sandbox/linux-aarch64").join(path)) + .unwrap() + .permissions() + .mode() + & 0o777, + mode, + "incorrect packaged mode for {path}" + ); + } + files[1].1[18..20].copy_from_slice(&62u16.to_le_bytes()); + assert!( + dever_sandbox::validate_assets_for_target(&files, "linux-aarch64") + .unwrap_err() + .contains("incompatible target") + ); +} + +#[test] +fn rejects_invalid_registry_metadata_and_unknown_ecosystems() { + for (ecosystem, field, value) in [ + ("pip", "python_markers", Value::Null), + ("pip", "python_wheel_tags", json!([])), + ("pip", "python_wheel_tags", json!(["invalid"])), + ( + "pip", + "python_wheel_tags", + json!(["py3-none-any", "py3-none-any"]), + ), + ("npm", "python_wheel_tags", json!(["py3-none-any"])), + ("npm", "name", json!("")), + ("npm", "name", json!("../node")), + ("go", "version", json!("latest")), + ("go", "unknown", json!(true)), + ] { + let mut author = runtime_author(); + author.settings["runtimes"][ecosystem][field] = value; + author.reject(); + } + let mut author = runtime_author(); + author.settings["runtimes"]["npm"]["python_markers"] = + author.settings["runtimes"]["pip"]["python_markers"].clone(); + author.reject(); + let mut author = runtime_author(); + author.settings["runtimes"]["exec"] = author.settings["runtimes"]["npm"].clone(); + author.reject(); + + let author = runtime_author(); + author.save(); + let path = author.root.join("config/setting.json"); + let config = fs::read_to_string(&path).unwrap(); + let duplicate = config.replacen("\"runtimes\":{", "\"runtimes\":{\"npm\":null,", 1); + assert_ne!(duplicate, config); + fs::write(path, duplicate).unwrap(); + assert!( + packaging::create(&author.root, &author.output("duplicate")) + .unwrap_err() + .contains("invalid author configuration") + ); + assert_eq!(fs::read_dir(author.directory.path()).unwrap().count(), 1); +} + +#[test] +fn rejects_runtime_path_conflicts_drift_and_private_key_aliases() { + for path in [ + "../outside", + "bin", + "bin/node/child", + "runtime.pack", + "bin/node", + ] { + let mut author = runtime_author(); + let mut input = author.settings["runtimes"]["npm"]["files"][0].clone(); + input["path"] = json!(path); + author.settings["runtimes"]["npm"]["files"] + .as_array_mut() + .unwrap() + .push(input); + author.reject(); + } + let author = runtime_author(); + fs::write(author.root.join("inputs/npm/bin/node"), b"changed").unwrap(); + assert!(author.reject().contains("SHA-256")); + let mut author = self::runtime_author(); + let alias = author.root.join("inputs/key-alias"); + fs::hard_link(author.root.join("signing.pk8"), &alias).unwrap(); + author.settings["runtimes"]["npm"]["files"].as_array_mut().unwrap().push(json!({"source":"inputs/key-alias","path":"secrets/key","sha256":sha256_file(&alias).unwrap()})); + assert!(author.reject().contains("cannot be included")); + let mut author = self::runtime_author(); + symlink("node", author.root.join("inputs/npm/bin/link")).unwrap(); + author.settings["runtimes"]["npm"]["files"][0]["source"] = json!("inputs/npm/bin/link"); + author.reject(); +} diff --git a/test/dever-cli-tests/native_release/targets.rs b/test/dever-cli-tests/native_release/targets.rs new file mode 100644 index 0000000..d481537 --- /dev/null +++ b/test/dever-cli-tests/native_release/targets.rs @@ -0,0 +1,178 @@ +use super::*; +use dever_cli::toolchain::runtime_pack::RuntimePack; +use dever_runtime::config::RuntimeProfile; + +#[cfg(target_arch = "x86_64")] +fn cross_author() -> Author { + let mut author = Author::new(); + let target = BuildTarget::LinuxAarch64; + let input = |name: &str, bytes: Vec<u8>| { + let source = format!("inputs/arm-{name}"); + fs::write(author.root.join(&source), bytes).unwrap(); + json!({"source":source, "sha256":sha256_file(&author.root.join(source)).unwrap()}) + }; + let mut profiles = json!({}); + for profile in ["base", "sqlite", "postgres", "both"] { + profiles[profile] = input(profile, native_objects::archive(target, profile)); + } + let link = |name: &str, bytes| { + let mut artifact = input(name, bytes); + artifact["path"] = json!(name); + artifact + }; + author.settings["native_targets"] = json!({"linux-aarch64": { + "profiles":profiles, + "start":[link("start.o", native_objects::object(target))], + "libraries":[link("system.a", native_objects::archive(target, "system.o"))], + "end":[link("end.o", native_objects::object(target))], + }}); + author +} + +#[test] +fn target_names_and_serde_are_exact() { + for (name, target, triple) in [ + ( + "linux-x86_64", + BuildTarget::LinuxX86_64, + "x86_64-unknown-linux-gnu", + ), + ( + "linux-aarch64", + BuildTarget::LinuxAarch64, + "aarch64-unknown-linux-gnu", + ), + ] { + assert_eq!(name.parse::<BuildTarget>().unwrap(), target); + assert_eq!(target.platform(), name); + assert_eq!(target.triple(), triple); + assert_eq!( + serde_json::to_string(&target).unwrap(), + format!("\"{name}\"") + ); + } + for invalid in [ + "", + "arm64", + "Linux-aarch64", + "linux-arm64", + "aarch64-unknown-linux-gnu", + "linux-aarch64 ", + ] { + assert!(invalid.parse::<BuildTarget>().is_err()); + assert!(serde_json::from_value::<BuildTarget>(json!(invalid)).is_err()); + } +} + +#[test] +#[cfg(target_arch = "x86_64")] +fn host_core_packages_both_native_targets_with_separate_profiles_and_signed_closures() { + let author = cross_author(); + author.save(); + let output = author.directory.path().join("cross-release"); + let manifest = packaging::create(&author.root, &output).unwrap(); + assert_eq!(manifest.platform, "linux-x86_64"); + assert_signed(&author, &output); + for target in [BuildTarget::LinuxX86_64, BuildTarget::LinuxAarch64] { + let closure = runtime_pack::validate(&output, env!("CARGO_PKG_VERSION"), target).unwrap(); + for input in closure { + assert!( + manifest + .artifacts + .iter() + .any(|artifact| artifact.path == input.path + && artifact.sha256 == input.sha256 + && artifact.bytes == input.bytes) + ); + } + for profile in [ + RuntimeProfile { + sqlite: false, + postgres: false, + }, + RuntimeProfile { + sqlite: true, + postgres: false, + }, + RuntimeProfile { + sqlite: false, + postgres: true, + }, + RuntimeProfile { + sqlite: true, + postgres: true, + }, + ] { + assert_eq!( + RuntimePack::load(&output, profile, target).unwrap().target, + target + ); + } + } + let profile = RuntimeProfile { + sqlite: false, + postgres: false, + }; + let arm = RuntimePack::load(&output, profile, BuildTarget::LinuxAarch64).unwrap(); + let host = RuntimePack::load(&output, profile, BuildTarget::LinuxX86_64).unwrap(); + assert_ne!(arm.identity, host.identity); + fs::write( + output.join("runtime/linux-aarch64/archives/base.a"), + b"tampered", + ) + .unwrap(); + assert!(RuntimePack::load(&output, profile, BuildTarget::LinuxAarch64).is_err()); + RuntimePack::load(&output, profile, BuildTarget::LinuxX86_64).unwrap(); +} + +#[test] +#[cfg(target_arch = "x86_64")] +fn target_packs_reject_wrong_architecture_and_duplicate_host_declarations() { + for (slot, source) in [ + ("/native_targets/linux-aarch64/start/0", "start.o"), + ("/native_targets/linux-aarch64/profiles/base", "base"), + ] { + let mut author = cross_author(); + let replacement = if source.ends_with(".o") { + native_objects::object(BuildTarget::LinuxX86_64) + } else { + native_objects::archive(BuildTarget::LinuxX86_64, "wrong.o") + }; + fs::write( + author.root.join(format!("inputs/arm-{source}")), + replacement, + ) + .unwrap(); + author.settings.pointer_mut(slot).unwrap()["sha256"] = + json!(sha256_file(&author.root.join(format!("inputs/arm-{source}"))).unwrap()); + assert!( + author + .reject() + .contains("relocatable ELF object for linux-aarch64") + ); + } + let mut author = cross_author(); + author.settings["native_targets"]["linux-x86_64"] = + author.settings["native_targets"]["linux-aarch64"].clone(); + assert!(author.reject().contains("duplicate release destination")); +} + +#[test] +#[cfg(target_arch = "x86_64")] +fn missing_target_pack_never_uses_host_inputs() { + let author = Author::new(); + author.save(); + let output = author.directory.path().join("host-release"); + packaging::create(&author.root, &output).unwrap(); + let error = RuntimePack::load( + &output, + RuntimeProfile { + sqlite: false, + postgres: false, + }, + BuildTarget::LinuxAarch64, + ) + .err() + .unwrap(); + assert!(error.contains("linux-aarch64"), "{error}"); +} diff --git a/test/dever-cli-tests/new_project.rs b/test/dever-cli-tests/new_project.rs new file mode 100644 index 0000000..b6aafc7 --- /dev/null +++ b/test/dever-cli-tests/new_project.rs @@ -0,0 +1,142 @@ +use std::fs; +use std::path::PathBuf; +use std::process::Command; +use std::sync::atomic::{AtomicU64, Ordering}; + +use dever_core::source::SourceMap; + +static NEXT_DIRECTORY: AtomicU64 = AtomicU64::new(0); + +struct Workspace(PathBuf); + +impl Workspace { + fn new() -> Self { + let sequence = NEXT_DIRECTORY.fetch_add(1, Ordering::Relaxed); + let path = + std::env::temp_dir().join(format!("dever-new-test-{}-{sequence}", std::process::id())); + fs::create_dir(&path).unwrap(); + Self(path) + } +} + +impl Drop for Workspace { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +#[test] +fn both_formats_create_checked_formatted_projects_with_real_application_tests() { + let workspace = Workspace::new(); + for markdown in [false, true] { + let root = workspace + .0 + .join(if markdown { "markdown" } else { "plain" }); + dever_cli::project::create(&root, markdown).unwrap(); + let sources = SourceMap::load_project(&root.join("module"), &root.join("test")).unwrap(); + let result = dever_core::check(&sources); + assert!(result.is_ok(), "{result:?}"); + assert_eq!(sources.files().len(), 3); + for source in sources.files() { + let formatted = dever_core::format::format(source).unwrap(); + assert_eq!(formatted, source.text(), "{}", source.path().display()); + } + let settings: serde_json::Value = + serde_json::from_slice(&fs::read(root.join("config/setting.json")).unwrap()).unwrap(); + assert_eq!(settings["log"]["level"], "info"); + assert!(root.join("AGENTS.md").is_file()); + assert!(!root.join("module/main.dever").exists()); + } +} + +#[test] +fn existing_directories_files_and_symlinks_are_preserved() { + let workspace = Workspace::new(); + let directory = workspace.0.join("existing"); + fs::create_dir(&directory).unwrap(); + assert!(dever_cli::project::create(&directory, false).is_err()); + assert_eq!(fs::read_dir(&directory).unwrap().count(), 0); + let file = workspace.0.join("file"); + fs::write(&file, "keep me").unwrap(); + assert!(dever_cli::project::create(&file, false).is_err()); + assert_eq!(fs::read_to_string(&file).unwrap(), "keep me"); + #[cfg(unix)] + { + let link = workspace.0.join("link"); + std::os::unix::fs::symlink(workspace.0.join("absent"), &link).unwrap(); + assert!(dever_cli::project::create(&link, true).is_err()); + assert!(fs::symlink_metadata(link).unwrap().is_symlink()); + } + assert!(fs::read_dir(&workspace.0).unwrap().all(|entry| { + !entry + .unwrap() + .file_name() + .to_string_lossy() + .starts_with(".dever-new-") + })); +} + +#[test] +fn concurrent_creation_publishes_one_complete_project_without_staging_leftovers() { + let workspace = Workspace::new(); + let root = workspace.0.join("project"); + let barrier = std::sync::Barrier::new(2); + let results = std::thread::scope(|scope| { + let workers: Vec<_> = [false, true] + .into_iter() + .map(|markdown| { + let root = &root; + let barrier = &barrier; + scope.spawn(move || { + barrier.wait(); + dever_cli::project::create(root, markdown) + }) + }) + .collect(); + workers + .into_iter() + .map(|worker| worker.join().unwrap()) + .collect::<Vec<_>>() + }); + assert_eq!(results.iter().filter(|result| result.is_ok()).count(), 1); + let sources = SourceMap::load_project(&root.join("module"), &root.join("test")).unwrap(); + assert!(dever_core::check(&sources).is_ok()); + assert_eq!(sources.files().len(), 3); + assert_eq!(fs::read_dir(&workspace.0).unwrap().count(), 1); +} + +#[test] +fn command_creates_before_reading_project_configuration_and_rejects_bad_arguments() { + let workspace = Workspace::new(); + let output = Command::new(env!("CARGO_BIN_EXE_dever")) + .env_clear() + .current_dir(&workspace.0) + .args(["new", "app", "--markdown"]) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!( + workspace + .0 + .join("app/module/hello/greeting/api.dever.md") + .is_file() + ); + for arguments in [ + vec!["new"], + vec!["new", "bad", "--unknown"], + vec!["new", "--markdown"], + ] { + let output = Command::new(env!("CARGO_BIN_EXE_dever")) + .env_clear() + .current_dir(&workspace.0) + .args(arguments) + .output() + .unwrap(); + assert!(!output.status.success()); + } + assert!(!workspace.0.join("bad").exists()); +} diff --git a/test/dever-cli-tests/packages.rs b/test/dever-cli-tests/packages.rs new file mode 100644 index 0000000..7143e9d --- /dev/null +++ b/test/dever-cli-tests/packages.rs @@ -0,0 +1,961 @@ +use std::collections::BTreeMap; +use std::fs; +#[cfg(unix)] +use std::io::Read; +use std::io::{Cursor, Write}; +#[cfg(unix)] +use std::path::Path; +#[cfg(unix)] +use std::process::Command; + +use dever_cli::libs::{ArtifactStore, FixtureArtifactStore, LockFile}; +#[cfg(unix)] +use dever_cli::libs::{LockedWorker, RuntimePack}; +#[cfg(unix)] +use dever_cli::packages::LockedPackage; +use dever_cli::packages::{ + PackageTransport, execute_with, owned_files_with, owned_worker_bytes_with, owns_worker, + sources_with, +}; +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +use dever_cli::toolchain::cache_status; +#[cfg(unix)] +use dever_cli::toolchain::{ + Artifact, Layout, MachineManager, ReleaseManifest, Version, artifact_get, artifact_put, +}; +#[cfg(unix)] +use dever_cli::workers; +#[cfg(unix)] +use dever_core::native::EmbeddedResource; +#[cfg(unix)] +use ring::rand::SystemRandom; +#[cfg(unix)] +use ring::signature::{Ed25519KeyPair, KeyPair}; +use sha2::{Digest, Sha256}; + +#[path = "../dever-tests/tests/support/temp.rs"] +mod temp; + +#[cfg(unix)] +#[path = "support/daemon.rs"] +mod daemon; +#[cfg(unix)] +use daemon::TestDaemon; + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "support/llvm_pack.rs"] +mod llvm_pack; + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "packages/managed_compile.rs"] +mod managed_compile; + +use temp::TemporaryDirectory; + +struct Registry(BTreeMap<String, Vec<u8>>); + +impl PackageTransport for Registry { + fn get(&self, path: &str, limit: usize) -> Result<Vec<u8>, String> { + let bytes = self + .0 + .get(path) + .ok_or_else(|| format!("missing registry path {path}"))?; + if bytes.len() > limit { + return Err("registry response exceeds limit".into()); + } + Ok(bytes.clone()) + } +} + +fn archive( + name: &str, + version: &str, + dependencies: serde_json::Value, + libs: &[&str], + source: &str, +) -> Vec<u8> { + archive_with_sources(name, version, dependencies, libs, source, &[]) +} + +fn archive_with_sources( + name: &str, + version: &str, + dependencies: serde_json::Value, + libs: &[&str], + source: &str, + additional_sources: &[(&str, &str)], +) -> Vec<u8> { + let mut writer = zip::ZipWriter::new(Cursor::new(Vec::new())); + for (path, bytes) in [ + ( + "dever-package.json".to_owned(), + serde_json::to_vec(&serde_json::json!({ + "format": "dever-package-v1", "name": name, "version": version, + "dependencies": dependencies, "lib": libs, + })) + .unwrap(), + ), + ( + format!("module/{name}/value/app.dever"), + source.as_bytes().to_vec(), + ), + ] { + writer + .start_file(path, zip::write::SimpleFileOptions::default()) + .unwrap(); + writer.write_all(&bytes).unwrap(); + } + for (path, source) in additional_sources { + writer + .start_file(*path, zip::write::SimpleFileOptions::default()) + .unwrap(); + writer.write_all(source.as_bytes()).unwrap(); + } + writer.finish().unwrap().into_inner() +} + +fn digest(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) +} + +fn registry(archives: &[(&str, &str, Vec<u8>)]) -> Registry { + let mut paths = BTreeMap::new(); + let mut versions = BTreeMap::<String, Vec<serde_json::Value>>::new(); + for (name, version, bytes) in archives { + versions + .entry((*name).into()) + .or_default() + .push(serde_json::json!({ + "version": version, "sha256": digest(bytes), "bytes": bytes.len(), + })); + paths.insert(format!("/v1/packages/{name}/{version}.zip"), bytes.clone()); + } + for (name, versions) in versions { + paths.insert( + format!("/v1/packages/{name}/index.json"), + serde_json::to_vec(&serde_json::json!({ + "format": "dever-package-index-v1", "name": name, "versions": versions, + })) + .unwrap(), + ); + } + Registry(paths) +} + +fn project() -> TemporaryDirectory { + let root = TemporaryDirectory::new(); + fs::create_dir_all(root.path().join("config")).unwrap(); + fs::create_dir_all(root.path().join("module/portal/value")).unwrap(); + fs::write( + root.path().join("config/setting.json"), + r#"{"package":{"registry":"https://packages.example.test","use":[]}}"#, + ) + .unwrap(); + fs::write( + root.path().join("module/portal/value/app.dever"), + "value() (result: Text) { result = storefront.value.value() }\n", + ) + .unwrap(); + fs::write( + root.path().join("module/portal/value/api.dever"), + "cmd show = app.value\n", + ) + .unwrap(); + fs::create_dir_all(root.path().join("test/portal/value")).unwrap(); + fs::write( + root.path().join("test/portal/value/smoke.dever"), + "smoke() () { assert_eq(portal.value.value(), \"two\") }\n", + ) + .unwrap(); + root +} + +#[test] +fn package_add_resolves_transitive_versions_and_feeds_the_common_lib_lock() { + let root = project(); + let registry = registry(&[ + ( + "storefront", + "1.0.0", + archive( + "storefront", + "1.0.0", + serde_json::json!({"catalog":"^1.0.0"}), + &[], + "value() (result: Text) { result = catalog.value.value() }\n", + ), + ), + ( + "catalog", + "1.0.0", + archive( + "catalog", + "1.0.0", + serde_json::json!({}), + &["pip:fixture@1.0.0"], + "value() (result: Text) { result = \"one\" }\n", + ), + ), + ( + "catalog", + "1.2.0", + archive( + "catalog", + "1.2.0", + serde_json::json!({}), + &["pip:fixture@1.0.0"], + "value() (result: Text) { result = \"two\" }\n", + ), + ), + ( + "catalog", + "2.0.0", + archive( + "catalog", + "2.0.0", + serde_json::json!({}), + &[], + "value() (result: Text) { result = \"wrong\" }\n", + ), + ), + ]); + let store = FixtureArtifactStore::default(); + execute_with( + "add", + root.path(), + &["storefront@^1.0.0".into()], + ®istry, + &store, + ) + .unwrap(); + let lock = LockFile::decode(&fs::read(root.path().join("dever.lock")).unwrap()).unwrap(); + assert_eq!( + lock.packages + .iter() + .map(|package| (package.name.as_str(), package.version.as_str())) + .collect::<Vec<_>>(), + [("catalog", "1.2.0"), ("storefront", "1.0.0")] + ); + assert_eq!(lock.libs.len(), 1); + assert_eq!(lock.libs[0].spec.key(), "pip:fixture@1.0.0"); + assert!(owns_worker(root.path(), "worker/catalog/tool").unwrap()); + assert!(owns_worker(root.path(), "module/catalog/value/worker/main.py").unwrap()); + assert!(!owns_worker(root.path(), "module/portal/value/worker/main.py").unwrap()); + let files = owned_files_with(root.path(), &store).unwrap(); + assert!(String::from_utf8_lossy(&files["module/catalog/value/app.dever"]).contains("two")); + assert!(execute_with("doctor", root.path(), &[], ®istry, &store).is_ok()); + + let mut damaged = lock.clone(); + damaged.packages[0].manifest_sha256 = "0".repeat(64); + damaged.write_atomic(root.path()).unwrap(); + assert!( + owned_files_with(root.path(), &store) + .unwrap_err() + .contains("manifest changed") + ); +} + +#[test] +fn package_resolution_rejects_conflicts_without_changing_project_configuration() { + let root = project(); + let before = fs::read(root.path().join("config/setting.json")).unwrap(); + let registry = registry(&[ + ( + "storefront", + "1.0.0", + archive( + "storefront", + "1.0.0", + serde_json::json!({"catalog":"^2.0.0"}), + &[], + "value() (result: Text) { result = catalog.value.value() }\n", + ), + ), + ( + "catalog", + "1.0.0", + archive( + "catalog", + "1.0.0", + serde_json::json!({}), + &[], + "value() (result: Text) { result = \"one\" }\n", + ), + ), + ]); + let store = FixtureArtifactStore::default(); + let error = execute_with( + "add", + root.path(), + &["storefront@1.0.0".into()], + ®istry, + &store, + ) + .unwrap_err(); + assert!(error.contains("no Package version satisfies"), "{error}"); + assert_eq!( + fs::read(root.path().join("config/setting.json")).unwrap(), + before + ); + assert!(!root.path().join("dever.lock").exists()); +} + +#[test] +fn package_archive_rejects_paths_owned_by_another_component() { + let root = project(); + let mut writer = zip::ZipWriter::new(Cursor::new(Vec::new())); + for (path, bytes) in [ + ( + "dever-package.json", + r#"{"format":"dever-package-v1","name":"storefront","version":"1.0.0"}"#, + ), + ( + "module/portal/value/app.dever", + "value() (result: Text) { result = \"bad\" }", + ), + ] { + writer + .start_file(path, zip::write::SimpleFileOptions::default()) + .unwrap(); + writer.write_all(bytes.as_bytes()).unwrap(); + } + let registry = registry(&[("storefront", "1.0.0", writer.finish().unwrap().into_inner())]); + let store = FixtureArtifactStore::default(); + let error = execute_with( + "add", + root.path(), + &["storefront@1.0.0".into()], + ®istry, + &store, + ) + .unwrap_err(); + assert!(error.contains("unowned path"), "{error}"); +} + +#[test] +fn package_rejects_local_component_even_when_file_paths_do_not_overlap() { + let root = project(); + fs::create_dir_all(root.path().join("module/storefront/other")).unwrap(); + fs::write( + root.path().join("module/storefront/other/app.dever"), + "value() (result: Text) { result = \"local\" }\n", + ) + .unwrap(); + let registry = registry(&[( + "storefront", + "1.0.0", + archive( + "storefront", + "1.0.0", + serde_json::json!({}), + &[], + "value() (result: Text) { result = \"package\" }\n", + ), + )]); + let store = FixtureArtifactStore::default(); + let error = execute_with( + "add", + root.path(), + &["storefront@1.0.0".into()], + ®istry, + &store, + ) + .unwrap_err(); + assert!(error.contains("collides with a local component"), "{error}"); + assert!(!root.path().join("dever.lock").exists()); +} + +#[test] +fn package_only_project_loads_without_a_local_module_directory() { + let root = TemporaryDirectory::new(); + fs::create_dir(root.path().join("config")).unwrap(); + fs::write( + root.path().join("config/setting.json"), + r#"{"package":{"registry":"https://packages.example.test","use":[]}}"#, + ) + .unwrap(); + let registry = registry(&[( + "catalog", + "1.0.0", + archive( + "catalog", + "1.0.0", + serde_json::json!({}), + &[], + "value() (result: Text) { result = \"ready\" }\n", + ), + )]); + let store = FixtureArtifactStore::default(); + execute_with( + "add", + root.path(), + &["catalog@1.0.0".into()], + ®istry, + &store, + ) + .unwrap(); + let packages = sources_with(root.path(), &store).unwrap(); + let sources = dever_core::source::SourceMap::load_with_packages( + &root.path().join("module"), + None, + &packages, + ) + .unwrap(); + assert_eq!(sources.files().len(), 1); + dever_core::check(&sources).unwrap(); +} + +#[test] +fn locked_package_worker_never_falls_back_to_a_local_file() { + let root = project(); + let registry = registry(&[( + "storefront", + "1.0.0", + archive( + "storefront", + "1.0.0", + serde_json::json!({}), + &[], + "value() (result: Text) { result = \"two\" }\n", + ), + )]); + let store = FixtureArtifactStore::default(); + execute_with( + "add", + root.path(), + &["storefront@1.0.0".into()], + ®istry, + &store, + ) + .unwrap(); + let local = root.path().join("worker/storefront/tool"); + fs::create_dir_all(local.parent().unwrap()).unwrap(); + fs::write(&local, b"local fallback must never execute").unwrap(); + assert!(owns_worker(root.path(), "worker/storefront/tool").unwrap()); + assert!(owns_worker(root.path(), "module/storefront/value/worker/main.py").unwrap()); + assert!(!owns_worker(root.path(), "worker/portal/tool").unwrap()); + let error = owned_worker_bytes_with(root.path(), "worker/storefront/tool", &store).unwrap_err(); + assert!(error.contains("missing from its locked archive"), "{error}"); + assert_eq!( + owned_worker_bytes_with(root.path(), "worker/portal/tool", &store).unwrap(), + None + ); +} + +#[cfg(unix)] +#[test] +fn managed_package_worker_requires_its_locked_entry_and_rejects_local_collision() { + let root = project(); + let (layout, executable) = signed_core(root.path(), &std::env::current_exe().unwrap(), false); + let mut daemon = TestDaemon::start(&layout); + let package_archive = archive_with_sources( + "storefront", + "1.0.0", + serde_json::json!({}), + &[], + "type SendResult { error Unavailable(message: Text) }\nvalue() (result: Text) { result = \"two\" }\nsend(value: Int) (receipt: Int) { receipt = port.send(value) }\n", + &[ + ( + "module/storefront/value/port.dever", + "send(value: Int) (receipt: Int) fails app.SendResult\n", + ), + ( + "module/storefront/value/adapter.dever", + "external pip \"worker/main.py\" { }\n", + ), + ], + ); + fs::write( + root.path().join("config/setting.json"), + r#"{"package":{"registry":"https://packages.example.test","use":["storefront@1.0.0"]}}"#, + ) + .unwrap(); + let store = ServiceStore(layout.clone()); + let package_sha256 = store.publish(&package_archive, "package").unwrap(); + let mut zip = zip::ZipArchive::new(Cursor::new(&package_archive)).unwrap(); + let mut manifest = Vec::new(); + zip.by_name("dever-package.json") + .unwrap() + .read_to_end(&mut manifest) + .unwrap(); + let mut lock = LockFile::new(Vec::new()).unwrap(); + lock.packages.push(LockedPackage { + name: "storefront".into(), + version: "1.0.0".into(), + sha256: package_sha256, + bytes: package_archive.len() as u64, + dependencies: Vec::new(), + manifest_sha256: digest(&manifest), + }); + lock.write_atomic(root.path()).unwrap(); + + let package_sources = dever_cli::packages::sources_with(root.path(), &store).unwrap(); + let sources = dever_core::source::SourceMap::load_with_packages( + &root.path().join("module"), + None, + &package_sources, + ) + .unwrap(); + let program = dever_core::check(&sources).unwrap(); + let contract = program.external_worker_contracts().remove(0); + assert_eq!(contract.entry, "module/storefront/value/worker/main.py"); + assert!(owns_worker(root.path(), &contract.entry).unwrap()); + + let target = dever_cli::toolchain::platform_identity(); + let metadata = serde_json::to_vec(&serde_json::json!({ + "format": "dever-worker-runtime-v1", "ecosystem": "pip", "target": target, + "executable": "bin/python3", "arguments": ["-I", "-S", "-B"], + "python_wheel_tags": ["py3-none-any"], + "python_extension_suffixes": [".cpython-312-x86_64-linux-gnu.so", ".abi3.so", ".so"], + "python_markers": { + "implementation_name":"cpython", "implementation_version":"3.12.0", "os_name":"posix", + "platform_machine":"x86_64", "platform_python_implementation":"CPython", "platform_release":"", + "platform_system":"Linux", "platform_version":"", "python_full_version":"3.12.0", + "python_version":"3.12", "sys_platform":"linux" + }, + })) + .unwrap(); + let encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::none()); + let mut tar = tar::Builder::new(encoder); + for (path, bytes) in [ + ("dever-runtime.json", metadata.as_slice()), + ("bin/python3", b"fixture-interpreter".as_slice()), + ] { + let mut header = tar::Header::new_gnu(); + header.set_size(bytes.len() as u64); + header.set_mode(0o755); + header.set_cksum(); + tar.append_data(&mut header, path, bytes).unwrap(); + } + let runtime_bytes = tar.into_inner().unwrap().finish().unwrap(); + let runtime_sha256 = digest(&runtime_bytes); + let runtime = RuntimePack { + name: "fixture-pip".into(), + version: "1".into(), + sha256: runtime_sha256.clone(), + }; + lock.workers.push(LockedWorker { + port: contract.port, + adapter: contract.adapter, + ecosystem: contract.ecosystem, + runtime: Some(runtime), + entry: contract.entry.clone(), + schema: contract.schema, + capabilities: contract.capabilities, + operations: contract.operations, + libs: Vec::new(), + }); + lock.write_atomic(root.path()).unwrap(); + fs::write(root.path().join("test-runtime.pack"), runtime_bytes).unwrap(); + let output = Command::new(executable) + .args([ + "--exact", + "installed_managed_package_worker_guard", + "--ignored", + "--nocapture", + ]) + .current_dir(root.path()) + .output() + .unwrap(); + assert!( + output.status.success(), + "signed child test failed: {}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + daemon.stop(); +} + +#[cfg(unix)] +#[test] +#[ignore = "run only from the signed private fixture"] +fn installed_managed_package_worker_guard() { + let root = std::env::current_dir().unwrap(); + let package_sources = dever_cli::packages::sources(&root).unwrap(); + let sources = dever_core::source::SourceMap::load_with_packages( + &root.join("module"), + None, + &package_sources, + ) + .unwrap(); + let program = dever_core::check(&sources).unwrap(); + let entry = &program.external_worker_contracts()[0].entry; + assert_eq!(entry, "module/storefront/value/worker/main.py"); + assert!(owns_worker(&root, entry).unwrap()); + let runtime_bytes = fs::read(root.join("test-runtime.pack")).unwrap(); + let resources = [EmbeddedResource { + path: format!( + "lib/runtime/pip/{}/runtime.pack", + dever_cli::toolchain::platform_identity() + ), + sha256: digest(&runtime_bytes), + bytes: runtime_bytes, + executable: false, + }]; + let error = workers::prepare(&root, &program, &resources).unwrap_err(); + assert!( + error.contains("locked Package Worker entry") && error.contains("missing"), + "{error}" + ); + + let local = root.join(entry); + fs::create_dir_all(local.parent().unwrap()).unwrap(); + fs::write(local, b"local fallback must never execute").unwrap(); + let error = workers::prepare(&root, &program, &resources).unwrap_err(); + assert!(error.contains("collides with a local component"), "{error}"); +} + +#[test] +fn declared_package_worker_requires_a_lock_before_local_fallback() { + let root = project(); + fs::write( + root.path().join("config/setting.json"), + r#"{"package":{"registry":"https://packages.example.test","use":["storefront@1.0.0"]}}"#, + ) + .unwrap(); + let error = owns_worker(root.path(), "worker/storefront/tool").unwrap_err(); + assert!(error.contains("dever.lock"), "{error}"); +} + +#[test] +fn removing_a_package_keeps_locked_versions_without_registry_requests() { + let root = project(); + let registry = registry(&[ + ( + "storefront", + "1.0.0", + archive( + "storefront", + "1.0.0", + serde_json::json!({"catalog":"^1.0.0"}), + &[], + "value() (result: Text) { result = catalog.value.value() }\n", + ), + ), + ( + "catalog", + "1.2.0", + archive( + "catalog", + "1.2.0", + serde_json::json!({}), + &["pip:fixture@1.0.0"], + "value() (result: Text) { result = \"two\" }\n", + ), + ), + ]); + let store = FixtureArtifactStore::default(); + execute_with( + "add", + root.path(), + &["storefront@1.0.0".into()], + ®istry, + &store, + ) + .unwrap(); + execute_with( + "add", + root.path(), + &["catalog@^1.0.0".into()], + ®istry, + &store, + ) + .unwrap(); + fs::write( + root.path().join("module/portal/value/app.dever"), + "value() (result: Text) { result = catalog.value.value() }\n", + ) + .unwrap(); + let offline = Registry(BTreeMap::new()); + execute_with( + "remove", + root.path(), + &["storefront".into()], + &offline, + &store, + ) + .unwrap(); + let lock = LockFile::decode(&fs::read(root.path().join("dever.lock")).unwrap()).unwrap(); + assert_eq!(lock.packages.len(), 1); + assert_eq!( + ( + lock.packages[0].name.as_str(), + lock.packages[0].version.as_str() + ), + ("catalog", "1.2.0") + ); + assert_eq!(lock.libs[0].spec.key(), "pip:fixture@1.0.0"); + assert!( + !String::from_utf8_lossy(&fs::read(root.path().join("config/setting.json")).unwrap()) + .contains("storefront@") + ); + fs::write( + root.path().join("module/portal/value/app.dever"), + "value() (result: Text) { result = \"two\" }\n", + ) + .unwrap(); + execute_with("remove", root.path(), &["catalog".into()], &offline, &store).unwrap(); + let settings: serde_json::Value = + serde_json::from_slice(&fs::read(root.path().join("config/setting.json")).unwrap()) + .unwrap(); + assert_eq!( + settings["package"]["registry"], + "https://packages.example.test" + ); + assert_eq!(settings["package"]["use"], serde_json::json!([])); + execute_with( + "add", + root.path(), + &["storefront@1.0.0".into()], + ®istry, + &store, + ) + .unwrap(); + assert_eq!( + LockFile::decode(&fs::read(root.path().join("dever.lock")).unwrap()) + .unwrap() + .packages + .len(), + 2 + ); +} + +#[cfg(unix)] +#[test] +fn package_update_restores_original_setting_bytes_when_lock_write_fails() { + use std::os::unix::fs::symlink; + + let root = project(); + let registry = registry(&[( + "storefront", + "1.0.0", + archive( + "storefront", + "1.0.0", + serde_json::json!({}), + &[], + "value() (result: Text) { result = \"two\" }\n", + ), + )]); + let store = FixtureArtifactStore::default(); + execute_with( + "add", + root.path(), + &["storefront@1.0.0".into()], + ®istry, + &store, + ) + .unwrap(); + let settings = fs::read(root.path().join("config/setting.json")).unwrap(); + let lock_path = root.path().join("dever.lock"); + let saved = root.path().join("saved.lock"); + fs::rename(&lock_path, &saved).unwrap(); + symlink(&saved, &lock_path).unwrap(); + let error = execute_with("update", root.path(), &[], ®istry, &store).unwrap_err(); + assert!(error.contains("symbolic link"), "{error}"); + assert_eq!( + fs::read(root.path().join("config/setting.json")).unwrap(), + settings + ); + assert_eq!(fs::read(&saved).unwrap(), fs::read(&lock_path).unwrap()); +} + +#[cfg(unix)] +struct ServiceStore(Layout); + +#[cfg(unix)] +impl ArtifactStore for ServiceStore { + fn has(&self, _: &str, _: &str) -> Result<bool, String> { + Err("exact length required".into()) + } + fn publish(&self, bytes: &[u8], _: &str) -> Result<String, String> { + Ok(artifact_put(&self.0, bytes)?.sha256) + } + fn get(&self, _: &str, _: &str) -> Result<Option<Vec<u8>>, String> { + Err("exact length required".into()) + } + fn get_exact(&self, sha256: &str, bytes: u64, _: &str) -> Result<Option<Vec<u8>>, String> { + artifact_get(&self.0, sha256, bytes).map(Some) + } +} + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +fn installed_dever(root: &Path) -> (Layout, std::path::PathBuf) { + signed_core(root, Path::new(env!("CARGO_BIN_EXE_dever")), true) +} + +#[cfg(unix)] +fn signed_core( + root: &Path, + source_executable: &Path, + health_checked_install: bool, +) -> (Layout, std::path::PathBuf) { + use std::os::unix::fs::PermissionsExt; + + let layout = Layout::new(root.join("machine")); + layout.initialize().unwrap(); + let key_bytes = Ed25519KeyPair::generate_pkcs8(&SystemRandom::new()).unwrap(); + let key = Ed25519KeyPair::from_pkcs8(key_bytes.as_ref()).unwrap(); + fs::write( + layout.state().join("trusted-release-key"), + format!("{}\n", hex(key.public_key().as_ref())), + ) + .unwrap(); + let release = if health_checked_install { + layout.downloads().join("0.1.0") + } else { + layout.versions().join("0.1.0") + }; + fs::create_dir(&release).unwrap(); + let core = release.join("dever-core"); + #[cfg(all(target_os = "linux", target_arch = "x86_64"))] + if health_checked_install { + llvm_pack::compiler(source_executable, &release, "dever-core"); + } else { + fs::copy(source_executable, &core).unwrap(); + } + #[cfg(not(all(target_os = "linux", target_arch = "x86_64")))] + fs::copy(source_executable, &core).unwrap(); + fs::set_permissions(&core, fs::Permissions::from_mode(0o755)).unwrap(); + let bytes = fs::read(&core).unwrap(); + let artifacts = vec![Artifact { + path: "dever-core".into(), + bytes: bytes.len() as u64, + sha256: digest(&bytes), + }]; + #[cfg(all(target_os = "linux", target_arch = "x86_64"))] + let artifacts = if health_checked_install { + artifacts + .into_iter() + .chain(native_release_inputs(&release)) + .collect() + } else { + artifacts + }; + let manifest = ReleaseManifest::new(Version::parse("0.1.0").unwrap(), artifacts); + let manifest_bytes = serde_json::to_vec(&manifest).unwrap(); + fs::write(release.join("manifest.json"), &manifest_bytes).unwrap(); + fs::write( + release.join("manifest.sig"), + format!("{}\n", hex(key.sign(&manifest_bytes).as_ref())), + ) + .unwrap(); + let manager = MachineManager::new(layout.clone()); + if health_checked_install { + manager.install_requested("0.1.0").unwrap(); + } else { + manager + .resolve_core(&Version::parse("0.1.0").unwrap()) + .unwrap(); + } + (layout.clone(), layout.versions().join("0.1.0/dever-core")) +} + +#[cfg(unix)] +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +/// Explicit author inputs included in an ephemeral signature; these are not +/// the four trimmed runtime profiles of a formal distribution. +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +fn native_release_inputs(root: &Path) -> Vec<Artifact> { + let document = llvm_pack::manifest(root, true); + let prefix = format!("runtime/{}", dever_cli::toolchain::platform_identity()); + let paths = document["files"] + .as_array() + .unwrap() + .iter() + .map(|file| format!("{prefix}/{}", file["path"].as_str().unwrap())) + .chain([format!("{prefix}/manifest.json")]) + .chain( + fs::read_dir(root.join("lib")) + .unwrap() + .map(|entry| format!("lib/{}", entry.unwrap().file_name().to_str().unwrap())), + ); + paths + .map(|path| { + let file = root.join(&path); + Artifact { + path, + bytes: fs::metadata(&file).unwrap().len(), + sha256: dever_cli::toolchain::sha256_file(&file).unwrap(), + } + }) + .collect() +} + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +fn assert_cli_success(executable: &Path, arguments: &[&str]) -> std::process::Output { + let output = Command::new(executable) + .env_clear() + .args(arguments) + .output() + .unwrap(); + assert!( + output.status.success(), + "{} exited {}\nstdout: {}\nstderr: {}", + arguments.join(" "), + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + output +} + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[test] +#[ignore = "requires explicitly prepared native runtime archive and owned daemon"] +fn installed_package_passes_check_test_run_build_and_standalone_execution() { + let root = project(); + let (layout, executable) = installed_dever(root.path()); + let mut daemon = TestDaemon::start(&layout); + let registry = registry(&[ + ( + "storefront", + "1.0.0", + archive( + "storefront", + "1.0.0", + serde_json::json!({"catalog":"1.2.0"}), + &[], + "value() (result: Text) { result = catalog.value.value() }\n", + ), + ), + ( + "catalog", + "1.2.0", + archive( + "catalog", + "1.2.0", + serde_json::json!({}), + &[], + "value() (result: Text) { result = \"two\" }\n", + ), + ), + ]); + execute_with( + "add", + root.path(), + &["storefront@1.0.0".into()], + ®istry, + &ServiceStore(layout.clone()), + ) + .unwrap(); + let project = root.path().to_str().unwrap(); + assert_cli_success(&executable, &["check", project]); + assert_cli_success(&executable, &["test", project]); + let run = assert_cli_success( + &executable, + &["run", project, "--", "portal.value.show", "{}"], + ); + assert!(String::from_utf8_lossy(&run.stdout).contains("two")); + let output = root.path().join("standalone"); + assert_cli_success( + &executable, + &["build", project, "--output", output.to_str().unwrap()], + ); + daemon.stop(); + let standalone = assert_cli_success(&output, &["portal.value.show", "{}"]); + assert!(String::from_utf8_lossy(&standalone.stdout).contains("two")); +} diff --git a/test/dever-cli-tests/packages/managed_compile.rs b/test/dever-cli-tests/packages/managed_compile.rs new file mode 100644 index 0000000..22c8ca1 --- /dev/null +++ b/test/dever-cli-tests/packages/managed_compile.rs @@ -0,0 +1,456 @@ +//! Actual managed compilation, using ephemeral signing keys and an owned daemon. +use super::*; +use dever_cli::toolchain::compilation::{CompileKind, CompileRequest}; +use dever_cli::toolchain::{BuildTarget, CacheStore, compile}; +use dever_core::source::SourceMap; +use dever_runtime::config::Settings; +use std::io::Read; +use std::os::unix::net::UnixStream; +use std::time::{Duration, Instant}; + +fn source_project() -> TemporaryDirectory { + let root = project(); + fs::write( + root.path().join("module/portal/value/app.dever"), + "value() (result: Text) { result = \"two\" }\n", + ) + .unwrap(); + root +} + +fn request(root: &Path, kind: CompileKind) -> CompileRequest { + let sources = SourceMap::load_with_packages( + &root.join("module"), + if kind == CompileKind::TestSuite { + Some(root.join("test")) + } else { + None + } + .as_deref(), + &[], + ) + .unwrap(); + let bindings = (kind == CompileKind::Application) + .then(|| Settings::load_project(root).unwrap().compilation_bindings()); + CompileRequest::new( + "0.1.0".into(), + kind, + &sources, + bindings, + &[], + BuildTarget::host().unwrap(), + ) + .unwrap() +} + +#[test] +fn compilation_snapshot_excludes_secrets_and_rejects_ambiguous_or_unsafe_inputs() { + let root = source_project(); + fs::write(root.path().join("config/setting.json"), serde_json::to_vec(&serde_json::json!({ + "database":{"default":{"type":"sqlite","path":"private-database-marker.sqlite"}}, + "auth":{"providers":{"session":{"verify":"user.account.verify", + "jwtSecret":"private-jwt-marker-long-enough-for-hmac-2026","ttlSeconds":3600,"cookie":"secret-cookie-name"}}}, + "sites":{"admin":{"path":"admin","auth":"session","origin":"https://private-origin.example"}} + })).unwrap()).unwrap(); + let original = request(root.path(), CompileKind::Application); + let encoded = original.encode().unwrap(); + let text = std::str::from_utf8(&encoded).unwrap(); + for secret in [ + "private-database-marker", + "private-jwt-marker", + "secret-cookie-name", + "private-origin", + root.path().to_str().unwrap(), + ] { + assert!(!text.contains(secret), "compile input leaked {secret}"); + } + assert!(text.contains("user.account.verify")); + assert!(text.contains("sqlite")); + CompileRequest::decode(&encoded).unwrap(); + let duplicate = text.replace("\"databases\":", "\"databases\":{},\"databases\":"); + assert!(CompileRequest::decode(duplicate.as_bytes()).is_err()); + for path in ["/tmp/read-me.dever", "../main.dever", "portal/value/app.rs"] { + let mut invalid = original.clone(); + invalid.sources[0].logical_path = path.into(); + assert!(invalid.encode().is_err(), "accepted source path {path}"); + } + let mut reserved = original.clone(); + reserved.sources[0].logical_path = "dever/auth/app.dever".into(); + assert!( + dever_core::check(&reserved.source_map().unwrap()).is_err(), + "reserved standard namespace bypassed checking" + ); + let mut invalid = original.clone(); + invalid.sources.push(invalid.sources[0].clone()); + assert!(invalid.encode().is_err()); + let mut invalid = original; + invalid.sources[0].is_test = true; + assert!(invalid.encode().is_err()); + let suite = request(root.path(), CompileKind::TestSuite); + assert!(suite.bindings.is_none()); + assert!( + suite + .source_map() + .unwrap() + .files() + .iter() + .any(|source| source.is_test()) + ); +} + +#[test] +fn compilation_target_is_required_canonical_and_part_of_cache_identity() { + let root = source_project(); + let mut input = request(root.path(), CompileKind::Application); + input.target = BuildTarget::LinuxX86_64; + let host = input.encode().unwrap(); + input.target = BuildTarget::LinuxAarch64; + let arm = input.encode().unwrap(); + assert_ne!(host, arm); + assert_eq!( + CompileRequest::decode(&arm).unwrap().target, + BuildTarget::LinuxAarch64 + ); + let text = std::str::from_utf8(&arm).unwrap(); + for invalid in [ + text.replace("\"target\":\"linux-aarch64\",", ""), + text.replace("linux-aarch64", "aarch64-unknown-linux-gnu"), + text.replace("\"target\":", "\"target\":\"linux-x86_64\",\"target\":"), + ] { + assert!(CompileRequest::decode(invalid.as_bytes()).is_err()); + } + let mut suite = request(root.path(), CompileKind::TestSuite); + suite.target = if BuildTarget::host().unwrap() == BuildTarget::LinuxX86_64 { + BuildTarget::LinuxAarch64 + } else { + BuildTarget::LinuxX86_64 + }; + assert!(suite.encode().unwrap_err().contains("host target")); +} + +#[test] +#[ignore = "requires explicitly prepared native archive; starts an owned daemon"] +fn trusted_compilation_cache_requires_sources_and_revalidates_signed_pack() { + let root = source_project(); + let (layout, _) = installed_dever(root.path()); + let _daemon = TestDaemon::start(&layout); + let input = request(root.path(), CompileKind::Application); + let started = Instant::now(); + let first = compile(&layout, &input).unwrap(); + let cold_seconds = started.elapsed().as_secs_f64(); + let initial_cache = cache_status(&layout).unwrap(); + assert_eq!(initial_cache.entries, 1); + let started = Instant::now(); + assert_eq!(compile(&layout, &input).unwrap(), first); + let hit_seconds = started.elapsed().as_secs_f64(); + assert_eq!(cache_status(&layout).unwrap().entries, 1); + assert!(artifact_get(&layout, &digest(&first), first.len() as u64).is_err()); + let mut invalid = input.clone(); + invalid + .sources + .iter_mut() + .find(|source| source.logical_path.ends_with("app.dever")) + .unwrap() + .text = "value() (result: Text) { result = missing_function() }".into(); + let error = compile(&layout, &invalid).unwrap_err(); + assert!(error.contains("missing_function"), "{error}"); + assert_eq!(cache_status(&layout).unwrap().entries, 1); + assert_eq!(compile(&layout, &input).unwrap(), first); + let mut changed = input.clone(); + changed + .sources + .iter_mut() + .find(|source| source.logical_path.ends_with("app.dever")) + .unwrap() + .text = "value() (result: Text) { result = \"three\" }".into(); + // 两个独立客户端同时提交同一份新源码，必须收敛到一个已验证产物。 + let barrier = std::sync::Barrier::new(2); + let concurrent = std::thread::scope(|scope| { + let clients: Vec<_> = (0..2) + .map(|_| { + scope.spawn(|| { + barrier.wait(); + let started = Instant::now(); + let bytes = compile(&layout, &changed).unwrap(); + (bytes, started.elapsed().as_secs_f64()) + }) + }) + .collect(); + clients + .into_iter() + .map(|client| client.join().unwrap()) + .collect::<Vec<_>>() + }); + assert_eq!(concurrent[0].0, concurrent[1].0); + assert_ne!(concurrent[0].0, first); + let shared_cache = cache_status(&layout).unwrap(); + assert_eq!(shared_cache.entries, 2); + + // The worker compiles the suite; only the caller runs its failing case. + fs::write( + root.path().join("test/portal/value/smoke.dever"), + "smoke() () { assert_eq(1, 2) }", + ) + .unwrap(); + let suite = compile(&layout, &request(root.path(), CompileKind::TestSuite)).unwrap(); + let program = dever_core::native::NativeProgram::from_verified_bytes(&suite).unwrap(); + assert!( + !Command::new(program.executable()) + .arg("0") + .env_clear() + .output() + .unwrap() + .status + .success() + ); + assert_eq!(cache_status(&layout).unwrap().entries, 3); + + // A valid signature over an incomplete list must not authorize unsigned + // link inputs, even when the native manifest still names those inputs. + let version_root = layout.versions().join("0.1.0"); + let trusted_key = fs::read(layout.state().join("trusted-release-key")).unwrap(); + let signed_manifest = fs::read(version_root.join("manifest.json")).unwrap(); + let signature = fs::read(version_root.join("manifest.sig")).unwrap(); + let mut manifest: ReleaseManifest = serde_json::from_slice(&signed_manifest).unwrap(); + manifest + .artifacts + .retain(|artifact| !artifact.path.ends_with("/crt1.o")); + let key_bytes = Ed25519KeyPair::generate_pkcs8(&SystemRandom::new()).unwrap(); + let key = Ed25519KeyPair::from_pkcs8(key_bytes.as_ref()).unwrap(); + let bytes = serde_json::to_vec(&manifest).unwrap(); + fs::write( + layout.state().join("trusted-release-key"), + hex(key.public_key().as_ref()), + ) + .unwrap(); + fs::write(version_root.join("manifest.json"), &bytes).unwrap(); + fs::write( + version_root.join("manifest.sig"), + hex(key.sign(&bytes).as_ref()), + ) + .unwrap(); + assert!( + compile(&layout, &input) + .unwrap_err() + .contains("not covered by the signed release") + ); + fs::write(layout.state().join("trusted-release-key"), trusted_key).unwrap(); + fs::write(version_root.join("manifest.json"), signed_manifest).unwrap(); + fs::write(version_root.join("manifest.sig"), signature).unwrap(); + + let pack = layout.versions().join("0.1.0/runtime/linux-x86_64/crt1.o"); + fs::write(pack, b"changed signed link input").unwrap(); + let error = compile(&layout, &input).unwrap_err(); + assert!(error.contains("integrity"), "{error}"); + assert_eq!(cache_status(&layout).unwrap().entries, 3); + assert_eq!( + fs::read_dir(layout.native_cache().join("staging")) + .unwrap() + .count(), + 0 + ); + assert_eq!( + fs::read_dir(layout.native_cache().join("operations")) + .unwrap() + .count(), + 0 + ); + let report = serde_json::json!({ + "format": "dever-shared-compile-measurement-v1", + "scope": "owned signed compiler and daemon; client wall time includes verification and IPC; warm OS caches", + "cold_seconds": cold_seconds, + "cache_hit_seconds": hit_seconds, + "artifact_bytes": first.len(), + "artifact_sha256": digest(&first), + "initial_cache": initial_cache, + "concurrent_clients": concurrent.len(), + "concurrent_miss_seconds": concurrent.iter().map(|(_, seconds)| seconds).collect::<Vec<_>>(), + "concurrent_same_artifact": true, + "shared_cache": shared_cache, + }); + fs::write( + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../target/closure-shared-compile-measurement.json"), + serde_json::to_vec_pretty(&report).unwrap(), + ) + .unwrap(); +} + +#[test] +#[ignore = "requires root and prepared archive; only owned clients change UID"] +fn different_machine_users_share_compilation_without_private_cache() { + use std::os::unix::fs::PermissionsExt; + use std::os::unix::process::CommandExt; + assert_eq!(rustix::process::geteuid().as_raw(), 0); + let root = source_project(); + let other = source_project(); + // Only these owned fixtures are shared; the usual temporary root is 0700. + fs::set_permissions(root.path(), fs::Permissions::from_mode(0o755)).unwrap(); + fs::set_permissions(other.path(), fs::Permissions::from_mode(0o755)).unwrap(); + let (layout, core) = installed_dever(root.path()); + let _daemon = TestDaemon::start(&layout); + assert_cli_success( + &core, + &[ + "run", + root.path().to_str().unwrap(), + "--", + "portal.value.show", + "{}", + ], + ); + assert_eq!(cache_status(&layout).unwrap().entries, 1); + let result = Command::new(&core) + .uid(65534) + .gid(65534) + .env_clear() + .args([ + "run", + other.path().to_str().unwrap(), + "--", + "portal.value.show", + "{}", + ]) + .output() + .unwrap(); + assert!( + result.status.success(), + "{}", + String::from_utf8_lossy(&result.stderr) + ); + assert!(String::from_utf8_lossy(&result.stdout).contains("two")); + assert_eq!(cache_status(&layout).unwrap().entries, 1); + assert!(!core.parent().unwrap().join("cache").exists()); + assert!(!other.path().join("cache").exists()); +} + +fn frame(stream: &mut UnixStream) -> serde_json::Value { + let mut size = [0; 4]; + stream.read_exact(&mut size).unwrap(); + let mut bytes = vec![0; u32::from_be_bytes(size) as usize]; + stream.read_exact(&mut bytes).unwrap(); + serde_json::from_slice(&bytes).unwrap() +} + +fn upload(layout: &Layout, input: &CompileRequest) -> UnixStream { + let payload = input.encode().unwrap(); + let mut stream = UnixStream::connect(layout.service_socket()).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(10))) + .unwrap(); + let token = fs::read_to_string(layout.service_token()).unwrap(); + let control = serde_json::to_vec(&serde_json::json!({"token":token.trim(), + "operation":{"compile":{"bytes":payload.len()}},"installed":[]})) + .unwrap(); + stream + .write_all(&(control.len() as u32).to_be_bytes()) + .unwrap(); + stream.write_all(&control).unwrap(); + assert_eq!(frame(&mut stream)["ok"], true); + stream.write_all(&payload).unwrap(); + stream +} + +fn wait_for<T>(mode: &str, phase: &str, mut action: impl FnMut() -> Option<T>) -> T { + let started = Instant::now(); + loop { + if let Some(value) = action() { + return value; + } + assert!( + started.elapsed() < Duration::from_secs(10), + "owned compilation mode={mode} timed out waiting for {phase}" + ); + std::thread::sleep(Duration::from_millis(10)); + } +} + +#[test] +#[ignore = "requires prepared archive, explicit author cc/LLVM inputs; supervises an owned signed sleeper"] +fn cancellation_disconnect_and_version_lease_reap_worker_and_staging() { + let root = source_project(); + let sleeper = root.path().join("signed-sleeper"); + let source = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../test/dever-cli-tests/support/compiler_sleeper.c"); + let status = Command::new("/usr/bin/cc") + .env_clear() + .args([ + "-B/usr/bin/", + "-O2", + "-Wl,--disable-new-dtags,-rpath,$ORIGIN/lib,--no-as-needed", + "-L/usr/lib/llvm-18/lib", + "-lLLVM-18", + "-lstdc++", + "-lz", + "-lgcc_s", + ]) + .arg(source) + .arg("-o") + .arg(&sleeper) + .status() + .unwrap(); + assert!( + status.success(), + "explicit native sleeper preparation failed" + ); + let (layout, _) = signed_core(root.path(), &sleeper, true); + let _daemon = TestDaemon::start(&layout); + for mode in ["cancel", "disconnect", "deadline"] { + let mut stream = upload(&layout, &request(root.path(), CompileKind::Application)); + let pid = wait_for(mode, "worker PID publication", || { + fs::read_dir(layout.native_cache().join("staging")) + .ok()? + .filter_map(Result::ok) + .find_map(|entry| { + fs::read_to_string(entry.path().join("worker.pid")) + .ok()? + .parse::<i32>() + .ok() + .filter(|pid| *pid > 0) + .map(|pid| pid.to_string()) + }) + }); + let lock = fs::OpenOptions::new() + .read(true) + .write(true) + .open(layout.state().join("install.lock")) + .unwrap(); + assert!( + fs2::FileExt::try_lock_exclusive(&lock).is_err(), + "active compiler version was not leased" + ); + assert!( + CacheStore::new(&layout) + .clean(&Default::default()) + .unwrap_err() + .contains("busy") + ); + if mode == "cancel" { + stream.write_all(&[1]).unwrap(); + let response = frame(&mut stream); + assert_eq!(response["ok"], false); + assert!(response["error"].as_str().unwrap().contains("cancel")); + } else if mode == "deadline" { + stream + .set_read_timeout(Some(Duration::from_secs(190))) + .unwrap(); + let response = frame(&mut stream); + assert_eq!(response["ok"], false); + assert!(response["error"].as_str().unwrap().contains("deadline")); + } + drop(stream); + wait_for(mode, "worker exit and staging/operation cleanup", || { + let staging = fs::read_dir(layout.native_cache().join("staging")) + .ok()? + .count(); + let operations = fs::read_dir(layout.native_cache().join("operations")) + .ok()? + .count(); + (staging == 0 && operations == 0 && !Path::new("/proc").join(&pid).exists()) + .then_some(()) + }); + fs2::FileExt::try_lock_exclusive(&lock).unwrap(); + fs2::FileExt::unlock(&lock).unwrap(); + assert_eq!(cache_status(&layout).unwrap().entries, 0); + } +} diff --git a/test/dever-cli-tests/release_source.rs b/test/dever-cli-tests/release_source.rs new file mode 100644 index 0000000..ed45f3a --- /dev/null +++ b/test/dever-cli-tests/release_source.rs @@ -0,0 +1,289 @@ +use super::super::release::Artifact; +use super::*; +use sha2::{Digest, Sha256}; + +fn manifest() -> ReleaseManifest { + ReleaseManifest::new( + Version::parse("1.2.3").unwrap(), + vec![Artifact { + path: "dever-core".into(), + bytes: 4, + sha256: super::super::release::hex(&Sha256::digest(b"core")), + }], + ) +} + +fn archive(entries: &[(&str, &[u8])]) -> Vec<u8> { + let mut archive = tar::Builder::new(Vec::new()); + for (name, contents) in entries { + let mut header = tar::Header::new_ustar(); + header.set_size(contents.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + archive.append_data(&mut header, name, *contents).unwrap(); + } + archive.into_inner().unwrap() +} + +#[test] +fn release_archive_accepts_only_the_exact_signed_regular_files() { + let root = Download::new(&std::env::temp_dir()).unwrap(); + unpack( + archive(&[("dever-core", b"core")]).as_slice(), + &root.0, + &manifest(), + ) + .unwrap(); + assert_eq!(fs::read(root.0.join("dever-core")).unwrap(), b"core"); + for entries in [ + vec![ + ("dever-core", b"core".as_slice()), + ("extra", b"x".as_slice()), + ], + vec![ + ("dever-core", b"core".as_slice()), + ("dever-core", b"core".as_slice()), + ], + vec![("dever-core", b"bad".as_slice())], + vec![], + ] { + let root = Download::new(&std::env::temp_dir()).unwrap(); + assert!(unpack(archive(&entries).as_slice(), &root.0, &manifest()).is_err()); + } +} + +#[test] +fn release_archive_rejects_links_and_extension_headers() { + for kind in [ + tar::EntryType::Symlink, + tar::EntryType::Link, + tar::EntryType::XGlobalHeader, + ] { + let root = Download::new(&std::env::temp_dir()).unwrap(); + let mut archive = tar::Builder::new(Vec::new()); + let mut header = tar::Header::new_ustar(); + header.set_path("dever-core").unwrap(); + header.set_entry_type(kind); + header.set_size(0); + header.set_mode(0o644); + header.set_link_name("/outside").unwrap(); + header.set_cksum(); + archive.append(&header, io::empty()).unwrap(); + assert!( + unpack( + archive.into_inner().unwrap().as_slice(), + &root.0, + &manifest() + ) + .is_err() + ); + assert!(!root.0.join("dever-core").exists()); + } +} + +#[test] +fn signed_catalog_paths_and_total_size_are_bounded_before_extraction() { + for path in [ + "/absolute", + "../outside", + "a/../b", + "./file", + "a//b", + "a\\b", + "C:drive", + "manifest.json", + "manifest.sig", + ] { + let mut manifest = manifest(); + manifest.artifacts[0].path = path.into(); + assert!(validate_catalog(&manifest).is_err(), "{path}"); + } + let mut excessive = manifest(); + excessive.artifacts[0].bytes = RELEASE_LIMIT + 1; + assert!(validate_catalog(&excessive).is_err()); + let mut duplicate = manifest(); + duplicate.artifacts.push(duplicate.artifacts[0].clone()); + assert!(validate_catalog(&duplicate).is_err()); +} + +#[test] +fn official_redirects_reject_http_credentials_ports_and_foreign_hosts() { + let current = "https://github.com/shemic/dever-main/releases/latest/download/a"; + assert!( + redirect( + current, + "https://release-assets.githubusercontent.com/asset?token=opaque" + ) + .is_ok() + ); + assert!(redirect(current, "/shemic/dever-main/releases/download/v1.2.3/a").is_ok()); + for address in [ + "http://github.com/a", + "https://evil.example/a", + "https://github.com.evil.example/a", + "https://user@github.com/a", + "https://github.com:444/a", + "https://github.com/a#fragment", + ] { + assert!(redirect(current, address).is_err(), "{address}"); + } +} + +#[test] +fn official_asset_names_are_versioned_and_platform_specific() { + let source = ReleaseSource::official(); + assert_eq!( + source.asset("latest", "manifest.json"), + format!( + "{OFFICIAL_RELEASES}/latest/download/dever-{}.manifest.json", + platform_identity() + ) + ); + assert_eq!( + source.asset("1.2.3", "tar.gz"), + format!( + "{OFFICIAL_RELEASES}/download/v1.2.3/dever-{}.tar.gz", + platform_identity() + ) + ); +} + +fn signed_release() -> (ReleaseManifest, Vec<u8>, Vec<u8>, Vec<u8>) { + use ring::signature::{Ed25519KeyPair, KeyPair}; + let pair = Ed25519KeyPair::from_seed_unchecked(&[37; 32]).unwrap(); + let mut manifest = manifest(); + manifest.artifacts.push(Artifact { + path: "skills/dever-language/SKILL.md".into(), + bytes: 5, + sha256: super::super::release::hex(&Sha256::digest(b"skill")), + }); + let bytes = serde_json::to_vec(&manifest).unwrap(); + let signature = super::super::release::hex(pair.sign(&bytes).as_ref()).into_bytes(); + ( + manifest, + bytes, + signature, + pair.public_key().as_ref().to_vec(), + ) +} + +fn server(responses: Vec<Vec<u8>>) -> (ReleaseSource, std::thread::JoinHandle<()>) { + use std::io::BufRead; + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let base = format!("http://{}/releases", listener.local_addr().unwrap()); + let worker = std::thread::spawn(move || { + let deadline = Instant::now() + Duration::from_secs(5); + for response in responses { + let mut stream = loop { + match listener.accept() { + Ok((stream, _)) => break stream, + Err(error) if error.kind() == io::ErrorKind::WouldBlock => { + assert!( + Instant::now() < deadline, + "expected release request did not arrive" + ); + std::thread::sleep(Duration::from_millis(5)); + } + Err(error) => panic!("{error}"), + } + }; + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let mut reader = io::BufReader::new(stream.try_clone().unwrap()); + let mut request = String::new(); + reader.read_line(&mut request).unwrap(); + assert!(request.starts_with("GET /releases/")); + loop { + let mut line = String::new(); + reader.read_line(&mut line).unwrap(); + if line == "\r\n" { + break; + } + } + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + response.len() + ) + .unwrap(); + stream.write_all(&response).unwrap(); + } + }); + (ReleaseSource::new(base), worker) +} + +fn compressed_release(core: &[u8]) -> Vec<u8> { + let bytes = archive(&[ + ("dever-core", core), + ("skills/dever-language/SKILL.md", b"skill"), + ]); + let mut gzip = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + gzip.write_all(&bytes).unwrap(); + gzip.finish().unwrap() +} + +#[test] +fn explicit_download_verifies_the_complete_bundle_before_publishing_catalog() { + let root = Download::new(&std::env::temp_dir()).unwrap(); + let layout = Layout::new(root.0.join("machine")); + layout.initialize().unwrap(); + let (manifest, bytes, signature, key) = signed_release(); + fs::write( + layout.state().join("trusted-release-key"), + super::super::release::hex(&key), + ) + .unwrap(); + let (source, worker) = server(vec![bytes, signature, compressed_release(b"core")]); + let version = prepare_from(&layout, "latest", &source).unwrap(); + worker.join().unwrap(); + assert_eq!(version, manifest.version); + let installed = layout.downloads().join(version.as_str()); + MachineManager::verify_package(&installed, Some(&version), &key).unwrap(); + assert_eq!( + fs::read(installed.join("skills/dever-language/SKILL.md")).unwrap(), + b"skill" + ); + assert_eq!( + fs::read_to_string(layout.downloads().join("latest")).unwrap(), + "1.2.3\n" + ); + assert_eq!(fs::read_dir(layout.staging()).unwrap().count(), 0); +} + +#[test] +fn bad_signatures_and_corrupt_payloads_preserve_the_previous_catalog_and_active_version() { + for bad_signature in [true, false] { + let root = Download::new(&std::env::temp_dir()).unwrap(); + let layout = Layout::new(root.0.join("machine")); + layout.initialize().unwrap(); + let (_, bytes, signature, key) = signed_release(); + fs::write( + layout.state().join("trusted-release-key"), + super::super::release::hex(&key), + ) + .unwrap(); + fs::write(layout.downloads().join("latest"), "0.9.0\n").unwrap(); + let active = layout.state().join("active-version"); + super::super::release::append_active_version(&active, &Version::parse("0.9.0").unwrap()) + .unwrap(); + let previous = fs::read(&active).unwrap(); + let responses = if bad_signature { + vec![bytes, vec![b'0'; 128]] + } else { + vec![bytes, signature, compressed_release(b"evil")] + }; + let (source, worker) = server(responses); + let error = prepare_from(&layout, "latest", &source).unwrap_err(); + worker.join().unwrap(); + assert!(error.contains("verification"), "{error}"); + assert_eq!(fs::read(&active).unwrap(), previous); + assert_eq!( + fs::read_to_string(layout.downloads().join("latest")).unwrap(), + "0.9.0\n" + ); + assert!(!layout.downloads().join("1.2.3").exists()); + assert_eq!(fs::read_dir(layout.staging()).unwrap().count(), 0); + } +} diff --git a/test/dever-cli-tests/shared_toolchain.rs b/test/dever-cli-tests/shared_toolchain.rs new file mode 100644 index 0000000..eb9f486 --- /dev/null +++ b/test/dever-cli-tests/shared_toolchain.rs @@ -0,0 +1,1375 @@ +use std::collections::BTreeSet; +use std::ffi::OsString; +use std::fs; +#[cfg(unix)] +use std::io::Read; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use dever_cli::toolchain::{ + Artifact, BuildIdentity, CacheStore, CommandResult, Layout, MachineManager, PlatformContract, + ReleaseManifest, Version, artifact_get, artifact_put, execute_in, platform_contract, +}; +use ring::rand::SystemRandom; +use ring::signature::{Ed25519KeyPair, KeyPair}; +use sha2::{Digest, Sha256}; + +static NEXT_TEMPORARY: AtomicU64 = AtomicU64::new(0); + +struct TemporaryDirectory(PathBuf); + +impl TemporaryDirectory { + fn new() -> Self { + loop { + let path = std::env::temp_dir().join(format!( + "dever-toolchain-test-{}-{}", + std::process::id(), + NEXT_TEMPORARY.fetch_add(1, Ordering::Relaxed), + )); + match fs::create_dir(&path) { + Ok(()) => return Self(path), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => panic!("cannot create test directory: {error}"), + } + } + } + + fn path(&self) -> &Path { + &self.0 + } +} + +impl Drop for TemporaryDirectory { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +#[cfg(unix)] +struct TestDaemon(Child); + +#[cfg(unix)] +impl TestDaemon { + fn start(layout: &Layout) -> Self { + let child = Command::new(env!("CARGO_BIN_EXE_deverd")) + .env_clear() + .args(["--root", layout.root().to_str().unwrap()]) + .spawn() + .unwrap(); + let mut daemon = Self(child); + for _ in 0..100 { + if dever_cli::toolchain::cache_status(layout).is_ok() { + return daemon; + } + assert!( + daemon.0.try_wait().unwrap().is_none(), + "deverd exited before becoming ready" + ); + std::thread::sleep(std::time::Duration::from_millis(10)); + } + panic!("deverd did not publish its endpoint"); + } + + fn stop(&mut self) { + self.0.kill().unwrap(); + self.0.wait().unwrap(); + } +} + +#[cfg(unix)] +impl Drop for TestDaemon { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +#[cfg(unix)] +fn raw_request(layout: &Layout, token: &str, operation: &str) -> serde_json::Value { + raw_operation(layout, token, serde_json::Value::String(operation.into())) +} + +#[cfg(unix)] +fn raw_operation(layout: &Layout, token: &str, operation: serde_json::Value) -> serde_json::Value { + use std::os::unix::net::UnixStream; + + let mut stream = UnixStream::connect(layout.service_socket()).unwrap(); + stream + .set_read_timeout(Some(std::time::Duration::from_secs(2))) + .unwrap(); + let payload = serde_json::to_vec(&serde_json::json!({ + "token": token, + "operation": operation, + "installed": [], + })) + .unwrap(); + stream + .write_all(&(payload.len() as u32).to_be_bytes()) + .unwrap(); + stream.write_all(&payload).unwrap(); + read_response(&mut stream) +} + +#[cfg(unix)] +#[test] +fn compilation_control_rejects_client_ir_paths_and_oversized_bodies() { + let temporary = TemporaryDirectory::new(); + let layout = Layout::new(temporary.path().join("machine")); + layout.initialize().unwrap(); + let _daemon = TestDaemon::start(&layout); + let token = fs::read_to_string(layout.service_token()).unwrap(); + for extra in ["ir", "output", "cache_key", "compiler"] { + let mut operation = serde_json::json!({"compile":{"bytes":1}}); + operation["compile"][extra] = serde_json::json!("caller-controlled"); + let response = raw_operation(&layout, token.trim(), operation); + assert_eq!(response["ok"], false); + assert!( + response["error"] + .as_str() + .unwrap() + .contains("unknown field"), + "{response}" + ); + } + for bytes in [0, 384 * 1024 * 1024 + 1, u64::MAX] { + let response = raw_operation( + &layout, + token.trim(), + serde_json::json!({"compile":{"bytes":bytes}}), + ); + assert_eq!(response["ok"], false); + assert!( + response["error"] + .as_str() + .unwrap() + .contains("transfer limit") + ); + } + assert_eq!( + dever_cli::toolchain::cache_status(&layout).unwrap().entries, + 0 + ); +} + +#[cfg(unix)] +fn read_response(stream: &mut std::os::unix::net::UnixStream) -> serde_json::Value { + let mut header = [0_u8; 4]; + stream.read_exact(&mut header).unwrap(); + let mut payload = vec![0; u32::from_be_bytes(header) as usize]; + stream.read_exact(&mut payload).unwrap(); + serde_json::from_slice(&payload).unwrap() +} + +#[test] +fn runtime_accepts_only_an_exact_project_toolchain_version() { + let temporary = TemporaryDirectory::new(); + let config = temporary.path().join("config"); + fs::create_dir(&config).unwrap(); + fs::write( + config.join("setting.json"), + r#"{"dever":{"version":"1.2.3"}}"#, + ) + .unwrap(); + let settings = dever_runtime::config::Settings::load_project(temporary.path()).unwrap(); + assert_eq!(settings.dever_version(), Some("1.2.3")); + + for invalid in [ + r#"{"dever":{"version":"latest"}}"#, + r#"{"dever":{"version":"01.2.3"}}"#, + r#"{"dever":{"version":"1.2"}}"#, + r#"{"dever":{"version":"1.2.3","channel":"stable"}}"#, + r#"{"dever":{"version":"1.2.3","version":"2.0.0"}}"#, + ] { + fs::write(config.join("setting.json"), invalid).unwrap(); + assert!( + dever_runtime::config::Settings::load_project(temporary.path()).is_err(), + "accepted {invalid}" + ); + } + assert!(Version::parse("1.10.0").unwrap() > Version::parse("1.2.0").unwrap()); +} + +#[test] +fn all_supported_platforms_have_one_machine_entry_and_service_contract() { + for (os, expected) in [ + ("linux", ("/opt/dever", "/usr/local/bin/dever")), + ( + "macos", + ("/Library/Application Support/Dever", "/usr/local/bin/dever"), + ), + ( + "windows", + ( + r"C:\Program Files\Dever", + r"C:\Program Files\Dever\bin\dever.exe", + ), + ), + ] { + let PlatformContract { + machine_root, + public_entry, + service_definition, + ipc_endpoint, + } = platform_contract(os).expect("supported platform contract"); + assert_eq!((machine_root, public_entry), expected); + assert!(!service_definition.is_empty()); + assert!(!ipc_endpoint.is_empty()); + } + assert!(platform_contract("freebsd").is_none()); +} + +#[cfg(unix)] +#[test] +fn signed_versions_install_select_dispatch_update_and_rollback_atomically() { + use std::os::unix::fs::PermissionsExt; + + let temporary = TemporaryDirectory::new(); + let layout = Layout::new(temporary.path().join("machine")); + layout.initialize().unwrap(); + let key = signing_key(); + fs::write( + layout.state().join("trusted-release-key"), + format!("{}\n", hex(key.public_key().as_ref())), + ) + .unwrap(); + let dispatch_log = temporary.path().join("dispatch.log"); + for (version, healthy, valid_signature) in [ + ("1.0.0", true, true), + ("2.0.0", true, true), + ("3.0.0", false, true), + ("4.0.0", true, false), + ] { + write_release( + &layout, + &key, + version, + &dispatch_log, + healthy, + valid_signature, + ); + } + fs::write(layout.downloads().join("latest"), "2.0.0\n").unwrap(); + let manager = MachineManager::new(layout.clone()); + + fs::write( + layout.state().join("active-version"), + "dever-active-version-v1\n\npartial", + ) + .unwrap(); + assert_eq!(manager.active_version().unwrap(), None); + assert_eq!( + manager.install_requested("1.0.0").unwrap().as_str(), + "1.0.0" + ); + assert_eq!( + fs::metadata(layout.versions().join("1.0.0/dever-core")) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o755 + ); + assert_eq!( + fs::metadata(layout.versions().join("1.0.0/runtime.pack")) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o644 + ); + assert_eq!(manager.active_version().unwrap().unwrap().as_str(), "1.0.0"); + fs::OpenOptions::new() + .append(true) + .open(layout.state().join("active-version")) + .unwrap() + .write_all(b"\npartial-record") + .unwrap(); + assert_eq!(manager.active_version().unwrap().unwrap().as_str(), "1.0.0"); + manager.install_requested("2.0.0").unwrap(); + assert_eq!(manager.active_version().unwrap().unwrap().as_str(), "1.0.0"); + + let skill_path = |layout: &Layout| { + let CommandResult::Message(path) = + execute_in(layout, &["skill".into(), "path".into()]).unwrap() + else { + panic!("skill path must return the active signed directory") + }; + PathBuf::from(path) + }; + assert_eq!( + fs::read_to_string(skill_path(&layout).join("SKILL.md")).unwrap(), + "skill 1.0.0\n" + ); + let skill_entry = temporary.path().join("dever-language"); + execute_in( + &layout, + &[ + "skill".into(), + "install".into(), + skill_entry.as_os_str().into(), + ], + ) + .unwrap(); + let loader = fs::read(skill_entry.join("SKILL.md")).unwrap(); + assert!(String::from_utf8_lossy(&loader).contains("skill path")); + assert!( + execute_in( + &layout, + &[ + "skill".into(), + "install".into(), + skill_entry.as_os_str().into() + ] + ) + .is_err() + ); + assert_eq!(fs::read(skill_entry.join("SKILL.md")).unwrap(), loader); + + let new_project = temporary.path().join("not-created-yet"); + let CommandResult::Exited(code) = execute_in( + &layout, + &[ + "new".into(), + new_project.as_os_str().into(), + "--markdown".into(), + ], + ) + .unwrap() else { + panic!("new must dispatch without project settings") + }; + assert_eq!(code, 0); + assert_eq!( + fs::read_to_string(&dispatch_log).unwrap(), + format!("new\n{}\n--markdown\n", new_project.display()) + ); + + let project = temporary.path().join("project"); + fs::create_dir_all(project.join("config")).unwrap(); + fs::write( + project.join("config/setting.json"), + r#"{"dever":{"version":"2.0.0"}}"#, + ) + .unwrap(); + let arguments = ["check", project.to_str().unwrap()].map(OsString::from); + let CommandResult::Message(pinned_skill) = execute_in( + &layout, + &["skill".into(), "path".into(), project.as_os_str().into()], + ) + .unwrap() else { + panic!("project skill must match its selected compiler") + }; + assert_eq!( + Path::new(&pinned_skill), + layout.versions().join("2.0.0/skills/dever-language") + ); + let CommandResult::Exited(code) = execute_in(&layout, &arguments).unwrap() else { + panic!("project command must dispatch") + }; + assert_eq!(code, 0); + assert_eq!( + fs::read_to_string(&dispatch_log).unwrap(), + format!("check\n{}\n", project.display()) + ); + + let clean_arguments = ["clean", project.to_str().unwrap()].map(OsString::from); + let CommandResult::Exited(code) = execute_in(&layout, &clean_arguments).unwrap() else { + panic!("clean command must use the selected project core") + }; + assert_eq!(code, 0); + assert_eq!( + fs::read_to_string(&dispatch_log).unwrap(), + format!("clean\n{}\n", project.display()) + ); + + assert_eq!(manager.update().unwrap().as_str(), "2.0.0"); + assert_eq!(manager.active_version().unwrap().unwrap().as_str(), "2.0.0"); + assert_eq!( + fs::read_to_string(skill_path(&layout).join("SKILL.md")).unwrap(), + "skill 2.0.0\n" + ); + assert_eq!(fs::read(skill_entry.join("SKILL.md")).unwrap(), loader); + fs::write(layout.downloads().join("latest"), "1.0.0\n").unwrap(); + assert!( + manager + .update() + .unwrap_err() + .contains("refusing to downgrade") + ); + assert_eq!(manager.active_version().unwrap().unwrap().as_str(), "2.0.0"); + fs::write(layout.downloads().join("latest"), "2.0.0\n").unwrap(); + assert!( + manager + .uninstall(&Version::parse("2.0.0").unwrap()) + .unwrap_err() + .contains("active") + ); + manager.activate(&Version::parse("1.0.0").unwrap()).unwrap(); + assert_eq!( + fs::read_to_string(skill_path(&layout).join("SKILL.md")).unwrap(), + "skill 1.0.0\n" + ); + manager + .uninstall(&Version::parse("2.0.0").unwrap()) + .unwrap(); + assert!(!layout.versions().join("2.0.0").exists()); + + let before = manager.active_version().unwrap(); + assert!( + manager + .install_requested("3.0.0") + .unwrap_err() + .contains("health check") + ); + assert_eq!(manager.active_version().unwrap(), before); + assert!(!layout.versions().join("3.0.0").exists()); + assert!( + manager + .install_requested("4.0.0") + .unwrap_err() + .contains("signature") + ); + assert_eq!(manager.active_version().unwrap(), before); + write_release(&layout, &key, "5.0.0", &dispatch_log, true, true); + fs::write( + layout.downloads().join("5.0.0/runtime.pack"), + "tampered runtime", + ) + .unwrap(); + assert!( + manager + .install_requested("5.0.0") + .unwrap_err() + .contains("integrity") + ); + assert_eq!(fs::read_dir(layout.staging()).unwrap().count(), 0); + + let install_lock = fs::OpenOptions::new() + .read(true) + .write(true) + .open(layout.state().join("install.lock")) + .unwrap(); + fs2::FileExt::lock_exclusive(&install_lock).unwrap(); + assert!( + manager + .install_requested("1.0.0") + .unwrap_err() + .contains("active") + ); + fs2::FileExt::unlock(&install_lock).unwrap(); + drop(install_lock); + + fs::set_permissions(layout.state(), fs::Permissions::from_mode(0o755)).unwrap(); + fs::write( + project.join("config/setting.json"), + r#"{"dever":{"version":"9.9.9"}}"#, + ) + .unwrap(); + let error = match execute_in(&layout, &arguments) { + Ok(_) => panic!("missing locked version must fail"), + Err(error) => error, + }; + assert!(error.contains("dever install 9.9.9"), "{error}"); + + fs::set_permissions( + layout.versions().join("1.0.0/dever-core"), + fs::Permissions::from_mode(0o777), + ) + .unwrap(); + let error = manager + .resolve_core(&Version::parse("1.0.0").unwrap()) + .unwrap_err(); + assert!( + error.contains("not writable by group or other users"), + "{error}" + ); +} + +#[cfg(unix)] +#[test] +fn installed_launcher_is_one_shared_entry_for_multiple_projects() { + let temporary = TemporaryDirectory::new(); + let layout = Layout::new(temporary.path().join("machine")); + layout.initialize().unwrap(); + let key = signing_key(); + fs::write( + layout.state().join("trusted-release-key"), + format!("{}\n", hex(key.public_key().as_ref())), + ) + .unwrap(); + let dispatch_log = temporary.path().join("dispatch.log"); + write_release(&layout, &key, "1.0.0", &dispatch_log, true, true); + MachineManager::new(layout.clone()) + .install_requested("1.0.0") + .unwrap(); + + let launcher = layout.bin().join("dever"); + fs::copy(env!("CARGO_BIN_EXE_dever-launcher"), &launcher).unwrap(); + let version = Command::new(&launcher) + .env_clear() + .arg("version") + .output() + .unwrap(); + assert!( + version.status.success(), + "{}", + String::from_utf8_lossy(&version.stderr) + ); + assert_eq!(version.stdout, b"1.0.0\n"); + + for name in ["first-project", "second-project"] { + let project = temporary.path().join(name); + fs::create_dir_all(project.join("config")).unwrap(); + fs::write(project.join("config/setting.json"), "{}\n").unwrap(); + for prefix in [vec!["check"], vec!["lib", "list"], vec!["package", "list"]] { + let output = Command::new(&launcher) + .env_clear() + .env("DEVER_TEST_SENTINEL", "must-not-reach-version-core") + .args(&prefix) + .arg(&project) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + fs::read_to_string(&dispatch_log).unwrap(), + format!("{}\n{}\n", prefix.join("\n"), project.display()) + ); + } + } + let project = temporary.path().join("exit-42"); + fs::create_dir_all(project.join("config")).unwrap(); + fs::write(project.join("config/setting.json"), "{}\n").unwrap(); + let status = Command::new(&launcher) + .env_clear() + .args(["check", project.to_str().unwrap()]) + .status() + .unwrap(); + assert_eq!(status.code(), Some(42)); +} + +#[test] +fn shared_cache_uses_complete_crypto_identity_integrity_leases_and_safe_cleanup() { + let temporary = TemporaryDirectory::new(); + let layout = Layout::new(temporary.path().join("machine")); + layout.initialize().unwrap(); + let version = Version::parse("1.0.0").unwrap(); + let program = temporary.path().join("program"); + fs::write(&program, b"verified executable bytes").unwrap(); + let build = identity(&version, b"generated one", "linux-x86_64"); + let service = CacheStore::new(&layout); + + let keys = std::thread::scope(|scope| { + let first = scope.spawn(|| CacheStore::new(&layout).publish(&build, &program).unwrap()); + let second = scope.spawn(|| CacheStore::new(&layout).publish(&build, &program).unwrap()); + [first.join().unwrap(), second.join().unwrap()] + }); + assert_eq!(keys[0], keys[1]); + let restored = service.restore(&build).unwrap().expect("cache hit"); + assert_eq!(restored.as_bytes(), b"verified executable bytes"); + assert!( + service + .restore(&identity(&version, b"generated two", "linux-x86_64")) + .unwrap() + .is_none() + ); + for target in ["linux-aarch64", "windows-x86_64"] { + assert!( + service + .restore(&identity(&version, b"generated one", target)) + .unwrap() + .is_none() + ); + } + + let status = service.status().unwrap(); + assert_eq!((status.entries, status.corrupt_entries), (1, 0)); + assert_eq!(status.versions.get("1.0.0"), Some(&1)); + service.clean(&BTreeSet::new()).unwrap(); + assert_eq!( + service.status().unwrap().entries, + 1, + "leased entry was deleted" + ); + drop(restored); + fs::create_dir_all(layout.native_cache().join("operations")).unwrap(); + fs::write( + layout.native_cache().join("operations/active"), + "fixture operation", + ) + .unwrap(); + assert!( + service + .clean(&BTreeSet::new()) + .unwrap_err() + .contains("busy") + ); + fs::remove_file(layout.native_cache().join("operations/active")).unwrap(); + service.clean(&BTreeSet::new()).unwrap(); + assert_eq!(service.status().unwrap().entries, 0); + + let key = service.publish(&build, &program).unwrap(); + fs::write( + layout + .native_cache() + .join("entries") + .join(&key) + .join("program"), + b"tampered", + ) + .unwrap(); + let error = match service.restore(&build) { + Ok(_) => panic!("modified cache entry must fail"), + Err(error) => error, + }; + assert!(error.contains("modified"), "{error}"); + assert_eq!(service.status().unwrap().corrupt_entries, 1); + let installed = BTreeSet::from([version]); + service.clean(&installed).unwrap(); + assert_eq!(service.status().unwrap().corrupt_entries, 0); +} + +#[test] +fn launcher_does_not_bypass_an_unavailable_authenticated_cache_service() { + let temporary = TemporaryDirectory::new(); + let layout = Layout::new(temporary.path().join("machine")); + layout.initialize().unwrap(); + for arguments in [["cache", "status"], ["cache", "clean"]] { + let arguments = arguments.map(OsString::from); + let error = match execute_in(&layout, &arguments) { + Ok(_) => panic!("cache command must not open the service store directly"), + Err(error) => error, + }; + assert!(error.contains("authenticated deverd"), "{error}"); + } +} + +#[cfg(unix)] +#[test] +fn deverd_authenticates_cache_commands_over_local_ipc() { + use std::os::unix::fs::PermissionsExt; + + let temporary = TemporaryDirectory::new(); + let layout = Layout::new(temporary.path().join("machine")); + layout.initialize().unwrap(); + let _daemon = TestDaemon::start(&layout); + let status = execute_in( + &layout, + &[OsString::from("cache"), OsString::from("status")], + ) + .unwrap(); + let CommandResult::Message(status) = status else { + panic!("cache status must return a message"); + }; + let status: serde_json::Value = serde_json::from_str(&status).unwrap(); + assert_eq!(status["entries"], 0); + let version = Version::parse("1.0.0").unwrap(); + let artifact = temporary.path().join("program"); + fs::write(&artifact, b"authenticated cleanup fixture").unwrap(); + CacheStore::new(&layout) + .publish(&identity(&version, b"auth", "linux-x86_64"), &artifact) + .unwrap(); + let token = fs::read_to_string(layout.service_token()).unwrap(); + let padded = format!("{}{}", token.trim(), "\0".repeat(256)); + assert_eq!(raw_request(&layout, &padded, "clean")["ok"], false); + assert_eq!( + dever_cli::toolchain::cache_status(&layout).unwrap().entries, + 1 + ); + assert_eq!(raw_request(&layout, token.trim(), "status")["ok"], true); + let moved = layout.state().join("deverd.token.saved"); + fs::rename(layout.service_token(), &moved).unwrap(); + assert_eq!(raw_request(&layout, "", "clean")["ok"], false); + assert!(dever_cli::toolchain::cache_clean(&layout, &BTreeSet::new()).is_err()); + assert!(!layout.service_token().exists()); + fs::rename(moved, layout.service_token()).unwrap(); + fs::set_permissions(layout.service_token(), fs::Permissions::from_mode(0o644)).unwrap(); + assert_eq!(raw_request(&layout, token.trim(), "clean")["ok"], false); + fs::set_permissions(layout.service_token(), fs::Permissions::from_mode(0o600)).unwrap(); + assert_eq!( + dever_cli::toolchain::cache_status(&layout).unwrap().entries, + 1 + ); + fs::write(layout.service_token(), format!("{token}tampered")).unwrap(); + let error = execute_in( + &layout, + &[OsString::from("cache"), OsString::from("status")], + ) + .unwrap_err(); + assert!( + error.contains("authentication token") || error.contains("invalid"), + "{error}" + ); +} + +#[cfg(unix)] +#[test] +fn deverd_rejects_a_second_instance_and_recovers_its_socket_after_exit() { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + let temporary = TemporaryDirectory::new(); + let layout = Layout::new(temporary.path().join("machine")); + layout.initialize().unwrap(); + let mut daemon = TestDaemon::start(&layout); + let socket_inode = fs::symlink_metadata(layout.service_socket()).unwrap().ino(); + let second = Command::new(env!("CARGO_BIN_EXE_deverd")) + .args(["--root", layout.root().to_str().unwrap()]) + .output() + .unwrap(); + assert!(!second.status.success()); + assert!(String::from_utf8_lossy(&second.stderr).contains("already running")); + assert_eq!( + fs::symlink_metadata(layout.service_socket()).unwrap().ino(), + socket_inode + ); + assert!(dever_cli::toolchain::cache_status(&layout).is_ok()); + + daemon.stop(); + assert!(layout.service_socket().exists()); + let lock = layout.state().join("deverd.lock"); + fs::set_permissions(&lock, fs::Permissions::from_mode(0o644)).unwrap(); + let insecure = Command::new(env!("CARGO_BIN_EXE_deverd")) + .args(["--root", layout.root().to_str().unwrap()]) + .output() + .unwrap(); + assert!(!insecure.status.success()); + assert!(String::from_utf8_lossy(&insecure.stderr).contains("deverd lock")); + fs::set_permissions(&lock, fs::Permissions::from_mode(0o600)).unwrap(); + let _recovered = TestDaemon::start(&layout); + assert!(dever_cli::toolchain::cache_status(&layout).is_ok()); +} + +#[cfg(target_os = "linux")] +#[test] +fn deverd_bounds_workers_when_clients_stall() { + use std::os::unix::net::UnixStream; + + let temporary = TemporaryDirectory::new(); + let layout = Layout::new(temporary.path().join("machine")); + layout.initialize().unwrap(); + let daemon = TestDaemon::start(&layout); + let clients: Vec<_> = (0..48) + .map(|_| { + let mut stream = UnixStream::connect(layout.service_socket()).unwrap(); + let _ = stream.write_all(&[0]); + stream + }) + .collect(); + let status = fs::read_to_string(format!("/proc/{}/status", daemon.0.id())).unwrap(); + let threads: usize = status + .lines() + .find_map(|line| { + line.strip_prefix("Threads:") + .map(|value| value.trim().parse().unwrap()) + }) + .unwrap(); + assert!( + threads <= 10, + "deverd created {threads} threads for stalled clients" + ); + drop(clients); + let recovered = (0..100).any(|_| { + if dever_cli::toolchain::cache_status(&layout).is_ok() { + return true; + } + std::thread::sleep(std::time::Duration::from_millis(10)); + false + }); + assert!(recovered); +} + +#[cfg(unix)] +#[test] +fn deverd_keeps_serving_after_bad_and_slow_clients() { + use std::net::Shutdown; + use std::os::unix::net::UnixStream; + use std::time::{Duration, Instant}; + + let temporary = TemporaryDirectory::new(); + let layout = Layout::new(temporary.path().join("machine")); + layout.initialize().unwrap(); + let _daemon = TestDaemon::start(&layout); + + let mut slow = UnixStream::connect(layout.service_socket()).unwrap(); + slow.write_all(&[0]).unwrap(); + let start = Instant::now(); + assert!(dever_cli::toolchain::cache_status(&layout).is_ok()); + assert!(start.elapsed() < Duration::from_secs(2)); + + let mut oversized = UnixStream::connect(layout.service_socket()).unwrap(); + oversized + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + oversized.write_all(&u32::MAX.to_be_bytes()).unwrap(); + assert_eq!(read_response(&mut oversized)["ok"], false); + + let mut partial = UnixStream::connect(layout.service_socket()).unwrap(); + partial + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + partial.write_all(&100_u32.to_be_bytes()).unwrap(); + partial.shutdown(Shutdown::Write).unwrap(); + assert_eq!(read_response(&mut partial)["ok"], false); + + let mut disconnected = UnixStream::connect(layout.service_socket()).unwrap(); + disconnected.write_all(&0_u32.to_be_bytes()).unwrap(); + disconnected.shutdown(Shutdown::Both).unwrap(); + drop(slow); + assert!(dever_cli::toolchain::cache_status(&layout).is_ok()); +} + +#[cfg(unix)] +#[test] +fn deverd_transfers_verified_artifacts_without_client_cache_paths() { + let temporary = TemporaryDirectory::new(); + let layout = Layout::new(temporary.path().join("machine")); + layout.initialize().unwrap(); + let _daemon = TestDaemon::start(&layout); + // Larger than a transport scratch buffer; the daemon streams to staging. + let bytes = (0..262_147) + .map(|index| (index % 251) as u8) + .collect::<Vec<_>>(); + let receipt = artifact_put(&layout, &bytes).unwrap(); + assert_eq!(receipt.bytes, bytes.len() as u64); + assert_eq!( + artifact_get(&layout, &receipt.sha256, receipt.bytes).unwrap(), + bytes + ); + std::thread::scope(|scope| { + for _ in 0..6 { + scope.spawn(|| assert_eq!(artifact_put(&layout, &bytes).unwrap(), receipt)); + } + }); + let status = dever_cli::toolchain::cache_status(&layout).unwrap(); + assert_eq!(status.artifact_entries, 1); + assert_eq!(status.artifact_bytes, receipt.bytes); + assert_eq!(status.staging_entries, 0); + assert!(artifact_get(&layout, "../outside", 1).is_err()); + assert!(artifact_get(&layout, &receipt.sha256, receipt.bytes + 1).is_err()); + assert!(artifact_get(&layout, &receipt.sha256, 64 * 1024 * 1024 + 1).is_err()); + fs::write( + layout + .native_cache() + .join("artifacts") + .join(&receipt.sha256), + b"tampered", + ) + .unwrap(); + assert!(artifact_get(&layout, &receipt.sha256, receipt.bytes).is_err()); + assert_eq!( + dever_cli::toolchain::cache_status(&layout) + .unwrap() + .corrupt_artifacts, + 1 + ); + dever_cli::toolchain::cache_clean(&layout, &BTreeSet::new()).unwrap(); + assert!(artifact_get(&layout, &receipt.sha256, receipt.bytes).is_err()); +} + +#[cfg(unix)] +#[test] +fn deverd_rejects_bad_and_incomplete_artifact_uploads_before_publication() { + use std::os::unix::net::UnixStream; + + let temporary = TemporaryDirectory::new(); + let layout = Layout::new(temporary.path().join("machine")); + layout.initialize().unwrap(); + let _daemon = TestDaemon::start(&layout); + let token = fs::read_to_string(layout.service_token()).unwrap(); + for (bytes, digest, body, complete) in [ + ( + 4_u64, + hex(&Sha256::digest(b"good")), + b"bad!".as_slice(), + true, + ), + (4, hex(&Sha256::digest(b"good")), b"go".as_slice(), false), + ] { + let mut stream = UnixStream::connect(layout.service_socket()).unwrap(); + stream + .set_read_timeout(Some(std::time::Duration::from_secs(2))) + .unwrap(); + let request = serde_json::to_vec(&serde_json::json!({ + "token": token.trim(), "operation": {"artifact_put": {"sha256": digest, "bytes": bytes}}, "installed": [] + })).unwrap(); + stream + .write_all(&(request.len() as u32).to_be_bytes()) + .unwrap(); + stream.write_all(&request).unwrap(); + assert_eq!(read_response(&mut stream)["ok"], true); + stream.write_all(body).unwrap(); + if complete { + assert_eq!(read_response(&mut stream)["ok"], false); + } + drop(stream); + for attempt in 0..100 { + let status = dever_cli::toolchain::cache_status(&layout).unwrap(); + if status.staging_entries == 0 { + assert_eq!(status.artifact_entries, 0); + break; + } + assert!(attempt < 99, "incomplete upload staging was not removed"); + std::thread::sleep(std::time::Duration::from_millis(10)); + } + } + let receipt = artifact_put(&layout, b"good").unwrap(); + assert_eq!( + artifact_get(&layout, &receipt.sha256, receipt.bytes).unwrap(), + b"good" + ); +} + +#[cfg(unix)] +#[test] +fn shared_artifact_quota_counts_bytes_and_entries_and_cleans_failed_staging() { + for (entries, artifact_bytes) in [(32_u32, 64 * 1024 * 1024), (4096, 0)] { + let temporary = TemporaryDirectory::new(); + let layout = Layout::new(temporary.path().join("machine")); + layout.initialize().unwrap(); + let artifacts = layout.native_cache().join("artifacts"); + fs::create_dir(&artifacts).unwrap(); + // Sparse owned fixtures test byte and entry quotas without allocating 2 GiB. + for index in 0_u32..entries { + let file = fs::File::create(artifacts.join(hex(&Sha256::digest(index.to_le_bytes())))) + .unwrap(); + file.set_len(artifact_bytes).unwrap(); + } + let body = b"new verified artifact"; + let receipt = dever_cli::toolchain::ArtifactReceipt { + sha256: hex(&Sha256::digest(body)), + bytes: body.len() as u64, + }; + let store = CacheStore::new(&layout); + assert!( + store + .publish_artifact(&receipt, &mut body.as_slice()) + .unwrap_err() + .contains("cache is full") + ); + assert!(!artifacts.join(&receipt.sha256).exists()); + assert_eq!( + fs::read_dir(layout.native_cache().join("staging")) + .unwrap() + .count(), + 0 + ); + assert_eq!( + fs::read_dir(layout.native_cache().join("operations")) + .unwrap() + .count(), + 0 + ); + } +} + +#[cfg(unix)] +#[test] +fn shared_cache_summary_cannot_claim_payload_integrity() { + let temporary = TemporaryDirectory::new(); + let layout = Layout::new(temporary.path().join("machine")); + layout.initialize().unwrap(); + let store = CacheStore::new(&layout); + let bytes = b"good"; + let receipt = dever_cli::toolchain::ArtifactReceipt { + sha256: hex(&Sha256::digest(bytes)), + bytes: bytes.len() as u64, + }; + store + .publish_artifact(&receipt, &mut bytes.as_slice()) + .unwrap(); + fs::write( + layout + .native_cache() + .join("artifacts") + .join(&receipt.sha256), + b"bad!", + ) + .unwrap(); + let summary = store.summary().unwrap(); + assert!(!summary.verified); + assert_eq!(summary.artifact_entries, 1); + assert_eq!(summary.corrupt_artifacts, 0); + let checked = store.status().unwrap(); + assert!(checked.verified); + assert_eq!(checked.artifact_entries, 0); + assert_eq!(checked.corrupt_artifacts, 1); + assert!(store.artifact(&receipt).is_err()); +} + +#[test] +fn compiled_cache_has_independent_byte_entry_and_output_limits() { + for (count, bytes) in [(32, 64 * 1024 * 1024), (4096, 1)] { + let temporary = TemporaryDirectory::new(); + let layout = Layout::new(temporary.path().join("machine")); + layout.initialize().unwrap(); + let version = Version::parse("0.1.0").unwrap(); + for index in 0..count { + let key = format!("{index:064x}"); + let directory = layout.native_cache().join("entries").join(&key); + fs::create_dir_all(&directory).unwrap(); + fs::File::create(directory.join("program")) + .unwrap() + .set_len(bytes) + .unwrap(); + let entry = dever_cli::toolchain::CacheEntry { + format: "dever-machine-cache-entry-v1".into(), + key, + version: version.clone(), + target: "linux-x86_64".into(), + bytes, + sha256: "0".repeat(64), + }; + fs::write( + directory.join("entry.json"), + serde_json::to_vec(&entry).unwrap(), + ) + .unwrap(); + } + let program = temporary.path().join("program"); + fs::write(&program, b"new compiled output").unwrap(); + let store = CacheStore::new(&layout); + assert!( + store + .publish(&identity(&version, b"new", "linux-x86_64"), &program) + .unwrap_err() + .contains("quota exceeded") + ); + fs::File::create(&program) + .unwrap() + .set_len(320 * 1024 * 1024 + 1) + .unwrap(); + assert!( + store + .publish(&identity(&version, b"large", "linux-x86_64"), &program) + .unwrap_err() + .contains("320 MiB") + ); + assert_eq!( + fs::read_dir(layout.native_cache().join("staging")) + .unwrap() + .count(), + 0 + ); + assert_eq!( + fs::read_dir(layout.native_cache().join("operations")) + .unwrap() + .count(), + 0 + ); + } +} + +#[cfg(target_os = "linux")] +#[test] +#[ignore = "requires root; drops only an owned skill client to an unprivileged uid"] +fn active_skill_is_readable_by_other_machine_users_without_install_lock_access() { + use std::os::unix::fs::PermissionsExt; + use std::os::unix::process::CommandExt; + + assert_eq!(rustix::process::geteuid().as_raw(), 0); + let temporary = TemporaryDirectory::new(); + let layout = Layout::new(temporary.path().join("machine")); + layout.initialize().unwrap(); + let key = signing_key(); + fs::write( + layout.state().join("trusted-release-key"), + format!("{}\n", hex(key.public_key().as_ref())), + ) + .unwrap(); + write_release( + &layout, + &key, + "1.0.0", + &temporary.path().join("dispatch"), + true, + true, + ); + MachineManager::new(layout.clone()) + .install_requested("1.0.0") + .unwrap(); + let launcher = layout.bin().join("dever"); + fs::copy(env!("CARGO_BIN_EXE_dever-launcher"), &launcher).unwrap(); + fs::set_permissions(&launcher, fs::Permissions::from_mode(0o755)).unwrap(); + let output = Command::new(&launcher) + .args(["skill", "path"]) + .uid(65534) + .gid(65534) + .env_clear() + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let path = String::from_utf8(output.stdout).unwrap(); + assert_eq!( + Path::new(path.trim()), + layout.versions().join("1.0.0/skills/dever-language") + ); + let read = Command::new("/bin/cat") + .arg(Path::new(path.trim()).join("SKILL.md")) + .uid(65534) + .gid(65534) + .env_clear() + .output() + .unwrap(); + assert!(read.status.success()); + assert_eq!(read.stdout, b"skill 1.0.0\n"); +} + +#[cfg(unix)] +#[test] +fn corrupt_skill_cannot_be_published_or_resolved() { + let temporary = TemporaryDirectory::new(); + let layout = Layout::new(temporary.path().join("machine")); + layout.initialize().unwrap(); + let key = signing_key(); + fs::write( + layout.state().join("trusted-release-key"), + format!("{}\n", hex(key.public_key().as_ref())), + ) + .unwrap(); + for version in ["1.0.0", "2.0.0"] { + write_release( + &layout, + &key, + version, + &temporary.path().join("dispatch"), + true, + true, + ); + } + let manager = MachineManager::new(layout.clone()); + manager.install_requested("1.0.0").unwrap(); + fs::write( + layout + .downloads() + .join("2.0.0/skills/dever-language/SKILL.md"), + "tampered", + ) + .unwrap(); + fs::write(layout.downloads().join("latest"), "2.0.0\n").unwrap(); + assert!(manager.update().unwrap_err().contains("integrity")); + assert_eq!(manager.active_version().unwrap().unwrap().as_str(), "1.0.0"); + assert!(!layout.versions().join("2.0.0").exists()); + fs::write( + layout + .versions() + .join("1.0.0/skills/dever-language/SKILL.md"), + "tampered", + ) + .unwrap(); + assert!( + execute_in(&layout, &["skill".into(), "path".into()]) + .unwrap_err() + .contains("integrity") + ); +} + +#[cfg(target_os = "linux")] +#[test] +#[ignore = "requires root; drops only owned test clients to an unprivileged uid"] +fn deverd_uses_kernel_credentials_for_other_machine_users() { + use std::os::unix::fs::PermissionsExt; + use std::os::unix::process::CommandExt; + + assert_eq!( + rustix::process::geteuid().as_raw(), + 0, + "this scoped acceptance requires root" + ); + let temporary = TemporaryDirectory::new(); + let layout = Layout::new(temporary.path().join("machine")); + layout.initialize().unwrap(); + let _daemon = TestDaemon::start(&layout); + let launcher = layout.bin().join("dever"); + fs::copy(env!("CARGO_BIN_EXE_dever-launcher"), &launcher).unwrap(); + fs::set_permissions(&launcher, fs::Permissions::from_mode(0o755)).unwrap(); + let status = Command::new(&launcher) + .args(["cache", "status"]) + .uid(65534) + .gid(65534) + .env_clear() + .output() + .unwrap(); + assert!( + status.status.success(), + "{}", + String::from_utf8_lossy(&status.stderr) + ); + assert_eq!( + serde_json::from_slice::<serde_json::Value>(&status.stdout).unwrap()["verified"], + false + ); + let clean = Command::new(&launcher) + .args(["cache", "clean"]) + .uid(65534) + .gid(65534) + .env_clear() + .output() + .unwrap(); + assert!(!clean.status.success()); + assert!(String::from_utf8_lossy(&clean.stderr).contains("service owner")); + let client = r#" +import hashlib, json, socket, struct, sys +endpoint, token_path = sys.argv[1:] +try: + open(token_path, 'rb') +except PermissionError: + pass +else: + raise AssertionError('foreign user read the owner token') +def response(peer): + def read(n): + out = b'' + while len(out) < n: + part = peer.recv(n - len(out)) + if not part: raise AssertionError('incomplete response') + out += part + return out + length, = struct.unpack('>I', read(4)) + return json.loads(read(length)) +held = [] +try: + for _ in range(2): + peer = socket.socket(socket.AF_UNIX) + peer.settimeout(2) + peer.connect(endpoint) + held.append(peer) + with socket.socket(socket.AF_UNIX) as excess: + excess.settimeout(2) + excess.connect(endpoint) + assert excess.recv(1) == b'', 'foreign UID exceeded its two-client admission' + request = json.dumps(dict(operation='status')).encode() + for peer in held: + peer.sendall(struct.pack('>I', len(request)) + request) + assert response(peer)['status']['verified'] is False +finally: + for peer in held: + peer.close() +body = b'shared across machine users' +identity = dict(sha256=hashlib.sha256(body).hexdigest(), bytes=len(body)) +for operation in ('artifact_put', 'artifact_get'): + with socket.socket(socket.AF_UNIX) as peer: + peer.settimeout(5) + peer.connect(endpoint) + request = json.dumps(dict(operation={operation: identity})).encode() + peer.sendall(struct.pack('>I', len(request)) + request) + assert response(peer)['artifact'] == identity + if operation == 'artifact_put': + peer.sendall(body) + assert response(peer)['artifact'] == identity + else: + read = b'' + while len(read) < len(body): + part = peer.recv(len(body) - len(read)) + assert part + read += part + assert read == body +"#; + let result = Command::new("/usr/bin/python3") + .args(["-I", "-c", client]) + .arg(layout.service_socket()) + .arg(layout.service_token()) + .uid(65534) + .gid(65534) + .env_clear() + .output() + .unwrap(); + assert!( + result.status.success(), + "{}", + String::from_utf8_lossy(&result.stderr) + ); + let status = dever_cli::toolchain::cache_status(&layout).unwrap(); + assert_eq!(status.artifact_entries, 1); +} + +fn identity<'a>(version: &'a Version, generated: &'a [u8], target: &'a str) -> BuildIdentity<'a> { + BuildIdentity { + version, + compiler: b"compiler identity", + runtime: b"runtime identity", + target, + generated, + options: b"release options", + environment: b"backend environment", + } +} + +fn signing_key() -> Ed25519KeyPair { + let document = Ed25519KeyPair::generate_pkcs8(&SystemRandom::new()).unwrap(); + Ed25519KeyPair::from_pkcs8(document.as_ref()).unwrap() +} + +#[cfg(unix)] +fn write_release( + layout: &Layout, + key: &Ed25519KeyPair, + version: &str, + dispatch_log: &Path, + healthy: bool, + valid_signature: bool, +) { + use std::os::unix::fs::PermissionsExt; + + let root = layout.downloads().join(version); + fs::create_dir(&root).unwrap(); + let core = root.join("dever-core"); + let log = dispatch_log.display().to_string().replace('\'', "'\\''"); + fs::write( + &core, + format!( + "#!/bin/sh\nif [ \"${{DEVER_TEST_SENTINEL+x}}\" = x ]; then exit 98; fi\nif [ \"$1\" = \"--dever-health\" ]; then exit {}; fi\nif [ \"${{2##*/}}\" = \"exit-42\" ]; then exit 42; fi\nprintf '%s\\n' \"$@\" > '{}'\n", + if healthy { 0 } else { 9 }, + log, + ), + ).unwrap(); + fs::set_permissions(&core, fs::Permissions::from_mode(0o755)).unwrap(); + let runtime = root.join("runtime.pack"); + fs::write(&runtime, format!("runtime {version}")).unwrap(); + fs::set_permissions(&runtime, fs::Permissions::from_mode(0o666)).unwrap(); + fs::set_permissions(&core, fs::Permissions::from_mode(0o777)).unwrap(); + let skill = root.join("skills/dever-language/SKILL.md"); + fs::create_dir_all(skill.parent().unwrap()).unwrap(); + fs::write(&skill, format!("skill {version}\n")).unwrap(); + let manifest = ReleaseManifest::new( + Version::parse(version).unwrap(), + vec![ + artifact(&core, "dever-core"), + artifact(&runtime, "runtime.pack"), + artifact(&skill, "skills/dever-language/SKILL.md"), + ], + ); + let bytes = serde_json::to_vec(&manifest).unwrap(); + fs::write(root.join("manifest.json"), &bytes).unwrap(); + let mut signature = key.sign(&bytes).as_ref().to_vec(); + if !valid_signature { + signature[0] ^= 1; + } + fs::write(root.join("manifest.sig"), format!("{}\n", hex(&signature))).unwrap(); +} + +fn artifact(path: &Path, relative: &str) -> Artifact { + let bytes = fs::read(path).unwrap(); + Artifact { + path: relative.into(), + bytes: bytes.len() as u64, + sha256: hex(&Sha256::digest(&bytes)), + } +} + +fn hex(bytes: &[u8]) -> String { + const DIGITS: &[u8; 16] = b"0123456789abcdef"; + let mut output = String::with_capacity(bytes.len() * 2); + for byte in bytes { + output.push(DIGITS[usize::from(byte >> 4)] as char); + output.push(DIGITS[usize::from(byte & 0x0f)] as char); + } + output +} diff --git a/test/dever-cli-tests/support/command_library.c b/test/dever-cli-tests/support/command_library.c new file mode 100644 index 0000000..61edc16 --- /dev/null +++ b/test/dever-cli-tests/support/command_library.c @@ -0,0 +1,2 @@ +/* Deliberately separate DT_NEEDED dependency for native command packaging. */ +int command_value(void) { return 42; } diff --git a/test/dever-cli-tests/support/command_tool.c b/test/dever-cli-tests/support/command_tool.c new file mode 100644 index 0000000..b030894 --- /dev/null +++ b/test/dever-cli-tests/support/command_tool.c @@ -0,0 +1,19 @@ +#include <stdio.h> +#include <string.h> + +extern int command_value(void); + +int main(int argc, char **argv) { + const unsigned char expected[] = {0, 255, 128, 10}; + unsigned char input[sizeof(expected)]; + if (argc != 3 || strcmp(argv[1], "a b") != 0 || + strcmp(argv[2], "$(literal);*") != 0 || command_value() != 42) + return 90; + if (fread(input, 1, sizeof(input), stdin) != sizeof(input) || + memcmp(input, expected, sizeof(input)) != 0 || getchar() != EOF) + return 91; + if (fwrite(input, 1, sizeof(input), stdout) != sizeof(input) || + fputs("ordinary command", stderr) == EOF) + return 92; + return 23; +} diff --git a/test/dever-cli-tests/support/compiler_sleeper.c b/test/dever-cli-tests/support/compiler_sleeper.c new file mode 100644 index 0000000..dfd6d62 --- /dev/null +++ b/test/dever-cli-tests/support/compiler_sleeper.c @@ -0,0 +1,18 @@ +/* Native author fixture for compile cancellation and deadline supervision. */ +#include <stdio.h> +#include <errno.h> +#include <string.h> +#include <time.h> +#include <unistd.h> + +int main(int argc, char **argv) { + if (argc == 2 && strcmp(argv[1], "--dever-health") == 0) return 0; + FILE *pid = fopen("worker.pid", "w"); + if (!pid) return 2; + if (fprintf(pid, "%ld", (long)getpid()) < 0 || fclose(pid) != 0) return 3; + struct timespec remaining = {300, 0}; + while (nanosleep(&remaining, &remaining) != 0) { + if (errno != EINTR) return 4; + } + return 0; +} diff --git a/test/dever-cli-tests/support/daemon.rs b/test/dever-cli-tests/support/daemon.rs new file mode 100644 index 0000000..078abfa --- /dev/null +++ b/test/dever-cli-tests/support/daemon.rs @@ -0,0 +1,42 @@ +//! Lifetime of an explicitly requested test-owned shared compiler service. +use std::process::{Child, Command}; +use std::time::Duration; + +use dever_cli::toolchain::{Layout, cache_status}; + +pub struct TestDaemon(Child); + +impl TestDaemon { + pub fn start(layout: &Layout) -> Self { + let child = Command::new(env!("CARGO_BIN_EXE_deverd")) + .env_clear() + .current_dir(layout.root()) + .args(["--root", "."]) + .spawn() + .unwrap(); + let mut daemon = Self(child); + for _ in 0..100 { + if cache_status(layout).is_ok() { + return daemon; + } + assert!( + daemon.0.try_wait().unwrap().is_none(), + "test deverd exited early" + ); + std::thread::sleep(Duration::from_millis(10)); + } + panic!("test deverd did not become ready"); + } + + pub fn stop(&mut self) { + self.0.kill().unwrap(); + self.0.wait().unwrap(); + } +} + +impl Drop for TestDaemon { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} diff --git a/test/dever-cli-tests/support/llvm_inputs.rs b/test/dever-cli-tests/support/llvm_inputs.rs new file mode 100644 index 0000000..1d7ceda --- /dev/null +++ b/test/dever-cli-tests/support/llvm_inputs.rs @@ -0,0 +1,67 @@ +//! Explicit Linux author inputs shared by opt-in native fixtures. +use std::path::{Path, PathBuf}; + +pub const LLVM_LIBRARY: &str = "/usr/lib/llvm-18/lib/libLLVM.so.1"; + +pub fn core_libraries() -> Vec<(&'static str, PathBuf)> { + let mut inputs = vec![("libLLVM.so.18.1", PathBuf::from(LLVM_LIBRARY))]; + for name in [ + "libstdc++.so.6", + "libgcc_s.so.1", + "libz.so.1", + "libffi.so.8", + "libedit.so.2", + "libzstd.so.1", + "libtinfo.so.6", + "libxml2.so.2", + "libbsd.so.0", + "libmd.so.0", + "libicuuc.so.74", + "libicudata.so.74", + "liblzma.so.5", + ] { + inputs.push((name, Path::new("/usr/lib/x86_64-linux-gnu").join(name))); + } + inputs + .into_iter() + .map(|(name, path)| { + ( + name, + path.canonicalize() + .expect("prepare the explicit GNU compiler-library inputs"), + ) + }) + .collect() +} + +pub fn native_inputs() -> Vec<(&'static str, PathBuf)> { + let workspace = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + vec![ + ( + "runtime.a", + workspace.join("target/native-runtime-abi/debug/libdever_backend_bridge.a"), + ), + ("crt1.o", "/usr/lib/x86_64-linux-gnu/crt1.o".into()), + ("crti.o", "/usr/lib/x86_64-linux-gnu/crti.o".into()), + ( + "crtbeginT.o", + "/usr/lib/gcc/x86_64-linux-gnu/13/crtbeginT.o".into(), + ), + ( + "crtend.o", + "/usr/lib/gcc/x86_64-linux-gnu/13/crtend.o".into(), + ), + ("crtn.o", "/usr/lib/x86_64-linux-gnu/crtn.o".into()), + ("libc.a", "/usr/lib/x86_64-linux-gnu/libc.a".into()), + ("libm.a", "/usr/lib/x86_64-linux-gnu/libm-2.39.a".into()), + ("libmvec.a", "/usr/lib/x86_64-linux-gnu/libmvec.a".into()), + ( + "libgcc.a", + "/usr/lib/gcc/x86_64-linux-gnu/13/libgcc.a".into(), + ), + ( + "libgcc_eh.a", + "/usr/lib/gcc/x86_64-linux-gnu/13/libgcc_eh.a".into(), + ), + ] +} diff --git a/test/dever-cli-tests/support/llvm_pack.rs b/test/dever-cli-tests/support/llvm_pack.rs new file mode 100644 index 0000000..643a7e5 --- /dev/null +++ b/test/dever-cli-tests/support/llvm_pack.rs @@ -0,0 +1,128 @@ +//! Explicit test inputs: no downloads, recursive builds or host tool lookup. +use std::fs; +use std::path::{Path, PathBuf}; + +use dever_cli::toolchain::sha256_file as hash_file; +use serde_json::{Value, json}; + +#[path = "llvm_inputs.rs"] +mod inputs; +#[cfg(test)] +#[path = "native_objects.rs"] +mod native_objects; +use inputs::{core_libraries, native_inputs}; + +#[cfg(test)] +pub fn compiler(source: &Path, root: &Path, name: &str) -> PathBuf { + fs::create_dir_all(root).unwrap(); + let path = root.join(name); + fs::hard_link(source, &path).unwrap(); + compiler_library(root).unwrap(); + path +} + +pub fn pack_root(root: &Path) -> PathBuf { + root.join("runtime") + .join(dever_cli::toolchain::platform_identity()) +} + +#[cfg(test)] +pub fn manifest(root: &Path, real: bool) -> Value { + let destination = pack_root(root); + fs::create_dir_all(&destination).unwrap(); + let sources = native_inputs(); + let mut files = Vec::new(); + for (name, source) in sources { + let output = destination.join(name); + if real { + assert!( + source.is_file(), + "prepare the explicit native archive and CRT fixture first: {}", + source.display() + ); + // Only immutable reads follow. Corruption cases use synthetic files. + fs::hard_link(source, &output).unwrap(); + } else { + fs::write( + &output, + if name.ends_with(".a") { + native_objects::archive( + dever_cli::toolchain::BuildTarget::host().unwrap(), + "fixture.o", + ) + } else { + native_objects::object(dever_cli::toolchain::BuildTarget::host().unwrap()) + }, + ) + .unwrap(); + } + files.push(json!({"path":name, "bytes":fs::metadata(&output).unwrap().len(), "sha256":digest(&output)})); + } + let document = document(files); + write_manifest(root, &document); + document +} + +fn document(files: Vec<Value>) -> Value { + let profile = json!({ + "start":["crt1.o","crti.o","crtbeginT.o"], "runtime":"runtime.a", + "libraries":["libc.a","libm.a","libmvec.a","libgcc.a","libgcc_eh.a"], + "end":["crtend.o","crtn.o"] + }); + json!({ + "format":"dever-native-runtime-v1", "compiler_version":env!("CARGO_PKG_VERSION"), + "abi":dever_backend_bridge::RUNTIME_ABI_VERSION, + "platform":dever_cli::toolchain::platform_identity(), + "target":"x86_64-unknown-linux-gnu", + "profiles":{"base":profile,"sqlite":profile,"postgres":profile,"both":profile}, + "files":files + }) +} + +fn compiler_library(root: &Path) -> Result<(), String> { + let libraries = root.join("lib"); + create_or_check_directory(&libraries)?; + for (name, source) in core_libraries() { + let target = libraries.join(name); + match fs::symlink_metadata(&target) { + Ok(metadata) if metadata.is_file() && !metadata.file_type().is_symlink() => { + if hash_file(&target)? != hash_file(&source)? { + return Err( + "existing compiler library differs from the explicit author SDK".into(), + ); + } + } + Ok(_) => return Err("compiler library must be a real file".into()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + fs::hard_link(source, target).map_err(|error| error.to_string())?; + } + Err(error) => return Err(error.to_string()), + } + } + Ok(()) +} + +fn create_or_check_directory(path: &Path) -> Result<(), String> { + match fs::symlink_metadata(path) { + Ok(metadata) if metadata.is_dir() && !metadata.file_type().is_symlink() => Ok(()), + Ok(_) => Err(format!("'{}' must be a real directory", path.display())), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + fs::create_dir(path).map_err(|error| error.to_string()) + } + Err(error) => Err(error.to_string()), + } +} + +#[cfg(test)] +pub fn write_manifest(root: &Path, manifest: &Value) { + fs::write( + pack_root(root).join("manifest.json"), + serde_json::to_vec(manifest).unwrap(), + ) + .unwrap(); +} + +#[cfg(test)] +pub fn digest(path: &Path) -> String { + hash_file(path).unwrap() +} diff --git a/test/dever-cli-tests/support/native_acceptance_init.rs b/test/dever-cli-tests/support/native_acceptance_init.rs new file mode 100644 index 0000000..251ca70 --- /dev/null +++ b/test/dever-cli-tests/support/native_acceptance_init.rs @@ -0,0 +1,41 @@ +//! Test-only static entry for the already pivoted no-host acceptance namespace. +#[cfg(target_os = "linux")] +fn main() -> Result<(), Box<dyn std::error::Error>> { + use rustix::mount::{MountFlags, mount}; + use rustix::thread::{self, CapabilitySet}; + use std::os::unix::process::CommandExt; + + let mut arguments = std::env::args_os().skip(1); + let executable = arguments.next().ok_or("missing acceptance executable")?; + if !std::path::Path::new(&executable).is_absolute() { + return Err("acceptance executable must be absolute".into()); + } + // The root-only harness keeps bwrap's original namespace capabilities. + // Reject ordinary host execution before touching any mount. + if !thread::no_new_privs()? + || !thread::capabilities(None)? + .effective + .contains(CapabilitySet::SYS_ADMIN) + { + return Err("acceptance init requires its restricted fixture namespace".into()); + } + // bwrap's readonly proc submounts otherwise prevent a nested user + // namespace from mounting its own proc. Replace only this private proc. + mount( + "proc", + "/proc", + "proc", + MountFlags::NOSUID | MountFlags::NODEV | MountFlags::NOEXEC, + None, + )?; + Err(std::process::Command::new(executable) + .args(arguments) + .env_clear() + .exec() + .into()) +} + +#[cfg(not(target_os = "linux"))] +fn main() -> Result<(), &'static str> { + Err("native acceptance isolation requires Linux") +} diff --git a/test/dever-cli-tests/support/native_elf.rs b/test/dever-cli-tests/support/native_elf.rs new file mode 100644 index 0000000..7d05583 --- /dev/null +++ b/test/dever-cli-tests/support/native_elf.rs @@ -0,0 +1,90 @@ +//! Small ELF metadata fixtures; execution acceptance uses real author inputs. +pub fn compiler(needed: &[&str]) -> Vec<u8> { + elf(None, needed, Some("$ORIGIN/lib")) +} + +pub fn library(name: &str, needed: &[&str]) -> Vec<u8> { + elf(Some(name), needed, None) +} + +pub fn static_executable() -> Vec<u8> { + let mut bytes = elf(None, &[], None); + // Change the synthetic PT_INTERP entry to PT_NULL. Execution tests use the + // real static author input; this fixture tests only ELF admission rules. + bytes[64 + 2 * 56..68 + 2 * 56].copy_from_slice(&0u32.to_le_bytes()); + bytes +} + +pub fn elf(soname: Option<&str>, needed: &[&str], rpath: Option<&str>) -> Vec<u8> { + let mut strings = vec![0u8]; + let mut string = |text: &str| { + let offset = strings.len() as u64; + strings.extend_from_slice(text.as_bytes()); + strings.push(0); + offset + }; + let mut dynamic = needed + .iter() + .map(|name| (1u64, string(name))) + .collect::<Vec<_>>(); + if let Some(name) = soname { + dynamic.push((14, string(name))); + } + if let Some(path) = rpath { + dynamic.push((15, string(path))); + } + let interpreter = if cfg!(target_arch = "aarch64") { + b"/lib/ld-linux-aarch64.so.1\0".as_slice() + } else { + b"/lib64/ld-linux-x86-64.so.2\0".as_slice() + }; + let header_count: u16 = if soname.is_some() { 2 } else { 3 }; + let string_offset = 64 + usize::from(header_count) * 56; + let interpreter_offset = string_offset + strings.len(); + let dynamic_offset = (interpreter_offset + interpreter.len() + 7) & !7; + dynamic.extend([ + (5, string_offset as u64), + (10, strings.len() as u64), + (0, 0), + ]); + let length = dynamic_offset + dynamic.len() * 16; + let mut bytes = vec![0; length]; + bytes[..7].copy_from_slice(b"\x7fELF\x02\x01\x01"); + bytes[16..18].copy_from_slice(&3u16.to_le_bytes()); + let machine = if cfg!(target_arch = "aarch64") { + 183u16 + } else { + 62u16 + }; + bytes[18..20].copy_from_slice(&machine.to_le_bytes()); + bytes[20..24].copy_from_slice(&1u32.to_le_bytes()); + bytes[32..40].copy_from_slice(&64u64.to_le_bytes()); + bytes[52..54].copy_from_slice(&64u16.to_le_bytes()); + bytes[54..56].copy_from_slice(&56u16.to_le_bytes()); + bytes[56..58].copy_from_slice(&header_count.to_le_bytes()); + let mut segment = |index: usize, kind: u32, offset: usize, size: usize| { + let start = 64 + index * 56; + bytes[start..start + 4].copy_from_slice(&kind.to_le_bytes()); + bytes[start + 4..start + 8].copy_from_slice(&4u32.to_le_bytes()); + for at in [8, 16, 24] { + bytes[start + at..start + at + 8].copy_from_slice(&(offset as u64).to_le_bytes()); + } + for at in [32, 40] { + bytes[start + at..start + at + 8].copy_from_slice(&(size as u64).to_le_bytes()); + } + bytes[start + 48..start + 56].copy_from_slice(&1u64.to_le_bytes()); + }; + segment(0, 1, 0, length); + segment(1, 2, dynamic_offset, dynamic.len() * 16); + if soname.is_none() { + segment(2, 3, interpreter_offset, interpreter.len()); + } + bytes[string_offset..interpreter_offset].copy_from_slice(&strings); + bytes[interpreter_offset..interpreter_offset + interpreter.len()].copy_from_slice(interpreter); + for (index, (tag, value)) in dynamic.into_iter().enumerate() { + let start = dynamic_offset + index * 16; + bytes[start..start + 8].copy_from_slice(&tag.to_le_bytes()); + bytes[start + 8..start + 16].copy_from_slice(&value.to_le_bytes()); + } + bytes +} diff --git a/test/dever-cli-tests/support/native_objects.rs b/test/dever-cli-tests/support/native_objects.rs new file mode 100644 index 0000000..a743f70 --- /dev/null +++ b/test/dever-cli-tests/support/native_objects.rs @@ -0,0 +1,29 @@ +//! Small architecture-bearing link inputs for pack validation tests. +use dever_cli::toolchain::BuildTarget; + +pub fn object(target: BuildTarget) -> Vec<u8> { + let mut bytes = vec![0; 64]; + bytes[..7].copy_from_slice(b"\x7fELF\x02\x01\x01"); + bytes[16..18].copy_from_slice(&1_u16.to_le_bytes()); + let machine: u16 = match target { + BuildTarget::LinuxX86_64 => 62, + BuildTarget::LinuxAarch64 => 183, + }; + bytes[18..20].copy_from_slice(&machine.to_le_bytes()); + bytes +} + +pub fn archive(target: BuildTarget, member: &str) -> Vec<u8> { + let object = object(target); + let header = format!( + "{:<16}{:<12}{:<6}{:<6}{:<8}{:<10}`\n", + format!("{member}/"), + 0, + 0, + 0, + "644", + object.len() + ); + assert_eq!(header.len(), 60); + [b"!<arch>\n".as_slice(), header.as_bytes(), &object].concat() +} diff --git a/test/dever-cli-tests/support/wheel.rs b/test/dever-cli-tests/support/wheel.rs new file mode 100644 index 0000000..afff3f9 --- /dev/null +++ b/test/dever-cli-tests/support/wheel.rs @@ -0,0 +1,81 @@ +use std::io::{Cursor, Write}; + +use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; +use sha2::{Digest, Sha256}; + +pub fn metadata(name: &str, version: &str, requires: &[&str], extras: &[&str]) -> String { + let mut text = format!("Metadata-Version: 2.4\nName: {name}\nVersion: {version}\n"); + for requirement in requires { + text.push_str(&format!("Requires-Dist: {requirement}\n")); + } + for extra in extras { + text.push_str(&format!("Provides-Extra: {extra}\n")); + } + text +} + +pub fn pack( + name: &str, + version: &str, + wheel: &str, + metadata: &str, + extra: &[(&str, Vec<u8>)], +) -> Vec<u8> { + let info = format!("{}-{version}.dist-info", name.replace('-', "_")); + let mut files = vec![ + (format!("{info}/WHEEL"), wheel.as_bytes().to_vec()), + (format!("{info}/METADATA"), metadata.as_bytes().to_vec()), + ]; + files.extend( + extra + .iter() + .map(|(path, bytes)| ((*path).to_owned(), bytes.clone())), + ); + let mut record = csv::WriterBuilder::new() + .has_headers(false) + .from_writer(Vec::new()); + for (path, bytes) in &files { + record + .write_record([ + path.clone(), + format!("sha256={}", URL_SAFE_NO_PAD.encode(Sha256::digest(bytes))), + bytes.len().to_string(), + ]) + .unwrap(); + } + let path = format!("{info}/RECORD"); + record + .write_record([path.clone(), String::new(), String::new()]) + .unwrap(); + files.push((path, record.into_inner().unwrap())); + zip(&files) +} + +pub fn zip(files: &[(String, Vec<u8>)]) -> Vec<u8> { + let mut writer = zip::ZipWriter::new(Cursor::new(Vec::new())); + for (path, bytes) in files { + writer + .start_file(path, zip::write::SimpleFileOptions::default()) + .unwrap(); + writer.write_all(bytes).unwrap(); + } + writer.finish().unwrap().into_inner() +} + +pub fn markers() -> pep508_rs::MarkerEnvironment { + pep508_rs::MarkerEnvironmentBuilder { + implementation_name: "cpython", + implementation_version: "3.12.0", + os_name: "posix", + platform_machine: "x86_64", + platform_python_implementation: "CPython", + platform_release: "", + platform_system: "Linux", + platform_version: "", + python_full_version: "3.12.0", + python_version: "3.12", + sys_platform: "linux", + } + .try_into() + .unwrap() +} diff --git a/test/dever-sandbox-tests/apparmor_transition.c b/test/dever-sandbox-tests/apparmor_transition.c new file mode 100644 index 0000000..dbcd9dd --- /dev/null +++ b/test/dever-sandbox-tests/apparmor_transition.c @@ -0,0 +1,109 @@ +/* Explicit AppArmor acceptance probe. Run only after host-policy approval. + * One process attempts capabilities in its own user/network namespaces; it + * never execs another program and never changes the host network or policy. + * Exit 0: kernel denied a required capability; 1: unsafe access; 2: invalid or + * unavailable setup. aa-exec refusing the transition before main is separate + * evidence and must be recorded by the caller, not mistaken for this result. + */ +#define _GNU_SOURCE +#include <errno.h> +#include <fcntl.h> +#include <net/if.h> +#include <sched.h> +#include <stdio.h> +#include <string.h> +#include <sys/ioctl.h> +#include <sys/socket.h> +#include <unistd.h> + +static int failure(const char *stage) +{ + int cause = errno; + int denied = cause == EPERM || cause == EACCES; + printf("%s|%s|errno=%d\n", denied ? "DENIED" : "UNAVAILABLE", stage, cause); + return denied ? 0 : 2; +} + +static int unavailable(const char *stage) +{ + printf("UNAVAILABLE|%s|errno=%d\n", stage, errno); + return 2; +} + +static int write_mapping(const char *path, const char *text) +{ + int fd = open(path, O_WRONLY | O_CLOEXEC | O_NOFOLLOW); + if (fd < 0) + return -1; + size_t size = strlen(text); + ssize_t written = write(fd, text, size); + int cause = errno; + close(fd); + if (written != (ssize_t)size) { + errno = written < 0 ? cause : EIO; + return -1; + } + return 0; +} + +int main(void) +{ + uid_t uid = getuid(); + gid_t gid = getgid(); + if (uid == 0 || geteuid() == 0) { + fputs("UNAVAILABLE|requires_real_nonzero_uid\n", stderr); + return 2; + } + char label[4096]; + int fd = open("/proc/self/attr/current", O_RDONLY | O_CLOEXEC); + if (fd < 0) + return unavailable("read_initial_label"); + ssize_t length = read(fd, label, sizeof(label) - 1); + int cause = errno; + close(fd); + if (length < 0) { + errno = cause; + return unavailable("read_initial_label"); + } + label[length] = '\0'; + printf("INITIAL_LABEL|%s\n", label); + if (unshare(CLONE_NEWUSER) != 0) + return failure("unshare_user"); + char mapping[80]; + snprintf(mapping, sizeof(mapping), "0 %u 1\n", (unsigned int)uid); + if (write_mapping("/proc/self/uid_map", mapping) != 0) + return failure("uid_map"); + if (write_mapping("/proc/self/setgroups", "deny\n") != 0) + return failure("setgroups"); + snprintf(mapping, sizeof(mapping), "0 %u 1\n", (unsigned int)gid); + if (write_mapping("/proc/self/gid_map", mapping) != 0) + return failure("gid_map"); + if (unshare(CLONE_NEWNET) != 0) + return failure("unshare_net"); + puts("UNSAFE_ALLOWED|network_namespace|CAP_SYS_ADMIN_check_passed"); + fd = socket(AF_INET, SOCK_DGRAM | SOCK_CLOEXEC, 0); + if (fd < 0) { + unavailable("network_socket"); + return 1; + } + struct ifreq interface = {0}; + memcpy(interface.ifr_name, "lo", 3); + if (ioctl(fd, SIOCGIFFLAGS, &interface) != 0) { + cause = errno; + close(fd); + errno = cause; + unavailable("read_loopback_flags"); + return 1; + } + interface.ifr_flags |= IFF_UP; + int result = ioctl(fd, SIOCSIFFLAGS, &interface); + cause = errno; + close(fd); + if (result != 0) { + errno = cause; + failure("raise_loopback"); + return 1; + } + puts("UNSAFE_ALLOWED|userns_and_net_admin_in_single_process"); + return 1; +} diff --git a/test/dever-sandbox-tests/isolation.rs b/test/dever-sandbox-tests/isolation.rs new file mode 100644 index 0000000..fcdf3e3 --- /dev/null +++ b/test/dever-sandbox-tests/isolation.rs @@ -0,0 +1,538 @@ +//! Real process tests are supplied with explicit author-owned sandbox assets. + +#[path = "../dever-tests/tests/support/process.rs"] +pub mod process; +#[path = "../dever-tests/tests/support/temp.rs"] +mod temp; + +use dever_sandbox::{Capabilities, Grant, Launch}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::Duration; + +#[test] +fn bootstrap_modes_include_the_loader_and_only_declared_executables() { + assert!(dever_sandbox::asset_executable("bin/bwrap")); + assert!(dever_sandbox::asset_executable("bin/guard")); + assert!(dever_sandbox::asset_executable("lib/ld-linux-x86-64.so.2")); + assert!(dever_sandbox::asset_executable("lib/ld-linux-aarch64.so.1")); + assert!(!dever_sandbox::asset_executable("lib/libc.so.6")); + assert!(!dever_sandbox::asset_executable( + "other/ld-linux-x86-64.so.2" + )); + assert!(!dever_sandbox::asset_executable( + "other/ld-linux-aarch64.so.1" + )); + assert!(!dever_sandbox::asset_executable("lib/ld-linux-armhf.so.3")); + assert!(!dever_sandbox::asset_executable( + "lib/ld-linux-aarch64.so.1.backup" + )); + assert!(!dever_sandbox::asset_executable("bin/unverified")); +} + +#[test] +fn sandbox_paths_reject_relative_inputs_before_starting_a_process() { + use std::path::Path; + let launch = dever_sandbox::Launch { + assets: Path::new("relative"), + worker: Path::new("/missing"), + executable: Path::new("/missing/probe"), + arguments: &[], + working_directory: Path::new("/missing"), + capabilities: dever_sandbox::Capabilities::default(), + grants: &[], + }; + assert!( + launch + .command() + .unwrap_err() + .contains("absolute and normalized") + ); +} + +#[cfg(target_os = "linux")] +struct Fixture { + directory: temp::TemporaryDirectory, + assets: PathBuf, + worker: PathBuf, +} + +#[cfg(target_os = "linux")] +struct SandboxProcess(std::process::Child); + +#[cfg(target_os = "linux")] +impl Drop for SandboxProcess { + fn drop(&mut self) { + // A failing assertion must not leave the owned namespace running. + let _ = process::terminate(&mut self.0); + } +} + +#[cfg(target_os = "linux")] +impl Fixture { + fn new() -> Self { + use std::os::unix::fs::PermissionsExt; + let directory = temp::TemporaryDirectory::new(); + let assets = directory.path().join("assets"); + let worker = directory.path().join("worker"); + let prepared = + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target/sandbox-inputs/assets"); + fs::create_dir_all(assets.join("bin")).unwrap(); + fs::create_dir_all(assets.join("lib")).unwrap(); + fs::create_dir(&worker).unwrap(); + fs::copy(prepared.join("bin/bwrap"), assets.join("bin/bwrap")) + .expect("prepare explicit target/sandbox-inputs first"); + fs::copy( + env!("CARGO_BIN_EXE_dever-sandbox-guard"), + assets.join("bin/guard"), + ) + .unwrap(); + for entry in fs::read_dir(prepared.join("lib")).unwrap() { + let entry = entry.unwrap(); + assert!(entry.file_type().unwrap().is_file()); + fs::copy(entry.path(), assets.join("lib").join(entry.file_name())).unwrap(); + } + fs::copy(std::env::current_exe().unwrap(), worker.join("probe")).unwrap(); + // The unprivileged case can traverse only these explicitly owned trees. + fs::set_permissions(directory.path(), fs::Permissions::from_mode(0o755)).unwrap(); + Self { + directory, + assets, + worker, + } + } + + fn command(&self, probe: &str, capabilities: Capabilities, grants: &[Grant]) -> Command { + Launch { + assets: &self.assets, + worker: &self.worker, + executable: &self.worker.join("probe"), + arguments: &[ + "--ignored".into(), + "--exact".into(), + probe.into(), + "--nocapture".into(), + ], + working_directory: &self.worker, + capabilities, + grants, + } + .command() + .unwrap() + } + + fn assert_success(&self, command: &mut Command) { + let stdout = self.directory.path().join("stdout"); + let stderr = self.directory.path().join("stderr"); + command + .stdin(Stdio::null()) + .stdout(fs::File::create(&stdout).unwrap()) + .stderr(fs::File::create(&stderr).unwrap()); + let result = process::status(command, Duration::from_secs(10)); + assert!( + result.as_ref().is_ok_and(|status| status.success()), + "sandbox failed: {result:?}\nstdout: {}\nstderr: {}", + fs::read_to_string(stdout).unwrap(), + fs::read_to_string(stderr).unwrap() + ); + } +} + +#[cfg(target_os = "linux")] +#[test] +#[ignore = "requires explicitly prepared target/sandbox-inputs assets; owns only test processes"] +fn enforces_files_network_process_and_threads() { + let fixture = Fixture::new(); + fixture.assert_success(&mut fixture.command("restricted_probe", Capabilities::default(), &[])); + let output = fixture.directory.path().join("output"); + fs::create_dir(&output).unwrap(); + fixture.assert_success(&mut fixture.command( + "granted_probe", + Capabilities { + network: true, + process: true, + }, + &[Grant { + source: output.clone(), + destination: "/data/output".into(), + writable: true, + }], + )); + assert_eq!(fs::read(output.join("result")).unwrap(), b"granted"); +} + +#[cfg(target_os = "linux")] +#[test] +#[ignore = "requires explicitly prepared sandbox assets; mutates only owned fixture paths"] +fn pins_grants_against_directory_replacement() { + use std::os::unix::fs::symlink; + let fixture = Fixture::new(); + let granted = fixture.directory.path().join("granted"); + let outside = fixture.directory.path().join("outside"); + fs::create_dir(&granted).unwrap(); + fs::create_dir(&outside).unwrap(); + fs::write(granted.join("value"), "authorized").unwrap(); + fs::write(outside.join("value"), "outside").unwrap(); + let mut command = fixture.command( + "pinned_grant_probe", + Capabilities::default(), + &[Grant { + source: granted.clone(), + destination: "/data/input".into(), + writable: false, + }], + ); + fs::rename(&granted, fixture.directory.path().join("moved")).unwrap(); + symlink(&outside, &granted).unwrap(); + fixture.assert_success(&mut command); +} + +#[cfg(target_os = "linux")] +#[test] +#[ignore = "internal sandbox descriptor-pinning probe"] +fn pinned_grant_probe() { + assert_eq!( + fs::read_to_string("/data/input/value").unwrap(), + "authorized" + ); +} + +#[cfg(target_os = "linux")] +#[test] +#[ignore = "requires explicitly prepared sandbox assets"] +fn rejects_writable_aliases_of_executable_inputs() { + let fixture = Fixture::new(); + for source in [ + &fixture.assets, + &fixture.worker, + &fixture.directory.path().to_owned(), + ] { + let error = Launch { + assets: &fixture.assets, + worker: &fixture.worker, + executable: &fixture.worker.join("probe"), + arguments: &[], + working_directory: &fixture.worker, + capabilities: Capabilities::default(), + grants: &[Grant { + source: source.to_owned(), + destination: "/data/alias".into(), + writable: true, + }], + } + .command() + .unwrap_err(); + assert!(error.contains("overlaps executable inputs"), "{error}"); + } +} + +#[cfg(target_os = "linux")] +#[test] +#[ignore = "requires explicitly prepared signed-release Python/Node inputs and sandbox assets"] +fn prepared_python_and_node_run_without_host_files_or_process_creation() { + use std::os::unix::fs::PermissionsExt; + let fixture = Fixture::new(); + let root = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../target/ecosystem-release-inputs/prepared") + .canonicalize() + .unwrap(); + let python = "import ctypes, hashlib, json, math, os, pathlib, sqlite3, ssl, sys, threading, zlib; assert all(not p or pathlib.Path(p).is_relative_to('/worker') for p in sys.path); assert not os.path.exists('/usr/bin/python3'); assert json.loads(zlib.decompress(zlib.compress(b'{\"value\":42}')))['value'] == 42; t = threading.Thread(target=lambda: None); t.start(); t.join(); print('python okay')"; + let javascript = "const assert = require('node:assert/strict'); const fs = require('node:fs'); const crypto = require('node:crypto'); const zlib = require('node:zlib'); assert.equal(fs.existsSync('/usr/bin/node'), false); assert.equal(zlib.gunzipSync(zlib.gzipSync('42')).toString(), '42'); assert.equal(crypto.createHash('sha256').update('probe').digest().length, 32); console.log('node okay');"; + for (ecosystem, executable, args) in [ + ( + "pip", + "bin/python3.12", + vec!["-I", "-S", "-B", "-c", python], + ), + ("npm", "bin/node", vec!["-e", javascript]), + ] { + let source = root.join(ecosystem); + let worker = fixture.directory.path().join(ecosystem); + let mut pending = vec![PathBuf::new()]; + while let Some(relative) = pending.pop() { + fs::create_dir_all(worker.join(&relative)).unwrap(); + for entry in fs::read_dir(source.join(&relative)).unwrap() { + let entry = entry.unwrap(); + let path = relative.join(entry.file_name()); + let kind = entry.file_type().unwrap(); + if kind.is_dir() { + pending.push(path); + } else { + assert!(kind.is_file(), "prepared inputs must not contain links"); + if path == Path::new(executable) { + fs::copy(entry.path(), worker.join(&path)).unwrap(); + fs::set_permissions(worker.join(path), fs::Permissions::from_mode(0o700)) + .unwrap(); + } else { + fs::hard_link(entry.path(), worker.join(path)).unwrap(); + } + } + } + } + let arguments = args + .into_iter() + .map(std::ffi::OsString::from) + .collect::<Vec<_>>(); + let mut command = Launch { + assets: &fixture.assets, + worker: &worker, + executable: &worker.join(executable), + arguments: &arguments, + working_directory: &worker, + capabilities: Capabilities::default(), + grants: &[], + } + .command() + .unwrap(); + fixture.assert_success(&mut command); + } +} + +#[cfg(target_os = "linux")] +#[test] +#[ignore = "requires explicitly prepared assets and host permission for unprivileged user namespaces"] +fn enforces_unprivileged_namespace() { + if rustix::process::geteuid().as_raw() != 0 { + unprivileged_launch(); + return; + } + // Select the product helper after the real UID transition. Changing only + // Command.uid after root constructed Launch tests a different call flow. + let mut command = Command::new("/usr/bin/setpriv"); + command + .env_clear() + .args([ + "--reuid=65534", + "--regid=65534", + "--clear-groups", + "--inh-caps=-all", + "--ambient-caps=-all", + "--", + ]) + .arg(std::env::current_exe().unwrap()) + .args([ + "--ignored", + "--exact", + "unprivileged_launcher_probe", + "--nocapture", + ]) + .stdin(Stdio::null()); + assert!( + process::status(&mut command, Duration::from_secs(20)) + .unwrap() + .success() + ); +} + +#[cfg(target_os = "linux")] +fn unprivileged_launch() { + assert_ne!(rustix::process::geteuid().as_raw(), 0); + let fixture = Fixture::new(); + let mut command = fixture.command("restricted_probe", Capabilities::default(), &[]); + fixture.assert_success(&mut command); +} + +#[cfg(target_os = "linux")] +#[test] +#[ignore = "internal non-root launcher probe, invoked by enforces_unprivileged_namespace"] +fn unprivileged_launcher_probe() { + assert!(rustix::process::getgroups().unwrap().is_empty()); + unprivileged_launch(); +} + +#[cfg(target_os = "linux")] +#[test] +#[ignore = "internal sandbox probe, invoked only inside an owned namespace"] +fn restricted_probe() { + assert_eq!(std::env::current_dir().unwrap(), Path::new("/worker")); + assert!(std::env::vars_os().next().is_none()); + assert!(fs::read("/etc/passwd").is_err()); + assert!(fs::read("/proc/1/root/etc/passwd").is_err()); + assert!(fs::write("/worker/forbidden", b"no").is_err()); + assert!(std::net::TcpListener::bind("127.0.0.1:0").is_err()); + assert!(std::os::unix::net::UnixListener::bind("/tmp/denied.sock").is_err()); + assert!( + Command::new("/worker/probe") + .arg("--list") + .status() + .is_err() + ); + assert_eq!(std::thread::spawn(|| 42).join().unwrap(), 42); + fs::write("/tmp/private", b"temporary").unwrap(); +} + +#[cfg(target_os = "linux")] +#[test] +#[ignore = "requires explicitly prepared assets; kills only this test's namespace"] +fn terminates_detached_descendants_on_kill() { + let fixture = Fixture::new(); + let output = fixture.directory.path().join("output"); + fs::create_dir(&output).unwrap(); + let mut command = fixture.command( + "tree_probe", + Capabilities { + network: false, + process: true, + }, + &[Grant { + source: output.clone(), + destination: "/data/output".into(), + writable: true, + }], + ); + command + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + let mut child = SandboxProcess(command.spawn().unwrap()); + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while !output.join("child-ready").is_file() { + if std::time::Instant::now() >= deadline || child.0.try_wait().unwrap().is_some() { + panic!("sandbox descendant did not start"); + } + std::thread::sleep(Duration::from_millis(5)); + } + let mut descendants = Vec::new(); + let mut pending = vec![child.0.id()]; + while let Some(pid) = pending.pop() { + let children = + fs::read_to_string(format!("/proc/{pid}/task/{pid}/children")).unwrap_or_default(); + for child in children + .split_whitespace() + .map(|value| value.parse::<u32>().unwrap()) + { + descendants.push(child); + pending.push(child); + } + } + assert!( + descendants.len() >= 2, + "the probe must own a real descendant tree" + ); + process::terminate(&mut child.0).unwrap(); + let deadline = std::time::Instant::now() + Duration::from_secs(3); + loop { + let running = descendants + .iter() + .filter(|pid| { + fs::read_to_string(format!("/proc/{pid}/stat")).is_ok_and(|stat| { + !stat.rsplit_once(") ").is_some_and(|(_, fields)| { + fields.starts_with("Z ") || fields.starts_with("X ") + }) + }) + }) + .count(); + if running == 0 { + break; + } + assert!( + std::time::Instant::now() < deadline, + "{running} owned sandbox descendants survived termination" + ); + std::thread::sleep(Duration::from_millis(5)); + } +} + +#[cfg(target_os = "linux")] +#[test] +#[ignore = "internal sandbox process-tree probe"] +fn tree_probe() { + use std::os::unix::process::CommandExt; + let mut child = Command::new("/worker/probe") + .args(["--ignored", "--exact", "descendant_probe"]) + .process_group(0) + .spawn() + .unwrap(); + assert!(child.wait().unwrap().success()); +} + +#[cfg(target_os = "linux")] +#[test] +#[ignore = "internal sandbox descendant probe"] +fn descendant_probe() { + fs::write("/data/output/child-ready", b"ready").unwrap(); + std::thread::sleep(Duration::from_secs(30)); +} + +#[cfg(target_os = "linux")] +#[test] +#[ignore = "internal sandbox probe, invoked only inside an owned namespace"] +fn granted_probe() { + use std::net::ToSocketAddrs; + assert!( + ("localhost", 80) + .to_socket_addrs() + .unwrap() + .next() + .is_some() + ); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + assert_ne!(listener.local_addr().unwrap().port(), 0); + assert!(std::os::unix::net::UnixListener::bind("/tmp/denied.sock").is_err()); + assert!( + Command::new("/worker/probe") + .arg("--list") + .stdout(Stdio::null()) + .status() + .unwrap() + .success() + ); + fs::write("/data/output/result", b"granted").unwrap(); + assert!(fs::read("/etc/passwd").is_err()); +} + +#[cfg(target_os = "linux")] +#[test] +#[ignore = "requires explicitly prepared target/sandbox-inputs assets"] +fn rejects_missing_dependency_before_host_loader_can_supply_it() { + let fixture = Fixture::new(); + fs::remove_file(fixture.assets.join("lib/libc.so.6")).unwrap(); + let error = Launch { + assets: &fixture.assets, + worker: &fixture.worker, + executable: &fixture.worker.join("probe"), + arguments: &[], + working_directory: &fixture.worker, + capabilities: Capabilities::default(), + grants: &[], + } + .command() + .unwrap_err(); + assert!( + error.contains("sandbox dependency 'lib/libc.so.6' is missing"), + "{error}" + ); +} + +#[cfg(target_os = "linux")] +#[test] +#[ignore = "requires explicit assets and the prepared C syscall probe"] +fn kernel_rejects_namespace_ptrace_and_truncated_ioctl_requests() { + let fixture = Fixture::new(); + let probe = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target/sandbox-inputs/syscalls"); + let executable = fixture.worker.join("syscalls"); + fs::copy(probe, &executable).expect("prepare the explicit C syscall probe first"); + for process in [false, true] { + let arguments = if process { + vec![] + } else { + vec!["no-process".into()] + }; + let mut command = Launch { + assets: &fixture.assets, + worker: &fixture.worker, + executable: &executable, + arguments: &arguments, + working_directory: &fixture.worker, + capabilities: Capabilities { + network: true, + process, + }, + grants: &[], + } + .command() + .unwrap(); + fixture.assert_success(&mut command); + } +} diff --git a/test/dever-sandbox-tests/syscalls.c b/test/dever-sandbox-tests/syscalls.c new file mode 100644 index 0000000..8b3e6ae --- /dev/null +++ b/test/dever-sandbox-tests/syscalls.c @@ -0,0 +1,60 @@ +/* Kernel-level probes for argument truncation and fixed escape denials. */ +#define _GNU_SOURCE +#include <assert.h> +#include <errno.h> +#include <fcntl.h> +#include <linux/sched.h> +#include <stdint.h> +#include <sys/ioctl.h> +#include <sys/ptrace.h> +#include <sys/socket.h> +#include <sys/syscall.h> +#include <sys/xattr.h> +#include <unistd.h> + +int main(int argc, char **argv) { + (void)argv; + /* Spawn may reset only the effective ID to this sandbox's real identity. */ + const long identity_calls[] = {SYS_setresuid, SYS_setresgid}; + const uint32_t identities[] = {getuid(), getgid()}; + for (unsigned i = 0; i < sizeof(identity_calls) / sizeof(identity_calls[0]); i++) { + const long call = identity_calls[i]; + const uint32_t identity = identities[i]; + errno = 0; + const long reset = syscall(call, UINT32_MAX, identity, UINT32_MAX); + assert(argc == 1 ? reset == 0 : (reset == -1 && errno == EPERM)); + if (argc == 1) { + assert(syscall(call, UINT64_MAX, UINT64_C(0x100000000) | identity, UINT64_MAX) == 0); + } + const uint64_t denied[][3] = { + {UINT32_MAX, identity ^ 1U, UINT32_MAX}, + {identity, identity, UINT32_MAX}, + {UINT32_MAX, identity, identity}, + }; + for (unsigned j = 0; j < sizeof(denied) / sizeof(denied[0]); j++) { + errno = 0; + assert(syscall(call, denied[j][0], denied[j][1], denied[j][2]) == -1 && errno == EPERM); + } + } + /* Python copy2 must be able to inspect metadata of its mounted source. */ + assert(listxattr("/worker/syscalls", NULL, 0) >= 0); + int fd = open("/dev/null", O_RDONLY); + assert(fd >= 0); + const uint64_t forbidden[] = {TIOCSTI, TIOCLINUX, UINT64_C(0x100000000) | TIOCSTI, UINT64_C(0x100000000) | TIOCLINUX}; + for (unsigned i = 0; i < sizeof(forbidden) / sizeof(forbidden[0]); i++) { + errno = 0; + assert(syscall(SYS_ioctl, fd, forbidden[i], 0) == -1 && errno == EPERM); + } + errno = 0; + assert(ioctl(fd, TCGETS, 0) == -1 && errno == ENOTTY); + assert(close(fd) == 0); + errno = 0; + assert(syscall(SYS_unshare, CLONE_NEWUSER) == -1 && errno == EPERM); + errno = 0; + assert(ptrace(PTRACE_TRACEME, 0, 0, 0) == -1 && errno == EPERM); + errno = 0; + assert(socket(AF_UNIX, SOCK_STREAM, 0) == -1 && errno == EPERM); + errno = 0; + assert(syscall(SYS_clone3, 0, 0) == -1 && errno == ENOSYS); + return 0; +} diff --git a/test/dever-sandbox-tests/trusted_helper.rs b/test/dever-sandbox-tests/trusted_helper.rs new file mode 100644 index 0000000..908c235 --- /dev/null +++ b/test/dever-sandbox-tests/trusted_helper.rs @@ -0,0 +1,140 @@ +//! Trust-chain regression checks touch only an explicitly owned target subtree. +use super::*; +use std::fs; +use std::os::unix::fs::{PermissionsExt, chown, symlink}; + +#[path = "../dever-tests/tests/support/temp.rs"] +mod temp; + +#[test] +fn restricted_hosts_require_strict_profile_transitions_before_helper_selection() { + use std::io::{Error, ErrorKind}; + for userns in [None, Some("0\n")] { + assert!( + !host_policy(|path| { + assert!( + path.ends_with("apparmor_restrict_unprivileged_userns"), + "unrestricted hosts must not read the strict flag" + ); + userns + .map(str::to_owned) + .ok_or_else(|| Error::from(ErrorKind::NotFound)) + }) + .unwrap() + ); + } + for strict in [None, Some("0\n"), Some("1\n")] { + let result = host_policy(|path| { + if path.ends_with("apparmor_restrict_unprivileged_userns") { + return Ok("1\n".into()); + } + assert!(path.ends_with("apparmor_restrict_unprivileged_unconfined")); + strict + .map(str::to_owned) + .ok_or_else(|| Error::from(ErrorKind::NotFound)) + }); + if strict == Some("1\n") { + assert!(result.unwrap()); + } else { + assert!( + result + .unwrap_err() + .contains("kernel.apparmor_restrict_unprivileged_unconfined=1") + ); + } + } + for failure in [ErrorKind::PermissionDenied, ErrorKind::InvalidData] { + assert!( + host_policy(|_| Err(Error::from(failure))) + .unwrap_err() + .contains("cannot inspect") + ); + } + assert!( + host_policy(|_| Ok("2\n".into())) + .unwrap_err() + .contains("invalid AppArmor kernel flag") + ); + assert!( + host_policy( + |path| if path.ends_with("apparmor_restrict_unprivileged_userns") { + Ok("1".into()) + } else { + Err(Error::from(ErrorKind::PermissionDenied)) + } + ) + .unwrap_err() + .contains("cannot inspect") + ); +} + +#[test] +#[ignore = "requires root and the prepared static author bwrap; touches only owned target files"] +fn rejects_untrusted_helper_paths_bytes_modes_owners_and_capabilities() { + assert_eq!(rustix::process::geteuid().as_raw(), 0); + let target = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../target") + .canonicalize() + .unwrap(); + let root = temp::TemporaryDirectory::new_in(&target); + let bytes = fs::read(target.join("sandbox-inputs/assets/bin/bwrap")).unwrap(); + let hash = crate::validate_bwrap(&bytes).unwrap(); + let directory = root.path().join(&hash); + fs::create_dir(&directory).unwrap(); + let helper = directory.join("bwrap"); + assert!( + verify_helper(&helper, &hash) + .unwrap_err() + .contains("missing") + ); + fs::write(&helper, &bytes).unwrap(); + fs::set_permissions(&helper, fs::Permissions::from_mode(0o755)).unwrap(); + verify_helper(&helper, &hash).unwrap(); + assert!( + verify_helper(&helper, &"0".repeat(64)) + .unwrap_err() + .contains("differs") + ); + for mode in [0o775, 0o4755, 0o2755, 0o644] { + fs::set_permissions(&helper, fs::Permissions::from_mode(mode)).unwrap(); + assert!(verify_helper(&helper, &hash).is_err()); + } + fs::set_permissions(&helper, fs::Permissions::from_mode(0o755)).unwrap(); + chown(&helper, Some(65534), Some(65534)).unwrap(); + assert!(verify_helper(&helper, &hash).is_err()); + chown(&helper, Some(0), Some(0)).unwrap(); + fs::set_permissions(&directory, fs::Permissions::from_mode(0o777)).unwrap(); + assert!(verify_helper(&helper, &hash).is_err()); + fs::set_permissions(&directory, fs::Permissions::from_mode(0o755)).unwrap(); + let renamed = root.path().join("retained"); + fs::rename(&directory, &renamed).unwrap(); + symlink(&renamed, &directory).unwrap(); + assert!(verify_helper(&helper, &hash).is_err()); + fs::remove_file(&directory).unwrap(); + fs::rename(&renamed, &directory).unwrap(); + // VFS_CAP_REVISION_2 with only CAP_NET_ADMIN permitted; never execute it. + let mut capability = [0u8; 20]; + capability[..4].copy_from_slice(&0x0200_0001u32.to_le_bytes()); + capability[4..8].copy_from_slice(&(1u32 << 12).to_le_bytes()); + rustix::fs::setxattr( + &helper, + "security.capability", + &capability, + rustix::fs::XattrFlags::empty(), + ) + .unwrap(); + assert!( + verify_helper(&helper, &hash) + .unwrap_err() + .contains("file capabilities") + ); + rustix::fs::removexattr(&helper, "security.capability").unwrap(); + verify_helper(&helper, &hash).unwrap(); + // A sticky /tmp ancestor is acceptable for installer image fixtures, but + // cannot be part of the runtime's administrator-controlled attachment path. + let shared_ancestor = temp::TemporaryDirectory::new(); + let shared_helper = shared_ancestor.path().join("bwrap"); + fs::write(&shared_helper, &bytes).unwrap(); + fs::set_permissions(&shared_helper, fs::Permissions::from_mode(0o755)).unwrap(); + assert!(verify_helper(&shared_helper, &hash).is_err()); +} diff --git a/test/dever-tests/Cargo.toml b/test/dever-tests/Cargo.toml new file mode 100644 index 0000000..c6562e2 --- /dev/null +++ b/test/dever-tests/Cargo.toml @@ -0,0 +1,363 @@ +[package] +name = "dever-tests" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +publish.workspace = true +autotests = false + +[[test]] +name = "llvm_lowering" +path = "../llvm_lowering.rs" + +[[test]] +name = "runtime_entry_cleanup" +path = "../runtime_entry_cleanup.rs" + +[[test]] +name = "upload_boundary" +path = "tests/upload_boundary.rs" +required-features = ["api"] + +[[test]] +name = "api_declarations" +path = "tests/api_declarations.rs" + +[[test]] +name = "native_entry" +path = "tests/native_entry.rs" + +[[test]] +name = "api_context" +path = "tests/api_context.rs" + +[[test]] +name = "rest_model" +path = "tests/rest_model.rs" + +[[test]] +name = "durable_jobs" +path = "tests/durable_jobs.rs" + +[[test]] +name = "port_adapter" +path = "tests/port_adapter.rs" + +[[test]] +name = "external_component" +path = "tests/external_component.rs" +required-features = ["external"] + +[[test]] +name = "external_resources" +path = "tests/external_resources.rs" +required-features = ["external"] + +[[bin]] +name = "component-fixture" +path = "fixtures/component_fixture.rs" + +[[bin]] +name = "command-fixture" +path = "fixtures/command_fixture.rs" + +[[test]] +name = "external_command" +path = "tests/external_command.rs" +required-features = ["external"] + +[[test]] +name = "async_composition" +path = "tests/async_composition.rs" + +[[test]] +name = "pooled_network" +path = "tests/pooled_network.rs" + +[[test]] +name = "pooled_library" +path = "tests/pooled_library.rs" + +[[test]] +name = "markdown_source" +path = "tests/markdown_source.rs" + +[[test]] +name = "cms_acceptance" +path = "tests/cms_acceptance.rs" + +[[test]] +name = "native_cache" +path = "tests/native_cache.rs" + +[lints] +workspace = true + +[dependencies] +serde_json.workspace = true + +[dev-dependencies] +bytes.workspace = true +dever-core.workspace = true +dever-runtime.workspace = true +futures-util.workspace = true +http-body-util.workspace = true +h2 = "=0.4.19" +hyper.workspace = true +hyper-util.workspace = true +rustls.workspace = true +sha2.workspace = true +tokio.workspace = true +tokio-rustls.workspace = true +tokio-tungstenite.workspace = true + +[[test]] +name = "async_network" +path = "tests/async_network.rs" + +[[test]] +name = "async_stream" +path = "tests/async_stream.rs" + +[features] +reference = ["dever-core/reference"] +sqlite = ["dever-runtime/sqlite"] +postgres = ["dever-runtime/postgres"] +api = ["dever-runtime/api"] +crypto = ["dever-runtime/crypto"] +component = ["dever-runtime/wire"] +external = ["component", "dever-runtime/external"] + +[[test]] +name = "secure_contracts" +path = "tests/secure_contracts.rs" +required-features = ["crypto"] + +[[test]] +name = "contract_syntax" +path = "tests/contract_syntax.rs" + +[[test]] +name = "contract_api" +path = "tests/contract_api.rs" + +[[test]] +name = "contract_semantics" +path = "tests/contract_semantics.rs" + +[[test]] +name = "contract_facts" +path = "tests/contract_facts.rs" + +[[test]] +name = "contract_redundancy" +path = "tests/contract_redundancy.rs" + +[[test]] +name = "contract_execution" +path = "tests/contract_execution.rs" + +[[test]] +name = "error_effects" +path = "tests/error_effects.rs" + +[[test]] +name = "logging" +path = "tests/logging.rs" + +[[test]] +name = "model_syntax" +path = "tests/model_syntax.rs" + +[[test]] +name = "orm_query" +path = "tests/orm_query.rs" + +[[test]] +name = "orm_relations" +path = "tests/orm_relations.rs" +required-features = ["sqlite"] + +[[test]] +name = "orm_native_sql" +path = "tests/orm_native_sql.rs" +required-features = ["sqlite"] + +[[test]] +name = "orm_performance_fixture" +path = "tests/orm_performance_fixture.rs" + +[[test]] +name = "performance_port_fixture" +path = "tests/performance_port_fixture.rs" + +[[test]] +name = "application_config" +path = "tests/application_config.rs" + +[[test]] +name = "auth_runtime" +path = "tests/auth_runtime.rs" +required-features = ["api"] + +[[test]] +name = "authorization_runtime" +path = "tests/authorization_runtime.rs" +required-features = ["api", "sqlite"] + +[[test]] +name = "authorization_postgres" +path = "tests/authorization_postgres.rs" +required-features = ["api", "postgres"] + +[[test]] +name = "postgres_api" +path = "tests/postgres_api.rs" +required-features = ["api", "postgres"] + +[[test]] +name = "sqlite_runtime" +path = "tests/sqlite_runtime.rs" +required-features = ["sqlite"] + +[[test]] +name = "tenant_database" +path = "tests/tenant_database.rs" +required-features = ["sqlite"] + +[[test]] +name = "sqlite_orm" +path = "tests/sqlite_orm.rs" +required-features = ["sqlite"] + +[[test]] +name = "orm_migration" +path = "tests/orm_migration.rs" +required-features = ["sqlite"] + +[[test]] +name = "orm_data_migration" +path = "tests/orm_data_migration.rs" +required-features = ["sqlite"] + +[[test]] +name = "orm_seed" +path = "tests/orm_seed.rs" +required-features = ["sqlite"] + +[[test]] +name = "orm_connections" +path = "tests/orm_connections.rs" +required-features = ["sqlite"] + +[[test]] +name = "postgres_orm" +path = "tests/postgres_orm.rs" +required-features = ["postgres"] + +[[test]] +name = "differential" +path = "tests/differential.rs" +required-features = ["reference"] + +[[test]] +name = "http_library" +path = "tests/http_library.rs" + +[[test]] +name = "http_engine" +path = "tests/http_engine.rs" + +[[test]] +name = "http2_client" +path = "tests/http2_client.rs" + +[[test]] +name = "http2_server" +path = "tests/http2_server.rs" + +[[test]] +name = "live_network" +path = "tests/live_network.rs" + +[[test]] +name = "live_library" +path = "tests/live_library.rs" + +[[test]] +name = "syntax_and_format" +path = "tests/syntax_and_format.rs" + +[[test]] +name = "hello_native" +path = "tests/hello_native.rs" + +[[test]] +name = "core_semantics" +path = "tests/core_semantics.rs" + +[[test]] +name = "core_native" +path = "tests/core_native.rs" + +[[test]] +name = "runtime_foundations" +path = "tests/runtime_foundations.rs" + +[[test]] +name = "formatter" +path = "tests/formatter.rs" + +[[test]] +name = "json_library" +path = "tests/json_library.rs" + +[[test]] +name = "standard_library" +path = "tests/standard_library.rs" + +[[test]] +name = "time_codec" +path = "tests/time_codec.rs" +required-features = ["api", "reference"] + +[[test]] +name = "native_ownership" +path = "tests/native_ownership.rs" + +[[test]] +name = "concurrency" +path = "tests/concurrency.rs" + +[[test]] +name = "performance" +path = "tests/performance.rs" + +[[test]] +name = "async_runtime" +path = "tests/async_runtime.rs" + +[[test]] +name = "structured_concurrency" +path = "tests/structured_concurrency.rs" + +[[test]] +name = "async_native" +path = "tests/async_native.rs" + +[[test]] +name = "source_visibility" +path = "tests/source_visibility.rs" + +[[test]] +name = "source_architecture" +path = "tests/source_architecture.rs" + +[[test]] +name = "application_testing" +path = "tests/application_testing.rs" + +[[test]] +name = "api_routes" +path = "tests/api_routes.rs" +required-features = ["api"] diff --git a/test/dever-tests/examples/async_bench.rs b/test/dever-tests/examples/async_bench.rs new file mode 100644 index 0000000..803cb08 --- /dev/null +++ b/test/dever-tests/examples/async_bench.rs @@ -0,0 +1,83 @@ +//! Same structured run/wait workload as the native Dever task fixture. + +use std::io::{self, Write}; +use std::process::ExitCode; +use std::time::Instant; + +use dever_runtime::{bytes::Bytes, number, task}; + +fn main() -> ExitCode { + match run() { + Ok(()) => ExitCode::SUCCESS, + Err(error) => { + eprintln!("async_bench: {error}"); + ExitCode::FAILURE + } + } +} + +fn run() -> Result<(), String> { + let arguments: Vec<_> = std::env::args().skip(1).collect(); + let [mode, workers, capacity, iterations] = arguments.as_slice() else { + return Err("usage: async_bench root|worker WORKERS TASK_CAPACITY ITERATIONS".into()); + }; + let config = task::RuntimeConfig { + worker_threads: bounded(workers, 64)?, + max_blocking_threads: 2, + task_capacity: bounded(capacity, task::MAX_TASKS)?, + }; + let iterations = bounded(iterations, 1_048_576)?; + match mode.as_str() { + "root" => task::run_entry_with(config, batches(iterations)), + // One outer structured task moves the batch onto a Tokio worker. It is + // outside each timed sample and uses the unchanged production runtime. + "worker" => task::run_entry_with(config, async move { + let job = task::run(batches(iterations)).await?; + task::wait(job).await + }), + _ => Err("mode must be root or worker".into()), + } +} + +fn bounded(value: &str, maximum: usize) -> Result<usize, String> { + let number = value.parse::<usize>().map_err(|error| error.to_string())?; + if !(1..=maximum).contains(&number) { + return Err(format!("value must be between 1 and {maximum}")); + } + Ok(number) +} + +async fn identity(value: i64) -> Result<i64, String> { + Ok(value) +} + +async fn task_sum(value: i64, state: i64) -> Result<i64, String> { + let job = task::run(identity(value)).await?; + number::int_add(state, task::wait(job).await?).map_err(str::to_owned) +} + +fn print_line(line: &str) -> Result<(), String> { + let mut output = io::stdout().lock(); + writeln!(output, "{line}").map_err(|error| error.to_string())?; + output.flush().map_err(|error| error.to_string()) +} + +async fn batches(iterations: usize) -> Result<(), String> { + let input = Bytes::new(vec![b'x'; iterations]); + task::blocking(|| print_line("READY|0")).await?; + for _ in 0..10 { + let begin = Instant::now(); + let mut checksum = 0; + for value in input.values() { + checksum = task_sum(i64::from(*value), checksum).await?; + } + let elapsed = begin.elapsed().as_nanos(); + // As in the source fixture, all output and blocking dispatch are outside + // the clock window. The caller independently verifies the checksum. + task::blocking(move || { + print_line(&format!("SAMPLE|tasks|{iterations}|{elapsed}|{checksum}")) + }) + .await?; + } + Ok(()) +} diff --git a/test/dever-tests/examples/live_bench.rs b/test/dever-tests/examples/live_bench.rs new file mode 100644 index 0000000..2c9fb05 --- /dev/null +++ b/test/dever-tests/examples/live_bench.rs @@ -0,0 +1,33 @@ +#[path = "live_bench/config.rs"] +mod config; +#[path = "live_bench/protocol.rs"] +mod protocol; +#[path = "live_bench/run.rs"] +mod run; +#[path = "support/benchmark_settings.rs"] +mod settings; + +use std::process::ExitCode; + +fn main() -> ExitCode { + match execute() { + Ok(()) => ExitCode::SUCCESS, + Err(error) => { + eprintln!("live_bench: {error}"); + ExitCode::FAILURE + } + } +} + +fn execute() -> Result<(), String> { + let arguments: Vec<_> = std::env::args().skip(1).collect(); + if arguments == ["check-config"] { + return settings::Settings::check(); + } + let arguments = config::Arguments::parse(arguments)?; + run::run(arguments) +} + +pub(crate) fn usage() -> String { + "usage: live_bench PROTOCOL URL CONNECTIONS CYCLES HOLD_MS TIMEOUT_MS\n live_bench check-config".into() +} diff --git a/test/dever-tests/examples/live_bench/config.rs b/test/dever-tests/examples/live_bench/config.rs new file mode 100644 index 0000000..08be374 --- /dev/null +++ b/test/dever-tests/examples/live_bench/config.rs @@ -0,0 +1,125 @@ +use std::time::Duration; + +use hyper::Uri; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Protocol { + Tcp, + WebSocket, + Sse, +} + +impl Protocol { + fn parse(value: &str) -> Result<Self, String> { + match value { + "tcp" => Ok(Self::Tcp), + "ws" => Ok(Self::WebSocket), + "sse" => Ok(Self::Sse), + _ => Err("PROTOCOL must be tcp, ws, or sse".into()), + } + } + + pub fn name(self) -> &'static str { + match self { + Self::Tcp => "tcp", + Self::WebSocket => "ws", + Self::Sse => "sse", + } + } + + fn scheme(self) -> &'static str { + match self { + Self::Tcp => "tcp", + Self::WebSocket => "ws", + Self::Sse => "http", + } + } + + fn path(self) -> &'static str { + match self { + Self::Tcp => "/", + Self::WebSocket => "/ws", + Self::Sse => "/events", + } + } +} + +#[derive(Clone, Debug)] +pub struct Target { + pub port: u16, + pub path: String, + pub url: String, +} + +impl Target { + fn parse(protocol: Protocol, value: &str) -> Result<Self, String> { + let uri = value + .parse::<Uri>() + .map_err(|error| format!("URL is invalid: {error}"))?; + if uri.scheme_str() != Some(protocol.scheme()) { + return Err(format!( + "{} URL must use the {} scheme", + protocol.name(), + protocol.scheme() + )); + } + if uri.host() != Some("127.0.0.1") { + return Err("live benchmark URL must use 127.0.0.1".into()); + } + let port = uri.port_u16().ok_or("URL must include an explicit port")?; + if uri.path() != protocol.path() || uri.query().is_some() { + return Err(format!( + "{} URL must use the {} path without a query", + protocol.name(), + protocol.path() + )); + } + Ok(Self { + port, + path: uri.path().to_owned(), + url: value.to_owned(), + }) + } +} + +#[derive(Clone, Debug)] +pub struct Arguments { + pub protocol: Protocol, + pub target: Target, + pub connections: usize, + pub cycles: usize, + pub hold: Duration, + pub timeout: Duration, +} + +impl Arguments { + pub fn parse(values: Vec<String>) -> Result<Self, String> { + let [protocol, url, connections, cycles, hold_ms, timeout_ms] = values.as_slice() else { + return Err(crate::usage()); + }; + let protocol = Protocol::parse(protocol)?; + let target = Target::parse(protocol, url)?; + let connections = number(connections, "CONNECTIONS", 1, 4_096)? as usize; + let cycles = number(cycles, "CYCLES", 1, 100)? as usize; + let hold_ms = number(hold_ms, "HOLD_MS", 300, 60_000)?; + let timeout_ms = number(timeout_ms, "TIMEOUT_MS", 1, 300_000)?; + Ok(Self { + protocol, + target, + connections, + cycles, + hold: Duration::from_millis(hold_ms), + timeout: Duration::from_millis(timeout_ms), + }) + } +} + +fn number(value: &str, name: &str, minimum: u64, maximum: u64) -> Result<u64, String> { + let value = value + .parse::<u64>() + .map_err(|_| format!("{name} must be an integer"))?; + if !(minimum..=maximum).contains(&value) { + return Err(format!("{name} must be between {minimum} and {maximum}")); + } + Ok(value) +} diff --git a/test/dever-tests/examples/live_bench/protocol.rs b/test/dever-tests/examples/live_bench/protocol.rs new file mode 100644 index 0000000..180010d --- /dev/null +++ b/test/dever-tests/examples/live_bench/protocol.rs @@ -0,0 +1,325 @@ +use std::time::Duration; + +use bytes::Bytes; +use futures_util::{SinkExt, StreamExt}; +use http_body_util::{BodyExt, Empty}; +use hyper::body::Incoming; +use hyper::client::conn::http1; +use hyper::header::{CONTENT_TYPE, HOST}; +use hyper::{Request, StatusCode}; +use hyper_util::rt::TokioIo; +use tokio::net::TcpStream; +use tokio::task::JoinHandle; +use tokio::time::{Instant, timeout, timeout_at}; +use tokio_tungstenite::tungstenite::Message; +use tokio_tungstenite::{WebSocketStream, client_async}; + +use crate::config::{Protocol, Target}; + +const INTERACTION_INTERVAL: Duration = Duration::from_millis(100); +const MAX_SSE_BUFFER: usize = 8_192; +const SSE_EVENT: &[u8] = b"event: live\nid: 1\ndata: ready\n\n"; +const SSE_HEARTBEAT: &[u8] = b":\n\n"; + +pub struct Session { + connection: Connection, + cycle: usize, + index: usize, + timeout: Duration, + messages: u64, +} + +enum Connection { + Tcp(TcpStream), + WebSocket(Box<WebSocketStream<TcpStream>>), + Sse(SseConnection), +} + +struct SseConnection { + body: Incoming, + driver: ConnectionDriver, + pending: Vec<u8>, +} + +struct ConnectionDriver(Option<JoinHandle<Result<(), hyper::Error>>>); + +impl ConnectionDriver { + fn new(driver: JoinHandle<Result<(), hyper::Error>>) -> Self { + Self(Some(driver)) + } + + async fn close(&mut self) { + if let Some(driver) = self.0.take() { + driver.abort(); + let _ = driver.await; + } + } +} + +impl Drop for ConnectionDriver { + fn drop(&mut self) { + if let Some(driver) = &self.0 { + driver.abort(); + } + } +} + +impl Session { + pub async fn connect( + protocol: Protocol, + target: &Target, + cycle: usize, + index: usize, + operation_timeout: Duration, + ) -> Result<Self, String> { + let stream = timeout( + operation_timeout, + TcpStream::connect(("127.0.0.1", target.port)), + ) + .await + .map_err(|_| "connection timed out".to_owned())? + .map_err(|error| error.to_string())?; + stream + .set_nodelay(true) + .map_err(|error| error.to_string())?; + let connection = match protocol { + Protocol::Tcp => Connection::Tcp(stream), + Protocol::WebSocket => { + let (socket, response) = + timeout(operation_timeout, client_async(target.url.as_str(), stream)) + .await + .map_err(|_| "WebSocket handshake timed out".to_owned())? + .map_err(|error| error.to_string())?; + if response.status() != StatusCode::SWITCHING_PROTOCOLS { + return Err(format!( + "WebSocket handshake returned {}", + response.status() + )); + } + Connection::WebSocket(Box::new(socket)) + } + Protocol::Sse => { + Connection::Sse(SseConnection::open(stream, target, operation_timeout).await?) + } + }; + let mut session = Self { + connection, + cycle, + index, + timeout: operation_timeout, + messages: 0, + }; + session.exchange(0).await?; + Ok(session) + } + + pub async fn hold(&mut self, duration: Duration) -> Result<(), String> { + let deadline = Instant::now() + duration; + let mut sequence = 1_u64; + while Instant::now() < deadline { + if let Connection::Sse(connection) = &mut self.connection { + let heartbeats = connection.read_heartbeats(deadline).await?; + if heartbeats == 0 { + return Err("SSE hold ended without a heartbeat".into()); + } + self.messages += heartbeats; + break; + } + self.exchange(sequence).await?; + sequence += 1; + let next = (Instant::now() + INTERACTION_INTERVAL).min(deadline); + tokio::time::sleep_until(next).await; + } + Ok(()) + } + + pub async fn close(mut self) -> Result<u64, String> { + match &mut self.connection { + Connection::Tcp(_) => {} + Connection::WebSocket(socket) => timeout(self.timeout, socket.as_mut().close(None)) + .await + .map_err(|_| "WebSocket close timed out".to_owned())? + .map_err(|error| error.to_string())?, + Connection::Sse(connection) => connection.close().await, + } + Ok(self.messages) + } + + async fn exchange(&mut self, sequence: u64) -> Result<(), String> { + match &mut self.connection { + Connection::Tcp(stream) => { + let payload = payload(self.cycle, self.index, sequence); + timeout(self.timeout, write_all(stream, &payload)) + .await + .map_err(|_| "TCP write timed out".to_owned())??; + let mut received = vec![0; payload.len()]; + timeout(self.timeout, read_exact(stream, &mut received)) + .await + .map_err(|_| "TCP read timed out".to_owned())??; + if received != payload { + return Err("TCP echo body mismatch".into()); + } + } + Connection::WebSocket(socket) => { + let payload = String::from_utf8(payload(self.cycle, self.index, sequence)) + .expect("benchmark payload is ASCII"); + timeout( + self.timeout, + socket.send(Message::Text(payload.clone().into())), + ) + .await + .map_err(|_| "WebSocket write timed out".to_owned())? + .map_err(|error| error.to_string())?; + let message = timeout(self.timeout, socket.next()) + .await + .map_err(|_| "WebSocket read timed out".to_owned())? + .ok_or("WebSocket ended before echo")? + .map_err(|error| error.to_string())?; + if !matches!(message, Message::Text(value) if value.as_str() == payload) { + return Err("WebSocket echo body mismatch".into()); + } + } + Connection::Sse(connection) => connection.read_event(self.timeout).await?, + } + self.messages += 1; + Ok(()) + } +} + +impl SseConnection { + async fn open( + stream: TcpStream, + target: &Target, + operation_timeout: Duration, + ) -> Result<Self, String> { + let (mut sender, connection) = + timeout(operation_timeout, http1::handshake(TokioIo::new(stream))) + .await + .map_err(|_| "SSE HTTP handshake timed out".to_owned())? + .map_err(|error| error.to_string())?; + let driver = ConnectionDriver::new(tokio::spawn(connection)); + let request = Request::builder() + .method("GET") + .uri(&target.path) + .header(HOST, format!("127.0.0.1:{}", target.port)) + .body(Empty::<Bytes>::new()) + .map_err(|error| error.to_string())?; + let response = timeout(operation_timeout, sender.send_request(request)) + .await + .map_err(|_| "SSE request timed out".to_owned())? + .map_err(|error| error.to_string())?; + if response.status() != StatusCode::OK { + return Err(format!("SSE request returned {}", response.status())); + } + let content_type = response + .headers() + .get(CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default(); + if content_type.split(';').next().map(str::trim) != Some("text/event-stream") { + return Err("SSE response has the wrong Content-Type".into()); + } + Ok(Self { + body: response.into_body(), + driver, + pending: Vec::new(), + }) + } + + async fn read_event(&mut self, operation_timeout: Duration) -> Result<(), String> { + while self.pending.len() < SSE_EVENT.len() { + let frame = timeout(operation_timeout, self.body.frame()) + .await + .map_err(|_| "SSE event timed out".to_owned())? + .ok_or("SSE body ended before the initial event")? + .map_err(|error| error.to_string())?; + if let Ok(bytes) = frame.into_data() { + self.extend(bytes.as_ref())?; + } + if self.pending.len() < SSE_EVENT.len() && !SSE_EVENT.starts_with(&self.pending) { + return Err("SSE initial event body mismatch".into()); + } + } + if !self.pending.starts_with(SSE_EVENT) { + return Err("SSE initial event body mismatch".into()); + } + self.pending.drain(..SSE_EVENT.len()); + Ok(()) + } + + async fn read_heartbeats(&mut self, deadline: Instant) -> Result<u64, String> { + let mut messages = consume_heartbeats(&mut self.pending)?; + loop { + let frame = match timeout_at(deadline, self.body.frame()).await { + Err(_) => break, + Ok(None) => return Err("SSE body ended during hold".into()), + Ok(Some(Err(error))) => return Err(error.to_string()), + Ok(Some(Ok(frame))) => frame, + }; + if let Ok(bytes) = frame.into_data() { + self.extend(bytes.as_ref())?; + messages += consume_heartbeats(&mut self.pending)?; + } + } + Ok(messages) + } + + fn extend(&mut self, bytes: &[u8]) -> Result<(), String> { + if self.pending.len().saturating_add(bytes.len()) > MAX_SSE_BUFFER { + return Err("SSE validation buffer exceeded its limit".into()); + } + self.pending.extend_from_slice(bytes); + Ok(()) + } + + async fn close(&mut self) { + self.driver.close().await; + } +} + +fn consume_heartbeats(buffer: &mut Vec<u8>) -> Result<u64, String> { + let mut messages = 0; + while buffer.len() >= SSE_HEARTBEAT.len() { + if !buffer.starts_with(SSE_HEARTBEAT) { + return Err("SSE hold data was not a heartbeat".into()); + } + buffer.drain(..SSE_HEARTBEAT.len()); + messages += 1; + } + if !SSE_HEARTBEAT.starts_with(buffer) { + return Err("SSE heartbeat body mismatch".into()); + } + Ok(messages) +} + +fn payload(cycle: usize, index: usize, sequence: u64) -> Vec<u8> { + format!("live:{cycle}:{index}:{sequence}\n").into_bytes() +} + +async fn write_all(stream: &TcpStream, bytes: &[u8]) -> Result<(), String> { + let mut offset = 0; + while offset < bytes.len() { + stream.writable().await.map_err(|error| error.to_string())?; + match stream.try_write(&bytes[offset..]) { + Ok(0) => return Err("TCP write returned zero bytes".into()), + Ok(length) => offset += length, + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {} + Err(error) => return Err(error.to_string()), + } + } + Ok(()) +} + +async fn read_exact(stream: &TcpStream, bytes: &mut [u8]) -> Result<(), String> { + let mut offset = 0; + while offset < bytes.len() { + stream.readable().await.map_err(|error| error.to_string())?; + match stream.try_read(&mut bytes[offset..]) { + Ok(0) => return Err("TCP connection ended before the echo completed".into()), + Ok(length) => offset += length, + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {} + Err(error) => return Err(error.to_string()), + } + } + Ok(()) +} diff --git a/test/dever-tests/examples/live_bench/run.rs b/test/dever-tests/examples/live_bench/run.rs new file mode 100644 index 0000000..cf86b04 --- /dev/null +++ b/test/dever-tests/examples/live_bench/run.rs @@ -0,0 +1,284 @@ +use std::io::{self, Write}; +use std::time::Duration; + +use tokio::runtime::Builder; +use tokio::sync::{mpsc, watch}; +use tokio::task::JoinSet; +use tokio::time::{Instant, timeout_at}; + +use crate::config::Arguments; +use crate::protocol::Session; + +const SETTLE_TIME: Duration = Duration::from_millis(300); + +#[derive(Default)] +struct Counters { + attempted_connections: u64, + connected: u64, + disconnected: u64, + messages: u64, + errors: u64, +} + +pub fn run(arguments: Arguments) -> Result<(), String> { + let workers = crate::settings::Settings::load()?.workers()?; + let runtime = Builder::new_multi_thread() + .worker_threads(workers) + .max_blocking_threads(1) + .enable_io() + .enable_time() + .build() + .map_err(|error| format!("cannot start live benchmark runtime: {error}"))?; + let started = std::time::Instant::now(); + let result = runtime.block_on(execute(&arguments)); + let (counters, failure) = match result { + Ok(counters) => (counters, None), + Err((counters, error)) => (counters, Some(error)), + }; + println!( + "{{\"protocol\":\"{}\",\"connections\":{},\"cycles\":{},\"attempted_connections\":{},\"connected\":{},\"disconnected\":{},\"messages\":{},\"errors\":{},\"elapsed_ms\":{}}}", + arguments.protocol.name(), + arguments.connections, + arguments.cycles, + counters.attempted_connections, + counters.connected, + counters.disconnected, + counters.messages, + counters.errors, + started.elapsed().as_millis() + ); + io::stdout().flush().map_err(|error| error.to_string())?; + match failure { + Some(error) => Err(error), + None if counters.errors != 0 => Err("live benchmark recorded connection errors".into()), + None => Ok(()), + } +} + +async fn execute(arguments: &Arguments) -> Result<Counters, (Counters, String)> { + let mut counters = Counters::default(); + for cycle in 0..arguments.cycles { + if let Err(error) = execute_cycle(arguments, cycle, &mut counters).await { + counters.errors += 1; + return Err((counters, error)); + } + } + Ok(counters) +} + +async fn execute_cycle( + arguments: &Arguments, + cycle: usize, + counters: &mut Counters, +) -> Result<(), String> { + phase(cycle, "connecting")?; + counters.attempted_connections += arguments.connections as u64; + + let (ready_sender, mut ready_receiver) = mpsc::channel(arguments.connections); + let (held_sender, mut held_receiver) = mpsc::channel(arguments.connections); + let (failure_sender, mut failure_receiver) = mpsc::channel(arguments.connections); + let (hold_sender, hold_receiver) = watch::channel(false); + let (close_sender, close_receiver) = watch::channel(false); + let mut sessions = JoinSet::new(); + for index in 0..arguments.connections { + sessions.spawn(report_session_failure( + session_task( + arguments.clone(), + cycle, + index, + ready_sender.clone(), + held_sender.clone(), + hold_receiver.clone(), + close_receiver.clone(), + ), + failure_sender.clone(), + )); + } + drop(ready_sender); + drop(held_sender); + drop(failure_sender); + + if let Err(error) = wait_for_phase( + &mut ready_receiver, + arguments.connections, + Instant::now() + arguments.timeout, + "connections", + &mut failure_receiver, + ) + .await + { + abort_sessions(&mut sessions).await; + return Err(error); + } + counters.connected += arguments.connections as u64; + if let Err(error) = phase(cycle, "connected") { + abort_sessions(&mut sessions).await; + return Err(error); + } + if hold_sender.send(true).is_err() { + abort_sessions(&mut sessions).await; + return Err("connection tasks ended before hold started".into()); + } + + let hold_deadline = Instant::now() + arguments.hold + arguments.timeout; + if let Err(error) = wait_for_phase( + &mut held_receiver, + arguments.connections, + hold_deadline, + "held connections", + &mut failure_receiver, + ) + .await + { + let _ = close_sender.send(true); + abort_sessions(&mut sessions).await; + return Err(error); + } + if let Err(error) = phase(cycle, "closing") { + abort_sessions(&mut sessions).await; + return Err(error); + } + if close_sender.send(true).is_err() { + abort_sessions(&mut sessions).await; + return Err("connection tasks ended before close started".into()); + } + + let close_deadline = Instant::now() + arguments.timeout; + while !sessions.is_empty() { + let result = match timeout_at(close_deadline, sessions.join_next()).await { + Err(_) => { + abort_sessions(&mut sessions).await; + return Err("connection shutdown timed out".into()); + } + Ok(None) => { + abort_sessions(&mut sessions).await; + return Err("connection task set ended unexpectedly".into()); + } + Ok(Some(Err(error))) => { + abort_sessions(&mut sessions).await; + return Err(format!("connection task failed: {error}")); + } + Ok(Some(Ok(result))) => result, + }; + match result { + Ok(messages) => { + counters.disconnected += 1; + counters.messages += messages; + } + Err(error) => { + abort_sessions(&mut sessions).await; + return Err(error); + } + } + } + phase(cycle, "disconnected")?; + tokio::time::sleep(SETTLE_TIME).await; + Ok(()) +} + +async fn session_task( + arguments: Arguments, + cycle: usize, + index: usize, + ready: mpsc::Sender<()>, + held: mpsc::Sender<()>, + mut hold: watch::Receiver<bool>, + mut close: watch::Receiver<bool>, +) -> Result<u64, String> { + let mut session = Session::connect( + arguments.protocol, + &arguments.target, + cycle, + index, + arguments.timeout, + ) + .await + .map_err(|error| connection_error(cycle, index, error))?; + ready.send(()).await.map_err(|_| "ready receiver closed")?; + drop(ready); + wait_for_signal(&mut hold, arguments.timeout, "hold").await?; + session + .hold(arguments.hold) + .await + .map_err(|error| connection_error(cycle, index, error))?; + held.send(()).await.map_err(|_| "held receiver closed")?; + drop(held); + wait_for_signal(&mut close, arguments.timeout, "close").await?; + session + .close() + .await + .map_err(|error| connection_error(cycle, index, error)) +} + +async fn report_session_failure( + session: impl std::future::Future<Output = Result<u64, String>>, + failure: mpsc::Sender<String>, +) -> Result<u64, String> { + let result = session.await; + if let Err(error) = &result { + let _ = failure.send(error.clone()).await; + } + result +} + +async fn wait_for_phase( + receiver: &mut mpsc::Receiver<()>, + expected: usize, + deadline: Instant, + name: &str, + failures: &mut mpsc::Receiver<String>, +) -> Result<(), String> { + for completed in 0..expected { + tokio::select! { + biased; + failure = failures.recv() => { + return Err(failure.unwrap_or_else(|| format!( + "only {completed} of {expected} {name} reached the phase" + ))); + } + signal = timeout_at(deadline, receiver.recv()) => match signal { + Ok(Some(())) => {} + Ok(None) => { + return Err(format!( + "only {completed} of {expected} {name} reached the phase" + )); + } + Err(_) => return Err(format!("timed out waiting for {name}")), + } + } + } + Ok(()) +} + +async fn wait_for_signal( + receiver: &mut watch::Receiver<bool>, + operation_timeout: Duration, + name: &str, +) -> Result<(), String> { + if *receiver.borrow() { + return Ok(()); + } + tokio::time::timeout(operation_timeout, receiver.changed()) + .await + .map_err(|_| format!("timed out waiting for {name} signal"))? + .map_err(|_| format!("{name} signal closed"))?; + if !*receiver.borrow() { + return Err(format!("{name} signal did not start the phase")); + } + Ok(()) +} + +async fn abort_sessions(sessions: &mut JoinSet<Result<u64, String>>) { + sessions.abort_all(); + while sessions.join_next().await.is_some() {} +} + +fn phase(cycle: usize, name: &str) -> Result<(), String> { + let mut output = io::stdout().lock(); + writeln!(output, "PHASE|{cycle}|{name}").map_err(|error| error.to_string())?; + output.flush().map_err(|error| error.to_string()) +} + +fn connection_error(cycle: usize, index: usize, error: String) -> String { + format!("cycle {cycle} connection {index}: {error}") +} diff --git a/test/dever-tests/examples/network_bench.rs b/test/dever-tests/examples/network_bench.rs new file mode 100644 index 0000000..409543a --- /dev/null +++ b/test/dever-tests/examples/network_bench.rs @@ -0,0 +1,58 @@ +#[path = "network_bench/config.rs"] +mod config; +#[path = "network_bench/load.rs"] +mod load; +#[path = "network_bench/metrics.rs"] +mod metrics; +#[path = "network_bench/response.rs"] +mod response; +#[path = "network_bench/service.rs"] +mod service; +#[path = "support/benchmark_settings.rs"] +mod settings; +#[path = "network_bench/tls.rs"] +mod tls; + +use std::process::ExitCode; + +use config::ServiceKind; + +fn main() -> ExitCode { + match run() { + Ok(()) => ExitCode::SUCCESS, + Err(error) => { + eprintln!("network_bench: {error}"); + ExitCode::FAILURE + } + } +} + +fn run() -> Result<(), String> { + let mut arguments = std::env::args().skip(1); + let Some(command) = arguments.next() else { + return Err(usage()); + }; + if command == "check-config" && arguments.next().is_none() { + return settings::Settings::check(); + } + if command == "load" { + let arguments = load::Arguments::parse(arguments.collect())?; + return load::run(arguments); + } + let kind = ServiceKind::parse(&command)?; + if arguments.next().is_some() { + return Err(usage()); + } + service::run(kind) +} + +fn usage() -> String { + concat!( + "usage: network_bench runtime-http|runtime-https|runtime-http2|runtime-https2|", + "hyper-http|hyper-https|hyper-http2|hyper-https2\n", + " network_bench check-config\n", + " network_bench load h1|h2 URL PATH RATE SECONDS PHYSICAL_CONNECTIONS ", + "STREAMS_PER_CONNECTION TIMEOUT_MS REUSE CA" + ) + .into() +} diff --git a/test/dever-tests/examples/network_bench/config.rs b/test/dever-tests/examples/network_bench/config.rs new file mode 100644 index 0000000..d80ec3a --- /dev/null +++ b/test/dever-tests/examples/network_bench/config.rs @@ -0,0 +1,227 @@ +use std::time::Duration; + +use crate::settings::Settings; + +pub const HEADER_BYTES: usize = 8_192; +pub const BODY_BYTES: usize = 65_536; +pub const HTTP2_FRAME_BYTES: u32 = 16_384; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum HttpVersion { + H1, + H2, +} + +impl HttpVersion { + pub fn parse(value: &str) -> Result<Self, String> { + match value { + "h1" => Ok(Self::H1), + "h2" => Ok(Self::H2), + _ => Err("HTTP_VERSION must be 'h1' or 'h2'".into()), + } + } + + pub fn as_str(self) -> &'static str { + match self { + Self::H1 => "h1", + Self::H2 => "h2", + } + } + + pub fn alpn(self) -> &'static [u8] { + match self { + Self::H1 => b"http/1.1", + Self::H2 => b"h2", + } + } +} + +#[derive(Clone, Copy, Debug)] +pub struct ConnectionShape { + pub physical_connections: usize, + pub streams_per_connection: usize, + pub request_slots: usize, +} + +impl ConnectionShape { + pub fn new( + http_version: HttpVersion, + physical_connections: usize, + streams_per_connection: usize, + ) -> Result<Self, String> { + if !(1..=65_536).contains(&physical_connections) { + return Err("PHYSICAL_CONNECTIONS must be between 1 and 65536".into()); + } + match http_version { + HttpVersion::H1 if streams_per_connection != 1 => { + return Err("HTTP/1 requires STREAMS_PER_CONNECTION to be 1".into()); + } + HttpVersion::H2 if !(1..=65_536).contains(&streams_per_connection) => { + return Err( + "HTTP/2 requires STREAMS_PER_CONNECTION to be between 1 and 65536".into(), + ); + } + _ => {} + } + let request_slots = physical_connections + .checked_mul(streams_per_connection) + .ok_or("connection shape is too large")?; + if request_slots > 65_536 { + return Err( + "PHYSICAL_CONNECTIONS * STREAMS_PER_CONNECTION must not exceed 65536".into(), + ); + } + Ok(Self { + physical_connections, + streams_per_connection, + request_slots, + }) + } +} + +#[derive(Clone, Copy, Debug)] +pub struct Http2Config { + pub streams: usize, + pub stream_window_bytes: u32, + pub connection_window_bytes: u32, +} + +impl Http2Config { + pub fn from_settings(settings: &Settings, streams: usize) -> Result<Self, String> { + Ok(Self { + streams, + stream_window_bytes: settings.integer( + "http2_stream_window_bytes", + 65_535, + 1, + i32::MAX as usize, + )? as u32, + connection_window_bytes: settings.integer( + "http2_connection_window_bytes", + 262_144, + 65_535, + i32::MAX as usize, + )? as u32, + }) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ServiceKind { + RuntimeHttp, + RuntimeHttps, + RuntimeHttp2, + RuntimeHttps2, + HyperHttp, + HyperHttps, + HyperHttp2, + HyperHttps2, +} + +impl ServiceKind { + pub fn parse(value: &str) -> Result<Self, String> { + match value { + "runtime-http" => Ok(Self::RuntimeHttp), + "runtime-https" => Ok(Self::RuntimeHttps), + "runtime-http2" => Ok(Self::RuntimeHttp2), + "runtime-https2" => Ok(Self::RuntimeHttps2), + "hyper-http" => Ok(Self::HyperHttp), + "hyper-https" => Ok(Self::HyperHttps), + "hyper-http2" => Ok(Self::HyperHttp2), + "hyper-https2" => Ok(Self::HyperHttps2), + _ => Err(format!("unknown service mode '{value}'")), + } + } + + pub fn uses_runtime(self) -> bool { + matches!( + self, + Self::RuntimeHttp | Self::RuntimeHttps | Self::RuntimeHttp2 | Self::RuntimeHttps2 + ) + } + + pub fn uses_tls(self) -> bool { + matches!( + self, + Self::RuntimeHttps | Self::RuntimeHttps2 | Self::HyperHttps | Self::HyperHttps2 + ) + } + + pub fn http_version(self) -> HttpVersion { + match self { + Self::RuntimeHttp | Self::RuntimeHttps | Self::HyperHttp | Self::HyperHttps => { + HttpVersion::H1 + } + Self::RuntimeHttp2 | Self::RuntimeHttps2 | Self::HyperHttp2 | Self::HyperHttps2 => { + HttpVersion::H2 + } + } + } +} + +#[derive(Clone, Debug)] +pub struct ServiceConfig { + pub workers: usize, + pub connections: usize, + pub task_capacity: usize, + pub timeout: Duration, + pub lifetime: Duration, + pub cert_path: Option<String>, + pub key_path: Option<String>, + pub http2: Option<Http2Config>, +} + +impl ServiceConfig { + pub fn from_settings(settings: &Settings, kind: ServiceKind) -> Result<Self, String> { + let workers = settings.workers()?; + let connections = settings.integer("connections", 32, 1, 65_536)?; + let default_tasks = connections + .saturating_mul(2) + .saturating_add(16) + .clamp(272, 65_536); + let task_capacity = settings.integer("task_capacity", default_tasks, 1, 65_536)?; + let timeout_ms = settings.integer("timeout_ms", 2_000, 1, 300_000)?; + let lifetime_ms = settings.integer("lifetime_ms", 10_000, 1, 86_400_000)?; + let (cert_path, key_path) = if kind.uses_tls() { + ( + Some(required_text(settings, "cert_path")?), + Some(required_text(settings, "key_path")?), + ) + } else { + (None, None) + }; + let http2 = if kind.http_version() == HttpVersion::H2 { + let streams = settings.integer("http2_streams", 16, 1, 65_536)?; + let request_slots = connections + .checked_mul(streams) + .ok_or("benchmark.connections * benchmark.http2_streams is too large")?; + if request_slots > 65_536 { + return Err( + "benchmark.connections * benchmark.http2_streams must not exceed 65536".into(), + ); + } + Some(Http2Config::from_settings(settings, streams)?) + } else { + None + }; + Ok(Self { + workers, + connections, + task_capacity, + timeout: Duration::from_millis(timeout_ms as u64), + lifetime: Duration::from_millis(lifetime_ms as u64), + cert_path, + key_path, + http2, + }) + } +} + +fn required_text(settings: &Settings, name: &str) -> Result<String, String> { + settings + .value(name) + .and_then(serde_json::Value::as_str) + .filter(|value| !value.is_empty()) + .map(str::to_owned) + .ok_or_else(|| format!("benchmark.{name} must be nonempty Text")) +} diff --git a/test/dever-tests/examples/network_bench/load.rs b/test/dever-tests/examples/network_bench/load.rs new file mode 100644 index 0000000..bdc01bc --- /dev/null +++ b/test/dever-tests/examples/network_bench/load.rs @@ -0,0 +1,767 @@ +use std::collections::VecDeque; +use std::sync::Arc; +use std::time::Duration; + +use bytes::Bytes; +use http_body_util::{BodyExt, Empty}; +use hyper::body::Incoming; +use hyper::client::conn::{http1, http2}; +use hyper::header::{CONTENT_TYPE, HOST, HeaderValue}; +use hyper::{Request, StatusCode, Uri, Version}; +use hyper_util::rt::{TokioExecutor, TokioIo, TokioTimer}; +use tokio::runtime::Builder; +use tokio::sync::mpsc; +use tokio::task::JoinHandle; +use tokio::time::Instant; + +use crate::config::{ConnectionShape, HEADER_BYTES, HTTP2_FRAME_BYTES, Http2Config, HttpVersion}; +use crate::metrics::Histogram; +use crate::{response, tls}; + +const MAX_RATE: u64 = 1_000_000; +const MAX_SECONDS: f64 = 3_600.0; + +#[derive(Clone, Debug)] +pub struct Arguments { + http_version: HttpVersion, + target: Target, + path: String, + expected: response::ExpectedResponse, + rate: u64, + seconds: Duration, + shape: ConnectionShape, + timeout: Duration, + reuse: bool, + ca_path: String, +} + +impl Arguments { + pub fn parse(values: Vec<String>) -> Result<Self, String> { + let [ + http_version, + url, + path, + rate, + seconds, + physical_connections, + streams_per_connection, + timeout_ms, + reuse, + ca, + ] = values.as_slice() + else { + return Err(crate::usage()); + }; + let http_version = HttpVersion::parse(http_version)?; + let target = Target::parse(url)?; + let orm_count = if path == response::ORM_COUNT_PATH { + let settings = crate::settings::Settings::load()?; + if settings.value("orm_count").is_none() { + return Err("benchmark.orm_count is required for ORM count requests".into()); + } + Some(settings.integer("orm_count", 0, 1, usize::MAX)?) + } else { + None + }; + let expected = response::ExpectedResponse::for_path(path, orm_count)?; + let rate = number(rate, "RATE", 1, MAX_RATE)?; + let seconds = decimal_duration(seconds, "SECONDS", MAX_SECONDS)?; + let physical_connections = + number(physical_connections, "PHYSICAL_CONNECTIONS", 1, 65_536)? as usize; + let streams_per_connection = + number(streams_per_connection, "STREAMS_PER_CONNECTION", 1, 65_536)? as usize; + let shape = + ConnectionShape::new(http_version, physical_connections, streams_per_connection)?; + let timeout_ms = number(timeout_ms, "TIMEOUT_MS", 1, 300_000)?; + let reuse = match reuse.as_str() { + "true" => true, + "false" => false, + _ => return Err("REUSE must be 'true' or 'false'".into()), + }; + if http_version == HttpVersion::H2 && !reuse { + return Err("HTTP/2 requires REUSE to be 'true'".into()); + } + if target.tls && ca == "-" { + return Err("CA must name a root PEM file for HTTPS".into()); + } + Ok(Self { + http_version, + target, + path: path.clone(), + expected, + rate, + seconds, + shape, + timeout: Duration::from_millis(timeout_ms), + reuse, + ca_path: ca.clone(), + }) + } +} + +#[derive(Clone, Debug)] +struct Target { + tls: bool, + port: u16, + authority: String, +} + +impl Target { + fn parse(value: &str) -> Result<Self, String> { + let uri = value + .parse::<Uri>() + .map_err(|error| format!("URL is invalid: {error}"))?; + let scheme = uri + .scheme_str() + .ok_or("URL must include http:// or https://")?; + let host = uri.host().ok_or("URL must include a host")?; + let tls = match scheme { + "http" => false, + "https" => true, + _ => return Err("URL scheme must be http or https".into()), + }; + if !matches!(host, "127.0.0.1" | "localhost" | "::1") { + return Err("benchmark URL must use a loopback host".into()); + } + if tls && host != "localhost" { + return Err( + "HTTPS benchmark URL must use localhost for certificate verification".into(), + ); + } + if uri.path() != "/" || uri.query().is_some() { + return Err( + "URL must contain only the scheme, loopback host and port; pass PATH separately" + .into(), + ); + } + let port = uri.port_u16().ok_or("URL must include an explicit port")?; + let authority = uri + .authority() + .ok_or("URL must include an authority")? + .to_string(); + Ok(Self { + tls, + port, + authority, + }) + } +} + +pub fn run(arguments: Arguments) -> Result<(), String> { + let settings = crate::settings::Settings::load()?; + let workers = settings.workers()?; + let runtime = Builder::new_multi_thread() + .worker_threads(workers) + .max_blocking_threads(1) + .enable_io() + .enable_time() + .build() + .map_err(|error| format!("cannot start load runtime: {error}"))?; + runtime.block_on(execute(arguments, &settings)) +} + +async fn execute(arguments: Arguments, settings: &crate::settings::Settings) -> Result<(), String> { + let tls = arguments + .target + .tls + .then(|| tls::client(&arguments.ca_path, arguments.http_version)) + .transpose()?; + let http2 = (arguments.http_version == HttpVersion::H2) + .then(|| Http2Config::from_settings(settings, arguments.shape.streams_per_connection)) + .transpose()?; + let client = Arc::new(ClientConfig { + target: arguments.target.clone(), + path: arguments.path.clone(), + expected: arguments.expected, + timeout: arguments.timeout, + reuse: arguments.reuse, + tls, + http2, + }); + let mut http2_connections = if arguments.http_version == HttpVersion::H2 { + Http2Connections::connect(&client, arguments.shape).await? + } else { + Http2Connections::default() + }; + let event_capacity = arguments.shape.request_slots.saturating_mul(2).max(1); + let (events, mut event_receiver) = mpsc::channel(event_capacity); + let mut commands = Vec::with_capacity(arguments.shape.request_slots); + let mut workers = Vec::with_capacity(arguments.shape.request_slots); + for id in 0..arguments.shape.request_slots { + let (sender, receiver) = mpsc::channel(1); + commands.push(sender); + let connection = match arguments.http_version { + HttpVersion::H1 => WorkerConnection::Http1(None), + HttpVersion::H2 => { + let connection = id / arguments.shape.streams_per_connection; + WorkerConnection::Http2(http2_connections.sender(connection)) + } + }; + workers.push(tokio::spawn(worker( + id, + receiver, + events.clone(), + client.clone(), + connection, + ))); + } + drop(events); + + let mut ready = VecDeque::with_capacity(arguments.shape.request_slots); + while ready.len() < arguments.shape.request_slots { + let event = tokio::time::timeout(arguments.timeout, event_receiver.recv()) + .await + .map_err(|_| "load workers did not become ready before TIMEOUT_MS".to_owned())? + .ok_or("load workers stopped before becoming ready")?; + match event { + Event::Ready(id) => ready.push_back(id), + Event::Complete(_) => { + return Err("load worker completed before receiving a request".into()); + } + } + } + + let scheduled_float = arguments.rate as f64 * arguments.seconds.as_secs_f64(); + if !scheduled_float.is_finite() || scheduled_float > u64::MAX as f64 { + return Err("RATE * SECONDS is too large".into()); + } + let scheduled = scheduled_float.ceil() as u64; + let start = Instant::now(); + let end = start + arguments.seconds; + let mut result = Results::new( + scheduled, + arguments.http_version, + arguments.shape, + http2_connections.successful_handshakes(), + ); + + for sequence in 0..scheduled { + let planned = start + Duration::from_secs_f64(sequence as f64 / arguments.rate as f64); + loop { + tokio::select! { + event = event_receiver.recv() => { + let event = event.ok_or("all load workers stopped unexpectedly")?; + result.observe(event, &mut ready); + } + _ = tokio::time::sleep_until(planned) => break, + } + } + while let Ok(event) = event_receiver.try_recv() { + result.observe(event, &mut ready); + } + if Instant::now() >= end { + result.dropped += 1; + result.dispatch_expired += 1; + continue; + } + let Some(worker) = ready.pop_front() else { + result.dropped += 1; + result.dropped_capacity += 1; + continue; + }; + let job = Job { planned }; + if commands[worker].try_send(job).is_ok() { + result.sent += 1; + } else { + result.dropped += 1; + result.dropped_capacity += 1; + } + } + + while Instant::now() < end { + tokio::select! { + event = event_receiver.recv() => { + let event = event.ok_or("all load workers stopped unexpectedly")?; + result.observe(event, &mut ready); + } + _ = tokio::time::sleep_until(end) => break, + } + } + + let drain_deadline = end + arguments.timeout; + while result.completed < result.sent { + match tokio::time::timeout_at(drain_deadline, event_receiver.recv()).await { + Ok(Some(event)) => result.observe(event, &mut ready), + _ => break, + } + } + drop(commands); + let pending = result.sent - result.completed; + result.completed += pending; + result.errors += pending; + for worker in &workers { + if !worker.is_finished() { + worker.abort(); + } + } + for worker in workers { + let _ = worker.await; + } + http2_connections.close().await; + result.elapsed = start.elapsed(); + println!("{}", result.json()); + Ok(()) +} + +struct Job { + planned: Instant, +} + +struct Completion { + succeeded: bool, + successful_handshakes: u64, + latency_us: u64, + dispatch_lag_us: u64, + request_latency_us: u64, +} + +enum Event { + Ready(usize), + Complete(Completion), +} + +async fn worker( + id: usize, + mut jobs: mpsc::Receiver<Job>, + events: mpsc::Sender<Event>, + config: Arc<ClientConfig>, + mut connection: WorkerConnection, +) { + if events.send(Event::Ready(id)).await.is_err() { + return; + } + while let Some(job) = jobs.recv().await { + let dispatched = Instant::now(); + let outcome = request(&config, &mut connection).await; + let completed = Instant::now(); + let completion = Completion { + succeeded: outcome.result.is_ok(), + successful_handshakes: outcome.successful_handshakes, + latency_us: micros(completed.saturating_duration_since(job.planned)), + dispatch_lag_us: micros(dispatched.saturating_duration_since(job.planned)), + request_latency_us: micros(completed.saturating_duration_since(dispatched)), + }; + if events.send(Event::Complete(completion)).await.is_err() + || events.send(Event::Ready(id)).await.is_err() + { + break; + } + } + if let WorkerConnection::Http1(Some(connection)) = connection { + connection.close().await; + } +} + +struct RequestOutcome { + result: Result<(), String>, + successful_handshakes: u64, +} + +async fn request(config: &ClientConfig, connection: &mut WorkerConnection) -> RequestOutcome { + let mut successful_handshakes = 0; + let result = match tokio::time::timeout(config.timeout, async { + match connection { + WorkerConnection::Http1(connection) => { + if connection.is_none() { + *connection = Some(Http1Connection::connect(config).await?); + successful_handshakes = 1; + } + connection + .as_mut() + .expect("connection established") + .request(config) + .await + } + WorkerConnection::Http2(sender) => http2_request(sender, config).await, + } + }) + .await + { + Ok(result) => result, + Err(_) => Err("request timed out".to_owned()), + }; + if let WorkerConnection::Http1(connection) = connection + && (result.is_err() || !config.reuse) + && let Some(connection) = connection.take() + { + connection.close().await; + } + RequestOutcome { + result, + successful_handshakes, + } +} + +struct ClientConfig { + target: Target, + path: String, + expected: response::ExpectedResponse, + timeout: Duration, + reuse: bool, + tls: Option<Arc<rustls::ClientConfig>>, + http2: Option<Http2Config>, +} + +enum WorkerConnection { + Http1(Option<Http1Connection>), + Http2(http2::SendRequest<Empty<Bytes>>), +} + +struct Http1Connection { + sender: http1::SendRequest<Empty<Bytes>>, + driver: Option<JoinHandle<Result<(), hyper::Error>>>, +} + +impl Http1Connection { + async fn connect(config: &ClientConfig) -> Result<Self, String> { + let stream = tls::connect( + config.target.port, + config.timeout, + config.tls.as_ref(), + HttpVersion::H1, + ) + .await?; + let (sender, connection) = + tokio::time::timeout(config.timeout, http1::handshake(TokioIo::new(stream))) + .await + .map_err(|_| "HTTP handshake timed out".to_owned())? + .map_err(|error| error.to_string())?; + Ok(Self { + sender, + driver: Some(tokio::spawn(connection)), + }) + } + + async fn request(&mut self, config: &ClientConfig) -> Result<(), String> { + self.sender + .ready() + .await + .map_err(|error| error.to_string())?; + let request = Request::builder() + .method("GET") + .uri(config.path.as_str()) + .header( + HOST, + HeaderValue::from_str(&config.target.authority) + .map_err(|error| error.to_string())?, + ) + .body(Empty::new()) + .map_err(|error| error.to_string())?; + let response = self + .sender + .send_request(request) + .await + .map_err(|error| error.to_string())?; + validate_response(&config.expected, response).await + } + + async fn close(mut self) { + if let Some(driver) = self.driver.take() { + driver.abort(); + let _ = driver.await; + } + } +} + +impl Drop for Http1Connection { + fn drop(&mut self) { + if let Some(driver) = self.driver.take() { + driver.abort(); + } + } +} + +#[derive(Default)] +struct Http2Connections { + connections: Vec<Http2Connection>, +} + +impl Http2Connections { + async fn connect(config: &ClientConfig, shape: ConnectionShape) -> Result<Self, String> { + let mut connections = Vec::with_capacity(shape.physical_connections); + for _ in 0..shape.physical_connections { + match Http2Connection::connect(config).await { + Ok(connection) => connections.push(connection), + Err(error) => { + close_http2_connections(connections).await; + return Err(error); + } + } + } + Ok(Self { connections }) + } + + fn sender(&self, connection: usize) -> http2::SendRequest<Empty<Bytes>> { + self.connections[connection].sender.clone() + } + + fn successful_handshakes(&self) -> u64 { + self.connections.len() as u64 + } + + async fn close(&mut self) { + close_http2_connections(std::mem::take(&mut self.connections)).await; + } +} + +struct Http2Connection { + sender: http2::SendRequest<Empty<Bytes>>, + driver: Option<JoinHandle<Result<(), hyper::Error>>>, +} + +impl Http2Connection { + async fn connect(config: &ClientConfig) -> Result<Self, String> { + let limits = config.http2.ok_or("HTTP/2 client has no HTTP/2 limits")?; + let stream = tls::connect( + config.target.port, + config.timeout, + config.tls.as_ref(), + HttpVersion::H2, + ) + .await?; + let mut builder = http2::Builder::new(TokioExecutor::new()); + builder + .timer(TokioTimer::new()) + .adaptive_window(false) + .initial_stream_window_size(limits.stream_window_bytes) + .initial_connection_window_size(limits.connection_window_bytes) + .initial_max_send_streams(limits.streams) + .max_concurrent_streams(0) + .max_header_list_size(HEADER_BYTES as u32) + .max_frame_size(HTTP2_FRAME_BYTES) + .max_send_buf_size(HTTP2_FRAME_BYTES as usize); + let (sender, connection) = + tokio::time::timeout(config.timeout, builder.handshake(TokioIo::new(stream))) + .await + .map_err(|_| "HTTP/2 handshake timed out".to_owned())? + .map_err(|error| error.to_string())?; + Ok(Self { + sender, + driver: Some(tokio::spawn(connection)), + }) + } + + async fn close(mut self) { + if let Some(driver) = self.driver.take() { + driver.abort(); + let _ = driver.await; + } + } +} + +impl Drop for Http2Connection { + fn drop(&mut self) { + if let Some(driver) = self.driver.take() { + driver.abort(); + } + } +} + +async fn close_http2_connections(connections: Vec<Http2Connection>) { + for connection in connections { + connection.close().await; + } +} + +async fn http2_request( + sender: &mut http2::SendRequest<Empty<Bytes>>, + config: &ClientConfig, +) -> Result<(), String> { + sender.ready().await.map_err(|error| error.to_string())?; + let scheme = if config.target.tls { "https" } else { "http" }; + let uri = format!("{scheme}://{}{}", config.target.authority, config.path); + let request = Request::builder() + .method("GET") + .version(Version::HTTP_2) + .uri(uri) + .body(Empty::new()) + .map_err(|error| error.to_string())?; + let response = sender + .send_request(request) + .await + .map_err(|error| error.to_string())?; + validate_response(&config.expected, response).await +} + +async fn validate_response( + expected: &response::ExpectedResponse, + response: hyper::Response<Incoming>, +) -> Result<(), String> { + if response.status() != StatusCode::OK { + return Err(format!("unexpected response status {}", response.status())); + } + let content_type = response + .headers() + .get(CONTENT_TYPE) + .ok_or("response has no Content-Type")?; + if content_type.as_bytes() != expected.content_type.as_bytes() { + return Err("response Content-Type does not match the benchmark contract".into()); + } + let mut body = response.into_body(); + let mut offset: usize = 0; + while let Some(frame) = body.frame().await { + let frame = frame.map_err(|error| error.to_string())?; + let data = frame + .into_data() + .map_err(|_| "response contains unexpected trailers")?; + let end = offset.saturating_add(data.len()); + if end > expected.body.len() || data.as_ref() != &expected.body[offset..end] { + return Err("response body does not match the benchmark contract".into()); + } + offset = end; + } + if offset != expected.body.len() { + return Err("response body is shorter than the benchmark contract".into()); + } + Ok(()) +} + +struct Results { + http_version: HttpVersion, + shape: ConnectionShape, + successful_handshakes: u64, + scheduled: u64, + sent: u64, + completed: u64, + succeeded: u64, + errors: u64, + dropped: u64, + dropped_capacity: u64, + dispatch_expired: u64, + elapsed: Duration, + latency: Histogram, + dispatch_lag: Histogram, + request_latency: Histogram, +} + +impl Results { + fn new( + scheduled: u64, + http_version: HttpVersion, + shape: ConnectionShape, + successful_handshakes: u64, + ) -> Self { + Self { + http_version, + shape, + successful_handshakes, + scheduled, + sent: 0, + completed: 0, + succeeded: 0, + errors: 0, + dropped: 0, + dropped_capacity: 0, + dispatch_expired: 0, + elapsed: Duration::ZERO, + latency: Histogram::new(), + dispatch_lag: Histogram::new(), + request_latency: Histogram::new(), + } + } + + fn observe(&mut self, event: Event, ready: &mut VecDeque<usize>) { + match event { + Event::Ready(id) => ready.push_back(id), + Event::Complete(completion) => { + self.completed += 1; + self.successful_handshakes += completion.successful_handshakes; + self.dispatch_lag.record(completion.dispatch_lag_us); + if completion.succeeded { + self.succeeded += 1; + self.latency.record(completion.latency_us); + self.request_latency.record(completion.request_latency_us); + } else { + self.errors += 1; + } + } + } + } + + fn json(&self) -> String { + let elapsed = self.elapsed.as_secs_f64(); + let qps = if elapsed == 0.0 { + 0.0 + } else { + self.succeeded as f64 / elapsed + }; + let latency = percentiles_json(&self.latency); + let dispatch_lag = percentiles_json(&self.dispatch_lag); + let request_latency = percentiles_json(&self.request_latency); + format!( + concat!( + "{{\"http_version\":\"{}\",\"physical_connections\":{},", + "\"streams_per_connection\":{},\"request_slots\":{},", + "\"successful_handshakes\":{},", + "\"scheduled\":{},\"sent\":{},\"completed\":{},", + "\"succeeded\":{},\"errors\":{},\"dropped\":{},", + "\"dropped_capacity\":{},\"dispatch_expired\":{},", + "\"latency_samples\":{},\"dispatch_lag_samples\":{},", + "\"request_latency_samples\":{},", + "\"elapsed_seconds\":{:.6},\"actual_success_qps\":{:.3},", + "\"latency_ms\":{},\"dispatch_lag_ms\":{},", + "\"request_latency_ms\":{}}}" + ), + self.http_version.as_str(), + self.shape.physical_connections, + self.shape.streams_per_connection, + self.shape.request_slots, + self.successful_handshakes, + self.scheduled, + self.sent, + self.completed, + self.succeeded, + self.errors, + self.dropped, + self.dropped_capacity, + self.dispatch_expired, + self.latency.count(), + self.dispatch_lag.count(), + self.request_latency.count(), + elapsed, + qps, + latency, + dispatch_lag, + request_latency, + ) + } +} + +fn percentiles_json(histogram: &Histogram) -> String { + if histogram.count() == 0 { + return "null".into(); + } + format!( + "{{\"sample_count\":{},\"p50\":{:.3},\"p95\":{:.3},\"p99\":{:.3}}}", + histogram.count(), + histogram.percentile_ms(50), + histogram.percentile_ms(95), + histogram.percentile_ms(99), + ) +} + +fn micros(duration: Duration) -> u64 { + duration.as_micros().min(u128::from(u64::MAX)) as u64 +} + +fn number(value: &str, name: &str, minimum: u64, maximum: u64) -> Result<u64, String> { + let parsed = value + .parse::<u64>() + .map_err(|_| format!("{name} must be an integer"))?; + if !(minimum..=maximum).contains(&parsed) { + return Err(format!("{name} must be between {minimum} and {maximum}")); + } + Ok(parsed) +} + +fn decimal_duration(value: &str, name: &str, maximum: f64) -> Result<Duration, String> { + let seconds = value + .parse::<f64>() + .map_err(|_| format!("{name} must be a number"))?; + if !seconds.is_finite() || seconds <= 0.0 || seconds > maximum { + return Err(format!( + "{name} must be greater than 0 and at most {maximum}" + )); + } + Ok(Duration::from_secs_f64(seconds)) +} diff --git a/test/dever-tests/examples/network_bench/metrics.rs b/test/dever-tests/examples/network_bench/metrics.rs new file mode 100644 index 0000000..feb4ae6 --- /dev/null +++ b/test/dever-tests/examples/network_bench/metrics.rs @@ -0,0 +1,67 @@ +const EXACT_BUCKETS: usize = 1_024; +const SUB_BUCKETS: usize = 64; +const BUCKETS: usize = 4_096; + +pub struct Histogram { + buckets: Box<[u64; BUCKETS]>, + count: u64, +} + +impl Histogram { + pub fn new() -> Self { + Self { + buckets: Box::new([0; BUCKETS]), + count: 0, + } + } + + pub fn record(&mut self, microseconds: u64) { + self.buckets[index(microseconds)] += 1; + self.count += 1; + } + + pub fn count(&self) -> u64 { + self.count + } + + pub fn percentile_ms(&self, percentile: u64) -> f64 { + if self.count == 0 { + return 0.0; + } + let target = self.count.saturating_mul(percentile).div_ceil(100); + let mut observed = 0; + for (index, count) in self.buckets.iter().enumerate() { + observed += count; + if observed >= target { + return upper_bound(index) as f64 / 1_000.0; + } + } + upper_bound(BUCKETS - 1) as f64 / 1_000.0 + } +} + +fn index(microseconds: u64) -> usize { + if microseconds < EXACT_BUCKETS as u64 { + return microseconds as usize; + } + let exponent = 63 - microseconds.leading_zeros() as usize; + let lower = 1_u64 << exponent; + let step = (lower / SUB_BUCKETS as u64).max(1); + let sub = ((microseconds - lower) / step).min((SUB_BUCKETS - 1) as u64) as usize; + (EXACT_BUCKETS + (exponent - 10) * SUB_BUCKETS + sub).min(BUCKETS - 1) +} + +fn upper_bound(index: usize) -> u64 { + if index < EXACT_BUCKETS { + return index as u64; + } + let offset = index - EXACT_BUCKETS; + let exponent = 10 + offset / SUB_BUCKETS; + if exponent >= 63 { + return u64::MAX; + } + let sub = offset % SUB_BUCKETS; + let lower = 1_u64 << exponent; + let step = lower / SUB_BUCKETS as u64; + lower.saturating_add(step.saturating_mul(sub as u64 + 1)) +} diff --git a/test/dever-tests/examples/network_bench/response.rs b/test/dever-tests/examples/network_bench/response.rs new file mode 100644 index 0000000..86ff3fc --- /dev/null +++ b/test/dever-tests/examples/network_bench/response.rs @@ -0,0 +1,117 @@ +use bytes::Bytes; +use http_body_util::Full; +use hyper::header::{CONTENT_TYPE, HeaderValue}; +use hyper::{Response, StatusCode}; + +pub const PLAIN_PATH: &str = "/plain"; +pub const JSON_PATH: &str = "/json"; +pub const BYTES_PATH: &str = "/bytes"; +pub const ORM_COUNT_PATH: &str = "/benchmark/item/count"; + +const PLAIN_BODY: &[u8] = b"hello"; +const JSON_BODY: &[u8] = br#"{"ok":true}"#; + +#[derive(Clone, Debug)] +pub struct ExpectedResponse { + pub body: Bytes, + pub content_type: &'static str, +} + +impl ExpectedResponse { + pub fn for_path(path: &str, orm_count: Option<usize>) -> Result<Self, String> { + match path { + PLAIN_PATH => Ok(Self { + body: Bytes::from_static(PLAIN_BODY), + content_type: "text/plain", + }), + JSON_PATH => Ok(Self { + body: Bytes::from_static(JSON_BODY), + content_type: "application/json", + }), + BYTES_PATH => Ok(Self { + body: Bytes::from(vec![b'x'; 65_536]), + content_type: "application/octet-stream", + }), + ORM_COUNT_PATH => Ok(Self { + body: Bytes::from(format!( + "{{\"code\":0,\"message\":\"ok\",\"data\":{}}}", + orm_count.ok_or("benchmark.orm_count is required for ORM count requests")? + )), + content_type: "application/json; charset=utf-8", + }), + _ => Err(format!("unsupported benchmark path '{path}'")), + } + } +} + +pub struct HyperResponses { + bytes: Bytes, +} + +impl HyperResponses { + pub fn new() -> Self { + Self { + bytes: Bytes::from(vec![b'x'; 65_536]), + } + } + + pub fn response(&self, path: &str) -> Response<Full<Bytes>> { + let (body, content_type) = match path { + PLAIN_PATH => (Bytes::from_static(PLAIN_BODY), "text/plain"), + JSON_PATH => (Bytes::from_static(JSON_BODY), "application/json"), + BYTES_PATH => (self.bytes.clone(), "application/octet-stream"), + _ => { + let mut response = Response::new(Full::new(Bytes::new())); + *response.status_mut() = StatusCode::NOT_FOUND; + return response; + } + }; + let mut response = Response::new(Full::new(body)); + response + .headers_mut() + .insert(CONTENT_TYPE, HeaderValue::from_static(content_type)); + response + } +} + +#[derive(Clone)] +pub struct RuntimeResponses { + bytes: dever_runtime::bytes::Bytes, +} + +impl RuntimeResponses { + pub fn new() -> Self { + Self { + bytes: dever_runtime::bytes::Bytes::new(vec![b'x'; 65_536]), + } + } + + pub fn response(&self, path: &str) -> dever_runtime::http::Response { + let (body, content_type) = match path { + PLAIN_PATH => ( + dever_runtime::bytes::Bytes::from_text("hello"), + "text/plain", + ), + JSON_PATH => ( + dever_runtime::bytes::Bytes::from_text(r#"{"ok":true}"#), + "application/json", + ), + BYTES_PATH => (self.bytes.clone(), "application/octet-stream"), + _ => { + return dever_runtime::http::Response { + status: 404, + headers: vec![], + body: dever_runtime::bytes::Bytes::new(vec![]), + }; + } + }; + dever_runtime::http::Response { + status: 200, + headers: vec![dever_runtime::http::Header { + name: "content-type".into(), + value: dever_runtime::bytes::Bytes::from_text(content_type), + }], + body, + } + } +} diff --git a/test/dever-tests/examples/network_bench/service.rs b/test/dever-tests/examples/network_bench/service.rs new file mode 100644 index 0000000..d978f8c --- /dev/null +++ b/test/dever-tests/examples/network_bench/service.rs @@ -0,0 +1,369 @@ +use std::future::Future; +use std::io::{self, Write}; +use std::pin::Pin; +use std::sync::Arc; + +use bytes::{Bytes, BytesMut}; +use http_body_util::{BodyExt, Full}; +use hyper::body::Incoming; +use hyper::header::{CONNECTION, HOST, UPGRADE}; +use hyper::server::conn::{http1, http2}; +use hyper::service::service_fn; +use hyper::{Method, Request, Response, StatusCode, Version}; +use hyper_util::rt::{TokioExecutor, TokioIo, TokioTimer}; +use tokio::net::TcpListener; +use tokio::runtime::Builder; +use tokio::sync::{Semaphore, watch}; +use tokio::task::JoinSet; +use tokio::time::{Instant, timeout, timeout_at}; + +use crate::config::{ + BODY_BYTES, HEADER_BYTES, HTTP2_FRAME_BYTES, Http2Config, HttpVersion, ServiceConfig, + ServiceKind, +}; +use crate::{response, tls}; + +pub fn run(kind: ServiceKind) -> Result<(), String> { + let settings = crate::settings::Settings::load()?; + let config = ServiceConfig::from_settings(&settings, kind)?; + if kind.uses_runtime() { + run_runtime(config, kind) + } else { + run_hyper(config, kind) + } +} + +fn run_runtime(config: ServiceConfig, kind: ServiceKind) -> Result<(), String> { + let server_tls = config + .cert_path + .as_deref() + .zip(config.key_path.as_deref()) + .map(|(cert, key)| tls::runtime_server(cert, key)) + .transpose()?; + let runtime = dever_runtime::task::RuntimeConfig { + worker_threads: config.workers, + max_blocking_threads: 2, + task_capacity: config.task_capacity, + }; + dever_runtime::task::run_entry_with(runtime, async move { + let responses = response::RuntimeResponses::new(); + let listener = dever_runtime::net::listen("127.0.0.1", 0).await?; + let port = dever_runtime::net::port(&listener)?; + ready(u16::try_from(port).map_err(|_| "listener returned an invalid port")?)?; + let limits = dever_runtime::http::Limits { + header_bytes: HEADER_BYTES as i64, + body_bytes: BODY_BYTES as i64, + timeout_ms: config.timeout.as_millis() as i64, + connections: config.connections as i64, + http2: config.http2.map(|limits| dever_runtime::http::Http2Limits { + streams: limits.streams as i64, + stream_window_bytes: i64::from(limits.stream_window_bytes), + connection_window_bytes: i64::from(limits.connection_window_bytes), + }), + }; + let owned = listener.clone(); + let route = move |request: dever_runtime::http::Request| { + let response = responses.response(&request.target); + async move { Ok(response) } + }; + let server: Pin<Box<dyn Future<Output = Result<(), String>> + Send>> = match server_tls { + Some(tls) if kind.uses_tls() => { + Box::pin(dever_runtime::http::serve_tls(route, owned, limits, tls)) + } + _ => Box::pin(dever_runtime::http::serve(route, owned, limits)), + }; + tokio::pin!(server); + tokio::select! { + result = server.as_mut() => result, + _ = tokio::time::sleep(config.lifetime) => { + listener.close()?; + timeout(config.timeout, server.as_mut()) + .await + .map_err(|_| "runtime server shutdown timed out".to_owned())? + } + } + }) +} + +fn run_hyper(config: ServiceConfig, kind: ServiceKind) -> Result<(), String> { + let server_tls = config + .cert_path + .as_deref() + .zip(config.key_path.as_deref()) + .map(|(cert, key)| tls::server(cert, key, kind.http_version())) + .transpose()?; + let runtime = Builder::new_multi_thread() + .worker_threads(config.workers) + .max_blocking_threads(1) + .enable_io() + .enable_time() + .build() + .map_err(|error| format!("cannot start benchmark runtime: {error}"))?; + runtime.block_on(hyper_server( + config, + kind.http_version(), + kind.uses_tls().then_some(server_tls).flatten(), + )) +} + +async fn hyper_server( + config: ServiceConfig, + http_version: HttpVersion, + server_tls: Option<Arc<rustls::ServerConfig>>, +) -> Result<(), String> { + let responses = Arc::new(response::HyperResponses::new()); + let listener = TcpListener::bind(("127.0.0.1", 0)) + .await + .map_err(|error| error.to_string())?; + ready( + listener + .local_addr() + .map_err(|error| error.to_string())? + .port(), + )?; + + let connections = Arc::new(Semaphore::new(config.connections)); + let (shutdown, shutdown_receiver) = watch::channel(false); + let mut tasks = JoinSet::new(); + let deadline = Instant::now() + config.lifetime; + + loop { + while let Some(result) = tasks.try_join_next() { + result.map_err(|error| format!("connection task failed: {error}"))??; + } + let connections = connections.clone(); + let accepted = timeout_at(deadline, async { + let permit = connections + .acquire_owned() + .await + .map_err(|_| "connection limit closed".to_owned())?; + let (stream, _) = listener.accept().await.map_err(|error| error.to_string())?; + stream + .set_nodelay(true) + .map_err(|error| error.to_string())?; + Ok::<_, String>((permit, stream)) + }) + .await; + let Ok(accepted) = accepted else { + break; + }; + let (permit, stream) = accepted?; + let connection_config = config.clone(); + let connection_tls = server_tls.clone(); + let connection_shutdown = shutdown_receiver.clone(); + let connection_responses = responses.clone(); + tasks.spawn(async move { + let _permit = permit; + let stream = match tls::accept( + stream, + connection_config.timeout, + connection_tls.as_ref(), + http_version, + ) + .await + { + Ok(stream) => stream, + Err(_) => return Ok(()), + }; + serve_connection( + stream, + connection_config.timeout, + connection_shutdown, + connection_responses, + http_version, + connection_config.http2, + ) + .await + }); + } + + let _ = shutdown.send(true); + if timeout(config.timeout, drain_connections(&mut tasks)) + .await + .is_err() + { + tasks.abort_all(); + while tasks.join_next().await.is_some() {} + return Err("hyper server shutdown timed out".into()); + } + Ok(()) +} + +async fn serve_connection( + stream: tls::BoxedStream, + operation_timeout: std::time::Duration, + shutdown: watch::Receiver<bool>, + responses: Arc<response::HyperResponses>, + http_version: HttpVersion, + http2_config: Option<Http2Config>, +) -> Result<(), String> { + match http_version { + HttpVersion::H1 => serve_http1(stream, operation_timeout, shutdown, responses).await, + HttpVersion::H2 => { + let limits = http2_config.ok_or("HTTP/2 service has no HTTP/2 limits")?; + serve_http2(stream, operation_timeout, shutdown, responses, limits).await + } + } +} + +async fn serve_http1( + stream: tls::BoxedStream, + operation_timeout: std::time::Duration, + mut shutdown: watch::Receiver<bool>, + responses: Arc<response::HyperResponses>, +) -> Result<(), String> { + let service = service_fn(move |request| { + dispatch( + request, + operation_timeout, + responses.clone(), + HttpVersion::H1, + ) + }); + let mut builder = http1::Builder::new(); + builder + .timer(TokioTimer::new()) + .header_read_timeout(operation_timeout) + .max_buf_size(HEADER_BYTES); + let connection = builder.serve_connection(TokioIo::new(stream), service); + tokio::pin!(connection); + let result = tokio::select! { + result = connection.as_mut() => result, + _ = shutdown.changed() => { + connection.as_mut().graceful_shutdown(); + timeout(operation_timeout, connection.as_mut()) + .await + .map_err(|_| "connection shutdown timed out".to_owned())? + } + }; + // Peer disconnects and malformed requests end only their own connection. + let _ = result; + Ok(()) +} + +async fn serve_http2( + stream: tls::BoxedStream, + operation_timeout: std::time::Duration, + mut shutdown: watch::Receiver<bool>, + responses: Arc<response::HyperResponses>, + limits: Http2Config, +) -> Result<(), String> { + let service = service_fn(move |request| { + dispatch( + request, + operation_timeout, + responses.clone(), + HttpVersion::H2, + ) + }); + let mut builder = http2::Builder::new(TokioExecutor::new()); + builder + .timer(TokioTimer::new()) + .adaptive_window(false) + .initial_stream_window_size(limits.stream_window_bytes) + .initial_connection_window_size(limits.connection_window_bytes) + .max_concurrent_streams(limits.streams as u32) + .max_header_list_size(HEADER_BYTES as u32) + .max_frame_size(HTTP2_FRAME_BYTES) + .max_send_buf_size(HTTP2_FRAME_BYTES as usize); + let connection = builder.serve_connection(TokioIo::new(stream), service); + tokio::pin!(connection); + let result = tokio::select! { + result = connection.as_mut() => result, + _ = shutdown.changed() => { + connection.as_mut().graceful_shutdown(); + timeout(operation_timeout, connection.as_mut()) + .await + .map_err(|_| "connection shutdown timed out".to_owned())? + } + }; + // Peer disconnects and malformed requests end only their own connection. + let _ = result; + Ok(()) +} + +async fn drain_connections(tasks: &mut JoinSet<Result<(), String>>) -> Result<(), String> { + while let Some(result) = tasks.join_next().await { + result.map_err(|error| format!("connection task failed: {error}"))??; + } + Ok(()) +} + +async fn dispatch( + request: Request<Incoming>, + operation_timeout: std::time::Duration, + responses: Arc<response::HyperResponses>, + http_version: HttpVersion, +) -> Result<Response<Full<Bytes>>, std::convert::Infallible> { + let response = timeout( + operation_timeout, + read_and_route(request, responses, http_version), + ) + .await + .unwrap_or_else(|_| reject(StatusCode::REQUEST_TIMEOUT, http_version)); + Ok(response) +} + +async fn read_and_route( + request: Request<Incoming>, + responses: Arc<response::HyperResponses>, + http_version: HttpVersion, +) -> Response<Full<Bytes>> { + if request.method() == Method::CONNECT || request.headers().contains_key(UPGRADE) { + return reject(StatusCode::NOT_IMPLEMENTED, http_version); + } + if request.version() == Version::HTTP_11 && request.headers().get_all(HOST).iter().count() != 1 + { + return reject(StatusCode::BAD_REQUEST, http_version); + } + if header_size(&request) > HEADER_BYTES { + return reject(StatusCode::REQUEST_HEADER_FIELDS_TOO_LARGE, http_version); + } + let (head, mut body) = request.into_parts(); + let mut received = BytesMut::new(); + while let Some(frame) = body.frame().await { + let Ok(frame) = frame else { + return reject(StatusCode::BAD_REQUEST, http_version); + }; + if frame.is_trailers() { + return reject(StatusCode::NOT_IMPLEMENTED, http_version); + } + if let Ok(data) = frame.into_data() { + if received.len().saturating_add(data.len()) > BODY_BYTES { + return reject(StatusCode::PAYLOAD_TOO_LARGE, http_version); + } + received.extend_from_slice(&data); + } + } + responses.response(head.uri.path()) +} + +fn header_size(request: &Request<Incoming>) -> usize { + let start = request.method().as_str().len() + request.uri().to_string().len() + 14; + request + .headers() + .iter() + .fold(start, |total, (name, value)| { + total + .saturating_add(name.as_str().len()) + .saturating_add(value.as_bytes().len()) + .saturating_add(4) + }) +} + +fn reject(status: StatusCode, http_version: HttpVersion) -> Response<Full<Bytes>> { + let mut response = Response::new(Full::new(Bytes::new())); + *response.status_mut() = status; + if http_version == HttpVersion::H1 { + response + .headers_mut() + .insert(CONNECTION, hyper::header::HeaderValue::from_static("close")); + } + response +} + +fn ready(port: u16) -> Result<(), String> { + let mut output = io::stdout().lock(); + writeln!(output, "READY|{port}").map_err(|error| error.to_string())?; + output.flush().map_err(|error| error.to_string()) +} diff --git a/test/dever-tests/examples/network_bench/tls.rs b/test/dever-tests/examples/network_bench/tls.rs new file mode 100644 index 0000000..22508d8 --- /dev/null +++ b/test/dever-tests/examples/network_bench/tls.rs @@ -0,0 +1,128 @@ +use std::path::Path; +use std::sync::Arc; +use std::time::Duration; + +use rustls::pki_types::{CertificateDer, PrivateKeyDer, ServerName, pem::PemObject}; +use rustls::{ClientConfig, RootCertStore, ServerConfig}; +use tokio::io::{AsyncRead, AsyncWrite}; +use tokio::net::TcpStream; +use tokio_rustls::{TlsAcceptor, TlsConnector}; + +use crate::config::HttpVersion; + +pub trait Stream: AsyncRead + AsyncWrite {} +impl<T: AsyncRead + AsyncWrite> Stream for T {} + +pub type BoxedStream = Box<dyn Stream + Unpin + Send>; + +pub fn runtime_server( + cert_path: &str, + key_path: &str, +) -> Result<dever_runtime::tls::ServerTls, String> { + let certificate = std::fs::read(cert_path) + .map_err(|error| format!("cannot read certificate {cert_path}: {error}"))?; + let key = std::fs::read(key_path) + .map_err(|error| format!("cannot read private key {key_path}: {error}"))?; + dever_runtime::tls::server( + &dever_runtime::bytes::Bytes::new(certificate), + &dever_runtime::bytes::Bytes::new(key), + ) +} + +pub fn server( + cert_path: &str, + key_path: &str, + http_version: HttpVersion, +) -> Result<Arc<ServerConfig>, String> { + let certificates = certificates(cert_path)?; + let key_bytes = std::fs::read(key_path) + .map_err(|error| format!("cannot read private key {key_path}: {error}"))?; + let key = PrivateKeyDer::from_pem_slice(&key_bytes).map_err(|error| error.to_string())?; + let mut config = ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(certificates, key) + .map_err(|error| error.to_string())?; + config.session_storage = rustls::server::ServerSessionMemoryCache::new(32); + config.send_tls13_tickets = 1; + config.alpn_protocols = vec![http_version.alpn().to_vec()]; + Ok(Arc::new(config)) +} + +pub fn client(ca_path: &str, http_version: HttpVersion) -> Result<Arc<ClientConfig>, String> { + let mut roots = RootCertStore::empty(); + for certificate in certificates(ca_path)? { + roots.add(certificate).map_err(|error| error.to_string())?; + } + let mut config = ClientConfig::builder() + .with_root_certificates(roots) + .with_no_client_auth(); + config.resumption = rustls::client::Resumption::in_memory_sessions(32); + config.alpn_protocols = vec![http_version.alpn().to_vec()]; + Ok(Arc::new(config)) +} + +pub async fn connect( + port: u16, + timeout: Duration, + tls: Option<&Arc<ClientConfig>>, + http_version: HttpVersion, +) -> Result<BoxedStream, String> { + let tcp = tokio::time::timeout(timeout, TcpStream::connect(("127.0.0.1", port))) + .await + .map_err(|_| "TCP connection timed out".to_owned())? + .map_err(|error| error.to_string())?; + tcp.set_nodelay(true).map_err(|error| error.to_string())?; + let Some(config) = tls else { + return Ok(Box::new(tcp)); + }; + let name = ServerName::try_from("localhost").map_err(|error| error.to_string())?; + let stream = tokio::time::timeout( + timeout, + TlsConnector::from(config.clone()).connect(name, tcp), + ) + .await + .map_err(|_| "TLS handshake timed out".to_owned())? + .map_err(|error| error.to_string())?; + verify_alpn(stream.get_ref().1.alpn_protocol(), http_version)?; + Ok(Box::new(stream)) +} + +pub async fn accept( + tcp: TcpStream, + timeout: Duration, + tls: Option<&Arc<ServerConfig>>, + http_version: HttpVersion, +) -> Result<BoxedStream, String> { + let Some(config) = tls else { + return Ok(Box::new(tcp)); + }; + let stream = tokio::time::timeout(timeout, TlsAcceptor::from(config.clone()).accept(tcp)) + .await + .map_err(|_| "TLS handshake timed out".to_owned())? + .map_err(|error| error.to_string())?; + verify_alpn(stream.get_ref().1.alpn_protocol(), http_version)?; + Ok(Box::new(stream)) +} + +fn verify_alpn(negotiated: Option<&[u8]>, http_version: HttpVersion) -> Result<(), String> { + if http_version == HttpVersion::H2 && negotiated != Some(http_version.alpn()) { + return Err("TLS peer did not negotiate h2".into()); + } + Ok(()) +} + +fn certificates(path: impl AsRef<Path>) -> Result<Vec<CertificateDer<'static>>, String> { + let path = path.as_ref(); + let bytes = std::fs::read(path) + .map_err(|error| format!("cannot read certificate {}: {error}", path.display()))?; + let certificates = CertificateDer::pem_slice_iter(&bytes) + .collect::<Result<Vec<_>, _>>() + .map_err(|error| error.to_string())?; + if certificates.is_empty() { + return Err(format!( + "certificate {} contains no certificates", + path.display() + )); + } + Ok(certificates) +} diff --git a/test/dever-tests/examples/support/benchmark_settings.rs b/test/dever-tests/examples/support/benchmark_settings.rs new file mode 100644 index 0000000..e40bfbe --- /dev/null +++ b/test/dever-tests/examples/support/benchmark_settings.rs @@ -0,0 +1,80 @@ +use std::path::Path; + +use serde_json::{Map, Value}; + +pub struct Settings(Map<String, Value>); + +impl Settings { + pub fn load() -> Result<Self, String> { + let executable = std::env::current_exe() + .map_err(|error| format!("cannot locate benchmark executable: {error}"))?; + let root = executable + .parent() + .ok_or("benchmark executable has no parent")?; + Self::read(&root.join("config/setting.json")) + } + + pub fn check() -> Result<(), String> { + println!("{{\"workers\":{}}}", Self::load()?.workers()?); + Ok(()) + } + + pub fn read(path: &Path) -> Result<Self, String> { + let bytes = std::fs::read(path) + .map_err(|error| format!("cannot read {}: {error}", path.display()))?; + let mut document: Map<String, Value> = serde_json::from_slice(&bytes) + .map_err(|error| format!("invalid {}: {error}", path.display()))?; + let Some(Value::Object(fields)) = document.remove("benchmark") else { + return Err("config/setting.json must contain a benchmark object".into()); + }; + for key in fields.keys() { + if !matches!( + key.as_str(), + "workers" + | "connections" + | "task_capacity" + | "timeout_ms" + | "lifetime_ms" + | "http2_streams" + | "http2_stream_window_bytes" + | "http2_connection_window_bytes" + | "cert_path" + | "key_path" + | "orm_count" + ) { + return Err(format!("unknown benchmark setting '{key}'")); + } + } + Ok(Self(fields)) + } + + pub fn integer( + &self, + name: &str, + default: usize, + minimum: usize, + maximum: usize, + ) -> Result<usize, String> { + let value = match self.value(name) { + None => default, + Some(value) => value + .as_u64() + .and_then(|value| usize::try_from(value).ok()) + .ok_or_else(|| format!("benchmark.{name} must be an unsigned integer"))?, + }; + if !(minimum..=maximum).contains(&value) { + return Err(format!( + "benchmark.{name} must be between {minimum} and {maximum}" + )); + } + Ok(value) + } + + pub fn workers(&self) -> Result<usize, String> { + self.integer("workers", 1, 1, 64) + } + + pub fn value(&self, name: &str) -> Option<&Value> { + self.0.get(name) + } +} diff --git a/test/dever-tests/fixtures/command_fixture.rs b/test/dever-tests/fixtures/command_fixture.rs new file mode 100644 index 0000000..395249c --- /dev/null +++ b/test/dever-tests/fixtures/command_fixture.rs @@ -0,0 +1,101 @@ +//! Ordinary argv/stdin/stdout fixture: deliberately has no Dever protocol. +use std::io::{Read, Write}; +use std::path::Path; +use std::time::Duration; + +fn main() { + match run() { + Ok(code) => std::process::exit(code), + Err(error) => { + eprintln!("{error}"); + if matches!( + std::env::args().nth(1).as_deref(), + Some("tree" | "wait" | "heartbeat") + ) && let Some(directory) = std::env::args().nth(2) + { + let _ = std::fs::write( + Path::new(&directory).join("fixture-error"), + error.to_string(), + ); + } + std::process::exit(91); + } + } +} + +fn run() -> Result<i32, Box<dyn std::error::Error>> { + let mut arguments = std::env::args().skip(1); + match arguments.next().as_deref() { + Some("echo") => { + for argument in arguments { + std::io::stdout().write_all(argument.as_bytes())?; + std::io::stdout().write_all(&[0])?; + } + } + Some("stdin") => { + std::io::copy(&mut std::io::stdin(), &mut std::io::stdout())?; + std::io::stderr().write_all(&[0, 255, 128, 10])?; + } + Some("exit") => return Ok(arguments.next().ok_or("missing exit code")?.parse()?), + Some("fds") => { + let descriptors = std::fs::read_dir("/proc/self/fd")? + .map(|entry| entry.map(|entry| entry.path())) + .collect::<Result<Vec<_>, _>>()?; + for descriptor in descriptors { + let Some(number) = descriptor + .file_name() + .and_then(|name| name.to_str()) + .and_then(|name| name.parse::<u32>().ok()) + else { + continue; + }; + if number > 2 && std::fs::read_link(&descriptor).is_ok() { + writeln!(std::io::stdout(), "{number}")?; + } + } + } + Some("duplex") => { + let count: usize = arguments.next().ok_or("missing output length")?.parse()?; + let stdout = std::thread::spawn(move || std::io::stdout().write_all(&vec![254; count])); + let stderr = std::thread::spawn(move || std::io::stderr().write_all(&vec![253; count])); + let mut input = Vec::new(); + std::io::stdin().read_to_end(&mut input)?; + stdout.join().map_err(|_| "stdout thread panicked")??; + stderr.join().map_err(|_| "stderr thread panicked")??; + } + Some("wait") | Some("tree") => { + let directory = arguments.next().ok_or("missing marker directory")?; + let directory = Path::new(&directory); + let mut starts = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(directory.join("starts"))?; + starts.write_all(b"started\n")?; + // Every lifecycle case gets a real descendant. PID namespace exit + // must stop its heartbeat before the runtime returns cancellation. + let mut child = std::process::Command::new(std::env::current_exe()?) + .arg("heartbeat") + .arg(directory) + .spawn()?; + std::fs::write(directory.join("ready"), b"ready")?; + let status = child.wait()?; + return Ok(status.code().unwrap_or(92)); + } + Some("heartbeat") => { + let directory = arguments.next().ok_or("missing marker directory")?; + let lifetime = std::fs::OpenOptions::new() + .create(true) + .truncate(false) + .write(true) + .open(Path::new(&directory).join("lifetime.lock"))?; + lifetime.lock()?; + let marker = Path::new(&directory).join("heartbeat"); + for count in 0..3000 { + std::fs::write(&marker, count.to_string())?; + std::thread::sleep(Duration::from_millis(10)); + } + } + _ => return Err("unknown ordinary command mode".into()), + } + Ok(0) +} diff --git a/test/dever-tests/fixtures/component_fixture.rs b/test/dever-tests/fixtures/component_fixture.rs new file mode 100644 index 0000000..c2fa127 --- /dev/null +++ b/test/dever-tests/fixtures/component_fixture.rs @@ -0,0 +1,225 @@ +use std::io::{self, Read, Write}; + +use serde_json::{Value, json}; + +const MAX_BYTES: usize = 16 * 1024 * 1024; + +fn main() { + if std::env::args().skip(1).collect::<Vec<_>>() != ["--dever-component"] { + std::process::exit(2); + } + if let Err(error) = serve() { + eprintln!("component fixture: {error}"); + std::process::exit(1); + } +} + +fn serve() -> Result<(), String> { + let hello = read_json()?; + require_kind(&hello, "hello")?; + let mode = hello + .pointer("/setting/mode") + .and_then(Value::as_str) + .unwrap_or("normal"); + if mode == "delayed_start" { + mark(&hello, "starting")?; + std::thread::sleep(std::time::Duration::from_millis(100)); + } + let ready = json!({ + "kind": "ready", + "version": field(&hello, "version")?, + "port": field(&hello, "port")?, + "schema": field(&hello, "schema")?, + "adapter": field(&hello, "adapter")?, + "capabilities": field(&hello, "capabilities")?, + "operations": field(&hello, "operations")?, + }); + match mode { + "duplicate_ready" => write_raw(r#"{"kind":"ready","kind":"ready"}"#)?, + "capability_mismatch" => { + let mut mismatched = ready; + mismatched["capabilities"] = json!(["network", "process"]); + write_json(&mismatched)?; + } + _ => write_json(&ready)?, + } + if matches!(mode, "duplicate_ready" | "capability_mismatch") { + return Ok(()); + } + + let health = read_json()?; + require_kind(&health, "health")?; + write_json(&health)?; + eprintln!("fixture ready"); + + loop { + let message = read_json()?; + match string_field(&message, "kind")? { + "call" => handle_call(&message, mode, &hello)?, + "shutdown" => { + if mode == "delayed_shutdown" { + mark(&hello, "stopping")?; + std::thread::sleep(std::time::Duration::from_millis(100)); + } + write_json(&json!({"kind": "shutdown"}))?; + if mode != "embedded" { + mark(&hello, "shutdown")?; + } + return Ok(()); + } + kind => return Err(format!("unexpected {kind} message")), + } + } +} + +fn mark(hello: &Value, extension: &str) -> Result<(), String> { + let Some(marker) = hello.pointer("/setting/marker").and_then(Value::as_str) else { + return Ok(()); + }; + let path = std::path::Path::new(marker).with_extension(extension); + std::fs::write(path, b"ready").map_err(io_error) +} + +fn handle_call(call: &Value, mode: &str, hello: &Value) -> Result<(), String> { + let id = field(call, "id")?; + // Deliberately malformed business payloads exercise the caller's typed boundary. + let payload = match mode { + "missing_output" => Some(json!({})), + "extra_output" => Some(json!({"value": 7, "unknown": true})), + "wrong_output" => Some(json!({"value": "invalid number"})), + _ => None, + }; + if let Some(payload) = payload { + return write_json(&json!({"kind": "result", "id": id, "payload": payload})); + } + if matches!(mode, "declared_error" | "invalid_error" | "missing_error") { + let identity = hello + .pointer("/setting/identity") + .ok_or("missing error identity")?; + let payload = match mode { + "invalid_error" => json!({"reason": 7}), + "missing_error" => json!({}), + _ => json!({"reason": "fixture"}), + }; + return write_json( + &json!({"kind": "error", "id": id, "error": identity, "payload": payload}), + ); + } + match string_field(call, "operation")? { + "echo" => write_json(&json!({ + "kind": "result", + "id": id, + "payload": field(call, "payload")?, + })), + "read" => write_json(&json!({ + "kind": "result", + "id": id, + "payload": {"value": 7}, + })), + "fail" => write_json(&json!({ + "kind": "error", + "id": id, + "error": "notification.delivery.Rejected", + "payload": {"reason": "fixture"}, + })), + "wait" => { + let cancel = read_json()?; + require_kind(&cancel, "cancel")?; + if field(&cancel, "id")? != id { + return Err("cancel id does not match call".into()); + } + write_json(&json!({ + "kind": "error", + "id": id, + "error": "notification.delivery.Cancelled", + "payload": {}, + })) + } + "timeout" => { + let cancel = read_json()?; + require_kind(&cancel, "cancel")?; + if field(&cancel, "id")? != id { + return Err("cancel id does not match timed out call".into()); + } + write_json(&json!({"kind": "result", "id": id, "payload": {}})) + } + "crash" => std::process::exit(7), + "duplicate" => write_raw(&format!( + r#"{{"kind":"result","id":{id},"id":{id},"payload":{{}}}}"#, + id = id + )), + "out_of_order" => { + let id = id.as_u64().ok_or_else(|| "id is not u64".to_owned())?; + write_json(&json!({"kind": "result", "id": id + 1, "payload": {}})) + } + "unknown" => write_json(&json!({"kind": "surprise", "id": id})), + "oversize" => { + let mut stdout = io::stdout().lock(); + stdout + .write_all(&((MAX_BYTES as u32) + 1).to_be_bytes()) + .map_err(io_error)?; + stdout.flush().map_err(io_error)?; + std::process::exit(0); + } + "truncated" => { + let mut stdout = io::stdout().lock(); + stdout.write_all(&32_u32.to_be_bytes()).map_err(io_error)?; + stdout.write_all(b"{").map_err(io_error)?; + stdout.flush().map_err(io_error)?; + std::process::exit(0); + } + operation => Err(format!("unknown operation {operation}")), + } +} + +fn read_json() -> Result<Value, String> { + let text = read_frame()?; + serde_json::from_str(&text).map_err(|error| error.to_string()) +} + +fn read_frame() -> Result<String, String> { + let mut stdin = io::stdin().lock(); + let mut length = [0_u8; 4]; + stdin.read_exact(&mut length).map_err(io_error)?; + let length = u32::from_be_bytes(length) as usize; + if length == 0 || length > MAX_BYTES { + return Err("invalid frame length".into()); + } + let mut body = vec![0_u8; length]; + stdin.read_exact(&mut body).map_err(io_error)?; + String::from_utf8(body).map_err(|error| error.to_string()) +} + +fn write_json(value: &Value) -> Result<(), String> { + write_raw(&serde_json::to_string(value).map_err(|error| error.to_string())?) +} + +fn write_raw(body: &str) -> Result<(), String> { + let length = u32::try_from(body.len()).map_err(|_| "fixture frame is too large")?; + let mut stdout = io::stdout().lock(); + stdout.write_all(&length.to_be_bytes()).map_err(io_error)?; + stdout.write_all(body.as_bytes()).map_err(io_error)?; + stdout.flush().map_err(io_error) +} + +fn require_kind(message: &Value, expected: &str) -> Result<(), String> { + if string_field(message, "kind")? == expected { + Ok(()) + } else { + Err(format!("expected {expected} message")) + } +} + +fn string_field<'a>(message: &'a Value, name: &str) -> Result<&'a str, String> { + field(message, name)? + .as_str() + .ok_or_else(|| format!("{name} is not a string")) +} + +fn field<'a>(message: &'a Value, name: &str) -> Result<&'a Value, String> { + message.get(name).ok_or_else(|| format!("missing {name}")) +} + +fn io_error(error: io::Error) -> String { + error.to_string() +} diff --git a/test/dever-tests/fixtures/performance/benchmark/allocation/app.dever b/test/dever-tests/fixtures/performance/benchmark/allocation/app.dever new file mode 100644 index 0000000..2b0f56f --- /dev/null +++ b/test/dever-tests/fixtures/performance/benchmark/allocation/app.dever @@ -0,0 +1,95 @@ +type Buffer { + text: Text + count: Int +} + +type Index { + values: Map<Int, Int> + next: Int +} + +double(value: Int, text: Text) (next: Text) { + next = text + text +} + +grow(value: Int, state: Buffer) (next: Buffer) { + next = state + next.text = state.text + "a" + next.count = state.count + 1 +} + +insert(value: Int, state: Index) (next: Index) { + next = state + next.values = put(state.values, state.next, value) + next.next = state.next + 1 +} + +word(value: Int) (result: Text) { + result = "abcdefgh你好" +} + +copy(value: Text) (result: Text) { + result = value +} + +clock(value: dever.time.ClockResult.Read(nanos)) (result: Int) recover("benchmark clock failures are reported") { + result = nanos +} + +clock(value: dever.time.ClockResult.Failed(message)) (result: Int) recover("benchmark clock failures are reported") { + dever.io.println("FAILED:" + message) + result = 0 +} + +json(value: dever.json.value.ParseResult.Parsed(document)) (result: Int) recover("benchmark parse failures are reported") { + result = length(document.nodes) +} + +json(value: dever.json.value.ParseResult.Failed(position, message)) (result: Int) recover("benchmark parse failures are reported") { + dever.io.println("FAILED:" + message) + result = 0 +} + +sample(index: Int, bytes: Bytes) () { + start = clock(result(dever.time.monotonic_nanos())) + buffer = reduce( + grow, + bytes, + Buffer { + text = "" + count = 0 + } + ) + text_end = clock(result(dever.time.monotonic_nanos())) + mapped = each(copy, each(word, bytes)) + list_end = clock(result(dever.time.monotonic_nanos())) + document = json(result(dever.json.parse("\"" + buffer.text + "\""))) + json_end = clock(result(dever.time.monotonic_nanos())) + dever.io.println( + "text|" + int.to_text(buffer.count) + "|" + int.to_text(text_end - start) + ) + dever.io.println( + "list|" + int.to_text(length(mapped)) + "|" + int.to_text(list_end - text_end) + ) + dever.io.println( + "json|" + int.to_text(document) + "|" + int.to_text(json_end - list_end) + ) +} + +run() () { + text = reduce(double, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "a") + each(sample, [0, 1, 2, 3, 4, 5], dever.bytes.from_text(text)) + start = clock(result(dever.time.monotonic_nanos())) + index = reduce( + insert, + dever.bytes.from_text(text), + Index { + values = {} + next = 0 + } + ) + end = clock(result(dever.time.monotonic_nanos())) + dever.io.println( + "map|" + int.to_text(length(index.values)) + "|" + int.to_text(end - start) + ) +} diff --git a/test/dever-tests/fixtures/performance/main.dever b/test/dever-tests/fixtures/performance/main.dever new file mode 100644 index 0000000..06fda55 --- /dev/null +++ b/test/dever-tests/fixtures/performance/main.dever @@ -0,0 +1,3 @@ +main() () { + benchmark.allocation.run() +} diff --git a/test/dever-tests/fixtures/postgres_api/account_api.dever b/test/dever-tests/fixtures/postgres_api/account_api.dever new file mode 100644 index 0000000..7fe701e --- /dev/null +++ b/test/dever-tests/fixtures/postgres_api/account_api.dever @@ -0,0 +1 @@ +cmd fixture_member = app.fixture_member diff --git a/test/dever-tests/fixtures/postgres_api/account_app.dever b/test/dever-tests/fixtures/postgres_api/account_app.dever new file mode 100644 index 0000000..157302c --- /dev/null +++ b/test/dever-tests/fixtures/postgres_api/account_app.dever @@ -0,0 +1,27 @@ +fixture_member( + selected_tenant_id: platform.tenant.model.id, + email: Text, + display_name: Text, + password: Secret +) (result: BootstrapView) { + input = domain.prepare(email, display_name) + hash = blocking(dever.crypto.password_hash(password)) + principal = model.principal.create( + { + display_name = input.name + } + ) + stored = model.create( + { + principal_id = principal.id + email = input.address + password_hash = hash + } + ) + user.membership.fixture_join(principal.id, selected_tenant_id) + result = BootstrapView { + tenant_id = selected_tenant_id + user_id = principal.id + credential_id = stored.id + } +} diff --git a/test/dever-tests/fixtures/postgres_api/membership_app.dever b/test/dever-tests/fixtures/postgres_api/membership_app.dever new file mode 100644 index 0000000..a58dfd6 --- /dev/null +++ b/test/dever-tests/fixtures/postgres_api/membership_app.dever @@ -0,0 +1,13 @@ +fixture_join( + account_id: user.account.model.principal.id, + selected_tenant_id: platform.tenant.model.id +) () { + stored = model.create( + { + user_id = account_id + tenant_id = selected_tenant_id + role = model.MembershipRole.Reader + } + ) + discarded = stored +} diff --git a/test/dever-tests/fixtures/tls/root.pem b/test/dever-tests/fixtures/tls/root.pem new file mode 100644 index 0000000..43e3be1 --- /dev/null +++ b/test/dever-tests/fixtures/tls/root.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBmTCCAT+gAwIBAgIUYx/Ad4tJiQ4uakAPbpXu5Senv2YwCgYIKoZIzj0EAwIw +GjEYMBYGA1UEAwwPRGV2ZXItVGVzdC1Sb290MB4XDTI2MDkxMjA4MTY1MFoXDTM2 +MDkwOTA4MTY1MFowGjEYMBYGA1UEAwwPRGV2ZXItVGVzdC1Sb290MFkwEwYHKoZI +zj0CAQYIKoZIzj0DAQcDQgAECOux92VpsFwb2YOTRs6UATfp7HPm+JeCTMkJNXPo +rqLZa4Vi/71dLQzNFxxsXxqFyf4oBYoCD0XMEicxYUgnO6NjMGEwHQYDVR0OBBYE +FIo/Vdu7qm5dIikkwg6A0yRp1YzVMB8GA1UdIwQYMBaAFIo/Vdu7qm5dIikkwg6A +0yRp1YzVMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49 +BAMCA0gAMEUCICmmB/eG055dU7F9OP7K9BUxOkFnYNtzIpYJfd0VwG6LAiEA4oiP +pnep4mLGHAnvsucvvqDA5SVdHzYh22OYPdSgNeY= +-----END CERTIFICATE----- diff --git a/test/dever-tests/fixtures/tls/server-key.pem b/test/dever-tests/fixtures/tls/server-key.pem new file mode 100644 index 0000000..62956e6 --- /dev/null +++ b/test/dever-tests/fixtures/tls/server-key.pem @@ -0,0 +1,5 @@ +-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgYl2wcdhzZMZrQxSV +ClBRv8AQqRyuJf5bSVXw5w6XLWmhRANCAAROwX6ZplJMfQicbaDXSZ7qoLPGDdr6 +EMtvQvlJdwVI44/NVX/lIS2dhF0h1GxKAippeSiLj9+Srh2mmg7+zEvK +-----END PRIVATE KEY----- diff --git a/test/dever-tests/fixtures/tls/server.pem b/test/dever-tests/fixtures/tls/server.pem new file mode 100644 index 0000000..e09c2bd --- /dev/null +++ b/test/dever-tests/fixtures/tls/server.pem @@ -0,0 +1,10 @@ +-----BEGIN CERTIFICATE----- +MIIBdjCCARugAwIBAgIBATAKBggqhkjOPQQDAjAaMRgwFgYDVQQDDA9EZXZlci1U +ZXN0LVJvb3QwHhcNMjYwOTEyMDgxNjUwWhcNMzYwOTA5MDgxNjUwWjAUMRIwEAYD +VQQDDAlsb2NhbGhvc3QwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAROwX6ZplJM +fQicbaDXSZ7qoLPGDdr6EMtvQvlJdwVI44/NVX/lIS2dhF0h1GxKAippeSiLj9+S +rh2mmg7+zEvKo1gwVjAUBgNVHREEDTALgglsb2NhbGhvc3QwHQYDVR0OBBYEFEc4 +R7JUtHLl16fHXxBvGDz0qwnKMB8GA1UdIwQYMBaAFIo/Vdu7qm5dIikkwg6A0yRp +1YzVMAoGCCqGSM49BAMCA0kAMEYCIQCzl7jB3XrkRoQXTXMoi9j1OzmCPO3UXLr5 +eDq/F8G/twIhALgVU/5C4hx6hpVtlHUmd49EibizuPlNeaCjRPB1WqDM +-----END CERTIFICATE----- diff --git a/test/dever-tests/tests/api_context.rs b/test/dever-tests/tests/api_context.rs new file mode 100644 index 0000000..11f0c6e --- /dev/null +++ b/test/dever-tests/tests/api_context.rs @@ -0,0 +1,250 @@ +use dever_runtime::api::{self, CookieOptions, SameSite}; +use dever_runtime::bytes::Bytes; +use dever_runtime::http::{Header, Request}; +use dever_runtime::{http, net, task}; +use std::time::Duration; + +fn request(path: &str, cookie: &str) -> Request { + Request { + method: "GET".into(), + target: path.into(), + headers: vec![Header { + name: "cookie".into(), + value: Bytes::from_text(cookie), + }], + body: Bytes::from_text(""), + } +} + +fn cookie_options() -> CookieOptions { + CookieOptions { + path: "/".into(), + domain: None, + same_site: SameSite::Lax, + secure: true, + http_only: true, + max_age: Some(3600), + } +} + +#[tokio::test] +async fn request_scope_is_isolated_and_merges_validated_response_metadata() { + let first = api::scoped_request( + request("/first?ignored=1", "session=v1.Zmlyc3Q; theme=dark"), + |_| async { + assert_eq!(api::method().unwrap(), "GET"); + assert_eq!(api::path().unwrap(), "/first"); + assert!(api::request_id().unwrap().parse::<u64>().is_ok()); + assert!(api::header("cookie").is_err()); + assert!(api::header("x-forwarded-for").is_err()); + assert_eq!(api::cookie("theme").unwrap().as_deref(), Some("dark")); + assert!(api::cookie("session").is_err()); + assert_eq!(api::cookie("missing").unwrap(), None); + let incoming = api::secret_cookie("session").unwrap().unwrap(); + assert_eq!(format!("{incoming:?}"), "Secret([REDACTED])"); + api::response_header("x-request-state", "first").unwrap(); + api::response_secret_cookie( + "session", + dever_runtime::secret::Secret::from_input(vec![0, 255, 0]), + cookie_options(), + ) + .unwrap(); + api::response_secret_cookie("copy", incoming, cookie_options()).unwrap(); + api::response_cookie("theme", "dark", cookie_options()).unwrap(); + let mut insecure = cookie_options(); + insecure.secure = false; + assert!( + api::response_secret_cookie( + "session", + dever_runtime::secret::Secret::from_input(b"unsafe".to_vec()), + insecure, + ) + .is_err() + ); + api::commit_response_metadata().unwrap(); + Ok(api::success(dever_runtime::wire::Encoded::null())) + }, + ); + let second = api::scoped_request(request("/second", "session=v1.c2Vjb25k"), |_| async { + assert_eq!(api::path().unwrap(), "/second"); + assert!(api::response_header("set-cookie", "bypass").is_err()); + assert!(api::response_header("content-type", "text/html").is_err()); + assert!(api::response_header("x-unsafe", "injected\r\nheader").is_err()); + api::response_header("x-request-state", "second").unwrap(); + api::commit_response_metadata().unwrap(); + Ok(api::success(dever_runtime::wire::Encoded::null())) + }); + let (first, second) = tokio::join!(first, second); + let first = first.unwrap(); + let second = second.unwrap(); + assert!( + first + .headers + .iter() + .any(|header| header.name == "x-request-state" && header.value.values() == b"first") + ); + assert!( + second + .headers + .iter() + .any(|header| header.name == "x-request-state" && header.value.values() == b"second") + ); + assert!(first.headers.iter().any(|header| { + header.name == "set-cookie" + && header + .value + .values() + .starts_with(b"session=v1.AP8A; Path=/") + })); + assert!(first.headers.iter().any(|header| { + header.name == "set-cookie" + && header + .value + .values() + .starts_with(b"copy=v1.Zmlyc3Q; Path=/") + })); + assert!( + first + .headers + .iter() + .any(|header| header.name == "set-cookie" + && header.value.values().starts_with(b"theme=dark; Path=/")) + ); + assert!( + !second + .headers + .iter() + .any(|header| header.name == "set-cookie") + ); + assert!(api::request_id().is_err()); +} + +#[tokio::test] +async fn malformed_or_duplicate_cookie_is_a_bad_request_before_handler_execution() { + for cookie in ["session=one; session=two", "session=one; broken"] { + let result = api::scoped_request(request("/profile", cookie), |_| async { + Err("malformed Cookie reached the API handler".into()) + }) + .await + .unwrap(); + assert_eq!(result.status, 400); + } +} + +#[tokio::test] +async fn quoted_cookie_value_uses_the_same_secret_codec() { + let response = api::scoped_request(request("/profile", "session=\"v1.Zmlyc3Q\""), |_| async { + let token = api::secret_cookie("session")?.ok_or("missing quoted session")?; + api::response_secret_cookie("copy", token, cookie_options())?; + api::commit_response_metadata()?; + Ok(api::success(dever_runtime::wire::Encoded::null())) + }) + .await + .unwrap(); + assert!(response.headers.iter().any(|header| { + header.name == "set-cookie" + && header + .value + .values() + .starts_with(b"copy=v1.Zmlyc3Q; Path=/") + })); +} + +#[tokio::test] +async fn uncommitted_response_metadata_is_discarded_on_failure() { + let response = api::scoped_request(request("/profile", ""), |_| async { + api::response_header("x-staged", "discarded")?; + Ok(api::internal_error()) + }) + .await + .unwrap(); + assert_eq!(response.status, 500); + assert!( + !response + .headers + .iter() + .any(|header| header.name == "x-staged") + ); +} + +#[test] +fn cookie_and_model_text_validation_reject_invalid_values() { + assert!(api::text_bounds("name", Some(""), 1, Some(64)).is_err()); + assert!(api::text_bounds("name", Some("abc"), 1, Some(2)).is_err()); + assert!(api::text_bounds("name", None, 1, Some(2)).is_ok()); + assert!(api::response_cookie("session", "a;b", cookie_options()).is_err()); + assert!(api::response_cookie("session", "valid", cookie_options()).is_err()); +} + +#[test] +fn client_address_comes_from_the_tcp_peer_not_forwarded_headers() { + let limits = http::Limits { + header_bytes: 8192, + body_bytes: 1024, + timeout_ms: 1000, + connections: 2, + http2: None, + }; + task::run_entry_with( + task::RuntimeConfig { + worker_threads: 1, + max_blocking_threads: 2, + task_capacity: 16, + }, + async { + tokio::time::timeout(Duration::from_secs(8), async { + let listener = net::listen("127.0.0.1", 0).await?; + let port = net::port(&listener)?; + let owned = listener.clone(); + let server = task::run(async move { + http::serve( + |request| { + api::scoped_request(request, |_| async { + let peer = + api::client_address()?.ok_or("missing TCP peer address")?; + let mut body = dever_runtime::wire::Encoder::default(); + body.text(&peer)?; + Ok(api::success(body.finish()?)) + }) + }, + owned, + limits, + ) + .await + }) + .await?; + let response = http::send( + "127.0.0.1", + port, + Request { + method: "GET".into(), + target: "/peer".into(), + headers: vec![ + Header { + name: "host".into(), + value: Bytes::from_text("localhost"), + }, + Header { + name: "x-forwarded-for".into(), + value: Bytes::from_text("203.0.113.99"), + }, + ], + body: Bytes::new(vec![]), + }, + limits, + ) + .await?; + assert_eq!(response.status, 200); + let envelope: serde_json::Value = serde_json::from_slice(response.body.values()) + .map_err(|error| error.to_string())?; + assert_eq!(envelope["data"], "127.0.0.1"); + task::stop(server).await?; + listener.close()?; + Ok::<(), String>(()) + }) + .await + .map_err(|_| "API peer test deadline exceeded".to_owned())? + }, + ) + .unwrap(); +} diff --git a/test/dever-tests/tests/api_declarations.rs b/test/dever-tests/tests/api_declarations.rs new file mode 100644 index 0000000..19f0c85 --- /dev/null +++ b/test/dever-tests/tests/api_declarations.rs @@ -0,0 +1,933 @@ +#[path = "support/temp.rs"] +mod temp; + +use std::fs; + +use dever_core::source::SourceMap; +use dever_runtime::config::Settings; + +fn sources(files: &[(&str, &str)]) -> SourceMap { + let mut sources = SourceMap::default(); + for (path, source) in files { + sources.add(*path, *source); + } + sources +} + +fn check(files: &[(&str, &str)]) { + let sources = sources(files); + dever_core::check(&sources).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }); +} + +fn reject(files: &[(&str, &str)], expected: &str) { + let sources = sources(files); + let errors = dever_core::check(&sources).expect_err("source must be rejected"); + let rendered = errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>(); + assert!( + rendered.contains(expected), + "expected {expected}: {rendered}" + ); +} + +fn auth_settings() -> (temp::TemporaryDirectory, Settings) { + let root = temp::TemporaryDirectory::new(); + fs::create_dir(root.path().join("config")).unwrap(); + fs::write( + root.path().join("config/setting.json"), + r#"{ + "auth": { + "providers": { + "session": { + "verify": "user.account.verify", + "jwtSecret": "0123456789abcdef0123456789abcdef" + } + } + }, + "sites": { + "admin": {"path": "admin", "auth": "session"} + }, + "database": { + "default": { + "type": "sqlite", + "path": "data/control.db", + "tenant_directory": "data/tenants" + } + }, + "tenant": { + "database": "default" + } + }"#, + ) + .unwrap(); + let settings = Settings::load_project(root.path()).unwrap(); + (root, settings) +} + +fn multi_provider_settings() -> (temp::TemporaryDirectory, Settings) { + let (root, _) = auth_settings(); + let setting_path = root.path().join("config/setting.json"); + let mut settings: serde_json::Value = + serde_json::from_str(&fs::read_to_string(&setting_path).unwrap()).unwrap(); + settings["auth"]["providers"]["other"] = serde_json::json!({ + "verify": "admin.principal.verify", + "jwtSecret": "0123456789abcdef0123456789abcdef", + }); + settings["sites"]["other"] = serde_json::json!({"path": "other", "auth": "other"}); + fs::write(&setting_path, settings.to_string()).unwrap(); + let settings = Settings::load_project(root.path()).unwrap(); + (root, settings) +} + +fn identity_sources() -> [(&'static str, &'static str); 3] { + [ + ( + "user/account/model.dever", + "global type Account { email: Text(1, 254) }", + ), + ( + "platform/tenant/model.dever", + "global type Tenant { name: Text(1, 64) }", + ), + ( + "user/account/app.dever", + r#"type Identity { + id: Text + user_id: user.account.model.id? + tenant_id: platform.tenant.model.id? +} +verify(claims: dever.auth.Claims) (identity: Identity) { + identity = Identity { + id = claims.subject + user_id = null + tenant_id = null + } +} +profile() (response: Text) { + response = dever.site.key() +}"#, + ), + ] +} + +fn configured_check( + files: &[(&str, &str)], + settings: &Settings, +) -> Result<dever_core::hir::Program, String> { + let sources = sources(files); + dever_core::check_with_settings(&sources, settings).map_err(|errors| { + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + }) +} + +#[test] +fn api_bindings_inherit_app_signatures_without_response_name_requirement() { + check(&[ + ( + "user/account/app.dever", + "greet(name: Text) (greeting: Text) { greeting = text.trim(name) }", + ), + ( + "user/account/api.dever", + "get greeting = app.greet\npost greeting = app.greet\nput greeting = app.greet\ndelete greeting = app.greet\ncmd greeting = app.greet", + ), + ]); +} + +#[test] +fn configured_sites_bind_nested_api_directories_to_checked_auth_hooks() { + let (_root, settings) = auth_settings(); + let mut files = identity_sources().to_vec(); + files.push(( + "user/account/api/admin/profile.dever", + "public get profile = app.profile", + )); + + let source_map = sources(&files); + let program = + dever_core::check_with_settings(&source_map, &settings).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&source_map)) + .collect::<String>() + ) + }); + let generated = dever_core::native::emit_project( + &program, + &source_map, + dever_runtime::config::RuntimeProfile::default(), + ) + .unwrap(); + assert!(generated.contains("dever_runtime::auth::prepare(&[\"admin\"]")); + assert!(generated.contains("dever_runtime::auth::scope(auth_prepared, auth_identity")); +} + +#[test] +fn configured_sites_reject_unmapped_api_directories_and_invalid_verify_records() { + let (_root, settings) = auth_settings(); + let mut unmapped = identity_sources().to_vec(); + unmapped.push(( + "user/account/api/front/profile.dever", + "get profile = app.profile", + )); + let error = configured_check(&unmapped, &settings).unwrap_err(); + assert!( + error.contains("not assigned to a configured site"), + "{error}" + ); + + let invalid_app = identity_sources()[2] + .1 + .replace("tenant_id: platform.tenant.model.id?", "tenant_id: Int?"); + let mut invalid = identity_sources().to_vec(); + invalid[2] = ("user/account/app.dever", &invalid_app); + invalid.push(( + "user/account/api/admin/profile.dever", + "get profile = app.profile", + )); + let error = configured_check(&invalid, &settings).unwrap_err(); + assert!( + error.contains("identity must be an App-owned public record"), + "{error}" + ); +} + +#[test] +fn verify_is_read_only() { + let (_root, settings) = auth_settings(); + let writing_verify = identity_sources()[2].1.replace( + "identity = Identity {", + "account = model.create({ email = claims.subject })\n identity = Identity {", + ); + let mut writing = identity_sources().to_vec(); + writing[2] = ("user/account/app.dever", &writing_verify); + writing.push(( + "user/account/api/admin/profile.dever", + "get profile = app.profile", + )); + let error = configured_check(&writing, &settings).unwrap_err(); + assert!( + error.contains("authentication verify cannot write"), + "{error}" + ); +} + +#[test] +fn trusted_identity_getters_preserve_nominal_ids_and_nullable_representation() { + let (_root, settings) = auth_settings(); + let mut files = identity_sources().to_vec(); + files.extend([ + ( + "news/reader/app.dever", + r#" +type Profile { + user: user.account.model.id? + tenant: platform.tenant.model.id? + raw: Int? +} +profile() (profile: Profile) { + profile = Profile { + user = dever.auth.user_id() + tenant = dever.auth.tenant_id() + raw = dever.auth.user_id() + } +} +current_user() (user: user.account.model.id) { + user = require_user(dever.auth.user_id()) +} +require_user(value: user.account.model.id) (user: user.account.model.id) { + user = value +} +require_user(value: null) (user: user.account.model.id) { + fail(dever.api.Error.Unauthorized) +} +owns(id: user.account.model.id) (owned: Bool) { + owned = dever.auth.owns_user(id) +} +"#, + ), + ( + "news/reader/api/admin/profile.dever", + "get profile = app.profile\nget current = app.current_user\nget owns = app.owns", + ), + ]); + let source_map = sources(&files); + dever_core::check(&source_map).expect("source-only checking defers provider binding"); + let program = + dever_core::check_with_settings(&source_map, &settings).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&source_map)) + .collect::<String>() + ) + }); + let native = dever_core::native::emit_project( + &program, + &source_map, + dever_runtime::config::RuntimeProfile { + sqlite: true, + postgres: false, + }, + ) + .unwrap(); + assert!(native.contains("dever_runtime::auth::user_id().map(|value| value.map(T")); + assert!(native.contains("dever_runtime::auth::tenant_id().map(|value| value.map(T")); + let llvm = dever_core::llvm::emit_application(&program, &source_map).unwrap(); + assert!(llvm.contains("@dever_rt_v1_auth_user_id")); + assert!(llvm.contains("@dever_rt_v1_auth_tenant_id")); +} + +#[test] +fn trusted_identity_getters_reject_wrong_ids_and_implicit_casts() { + let (_root, settings) = auth_settings(); + for (app, message) in [ + ( + "profile() (value: platform.tenant.model.id?) { value = dever.auth.user_id() }", + "nominal ID must match provider", + ), + ( + "profile() (value: user.account.model.id?) { value = dever.auth.tenant_id() }", + "nominal ID must match provider", + ), + ( + "profile(id: platform.tenant.model.id) (owned: Bool) { owned = dever.auth.owns_user(id) }", + "nominal ID must match provider", + ), + ( + "profile() (value: user.account.model.id) { value = dever.auth.user_id() }", + "C005", + ), + ( + "profile(raw: Int?) (value: user.account.model.id?) { value = raw }", + "C005", + ), + ( + "profile() (value: user.account.model.id?) { value = 1 }", + "C005", + ), + ( + "profile() (value: user.account.model.id?) { raw = dever.auth.user_id()\nvalue = raw }", + "C005", + ), + ] { + let mut files = identity_sources().to_vec(); + files.extend([ + ("news/reader/app.dever", app), + ( + "news/reader/api/admin/profile.dever", + "get profile = app.profile", + ), + ]); + let error = configured_check(&files, &settings).unwrap_err(); + assert!(error.contains(message), "{app}: {error}"); + } +} + +#[test] +fn trusted_identity_getters_require_matching_providers_at_every_route() { + let (_root, settings) = multi_provider_settings(); + let other_provider = identity_sources()[2] + .1 + .replace("user.account.model.id?", "admin.principal.model.id?"); + for app in [ + "profile() (value: user.account.model.id?) { value = dever.auth.user_id() }", + "profile(id: user.account.model.id) (owned: Bool) { owned = dever.auth.owns_user(id) }", + ] { + let mut files = identity_sources().to_vec(); + files.extend([ + ( + "admin/principal/model.dever", + "global type Principal { name: Text }", + ), + ("admin/principal/app.dever", other_provider.as_str()), + ("news/reader/app.dever", app), + ( + "news/reader/api/admin/profile.dever", + "get profile = app.profile", + ), + ( + "news/reader/api/other/profile.dever", + "get profile = app.profile", + ), + ]); + let error = configured_check(&files, &settings).unwrap_err(); + assert!( + error.contains("C006") && error.contains("provider 'other'"), + "{error}" + ); + } +} + +#[test] +fn rest_identity_bindings_preserve_matching_nominal_ids() { + let (_root, settings) = auth_settings(); + let mut files = identity_sources().to_vec(); + files.extend([ + ( + "news/article/model.dever", + r#"type Article { + private owner author: user.account.model.id = dever.auth.user_id() + title: Text + linked_user: user.account.model.id? create = dever.auth.user_id() replace = dever.auth.user_id() + linked_tenant: platform.tenant.model.id? create = dever.auth.tenant_id() replace = dever.auth.tenant_id() +}"#, + ), + ("news/article/api/admin/manage.dever", "rest model"), + ]); + check(&files); + let program = configured_check(&files, &settings).unwrap(); + let source_map = sources(&files); + let native = dever_core::native::emit_project( + &program, + &source_map, + dever_runtime::config::RuntimeProfile { + sqlite: true, + postgres: false, + }, + ) + .unwrap(); + assert!(native.contains("dever_runtime::auth::user_id().map(|value| value.map(T")); + assert!(native.contains("dever_runtime::auth::tenant_id().map(|value| value.map(T")); + let llvm = dever_core::llvm::emit_application(&program, &source_map).unwrap(); + assert!(llvm.contains("@dever_rt_v1_auth_user_id")); + assert!(llvm.contains("@dever_rt_v1_auth_tenant_id")); +} + +#[test] +fn rest_identity_bindings_reject_wrong_provider_ids() { + let (_root, settings) = auth_settings(); + for field in [ + "linked: platform.tenant.model.id? create = dever.auth.user_id()", + "linked: platform.tenant.model.id? replace = dever.auth.user_id()", + "linked: user.account.model.id? create = dever.auth.tenant_id()", + "linked: user.account.model.id? replace = dever.auth.tenant_id()", + "private owner linked: platform.tenant.model.id = dever.auth.user_id()", + "private owner linked: user.account.model.id = dever.auth.tenant_id()", + ] { + let model = format!("type Article {{\n title: Text\n {field}\n}}"); + let mut files = identity_sources().to_vec(); + files.extend([ + ("news/article/model.dever", model.as_str()), + ("news/article/api/admin/manage.dever", "rest model"), + ]); + check(&files); + let error = configured_check(&files, &settings).unwrap_err(); + assert!( + error.contains("C006") && error.contains("provider 'session'"), + "{field}: {error}" + ); + } +} + +#[test] +fn rest_identity_bindings_match_each_sites_provider() { + let (_root, settings) = multi_provider_settings(); + let other_provider = identity_sources()[2] + .1 + .replace("user.account.model.id?", "admin.principal.model.id?"); + for field in [ + "linked: user.account.model.id? create = dever.auth.user_id()", + "linked: user.account.model.id? replace = dever.auth.user_id()", + "private owner linked: user.account.model.id = dever.auth.user_id()", + ] { + let model = format!("type Article {{\n title: Text\n {field}\n}}"); + let mut files = identity_sources().to_vec(); + files.extend([ + ( + "admin/principal/model.dever", + "global type Principal { name: Text }", + ), + ("admin/principal/app.dever", other_provider.as_str()), + ("news/article/model.dever", model.as_str()), + ("news/article/api/admin/manage.dever", "rest model"), + ]); + configured_check(&files, &settings).expect("first site's provider matches"); + files.push(("news/article/api/other/manage.dever", "rest model")); + let error = configured_check(&files, &settings).unwrap_err(); + assert!( + error.contains("C006") && error.contains("provider 'other'"), + "{field}: {error}" + ); + } +} + +#[test] +fn nominal_identity_getters_do_not_gain_public_cmd_or_job_context() { + let (_root, settings) = auth_settings(); + let app = "profile() (value: user.account.model.id?) { value = dever.auth.user_id() }"; + for (path, declaration, message) in [ + ( + "news/reader/api/admin/profile.dever", + "public get profile = app.profile", + "requires a protected API", + ), + ( + "news/reader/api.dever", + "cmd profile = app.profile", + "not available from CMD or Job", + ), + ( + "news/reader/job.dever", + "database default\njob sample() () retry(3) timeout(1000) { value = app.profile() }", + "not available from CMD or Job", + ), + ] { + let mut files = identity_sources().to_vec(); + files.extend([("news/reader/app.dever", app), (path, declaration)]); + let error = configured_check(&files, &settings).unwrap_err(); + assert!(error.contains("C006") && error.contains(message), "{error}"); + } +} + +#[test] +fn nominal_identity_getters_remain_unavailable_to_ordinary_tests() { + let (_root, settings) = auth_settings(); + let mut files = identity_sources().to_vec(); + files.push(( + "news/reader/app.dever", + "profile() (value: user.account.model.id?) { value = dever.auth.user_id() }", + )); + let mut inputs = sources(&files).compilation_sources().unwrap(); + inputs.push(dever_core::source::CompilationSource { + logical_path: "news/reader/probe.dever".into(), + is_test: true, + text: "probe() () { value = app.profile() }".into(), + }); + let source_map = SourceMap::from_compilation_sources(&inputs).unwrap(); + let errors = dever_core::check_with_settings(&source_map, &settings).unwrap_err(); + let rendered = errors + .iter() + .map(|error| error.render(&source_map)) + .collect::<String>(); + assert!( + rendered.contains("C006") && rendered.contains("not available from tests"), + "{rendered}" + ); +} + +#[test] +fn protected_api_permissions_are_stable_and_unique_across_paths_and_methods() { + let (_root, settings) = auth_settings(); + let mut files = identity_sources().to_vec(); + files.push(( + "user/account/api/admin/profile.dever", + "get profile = app.profile", + )); + let program = configured_check(&files, &settings).unwrap(); + assert!(program.api_snapshot().contains( + "permission user.account.admin.profile method=GET component=user domain=account site=admin action=profile" + )); + + files.push(( + "user/account/api/admin/alternate.dever", + "post profile = app.profile", + )); + let error = configured_check(&files, &settings).unwrap_err(); + assert!( + error.contains("duplicate API permission 'user.account.admin.profile'"), + "{error}" + ); +} + +#[test] +fn public_visibility_and_identity_stay_on_http_app_bindings() { + reject( + &[ + ( + "user/account/app.dever", + "profile() (response: Text) { response = \"ok\" }", + ), + ( + "user/account/api/admin/profile.dever", + "public cmd profile = app.profile", + ), + ], + "application visibility is defined by the source role", + ); + reject( + &[( + "user/account/domain.dever", + "profile() (response: Text) { response = dever.auth.id() }", + )], + "available only from App functions", + ); + reject( + &[ + ( + "user/account/app.dever", + "manage() (ok: Bool) { dever.auth.disable_role(\"reader\")\nok = true }", + ), + ( + "user/account/api/admin/manage.dever", + "public post manage = app.manage", + ), + ], + "public API call chains cannot administer authorization", + ); +} + +#[test] +fn public_api_cannot_reach_a_tenant_component() { + let (_root, settings) = auth_settings(); + let mut files = identity_sources().to_vec(); + files.extend([ + ( + "news/article/model.dever", + "type Article { title: Text(1, 160) }", + ), + ( + "news/article/app.dever", + "count() (value: Int) { value = model.count() }", + ), + ( + "news/article/api/admin/browse.dever", + "public get count = app.count", + ), + ]); + let error = configured_check(&files, &settings).unwrap_err(); + assert!( + error.contains("public API cannot reach tenant components [news]"), + "{error}" + ); +} + +#[test] +fn json_bindings_accept_wire_records_but_query_bindings_require_scalars() { + let app = "type Input { name: Text }\nupdate(input: Input) (value: Text) { value = text.trim(input.name) }"; + check(&[ + ("user/account/app.dever", app), + ( + "user/account/api.dever", + "put profile = app.update\ncmd update = app.update", + ), + ]); + reject( + &[ + ("user/account/app.dever", app), + ("user/account/api.dever", "get profile = app.update"), + ], + "wire-safe scalars", + ); +} + +#[test] +fn api_bindings_reject_legacy_bodies_cross_domain_and_ambiguous_targets() { + reject( + &[( + "user/account/api.dever", + "get_name() (response: Text) { response = \"x\" }", + )], + "handler function bodies and legacy prefixes", + ); + reject( + &[("user/account/api.dever", "post save = user.profile.save")], + "same domain", + ); + reject( + &[("user/account/api.dever", "post save = app.missing")], + "unambiguous same-domain", + ); + reject( + &[ + ( + "user/account/app.dever", + "save() (value: Text) { value = \"x\" }\nsave(name: Text) (value: Text) { value = text.trim(name) }", + ), + ("user/account/api.dever", "post save = app.save"), + ], + "unambiguous same-domain", + ); +} + +#[test] +fn duplicate_routes_and_commands_are_rejected() { + let app = "greet() (value: Text) { value = \"hello\" }"; + for (declaration, diagnostic) in [ + ("get hello = app.greet", "duplicate HTTP route"), + ("cmd hello = app.greet", "duplicate command"), + ] { + let api = format!("{declaration}\n{declaration}"); + reject( + &[ + ("user/account/app.dever", app), + ("user/account/api.dever", &api), + ], + diagnostic, + ); + } + reject( + &[ + ("user/account/app.dever", app), + ("user/account/api.dever", "cmd hello = app.greet"), + ( + "user/account/api/maintenance.dever", + "cmd hello = app.greet", + ), + ], + "duplicate command", + ); +} + +fn sql_projection_files(sqlite: &str, postgres: &str, method: &str) -> [(&'static str, String); 3] { + [ + ( + "news/article/model.dever", + format!( + "type Article {{\n name: Text\n private draft: Bool default true\n}}\n\ + sql lookup(name: Text, size: Int) (rows: List<app.View>) {{\n\ + sqlite = {sqlite:?}\n postgres = {postgres:?}\n}}\n" + ), + ), + ( + "news/article/app.dever", + "type View { name: Text }\n\ + read(name: Text, size: Int) (rows: List<View>) {\n\ + rows = model.lookup(name, size)\n}\n" + .into(), + ), + ( + "news/article/api.dever", + format!("{method} read = app.read\n"), + ), + ] +} + +#[test] +fn get_accepts_only_proven_single_model_sql_projections() { + for sqlite in [ + "SELECT name FROM article WHERE name = ?1 ORDER BY id DESC LIMIT ?2", + "select \"name\" from \"article\" where \"name\" = ?1 AND name = ?1 order by id desc, name asc limit ?2;", + ] { + let postgres = sqlite.replace('?', "$"); + let files = sql_projection_files(sqlite, &postgres, "get"); + check( + &files + .iter() + .map(|(path, source)| (*path, source.as_str())) + .collect::<Vec<_>>(), + ); + } +} + +#[test] +fn unproven_sql_keeps_its_write_effect_in_each_dialect() { + let plain = "SELECT name FROM article WHERE name = ?1 ORDER BY id DESC LIMIT ?2"; + for unproven in [ + "SELECT name FROM other_article WHERE name = ?1 ORDER BY id DESC LIMIT ?2", + "SELECT missing FROM article WHERE name = ?1 ORDER BY id DESC LIMIT ?2", + "SELECT lower(name) FROM article WHERE name = ?1 ORDER BY id DESC LIMIT ?2", + "WITH changed AS (UPDATE article SET name = ?1 WHERE id = ?2 RETURNING name) SELECT name FROM changed", + "SELECT name FROM article WHERE name = ?1 LIMIT ?2; DELETE FROM article", + "UPDATE article SET name = ?1 WHERE id = ?2 RETURNING name", + "SELECT name INTO copied_article FROM article WHERE name = ?1 LIMIT ?2", + "SELECT name FROM article WHERE name = ?1 LIMIT ?2 FOR UPDATE", + "SELECT name FROM article WHERE name = ?1 LIMIT ?2 -- unproved comment", + ] { + let plain_postgres = plain.replace('?', "$"); + let unproven_postgres = unproven.replace('?', "$"); + for (sqlite, postgres) in [ + (unproven, plain_postgres.as_str()), + (plain, unproven_postgres.as_str()), + ] { + let files = sql_projection_files(sqlite, postgres, "get"); + reject( + &files + .iter() + .map(|(path, source)| (*path, source.as_str())) + .collect::<Vec<_>>(), + "GET App call chains cannot write", + ); + let files = sql_projection_files(sqlite, postgres, "post"); + check( + &files + .iter() + .map(|(path, source)| (*path, source.as_str())) + .collect::<Vec<_>>(), + ); + } + } +} + +#[test] +fn get_rejects_writes_reached_through_cross_domain_app_calls() { + let writer = "create(name: Text) (value: Text) { account = model.create({ name = text.trim(name) })\nvalue = account.name }"; + reject( + &[ + ( + "user/account/model.dever", + "type Account { name: Text(1, 64) }", + ), + ("user/account/app.dever", writer), + ( + "user/report/app.dever", + "greet(name: Text) (value: Text) { value = \"hello \" + user.account.create(name) }", + ), + ("user/report/api.dever", "get greeting = app.greet"), + ], + "GET App call chains cannot write", + ); + check(&[ + ( + "user/account/model.dever", + "type Account { name: Text(1, 64) }", + ), + ("user/account/app.dever", writer), + ("user/account/api.dever", "post create = app.create"), + ]); + check(&[ + ( + "user/account/model.dever", + "type Account { name: Text(1, 64) }", + ), + ( + "user/account/app.dever", + "count() (value: Text) { value = int.to_text(model.count()) }", + ), + ("user/account/api.dever", "get count = app.count"), + ]); + reject( + &[ + ( + "user/account/app.dever", + "disable() (ok: Bool) { dever.auth.disable_role(\"reader\")\nok = true }", + ), + ("user/account/api.dever", "get disable = app.disable"), + ], + "GET App call chains cannot write database or authorization state", + ); +} + +#[test] +fn rest_requires_unambiguous_same_domain_model_and_rejects_duplicate_registration() { + let model = "type Account { name: Text(1, 64) }"; + check(&[ + ("user/account/model.dever", model), + ("user/account/api.dever", "rest"), + ]); + check(&[ + ("user/account/model.dever", model), + ("user/account/api.dever", "rest model"), + ]); + reject( + &[("user/account/api.dever", "rest")], + "requires one same-domain Model", + ); + reject( + &[ + ("user/account/model.dever", model), + ("user/account/api.dever", "rest\nrest"), + ], + "duplicate HTTP route", + ); +} + +#[test] +fn rest_selector_disambiguates_main_and_topic_models() { + let main = ( + "user/account/model.dever", + "type Account { name: Text(1, 64) }", + ); + let detail = ( + "user/account/model/detail.dever", + "type Detail { title: Text(1, 64) }", + ); + reject( + &[main, detail, ("user/account/api.dever", "rest")], + "requires one same-domain Model", + ); + check(&[ + main, + detail, + ("user/account/api.dever", "rest model.detail"), + ]); + let alternate = ( + "user/account/model/archive.dever", + "type Archive { name: Text(1, 64) }", + ); + reject( + &[detail, alternate, ("user/account/api.dever", "rest")], + "requires one same-domain Model", + ); + reject( + &[main, ("user/account/api.dever", "rest user.profile.model")], + "requires one same-domain Model", + ); +} + +#[test] +fn generated_rest_and_custom_bindings_cannot_register_the_same_route() { + reject( + &[ + ( + "user/account/model.dever", + "type Account { name: Text(1, 64) }", + ), + ( + "user/account/app.dever", + "details() (value: Text) { value = \"details\" }", + ), + ("user/account/api.dever", "get details = app.details"), + ("user/account/api/details.dever", "rest model"), + ], + "duplicate HTTP route", + ); +} + +#[test] +fn api_declarations_format_idempotently() { + let mut sources = SourceMap::default(); + sources.add("user/account/api.dever", "# Actions\npost publish=app.publish # publish article\ncmd rebuild =app.rebuild\nrest model\n"); + let source = sources.files().first().unwrap(); + let formatted = dever_core::format::format(source).unwrap(); + assert!(formatted.contains("post publish = app.publish")); + assert!(formatted.contains("cmd rebuild = app.rebuild")); + assert!(formatted.contains("rest model")); + assert!(formatted.contains("# Actions")); + assert!(formatted.contains("# publish article")); + let mut roundtrip = SourceMap::default(); + roundtrip.add("user/account/api.dever", formatted.clone()); + assert_eq!( + dever_core::format::format(roundtrip.files().first().unwrap()).unwrap(), + formatted + ); +} + +#[test] +fn markdown_api_declarations_share_plain_source_contracts() { + let markdown = "# 账户接口\n\n绑定账户业务能力。\n\n- 包：`user.account.api`\n- 公开类型：无\n- 公开方法：无\n- 使用：无\n\n## 问候\n\n提供账户问候接口。\n\n- 声明：`get greeting`\n\n```dever\nget greeting = app.greet\n```\n"; + check(&[ + ( + "user/account/app.dever", + "greet() (value: Text) { value = \"hello\" }", + ), + ("user/account/api.dever.md", markdown), + ]); + let invalid = markdown.replace("`get greeting`", "`post greeting`"); + reject( + &[ + ( + "user/account/app.dever", + "greet() (value: Text) { value = \"hello\" }", + ), + ("user/account/api.dever.md", &invalid), + ], + "documented Model declaration", + ); +} diff --git a/test/dever-tests/tests/api_routes.rs b/test/dever-tests/tests/api_routes.rs new file mode 100644 index 0000000..180cab7 --- /dev/null +++ b/test/dever-tests/tests/api_routes.rs @@ -0,0 +1,241 @@ +#[path = "support/temp.rs"] +mod temp; + +use std::fs; + +use dever_core::source::SourceMap; +use dever_runtime::api::{self, Inputs}; +use dever_runtime::bytes::Bytes; +use dever_runtime::config::Settings; +use dever_runtime::http::{Header, Request}; +use temp::TemporaryDirectory; + +fn sources(main: &str, api_source: &str) -> SourceMap { + let mut sources = SourceMap::default(); + sources.add("main.dever", main); + sources.add("user/api/hello.dever", api_source); + sources.add( + "user/service/greeting.dever", + "public type Greeting { message: Text }\n\ + public greet(name: Text) (greeting: Greeting) {\n\ + greeting = Greeting { message = \"hello, \" + text.trim(name) }\n\ + }\n", + ); + sources +} + +fn diagnostics(sources: &SourceMap) -> String { + dever_core::check(sources) + .expect_err("source must be rejected") + .iter() + .map(|error| error.render(sources)) + .collect() +} + +#[test] +fn explicit_serve_compiles_recursive_module_routes_and_json_adapter() { + let mut sources = sources( + "main() () { dever.api.serve() }", + "get_greeting(name: Text) (response: user.service.greeting.Greeting) {\n\ + response = user.service.greeting.greet(name)\n\ + }\n\ + post_greeting(name: Text) (response: user.service.greeting.Greeting) {\n\ + response = user.service.greeting.greet(name)\n\ + }", + ); + sources.add( + "user/api/admin/health.dever", + "get_status() (response: Text) { response = \"ready\" }", + ); + let program = dever_core::check(&sources).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }); + let generated = dever_core::native::emit(&program, &sources, "main.main").unwrap(); + assert!(generated.contains("(\"GET\", \"/user/hello/greeting\")")); + assert!(generated.contains("(\"POST\", \"/user/hello/greeting\")")); + assert!(generated.contains("(\"GET\", \"/user/admin/health/status\")")); + assert!(generated.contains("dever_runtime::api::serve(api_route).await")); + assert!(generated.contains("inputs.finish()")); + assert!(generated.contains("dever_runtime::api::success(value)")); +} + +#[test] +fn generated_api_application_compiles_without_starting_a_listener() { + let sources = sources( + "main() () { dever.api.serve() }", + "get_greeting(name: Text) (response: user.service.greeting.Greeting) {\n\ + response = user.service.greeting.greet(name)\n\ + }\n\ + post_greeting(name: Text) (response: user.service.greeting.Greeting) {\n\ + response = user.service.greeting.greet(name)\n\ + }\n\ + delete_greeting(name: Text) (response: Text) { response = name }", + ); + let program = dever_core::check(&sources).unwrap(); + let rustc = std::env::var_os("RUSTC").unwrap_or_else(|| "rustc".into()); + let native = dever_core::native::compile_application( + &program, + &sources, + "main.main", + &rustc, + dever_runtime::config::RuntimeProfile::default(), + ) + .unwrap(); + assert!(native.executable().is_file()); +} + +#[test] +fn without_serve_api_sources_are_checked_but_not_emitted() { + let checked_sources = sources( + "main() (answer: Int) { answer = 42 }", + "get_greeting(name: Text) (response: user.service.greeting.Greeting) {\n\ + response = user.service.greeting.greet(name)\n\ + }", + ); + let program = dever_core::check(&checked_sources).unwrap(); + let generated = dever_core::native::emit(&program, &checked_sources, "main.main").unwrap(); + assert!(!generated.contains("fn api_route(")); + assert!(!generated.contains("dever_runtime::api::serve")); + let invalid = sources( + "main() () {}", + "fetch_greeting() (response: Text) { response = \"x\" }", + ); + assert!(diagnostics(&invalid).contains("API handler names must start with")); +} + +#[test] +fn api_only_accepts_private_http_handlers_with_response_output() { + for (source, expected) in [ + ( + "public get_ok() (response: Text) { response = \"x\" }", + "private ordinary", + ), + ( + "post_create() (result: Text) { result = \"x\" }", + "named response", + ), + ( + "delete_thing(file: dever.system.File) (response: Text) { response = \"x\" }", + "API inputs must", + ), + ( + "get_limit(value: Int >= 1) (response: Int) { response = value }", + "API inputs must be plain", + ), + ( + "type Temporary { value: Text }", + "only HTTP handler functions", + ), + ] { + let errors = diagnostics(&sources("main() () {}", source)); + assert!(errors.contains(expected), "{source}: {errors}"); + } +} + +#[test] +fn query_and_json_body_bind_the_same_typed_parameters() { + let request = Request { + method: "GET".into(), + target: "/user/hello/greeting?name=%E4%B8%AD%E6%96%87&count=12&active=true".into(), + headers: vec![], + body: Bytes::from_text(""), + }; + let mut inputs = Inputs::from_request(&request).unwrap(); + assert_eq!(inputs.text("name").unwrap(), "中文"); + assert_eq!(inputs.int("count").unwrap(), 12); + assert!(inputs.boolean("active").unwrap()); + inputs.finish().unwrap(); + + let request = Request { + method: "POST".into(), + target: "/user/hello/greeting".into(), + headers: vec![Header { + name: "content-type".into(), + value: Bytes::from_text("application/json"), + }], + body: Bytes::from_text("{\"name\":\"Ada\",\"count\":2,\"active\":false}"), + }; + let mut inputs = Inputs::from_request(&request).unwrap(); + assert_eq!(inputs.text("name").unwrap(), "Ada"); + assert_eq!(inputs.int("count").unwrap(), 2); + assert!(!inputs.boolean("active").unwrap()); + inputs.finish().unwrap(); +} + +#[test] +fn malformed_or_extra_inputs_get_structured_errors() { + for target in [ + "/user/hello?name=x&name=y", + "/user/hello?name=%GG", + "/user/hello?name=x&extra=1", + ] { + let request = Request { + method: "GET".into(), + target: target.into(), + headers: vec![], + body: Bytes::from_text(""), + }; + let parsed = Inputs::from_request(&request); + if let Ok(mut inputs) = parsed { + inputs.text("name").unwrap(); + assert!(inputs.finish().is_err()); + } + } + let response = api::invalid_input(api::InputError("bad value".into())); + assert_eq!(response.status, 400); + let body: api::Value = serde_json::from_slice(response.body.values()).unwrap(); + assert_eq!(body["code"], 400); + assert_eq!(body["message"], "bad value"); + assert!(body["data"].is_null()); + + for (status, message) in [ + (400, "invalid request"), + (401, "unauthorized"), + (403, "forbidden"), + (404, "not found"), + (409, "conflict"), + (429, "too many requests"), + ] { + let response = api::standard_error(status); + assert_eq!(response.status, status); + let body: api::Value = serde_json::from_slice(response.body.values()).unwrap(); + assert_eq!(body["code"], status); + assert_eq!(body["message"], message); + assert!(body["data"].is_null()); + } +} + +#[test] +fn listener_settings_are_required_only_for_api_and_come_from_project_config() { + let directory = TemporaryDirectory::new(); + fs::create_dir(directory.path().join("config")).unwrap(); + let setting = directory.path().join("config/setting.json"); + fs::write(&setting, "{}\n").unwrap(); + let settings = Settings::load_project(directory.path()).unwrap(); + assert!( + settings + .http() + .unwrap_err() + .contains("http.host and http.port") + ); + + fs::write(&setting, r#"{"http":{"host":"127.0.0.1","port":8181}}"#).unwrap(); + let settings = Settings::load_project(directory.path()).unwrap(); + let http = settings.http().unwrap(); + assert_eq!(http.host, "127.0.0.1"); + assert_eq!(http.port, 8181); + + fs::write(&setting, r#"{"http":{"host":"example.com","port":8181}}"#).unwrap(); + assert!( + Settings::load_project(directory.path()) + .unwrap() + .http() + .is_err() + ); +} diff --git a/test/dever-tests/tests/application_config.rs b/test/dever-tests/tests/application_config.rs new file mode 100644 index 0000000..ceb442f --- /dev/null +++ b/test/dever-tests/tests/application_config.rs @@ -0,0 +1,426 @@ +mod support; + +use std::fs; + +use dever_runtime::config::{Database, LogLevel, PostgresTls, Settings}; + +const MIXED_SETTING: &str = r#"{ + "http": {}, + "performance": { + "network_peer": "target/network_bench", + "live_peer": "target/live_bench", + "linux_ci": {"inputs": [], "required_inputs": {}, "author_outputs": []} + }, + "database": { + "default": { + "type": "sqlite", + "path": "data/db/cms.db", + "max_connections": 1, + "max_page_size": 100 + }, + "report": { + "type": "postgres", + "url": "postgres://example.invalid/report", + "tls": "system", + "min_connections": 0, + "max_connections": 4, + "max_page_size": 200 + } + }, + "log": {} +}"#; + +fn project(setting: &str) -> support::temp::TemporaryDirectory { + let root = support::temp::TemporaryDirectory::new(); + fs::create_dir(root.path().join("config")).unwrap(); + fs::write(root.path().join("config/setting.json"), setting).unwrap(); + root +} + +#[cfg(feature = "external")] +#[test] +fn worker_file_grants_are_explicit_and_cannot_write_executable_cache() { + let setting = |path: &str, write: bool| { + serde_json::json!({ + "adapter": {"sample.worker": {"files": {"content": {"path": path, "write": write}}}} + }) + .to_string() + }; + let capabilities = vec!["file".into()]; + let root = project(&setting("data/upload", true)); + let settings = Settings::load_project(root.path()).unwrap(); + let grants = settings + .external_grants("sample.worker", &capabilities) + .unwrap(); + assert_eq!(grants.len(), 1); + assert_eq!(grants[0].source, root.path().join("data/upload")); + assert_eq!(grants[0].destination, std::path::Path::new("/data/content")); + assert!(grants[0].writable); + assert!( + settings + .external_grants("sample.worker", &[]) + .unwrap_err() + .contains("file capability") + ); + for path in [ + "data", + "data/cache", + "data/cache/lib", + "data/cache/lib/worker", + ] { + let root = project(&setting(path, true)); + let settings = Settings::load_project(root.path()).unwrap(); + assert!( + settings + .external_grants("sample.worker", &capabilities) + .unwrap_err() + .contains("executable resource cache") + ); + } + for path in ["../private", "/etc", ""] { + let root = project(&setting(path, false)); + let settings = Settings::load_project(root.path()).unwrap(); + assert!( + settings + .external_grants("sample.worker", &capabilities) + .unwrap_err() + .contains("relative to the application") + ); + } +} + +#[test] +fn settings_resolve_connections_and_paths_from_the_application_directory() { + let root = project(MIXED_SETTING); + let settings = Settings::load_project(root.path()).unwrap(); + assert_eq!(settings.resolve_database(None, "cms").unwrap().0, "default"); + assert_eq!( + settings.resolve_database(None, "report").unwrap().0, + "report" + ); + assert!(settings.resolve_database(Some("missing"), "cms").is_err()); + match settings.database("default").unwrap() { + Database::Sqlite { + path, + max_connections, + max_page_size, + .. + } => { + assert_eq!(path, &root.path().join("data/db/cms.db")); + assert_eq!((*max_connections, *max_page_size), (1, 100)); + } + Database::Postgres { .. } => panic!("default must be SQLite"), + } + match settings.database("report").unwrap() { + Database::Postgres { + tls, + wait_timeout_ms, + io_timeout_ms, + .. + } => { + assert_eq!(*tls, PostgresTls::System); + assert_eq!((*wait_timeout_ms, *io_timeout_ms), (5_000, 30_000)); + } + Database::Sqlite { .. } => panic!("report must be PostgreSQL"), + } + assert!(settings.profile().sqlite); + assert!(settings.profile().postgres); + assert!( + settings + .validate_transaction_bindings(&[&[(None, "cms"), (None, "news")]]) + .is_ok() + ); + let error = settings + .validate_transaction_bindings(&[&[(None, "cms"), (Some("report"), "cms")]]) + .unwrap_err(); + assert!(error.contains("multiple database connections"), "{error}"); + + settings.initialize_data_directories().unwrap(); + for directory in ["db", "upload", "log", "cache", "tmp"] { + assert!(root.path().join("data").join(directory).is_dir()); + } +} + +#[test] +fn tenant_database_settings_use_only_controlled_physical_targets() { + let root = project( + r#"{ + "tenant": { + "database": "default", + "max_pools": 8, + "idle_timeout_ms": 60000 + }, + "database": { + "default": { + "type": "sqlite", + "path": "data/platform.db", + "tenant_directory": "data/tenants" + }, + "content": { + "type": "postgres", + "url": "postgres://example.invalid/platform", + "tls": "disabled", + "tenant_database_prefix": "cms_tenant" + }, + "platform": { + "type": "sqlite", + "path": "data/platform_only.db" + } + } + }"#, + ); + let settings = Settings::load_project(root.path()).unwrap(); + let tenant = settings.tenant().unwrap(); + assert_eq!(tenant.database(), "default"); + assert_eq!(tenant.max_pools(), 8); + assert_eq!(tenant.idle_timeout_ms(), 60_000); + match settings.database("default").unwrap() { + Database::Sqlite { + tenant_directory, .. + } => { + assert_eq!( + tenant_directory.as_ref().unwrap(), + &root.path().join("data/tenants") + ); + } + Database::Postgres { .. } => panic!("default must be SQLite"), + } + match settings.database("content").unwrap() { + Database::Postgres { + tenant_database_prefix, + .. + } => { + assert_eq!(tenant_database_prefix.as_deref(), Some("cms_tenant")); + } + Database::Sqlite { .. } => panic!("content must be PostgreSQL"), + } + + settings + .validate_scoped_database_bindings( + &[(Some("content"), "news", true), (None, "platform", false)], + &[], + ) + .unwrap(); + let error = settings + .validate_scoped_database_bindings(&[(Some("platform"), "news", true)], &[]) + .unwrap_err(); + assert!(error.contains("tenant_directory is required"), "{error}"); + + let transaction = [ + (Some("default"), "news", false), + (Some("default"), "news", true), + ]; + let error = settings + .validate_scoped_database_bindings(&[], &[&transaction]) + .unwrap_err(); + assert!( + error.contains("cannot mix global and tenant Models"), + "{error}" + ); +} + +#[test] +fn final_connection_resolution_rejects_table_collisions_and_cross_database_foreign_keys() { + let root = project(MIXED_SETTING); + let settings = Settings::load_project(root.path()).unwrap(); + + let mut collision_sources = dever_core::source::SourceMap::default(); + collision_sources.add( + "user/model/account.dever", + "public type Account { email: Text }", + ); + collision_sources.add( + "news/model/account.dever", + "public type Account { slug: Text }", + ); + let collision = dever_core::check(&collision_sources).unwrap(); + let error = collision.validate_database_settings(&settings).unwrap_err(); + assert!(error.contains("share table 'account'"), "{error}"); + + let mut foreign_key_sources = dever_core::source::SourceMap::default(); + foreign_key_sources.add( + "user/model/user.dever", + "database report\npublic type User { email: Text }", + ); + foreign_key_sources.add( + "news/model/news.dever", + "public type News { author_id: user.model.user.id }", + ); + let foreign_key = dever_core::check(&foreign_key_sources).unwrap(); + let error = foreign_key + .validate_database_settings(&settings) + .unwrap_err(); + assert!(error.contains("crosses database connections"), "{error}"); + + let long_field = + "field_name_that_is_longer_than_postgresql_identifier_limit_of_sixty_three_bytes"; + let mut identifier_sources = dever_core::source::SourceMap::default(); + identifier_sources.add( + "report/model/entry.dever", + format!("database report\npublic type Entry {{ {long_field}: Text }}"), + ); + let identifiers = dever_core::check(&identifier_sources).unwrap(); + let error = identifiers + .validate_database_settings(&settings) + .unwrap_err(); + assert!(error.contains("exceeds 63 bytes"), "{error}"); +} + +#[test] +fn settings_reject_unknown_fields_missing_default_and_escaping_paths() { + for (setting, expected) in [ + (r#"{"linux_ci":{}}"#, "unknown field `linux_ci`"), + ( + r#"{"database":{"default":{"type":"sqlite","path":"data/app.db","extra":true}}}"#, + "unknown field", + ), + (r#"{"database":{}}"#, "database.default is required"), + ( + r#"{"database":{"default":{"type":"sqlite","path":"../outside.db"}}}"#, + "must stay below", + ), + ( + r#"{"database":{"default":{"type":"postgres","url":"postgres://example.invalid/app","tls":"prefer"}}}"#, + "unknown variant", + ), + ( + r#"{"database":{"default":{"type":"sqlite","path":"data/app.db","max_connections":0}}}"#, + "must be between", + ), + ( + r#"{"database":{"default":{"type":"postgres","url":"postgres://example.invalid/app","tls":"disabled","wait_timeout_ms":0}}}"#, + "wait_timeout_ms must be between", + ), + ( + r#"{"database":{"default":{"type":"postgres","url":"postgres://example.invalid/app","tls":"disabled","io_timeout_ms":300001}}}"#, + "io_timeout_ms must be between", + ), + ( + r#"{"database":{"default":{"type":"postgres","url":"postgres://example.invalid/app","tls":"disabled","tenant_database_prefix":"tenant_database_prefix_that_exceeds_forty_three_bytes"}}}"#, + "at most 43 lower_snake_case bytes", + ), + ( + r#"{"database":{"default":{"type":"sqlite","path":"data/app.db","tenant_directory":"data/tenants"}}}"#, + "require top-level tenant settings", + ), + ( + r#"{"tenant":{"database":"missing"},"database":{"default":{"type":"sqlite","path":"data/app.db","tenant_directory":"data/tenants"}}}"#, + "unknown database", + ), + ( + r#"{"tenant":{"mode":"database","control_database":"default"},"database":{"default":{"type":"sqlite","path":"data/app.db","tenant_directory":"data/tenants"}}}"#, + "unknown field", + ), + ( + r#"{"tenant":{"max_pools":8},"database":{"default":{"type":"sqlite","path":"data/app.db","tenant_directory":"data/tenants"}}}"#, + "missing field `database`", + ), + ( + r#"{"tenant":{"database":"default","max_pools":0},"database":{"default":{"type":"sqlite","path":"data/app.db","tenant_directory":"data/tenants"}}}"#, + "max_pools must be between", + ), + ] { + let root = project(setting); + let error = Settings::load_project(root.path()) + .err() + .expect("invalid setting must fail"); + assert!(error.contains(expected), "{error}"); + } +} + +#[test] +fn sites_map_api_parent_directories_to_strict_auth_providers() { + let root = project( + r#"{ + "auth": { + "providers": { + "session": { + "verify": "user.account.verify", + "jwtSecret": "0123456789abcdef0123456789abcdef", + "ttlSeconds": 3600, + "cookie": "cms_session" + } + } + }, + "sites": { + "admin": {"path": "admin", "auth": "session", "hosts": ["ADMIN.EXAMPLE.COM:443"], "origin": "https://admin.example.com"}, + "front": {"path": "front", "auth": "session", "origin": "https://front.example.com:8443"} + }, + "log": {"level": "warn"} + }"#, + ); + let settings = Settings::load_project(root.path()).unwrap(); + let (key, site) = settings + .site_for_directory(&["admin", "article"]) + .unwrap() + .unwrap(); + assert_eq!(key, "admin"); + assert_eq!(site.path(), "admin"); + assert_eq!(site.auth(), "session"); + assert_eq!(site.hosts(), ["admin.example.com"]); + let origin = site.origin().unwrap(); + assert_eq!( + (origin.scheme(), origin.host(), origin.port()), + ("https", "admin.example.com", 443) + ); + let (_, front) = settings.site_for_directory(&["front"]).unwrap().unwrap(); + assert_eq!(front.hosts(), ["front.example.com"]); + let origin = front.origin().unwrap(); + assert_eq!( + (origin.scheme(), origin.host(), origin.port()), + ("https", "front.example.com", 8443) + ); + assert!(settings.site_for_directory(&[]).unwrap().is_none()); + assert!( + settings + .site_for_directory(&["administrator"]) + .unwrap() + .is_none() + ); + assert_eq!( + settings.auth().provider("session").unwrap().verify(), + "user.account.verify" + ); + assert_eq!(settings.log().level, LogLevel::Warn); +} + +#[test] +fn sites_reject_ambiguous_paths_unknown_providers_and_removed_app_settings() { + for (setting, expected) in [ + (r#"{"app":{"name":"cms"}}"#, "unknown field"), + ( + r#"{"sites":{"admin":{"path":"admin","auth":"missing"}}}"#, + "unknown provider", + ), + ( + r#"{ + "auth":{"providers":{"session":{"verify":"user.account.verify","jwtSecret":"0123456789abcdef0123456789abcdef"}}}, + "sites":{ + "admin":{"path":"admin","auth":"session"}, + "nested":{"path":"admin/reports","auth":"session"} + } + }"#, + "overlap", + ), + ( + r#"{ + "auth":{"providers":{"session":{"verify":"user.account.verify","jwtSecret":"0123456789abcdef0123456789abcdef"}}}, + "sites":{"front":{"path":"front","auth":"session","origin":"ftp://cms.example.com"}} + }"#, + "scheme must be http or https", + ), + ( + r#"{ + "auth":{"providers":{"session":{"verify":"user.account.verify","jwtSecret":"0123456789abcdef0123456789abcdef"}}}, + "sites":{"front":{"path":"front","auth":"session","hosts":["api.example.com"],"origin":"https://cms.example.com"}} + }"#, + "origin host must be present", + ), + ] { + let root = project(setting); + let error = Settings::load_project(root.path()) + .err() + .expect("invalid setting must fail"); + assert!(error.contains(expected), "{error}"); + } +} diff --git a/test/dever-tests/tests/application_testing.rs b/test/dever-tests/tests/application_testing.rs new file mode 100644 index 0000000..e0d8636 --- /dev/null +++ b/test/dever-tests/tests/application_testing.rs @@ -0,0 +1,411 @@ +#[path = "support/temp.rs"] +mod temp; + +use std::ffi::OsString; +use std::fs; +use std::path::Path; +use std::process::Command; + +use dever_core::diagnostic::Diagnostic; +use dever_core::source::SourceMap; +use temp::TemporaryDirectory; + +fn project(module: &[(&str, &str)], tests: &[(&str, &str)]) -> (TemporaryDirectory, SourceMap) { + let root = TemporaryDirectory::new(); + write_sources(&root.path().join("module"), module); + if !tests.is_empty() { + write_sources(&root.path().join("test"), tests); + } + let sources = + SourceMap::load_project(&root.path().join("module"), &root.path().join("test")).unwrap(); + (root, sources) +} + +fn write_sources(root: &Path, files: &[(&str, &str)]) { + fs::create_dir_all(root).unwrap(); + for (relative, text) in files { + let path = root.join(relative); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, text).unwrap(); + } +} + +fn checked(sources: &SourceMap) -> dever_core::hir::Program { + dever_core::check(sources).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(sources)) + .collect::<String>() + ) + }) +} + +fn rejected(sources: &SourceMap, code: &str, message: &str) -> Vec<Diagnostic> { + let errors = dever_core::check(sources).expect_err("source must be rejected"); + assert!( + errors + .iter() + .any(|error| error.code == code && error.message.contains(message)), + "expected {code}: {message}; got {}", + errors + .iter() + .map(|error| error.render(sources)) + .collect::<String>() + ); + errors +} + +#[test] +fn discovers_filename_entries_in_stable_order_and_allows_no_tests() { + let (_root, sources) = project(&[("main.dever", "main() () {}")], &[]); + assert!(checked(&sources).tests().is_empty()); + + let (_root, sources) = project( + &[("main.dever", "main() () {}")], + &[ + ( + "user/account/register.dever", + "register() () { assert(true) }", + ), + ( + "news/article/publish.dever", + "publish() () { assert_eq(1, 1) }", + ), + ], + ); + assert_eq!( + checked(&sources) + .tests() + .iter() + .map(|test| test.name()) + .collect::<Vec<_>>(), + ["news/article/publish", "user/account/register"] + ); +} + +#[test] +fn rejects_invalid_test_paths_and_entry_contracts() { + let (_root, sources) = project( + &[("main.dever", "main() () {}")], + &[("user/account/nested/register.dever", "register() () {}")], + ); + rejected(&sources, "C003", "test source must be"); + + let (_root, sources) = project( + &[("main.dever", "main() () {}")], + &[("user/account/register.dever", "helper_entry() () {}")], + ); + rejected( + &sources, + "C005", + "must declare the ordinary 'register() ()' entry", + ); + + let (_root, sources) = project( + &[("main.dever", "main() () {}")], + &[("user/account/register.dever", "register(value: Int) () {}")], + ); + rejected(&sources, "C005", "zero-input, zero-output"); +} + +#[test] +fn accepts_documented_markdown_tests_and_rejects_duplicate_topics() { + let (_root, sources) = project( + &[("main.dever", "main() () {}")], + &[( + "user/account/register.dever.md", + r#"# 注册账户测试 + +验证账户注册行为。 + +- 包：`user.account.register` +- 公开类型：无 +- 公开方法：无 +- 使用：无 + +## 注册账户 + +执行注册断言。 + +- 函数：`register` +- 输入：无 +- 输出：无 + +```dever +register() () { assert(true) } +``` +"#, + )], + ); + assert_eq!(checked(&sources).tests()[0].name(), "user/account/register"); + + let (_root, sources) = project( + &[("main.dever", "main() () {}")], + &[ + ("user/account/register.dever", "register() () {}"), + ( + "user/account/register.dever.md", + r#"# 注册账户测试 + +验证账户注册行为。 + +- 包：`user.account.register` +- 公开类型：无 +- 公开方法：无 +- 使用：无 + +## 注册账户 + +执行注册断言。 + +- 函数：`register` +- 输入：无 +- 输出：无 + +```dever +register() () {} +``` +"#, + ), + ], + ); + rejected(&sources, "C002", "duplicate package"); +} + +#[test] +fn test_visibility_uses_domain_boundaries_without_exposing_test_helpers() { + let (_root, sources) = project( + &[ + ("main.dever", "main() () {}"), + ( + "user/account/domain.dever", + "normalize(value: Text) (result: Text) { result = text.lower(text.trim(value)) }", + ), + ( + "user/account/app.dever", + "normalize(value: Text) (result: Text) { result = domain.normalize(value) }", + ), + ( + "news/article/app.dever", + "label() (result: Text) { result = \"article\" }", + ), + ], + &[( + "user/account/normalize.dever", + r#"normalize() () { + assert_eq(app.normalize(" Editor "), "editor") + assert_eq(domain.normalize(" ADMIN "), "admin") + assert_eq(news.article.label(), "article") + helper() +} +helper() () {}"#, + )], + ); + assert_eq!(checked(&sources).tests().len(), 1); + + let (_root, sources) = project( + &[ + ("main.dever", "main() () {}"), + ( + "news/article/domain.dever", + "secret() (result: Text) { result = \"private\" }", + ), + ], + &[( + "user/account/steal.dever", + "steal() () { assert_eq(news.article.domain.secret(), \"private\") }", + )], + ); + rejected(&sources, "C006", "not callable from this source role"); + + let (_root, sources) = project( + &[ + ("main.dever", "main() () {}"), + ("user/account/model.dever", "type Account { email: Text }"), + ], + &[( + "user/account/count.dever", + "count() () { assert_eq(model.count(), 0) }", + )], + ); + rejected(&sources, "C006", "Model operations are private"); + + let (_root, sources) = project( + &[ + ("main.dever", "main() () {}"), + ( + "user/account/app.dever", + "leak() () { user.account.register.helper() }", + ), + ], + &[( + "user/account/register.dever", + "register() () {}\nhelper() () {}", + )], + ); + rejected(&sources, "C006", "not callable from this source role"); + + let (_root, sources) = project( + &[("main.dever", "main() () {}")], + &[ + ( + "user/account/register.dever", + "register() () {}\nhelper() () {}", + ), + ( + "user/account/profile.dever", + "profile() () { user.account.register.helper() }", + ), + ], + ); + rejected(&sources, "C006", "not callable from this source role"); +} + +#[test] +fn assertions_are_test_only_exact_and_comparable() { + let (_root, sources) = project(&[("main.dever", "main() () { assert(true) }")], &[]); + rejected(&sources, "C004", "unknown function 'assert'"); + + let (_root, sources) = project( + &[("main.dever", "main() () {}")], + &[("user/account/value.dever", "value() () { assert(1) }")], + ); + rejected(&sources, "C005", "expected Bool"); + + let (_root, sources) = project( + &[("main.dever", "main() () {}")], + &[( + "user/account/types.dever", + "types() () { assert_eq(1, 1.0) }", + )], + ); + rejected(&sources, "C005", "same type on both sides"); + + let (_root, sources) = project( + &[("main.dever", "main() () {}")], + &[( + "user/account/stream.dever", + "stream() () { assert_eq(stream([1]), stream([1])) }", + )], + ); + rejected(&sources, "C005", "assert_eq does not support"); +} + +#[test] +fn tests_cannot_start_the_application_api() { + let (_root, sources) = project( + &[("main.dever", "main() () {}")], + &[( + "user/account/server.dever", + "server() () { dever.api.serve() }", + )], + ); + rejected(&sources, "C006", "cannot start the application HTTP API"); +} + +#[test] +fn tests_use_existing_result_choices_for_business_failures() { + let (_root, sources) = project( + &[ + ("main.dever", "main() () {}"), + ( + "user/account/app.dever", + r#"type RegistrationResult { + Created(account: Int) + error Invalid(message: Text) +} +register(value: 0) (account: Int) { + fail(RegistrationResult.Invalid("invalid account")) +} +register(value: other) (account: Int) { + account = value +}"#, + ), + ], + &[( + "user/account/register.dever", + r#"register() () { + verify(result(app.register(0))) +} +verify(result: app.RegistrationResult.Invalid(message)) () recover("test verifies expected failure") { + assert_eq(message, "invalid account") +} +verify(result: app.RegistrationResult.Created(account)) () recover("test verifies expected failure") { + assert(false) +}"#, + )], + ); + assert_eq!(checked(&sources).tests().len(), 1); +} + +#[test] +fn native_test_suite_dispatches_cases_reports_locations_and_reuses_cache() { + let (_root, sources) = project( + &[ + ("main.dever", "main() () {}"), + ("user/profile/model.dever", "type Profile { name: Text }"), + ], + &[ + ( + "news/article/publish.dever", + "publish() () { assert_eq(1, 1) }", + ), + ( + "user/account/register.dever", + r#"type Pair { + left: Int + right: Int +} +type Number { + Present(value: Int) + Missing +} +register() () { + actual = Pair { + left = 1 + right = 2 + } + expected = Pair { + left = 1 + right = 2 + } + assert_eq(actual, expected) + assert_eq(Number.Present(1), Number.Present(1)) + assert_eq(first([1]), first([1])) + assert_eq(missing(), null) + assert_eq([1, 2], [1, 2]) + values = { "one" = 1 } + assert_eq(values, { "one" = 1 }) + assert_eq(entries(values), entries({ "one" = 1 })) + assert_eq(1, 2) +} +missing() (value: Int?) { + value = null +}"#, + ), + ], + ); + let program = checked(&sources); + assert!(!program.tests()[0].uses_database()); + assert!(!program.tests()[1].uses_database()); + let rustc = std::env::var_os("RUSTC").unwrap_or_else(|| OsString::from("rustc")); + let native = dever_core::native::compile_test_suite(&program, &sources, &rustc).unwrap(); + let success = Command::new(native.executable()).arg("0").output().unwrap(); + assert!(success.status.success()); + let output = Command::new(native.executable()).arg("1").output().unwrap(); + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("test/user/account/register.dever:26:"), + "{stderr}" + ); + assert!( + stderr.contains("assert_eq failed: actual = 1, expected = 2"), + "{stderr}" + ); + drop(native); + let cached = dever_core::native::compile_test_suite(&program, &sources, &rustc).unwrap(); + assert!(cached.cache_hit()); +} diff --git a/test/dever-tests/tests/async_composition.rs b/test/dever-tests/tests/async_composition.rs new file mode 100644 index 0000000..ef22c0f --- /dev/null +++ b/test/dever-tests/tests/async_composition.rs @@ -0,0 +1,201 @@ +mod support; + +use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, +}; +use std::time::Duration; + +use dever_runtime::{channel::Channel, task}; + +fn run(future: impl std::future::Future<Output = Result<(), String>>) { + task::run_entry_with( + task::RuntimeConfig { + worker_threads: 1, + max_blocking_threads: 2, + task_capacity: 32, + }, + future, + ) + .unwrap(); +} + +#[test] +fn source_runtime_configuration_validates_limits_and_starts_once() { + assert!(task::configured(0, 1, 8).is_err()); + assert!(task::configured(1, -1, 8).is_err()); + assert!(task::configured(1, 1, 65537).is_err()); + let config = task::configured(1, 2, 8).unwrap(); + task::start_entry(config, async { Ok(()) }).unwrap(); + assert!( + task::start_entry(config, async { Ok(()) }) + .unwrap_err() + .contains("only be started once") + ); +} + +#[test] +fn timeout_drains_started_blocking_children_before_returning() { + run(async { + let started = Arc::new(tokio::sync::Semaphore::new(0)); + let finished = Arc::new(AtomicBool::new(false)); + let signal = started.clone(); + let completed = finished.clone(); + let job = task::run(async move { + task::blocking(move || { + signal.add_permits(1); + std::thread::sleep(Duration::from_millis(40)); + completed.store(true, Ordering::Release); + Ok(17) + }) + .await + }) + .await?; + started.acquire().await.unwrap().forget(); + assert_eq!(task::wait_timeout(job, 1).await?, None); + assert!(finished.load(Ordering::Acquire)); + Ok(()) + }); +} + +#[test] +fn race_stops_losers_and_preserves_program_faults() { + run(async { + struct Dropped(Arc<AtomicBool>); + impl Drop for Dropped { + fn drop(&mut self) { + self.0.store(true, Ordering::Release); + } + } + let ended = Arc::new(AtomicBool::new(false)); + let owner = Dropped(ended.clone()); + let slow = task::run(async move { + let _owner = owner; + std::future::pending::<Result<i64, String>>().await + }) + .await?; + let fast = task::run(async { Ok(42) }).await?; + assert_eq!(task::race(vec![slow, fast]).await?, 42); + assert!(ended.load(Ordering::Acquire)); + let failed = task::run(async { Err::<i64, _>("original fault".into()) }).await?; + let pending = task::run(std::future::pending::<Result<i64, String>>()).await?; + assert_eq!( + task::race(vec![failed, pending]).await.unwrap_err(), + "original fault" + ); + let failed = task::run(async { Err::<(), _>("timer fault".into()) }).await?; + assert_eq!( + task::wait_timeout(failed, 100).await.unwrap_err(), + "timer fault" + ); + Ok(()) + }); +} + +#[test] +fn channel_stream_drains_close_without_closing_independent_aliases() { + run(async { + let channel = Channel::new(2).unwrap(); + channel.send(Some(1)).await?; + channel.send(None).await?; + let stream = channel.clone().stream(); + assert_eq!(stream.pull().await, Some(Some(1))); + stream.close(); + assert_eq!(channel.receive().await?, Some(None)); + channel.send(Some(2)).await?; + channel.close().await?; + let stream = channel.stream(); + assert_eq!(stream.pull().await, Some(Some(2))); + assert_eq!(stream.pull().await, None); + Ok(()) + }); +} + +#[test] +fn ticks_delay_first_value_skip_missed_periods_and_close_pending_pull() { + run(async { + let ticks = task::ticks(20)?; + assert!( + tokio::time::timeout(Duration::from_millis(2), ticks.pull()) + .await + .is_err() + ); + assert_eq!(ticks.pull().await, Some(1)); + tokio::time::sleep(Duration::from_millis(70)).await; + assert_eq!(ticks.pull().await, Some(2)); + assert!( + tokio::time::timeout(Duration::from_millis(2), ticks.pull()) + .await + .is_err() + ); + let waiter = ticks.clone(); + let pending = task::run(async move { Ok(waiter.pull().await) }).await?; + ticks.close(); + assert_eq!(task::wait(pending).await?, None); + assert!(task::ticks(0).is_err()); + Ok(()) + }); +} + +#[test] +fn static_composition_rejects_wrong_shapes_and_reused_tasks() { + support::rejected( + "work() (value: Int) { value = 1 }\nfallback() (value: Text) { value = \"wrong\" }\nmain() () { job = run(work())\nvalue = timeout(job, 1, fallback) }", + "signature", + ); + support::rejected( + "work() (value: Int) { value = 1 }\nmain() () { job = run(work())\nvalue = race(job, job) }", + "consumed", + ); +} + +#[test] +fn native_composition_and_suspending_memory_handlers() { + let source = r#"delayed(value: Int, millis: Int) (answer: Int) recover("test timer") { + dever.task.sleep(millis) + answer = value +} +fallback() (answer: Int) { answer = 9 } +collect(value: Int, output: Channel<Int>) () { send(output, value) } +add(value: Int, state: Int) (result: Int) { result = state + value } +identity(value: Int) (result: Int) { result = value } +pair(millis: Int) (number: Int, text: Text) recover("test timer") { + dever.task.sleep(millis) + number = 8 + text = "pair" +} +pair_timeout() (number: Int, text: Text) { number = 0 + text = "timeout" } +worker() () { + slow = run(delayed(1, 1000)) + timed = timeout(slow, 1, fallback) + first = run(delayed(2, 1000)) + second = run(delayed(3, 1)) + fastest = race(first, second) + values = each(add, [1, 2], 10) + total = reduce(add, [1, 2, 3], 0) + mapped = each(identity, [7, 8]) + mapped_total = sum(mapped) + pair_task = run(pair(1000)) + pair_value = timeout(pair_task, 1, pair_timeout) + pair_text = pair_value.text + channel = channel(Int, 4) + each(collect, [4, 5], channel) + parallel_each(collect, dever.bytes.from_text("ab"), 2, channel) + close(channel) + received = reduce(add, stream(channel), 0) + timer = dever.task.ticks(10) + close(timer) + blocking(dever.io.println(int.to_text(timed))) + blocking(dever.io.println(int.to_text(fastest))) + blocking(dever.io.println(int.to_text(sum(values)))) + blocking(dever.io.println(int.to_text(total))) + blocking(dever.io.println(int.to_text(received))) + blocking(dever.io.println(int.to_text(mapped_total))) + blocking(dever.io.println(pair_text)) +} +public main() () { + worker() +}"#; + assert_eq!(support::stdout(source), "9\n3\n23\n6\n204\n15\ntimeout\n"); +} diff --git a/test/dever-tests/tests/async_native.rs b/test/dever-tests/tests/async_native.rs new file mode 100644 index 0000000..cd57bb2 --- /dev/null +++ b/test/dever-tests/tests/async_native.rs @@ -0,0 +1,148 @@ +mod support; + +use support::{checked, sources, stdout}; + +#[test] +fn async_program_compiles_and_runs_every_execution_class() { + let output = stdout( + r#"sleep_ok(result: dever.time.SleepResult.Done) () recover("test handles timer failures") {} +sleep_ok(result: dever.time.SleepResult.Failed(message)) () recover("test handles timer failures") {} +pause() (result: dever.time.SleepResult) { result = result(dever.time.sleep(1)) } +double(value: Int) (answer: Int) pure { answer = value * 2 } +delayed(value: Int, millis: Int) (answer: Int) { + sleep_ok(result(dever.task.sleep(millis))) + answer = value +} +send_value(channel: Channel<Int>, value: Int) () { + send(channel, value) +} +wait_until_stopped() () { + sleep_ok(result(dever.task.sleep(1000))) +} +public main() (direct: Int, first: Int, second: Int, received: Int?, grouped: Int?, computed: Int) { + direct = delayed(1, 1) + first_task = run(delayed(2, 10)) + second_task = run(delayed(3, 1)) + second = wait(second_task) + first = wait(first_task) + + channel = channel(Int, 2) + sender = run(send_value(channel, 4)) + received = receive(channel) + wait(sender) + + workers = group(2) + run(workers, send_value(channel, 5)) + wait(workers) + grouped = receive(channel) + close(channel) + + cancelled = run(wait_until_stopped()) + stop(cancelled) + computed = parallel(double(21)) + sleep_ok(blocking(pause())) +}"#, + ); + + assert_eq!( + output, + "direct = 1\nfirst = 2\nsecond = 3\nreceived = 4\ngrouped = 5\ncomputed = 42\n" + ); +} + +#[test] +fn only_inferred_suspending_entries_start_the_shared_runtime() { + let sync_sources = sources("public main() (answer: Int) { answer = 1 }"); + let sync = + dever_core::native::emit(&checked(&sync_sources), &sync_sources, "main.main").unwrap(); + assert!(sync.contains("match entry()"), "{sync}"); + assert!(!sync.contains("task::run_entry_typed(entry())"), "{sync}"); + + let async_sources = sources( + r#"sleep_ok(result: dever.time.SleepResult.Done) () recover("timer failure handled") {} +sleep_ok(result: dever.time.SleepResult.Failed(message)) () recover("timer failure handled") {} +public main() () { sleep_ok(result(dever.task.sleep(0))) }"#, + ); + let asynchronous = + dever_core::native::emit(&checked(&async_sources), &async_sources, "main.main").unwrap(); + assert!( + asynchronous.contains("match task::run_entry_typed(entry())"), + "{asynchronous}" + ); +} + +#[test] +fn timer_suspension_is_inferred_without_a_source_modifier() { + let program_sources = sources( + "public main() (result: dever.time.SleepResult) { result = dever.system.task_sleep(1) }", + ); + let generated = + dever_core::native::emit(&checked(&program_sources), &program_sources, "main.main") + .unwrap(); + assert!(generated.contains("async fn"), "{generated}"); + assert!( + generated.contains("task::run_entry_typed(entry())"), + "{generated}" + ); +} + +#[test] +fn async_parallel_each_runs_outside_the_runtime_worker() { + let source = r#"sleep_ok(result: dever.time.SleepResult.Done) () recover("timer failure handled") {} +sleep_ok(result: dever.time.SleepResult.Failed(message)) () recover("timer failure handled") {} +visit(value: Int) () {} +public main() (answer: Int) { + sleep_ok(result(dever.task.sleep(0))) + parallel_each(visit, [1, 2, 3], 2) + parallel_each(visit, dever.bytes.from_text("ab"), 2) + answer = 7 +}"#; + let program_sources = sources(source); + let generated = + dever_core::native::emit(&checked(&program_sources), &program_sources, "main.main") + .unwrap(); + assert!( + generated.contains("task::parallel_each_typed(a0.into_values()"), + "{generated}" + ); + assert_eq!(stdout(source), "answer = 7\n"); +} + +#[test] +fn nullable_channel_items_are_flattened_at_the_language_boundary() { + assert_eq!( + stdout( + "public main() (received: Int?) { channel = channel(Int?, 1)\nsend(channel, null)\nreceived = receive(channel)\nclose(channel) }", + ), + "received = null\n" + ); +} + +#[test] +fn inferred_suspension_requires_explicit_transitive_blocking_boundaries() { + support::rejected( + r#"sleep_ok(result: dever.time.SleepResult.Done) () recover("timer failure handled") {} +sleep_ok(result: dever.time.SleepResult.Failed(message)) () recover("timer failure handled") {} +pause() (result: dever.time.SleepResult) { result = result(dever.time.sleep(1)) } +public main() (result: dever.time.SleepResult) { + sleep_ok(result(dever.task.sleep(0))) + result = pause() +}"#, + "cannot run inline inside a suspending function", + ); + support::rejected( + r#"sleep_ok(result: dever.time.SleepResult.Done) () recover("timer failure handled") {} +sleep_ok(result: dever.time.SleepResult.Failed(message)) () recover("timer failure handled") {} +visit(value: Int) () {} +visit_all(values: List<Int>) () { parallel_each(visit, values, 2) } +public main() () { + sleep_ok(result(dever.task.sleep(0))) + visit_all([1, 2, 3]) +}"#, + "cannot run inline inside a suspending function", + ); + checked(&sources( + r#"visit(value: Int) () {} +visit_all(values: Stream<Int>) () { parallel_each(visit, values, 2) }"#, + )); +} diff --git a/test/dever-tests/tests/async_network.rs b/test/dever-tests/tests/async_network.rs new file mode 100644 index 0000000..a4d34b2 --- /dev/null +++ b/test/dever-tests/tests/async_network.rs @@ -0,0 +1,265 @@ +use std::future::Future; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::time::Duration; + +use dever_runtime::{ + async_stream::AsyncStream, bytes::Bytes, channel::Channel, net, resource::Pull, task, +}; +use futures_util::stream; + +fn run_test(test: impl Future<Output = Result<(), String>>) { + task::run_entry_with( + task::RuntimeConfig { + worker_threads: 1, + max_blocking_threads: 2, + task_capacity: 32, + }, + async { + tokio::time::timeout(Duration::from_secs(5), test) + .await + .map_err(|_| "async network test exceeded its deadline".to_owned())? + }, + ) + .unwrap(); +} + +struct Dropped(Arc<AtomicBool>); +impl Drop for Dropped { + fn drop(&mut self) { + self.0.store(true, Ordering::SeqCst); + } +} + +#[test] +fn closing_an_idle_stream_wakes_pull_and_drops_the_producer() { + run_test(async { + let entered = Channel::new(1)?; + let ready = entered.clone(); + let dropped = Arc::new(AtomicBool::new(false)); + let marker = Dropped(dropped.clone()); + let stream = AsyncStream::<i64>::new(stream::unfold(marker, move |marker| { + let ready = ready.clone(); + async move { + ready.send(()).await.unwrap(); + std::future::pending::<()>().await; + Some((Pull::Item(1), marker)) + } + })); + let alias = stream.clone(); + let reader = task::run(async move { Ok(alias.pull().await) }).await?; + entered.receive().await?; + stream.close(); + stream.close(); + assert_eq!(task::wait(reader).await?, None); + assert!(dropped.load(Ordering::SeqCst)); + assert_eq!(stream.pull().await, None); + Ok(()) + }); +} + +#[test] +fn mapped_stream_shares_cursor_and_emits_terminal_failure_once() { + run_test(async { + let stream = AsyncStream::new(stream::iter([ + Pull::Item(Ok(7)), + Pull::Last(Err("failed")), + Pull::Item(Ok(99)), + ])) + .map(|value| value.map(|value| value * 2)); + let alias = stream.clone(); + assert_eq!(stream.pull().await, Some(Ok(14))); + assert_eq!(alias.pull().await, Some(Err("failed"))); + assert_eq!(stream.pull().await, None); + Ok(()) + }); +} + +#[test] +fn stream_parallelism_applies_backpressure_before_pull_and_stop_drains_children() { + run_test(async { + let pulls = Arc::new(AtomicUsize::new(0)); + let observed = pulls.clone(); + let source_dropped = Arc::new(AtomicBool::new(false)); + let marker = Dropped(source_dropped.clone()); + let stream = AsyncStream::new(stream::unfold(marker, move |marker| { + observed.fetch_add(1, Ordering::SeqCst); + async move { Some((Pull::Item(1_i64), marker)) } + })); + let alias = stream.clone(); + let started = Channel::new(1)?; + let notify = started.clone(); + let child_dropped = Arc::new(AtomicBool::new(false)); + let observed_child = child_dropped.clone(); + let consumer = task::run(async move { + task::parallel_each_stream(stream, 1, move |_| { + let notify = notify.clone(); + let marker = Dropped(observed_child.clone()); + async move { + let _marker = marker; + notify.send(()).await?; + std::future::pending::<()>().await; + Ok(()) + } + }) + .await + }) + .await?; + started.receive().await?; + assert_eq!(pulls.load(Ordering::SeqCst), 1); + task::stop(consumer).await?; + assert!(source_dropped.load(Ordering::SeqCst)); + assert!(child_dropped.load(Ordering::SeqCst)); + assert_eq!(alias.pull().await, None); + Ok(()) + }); +} + +#[test] +fn handler_failure_interrupts_an_idle_producer() { + run_test(async { + let idle = Channel::new(1)?; + let notify_idle = idle.clone(); + let dropped = Arc::new(AtomicBool::new(false)); + let stream = AsyncStream::new(stream::unfold( + (false, Dropped(dropped.clone())), + move |(sent, marker)| { + let notify_idle = notify_idle.clone(); + async move { + if sent { + notify_idle.send(()).await.unwrap(); + std::future::pending::<()>().await; + } + Some((Pull::Item(1_i64), (true, marker))) + } + }, + )); + let result = task::parallel_each_stream(stream, 2, move |_| { + let idle = idle.clone(); + async move { + idle.receive().await?; + Err("expected handler failure".into()) + } + }) + .await; + assert_eq!(result.unwrap_err(), "expected handler failure"); + assert!(dropped.load(Ordering::SeqCst)); + Ok(()) + }); +} + +async fn pair() -> Result<(net::Socket, net::Socket), String> { + let listener = net::listen("127.0.0.1", 0).await?; + let client = net::connect_timeout("127.0.0.1", net::port(&listener)?, 1000).await?; + let server = net::accept(&listener).await?; + net::timeout(&client, 1000)?; + net::timeout(&server, 1000)?; + Ok((client, server)) +} + +#[test] +fn tcp_duplex_progresses_with_one_worker_and_pending_read() { + run_test(async { + let (client, server) = pair().await?; + let reading = client.clone(); + let reader = task::run(async move { net::read(&reading, 1).await }).await?; + // A read may be pending on the same Socket while its write completes. + net::write(&client, &Bytes::from_text("x")).await?; + assert_eq!(net::read(&server, 1).await?.unwrap().to_text()?, "x"); + net::write(&server, &Bytes::from_text("y")).await?; + assert_eq!(task::wait(reader).await?.unwrap().to_text()?, "y"); + let chunks = net::chunks(client.clone(), 1)?; + chunks.close(); + // Closing a stream releases its alias; the explicitly held socket stays usable. + net::write(&client, &Bytes::from_text("z")).await?; + assert_eq!(net::read(&server, 1).await?.unwrap().to_text()?, "z"); + client.close()?; + assert!(net::write(&client, &Bytes::from_text("x")).await.is_err()); + assert_eq!(net::read(&server, 1).await?, None); + Ok(()) + }); +} + +#[test] +fn closing_resources_wakes_pending_accepts_and_reads() { + run_test(async { + let listener = net::listen("127.0.0.1", 0).await?; + let accepting = listener.clone(); + let accept = task::run(async move { net::accept(&accepting).await }).await?; + let (client, server) = pair().await?; + let reading = server.clone(); + let read = task::run(async move { net::read(&reading, 1).await }).await?; + tokio::task::yield_now().await; + listener.close()?; + server.close()?; + assert_eq!(task::wait(accept).await.unwrap_err(), "resource is closed"); + assert_eq!(task::wait(read).await.unwrap_err(), "resource is closed"); + assert_eq!(net::read(&client, 1).await?, None); + Ok(()) + }); +} + +#[test] +fn read_timeout_does_not_close_or_consume_the_next_message() { + run_test(async { + let (client, server) = pair().await?; + net::timeout(&server, 10)?; + assert_eq!( + net::read(&server, 1).await.unwrap_err(), + "network operation timed out" + ); + net::timeout(&server, 1000)?; + net::write(&client, &Bytes::from_text("x")).await?; + assert_eq!(net::read(&server, 1).await?.unwrap().to_text()?, "x"); + assert!(net::read(&server, 0).await.is_err()); + assert!(net::connect_timeout("localhost", 0, 1).await.is_err()); + assert!(net::connect("127.0.0.1", -1).await.is_err()); + Ok(()) + }); +} + +#[test] +fn cancelling_a_partial_write_closes_socket_before_another_write() { + run_test(async { + let (client, server) = pair().await?; + // More than a loopback socket's send buffer, with no draining reader. + let sending = client.clone(); + let writer = + task::run( + async move { net::write(&sending, &Bytes::new(vec![7; 16 * 1024 * 1024])).await }, + ) + .await?; + assert_eq!(net::read(&server, 1).await?.unwrap().values(), &[7]); + task::stop(writer).await?; + assert_eq!( + net::write(&client, &Bytes::from_text("next")) + .await + .unwrap_err(), + "resource is closed" + ); + Ok(()) + }); +} + +#[test] +fn stopped_connection_consumer_releases_its_listener() { + run_test(async { + let listener = net::listen("127.0.0.1", 0).await?; + let port = net::port(&listener)?; + let stream = net::connections(listener); + let consumer = + task::run( + async move { task::parallel_each_stream(stream, 2, |_| async { Ok(()) }).await }, + ) + .await?; + tokio::task::yield_now().await; + task::stop(consumer).await?; + let rebound = net::listen("127.0.0.1", port).await?; + assert_eq!(net::port(&rebound)?, port); + let stream = net::connections(rebound.clone()); + rebound.close()?; + assert!(stream.pull().await.unwrap().is_err()); + assert!(stream.pull().await.is_none()); + Ok(()) + }); +} diff --git a/test/dever-tests/tests/async_runtime.rs b/test/dever-tests/tests/async_runtime.rs new file mode 100644 index 0000000..ee054cf --- /dev/null +++ b/test/dever-tests/tests/async_runtime.rs @@ -0,0 +1,295 @@ +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, mpsc}; +use std::time::Duration; + +use dever_runtime::{channel::Channel, task}; + +struct DropSignal(Arc<AtomicBool>); + +impl Drop for DropSignal { + fn drop(&mut self) { + self.0.store(true, Ordering::SeqCst); + } +} + +#[test] +fn tasks_return_values_and_stop_waits_for_cleanup() { + task::run_entry(async { + let task = task::run(async { Ok::<_, String>(42_i64) }).await?; + assert_eq!(task::wait(task).await?, 42); + + let started = Channel::new(1)?; + let child_started = started.clone(); + let dropped = Arc::new(AtomicBool::new(false)); + let child_dropped = Arc::clone(&dropped); + let task = task::run(async move { + let _drop_signal = DropSignal(child_dropped); + child_started.send(()).await?; + std::future::pending::<()>().await; + Ok(()) + }) + .await?; + assert_eq!(started.receive().await?, Some(())); + task::stop(task).await?; + assert!(dropped.load(Ordering::SeqCst)); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn group_failure_cancels_and_drains_remaining_tasks() { + task::run_entry(async { + let started = Channel::new(1)?; + let child_started = started.clone(); + let dropped = Arc::new(AtomicBool::new(false)); + let child_dropped = Arc::clone(&dropped); + let mut group = task::Group::new(2)?; + group + .run(async move { + let _drop_signal = DropSignal(child_dropped); + child_started.send(()).await?; + std::future::pending::<()>().await; + Ok(()) + }) + .await?; + assert_eq!(started.receive().await?, Some(())); + group + .run(async { Err::<(), _>("expected child failure".to_owned()) }) + .await?; + + assert_eq!(group.wait().await.unwrap_err(), "expected child failure"); + assert!(dropped.load(Ordering::SeqCst)); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn group_capacity_applies_backpressure_before_starting_more_work() { + task::run_entry(async { + let release = Channel::new(1)?; + let wait_for_release = release.clone(); + let released = Arc::new(AtomicBool::new(false)); + let release_recorded = Arc::clone(&released); + let mut group = task::Group::new(1)?; + group + .run(async move { + wait_for_release.receive().await?; + Ok(()) + }) + .await?; + + let unblock = task::run(async move { + task::sleep(1).await?; + release_recorded.store(true, Ordering::SeqCst); + release.send(()).await + }) + .await?; + group.run(async { Ok(()) }).await?; + assert!(released.load(Ordering::SeqCst)); + group.wait().await?; + task::wait(unblock).await?; + Ok(()) + }) + .unwrap(); +} + +#[test] +fn channels_are_bounded_and_close_after_buffered_values() { + task::run_entry(async { + let channel = Channel::new(1)?; + channel.send(7_i64).await?; + channel.close().await?; + assert_eq!(channel.receive().await?, Some(7)); + assert_eq!(channel.receive().await?, None); + assert!(channel.send(8).await.is_err()); + Ok(()) + }) + .unwrap(); + + assert!(Channel::<i64>::new(0).is_err()); + assert!(Channel::<i64>::new(65_537).is_err()); +} + +#[test] +fn closing_a_channel_wakes_blocked_senders_and_receivers() { + task::run_entry(async { + let full = Channel::new(1)?; + full.send(1_i64).await?; + let blocked_sender = full.clone(); + let send = task::run(async move { blocked_sender.send(2).await }).await?; + full.close().await?; + assert_eq!(task::wait(send).await.unwrap_err(), "channel is closed"); + assert_eq!(full.receive().await?, Some(1)); + assert_eq!(full.receive().await?, None); + + let empty = Channel::<i64>::new(1)?; + let blocked_receiver = empty.clone(); + let receive = task::run(async move { blocked_receiver.receive().await }).await?; + empty.close().await?; + assert_eq!(task::wait(receive).await?, None); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn parallel_and_blocking_use_the_bounded_blocking_pool() { + task::run_entry(async { + task::sleep(0).await?; + assert_eq!( + task::sleep(-1).await, + Err("sleep duration must not be negative") + ); + assert_eq!(task::parallel(|| Ok::<_, String>(6 * 7)).await?, 42); + assert_eq!(task::blocking(|| Ok::<_, String>("done")).await?, "done"); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn runtime_configuration_rejects_unbounded_thread_counts() { + let result = task::run_entry_with( + task::RuntimeConfig { + worker_threads: task::MAX_ASYNC_WORKERS + 1, + max_blocking_threads: 1, + task_capacity: 1, + }, + async { Ok::<_, String>(()) }, + ); + assert!(result.unwrap_err().contains("async worker count")); +} + +#[test] +fn root_and_parent_failures_wait_for_descendant_cleanup() { + let root_dropped = Arc::new(AtomicBool::new(false)); + let root_child_dropped = Arc::clone(&root_dropped); + let root_result = task::run_entry(async move { + let started = Channel::new(1)?; + let child_started = started.clone(); + let _child = task::run(async move { + let _drop_signal = DropSignal(root_child_dropped); + child_started.send(()).await?; + std::future::pending::<()>().await; + Ok(()) + }) + .await?; + assert_eq!(started.receive().await?, Some(())); + Err::<(), _>("root failure".to_owned()) + }); + assert_eq!(root_result.unwrap_err(), "root failure"); + assert!(root_dropped.load(Ordering::SeqCst)); + + let descendant_dropped = Arc::new(AtomicBool::new(false)); + let child_dropped = Arc::clone(&descendant_dropped); + task::run_entry(async move { + let child_started = Channel::new(1)?; + let wait_for_child = child_started.clone(); + let parent_ready = Channel::new(1)?; + let notify_root = parent_ready.clone(); + let parent = task::run(async move { + let _descendant = task::run(async move { + let _drop_signal = DropSignal(child_dropped); + child_started.send(()).await?; + std::future::pending::<()>().await; + Ok(()) + }) + .await?; + assert_eq!(wait_for_child.receive().await?, Some(())); + notify_root.send(()).await?; + Err::<(), _>("parent failure".to_owned()) + }) + .await?; + assert_eq!(parent_ready.receive().await?, Some(())); + assert_eq!(task::wait(parent).await.unwrap_err(), "parent failure"); + assert!(descendant_dropped.load(Ordering::SeqCst)); + Ok(()) + }) + .unwrap(); + + let group_dropped = Arc::new(AtomicBool::new(false)); + let group_child_dropped = Arc::clone(&group_dropped); + let group_result = task::run_entry(async move { + let started = Channel::new(1)?; + let child_started = started.clone(); + let mut group = task::Group::new(1)?; + group + .run(async move { + let _drop_signal = DropSignal(group_child_dropped); + child_started.send(()).await?; + std::future::pending::<()>().await; + Ok(()) + }) + .await?; + assert_eq!(started.receive().await?, Some(())); + Err::<(), _>("group parent failure".to_owned()) + }); + assert_eq!(group_result.unwrap_err(), "group parent failure"); + assert!(group_dropped.load(Ordering::SeqCst)); +} + +#[test] +fn stopping_a_task_waits_for_started_blocking_work() { + let finished = Arc::new(AtomicBool::new(false)); + let blocking_finished = Arc::clone(&finished); + let (started, running) = mpsc::channel(); + task::run_entry_with( + task::RuntimeConfig { + worker_threads: 1, + max_blocking_threads: 1, + task_capacity: 4, + }, + async move { + let child = task::run(async move { + task::blocking(move || { + started.send(()).unwrap(); + std::thread::sleep(Duration::from_millis(20)); + blocking_finished.store(true, Ordering::SeqCst); + Ok(()) + }) + .await + }) + .await?; + running.recv_timeout(Duration::from_secs(1)).unwrap(); + task::stop(child).await?; + assert!(finished.load(Ordering::SeqCst)); + Ok(()) + }, + ) + .unwrap(); +} + +#[test] +fn async_parallel_each_shares_the_configured_blocking_limit() { + let active = Arc::new(AtomicUsize::new(0)); + let peak = Arc::new(AtomicUsize::new(0)); + task::run_entry_with( + task::RuntimeConfig { + worker_threads: 2, + max_blocking_threads: 2, + task_capacity: 4, + }, + async { + let run_batch = |active: Arc<AtomicUsize>, peak: Arc<AtomicUsize>| async move { + task::parallel_each(0..32, 256, move |_| { + let current = active.fetch_add(1, Ordering::SeqCst) + 1; + peak.fetch_max(current, Ordering::SeqCst); + std::thread::sleep(Duration::from_millis(1)); + active.fetch_sub(1, Ordering::SeqCst); + Ok(()) + }) + .await + }; + let first = task::run(run_batch(Arc::clone(&active), Arc::clone(&peak))).await?; + let second = task::run(run_batch(Arc::clone(&active), Arc::clone(&peak))).await?; + task::wait(first).await?; + task::wait(second).await?; + assert_eq!(peak.load(Ordering::SeqCst), 2); + Ok(()) + }, + ) + .unwrap(); +} diff --git a/test/dever-tests/tests/async_stream.rs b/test/dever-tests/tests/async_stream.rs new file mode 100644 index 0000000..4f4d7a4 --- /dev/null +++ b/test/dever-tests/tests/async_stream.rs @@ -0,0 +1,164 @@ +mod support; + +use support::{checked, rejected, sources, stdout}; + +#[test] +fn tcp_server_example_checks_without_starting_a_service() { + checked(&sources(include_str!( + "../../../examples/old/dever/async_tcp/module/main.dever" + ))); +} + +#[test] +fn async_stream_handlers_and_transfer_rules_are_checked() { + let source = r#"visit(value: Int) () {} +public consume(route: handler(value: Int) (), stream: AsyncStream<Int>) () { + each(route, stream) + parallel_each(route, stream, 2) + close(stream) +}"#; + let input = sources(source); + checked(&input); + let formatted = dever_core::format::format(&input.files()[0]).unwrap(); + let roundtrip = sources(&formatted); + checked(&roundtrip); + assert_eq!( + dever_core::format::format(&roundtrip.files()[0]).unwrap(), + formatted + ); + checked(&sources( + "visit(value: Int) () {}\nconsume(stream: AsyncStream<Int>) () { each(visit, stream) }", + )); + checked(&sources( + "visit(value: Int) () {}\nconsume(stream: AsyncStream<Int>) () { parallel_each(visit, stream, 2) }", + )); + rejected( + "visit(value: Int) () { dever.io.println(int.to_text(value)) }\nconsume(stream: AsyncStream<Int>) () { each(visit, stream) }", + "cannot run inline inside a suspending function", + ); + rejected( + "consume(stream: AsyncStream<Stream<Int>>) () { close(stream) }", + "must be transferable between threads", + ); + checked(&sources( + "main() (result: dever.net.ListenResult) { result = result(dever.net.listen(\"127.0.0.1\", 0)) }", + )); +} + +#[test] +fn native_async_handler_forwarding_uses_bounded_stream_tasks() { + let source = r#"handle(event: dever.net.ConnectResult.Connected(socket), stream: AsyncStream<dever.net.ConnectResult>) () recover("test observes network failures through the client read") { + close(stream) + dever.net.write(socket, dever.bytes.from_text("x")) + dever.net.close(socket) +} +handle(event: dever.net.ConnectResult.Failed(message), stream: AsyncStream<dever.net.ConnectResult>) () recover("test observes network failures through the client read") { close(stream) } +dispatch(route: handler(event: dever.net.ConnectResult, stream: AsyncStream<dever.net.ConnectResult>) (), stream: AsyncStream<dever.net.ConnectResult>) () { + parallel_each(route, stream, 2, stream) +} +received(result: dever.io.ReadResult.Done(state)) (answer: Bool) recover("test asserts the received result") { answer = received_state(state) } +received(result: other) (answer: Bool) recover("test asserts the received result") { answer = false } +received_state(state: dever.io.ReadState.Read(bytes)) (answer: Bool) { answer = bytes == dever.bytes.from_text("x") } +received_state(state: other) (answer: Bool) { answer = false } +connected(result: dever.net.ConnectResult.Connected(client), listener: dever.system.Listener) (answer: Bool) recover("test asserts the connection result") { + dever.net.timeout(client, 1000) + stream = dever.net.connections(listener) + dispatch(handle, stream) + answer = received(result(dever.net.read(client, 1))) + dever.net.close(client) + dever.net.close_listener(listener) +} +connected(result: other, listener: dever.system.Listener) (answer: Bool) recover("test asserts the connection result") { answer = false } +bound(result: dever.net.PortResult.Bound(port), listener: dever.system.Listener) (answer: Bool) recover("test asserts the bound result") { answer = connected(result(dever.net.connect("127.0.0.1", port)), listener) } +bound(result: other, listener: dever.system.Listener) (answer: Bool) recover("test asserts the bound result") { answer = false } +listening(result: dever.net.ListenResult.Listening(listener)) (answer: Bool) recover("test asserts the listen result") { answer = bound(result(dever.net.port(listener)), listener) } +listening(result: other) (answer: Bool) recover("test asserts the listen result") { answer = false } +public main() (answer: Bool) { answer = listening(result(dever.net.listen("127.0.0.1", 0))) } +"#; + let input = sources(source); + let generated = dever_core::native::emit(&checked(&input), &input, "main.main").unwrap(); + assert!(generated.contains("task::parallel_each_stream_typed(")); + assert_eq!(stdout(source), "answer = true\n"); +} + +#[test] +fn closed_listener_stream_yields_one_failure() { + assert_eq!( + stdout( + r#"count(event: dever.net.ConnectResult.Failed(message), state: Int) (next: Int) recover("test observes count failures in its assertions") { next = state + 1 } +count(event: dever.net.ConnectResult.Connected(socket), state: Int) (next: Int) recover("test observes count failures in its assertions") { next = state + 100 } +listening(result: dever.net.ListenResult.Listening(listener)) (answer: Int) recover("test observes listening failures in its assertions") { + stream = dever.net.connections(listener) + dever.net.close_listener(listener) + answer = reduce(count, stream, 0) + reduce(count, stream, 0) +} +listening(result: other) (answer: Int) recover("test observes listening failures in its assertions") { answer = -1 } +public main() (answer: Int) { answer = listening(result(dever.net.listen("127.0.0.1", 0))) } +"# + ), + "answer = 1\n", + ); +} + +#[test] +fn native_socket_stream_each_executes_async_actions() { + assert_eq!( + stdout( + r#"show(event: dever.io.ReadEvent.Chunk(bytes)) () recover("test observes show failures in its assertions") { blocking(dever.io.println(int.to_text(dever.bytes.length(bytes)))) } +show(event: dever.io.ReadEvent.Failed(message)) () recover("test observes show failures in its assertions") { blocking(dever.io.println("failure")) } +consume(result: dever.net.ReadStreamResult.Streaming(stream)) () recover("test observes consume failures in its assertions") { each(show, stream) } +consume(result: dever.net.ReadStreamResult.Failed(message)) () recover("test observes consume failures in its assertions") { blocking(dever.io.println("failure")) } +accepted(result: dever.net.ConnectResult.Connected(server), client: dever.system.Socket, listener: dever.system.Listener) (answer: Bool) recover("test observes accepted failures in its assertions") { + dever.net.timeout(server, 1000) + dever.net.timeout(client, 1000) + dever.net.write(client, dever.bytes.from_text("x")) + dever.net.close(client) + consume(result(dever.net.chunks(server, 1))) + dever.net.close(server) + dever.net.close_listener(listener) + answer = true +} +accepted(result: other, client: dever.system.Socket, listener: dever.system.Listener) (answer: Bool) recover("test observes accepted failures in its assertions") { answer = false } +connected(result: dever.net.ConnectResult.Connected(client), listener: dever.system.Listener) (answer: Bool) recover("test observes connected failures in its assertions") { answer = accepted(result(dever.net.accept(listener)), client, listener) } +connected(result: other, listener: dever.system.Listener) (answer: Bool) recover("test observes connected failures in its assertions") { answer = false } +bound(result: dever.net.PortResult.Bound(port), listener: dever.system.Listener) (answer: Bool) recover("test observes bound failures in its assertions") { answer = connected(result(dever.net.connect("127.0.0.1", port)), listener) } +bound(result: other, listener: dever.system.Listener) (answer: Bool) recover("test observes bound failures in its assertions") { answer = false } +listening(result: dever.net.ListenResult.Listening(listener)) (answer: Bool) recover("test observes listening failures in its assertions") { answer = bound(result(dever.net.port(listener)), listener) } +listening(result: other) (answer: Bool) recover("test observes listening failures in its assertions") { answer = false } +public main() (answer: Bool) { answer = listening(result(dever.net.listen("127.0.0.1", 0))) } +"# + ), + "1\nanswer = true\n", + ); +} + +#[test] +fn last_use_optimization_preserves_socket_alias_lifetime_until_scope_exit() { + assert_eq!( + stdout( + r#"type Peer { socket: dever.system.Socket } +send(peer: Peer) () recover("test observes socket lifetime through the peer read") { dever.net.write(peer.socket, dever.bytes.from_text("x")) } +timeout(result: dever.io.ReadResult.Failed(message)) (answer: Bool) recover("test observes timeout failures in its assertions") { answer = true } +timeout(result: other) (answer: Bool) recover("test observes timeout failures in its assertions") { answer = false } +accepted(result: dever.net.ConnectResult.Connected(server), client: dever.system.Socket, listener: dever.system.Listener) (answer: Bool) recover("test observes accepted failures in its assertions") { + dever.net.timeout(server, 20) + peer = Peer { socket = client } + send(peer) + first = dever.net.read(server, 1) + answer = timeout(result(dever.net.read(server, 1))) + dever.net.close(server) + dever.net.close_listener(listener) +} +accepted(result: other, client: dever.system.Socket, listener: dever.system.Listener) (answer: Bool) recover("test observes accepted failures in its assertions") { answer = false } +connected(result: dever.net.ConnectResult.Connected(client), listener: dever.system.Listener) (answer: Bool) recover("test observes connected failures in its assertions") { answer = accepted(result(dever.net.accept(listener)), client, listener) } +connected(result: other, listener: dever.system.Listener) (answer: Bool) recover("test observes connected failures in its assertions") { answer = false } +bound(result: dever.net.PortResult.Bound(port), listener: dever.system.Listener) (answer: Bool) recover("test observes bound failures in its assertions") { answer = connected(result(dever.net.connect("127.0.0.1", port)), listener) } +bound(result: other, listener: dever.system.Listener) (answer: Bool) recover("test observes bound failures in its assertions") { answer = false } +listening(result: dever.net.ListenResult.Listening(listener)) (answer: Bool) recover("test observes listening failures in its assertions") { answer = bound(result(dever.net.port(listener)), listener) } +listening(result: other) (answer: Bool) recover("test observes listening failures in its assertions") { answer = false } +public main() (answer: Bool) { answer = listening(result(dever.net.listen("127.0.0.1", 0))) } +"# + ), + "answer = true\n", + ); +} diff --git a/test/dever-tests/tests/auth_runtime.rs b/test/dever-tests/tests/auth_runtime.rs new file mode 100644 index 0000000..1dec08b --- /dev/null +++ b/test/dever-tests/tests/auth_runtime.rs @@ -0,0 +1,249 @@ +#[path = "support/temp.rs"] +mod temp; + +use std::fs; + +use dever_runtime::api; +use dever_runtime::auth::{self, Error, Identity}; +use dever_runtime::bytes::Bytes; +use dever_runtime::config::Settings; +use dever_runtime::http::{Header, Request}; + +fn settings() -> (temp::TemporaryDirectory, Settings) { + let root = temp::TemporaryDirectory::new(); + fs::create_dir(root.path().join("config")).unwrap(); + fs::write(root.path().join("config/setting.json"), r#"{ + "auth": { + "providers": { + "session": { + "verify": "user.account.verify", + "jwtSecret": "0123456789abcdef0123456789abcdef", + "ttlSeconds": 3600, + "cookie": "cms_session" + } + } + }, + "sites": { + "admin": {"path": "admin", "auth": "session", "hosts": ["cms.example.com"], "origin": "https://cms.example.com"}, + "front": {"path": "front", "auth": "session", "hosts": ["cms.example.com"], "origin": "https://cms.example.com"} + } + }"#).unwrap(); + let settings = Settings::load_project(root.path()).unwrap(); + (root, settings) +} + +fn request(method: &str, extra_headers: Vec<Header>) -> Request { + let mut headers = vec![Header { + name: "host".into(), + value: Bytes::from_text("cms.example.com"), + }]; + headers.extend(extra_headers); + Request { + method: method.into(), + target: "/news/article/profile".into(), + headers, + body: Bytes::from_text(""), + } +} + +#[tokio::test] +async fn site_scope_issues_and_verifies_a_bound_identity() { + let (_root, settings) = settings(); + let login_request = request( + "POST", + vec![Header { + name: "origin".into(), + value: Bytes::from_text("https://cms.example.com"), + }], + ); + let login_settings = &settings; + let response = api::scoped_request(login_request, move |request| async move { + let prepared = auth::prepare_with_settings(login_settings, &["front"], &request, true) + .map_err(|error| error.to_string())?; + auth::scope(prepared, None, async { + auth::issue_cookie("account:7", "session:19", Some("tenant:3"))?; + api::commit_response_metadata()?; + Ok(api::success(dever_runtime::wire::Encoded::null())) + }) + .await + .map_err(|error| error.to_string())? + }) + .await + .unwrap(); + + let cookie = response + .headers + .iter() + .find(|header| header.name == "set-cookie") + .map(|header| { + std::str::from_utf8(header.value.values()) + .unwrap() + .split(';') + .next() + .unwrap() + .to_owned() + }) + .expect("issued session cookie"); + let authenticated = request( + "GET", + vec![Header { + name: "cookie".into(), + value: Bytes::from_string(cookie.clone()), + }], + ); + let prepared = + auth::prepare_with_settings(&settings, &["front", "profile"], &authenticated, false) + .unwrap(); + let claims = prepared.claims().unwrap(); + assert_eq!(claims.subject, "account:7"); + assert_eq!(claims.session, "session:19"); + assert_eq!(claims.tenant.as_deref(), Some("tenant:3")); + assert_eq!(claims.site, "front"); + + let identity = Identity { + id: "account:7".into(), + user_id: Some(7), + tenant_id: Some(3), + }; + auth::scope(prepared, Some(identity), async { + assert_eq!(auth::site_key().unwrap(), "front"); + assert_eq!(auth::id().unwrap(), "account:7"); + assert_eq!(auth::session().unwrap(), "session:19"); + assert_eq!(auth::user_id().unwrap(), Some(7)); + assert_eq!(auth::tenant_id().unwrap(), Some(3)); + #[cfg(feature = "sqlite")] + dever_runtime::job::bind_permission("news.article.front.read").unwrap(); + #[cfg(feature = "sqlite")] + assert!(matches!( + dever_runtime::job::execution_identity().unwrap(), + dever_runtime::job::ExecutionIdentity::User { permission_key, .. } + if permission_key == "news.article.front.read" + )); + }) + .await + .unwrap(); + + let cross_site = request( + "GET", + vec![Header { + name: "cookie".into(), + value: Bytes::from_string(cookie), + }], + ); + assert_eq!( + auth::prepare_with_settings(&settings, &["admin"], &cross_site, false) + .err() + .expect("site-bound token must fail"), + Error::Unauthorized + ); +} + +#[test] +fn public_routes_reject_bad_credentials_and_cookie_writes_require_origin() { + let (_root, settings) = settings(); + let invalid = request( + "GET", + vec![Header { + name: "authorization".into(), + value: Bytes::from_text("Bearer invalid"), + }], + ); + assert_eq!( + auth::prepare_with_settings(&settings, &["front"], &invalid, true) + .err() + .expect("invalid credential must not become anonymous"), + Error::Unauthorized + ); + + let cookie_write = request( + "DELETE", + vec![Header { + name: "cookie".into(), + value: Bytes::from_text("cms_session=v1.aW52YWxpZA"), + }], + ); + assert_eq!( + auth::prepare_with_settings(&settings, &["front"], &cookie_write, false) + .err() + .expect("unsafe cookie request requires Origin"), + Error::Forbidden + ); + + let stale_cookie = request( + "POST", + vec![Header { + name: "cookie".into(), + value: Bytes::from_text("cms_session=v1.invalid"), + }], + ); + let prepared = auth::prepare_with_settings(&settings, &["front"], &stale_cookie, true) + .expect("public login must ignore stale Cookie credentials"); + assert!(prepared.claims().is_none()); +} + +#[tokio::test] +async fn unsafe_cookie_writes_require_the_configured_external_origin() { + let (_root, settings) = settings(); + for origin in [ + None, + Some("http://cms.example.com"), + Some("https://cms.example.com:444"), + Some("https://other.example.com"), + ] { + let headers = origin + .into_iter() + .map(|value| Header { + name: "origin".into(), + value: Bytes::from_text(value), + }) + .collect(); + let request = request("POST", headers); + let test_settings = &settings; + let response = api::scoped_request(request, |request| async move { + let prepared = auth::prepare_with_settings(test_settings, &["front"], &request, true) + .map_err(|error| error.to_string())?; + auth::scope(prepared, None, async { + assert!(auth::issue_cookie("account:7", "session:19", Some("tenant:3")).is_err()); + Ok(api::success(dever_runtime::wire::Encoded::null())) + }) + .await + .map_err(|error| error.to_string())? + }) + .await + .unwrap(); + assert!( + !response + .headers + .iter() + .any(|header| header.name == "set-cookie") + ); + } + + let valid = request( + "POST", + vec![Header { + name: "origin".into(), + value: Bytes::from_text("https://cms.example.com"), + }], + ); + let test_settings = &settings; + let response = api::scoped_request(valid, |request| async move { + let prepared = auth::prepare_with_settings(test_settings, &["front"], &request, true) + .map_err(|error| error.to_string())?; + auth::scope(prepared, None, async { + auth::issue_cookie("account:7", "session:19", Some("tenant:3"))?; + api::commit_response_metadata()?; + Ok(api::success(dever_runtime::wire::Encoded::null())) + }) + .await + .map_err(|error| error.to_string())? + }) + .await + .unwrap(); + assert!( + response + .headers + .iter() + .any(|header| header.name == "set-cookie") + ); +} diff --git a/test/dever-tests/tests/authorization_postgres.rs b/test/dever-tests/tests/authorization_postgres.rs new file mode 100644 index 0000000..8a54417 --- /dev/null +++ b/test/dever-tests/tests/authorization_postgres.rs @@ -0,0 +1,59 @@ +mod support; + +use std::ffi::OsStr; + +use dever_runtime::{config, database}; + +#[test] +#[ignore = "requires an isolated database configured in config/setting.json"] +fn postgres_authorization_preserves_site_scoped_roles_and_catalog() { + let executable = std::env::current_exe().unwrap(); + if executable.file_stem().unwrap() != OsStr::new("authorization-postgres-case") { + support::postgres::run_in_isolated_schema("authorization_postgres", |setting| async move { + let project = support::temp::TemporaryDirectory::new(); + std::fs::create_dir(project.path().join("config")).unwrap(); + std::fs::write( + project.path().join("config/setting.json"), + setting.settings_json(), + ) + .unwrap(); + let child = project.path().join("authorization-postgres-case"); + std::fs::copy(executable, &child).unwrap(); + let output = std::process::Command::new(child) + .args([ + "--ignored", + "--exact", + "postgres_authorization_preserves_site_scoped_roles_and_catalog", + "--nocapture", + ]) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + Ok(()) + }); + return; + } + + config::bootstrap( + config::RuntimeProfile { + sqlite: false, + postgres: true, + }, + &[(Some("default"), "auth")], + &[], + ) + .unwrap(); + dever_runtime::task::run_entry(async { + database::prepare().await.unwrap(); + let catalog = database::database_for(Some("default"), "auth").unwrap(); + support::authorization::assert_contract(catalog.clone(), catalog).await; + database::shutdown().await.unwrap(); + Ok(()) + }) + .unwrap(); +} diff --git a/test/dever-tests/tests/authorization_runtime.rs b/test/dever-tests/tests/authorization_runtime.rs new file mode 100644 index 0000000..ec12b44 --- /dev/null +++ b/test/dever-tests/tests/authorization_runtime.rs @@ -0,0 +1,52 @@ +mod support; + +#[cfg(feature = "sqlite")] +#[test] +fn sqlite_authorization_uses_exact_site_scoped_role_permissions() { + use dever_runtime::{config, database}; + + let executable = std::env::current_exe().unwrap(); + if executable.file_stem().unwrap() != "authorization-runtime-case" { + let project = support::temp::TemporaryDirectory::new(); + std::fs::create_dir(project.path().join("config")).unwrap(); + std::fs::write( + project.path().join("config/setting.json"), + r#"{"database":{"default":{"type":"sqlite","path":"data/db/auth.db","max_connections":2}}}"#, + ) + .unwrap(); + let child = project.path().join("authorization-runtime-case"); + std::fs::copy(executable, &child).unwrap(); + let output = std::process::Command::new(child) + .args([ + "--exact", + "sqlite_authorization_uses_exact_site_scoped_role_permissions", + "--nocapture", + ]) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + return; + } + + config::bootstrap( + config::RuntimeProfile { + sqlite: true, + postgres: false, + }, + &[(Some("default"), "auth")], + &[], + ) + .unwrap(); + dever_runtime::task::run_entry(async { + let catalog = database::database_for(Some("default"), "auth").unwrap(); + support::authorization::assert_contract(catalog.clone(), catalog).await; + database::shutdown().await.unwrap(); + Ok(()) + }) + .unwrap(); +} diff --git a/test/dever-tests/tests/cms_acceptance.rs b/test/dever-tests/tests/cms_acceptance.rs new file mode 100644 index 0000000..7b34d3b --- /dev/null +++ b/test/dever-tests/tests/cms_acceptance.rs @@ -0,0 +1,27 @@ +use dever_core::source::SourceMap; +use std::path::Path; + +fn load(root: &Path) -> dever_core::hir::Program { + let sources = SourceMap::load_project(&root.join("module"), &root.join("test")) + .unwrap_or_else(|error| panic!("failed to load {}: {error:?}", root.display())); + dever_core::check(&sources).unwrap_or_else(|errors| { + panic!( + "{} failed:\n{}", + root.display(), + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<Vec<_>>() + .join("\n") + ) + }) +} + +#[test] +fn cms_plain_and_markdown_have_the_same_contracts() { + let workspace = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let plain = load(&workspace.join("examples/cms/dever")); + let markdown = load(&workspace.join("examples/cms/md")); + assert_eq!(plain.api_snapshot(), markdown.api_snapshot()); + assert_eq!(plain.model_snapshot(), markdown.model_snapshot()); +} diff --git a/test/dever-tests/tests/concurrency.rs b/test/dever-tests/tests/concurrency.rs new file mode 100644 index 0000000..3a899f3 --- /dev/null +++ b/test/dever-tests/tests/concurrency.rs @@ -0,0 +1,170 @@ +mod support; + +use dever_runtime::concurrent; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, mpsc}; +use std::time::Duration; + +#[test] +#[ignore = "explicit bounded local scheduling benchmark"] +fn parallel_task_granularity_measurement() { + use std::time::Instant; + let values: Vec<usize> = (0..32_768).collect(); + for rounds in [0, 512] { + for sample in 0..6 { + let total = AtomicUsize::new(0); + let apply = |value: usize| { + let mut result = value; + for _ in 0..rounds { + result = + std::hint::black_box(result.wrapping_mul(1664525).wrapping_add(1013904223)); + } + total.fetch_add(result, Ordering::Relaxed); + Ok(()) + }; + let start = Instant::now(); + concurrent::each(values.iter().copied(), 4, apply).unwrap(); + let parallel = start.elapsed(); + let expected = total.swap(0, Ordering::Relaxed); + let start = Instant::now(); + concurrent::each_slice(&values, 4, |value| apply(*value)).unwrap(); + let batched = start.elapsed(); + assert_eq!(total.swap(0, Ordering::Relaxed), expected); + let start = Instant::now(); + for value in &values { + apply(*value).unwrap(); + } + let serial = start.elapsed(); + assert_eq!(total.load(Ordering::Relaxed), expected); + println!( + "rounds={rounds} sample={sample} queue_ns={} batch_ns={} serial_ns={}", + parallel.as_nanos(), + batched.as_nanos(), + serial.as_nanos() + ); + } + } +} + +#[test] +fn batched_memory_actions_visit_once_and_stop_within_a_batch_on_failure() { + let values: Vec<_> = (0..4096).collect(); + let seen: Vec<_> = values.iter().map(|_| AtomicUsize::new(0)).collect(); + concurrent::each_slice(&values, 4, |value| { + seen[*value].fetch_add(1, Ordering::SeqCst); + Ok(()) + }) + .unwrap(); + assert!(seen.iter().all(|count| count.load(Ordering::SeqCst) == 1)); + let calls = AtomicUsize::new(0); + let failed = concurrent::each_slice(&values, 1, |_| { + calls.fetch_add(1, Ordering::SeqCst); + Err("expected".into()) + }); + assert_eq!(failed, Err("expected".into())); + assert_eq!(calls.load(Ordering::SeqCst), 1); + assert!( + concurrent::each_slice(&values, 0, |_| panic!("invalid workers invoked action")).is_err() + ); + + let calls = AtomicUsize::new(0); + concurrent::each_slice(&[1], 256, |_| { + calls.fetch_add(1, Ordering::SeqCst); + Ok(()) + }) + .unwrap(); + assert_eq!(calls.load(Ordering::SeqCst), 1); +} + +#[test] +fn bounded_workers_overlap_and_all_finish_before_returning() { + let (started, starts) = mpsc::channel(); + let (release, releases) = mpsc::channel(); + let releases = Mutex::new(releases); + let finished = Arc::new(AtomicUsize::new(0)); + let completed = Arc::clone(&finished); + let job = std::thread::spawn(move || { + concurrent::each(0..2, 2, |_| { + started.send(()).unwrap(); + releases + .lock() + .unwrap() + .recv_timeout(Duration::from_secs(2)) + .map_err(|error| error.to_string())?; + completed.fetch_add(1, Ordering::SeqCst); + Ok(()) + }) + }); + let first = starts.recv_timeout(Duration::from_secs(2)); + let second = starts.recv_timeout(Duration::from_secs(2)); + let _ = release.send(()); + let _ = release.send(()); + let result = job.join().unwrap(); + assert!( + first.is_ok() && second.is_ok(), + "both workers must start before either is released" + ); + result.unwrap(); + assert_eq!(finished.load(Ordering::SeqCst), 2); +} + +#[test] +fn invalid_limits_do_not_pull_and_failures_stop_dispatch_without_deadlock() { + for limit in [-1, 0, 257, i64::MAX] { + let pulled = AtomicUsize::new(0); + let values = std::iter::from_fn(|| { + pulled.fetch_add(1, Ordering::SeqCst); + Some(1) + }); + assert!(concurrent::each(values, limit, |_| Ok(())).is_err()); + assert_eq!(pulled.load(Ordering::SeqCst), 0); + } + let called = AtomicUsize::new(0); + let result = concurrent::each(0..100, 1, |_| { + called.fetch_add(1, Ordering::SeqCst); + Err("expected failure".into()) + }); + assert_eq!(result, Err("expected failure".into())); + assert_eq!(called.load(Ordering::SeqCst), 1); +} + +#[test] +fn native_parallel_handlers_are_static_and_join_before_following_statements() { + let output = support::stdout( + r#"show(value: Int, prefix: Text) () { dever.io.println(prefix + int.to_text(value)) } +invoke(route: handler(value: Int, prefix: Text) (), values: List<Int>) () { + parallel_each(route, values, 2, "value=") +} +public main() () { + invoke(show, [1, 2, 3]) + parallel_each(show, dever.bytes.from_text("ab"), 2, "byte=") + dever.io.println("joined") +} +"#, + ); + let mut lines: Vec<_> = output.lines().collect(); + assert_eq!(lines.pop(), Some("joined")); + lines.sort_unstable(); + assert_eq!( + lines, + ["byte=97", "byte=98", "value=1", "value=2", "value=3"] + ); +} + +#[test] +fn parallel_checker_rejects_outputs_and_shared_stream_inputs() { + support::rejected( + "f(value: Int) (out: Int) { out = value }\npublic main() () { parallel_each(f, [1], 2) }", + "zero-output handler", + ); + support::rejected( + "public type Box { stream: Stream<Int> }\nf(value: Box) () {}\npublic run(values: List<Box>) () { parallel_each(f, values, 2) }", + "cannot contain Stream", + ); + let failed = support::run("f(value: Int) () {}\npublic main() () { parallel_each(f, [1], 0) }"); + assert!(!failed.status.success()); + assert!( + String::from_utf8_lossy(&failed.stderr) + .contains("parallel worker count must be between 1 and 256") + ); +} diff --git a/test/dever-tests/tests/contract_api.rs b/test/dever-tests/tests/contract_api.rs new file mode 100644 index 0000000..3de7ddf --- /dev/null +++ b/test/dever-tests/tests/contract_api.rs @@ -0,0 +1,226 @@ +mod support; + +use dever_core::source::SourceMap; +use support::{checked, rejected, sources}; + +#[path = "../../../crates/dever-cli/src/api.rs"] +mod cli_api; + +#[test] +fn application_exports_are_checked_only_with_a_known_entry() { + let program = checked(&sources( + "public main() (answer: Int) { answer = helper() }\npublic helper() (answer: Int) { answer = 1 }\npublic unused() () {}", + )); + let warnings = program.application_warnings("main.main").unwrap(); + assert_eq!(warnings.len(), 2); + assert!(warnings.iter().all(|warning| warning.code == "W007")); + assert!(program.application_warnings("app.missing").is_err()); + let mut map = SourceMap::default(); + map.add( + "main.dever", + "public main() (answer: Int) { answer = utility.read() }", + ); + map.add( + "utility.dever", + "public read() (answer: Int) { answer = 1 }", + ); + let cross_package_program = checked(&map); + assert!( + cross_package_program + .application_warnings("main.main") + .unwrap() + .is_empty() + ); + assert!( + !cross_package_program + .api_snapshot() + .contains("package app depends") + ); +} + +#[test] +fn cli_baseline_is_explicit_never_overwrites_and_blocks_drift() { + let directory = support::temp::TemporaryDirectory::new(); + let path = directory.path().join("dever.api"); + let original = checked(&sources("public main() (answer: Int) { answer = 1 }")); + assert!( + original + .check_api( + &original + .api_snapshot() + .replace("dever-api 5", "dever-api 4") + ) + .unwrap_err() + .contains("expected dever-api 5") + ); + cli_api::check_baseline(&original, directory.path()).unwrap(); + assert!(!path.exists()); + cli_api::write(&original, Some(&path)).unwrap(); + cli_api::check_baseline(&original, directory.path()).unwrap(); + assert!(cli_api::write(&original, Some(&path)).is_err()); + let expanded = checked(&sources( + "public main() (answer: Int) { answer = 1 }\npublic extra() () {}", + )); + assert!( + cli_api::check_baseline(&expanded, directory.path()) + .unwrap_err() + .contains("+ function main.extra") + ); + assert_eq!( + std::fs::read_to_string(path).unwrap(), + original.api_snapshot() + ); +} + +#[test] +fn effects_follow_named_and_forwarded_handlers_without_tainting_pure_calls() { + let shared = r#" +apply(route: handler(value: Int) (answer: Int), value: Int) (answer: Int) { answer = route(value) } +forward(route: handler(value: Int) (answer: Int), value: Int) (answer: Int) { answer = apply(route, value) } +identity(value: Int) (answer: Int) pure { answer = value } +noisy(value: Int) (answer: Int) { dever.io.println("called") +answer = value } +"#; + checked(&sources(&format!( + "{shared}\nmain() (answer: Int) pure {{ answer = forward(identity, 1) }}" + ))); + rejected( + &format!("{shared}\nmain() (answer: Int) pure {{ answer = forward(noisy, 1) }}"), + "has effects: stdout", + ); + rejected( + "apply(route: handler(value: Int) (answer: Int), value: Int) (answer: Int) pure { answer = route(value) }", + "has effects: handler", + ); + checked(&sources(&format!( + "{shared}\nignore(route: handler(value: Int) (answer: Int), value: Int) (answer: Int) {{ answer = value }}\nmain() (answer: Int) pure {{ answer = ignore(noisy, 1) }}" + ))); + + let collection = r#" +map_values(route: handler(value: Int) (answer: Int), values: List<Int>) (answers: List<Int>) { answers = each(route, values) } +identity(value: Int) (answer: Int) pure { answer = value } +noisy(value: Int) (answer: Int) { dever.io.println("called") +answer = value } +"#; + checked(&sources(&format!( + "{collection}\nmain() (answers: List<Int>) pure {{ answers = map_values(identity, [1]) }}" + ))); + rejected( + &format!( + "{collection}\nmain() (answers: List<Int>) pure {{ answers = map_values(noisy, [1]) }}" + ), + "has effects: stdout", + ); +} + +#[test] +fn pure_arithmetic_and_text_are_allowed_but_resource_and_clock_effects_are_rejected() { + checked(&sources( + "public main() (answer: Text) pure { answer = text.trim(\" x \") }", + )); + rejected( + "clock() (result: Int) pure { result = dever.time.unix_millis() }", + "has effects: time", + ); + rejected( + "close_stream(stream: Stream<Int>) () pure { close(stream) }", + "has effects: stream", + ); +} + +#[test] +fn api_snapshot_detects_expansion_and_effect_changes_but_ignores_private_helpers() { + let original = checked(&sources("public main() (answer: Int) { answer = 1 }")); + let baseline = original.api_snapshot(); + assert!(baseline.contains("function main.main")); + assert!(!baseline.contains("function dever.")); + original.check_api(&baseline).unwrap(); + let helper = checked(&sources( + "public main() (answer: Int) { answer = helper() }\nhelper() (answer: Int) { answer = 2 }", + )); + helper.check_api(&baseline).unwrap(); + let expanded = checked(&sources( + "public main() (answer: Int) { answer = 1 }\npublic helper() (answer: Int) { answer = 2 }", + )); + let difference = expanded.check_api(&baseline).unwrap_err(); + assert!(difference.contains("+ function main.helper")); + let noisy = checked(&sources( + "public main() (answer: Int) { dever.io.println(\"x\")\nanswer = 1 }", + )); + assert!( + noisy + .check_api(&baseline) + .unwrap_err() + .contains("effects=[stdout]") + ); +} + +#[test] +fn api_includes_constraints_and_private_construction_without_internal_representation() { + let program = checked(&sources( + "public type Port { value: Int >= 1 and <= 65535\nprivate secret: Text }", + )); + let snapshot = program.api_snapshot(); + assert!(snapshot.contains("construct=private")); + assert!(snapshot.contains("1..65535")); + assert!(!snapshot.contains("secret")); +} + +#[test] +fn api_tracks_private_transitive_recovery_without_exposing_helper_names() { + let original = checked(&sources("public main() (answer: Int) pure { answer = 1 }")); + let changed = checked(&sources( + "public main() (answer: Int) pure { answer = bridge() }\nbridge() (answer: Int) pure { answer = fallback() }\nfallback() (answer: Int) pure recover(\"missing configuration uses default\") { answer = 1 }", + )); + let snapshot = changed.api_snapshot(); + assert!(snapshot.contains("missing configuration uses default")); + assert!(!snapshot.contains("bridge")); + assert!(!snapshot.contains("fallback")); + assert!(changed.check_api(&original.api_snapshot()).is_err()); + assert!( + snapshot.contains("effects=[]"), + "recovery itself is not an external effect" + ); +} + +#[test] +fn recovery_through_handlers_only_propagates_when_the_handler_is_invoked() { + let helpers = "fallback(value: Int) (answer: Int) recover(\"use default\") { answer = value }\napply(route: handler(value: Int) (answer: Int), value: Int) (answer: Int) { answer = route(value) }\nignore(route: handler(value: Int) (answer: Int), value: Int) (answer: Int) { answer = value }\n"; + let invoked = checked(&sources(&format!( + "{helpers}public main() (answer: Int) pure {{ answer = apply(fallback, 1) }}" + ))); + assert!(invoked.api_snapshot().contains("use default")); + let ignored = checked(&sources(&format!( + "{helpers}public main() (answer: Int) pure {{ answer = ignore(fallback, 1) }}" + ))); + assert!(!ignored.api_snapshot().contains("use default")); +} + +#[test] +fn effect_work_queue_preserves_long_chains_and_unused_handler_cycles() { + let mut source = String::from("public main() () { step0() }\n"); + for level in 0..64 { + source.push_str(&format!("step{level}() () {{ step{}() }}\n", level + 1)); + } + source + .push_str("step64() () recover(\"report optional event\") { dever.io.println(\"event\") }"); + let snapshot = checked(&sources(&source)).api_snapshot(); + assert!(snapshot.contains("effects=[stdout]"), "{snapshot}"); + assert!(snapshot.contains("report optional event"), "{snapshot}"); + rejected( + &source.replace("main() () {", "main() () pure {"), + "has effects: stdout", + ); + + let cycle = "public main() () pure { bridge() }\nbridge() () { ignore(bridge) }\nignore(route: handler() ()) () recover(\"skip optional callback\") {}"; + let snapshot = checked(&sources(cycle)).api_snapshot(); + assert!(snapshot.contains("effects=[]"), "{snapshot}"); + assert!(snapshot.contains("skip optional callback"), "{snapshot}"); + let unused = cycle.replace( + "bridge() () { ignore(bridge) }", + "bridge() () { ignore(bridge)\nignore(noisy) }\nnoisy() () recover(\"unused recovery\") { dever.io.println(\"unused\") }", + ); + let snapshot = checked(&sources(&unused)).api_snapshot(); + assert!(snapshot.contains("effects=[]"), "{snapshot}"); + assert!(!snapshot.contains("unused recovery"), "{snapshot}"); +} diff --git a/test/dever-tests/tests/contract_execution.rs b/test/dever-tests/tests/contract_execution.rs new file mode 100644 index 0000000..3789cf7 --- /dev/null +++ b/test/dever-tests/tests/contract_execution.rs @@ -0,0 +1,291 @@ +mod support; + +use dever_core::source::SourceMap; + +#[test] +fn cms_formats_share_source_model_api_and_configuration_contracts() { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../examples/cms"); + let plain = SourceMap::load(&root.join("dever/module")).unwrap(); + let markdown = SourceMap::load(&root.join("md/module")).unwrap(); + let plain_settings = + dever_runtime::config::Settings::load_project(&root.join("dever")).unwrap(); + let markdown_settings = + dever_runtime::config::Settings::load_project(&root.join("md")).unwrap(); + let plain_program = + dever_core::check_with_settings(&plain, &plain_settings).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&plain)) + .collect::<String>() + ) + }); + let markdown_program = dever_core::check_with_settings(&markdown, &markdown_settings) + .unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&markdown)) + .collect::<String>() + ) + }); + assert_no_application_errors(&plain_program, &plain); + assert_no_application_errors(&markdown_program, &markdown); + assert_eq!( + plain_program.model_snapshot(), + markdown_program.model_snapshot(), + ); + assert_eq!( + plain_program.api_snapshot(), + markdown_program.api_snapshot() + ); + // 应用测试也是双源码合同的一部分，避免 Markdown 测试退化为空断言。 + let plain_with_tests = + SourceMap::load_project(&root.join("dever/module"), &root.join("dever/test")).unwrap(); + let markdown_with_tests = + SourceMap::load_project(&root.join("md/module"), &root.join("md/test")).unwrap(); + assert_eq!( + canonical_cms_sources(&plain_with_tests), + canonical_cms_sources(&markdown_with_tests), + ); + + let configuration = |format: &str| -> serde_json::Value { + serde_json::from_str( + &std::fs::read_to_string(root.join(format).join("config/setting.json")).unwrap(), + ) + .unwrap() + }; + assert_eq!(configuration("dever"), configuration("md")); + for (format, program, settings) in [ + ("dever", &plain_program, &plain_settings), + ("md", &markdown_program, &markdown_settings), + ] { + let profile = program.validate_database_settings(settings).unwrap(); + assert!(profile.sqlite && !profile.postgres, "{format}"); + let sources = if format == "dever" { &plain } else { &markdown }; + let generated = dever_core::native::emit_project(program, sources, profile).unwrap(); + assert!(generated.contains( + "dever_runtime::job::scope_system(None, cmd_dispatch(&args[0], &args[1])).await" + )); + assert!(generated.contains( + "dever_runtime::lifecycle::serve_both(dever_runtime::api::serve(api_route), job_serve()).await" + )); + } +} + +fn assert_no_application_errors(program: &dever_core::hir::Program, sources: &SourceMap) { + let errors = program.application_errors().unwrap(); + assert!( + errors.is_empty(), + "{}", + errors + .iter() + .map(|error| error.render(sources)) + .collect::<String>() + ); +} + +fn canonical_cms_sources(sources: &SourceMap) -> std::collections::BTreeMap<String, String> { + use dever_core::syntax::Declaration; + + let mut declarations = std::collections::BTreeMap::new(); + for source in sources.files() { + // 使用编译器 AST 的边界抽取声明，避免测试另写 Markdown fence 解析器。 + let package = dever_core::parse(source).unwrap(); + let name = package + .name + .iter() + .map(|part| part.text.as_str()) + .collect::<Vec<_>>() + .join("."); + let mut code = String::new(); + for declaration in package.declarations { + let span = match declaration { + Declaration::Type(value) => value.span, + Declaration::Function(value) => value.span, + Declaration::External(value) => value.span, + Declaration::Api(value) => value.span, + Declaration::Rest(value) => value.span, + Declaration::Database(value) => value.span, + Declaration::Schedule(value) => value.span, + Declaration::ModelIndex(value) => value.span, + Declaration::Relation(value) => value.span, + Declaration::Seed(value) => value.span, + Declaration::Migration(value) => value.span, + Declaration::ModelSql(value) => value.span, + }; + code.push_str(&source.text()[span.start..span.end]); + code.push('\n'); + } + let mut normalized = SourceMap::default(); + normalized.add(format!("{}.dever", name.replace('.', "/")), code); + let canonical = dever_core::format::format(&normalized.files()[0]).unwrap(); + assert!(declarations.insert(name, canonical).is_none()); + } + declarations +} + +#[test] +fn maintained_examples_pass_the_same_contract_checker() { + for root in example_roots() { + let source_root = root.join("module"); + let program = support::checked(&SourceMap::load(&source_root).unwrap()); + let baseline = root.join("dever.api"); + if baseline.exists() { + program + .check_api(&std::fs::read_to_string(baseline).unwrap()) + .unwrap(); + } + } +} + +fn examples_root() -> std::path::PathBuf { + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../examples/old/dever") +} + +fn example_roots() -> Vec<std::path::PathBuf> { + let examples = examples_root() + .parent() + .unwrap() + .parent() + .unwrap() + .to_path_buf(); + vec![examples.join("cms/dever"), examples.join("cms/md")] +} + +fn archived_sources(root: &std::path::Path) -> SourceMap { + copy_archived_sources(root, false) +} + +fn archived_rejected_sources(root: &std::path::Path) -> SourceMap { + copy_archived_sources(root, true) +} + +fn copy_archived_sources(root: &std::path::Path, remap_main: bool) -> SourceMap { + let loaded = SourceMap::load(root).unwrap(); + let mut sources = SourceMap::default(); + for source in loaded.files() { + let path = if remap_main && source.path() == std::path::Path::new("main.dever") { + std::path::Path::new("app.dever") + } else { + source.path() + }; + sources.add(path, source.text()); + } + sources +} + +#[test] +fn rejected_examples_demonstrate_the_documented_diagnostic() { + for (name, code) in [ + ("range", "C011"), + ("failure", "C012"), + ("pure", "C013"), + ("coverage", "C009"), + ("private", "C006"), + ] { + let sources = + archived_rejected_sources(&examples_root().join("rejected").join(name).join("module")); + let errors = dever_core::check(&sources).unwrap_err(); + assert!( + errors.iter().any(|error| error.code == code), + "{name}: {errors:#?}" + ); + } +} + +#[test] +fn constraints_and_explicit_recovery_preserve_native_execution() { + assert_eq!( + run_example("contracts", "main.main"), + "configured = domain.Port { value = 8080 }\ninvalid = domain.PortResult.Failed(port must be between 1 and 65535)\nfallback = domain.Port { value = 8080 }\n" + ); +} + +fn run_example(name: &str, entry: &str) -> String { + let sources = archived_sources(&examples_root().join(name).join("module")); + let program = support::checked(&sources); + let compiler = std::env::var_os("RUSTC").unwrap_or_else(|| "rustc".into()); + let native = dever_core::native::compile(&program, &sources, entry, &compiler).unwrap(); + let output = std::process::Command::new(native.executable()) + .current_dir(std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../..")) + .arg("sample") + .output() + .unwrap(); + assert!( + output.status.success(), + "{name}: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert!( + output.stderr.is_empty(), + "{name}: {}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).unwrap() +} + +#[test] +fn source_roots_and_sequence_examples_preserve_expected_native_results() { + for (name, entry, expected) in [ + ("hello", "main.main", "hello\n"), + ("privacy", "main.main", "before = 0\nafter = 1\n"), + ( + "sequences", + "main.main", + "mapped = [2, 4, 6]\ntotal = 6\nstopped = main.Summary { total = 6, count = 3 }\nempty_total = 0\nbytes_total = 131\nfirst_value = 2\nmissing = null\n", + ), + ("stream", "main.main", "total = 208\n"), + ( + "library", + "main.main", + "json = {\"message\":\"你好 Dever\",\"total\":6}\n", + ), + ] { + assert_eq!(run_example(name, entry), expected, "{name}"); + } +} + +#[test] +fn text_example_preserves_unicode_null_and_explicit_conversion_failures() { + let output = run_example("text", "main.main"); + assert!(output.starts_with("cleaned = dever\ncharacter = 😀\nslice = 中😀\npieces = [a, , b]\njoined = a||b\nparsed = 42\ninvalid = null\nexact = 1.25\nrounded = 2.34\nutf8 = 你好\n"), "{output}"); + assert!( + output.contains("invalid_utf8 = dever.bytes.DecodeResult.Failed("), + "{output}" + ); + assert!( + output.contains("invalid_byte = dever.bytes.BufferResult.Failed("), + "{output}" + ); +} + +#[test] +fn parallel_example_joins_every_named_task_before_done() { + let output = run_example("parallel", "main.main"); + let mut lines: Vec<_> = output.lines().collect(); + assert_eq!(lines.pop(), Some("done")); + lines.sort(); + assert_eq!(lines, ["alpha", "beta", "gamma"]); +} + +#[test] +fn async_example_owns_tasks_groups_and_channels_until_completion() { + assert_eq!( + run_example("async", "main.main"), + "task_value = 7\ngrouped_value = 21\ncomputed = 144\n" + ); +} + +#[test] +fn system_example_reads_child_inputs_and_returns_invalid_sleep_as_failure() { + let output = run_example("system", "main.main"); + assert!(output.contains("arguments = [sample]"), "{output}"); + assert!(output.contains("clock = "), "{output}"); + assert!( + output.contains("invalid_sleep = dever.time.SleepResult.Failed("), + "{output}" + ); +} diff --git a/test/dever-tests/tests/contract_facts.rs b/test/dever-tests/tests/contract_facts.rs new file mode 100644 index 0000000..6284aa6 --- /dev/null +++ b/test/dever-tests/tests/contract_facts.rs @@ -0,0 +1,249 @@ +use dever_core::diagnostic::Diagnostic; +use dever_core::source::SourceMap; + +fn checked(declarations: &str) -> Result<dever_core::hir::Program, Vec<Diagnostic>> { + let mut sources = SourceMap::default(); + sources.add("main.dever", declarations); + dever_core::check(&sources) +} + +fn accepts(declarations: &str) { + checked(declarations).unwrap_or_else(|errors| panic!("unexpected errors: {errors:#?}")); +} + +fn rejects(declarations: &str) { + let errors = checked(declarations).expect_err("constraint proof must fail"); + assert!( + errors.iter().any(|error| error.code == "C011"), + "{errors:#?}" + ); +} + +#[test] +fn constrained_construction_requires_a_proof_even_in_unused_functions() { + rejects( + "type Port { value: Int >= 1 and <= 65535 }\nbad() (port: Port) { port = Port { value = 0 } }", + ); + rejects( + "type Port { value: Int >= 1 and <= 65535 }\nbad(value: Int) (port: Port) { port = Port { value = value } }", + ); + accepts( + "type Port { value: Int >= 1 and <= 65535 }\nmake() (port: Port) { port = Port { value = 8080 } }", + ); +} + +#[test] +fn clause_domains_establish_safe_construction_and_arithmetic() { + accepts( + "type Port { value: Int >= 1 and <= 65535 }\nmake(value: Int >= 1 and <= 65534) (port: Port) { port = Port { value = value + 1 } }\nmake(value: other) (port: Port) { port = Port { value = 8080 } }", + ); + rejects( + "type Port { value: Int >= 1 and <= 65535 }\nmake(value: Int >= 1 and <= 65535) (port: Port) { port = Port { value = value + 1 } }\nmake(value: other) (port: Port) { port = Port { value = 8080 } }", + ); +} + +#[test] +fn constrained_outputs_and_input_transfer_survive_function_boundaries() { + accepts( + "type Port { value: Int >= 1 and <= 65535 }\nidentity(value: Int) (result: Int) { result = value }\nnumber(port: Port) (value: Int >= 1 and <= 65535) { value = identity(port.value) }\nmake(port: Port) (result: Port) { result = Port { value = number(port) } }", + ); + rejects("bad(value: Int) (result: Int >= 1) { result = value }"); + accepts("valid() (result: Int >= 1) { result = 1 }"); +} + +#[test] +fn choice_payload_constraints_apply_at_construction_and_binding() { + rejects( + "type Result { Ready(value: Int >= 1) }\nbad() (result: Result) { result = Result.Ready(0) }", + ); + accepts( + "type Result { Ready(value: Int >= 1) }\nread(result: Result.Ready(value)) (number: Int >= 1) { number = value }", + ); +} + +#[test] +fn nested_mutation_rechecks_constraints_and_aliases_keep_independent_facts() { + rejects( + "type Port { value: Int >= 1 }\ntype Config { port: Port }\nbad(config: Config) (result: Config) { config.port.value = 0\nresult = config }", + ); + accepts( + "type Box { value: Int }\nmake() (result: Int >= 1) { original = Box { value = 1 }\nalias = original\noriginal.value = 0\nresult = alias.value }", + ); + rejects( + "type Box { value: Int }\nmake() (result: Int >= 1) { original = Box { value = 1 }\noriginal.value = 0\nresult = original.value }", + ); +} + +#[test] +fn reassignment_removes_previous_range_guarantees() { + rejects( + "type Port { value: Int >= 1 }\nmake(input: Int) (port: Port) { value = 1\nvalue = input\nport = Port { value = value } }", + ); + rejects( + "type Box { value: Int }\nmake(box: Box) (result: Int >= 1) { box.value = 0\nresult = box.value }", + ); +} + +#[test] +fn decimal_bounds_reuse_checked_numeric_semantics() { + accepts( + "type Amount { value: Decimal >= 0.0 and <= 10.0 }\nmake(value: Decimal >= 0.0 and <= 9.0) (result: Amount) { result = Amount { value = value + 1.0 } }\nmake(value: other) (result: Amount) { result = Amount { value = 0.0 } }", + ); + rejects( + "type Amount { value: Decimal >= 0.0 }\nmake(value: Decimal) (result: Amount) { result = Amount { value = value } }", + ); +} + +#[test] +fn handler_inputs_use_declared_bounds_including_sequence_adapters() { + accepts( + "call(step: handler(value: Int >= 1) (result: Int >= 1)) (result: Int >= 1) { result = step(1) }", + ); + rejects( + "call(step: handler(value: Int >= 1) (result: Int)) (result: Int) { result = step(0) }", + ); + rejects("call(step: handler(value: Int >= 1) ()) () { each(step, [0]) }"); +} + +#[test] +fn proven_comparisons_and_call_domains_have_evidence_without_rejecting_callees() { + let program = checked("type Port { value: Int >= 1 }\npositive(value: Int >= 1) (result: Bool) { result = true }\npositive(value: other) (result: Bool) { result = false }\ncheck(port: Port) (result: Bool) { compared = port.value < 0\nresult = positive(port.value) and compared }").unwrap(); + let warnings: Vec<_> = program + .warnings() + .iter() + .filter(|warning| warning.code == "W002") + .collect(); + assert!( + warnings + .iter() + .any(|warning| warning.message.contains("comparison")) + ); + assert!( + warnings + .iter() + .any(|warning| warning.message.contains("cannot run")) + ); + assert!(warnings.iter().all(|warning| !warning.related.is_empty())); + accepts( + "positive(value: Int >= 1) (result: Bool) { result = true }\npositive(value: other) (result: Bool) { result = false }", + ); +} + +#[test] +fn constant_calls_and_unknown_comparisons_do_not_create_redundancy_noise() { + let mut sources = SourceMap::default(); + let id = sources.add("main.dever", "positive(value: Int >= 1) (result: Bool) { result = true }\npositive(value: other) (result: Bool) { result = false }\nmain(value: Int) (result: Bool) { known = positive(1)\nresult = value < 0 and known }"); + let program = dever_core::check(&sources).unwrap(); + assert!( + !program + .warnings() + .iter() + .any(|warning| warning.code == "W002" && warning.primary.source == id) + ); +} + +#[test] +fn bool_null_and_choice_facts_reach_call_sites() { + let mut sources = SourceMap::default(); + let id = sources.add("main.dever", "type Choice { Yes\nNo }\ntype State { enabled: Bool\nchoice: Choice\noptional: Int? }\nflag(value: true) () {}\nflag(value: false) () {}\noption(value: Int) () {}\noption(value: null) () {}\nchoose(value: Choice.Yes) () {}\nchoose(value: Choice.No) () {}\nmain() () { state = State { enabled = true\nchoice = Choice.Yes\noptional = null }\nflag(state.enabled)\noption(state.optional)\nchoose(state.choice) }"); + let program = dever_core::check(&sources).unwrap(); + let warnings: Vec<_> = program + .warnings() + .iter() + .filter(|warning| warning.code == "W002" && warning.primary.source == id) + .collect(); + assert_eq!(warnings.len(), 3); +} + +#[test] +fn arithmetic_does_not_assume_wrapping_or_unproved_division_bounds() { + rejects( + "type Positive { value: Int >= 1 }\nmake(input: Int) (result: Positive) { result = Positive { value = input + 1 } }", + ); + rejects( + "type Positive { value: Int >= 1 }\nmake(input: Int) (result: Positive) { result = Positive { value = input // 2 } }", + ); +} + +#[test] +fn concrete_handler_outputs_are_proved_from_bodies_without_wrapper_annotations() { + accepts( + "apply(step: handler(value: Int >= 1) (result: Int >= 1)) (result: Int >= 1) { result = step(1) }\nidentity(value: Int) (result: Int) { result = value }\nmain() (result: Int >= 1) { result = apply(identity) }", + ); + accepts( + "apply(step: handler(value: Int >= 1) (result: Int >= 1)) (result: Int >= 1) { result = step(1) }\ngood(value: Int) (result: Int) { result = 2 }\nmain() (result: Int >= 1) { result = apply(good) }", + ); + rejects( + "apply(step: handler(value: Int >= 1) (result: Int >= 1)) (result: Int >= 1) { result = step(1) }\nbad(value: Int) (result: Int) { result = 0 }\nmain() (result: Int >= 1) { result = apply(bad) }", + ); +} + +#[test] +fn forwarded_handler_inputs_are_contravariant_and_outputs_covariant() { + accepts( + "apply(step: handler(value: Int >= 1) (result: Int >= 1)) (result: Int >= 1) { result = step(1) }\nforward(step: handler(value: Int >= 0) (result: Int >= 2)) (result: Int >= 1) { result = apply(step) }", + ); + for signature in [ + "handler(value: Int >= 2) (result: Int >= 1)", + "handler(value: Int >= 0) (result: Int >= 0)", + ] { + let source = format!( + "apply(step: handler(value: Int >= 1) (result: Int >= 1)) (result: Int >= 1) {{ result = step(1) }}\nforward(step: {signature}) (result: Int >= 1) {{ result = apply(step) }}" + ); + let errors = checked(&source).expect_err("unsafe handler variance"); + assert!( + errors.iter().any(|error| error.code == "C005"), + "{errors:#?}" + ); + } +} + +#[test] +fn shared_value_graphs_keep_linear_facts_and_independent_mutation_snapshots() { + const DEPTH: usize = 24; + let mut declarations = String::from("type Layer0 { value: Int >= 1 }\n"); + for level in 1..=DEPTH { + declarations.push_str(&format!( + "type Layer{level} {{ left: Layer{}\nright: Layer{} }}\n", + level - 1, + level - 1 + )); + } + for (flag, initial) in [("true", 1), ("false", 2)] { + declarations.push_str(&format!( + "build(flag: {flag}) (result: Layer{DEPTH}) {{\nv0 = Layer0 {{ value = {initial} }}\n" + )); + for level in 1..=DEPTH { + declarations.push_str(&format!( + "v{level} = Layer{level} {{ left = v{}\nright = v{} }}\n", + level - 1, + level - 1 + )); + } + declarations.push_str(&format!("result = v{DEPTH}\n}}\n")); + } + let path = format!("{}value", "left.".repeat(DEPTH)); + declarations.push_str(&format!("main() (result: Int >= 1 and <= 2) {{ value = build(true)\nalias = value\nvalue.{path} = 3\nresult = alias.{path}\n}}")); + accepts(&declarations); + let mutated = declarations.replace( + &format!("result = alias.{path}"), + &format!("result = value.{path}"), + ); + rejects(&mutated); +} + +#[test] +fn excessive_fact_work_is_an_explicit_diagnostic() { + // A broad but shallow source exercises the budget without allocating an + // exponentially nested value or risking the host process's memory. + let mut declarations = String::from("main(input: Int) (result: Int) {\n"); + for _ in 0..5500 { + declarations.push_str("result = input + 1\n"); + } + declarations.push('}'); + let errors = checked(&declarations).expect_err("proof budget must be enforced"); + assert!( + errors.iter().any(|error| error.code == "C015"), + "{errors:#?}" + ); +} diff --git a/test/dever-tests/tests/contract_redundancy.rs b/test/dever-tests/tests/contract_redundancy.rs new file mode 100644 index 0000000..a2b4b04 --- /dev/null +++ b/test/dever-tests/tests/contract_redundancy.rs @@ -0,0 +1,101 @@ +mod support; + +use support::{checked, sources}; + +fn warnings(source: &str, code: &str) -> Vec<dever_core::diagnostic::Diagnostic> { + let sources = sources(source); + let user_source = sources.files()[0].id(); + checked(&sources) + .warnings() + .iter() + .filter(|warning| warning.code == code && warning.primary.source == user_source) + .cloned() + .collect() +} + +#[test] +fn identical_private_implementations_are_reported_with_source_evidence() { + let source = "first(value: Int) (answer: Int) { answer = value + 1 }\nsecond(renamed: Int) (answer: Int) { answer = renamed + 1 }"; + let duplicates = warnings(source, "W003"); + assert_eq!(duplicates.len(), 1); + assert_eq!(duplicates[0].related.len(), 1); + assert!(duplicates[0].message.contains("main.first")); + assert!(duplicates[0].message.contains("main.second")); + assert_ne!(duplicates[0].primary, duplicates[0].related[0].span); +} + +#[test] +fn visibility_contract_and_operator_differences_are_not_duplicates() { + for source in [ + "public first(value: Int) (answer: Int) { answer = value + 1 }\nsecond(value: Int) (answer: Int) { answer = value + 1 }", + "first(value: Int) (answer: Int) pure { answer = value + 1 }\nsecond(value: Int) (answer: Int) { answer = value + 1 }", + "first(value: Int) (answer: Int) { answer = value + 1 }\nsecond(value: Int) (answer: Int) { answer = value - 1 }", + "first(value: Int) (answer: Int) { answer = value }\nsecond(value: Int) (different: Int) { different = value }", + ] { + assert!(warnings(source, "W003").is_empty(), "{source}"); + } +} + +#[test] +fn repeated_pure_calls_require_unchanged_inputs_and_an_available_result() { + let prefix = "calculate(value: Int) (answer: Int) { answer = value + 1 }\n"; + let repeated = format!( + "{prefix}main(value: Int) (answer: Int) {{ first = calculate(value)\nsecond = calculate(value)\nanswer = first + second }}" + ); + let diagnostics = warnings(&repeated, "W004"); + assert_eq!(diagnostics.len(), 1); + assert_eq!(diagnostics[0].related.len(), 1); + for body in [ + "first = calculate(value)\nvalue = value + 1\nsecond = calculate(value)\nanswer = first + second", + "first = calculate(value)\nfirst = 0\nsecond = calculate(value)\nanswer = first + second", + ] { + assert!( + warnings( + &format!("{prefix}main(value: Int) (answer: Int) {{ {body} }}"), + "W004" + ) + .is_empty() + ); + } +} + +#[test] +fn field_writes_and_effectful_calls_are_not_reuse_candidates() { + let field = "public type Counter { value: Int }\ncalculate(value: Int) (answer: Int) { answer = value + 1 }\npublic main(counter: Counter) (answer: Int) { first = calculate(counter.value)\ncounter.value = counter.value + 1\nsecond = calculate(counter.value)\nanswer = first + second }"; + assert!(warnings(field, "W004").is_empty()); + let effects = "noisy(value: Int) (answer: Int) { dever.io.println(\"called\")\nanswer = value }\npublic main(value: Int) (answer: Int) { first = noisy(value)\nsecond = noisy(value)\nanswer = first + second }"; + assert!(warnings(effects, "W004").is_empty()); +} + +#[test] +fn forwarding_warning_excludes_public_adapters_recovery_and_constrained_outputs() { + let direct = "calculate(value: Int) (answer: Int) { answer = value + 1 }\nwrapper(value: Int) (answer: Int) { answer = calculate(value) }"; + assert_eq!(warnings(direct, "W005").len(), 1); + for source in [ + "calculate(value: Int) (answer: Int) { answer = value + 1 }\npublic wrapper(value: Int) (answer: Int) { answer = calculate(value) }", + "calculate(value: Int) (answer: Int) { answer = value + 1 }\nwrapper(value: Int) (answer: Int) recover(\"application policy\") { answer = calculate(value) }", + "calculate() (answer: Int >= 1) { answer = 1 }\nwrapper() (answer: Int >= 1) { answer = calculate() }", + "type PrivateValue { private value: Int }\ncreate(value: Int) (answer: PrivateValue) { answer = PrivateValue { value = value } }\nwrapper(value: Int) (answer: PrivateValue) { answer = create(value) }", + ] { + assert!(warnings(source, "W005").is_empty(), "{source}"); + } +} + +#[test] +fn existing_unused_write_warning_is_not_duplicated_as_dead_computation() { + let program = checked(&sources( + "calculate(value: Int) (answer: Int) { answer = value + 1 }\npublic main() () { unused = calculate(1) }", + )); + assert!( + program + .warnings() + .iter() + .any(|warning| warning.code == "W001") + ); + assert!( + !program + .warnings() + .iter() + .any(|warning| warning.code == "W006") + ); +} diff --git a/test/dever-tests/tests/contract_semantics.rs b/test/dever-tests/tests/contract_semantics.rs new file mode 100644 index 0000000..8100c95 --- /dev/null +++ b/test/dever-tests/tests/contract_semantics.rs @@ -0,0 +1,236 @@ +mod support; + +use support::{checked, rejected, sources}; + +const RESULT: &str = r#"type Result { Ready(value: Int) +error Failed(message: Text) } +fail_value(message: Text) (value: Int) { fail(Result.Failed(message)) } +failure(message: Text) (value: Result) { value = result(fail_value(message)) } +"#; + +#[test] +fn failure_cannot_be_discarded_overwritten_logged_or_laundered() { + for code in [ + "run() () { result = failure(\"bad\") }", + "run() (result: Result) { result = failure(\"bad\")\nresult = Result.Ready(0) }", + "run(result: Result.Ready(value)) (out: Int) { out = value }\nrun(result: Result.Failed(message)) (out: Int) { dever.io.println(message)\nout = 0 }", + "ignore(result: Result) () {}\nrun() () { ignore(failure(\"bad\")) }", + ] { + rejected(&format!("{RESULT}{code}"), "failure is not handled"); + } +} + +#[test] +fn errors_can_be_returned_translated_or_explicitly_recovered() { + for code in [ + "run() (result: Result) { result = failure(\"bad\") }", + "run(result: Result) (out: Result) { out = result }", + "run(result: Result.Ready(value)) (out: Result) { out = Result.Ready(value) }\nrun(result: Result.Failed(_)) (out: Result) { fail(Result.Failed(\"translated\")) }", + "run(result: Result.Ready(value)) (out: Int) recover(\"optional configuration uses zero\") { out = value }\nrun(result: Result.Failed(_)) (out: Int) recover(\"optional configuration uses zero\") { out = 0 }", + ] { + checked(&sources(&format!("{RESULT}{code}"))); + } +} + +#[test] +fn nested_failure_survives_value_copies_and_projection() { + rejected( + &format!( + "{RESULT}type Box {{ result: Result\nnumber: Int }}\nrun() (out: Int) {{ box = Box {{ result = failure(\"bad\")\nnumber = 3 }}\nout = box.number }}" + ), + "failure is not handled", + ); + checked(&sources(&format!( + "{RESULT}type Box {{ result: Result }}\nrun() (out: Box) {{ box = Box {{ result = failure(\"bad\") }}\nout = box }}" + ))); +} + +#[test] +fn possibly_empty_or_null_outputs_cannot_hide_another_failure() { + for (ty, extra) in [ + ("List<Problem>", ""), + ("Box?", "public type Box { problem: Problem }"), + ] { + rejected( + &format!( + "public type Problem {{ error Bad }}\n{extra}\npublic run(problem: Problem.Bad, replacement: {ty}) (result: {ty}) {{ result = replacement }}" + ), + "failure is not handled", + ); + } + checked(&sources( + "public type Problem { error Bad }\npublic type Box { problem: Problem }\npublic run(box: Box, replacement: Problem) (result: Problem) { result = replacement }", + )); +} + +#[test] +fn empty_traversal_does_not_handle_context_and_singleton_projection_transfers_failure() { + rejected( + "public type Problem { error Bad }\nstep(value: Int, problem: Problem) () recover(\"report optional diagnostic\") {}\npublic run(problem: Problem) () { each(step, [], problem) }", + "failure is not handled", + ); + checked(&sources( + "public type Problem { error Bad }\npublic run(problem: Problem) (result: Problem?) { result = first([problem]) }", + )); + rejected( + "public type Problem { error Bad }\npublic run(first_problem: Problem, second_problem: Problem) (result: Problem?) { result = first([first_problem, second_problem]) }", + "failure is not handled", + ); + rejected( + "public type Result { Ready\nerror Failed }\npublic run(problem: Result) (result: Map<Text, Result>) { old = {\"key\" = problem}\nresult = put(old, \"key\", Result.Ready) }", + "failure is not handled", + ); + rejected( + "public type Result { Ready\nerror Failed }\npublic run(problem: Result, key: Text) (result: Map<Text, Result>) { result = {key = problem\n\"key\" = Result.Ready} }", + "failure is not handled", + ); +} + +#[test] +fn short_circuit_does_not_claim_a_skipped_handler_consumes_failures() { + for left in ["false", "condition"] { + rejected( + &format!( + "public type Problem {{ error Bad }}\nreport(problem: Problem) (result: Bool) recover(\"report optional failure\") {{ result = true }}\npublic run(problem: Problem, condition: Bool) (result: Bool) {{ result = {left} and report(problem) }}" + ), + "failure is not handled", + ); + } +} + +#[test] +fn empty_reduce_preserves_the_initial_failure_obligation() { + let declarations = "type Problem { error Bad }\nstep(value: Int, state: Problem) (next: Problem) { next = state }\n"; + rejected( + &format!( + "{declarations}run(problem: Problem) () {{ discarded = reduce(step, [], problem) }}" + ), + "failure is not handled", + ); + checked(&sources(&format!( + "{declarations}run(problem: Problem) (result: Problem) {{ result = reduce(step, [], problem) }}" + ))); +} + +#[test] +fn shared_type_graphs_do_not_expand_without_bounded_failure_work() { + for error in [false, true] { + let mut source = String::from(""); + source.push_str(if error { + "type T0 { error Bad }\n" + } else { + "type T0 { value: Int }\n" + }); + for level in 1..=24 { + source.push_str(&format!( + "type T{level} {{ left: T{}\nright: T{} }}\n", + level - 1, + level - 1 + )); + } + source.push_str("accept(value: T24) (result: T24) { result = value }"); + if error { + rejected(&source, "failure analysis exceeds its structural limit"); + } else { + checked(&sources(&source)); + } + } +} + +#[test] +fn nominal_depth_is_checked_before_recursive_contract_analysis() { + let mut source = String::from("type T0 { error Bad }\n"); + for level in 1..=256 { + source.push_str(&format!("type T{level} {{ value: T{} }}\n", level - 1)); + } + source.push_str("accept(value: T256) (result: T256) { result = value }"); + rejected( + &source, + "contract type nesting exceeds the structural depth limit", + ); +} + +#[test] +fn shared_nominal_properties_preserve_equality_and_worker_resource_boundaries() { + const DEPTH: usize = 48; + for (leaf, comparable, transferable) in [ + ("Text", true, true), + ("dever.system.File", false, true), + ("dever.system.Socket", false, true), + ("dever.system.Listener", false, true), + ("Stream<Int>", false, false), + ] { + let mut declarations = format!("type T0 {{ value: {leaf} }}\n"); + // Declare callers before their dependencies, so source order cannot stand in + // for the checked type graph. Each level shares the previous type twice. + for level in (1..=DEPTH).rev() { + declarations.push_str(&format!( + "type T{level} {{ left: T{}\nright: T{} }}\n", + level - 1, + level - 1 + )); + } + let equality = format!( + "{declarations}run(left: T{DEPTH}, right: T{DEPTH}) (same: Bool) {{ same = left == right }}" + ); + if comparable { + checked(&sources(&equality)); + } else { + rejected(&equality, "equality requires the same comparable type"); + } + let parallel = format!( + "{declarations}accept(value: T{DEPTH}) () {{}}\nrun(values: List<T{DEPTH}>) () {{ parallel_each(accept, values, 2) }}" + ); + if transferable { + checked(&sources(¶llel)); + } else { + rejected(¶llel, "cannot contain Stream"); + } + } +} + +#[test] +fn failure_summaries_follow_long_calls_and_unused_handler_dependency_cycles() { + // 已有失败 payload 保留“必定携带失败”的摘要；叶子允许恢复，负向只检验调用者的原失败。 + let prefix = "type Problem { error Bad }\ntype Result { Ready(problem: Problem)\nEmpty }\n"; + let mut declarations = String::from(prefix); + for level in 0..64 { + declarations.push_str(&format!( + "step{level}(replacement: Problem) (result: Result) {{ result = step{}(replacement) }}\n", + level + 1 + )); + } + declarations + .push_str("step64(replacement: Problem) (result: Result) recover(\"replacement policy\") { result = Result.Ready(replacement) }\n"); + declarations.push_str("run(problem: Problem, replacement: Problem) (result: Result) { result = step0(replacement) }"); + checked(&sources(&declarations)); + rejected( + &declarations.replace("Result.Ready(replacement)", "Result.Empty"), + "failure is not handled", + ); + + let cycle = format!( + "{prefix}run(problem: Problem, value: Problem) (result: Result) {{ result = replacement(value) }}\nreplacement(value: Problem) (result: Result) {{ result = ignore(replacement, value) }}\nignore(route: handler(value: Problem) (result: Result), value: Problem) (result: Result) recover(\"replacement policy\") {{ result = Result.Ready(value) }}" + ); + checked(&sources(&cycle)); + rejected( + &cycle.replace("Result.Ready(value)", "Result.Empty"), + "failure is not handled", + ); +} + +#[test] +fn private_fields_protect_access_mutation_and_construction() { + for expression in [ + "out = box.value", + "box.value = 2\nout = 0", + "box = secret.Box { value = 2 }\nout = 0", + ] { + let mut source = sources(&format!( + "public run(box: secret.Box) (out: Int) {{ {expression} }}" + )); + source.add("secret.dever", "public type Box { private value: Int }"); + let diagnostics = dever_core::check(&source).unwrap_err(); + assert!(diagnostics.iter().any(|error| error.code == "C006")); + } +} diff --git a/test/dever-tests/tests/contract_syntax.rs b/test/dever-tests/tests/contract_syntax.rs new file mode 100644 index 0000000..46e86a6 --- /dev/null +++ b/test/dever-tests/tests/contract_syntax.rs @@ -0,0 +1,168 @@ +use dever_core::source::SourceMap; +use dever_core::syntax::{Declaration, InputKind, Package, TypeShape}; + +fn parsed(source: &str) -> Package { + let mut sources = SourceMap::default(); + let id = sources.add("main.dever", source); + dever_core::parse(sources.get(id)).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }) +} + +fn formatted(source: &str) -> String { + let mut sources = SourceMap::default(); + let id = sources.add("main.dever", source); + dever_core::format::format(sources.get(id)).expect("valid contract syntax") +} + +#[test] +fn parses_package_type_and_function_contracts() { + let package = parsed( + r#"public type Port { + private value: Int >= 1 and <= 65535 + scale: Decimal > -1.5 +} +public type Result { + Ready(port: Port) + error Failed(message: Text) +} +public read(step: handler(value: Int >= 1) (result: Int <= 65535)) (result: Int >= 1) recover("configured fallback") pure { + result = 8080 +}"#, + ); + let Declaration::Type(port) = &package.declarations[0] else { + panic!("record") + }; + let TypeShape::Record(fields) = &port.shape else { + panic!("record") + }; + assert!(fields[0].private); + assert_eq!(fields[0].bounds.len(), 2); + assert!(!fields[1].private); + assert_eq!(fields[1].bounds[0].number, "-1.5"); + let Declaration::Type(result) = &package.declarations[1] else { + panic!("choice") + }; + let TypeShape::Choice(variants) = &result.shape else { + panic!("choice") + }; + assert!(!variants[0].error); + assert!(variants[1].error); + assert_eq!(variants[1].name.text, "Failed"); + let Declaration::Function(function) = &package.declarations[2] else { + panic!("function") + }; + assert!(function.pure); + assert_eq!(function.recovery.as_deref(), Some("configured fallback")); + assert_eq!(function.outputs[0].bounds.len(), 1); + let InputKind::Handler(handler) = &function.inputs[0].kind else { + panic!("handler") + }; + assert_eq!(handler.inputs[0].bounds.len(), 1); + assert_eq!(handler.outputs[0].bounds.len(), 1); +} + +#[test] +fn preserves_contextual_identifiers() { + let package = parsed( + "type Record { private: Int }\ntype Choice { error(value: Int) }\ndepends() () {}\npure() () {}\nrecover() () {}", + ); + let Declaration::Type(record) = &package.declarations[0] else { + panic!("record") + }; + let TypeShape::Record(fields) = &record.shape else { + panic!("record") + }; + assert!(!fields[0].private); + assert_eq!(fields[0].name.text, "private"); + let Declaration::Type(choice) = &package.declarations[1] else { + panic!("choice") + }; + let TypeShape::Choice(variants) = &choice.shape else { + panic!("choice") + }; + assert!(!variants[0].error); + assert_eq!(variants[0].name.text, "error"); +} + +#[test] +fn depends_remains_an_ordinary_function_name() { + let source = + "depends() (value: Int) { value = 1 }\npublic main() (value: Int) { value = depends() }"; + let output = formatted(source); + assert_eq!(formatted(&output), output); + + let mut sources = SourceMap::default(); + let id = sources.add("main.dever", output); + let package = dever_core::parse(sources.get(id)).unwrap(); + assert!(dever_core::check(&sources).is_ok()); + assert!(matches!(package.declarations[0], Declaration::Function(_))); +} + +#[test] +fn rejects_removed_package_dependency_clauses() { + let mut sources = SourceMap::default(); + let id = sources.add("main.dever", "package app exposes () depends ()"); + assert!(dever_core::parse(sources.get(id)).is_err()); +} + +#[test] +fn rejects_ambiguous_or_invalid_contract_markers() { + for declaration in [ + "f() () pure pure {}", + "f() () recover(\"one\") recover(\"two\") {}", + "f() () recover(\"\") {}", + "f() () recover(\" \") {}", + "f() () recover(1) {}", + "f() (private result: Int) {}", + "f(step: handler(private value: Int) ()) () {}", + "type Result { Ready(private value: Int) }", + "type Result { private Ready }", + "type Port { value: Int >= 1 and }", + ] { + let mut sources = SourceMap::default(); + let id = sources.add("main.dever", declaration.to_string()); + assert!( + dever_core::parse(sources.get(id)).is_err(), + "accepted: {declaration}" + ); + } +} + +#[test] +fn formatting_preserves_contracts_comments_and_canonical_marker_order() { + let source = r#"public type Port { +private value:Int >= 1 and <=65535 # valid range +} +public type Result { +Ready(value:Int >= 1) +error Failed(message:Text) # failure +} +public main() (result:Int >=1) recover("use \"configured\" port") # recovery +pure # effect +{result=8080} +"#; + let output = formatted(source); + assert_eq!(formatted(&output), output); + assert!(output.contains("private value: Int >= 1 and <= 65535")); + assert!(output.contains("error Failed(")); + for comment in ["# valid range", "# failure", "# recovery", "# effect"] { + assert_eq!(output.matches(comment).count(), 1); + } + assert!(output.find("pure # effect").unwrap() < output.find("recover(").unwrap()); + let package = parsed(&output); + let Declaration::Function(function) = &package.declarations[2] else { + panic!("function") + }; + assert!(function.pure); + assert_eq!( + function.recovery.as_deref(), + Some("use \"configured\" port") + ); +} diff --git a/test/dever-tests/tests/core_native.rs b/test/dever-tests/tests/core_native.rs new file mode 100644 index 0000000..b50d325 --- /dev/null +++ b/test/dever-tests/tests/core_native.rs @@ -0,0 +1,446 @@ +mod support; + +use support::{checked, run, sources, stdout}; + +#[test] +fn typed_core_compiles_to_native_values_with_named_outputs() { + let output = stdout(include_str!( + "../../../examples/old/dever/core/module/main.dever" + )); + assert_eq!( + output, + concat!( + "original = 1\nchanged = 99\nchoice = 7\ncategory = large\nexact = true\n", + "quotient = 2.5\nthird = 0.3333333333333333333333333333333333\nrounded = 2.34\n", + "integer_division = -2\nremainder = -1\nselected = [3, 6]\nfound = 3\ntotal = 18\n", + "empty_total = 0\nnames = ABCabc\nordered_keys = [second, first]\nmaps_equal = false\n", + "nan_equal = false\nfinite = true\noptionals = [null, 1, null]\nkey_value = 2\nidentity = true\n", + ) + ); +} + +#[test] +fn source_field_order_actions_and_short_circuiting_are_preserved() { + let output = stdout( + r#"type Pair { + first: Int + second: Int +} +mark(label: Text, value: Int) (answer: Int) { + dever.io.println(label) + answer = value +} +probe(value: Int) (answer: Bool) { + dever.io.println("probe") + answer = value == 2 +} +show(value: Int) () { + dever.io.println("each") +} +public main() (answer: Int?, first_value: Int?, short: Bool, pair_value: Int) { + pair = Pair { + second = mark("second", 2) + first = mark("first", 1) + } + pair_value = pair.first + pair.second + each(show, [1, 2]) + answer = find(probe, [1, 2, 3]) + short = false and probe(10) or true + first_value = first([]) +} +"#, + ); + assert_eq!( + output, + "second\nfirst\neach\neach\nprobe\nprobe\nanswer = 2\nfirst_value = null\nshort = true\npair_value = 3\n" + ); +} + +#[test] +fn nullable_variants_bind_payloads_and_optional_results_flatten() { + let output = stdout( + r#"type Result { + Found(value: Int) + Missing +} +choose(value: Result.Found(payload)) (answer: Int) { answer = payload } +choose(value: Result.Missing) (answer: Int) { answer = -1 } +choose(value: null) (answer: Int) { answer = -2 } +present(value: Int) (answer: Bool) { answer = true } +present(value: null) (answer: Bool) { answer = false } +select(values: List<Int?>) (answer: Int?) { answer = first(values) } +public main() (payload: Int, missing: Int, absent: Int, first_value: Int?, found: Int?, label: Int?) { + payload = choose(Result.Found(7)) + missing = choose(Result.Missing) + absent = choose(null) + first_value = select([null, 1]) + found = find(present, [null, 1, 2]) + labels = { "first" = optional(4) } + label = get(labels, "first") +} +optional(value: Int) (answer: Int?) { answer = value } +"#, + ); + assert_eq!( + output, + "payload = 7\nmissing = -1\nabsent = -2\nfirst_value = null\nfound = 1\nlabel = 4\n" + ); +} + +#[test] +fn dynamic_numeric_faults_retain_the_original_source_location() { + for (function, arguments, message) in [ + ( + "divide(left: Int, right: Int) (answer: Int) { answer = left // right }", + "1, 0", + "Int division by zero", + ), + ( + "divide(left: Decimal, right: Decimal) (answer: Decimal) { answer = left / right }", + "1.0, 0.0", + "Decimal division by zero", + ), + ( + "divide(left: Int, right: Int) (answer: Int) { answer = left + right }", + "9223372036854775807, 1", + "Int overflow", + ), + ] { + let output = run(&format!( + "{function}\npublic main() () {{ answer = divide({arguments}) }}" + )); + assert!(!output.status.success()); + let error = String::from_utf8(output.stderr).unwrap(); + assert!(error.contains("main.dever:2:"), "{error}"); + assert!(error.contains(message), "{error}"); + } +} + +#[test] +fn standard_bytes_are_normal_choice_results_and_not_dynamic_values() { + let output = stdout( + r#"decode(result: dever.bytes.DecodeResult.Decoded(text)) (answer: Text) recover("test observes decode failures in its assertions") { answer = text } +decode(result: dever.bytes.DecodeResult.Failed(message)) (answer: Text) recover("test observes decode failures in its assertions") { answer = "invalid" } +buffer(result: dever.bytes.BufferResult.Ready(bytes)) (answer: Bytes) recover("test observes buffer failures in its assertions") { answer = bytes } +buffer(result: dever.bytes.BufferResult.Failed(message)) (answer: Bytes) recover("test observes buffer failures in its assertions") { answer = dever.bytes.from_text(message) } +public main() (decoded: Text, invalid: Text, count: Int, byte: Int?, outside: Int?) { + original = dever.bytes.from_text("abc") + combined = dever.bytes.concat(original, dever.bytes.from_text("def")) + sliced = buffer(result(dever.bytes.slice(combined, 1, 4))) + decoded = decode(result(dever.bytes.to_text(sliced))) + invalid = decode(result(dever.bytes.to_text(buffer(result(dever.bytes.from_ints([255])))))) + count = dever.bytes.length(original) + byte = dever.bytes.at(original, 1) + outside = dever.bytes.at(original, -1) +} +"#, + ); + assert_eq!( + output, + "decoded = bcd\ninvalid = invalid\ncount = 3\nbyte = 98\noutside = null\n" + ); +} + +#[test] +fn official_file_source_compiles_with_explicit_resource_results() { + let directory = support::temp::TemporaryDirectory::new(); + let path = directory.path().join("native.txt"); + let functions = r#" +created(result: dever.io.OpenResult.Opened(file), path: Text) (answer: Bool) recover("test observes created failures in its assertions") { + alias = file + written = write_ok(result(dever.io.write(file, dever.bytes.from_text("hello")))) + closed = close_ok(result(dever.io.close(file))) + rejected = write_failed(result(dever.io.write(alias, dever.bytes.from_text("bad")))) + content = opened(result(dever.io.open(path))) + answer = written and closed and rejected and content +} +created(result: dever.io.OpenResult.Failed(message), path: Text) (answer: Bool) recover("test observes created failures in its assertions") { answer = false } +opened(result: dever.io.OpenResult.Opened(file)) (answer: Bool) recover("test observes opened failures in its assertions") { + read = read_ok(result(dever.io.read(file, 64))) + closed = close_ok(result(dever.io.close(file))) + answer = read and closed +} +opened(result: dever.io.OpenResult.Failed(message)) (answer: Bool) recover("test observes opened failures in its assertions") { answer = false } +read_ok(result: dever.io.ReadResult.Done(state)) (answer: Bool) recover("test observes read_ok failures in its assertions") { answer = read_state_ok(state) } +read_ok(result: other) (answer: Bool) recover("test observes read_ok failures in its assertions") { answer = false } +read_state_ok(state: dever.io.ReadState.Read(bytes)) (answer: Bool) { answer = decode_ok(result(dever.bytes.to_text(bytes))) } +read_state_ok(state: other) (answer: Bool) { answer = false } +decode_ok(result: dever.bytes.DecodeResult.Decoded(text)) (answer: Bool) recover("test observes decode_ok failures in its assertions") { answer = text == "hello" } +decode_ok(result: other) (answer: Bool) recover("test observes decode_ok failures in its assertions") { answer = false } +write_ok(result: dever.io.WriteResult.Done) (answer: Bool) recover("test observes write_ok failures in its assertions") { answer = true } +write_ok(result: other) (answer: Bool) recover("test observes write_ok failures in its assertions") { answer = false } +write_failed(result: dever.io.WriteResult.Failed(message)) (answer: Bool) recover("test observes write_failed failures in its assertions") { answer = true } +write_failed(result: other) (answer: Bool) recover("test observes write_failed failures in its assertions") { answer = false } +close_ok(result: dever.io.CloseResult.Closed) (answer: Bool) recover("test observes close_ok failures in its assertions") { answer = true } +close_ok(result: other) (answer: Bool) recover("test observes close_ok failures in its assertions") { answer = false } +"#; + let source = format!( + "public main() (result: Bool) {{ result = created(result(dever.io.create({path:?})), {path:?}) }}\n{functions}", + path = path.to_str().unwrap() + ); + assert_eq!(stdout(&source), "result = true\n"); + assert_eq!(std::fs::read_to_string(path).unwrap(), "hello"); +} + +#[test] +fn official_tcp_source_runs_a_single_local_connection_and_closes_it() { + let output = stdout( + r#"public main() (result: Bool) { result = listening(result(dever.net.listen("127.0.0.1", 0))) } +listening(result: dever.net.ListenResult.Listening(listener)) (answer: Bool) recover("test observes listening failures in its assertions") { + answer = bound(result(dever.net.port(listener)), listener) +} +listening(result: other) (answer: Bool) recover("test observes listening failures in its assertions") { answer = false } +bound(result: dever.net.PortResult.Bound(port), listener: dever.system.Listener) (answer: Bool) recover("test observes bound failures in its assertions") { + answer = connected(result(dever.net.connect("127.0.0.1", port)), listener) +} +bound(result: other, listener: dever.system.Listener) (answer: Bool) recover("test observes bound failures in its assertions") { answer = false } +connected(result: dever.net.ConnectResult.Connected(client), listener: dever.system.Listener) (answer: Bool) recover("test observes connected failures in its assertions") { + answer = accepted(result(dever.net.accept(listener)), client, listener) +} +connected(result: other, listener: dever.system.Listener) (answer: Bool) recover("test observes connected failures in its assertions") { answer = false } +accepted(result: dever.net.ConnectResult.Connected(server), client: dever.system.Socket, listener: dever.system.Listener) (answer: Bool) recover("test observes accepted failures in its assertions") { + client_timeout = write_ok(result(dever.net.timeout(client, 1000))) + server_timeout = write_ok(result(dever.net.timeout(server, 1000))) + written = write_ok(result(dever.net.write(client, dever.bytes.from_text("x")))) + received = read_ok(result(dever.net.read(server, 1))) + client_closed = close_ok(result(dever.net.close(client))) + server_closed = close_ok(result(dever.net.close(server))) + listener_closed = close_ok(result(dever.net.close_listener(listener))) + answer = client_timeout and server_timeout and written and received and client_closed and server_closed and listener_closed +} +accepted(result: other, client: dever.system.Socket, listener: dever.system.Listener) (answer: Bool) recover("test observes accepted failures in its assertions") { answer = false } +write_ok(result: dever.io.WriteResult.Done) (answer: Bool) recover("test observes write_ok failures in its assertions") { answer = true } +write_ok(result: other) (answer: Bool) recover("test observes write_ok failures in its assertions") { answer = false } +close_ok(result: dever.io.CloseResult.Closed) (answer: Bool) recover("test observes close_ok failures in its assertions") { answer = true } +close_ok(result: other) (answer: Bool) recover("test observes close_ok failures in its assertions") { answer = false } +read_ok(result: dever.io.ReadResult.Done(state)) (answer: Bool) recover("test observes read_ok failures in its assertions") { answer = read_state_ok(state) } +read_ok(result: other) (answer: Bool) recover("test observes read_ok failures in its assertions") { answer = false } +read_state_ok(state: dever.io.ReadState.Read(bytes)) (answer: Bool) { answer = bytes == dever.bytes.from_text("x") } +read_state_ok(state: other) (answer: Bool) { answer = false } +"#, + ); + assert_eq!(output, "result = true\n"); +} + +#[test] +fn grouping_and_null_only_collections_do_not_change_semantics() { + let output = stdout( + r#"same(value: Int?) (answer: Bool) { answer = (null) == value and value == ((null)) } +public main() (size: Int, equal: Bool, first_value: Int?, mapped: List<Text>, appended: List<Int>, found: Int?) { + values = [(null), 1] + size = length(values) + equal = same(null) + first_value = first([null]) + mapped = each((text.trim), [" a ", " b "]) + appended = append(([]), 1) + labels = put(({}), "a", 2) + found = get(labels, "a") +} +"#, + ); + assert_eq!( + output, + "size = 2\nequal = true\nfirst_value = null\nmapped = [a, b]\nappended = [1]\nfound = 2\n" + ); +} + +#[test] +fn float_patterns_preserve_signed_zero_and_literal_context_is_symmetric() { + let output = stdout( + r#"sign(value: 0.0e0) (answer: Float) { answer = 1.0e0 / value } +sign(value: other) (answer: Float) { answer = 0.0e0 } +compare(value: Float) (answer: Bool) { answer = value < 0.0 and 0.0 > value and value == (-1.0) } +public main() (negative: Bool, comparison: Bool, ordered_sum: Float) { + negative = sign(-0.0e0) < 0.0 + comparison = compare(-1.0e0) + ordered_sum = sum([1.0e16, 1.0e0, -1.0e16]) +} +"#, + ); + assert_eq!( + output, + "negative = true\ncomparison = true\nordered_sum = 0\n" + ); +} + +#[test] +fn reduce_bytes_and_forwarded_handlers_are_direct_native_operations() { + let output = stdout( + r#"add(value: Int, total: Int) (next: Int) { next = total + value } +until(value: Int, total: Int) (next: Int, stop: Bool) { next = total + value +stop = value == 2 } +increment(value: Int) (answer: Int) { answer = value + 1 } +apply(route: handler(value: Int) (answer: Int), value: Int) (answer: Int) { answer = route(value) } +forward(route: handler(value: Int) (answer: Int), value: Int) (answer: Int) { answer = apply(route, value) } +public main() (list_total: Int, empty_total: Int, stopped: Int, bytes_total: Int, mapped: List<Int>, forwarded: Int) { + list_total = reduce(add, [1, 2, 3], 0) + empty_total = reduce(add, [], 7) + stopped = reduce_until(until, [1, 2, 100], 0) + bytes = dever.bytes.from_text("ABC") + bytes_total = reduce(add, bytes, 0) + mapped = each(increment, bytes) + forwarded = forward(increment, 4) +} +"#, + ); + assert_eq!( + output, + "list_total = 6\nempty_total = 7\nstopped = 3\nbytes_total = 198\nmapped = [66, 67, 68]\nforwarded = 5\n" + ); +} + +#[test] +fn handler_codegen_emits_concrete_specializations_without_function_values() { + let sources = sources( + r#"increment(value: Int) (answer: Int) { answer = value + 1 } +decrement(value: Int) (answer: Int) { answer = value - 1 } +apply(route: handler(value: Int) (answer: Int), value: Int) (answer: Int) { answer = route(value) } +public main() (up: Int, down: Int) { up = apply(increment, 1) +down = apply(decrement, 1) } +"#, + ); + let generated = dever_core::native::emit(&checked(&sources), &sources, "main.main").unwrap(); + assert_eq!( + generated + .lines() + .filter(|line| line.starts_with("fn f") && line.contains('h')) + .count(), + 2 + ); + assert!(!generated.contains("dyn Fn")); + assert!(!generated.contains("fn(")); +} + +#[test] +fn bytes_reduce_codegen_iterates_bytes_without_materializing_a_list() { + let sources = sources( + r#"add(value: Int, total: Int) (next: Int) { next = total + value } +public main() (answer: Int) { + bytes = dever.bytes.from_text("ABC") + answer = reduce(add, bytes, 0) +} +"#, + ); + let generated = dever_core::native::emit(&checked(&sources), &sources, "main.main").unwrap(); + assert!(generated.contains("for element in a0.values()")); + assert!(generated.contains("i64::from(*element)")); + assert!(!generated.contains("List::<i64>::new")); +} + +#[test] +fn official_file_chunks_stop_and_resume_the_same_stream_alias() { + let directory = support::temp::TemporaryDirectory::new(); + let path = directory.path().join("chunks.txt"); + std::fs::write(&path, "abcdef").unwrap(); + let source = format!( + r#"decode(result: dever.bytes.DecodeResult.Decoded(text), state: Text) (next: Text) recover("test observes decode failures in its assertions") {{ next = state + text }} +decode(result: dever.bytes.DecodeResult.Failed(message), state: Text) (next: Text) recover("test observes decode failures in its assertions") {{ next = state }} +collect(event: dever.io.ReadEvent.Chunk(bytes), state: Text) (next: Text) recover("test observes collect failures in its assertions") {{ next = decode(result(dever.bytes.to_text(bytes)), state) }} +collect(event: dever.io.ReadEvent.Failed(message), state: Text) (next: Text) recover("test observes collect failures in its assertions") {{ next = state }} +first(event: dever.io.ReadEvent.Chunk(bytes), state: Text) (next: Text, stop: Bool) recover("test observes first failures in its assertions") {{ next = decode(result(dever.bytes.to_text(bytes)), state) +stop = true }} +first(event: dever.io.ReadEvent.Failed(message), state: Text) (next: Text, stop: Bool) recover("test observes first failures in its assertions") {{ next = state +stop = true }} +streaming(result: dever.io.ReadStreamResult.Streaming(stream)) (answer: Text) recover("test observes streaming failures in its assertions") {{ alias = stream +prefix = reduce_until(first, stream, "") +answer = reduce(collect, alias, prefix) +close(stream) }} +streaming(result: dever.io.ReadStreamResult.Failed(message)) (answer: Text) recover("test observes streaming failures in its assertions") {{ answer = message }} +opened(result: dever.io.OpenResult.Opened(file)) (answer: Text) recover("test observes opened failures in its assertions") {{ answer = streaming(result(dever.io.chunks(file, 2))) }} +opened(result: dever.io.OpenResult.Failed(message)) (answer: Text) recover("test observes opened failures in its assertions") {{ answer = message }} +public main() (answer: Text) {{ answer = opened(result(dever.io.open({path:?}))) }} +"#, + path = path.to_str().unwrap() + ); + assert_eq!(stdout(&source), "answer = abcdef\n"); +} + +#[test] +fn official_file_chunks_report_creation_failures_without_a_stream() { + let directory = support::temp::TemporaryDirectory::new(); + let path = directory.path().join("creation-failure.txt"); + std::fs::write(&path, "x").unwrap(); + let source = format!( + r#"failed(result: dever.io.ReadStreamResult.Streaming(stream)) (answer: Bool) recover("test observes failed failures in its assertions") {{ close(stream) +answer = false }} +failed(result: dever.io.ReadStreamResult.Failed(message)) (answer: Bool) recover("test observes failed failures in its assertions") {{ answer = true }} +opened(result: dever.io.OpenResult.Opened(file)) (invalid: Bool, closed: Bool) recover("test observes opened failures in its assertions") {{ + invalid = failed(result(dever.io.chunks(file, 0))) + dever.io.close(file) + closed = failed(result(dever.io.chunks(file, 1))) +}} +opened(result: dever.io.OpenResult.Failed(message)) (invalid: Bool, closed: Bool) recover("test observes opened failures in its assertions") {{ invalid = false +closed = false }} +public main() (invalid: Bool, closed: Bool) {{ result = opened(result(dever.io.open({path:?}))) +invalid = result.invalid +closed = result.closed }} +"#, + path = path.to_str().unwrap() + ); + assert_eq!(stdout(&source), "invalid = true\nclosed = true\n"); +} + +#[test] +fn official_loopback_connections_and_socket_chunks_are_streamed() { + let output = stdout( + r#"type State { + client: dever.system.Socket + text: Text +} +decode(result: dever.bytes.DecodeResult.Decoded(text), state: State) (next: State) recover("test observes decode failures in its assertions") { next = State { client = state.client +text = state.text + text } } +decode(result: dever.bytes.DecodeResult.Failed(message), state: State) (next: State) recover("test observes decode failures in its assertions") { next = state } +collect(event: dever.io.ReadEvent.Chunk(bytes), state: State) (next: State) recover("test observes collect failures in its assertions") { next = decode(result(dever.bytes.to_text(bytes)), state) } +collect(event: dever.io.ReadEvent.Failed(message), state: State) (next: State) recover("test observes collect failures in its assertions") { next = state } +reading(result: dever.net.ReadStreamResult.Streaming(stream), state: State) (next: State) recover("test observes reading failures in its assertions") { next = reduce(collect, stream, state) } +reading(result: dever.net.ReadStreamResult.Failed(message), state: State) (next: State) recover("test observes reading failures in its assertions") { next = state } +accepted(result: dever.net.ConnectResult.Connected(server), state: State) (next: State, stop: Bool) recover("test observes accepted failures in its assertions") { + dever.net.timeout(state.client, 1000) + dever.net.timeout(server, 1000) + dever.net.write(state.client, dever.bytes.from_text("xyz")) + dever.net.close(state.client) + next = reading(result(dever.net.chunks(server, 2)), state) + stop = true +} +accepted(result: dever.net.ConnectResult.Failed(message), state: State) (next: State, stop: Bool) recover("test observes accepted failures in its assertions") { next = state +stop = true } +connected(result: dever.net.ConnectResult.Connected(client), listener: dever.system.Listener) (answer: Text) recover("test observes connected failures in its assertions") { + stream = dever.net.connections(listener) + state = State { client = client +text = "" } + finished = reduce_until(accepted, stream, state) + close(stream) + answer = finished.text +} +connected(result: dever.net.ConnectResult.Failed(message), listener: dever.system.Listener) (answer: Text) recover("test observes connected failures in its assertions") { answer = message } +bound(result: dever.net.PortResult.Bound(port), listener: dever.system.Listener) (answer: Text) recover("test observes bound failures in its assertions") { answer = connected(result(dever.net.connect("127.0.0.1", port)), listener) } +bound(result: dever.net.PortResult.Failed(message), listener: dever.system.Listener) (answer: Text) recover("test observes bound failures in its assertions") { answer = message } +listening(result: dever.net.ListenResult.Listening(listener)) (answer: Text) recover("test observes listening failures in its assertions") { answer = bound(result(dever.net.port(listener)), listener) } +listening(result: dever.net.ListenResult.Failed(message)) (answer: Text) recover("test observes listening failures in its assertions") { answer = message } +public main() (answer: Text) { answer = listening(result(dever.net.listen("127.0.0.1", 0))) } +"#, + ); + assert_eq!(output, "answer = xyz\n"); +} + +#[test] +fn each_consumes_a_stream_only_for_zero_output_actions() { + let directory = support::temp::TemporaryDirectory::new(); + let path = directory.path().join("each.txt"); + std::fs::write(&path, "abcd").unwrap(); + let source = format!( + r#"print(result: dever.bytes.DecodeResult.Decoded(text)) () recover("test observes print failures in its assertions") {{ dever.io.println(text) }} +print(result: dever.bytes.DecodeResult.Failed(message)) () recover("test observes print failures in its assertions") {{ dever.io.println("failed") }} +show(event: dever.io.ReadEvent.Chunk(bytes)) () recover("test observes show failures in its assertions") {{ print(result(dever.bytes.to_text(bytes))) }} +show(event: dever.io.ReadEvent.Failed(message)) () recover("test observes show failures in its assertions") {{ dever.io.println("failed") }} +streaming(result: dever.io.ReadStreamResult.Streaming(stream)) (answer: Bool) recover("test observes streaming failures in its assertions") {{ each(show, stream) +answer = true }} +streaming(result: dever.io.ReadStreamResult.Failed(message)) (answer: Bool) recover("test observes streaming failures in its assertions") {{ answer = false }} +opened(result: dever.io.OpenResult.Opened(file)) (answer: Bool) recover("test observes opened failures in its assertions") {{ answer = streaming(result(dever.io.chunks(file, 2))) }} +opened(result: dever.io.OpenResult.Failed(message)) (answer: Bool) recover("test observes opened failures in its assertions") {{ answer = false }} +public main() (answer: Bool) {{ answer = opened(result(dever.io.open({path:?}))) }} +"#, + path = path.to_str().unwrap() + ); + assert_eq!(stdout(&source), "ab\ncd\nanswer = true\n"); +} diff --git a/test/dever-tests/tests/core_semantics.rs b/test/dever-tests/tests/core_semantics.rs new file mode 100644 index 0000000..5897774 --- /dev/null +++ b/test/dever-tests/tests/core_semantics.rs @@ -0,0 +1,398 @@ +mod support; + +use support::{checked, rejected, sources}; + +#[test] +fn full_language_contract_package_checks_with_the_official_sources() { + let prd = include_str!("../../../.trellis/tasks/09-04-dever-language-mvp/prd.md"); + let source = prd + .split_once("## Complete Package Example\n") + .unwrap() + .1 + .split_once("```dever\n") + .unwrap() + .1 + .split_once("\n```") + .unwrap() + .0; + let mut sources = dever_core::source::SourceMap::default(); + sources.add("account/user.dever", source); + checked(&sources); + assert_eq!(sources.files().len(), 1); +} + +#[test] +fn clause_domains_are_disjoint_exhaustive_and_order_independent() { + for clauses in [ + "choose(value: true) (answer: Int) { answer = 1 }\nchoose(value: false) (answer: Int) { answer = 2 }", + "choose(value: false) (answer: Int) { answer = 2 }\nchoose(value: true) (answer: Int) { answer = 1 }", + "choose(value: Text) (answer: Int) { answer = 1 }\nchoose(value: null) (answer: Int) { answer = 0 }", + "choose(value: Int < 0) (answer: Int) { answer = -1 }\nchoose(value: Int >= 0) (answer: Int) { answer = 1 }", + "choose(value: Decimal < 0.0) (answer: Int) { answer = -1 }\nchoose(value: other) (answer: Int) { answer = 1 }", + "choose(value: \"yes\") (answer: Int) { answer = 1 }\nchoose(value: other) (answer: Int) { answer = 0 }", + "choose(value: 1.0e0) (answer: Int) { answer = 1 }\nchoose(value: other) (answer: Int) { answer = 0 }", + ] { + checked(&sources(clauses)); + } + for (clauses, message) in [ + ("choose(value: true) () {}", "do not cover"), + ( + "choose(value: Bool) () {}\nchoose(value: true) () {}", + "overlap", + ), + ( + "choose(value: true) () {}\nchoose(value: false) () {}\nchoose(value: other) () {}", + "unreachable", + ), + ( + "choose(value: Text?) () {}\nchoose(value: null) () {}", + "overlap", + ), + ( + "choose(value: null) () {}\nchoose(value: other) () {}", + "cannot infer", + ), + ("choose(value: Int > 0 and < 1) () {}", "unreachable"), + ( + "choose(value: Int > 9223372036854775807) () {}", + "unreachable", + ), + ( + "choose(value: Decimal > 1.0 and < 1.000000000000000000000000000000001) () {}", + "unreachable", + ), + ("choose(value: Float > 0.0) () {}", "range bounds"), + ("choose(value: Int > 0 and > 1) () {}", "one lower"), + ( + "choose(left: true, right: Bool) () {}\nchoose(left: other, right: false) () {}", + "every input combination", + ), + ("choose(left: Bool, right: other) () {}", "cannot infer"), + ("choose() () {}\nchoose() () {}", "overlap"), + ( + "choose(value: true) (answer: Int) { answer = 1 }\nchoose(value: false) (different: Int) { different = 2 }", + "identical ordered output", + ), + ] { + rejected(clauses, message); + } +} + +#[test] +fn four_quadrants_cover_a_multi_input_product() { + checked(&sources( + "public choose(a: true, b: true) () {}\npublic choose(a: true, b: other) () {}\npublic choose(a: other, b: true) () {}\npublic choose(a: other, b: other) () {}", + )); +} + +#[test] +fn construction_initialization_and_stable_types_are_checked() { + let prefix = + "type User { name: Text\nphone: Text? }\ntype Result { Found(user: User)\nMissing }\n"; + for (body, message) in [ + ("main() () { user = User { name = \"a\" } }", "every field"), + ( + "main() () { user = User { name = \"a\"\nphone = null\nextra = 1 } }", + "unknown constructor field", + ), + ( + "main() () { user = User { name = \"a\"\nname = \"b\"\nphone = null } }", + "duplicate constructor", + ), + ("main() (user: User) { user.name = \"a\" }", "uninitialized"), + ("main() (answer: Int) {}", "uninitialized"), + ("main() () { value = null }", "nullable type context"), + ("main() () { value = 1\nvalue = \"a\" }", "expected Int"), + ("main() () { value = Result.Found() }", "payload arity"), + ("main() () { value = Result.Missing(1) }", "payload arity"), + ("main() () { values = [] }", "type context"), + ("main() () { labels = {} }", "type context"), + ("main() () { values = [1, true] }", "one stable type"), + ("main() () { values = { 1.5 = 1 } }", "Map key"), + ( + "main() () { values = { \"a\" = 1\n\"a\" = 2 } }", + "duplicate Map", + ), + ( + "main() (a: Int, b: Int) { a = 1\nb = 2 }\ninvalid() () { values = [main()] }", + "multi-output", + ), + ( + "main() (a: Int, b: Int) { a = 1\nb = 2 }\ninvalid() (same: Bool) { same = main() == main() }", + "comparable", + ), + ] { + rejected(&format!("{prefix}{body}"), message); + } +} + +#[test] +fn scalar_rules_and_static_faults_are_enforced() { + for (expression, message) in [ + ("9223372036854775808", "64-bit"), + ("-9223372036854775809", "64-bit"), + ("0.12345678901234567890123456789012345", "34 digits"), + ("9223372036854775807 + 1", "overflow"), + ("-9223372036854775808 // -1", "overflow"), + ("1 // 0", "division by zero"), + ("1 / 0", "division by zero"), + ("1 == 1.0", "does not promote"), + ("1 + 1.0e0", "cannot implicitly mix"), + ("1.0 // 2", "require Int"), + ("1 and true", "expected Bool"), + ] { + rejected(&format!("main() () {{ value = {expression} }}"), message); + } + checked(&sources( + "public main() (min: Int, exact: Bool, quotient: Decimal, float: Float) { min = -9223372036854775808\nexact = 0.1 + 0.2 == 0.3\nquotient = 5 / 2\nfloat = 1.5 + 2.5 }", + )); +} + +#[test] +fn graphs_cover_type_references_and_collection_handlers() { + rejected("type Recursive { next: Recursive? }", "recursive type"); + rejected("type A { b: B }\ntype B { a: List<A> }", "recursive type"); + rejected( + "repeat_value(value: Int) (answer: Int) { values = each(repeat_value, [value])\nanswer = sum(values) }", + "recursive function", + ); + rejected( + "f(value: true) () {}\ng() () {}\nf(value: false) () {}", + "contiguous", + ); + let mut sources = dever_core::source::SourceMap::default(); + sources.add( + "a.dever", + "public type A { value: Int }\nstart() () { b.run() }\nfinish() () {}", + ); + sources.add("b.dever", "public run() () {}\nunused(value: a.A) () {}"); + assert!( + dever_core::check(&sources) + .unwrap_err() + .iter() + .any(|error| error.message.contains("cyclic package")) + ); +} + +#[test] +fn invalid_handlers_are_rejected_before_build() { + for (declarations, call, message) in [ + ( + "handler(value: Int) (a: Int, b: Int) { a = value\nb = value }", + "each(handler, [1])", + "invalid outputs", + ), + ( + "handler(value: Int) (value: Int) { value = 1 }", + "filter(handler, [1])", + "duplicate local", + ), + ( + "handler(value: Int) (answer: Int) { answer = value }", + "filter(handler, [1])", + "invalid outputs", + ), + ( + "handler(value: Int) () {}", + "sum(handler, [1])", + "invalid outputs", + ), + ( + "handler(value: Int) () {}", + "each(handler, [1], 2)", + "unknown function", + ), + ] { + rejected( + &format!("{declarations}\nmain() () {{ result = {call} }}"), + message, + ); + } +} + +#[test] +fn unused_writes_are_warnings_not_errors() { + let sources = sources("public main() () { value = 1\nvalue = 2 }"); + let program = checked(&sources); + assert_eq!( + program + .warnings() + .iter() + .filter(|warning| warning.code == "W001") + .count(), + 2 + ); + assert!( + program.warnings()[0] + .render(&sources) + .contains("warning[W001]") + ); +} + +#[test] +fn known_scalar_and_choice_keys_are_rejected_when_duplicated() { + for body in [ + "labels = { Key.First = 1\nKey.First = 2 }", + "key = Key.First\nlabels = { key = 1\nKey.First = 2 }", + "key = \"same\"\nlabels = { key = 1\n\"same\" = 2 }", + ] { + rejected( + &format!("type Key {{ First\nSecond }}\nmain() () {{ {body} }}"), + "duplicate Map literal key", + ); + } + rejected( + "invalid(key: Id) () { labels = { key = 1\nkey = 2 } }", + "duplicate Map literal key", + ); +} + +#[test] +fn known_numeric_faults_include_local_constants_and_constant_divisors() { + rejected( + "invalid(value: Int) (answer: Int) { answer = value // 0 }", + "division by zero", + ); + rejected( + "invalid(value: Decimal) (answer: Decimal) { zero = 0\nanswer = value / zero }", + "division by zero", + ); + rejected( + "invalid() () { value = 9223372036854775807\nanswer = value + 1 }", + "overflow", + ); +} + +#[test] +fn literal_clause_inference_is_stable_under_permutation() { + for clauses in [ + "choose(value: 1) () {}\nchoose(value: 2.0) () {}\nchoose(value: other) () {}", + "choose(value: 2.0) () {}\nchoose(value: other) () {}\nchoose(value: 1) () {}", + ] { + checked(&sources(clauses)); + } +} + +#[test] +fn names_follow_the_existing_language_conventions() { + rejected( + "package App exposes ()", + "package/exposes declarations were removed", + ); + for source in [ + "type user { name: Text }", + "Main() () {}", + "main() () { bad__name = 1 }", + ] { + rejected(source, "ASCII"); + } +} + +#[test] +fn float_literal_context_never_retypes_existing_numeric_values() { + checked(&sources( + "compare(value: Float) (answer: Bool) { answer = value < 0.0 or 0.0 == value or (-0.0) >= value }", + )); + rejected( + "compare(value: Float) (answer: Bool) { decimal = 0.0\nanswer = value == decimal }", + "same comparable type", + ); + rejected( + "compare(value: Float) (answer: Bool) { answer = value == 0 }", + "same comparable type", + ); + rejected( + "invalid() () { decimal = 1.0\nvalue = decimal + 1.0e0 }", + "cannot implicitly mix", + ); +} + +#[test] +fn reduce_bytes_stream_and_static_handler_contracts_are_checked() { + checked(&sources( + r#"add(value: Int, total: Int) (next: Int) { next = total + value } +until(value: Int, total: Int) (next: Int, stop: Bool) { next = total + value +stop = value == 2 } +show(value: Int) () {} +apply(route: handler(value: Int) (answer: Int), value: Int) (answer: Int) { answer = route(value) } +forward(route: handler(value: Int) (answer: Int), value: Int) (answer: Int) { answer = apply(route, value) } +consume(stream: Stream<Int>) (total: Int) { each(show, stream) +total = 0 } +main() () { total = reduce(add, [1, 2], 0) +partial = reduce_until(until, dever.bytes.from_text("abc"), 0) }"#, + )); + + for (source, message) in [ + ( + "map(value: Int) (answer: Int) { answer = value }\nbad(stream: Stream<Int>) () { values = each(map, stream) }", + "each on a stream", + ), + ( + "bad(value: Int, state: Int) (next: Int, done: Bool) { next = state\ndone = true }\nmain() () { value = reduce_until(bad, [1], 0) }", + "stop: Bool", + ), + ( + "apply(route: handler(value: Int) (answer: Int)) (answer: Int) { answer = route }", + "not runtime values", + ), + ( + "text(value: Text) (answer: Text) { answer = value }\napply(route: handler(value: Int) (answer: Int)) (answer: Int) { answer = route(1) }\nmain() () { answer = apply(text) }", + "signature does not match", + ), + ( + "wrong(value: Int) (result: Int) { result = value }\napply(route: handler(value: Int) (answer: Int)) (answer: Int) { answer = route(1) }\nmain() () { answer = apply(wrong) }", + "signature does not match", + ), + ( + "apply(route: handler(value: Int) (answer: Int)) () { values = [route] }", + "not runtime values", + ), + ( + "apply(route: handler(value: Int) (answer: Int)) (answer: Bool) { answer = route == route }", + "not runtime values", + ), + ( + "loop(next: handler(value: Int) (answer: Int), value: Int) (answer: Int) { answer = next(value) }\nagain(value: Int) (answer: Int) { answer = loop(again, value) }", + "specialized handler", + ), + ] { + rejected(source, message); + } +} + +#[test] +fn handler_functions_can_cross_packages_and_forward_statically() { + let mut sources = dever_core::source::SourceMap::default(); + sources.add( + "worker.dever", + "public double(value: Int) (answer: Int) { answer = value * 2 }", + ); + sources.add( + "main.dever", + "apply(route: handler(value: Int) (answer: Int), value: Int) (answer: Int) { answer = route(value) }\nforward(route: handler(value: Int) (answer: Int), value: Int) (answer: Int) { answer = apply(route, value) }\npublic main() (answer: Int) { answer = forward(worker.double, 3) }", + ); + checked(&sources); +} + +#[test] +fn handler_signatures_and_streams_reuse_value_type_restrictions() { + for source in [ + "apply(route: handler(value: Map<List<Int>, Int>) ()) () {}", + "apply(route: handler() (value: Map<List<Int>, Int>)) () {}", + "bad(values: Map<Stream<Int>, Int>) () {}", + ] { + rejected(source, "invalid Map key type"); + } + rejected( + "bad(stream: Stream) () {}", + "incorrect number of type arguments", + ); + rejected( + "bad(stream: Stream<Int>) (same: Bool) { same = stream == stream }", + "same comparable type", + ); + rejected( + "bad(values: List<Int>) () { close(values) }", + "close requires a Stream", + ); +} diff --git a/test/dever-tests/tests/differential.rs b/test/dever-tests/tests/differential.rs new file mode 100644 index 0000000..41af63e --- /dev/null +++ b/test/dever-tests/tests/differential.rs @@ -0,0 +1,353 @@ +mod support; + +use dever_core::{native, reference, source::SourceMap}; +use std::ffi::OsString; +use std::process::Command; +use support::{checked, sources}; + +fn compare_sources(sources: &SourceMap, expected: &str) { + let program = checked(sources); + let evaluated = reference::evaluate(&program, "main.main").unwrap(); + assert!(evaluated.fault.is_none(), "{:?}", evaluated.fault); + assert_eq!(evaluated.stdout, expected); + let rustc = std::env::var_os("RUSTC").unwrap_or_else(|| OsString::from("rustc")); + let executable = native::compile(&program, sources, "main.main", &rustc).unwrap(); + let output = Command::new(executable.executable()).output().unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!(output.stderr.is_empty()); + assert_eq!(String::from_utf8(output.stdout).unwrap(), evaluated.stdout); +} + +fn compare(source: &str, expected: &str) { + compare_sources(&sources(source), expected); +} + +#[test] +fn uuid_parsing_and_canonical_text_agree() { + compare( + r#"display(value: Uuid) (text: Text) { text = uuid.to_text(value) } +display(value: null) (text: Text) { text = "invalid" } +public main() (parsed: Text, invalid: Text, literal: Text) { + parsed = display(uuid.parse("018F08D8-59B0-7CC2-98C4-DC0C0C07398F")) + invalid = display(uuid.parse("not-a-uuid")) + literal = uuid.to_text("12345678-1234-5678-9012-123456789abc") +}"#, + "parsed = 018f08d8-59b0-7cc2-98c4-dc0c0c07398f\ninvalid = invalid\nliteral = 12345678-1234-5678-9012-123456789abc\n", + ); +} + +#[test] +fn complete_core_example_has_independent_expected_results() { + compare( + include_str!("../../../examples/old/dever/core/module/main.dever"), + concat!( + "original = 1\nchanged = 99\nchoice = 7\ncategory = large\nexact = true\n", + "quotient = 2.5\nthird = 0.3333333333333333333333333333333333\nrounded = 2.34\n", + "integer_division = -2\nremainder = -1\nselected = [3, 6]\nfound = 3\ntotal = 18\n", + "empty_total = 0\nnames = ABCabc\nordered_keys = [second, first]\nmaps_equal = false\n", + "nan_equal = false\nfinite = true\noptionals = [null, 1, null]\nkey_value = 2\nidentity = true\n", + ), + ); +} + +#[test] +fn expression_order_nested_aliases_and_named_output_projections_agree() { + compare( + r#"public type Pair { left: Int +right: Int } +public type Box { pair: Pair +values: List<Int> } +mark(label: Text, value: Int) (answer: Int) { dever.io.println(label) +answer = value } +key(label: Text) (answer: Text) { dever.io.println(label) +answer = label } +parts() (left: Int, right: Int) { left = 3 +right = 4 } +never() (answer: Bool) { dever.io.println("unexpected") +answer = true } +public main() (saved: Box, changed: Box, pair: Pair, ordered: Map<Text, Int>, old: List<Int>, short: Bool) { + pair = Pair { right = mark("right", 2) +left = mark("left", 1) } + saved = Box { pair = pair +values = [1] } + changed = saved + changed.pair.left = 9 + changed.values = append(changed.values, 2) + old = saved.values + outputs = parts() + copy = outputs + pair.right = copy.left + ordered = { key("a") = mark("av", 1) +key("b") = mark("bv", 2) } + short = false and never() or true or never() +} +"#, + "right\nleft\na\nav\nb\nbv\nsaved = main.Box { pair = main.Pair { left = 1, right = 2 }, values = [1] }\nchanged = main.Box { pair = main.Pair { left = 9, right = 2 }, values = [1, 2] }\npair = main.Pair { left = 1, right = 3 }\nordered = {a = 1, b = 2}\nold = [1]\nshort = true\n", + ); +} + +#[test] +fn nullable_choice_payloads_domains_and_signed_zero_agree() { + compare( + r#"type Result { Found(value: Int) +Missing } +unwrap(value: Result.Found(payload)) (answer: Int) { answer = payload } +unwrap(value: Result.Missing) (answer: Int) { answer = -1 } +unwrap(value: null) (answer: Int) { answer = -2 } +decimal_kind(value: Decimal < 0) (answer: Int) { answer = -1 } +decimal_kind(value: Decimal >= 0 and <= 1) (answer: Int) { answer = 0 } +decimal_kind(value: Decimal > 1) (answer: Int) { answer = 1 } +label(value: "x") (answer: Int) { answer = 1 } +label(value: other) (answer: Int) { answer = 2 } +sign(value: 0.0e0) (answer: Float) { answer = 1.0e0 / value } +sign(value: other) (answer: Float) { answer = 0.0e0 } +present(value: Int) (answer: Bool) { answer = true } +present(value: null) (answer: Bool) { answer = false } +optional(value: Int) (answer: Int?) { answer = value } +public main() (values: List<Int>, first_value: Int?, found: Int?, missing: Int?, negative: Bool, kind: Int, text_kind: Int, ordered: Float) { + values = [unwrap(Result.Found(7)), unwrap(Result.Missing), unwrap(null)] + first_value = first([null, 1]) + found = find(present, [null, 2]) + mapping = { "x" = optional(1) } + missing = get(mapping, "none") + negative = sign(-0.0e0) < 0.0 + kind = decimal_kind(-0.1) + decimal_kind(0.5) + decimal_kind(1.1) + text_kind = label("x") + label("y") + ordered = sum([1.0e16, 1.0e0, -1.0e16]) +} +"#, + "values = [7, -1, -2]\nfirst_value = null\nfound = 2\nmissing = null\nnegative = true\nkind = 0\ntext_kind = 3\nordered = 0\n", + ); +} + +#[test] +fn static_handlers_cross_packages_and_reduce_all_finite_sequences() { + let mut sources = sources( + r#"add(value: Int, state: Int) (next: Int) { next = state + value } +until(value: Int, state: Int) (next: Int, stop: Bool) { next = state + value +stop = value == 2 } +increment(value: Int) (answer: Int) { answer = value + 1 } +show(value: Int) () { dever.io.println(int.to_text(value)) } +public main() (list: Int, empty: Int, stopped: Int, bytes: Int, mapped: List<Int>, forwarded: Int) { + list = reduce(add, [1, 2, 3], 0) + empty = reduce(add, [], 9) + stopped = reduce_until(until, [1, 2, 100], 0) + raw = dever.bytes.from_text("AB") + bytes = reduce(add, raw, 0) + mapped = each(increment, raw) + each(show, [1, 2]) + each(show, dever.bytes.from_text("A")) + forwarded = helper.forward(increment, 4) +} +"#, + ); + sources.add("helper.dever", r#"public forward(route: handler(value: Int) (answer: Int), value: Int) (answer: Int) { answer = apply(value, route) } +apply(value: Int, route: handler(value: Int) (answer: Int)) (answer: Int) { answer = route(value) } +"#); + compare_sources( + &sources, + "1\n2\n65\nlist = 6\nempty = 9\nstopped = 3\nbytes = 131\nmapped = [66, 67]\nforwarded = 5\n", + ); +} + +#[test] +fn runtime_faults_match_exact_locations_call_traces_and_prior_stdout() { + for (function, arguments, expected) in [ + ( + "compute_failure(a: Int, b: Int) (answer: Int) { answer = a // b }", + "1, 0", + "Int division by zero", + ), + ( + "compute_failure(a: Int, b: Int) (answer: Int) { answer = a + b }", + "9223372036854775807, 1", + "Int overflow", + ), + ( + "compute_failure(a: Decimal, b: Decimal) (answer: Decimal) { answer = a / b }", + "1.0, 0.0", + "Decimal division by zero", + ), + ( + "compute_failure(a: Int, b: Int) (answer: Map<Int, Int>) { answer = { a = 1\nb = 2 } }", + "1, 1", + "duplicate Map key", + ), + ] { + let sources = sources(&format!( + "{function}\npublic main() () {{ dever.io.println(\"before\")\nanswer = compute_failure({arguments}) }}" + )); + let program = checked(&sources); + let evaluated = reference::evaluate(&program, "main.main").unwrap(); + assert_eq!(evaluated.stdout, "before\n"); + let fault = evaluated.fault.unwrap(); + assert_eq!(fault.message, expected); + assert!(fault.spans.len() >= 2); + assert_eq!( + sources + .get(fault.spans[0].source) + .position(fault.spans[0].start) + .0, + 1 + ); + let rustc = std::env::var_os("RUSTC").unwrap_or_else(|| OsString::from("rustc")); + let executable = native::compile(&program, &sources, "main.main", &rustc).unwrap(); + let output = Command::new(executable.executable()).output().unwrap(); + assert!(!output.status.success()); + assert_eq!(String::from_utf8(output.stdout).unwrap(), evaluated.stdout); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert_eq!(stderr.lines().count(), 1, "{stderr}"); + let record: serde_json::Value = serde_json::from_str(&stderr).unwrap(); + assert_eq!( + record, + serde_json::json!({"level": "error", "message": fault.render(&sources), "fields": {}}) + ); + } +} + +#[test] +fn standard_text_numeric_and_bytes_results_agree() { + compare( + r#"decode(value: dever.bytes.DecodeResult.Decoded(text)) (answer: Text) recover("test observes decode failures in its assertions") { answer = text } +decode(value: other) (answer: Text) recover("test observes decode failures in its assertions") { answer = "invalid" } +buffer(value: dever.bytes.BufferResult.Ready(bytes)) (answer: Bytes) recover("test observes buffer failures in its assertions") { answer = bytes } +buffer(value: other) (answer: Bytes) recover("test observes buffer failures in its assertions") { answer = dever.bytes.from_text("") } +public main() (text_value: Text, unicode: Text?, slice: Text?, index: Int?, predicates: Bool, integer: Int?, decimal_value: Decimal?, float_value: Float?, invalid: Int?, numbers: Text, bytes: Text, outside: Int?, bad: Text) { + text_value = text.join(text.split(text.upper("a,b"), ","), "-") + unicode = text.at("中文", 1) + slice = text.slice("中文a", 1, 3) + index = text.index_of("中文a", "a") + predicates = text.contains("abc", "b") and text.starts_with("abc", "a") and text.ends_with("abc", "c") and text.replace("abc", "b", "x") == "axc" + integer = int.parse("-12") + decimal_value = decimal.parse("1.25") + float_value = float.parse("1.5") + invalid = int.parse("no") + numbers = int.to_text(4) + decimal.to_text(decimal.from_int(2)) + float.to_text(1.5e0) + combined = dever.bytes.concat(dever.bytes.from_text("ab"), dever.bytes.from_text("cd")) + bytes = decode(result(dever.bytes.to_text(buffer(result(dever.bytes.slice(combined, 1, 3)))))) + outside = dever.bytes.at(combined, -1) + bad = decode(result(dever.bytes.to_text(buffer(result(dever.bytes.from_ints([255])))))) +} +"#, + "text_value = A-B\nunicode = 文\nslice = 文a\nindex = 2\npredicates = true\ninteger = -12\ndecimal_value = 1.25\nfloat_value = 1.5\ninvalid = null\nnumbers = 421.5\nbytes = bc\noutside = null\nbad = invalid\n", + ); +} + +#[test] +fn file_stream_aliases_early_stop_resume_and_terminal_close_agree() { + let directory = support::temp::TemporaryDirectory::new(); + let path = directory.path().join("stream.txt"); + std::fs::write(&path, "ABC").unwrap(); + let source = r#"add(byte: Int, state: Int) (next: Int) { next = state + byte } +collect(event: dever.io.ReadEvent.Chunk(bytes), state: Int) (next: Int) recover("test observes collect failures in its assertions") { next = reduce(add, bytes, state) } +collect(event: dever.io.ReadEvent.Failed(message), state: Int) (next: Int) recover("test observes collect failures in its assertions") { next = state + 1000 } +first(event: dever.io.ReadEvent, state: Int) (next: Int, stop: Bool) { next = collect(event, state) +stop = true } +show(event: dever.io.ReadEvent) () recover("test observes show failures in its assertions") { dever.io.println("unexpected") } +streaming(result: dever.io.ReadStreamResult.Streaming(stream), file: dever.system.File) (answer: List<Int>) recover("test observes streaming failures in its assertions") { + alias = stream + prefix = reduce_until(first, stream, 0) + suffix = reduce(collect, alias, 0) + exhausted = reduce(collect, stream, 7) + close(stream) + close(alias) + each(show, alias) + answer = [prefix, suffix, exhausted] +} +streaming(result: other, file: dever.system.File) (answer: List<Int>) recover("test observes streaming failures in its assertions") { answer = [] } +terminal(result: dever.io.ReadStreamResult.Streaming(stream), file: dever.system.File) (answer: List<Int>) recover("test observes terminal failures in its assertions") { + dever.io.close(file) + once = reduce(collect, stream, 0) + again = reduce(collect, stream, 0) + answer = [once, again] +} +terminal(result: other, file: dever.system.File) (answer: List<Int>) recover("test observes terminal failures in its assertions") { answer = [] } +opened(result: dever.io.OpenResult.Opened(file)) (answer: List<Int>) recover("test observes opened failures in its assertions") { answer = streaming(result(dever.io.chunks(file, 1)), file) } +opened(result: other) (answer: List<Int>) recover("test observes opened failures in its assertions") { answer = [] } +closed(result: dever.io.OpenResult.Opened(file)) (answer: List<Int>) recover("test observes closed failures in its assertions") { answer = terminal(result(dever.io.chunks(file, 1)), file) } +closed(result: other) (answer: List<Int>) recover("test observes closed failures in its assertions") { answer = [] } +public main() (values: List<Int>, failure: List<Int>) { + values = opened(result(dever.io.open(PATH))) + failure = closed(result(dever.io.open(PATH))) +} +"#.replace("PATH", &format!("{:?}", path.to_str().unwrap())); + compare(&source, "values = [65, 133, 7]\nfailure = [1000, 0]\n"); +} + +#[test] +fn repeated_arguments_live_fields_and_growing_reduction_state_keep_value_semantics() { + compare( + r#"public type Pair { left: List<Int> +right: List<Int> } +combine(left: List<Int>, right: List<Int>) (answer: List<Int>) { + left = append(left, 8) + answer = append(right, length(left)) +} +grow(value: Int, state: List<Int>) (next: List<Int>) { next = append(state, value) } +public main() (original: List<Int>, combined: List<Int>, old: Pair, changed: Pair, reduced: List<Int>, shared: List<Int>) { + original = [1] + combined = combine(original, original) + changed = Pair { left = original +right = original } + old = changed + changed.left = append(changed.right, length(changed.left)) + changed.right = changed.left + shared = [9] + reduced = reduce(grow, [1, 2, 3], shared) +} +"#, + "original = [1]\ncombined = [1, 2]\nold = main.Pair { left = [1], right = [1] }\nchanged = main.Pair { left = [1, 1], right = [1, 1] }\nreduced = [9, 1, 2, 3]\nshared = [9]\n", + ); +} + +#[test] +fn numeric_operators_math_nullable_collections_and_key_types_agree() { + compare( + r#"arithmetic(a: Int, b: Int) (answer: Int) { answer = -(a - b) * b + a % b } +decimal_math(a: Decimal, b: Decimal) (answer: Decimal) { answer = -(a - b) * b / 2 } +floating(a: Float, b: Float) (answer: Float) { answer = -(a - b) * b + a / b } +entry_key(entry: MapEntry<Bool, Text>) (answer: Bool) { answer = entry.key } +absent() (answer: Int?) { answer = null } +public main() (integer: Int, fixed: Decimal, float_value: Float, math_ok: Bool, no_nan_order: Bool, first_value: Int?, found: Int?, identity: Text, keys: List<Bool>, entry: MapEntry<Bool, Text>?) { + integer = arithmetic(7, 3) + fixed = decimal_math(1.5, 0.5) + float_value = floating(3.0e0, 2.0e0) + math_ok = math.sqrt(4.0e0) == 2.0e0 and math.sin(0.0e0) == 0.0e0 and math.cos(0.0e0) == 1.0e0 and math.log(1.0e0) == 0.0e0 and math.pow(2.0e0, 3.0e0) == 8.0e0 and math.is_infinite(1.0e0 / 0.0e0) + nan = 0.0e0 / 0.0e0 + no_nan_order = not (nan < 1.0e0 or nan <= 1.0e0 or nan > 1.0e0 or nan >= 1.0e0) and math.is_nan(nan) + first_value = first([]) + optional_map = { "x" = absent() } + found = get(optional_map, "x") + id = dever.id.from_text("stable") + ids = { id = 7 } + identity = dever.id.to_text(id) + values = { true = "a" +false = "b" } + keys = each(entry_key, entries(values)) + entry = first(entries(values)) +} +"#, + "integer = -11\nfixed = -0.25\nfloat_value = -0.5\nmath_ok = true\nno_nan_order = true\nfirst_value = null\nfound = null\nidentity = stable\nkeys = [true, false]\nentry = {key = true, value = a}\n", + ); +} + +#[test] +fn time_failures_are_source_defined_results() { + compare( + r#"sleep_failed(result: dever.time.SleepResult.Failed(message)) (answer: Bool) recover("test observes sleep_failed failures in its assertions") { answer = true } +sleep_failed(result: other) (answer: Bool) recover("test observes sleep_failed failures in its assertions") { answer = false } +clock_ok(result: dever.time.ClockResult.Read(value)) (answer: Bool) recover("test observes clock_ok failures in its assertions") { answer = value >= 0 } +clock_ok(result: other) (answer: Bool) recover("test observes clock_ok failures in its assertions") { answer = false } +public main() (sleep: Bool, wall: Bool, monotonic: Bool) { + sleep = sleep_failed(result(dever.time.sleep(-1))) + wall = clock_ok(result(dever.time.unix_millis())) + monotonic = clock_ok(result(dever.time.monotonic_nanos())) +} +"#, + "sleep = true\nwall = true\nmonotonic = true\n", + ); +} diff --git a/test/dever-tests/tests/durable_jobs.rs b/test/dever-tests/tests/durable_jobs.rs new file mode 100644 index 0000000..b41cd26 --- /dev/null +++ b/test/dever-tests/tests/durable_jobs.rs @@ -0,0 +1,1573 @@ +use dever_core::source::SourceMap; +#[path = "support/temp.rs"] +mod temp; + +fn sources(job: &str, app: &str) -> SourceMap { + let mut sources = SourceMap::default(); + sources.add("content/article/job.dever", job); + sources.add("content/article/app.dever", app); + sources +} + +fn checked(sources: &SourceMap) -> dever_core::hir::Program { + dever_core::check(sources).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(sources)) + .collect::<Vec<_>>() + .join("\n") + ) + }) +} + +fn rejected(sources: &SourceMap, message: &str) { + let errors = dever_core::check(sources).expect_err("invalid Job must be rejected"); + assert!( + errors.iter().any(|error| error.message.contains(message)), + "{errors:?}" + ); +} + +#[cfg(feature = "sqlite")] +fn clock_at(millis: i64) -> dever_runtime::time::Clock { + let clock = dever_runtime::time::Clock::for_test(); + clock.advance(millis).unwrap(); + clock +} + +const JOB: &str = "database default\njob publish(input: app.Publish) () retry(5) timeout(30000) { app.apply(input) }"; +const APP: &str = "type Publish { title: Text }\napply(input: Publish) () {}\ntransaction submit(input: Publish) (id: Id) { id = dever.job.enqueue(job.publish, input, input.title) }"; + +#[test] +fn static_jobs_and_enqueue_only_transactions_check_and_format() { + let sources = sources(JOB, APP); + checked(&sources); + let source = &sources.files()[0]; + let formatted = dever_core::format::format(source).unwrap(); + let mut round_trip = SourceMap::default(); + let id = round_trip.add("content/article/job.dever", &formatted); + assert_eq!( + formatted, + dever_core::format::format(round_trip.get(id)).unwrap() + ); +} + +#[test] +fn jobs_enforce_private_entry_payload_and_policy_boundaries() { + rejected( + &sources( + JOB, + &APP.replace( + "dever.job.enqueue(job.publish, input, input.title)", + "job.publish(input)", + ), + ), + "not callable", + ); + rejected( + &sources(&JOB.replace("retry(5)", "retry(0)"), APP), + "retry must", + ); + rejected( + &sources(&JOB.replace("app.Publish", "Text"), APP), + "payload must", + ); + rejected( + &sources(&JOB.replace("app.apply(input)", "model.count()"), APP), + "unknown", + ); + rejected( + &sources(JOB, &APP.replace("title: Text", "title: Secret")), + "Secret", + ); + rejected( + &sources(&JOB.replace("database default\n", ""), APP), + "database binding", + ); + rejected( + &sources( + &JOB.replace("app.apply(input)", "dever.system.time_now()"), + APP, + ), + "same-domain App", + ); + rejected( + &sources( + &JOB.replace( + "app.apply(input)", + "task = run(app.apply(input))\nwait(task)", + ), + APP, + ), + "same-domain App", + ); + + let mut public_enqueue = sources(JOB, APP); + public_enqueue.add( + "content/article/api.dever", + "public post submit = app.submit", + ); + rejected(&public_enqueue, "trusted user identity"); +} + +#[test] +fn queue_boundary_does_not_add_synchronous_recursion() { + checked(&sources( + "database default\njob retry_later(input: app.Work) () retry(2) timeout(1000) { id = app.apply(input) }", + "type Work { key: Text }\napply(input: Work) (id: Id) { id = dever.job.enqueue(job.retry_later, input, input.key) }", + )); +} + +#[test] +fn schedules_validate_utc_fields_and_zero_input_targets() { + checked(&sources( + "database default\njob cleanup() () retry(3) timeout(1000) { app.clean() }\nschedule cleanup = \"0 * * * *\"", + "clean() () {}", + )); + rejected( + &sources(&format!("{JOB}\nschedule publish = \"0 * * * *\""), APP), + "zero-input", + ); + rejected( + &sources( + "database default\njob cleanup() () retry(3) timeout(1000) {}\nschedule cleanup = \"60 * * * *\"", + "clean() () {}", + ), + "cron field", + ); +} + +#[test] +fn nominal_model_id_wire_identity_is_preserved() { + let mut sources = sources( + JOB, + &APP.replace("title: Text", "title: Text\narticle_id: model.id"), + ); + sources.add( + "content/article/model.dever", + "type Article { title: Text }", + ); + let program = checked(&sources); + let schema = program + .wire_schema("content.article.Publish", dever_core::wire::Policy::Job) + .unwrap(); + assert!( + schema + .fingerprint() + .contains("ModelId(\"content.article.model.id\")") + ); +} + +#[test] +fn markdown_job_and_schedule_share_the_source_contract() { + let markdown = "# 文章清理任务\n\n清理已失效文章。\n\n- 包：`content.article.job`\n- 公开类型：无\n- 公开方法：无\n- 使用：无\n\n## 队列连接\n\n使用默认连接。\n\n- 声明：`database default`\n\n```dever\ndatabase default\n```\n\n## 清理入口\n\n调用领域能力。\n\n- 函数：`cleanup`\n- 输入：无\n- 输出：无\n\n```dever\njob cleanup() () retry(3) timeout(1000) { app.clean() }\n```\n\n## 每小时清理\n\n按 UTC 整点运行。\n\n- 声明：`schedule cleanup`\n\n```dever\nschedule cleanup = \"0 * * * *\"\n```\n"; + let mut sources = SourceMap::default(); + let id = sources.add("content/article/job.dever.md", markdown); + sources.add("content/article/app.dever", "clean() () {}"); + checked(&sources); + let formatted = dever_core::format::format(sources.get(id)).unwrap(); + let mut round_trip = SourceMap::default(); + round_trip.add("content/article/job.dever.md", &formatted); + round_trip.add("content/article/app.dever", "clean() () {}"); + checked(&round_trip); +} + +#[test] +fn transaction_rejects_model_and_job_on_different_explicit_connections() { + let mut sources = sources( + JOB, + "type Publish { title: Text }\napply(input: Publish) () {}\ntransaction submit(input: Publish) (id: Id) {\n article = model.create({ title = input.title })\n id = dever.job.enqueue(job.publish, input, input.title)\n}", + ); + sources.add( + "content/article/model.dever", + "database report\ntype Article { title: Text }", + ); + rejected(&sources, "multiple explicit database connections"); +} + +#[test] +fn tenant_model_jobs_generate_scoped_storage_and_private_schema_migration() { + let mut sources = sources( + "database default\njob publish(input: app.Publish) () retry(5) timeout(30000) { app.apply(input) }\njob cleanup() () retry(2) timeout(1000) { app.clean() }\nschedule cleanup = \"0 * * * *\"", + "type Publish { title: Text }\napply(input: Publish) () { article = model.create({ title = input.title }) }\nclean() () {}\ntransaction submit(input: Publish) (id: Id) { article = model.create({ title = input.title })\nid = dever.job.enqueue(job.publish, input, input.title) }", + ); + sources.add( + "content/article/model.dever", + "type Article { title: Text }", + ); + let program = checked(&sources); + let generated = dever_core::native::emit_project( + &program, + &sources, + dever_runtime::config::RuntimeProfile { + sqlite: true, + postgres: false, + }, + ) + .unwrap(); + assert!(generated.contains("JOB_STORAGE_")); + assert!(generated.contains("ModelScope::Tenant")); + assert!(generated.contains("job_migrate_tenant(tenant_id")); + assert!(generated.contains("dever_runtime::job::store::initialize")); + assert!(!generated.contains("JOB_DATABASE_")); + let storage_initializers = generated + .lines() + .filter(|line| line.contains("JOB_STORAGE_") && line.contains(".set(")) + .collect::<Vec<_>>(); + assert_eq!(storage_initializers.len(), 2); + assert!( + storage_initializers + .iter() + .all(|line| line.contains("ModelScope::Tenant")) + ); + dever_core::native::compile_project( + &program, + &sources, + std::ffi::OsStr::new("rustc"), + dever_runtime::config::RuntimeProfile { + sqlite: true, + postgres: false, + }, + ) + .unwrap(); +} + +#[test] +fn runtime_modes_and_job_limits_are_strict_and_validate_permissions() { + let project = temp::TemporaryDirectory::new(); + std::fs::create_dir(project.path().join("config")).unwrap(); + let file = project.path().join("config/setting.json"); + let valid = r#"{"runtime":{"mode":"worker","shutdown_ms":30000},"job":{"workers":2,"poll_ms":250,"lease_ms":60000,"retry_base_ms":1000,"retry_max_ms":300000}}"#; + std::fs::write(&file, valid).unwrap(); + let settings = dever_runtime::config::Settings::load_project(project.path()).unwrap(); + settings.services(false, true).unwrap(); + assert!(settings.services(true, false).is_err()); + let inferred_worker = valid.replace(r#""runtime":{"mode":"worker","shutdown_ms":30000},"#, ""); + std::fs::write(&file, inferred_worker).unwrap(); + let inferred = dever_runtime::config::Settings::load_project(project.path()).unwrap(); + assert_eq!( + inferred.services(false, true).unwrap().mode, + dever_runtime::config::RuntimeMode::Worker + ); + for mode in ["api", "all"] { + let document = valid.replace("\"worker\"", &format!("\"{mode}\"")); + std::fs::write(&file, &document).unwrap(); + assert!( + dever_runtime::config::Settings::load_project(project.path()) + .unwrap() + .services(true, true) + .is_err() + ); + let document = + document.replacen('{', "{\"http\":{\"host\":\"127.0.0.1\",\"port\":8181},", 1); + std::fs::write(&file, document).unwrap(); + dever_runtime::config::Settings::load_project(project.path()) + .unwrap() + .services(true, true) + .unwrap(); + } + for invalid in [ + valid.replace("\"worker\"", "\"unknown\""), + valid.replace("\"workers\":2", "\"workers\":0"), + valid.replace("\"lease_ms\":60000", "\"lease_ms\":0"), + valid.replace("\"shutdown_ms\":30000", "\"shutdown_ms\":-1"), + "{\"runtime\":null}".into(), + "{\"job\":null}".into(), + ] { + std::fs::write(&file, invalid).unwrap(); + assert!(dever_runtime::config::Settings::load_project(project.path()).is_err()); + } + let document = valid.replacen('{', "{\"database\":{\"default\":{\"type\":\"sqlite\",\"path\":\"data/db/jobs.db\",\"max_connections\":3}},", 1); + std::fs::write(&file, document).unwrap(); + let settings = dever_runtime::config::Settings::load_project(project.path()).unwrap(); + settings + .validate_job_connection(Some("default"), "content", 59_999) + .unwrap(); + for timeout in [60_000, 60_001] { + assert!( + settings + .validate_job_connection(Some("default"), "content", timeout) + .unwrap_err() + .contains("must exceed Job timeout_ms") + ); + } + std::fs::write( + &file, + valid.replace("\"lease_ms\":60000", "\"lease_ms\":7200000"), + ) + .unwrap(); + dever_runtime::config::Settings::load_project(project.path()).unwrap(); + std::fs::write( + &file, + valid.replace("\"lease_ms\":60000", "\"lease_ms\":7200001"), + ) + .unwrap(); + assert!(dever_runtime::config::Settings::load_project(project.path()).is_err()); +} + +#[test] +fn utc_cron_handles_calendar_ranges_steps_and_day_or_semantics() { + let minute = |text| dever_runtime::time::parse_datetime(text).unwrap() / 60000; + let cron = dever_runtime::cron::Cron::parse("*/15 3 20 9 1").unwrap(); + assert!(cron.matches_minute(minute("2026-09-20T03:30:00Z"))); + assert!(cron.matches_minute(minute("2026-09-21T03:30:00Z"))); + assert!(!cron.matches_minute(minute("2026-09-22T03:30:00Z"))); + assert!(!cron.matches_minute(minute("2026-09-20T03:31:00Z"))); + for invalid in [ + "* * * *", + "0 24 * * *", + "0 0 0 * *", + "0 0 * 13 *", + "0 0 * * 7", + "*/0 * * * *", + "5-2 * * * *", + "0 0 31 2 *", + ] { + assert!(dever_runtime::cron::Cron::parse(invalid).is_err()); + } +} + +#[test] +fn application_jobs_use_case_local_sqlite_clock_and_port_fakes() { + let project = temp::TemporaryDirectory::new(); + let long_job = JOB.replace("timeout(30000)", "timeout(3600000)"); + let files = [ + ("module/main.dever", "main() () {}"), + ( + "module/content/article/model.dever", + "type Article { title: Text }", + ), + ( + "module/content/article/app.dever", + "type Publish { title: Text\narticle_id: model.id }\ntype DeliveryError { error Failed(message: Text) }\ntransaction submit(title: Text) (id: Id) {\n article = model.create({ title = title })\n id = dever.job.enqueue(job.publish, Publish { title = title\narticle_id = article.id }, title)\n}\napply(input: Publish) () {\n original = model.get(input.article_id)\n article = model.create({ title = port.title(original.title) })\n}\ncount(title: Text) (total: Int) { total = model.count({ where = title == title }) }", + ), + ("module/content/article/job.dever", long_job.as_str()), + ( + "module/content/article/port.dever", + "title(value: Text) (title: Text) fails app.DeliveryError", + ), + ( + "module/content/article/adapter.dever", + "port.title(value: Text) (title: Text) { title = \"production\" }", + ), + ( + "test/content/article/first.dever", + "port.title(value: Text) (title: Text) { title = \"first\" }\nfirst() () {\n assert_eq(app.count(\"first\"), 0)\n id = app.submit(\"same-key\")\n assert_eq(dever.test.drain_jobs(5), 1)\n assert_eq(app.count(\"first\"), 1)\n dever.test.advance_clock(1000)\n assert_eq(dever.test.drain_jobs(5), 0)\n}", + ), + ( + "test/content/article/second.dever", + "port.title(value: Text) (title: Text) { title = \"second\" }\nsecond() () {\n assert_eq(app.count(\"first\"), 0)\n id = app.submit(\"same-key\")\n assert_eq(dever.test.drain_jobs(5), 1)\n assert_eq(app.count(\"second\"), 1)\n}", + ), + ( + "test/content/article/empty.dever", + "empty() () { dever.test.advance_clock(1000)\nassert_eq(dever.test.drain_jobs(5), 0) }", + ), + ]; + for (path, source) in files { + let path = project.path().join(path); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, source).unwrap(); + } + let sources = + SourceMap::load_project(&project.path().join("module"), &project.path().join("test")) + .unwrap(); + let program = checked(&sources); + assert_eq!( + program + .tests() + .iter() + .filter(|case| case.uses_database()) + .count(), + 2 + ); + let native = + dever_core::native::compile_test_suite(&program, &sources, std::ffi::OsStr::new("rustc")) + .unwrap(); + for index in 0..program.tests().len() { + let case = temp::TemporaryDirectory::new(); + std::fs::create_dir(case.path().join("config")).unwrap(); + std::fs::write(case.path().join("config/setting.json"), r#"{"database":{"default":{"type":"sqlite","path":"data/db/case.db","max_connections":1}}}"#).unwrap(); + let binary = case.path().join("case"); + native.save(&binary).unwrap(); + let output = std::process::Command::new(binary) + .arg(index.to_string()) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } +} + +#[cfg(feature = "sqlite")] +#[test] +fn native_dispatch_blocks_changed_schema_corrupt_payload_and_unknown_target() { + let project = temp::TemporaryDirectory::new(); + for (path, source) in [ + ("module/main.dever", "main() () {}"), + ( + "module/content/article/model.dever", + "type Article { title: Text }", + ), + ( + "module/content/article/app.dever", + "type Work { title: Text }\nsubmit() (id: Id) { id = dever.job.enqueue(job.publish, Work { title = \"delivered\" }, \"same\") }\napply(input: Work) () { article = model.create({ title = input.title }) }\ncount() (total: Int) { total = model.count() }", + ), + ( + "module/content/article/job.dever", + "database default\njob publish(input: app.Work) () retry(2) timeout(1000) { app.apply(input) }", + ), + ( + "test/content/article/prepare.dever", + "prepare() () { id = app.submit() }", + ), + ( + "test/content/article/blocked.dever", + "blocked() () { id = app.submit()\nassert_eq(dever.test.drain_jobs(5), 1)\nassert_eq(app.count(), 0) }", + ), + ( + "test/content/article/unknown.dever", + "unknown() () { id = app.submit()\nassert_eq(dever.test.drain_jobs(5), 2)\nassert_eq(app.count(), 1) }", + ), + ] { + let path = project.path().join(path); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, source).unwrap(); + } + let sources = + SourceMap::load_project(&project.path().join("module"), &project.path().join("test")) + .unwrap(); + let program = checked(&sources); + let native = + dever_core::native::compile_test_suite(&program, &sources, std::ffi::OsStr::new("rustc")) + .unwrap(); + let index = |suffix: &str| { + program + .tests() + .iter() + .position(|case| case.name().ends_with(suffix)) + .unwrap() + .to_string() + }; + for (mutation, expected, entry) in [ + ( + "UPDATE _dever_jobs SET schema='old-version'", + "schema_mismatch", + "blocked", + ), + ( + "UPDATE _dever_jobs SET payload='{}'", + "invalid_payload", + "blocked", + ), + ( + "UPDATE _dever_jobs SET timeout_ms=3600001", + "invalid_policy", + "blocked", + ), + ( + "UPDATE _dever_jobs SET target='removed.job'", + "unknown_target", + "unknown", + ), + ] { + let case = temp::TemporaryDirectory::new(); + std::fs::create_dir(case.path().join("config")).unwrap(); + std::fs::write(case.path().join("config/setting.json"), r#"{"database":{"default":{"type":"sqlite","path":"data/db/jobs.db","max_connections":1}}}"#).unwrap(); + let binary = case.path().join("case"); + native.save(&binary).unwrap(); + let run = |entry| { + let output = std::process::Command::new(&binary) + .arg(index(entry)) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + }; + run("prepare"); + let path = case.path().join("data/db/jobs.db"); + let update_path = path.clone(); + dever_runtime::task::run_entry(async move { + let database = dever_runtime::sqlite::Database::open(update_path, 1, 100) + .map_err(|error| error.to_string())?; + database + .execute(mutation, vec![]) + .await + .map_err(|error| error.to_string())?; + Ok(()) + }) + .unwrap(); + run(entry); + dever_runtime::task::run_entry(async move { + let database = dever_runtime::sqlite::Database::open(path, 1, 100) + .map_err(|error| error.to_string())?; + let rows = database + .query( + "SELECT error,attempt FROM _dever_jobs WHERE state='blocked'", + vec![], + ) + .await + .map_err(|error| error.to_string())?; + assert_eq!(rows.len(), 1); + assert_eq!( + rows[0].get(0).unwrap(), + &dever_runtime::orm::Value::Text(expected.into()) + ); + assert_eq!(rows[0].get(1).unwrap(), &dever_runtime::orm::Value::Int(1)); + Ok(()) + }) + .unwrap(); + } +} + +#[cfg(feature = "sqlite")] +#[test] +fn sqlite_durable_state_machine() { + use database::{Executor, Sql}; + use dever_runtime::{ + config, database, + job::{self, Spec, store}, + orm::Value, + wire::Encoded, + }; + let executable = std::env::current_exe().unwrap(); + if executable.file_stem().unwrap() != "durable-job-case" { + // The child owns all runtime globals and its setting.json; deployment config is never read. + let project = temp::TemporaryDirectory::new(); + std::fs::create_dir(project.path().join("config")).unwrap(); + std::fs::write(project.path().join("config/setting.json"), r#"{"database":{"default":{"type":"sqlite","path":"data/db/jobs.db","max_connections":2}}}"#).unwrap(); + let child = project.path().join("durable-job-case"); + std::fs::copy(executable, &child).unwrap(); + let output = std::process::Command::new(child) + .args(["--exact", "sqlite_durable_state_machine", "--nocapture"]) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + return; + } + config::bootstrap( + config::RuntimeProfile { + sqlite: true, + postgres: false, + }, + &[(Some("default"), "content")], + &[], + ) + .unwrap(); + dever_runtime::task::run_entry(async { + job::scope_system(None, async { + let db = database::database_for(Some("default"), "content").unwrap(); + store::initialize(db.clone()).await.unwrap(); + store::initialize(db.clone()).await.unwrap(); + let spec = Spec { target: "content.article.job.publish", schema: "wire-v1:Job:Unit", attempts: 2, timeout_ms: 1000 }; + let sql = |text| Sql { sqlite: text, postgres: text }; + db.execute(sql("CREATE TABLE business (id BIGINT PRIMARY KEY)"), vec![]).await.unwrap(); + let tx = db.begin().await.unwrap(); + Executor::new(db.clone(), Some(&tx)).unwrap().execute(sql("INSERT INTO business(id) VALUES (1)"), vec![]).await.unwrap(); + job::enqueue(db.clone(), Some(&tx), &spec, Encoded::null(), "rollback", 0).await.unwrap(); + tx.rollback().await.unwrap(); + assert!(db.query(sql("SELECT id FROM business"), vec![]).await.unwrap().is_empty()); + assert!(db.query(sql("SELECT id FROM _dever_jobs"), vec![]).await.unwrap().is_empty()); + let tx = db.begin().await.unwrap(); + Executor::new(db.clone(), Some(&tx)).unwrap().execute(sql("INSERT INTO business(id) VALUES (2)"), vec![]).await.unwrap(); + let id = job::enqueue(db.clone(), Some(&tx), &spec, Encoded::null(), "commit", 1000).await.unwrap(); + tx.commit().await.unwrap(); + let (first, second) = tokio::join!(job::enqueue(db.clone(), None, &spec, Encoded::null(), "commit", 1000), job::enqueue(db.clone(), None, &spec, Encoded::null(), "commit", 1000)); + assert_eq!(first.unwrap(), id); + assert_eq!(second.unwrap(), id); + assert!(store::claim(db.clone(), &clock_at(999), 1000, 1).await.unwrap().is_empty()); + let claim = store::claim(db.clone(), &clock_at(1000), 1000, 1).await.unwrap().remove(0); + assert_eq!(claim.attempt, 1); + assert!(store::claim(db.clone(), &clock_at(1001), 1000, 1).await.unwrap().is_empty()); + let reclaimed = store::claim(db.clone(), &clock_at(2000), 1000, 1).await.unwrap().remove(0); + assert_eq!(reclaimed.attempt, 2); + assert_ne!(reclaimed.token, claim.token); + assert!(!store::renew(db.clone(), &claim, &clock_at(2001), 1000).await.unwrap()); + assert!(!store::complete(db.clone(), &claim, store::Outcome::Success, &clock_at(2001), 100, 1000).await.unwrap()); + assert!(store::complete(db.clone(), &reclaimed, store::Outcome::Success, &clock_at(2001), 100, 1000).await.unwrap()); + let reuse = job::enqueue(db.clone(), None, &spec, Encoded::null(), "commit", 3000).await.unwrap(); + assert_ne!(reuse, id); + let attempt = store::claim(db.clone(), &clock_at(3000), 1000, 1).await.unwrap().remove(0); + assert!(store::complete(db.clone(), &attempt, store::Outcome::Retry, &clock_at(3000), 100, 1000).await.unwrap()); + assert!(store::claim(db.clone(), &clock_at(3099), 1000, 1).await.unwrap().is_empty()); + assert_eq!(store::claim(db.clone(), &clock_at(3100), 1000, 1).await.unwrap()[0].attempt, 2); + assert!(store::claim(db.clone(), &clock_at(4100), 1000, 1).await.unwrap().is_empty()); + let states = db.query(sql("SELECT state,error FROM _dever_jobs WHERE state='dead'"), vec![]).await.unwrap(); + assert_eq!(states[0].get(1).unwrap(), &Value::Text("attempts_exhausted".into())); + // Advance the injected clock while each operation waits for SQLite's + // write lock. Claim starts a fresh lease; renew/finish cannot revive it. + let clock = clock_at(5000); + job::enqueue(db.clone(), None, &spec, Encoded::null(), "lock-wait", 5000).await.unwrap(); + let lock = db.begin().await.unwrap(); + Executor::new(db.clone(), Some(&lock)).unwrap().execute(sql("UPDATE business SET id=id WHERE id=2"), vec![]).await.unwrap(); + let pending = store::claim(db.clone(), &clock, 1000, 1); + tokio::pin!(pending); + assert!(tokio::time::timeout(std::time::Duration::from_millis(20), &mut pending).await.is_err()); + clock.advance(1000).unwrap(); + lock.commit().await.unwrap(); + let locked_claim = pending.await.unwrap().remove(0); + let leases = db.query(sql("SELECT lease_until FROM _dever_jobs WHERE dedupe_key='business:lock-wait'"), vec![]).await.unwrap(); + assert_eq!(leases[0].get(0).unwrap(), &Value::Int(7000)); + let lock = db.begin().await.unwrap(); + Executor::new(db.clone(), Some(&lock)).unwrap().execute(sql("UPDATE business SET id=id WHERE id=2"), vec![]).await.unwrap(); + let pending = store::renew(db.clone(), &locked_claim, &clock, 1000); + tokio::pin!(pending); + assert!(tokio::time::timeout(std::time::Duration::from_millis(20), &mut pending).await.is_err()); + clock.advance(1001).unwrap(); + lock.commit().await.unwrap(); + assert!(!pending.await.unwrap()); + let reclaimed = store::claim(db.clone(), &clock, 1000, 1).await.unwrap().remove(0); + let lock = db.begin().await.unwrap(); + Executor::new(db.clone(), Some(&lock)).unwrap().execute(sql("UPDATE business SET id=id WHERE id=2"), vec![]).await.unwrap(); + let pending = store::complete(db.clone(), &reclaimed, store::Outcome::Success, &clock, 100, 1000); + tokio::pin!(pending); + assert!(tokio::time::timeout(std::time::Duration::from_millis(20), &mut pending).await.is_err()); + clock.advance(1001).unwrap(); + lock.commit().await.unwrap(); + assert!(!pending.await.unwrap()); + static NEXT: Spec = Spec { target: "content.article.job.next", schema: "unit", attempts: 2, timeout_ms: 1000 }; + job::enqueue(db.clone(), None, &NEXT, Encoded::null(), "after-exhausted", clock.now().unwrap()).await.unwrap(); + let bindings = vec![job::worker::Binding::fixed(db.clone(), &NEXT, None)]; + assert_eq!(job::worker::drain(bindings, clock.clone(), 5, |_| async { store::Outcome::Success }).await.unwrap(), 1); + assert!(store::claim(db.clone(), &clock, 1000, 1).await.unwrap().is_empty()); + let cron = dever_runtime::cron::Cron::parse("* * * * *").unwrap(); + assert_eq!(store::materialize(db.clone(), &spec, &cron, 60_000).await.unwrap(), 0); + let (a, b) = tokio::join!(store::materialize(db.clone(), &spec, &cron, 120_000), store::materialize(db.clone(), &spec, &cron, 120_000)); + assert_eq!(a.unwrap() + b.unwrap(), 1); + let slot = store::claim(db.clone(), &clock_at(120_000), 1000, 1).await.unwrap().remove(0); + store::complete(db.clone(), &slot, store::Outcome::Success, &clock_at(120_001), 100, 1000).await.unwrap(); + assert_eq!(store::materialize(db.clone(), &spec, &cron, 120_000).await.unwrap(), 0); + assert!(store::materialize(db.clone(), &spec, &dever_runtime::cron::Cron::parse("0 * * * *").unwrap(), 120_000).await.is_err()); + assert_eq!(store::materialize(db.clone(), &spec, &cron, 60_000 * 1000).await.unwrap(), 256); + // A failure after inserting an occurrence must roll back both it and its cursor. + db.execute(sql("CREATE TRIGGER reject_job_cursor BEFORE UPDATE ON _dever_job_schedule BEGIN SELECT RAISE(ABORT, 'test rollback'); END"), vec![]).await.unwrap(); + let before = db.query(sql("SELECT COUNT(*) FROM _dever_jobs"), vec![]).await.unwrap(); + assert!(store::materialize(db.clone(), &spec, &cron, 60_000 * 259).await.is_err()); + let after = db.query(sql("SELECT COUNT(*) FROM _dever_jobs"), vec![]).await.unwrap(); + assert_eq!(before[0].get(0).unwrap(), after[0].get(0).unwrap()); + db.execute(sql("DROP TRIGGER reject_job_cursor"), vec![]).await.unwrap(); + store::initialize(db.clone()).await.unwrap(); + assert_eq!(store::materialize(db.clone(), &spec, &cron, 60_000 * 259).await.unwrap(), 1); + db.execute(sql("UPDATE _dever_job_version SET version=99"), vec![]).await.unwrap(); + assert!(store::initialize(db.clone()).await.is_err()); + Ok(()) + }).await + }).unwrap(); +} + +#[cfg(feature = "sqlite")] +#[test] +fn sqlite_job_identity_migration_blocks_legacy_work_and_persists_trusted_identity() { + use dever_runtime::{ + config, database, + job::{self, ExecutionIdentity, Spec, store}, + orm::Value, + wire::Encoded, + }; + + let executable = std::env::current_exe().unwrap(); + if executable.file_stem().unwrap() != "job-identity-migration-case" { + let project = temp::TemporaryDirectory::new(); + std::fs::create_dir(project.path().join("config")).unwrap(); + std::fs::write(project.path().join("config/setting.json"), r#"{"database":{"default":{"type":"sqlite","path":"data/db/jobs.db","max_connections":2}}}"#).unwrap(); + let child = project.path().join("job-identity-migration-case"); + std::fs::copy(executable, &child).unwrap(); + let output = std::process::Command::new(child) + .args([ + "--exact", + "sqlite_job_identity_migration_blocks_legacy_work_and_persists_trusted_identity", + "--nocapture", + ]) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + return; + } + + config::bootstrap( + config::RuntimeProfile { + sqlite: true, + postgres: false, + }, + &[(Some("default"), "content")], + &[], + ) + .unwrap(); + dever_runtime::task::run_entry(async { + let db = database::database_for(Some("default"), "content").unwrap(); + let sql = |text| database::Sql { sqlite: text, postgres: text }; + for statement in [ + "CREATE TABLE _dever_job_version (singleton INTEGER PRIMARY KEY CHECK(singleton=1), version BIGINT NOT NULL)", + "INSERT INTO _dever_job_version(singleton,version) VALUES (1,2)", + "CREATE TABLE _dever_jobs (id TEXT PRIMARY KEY, tenant_id BIGINT NOT NULL, target TEXT NOT NULL, schema TEXT NOT NULL, payload TEXT NOT NULL, dedupe_key TEXT NOT NULL, state TEXT NOT NULL, run_at BIGINT NOT NULL, attempt BIGINT NOT NULL, max_attempts BIGINT NOT NULL, timeout_ms BIGINT NOT NULL, claim_token TEXT NOT NULL, lease_until BIGINT NOT NULL, error TEXT NOT NULL, updated_at BIGINT NOT NULL)", + "CREATE UNIQUE INDEX _dever_jobs_active ON _dever_jobs(tenant_id,target,dedupe_key) WHERE state IN ('pending','running')", + "CREATE INDEX _dever_jobs_due ON _dever_jobs(state,run_at,lease_until)", + "CREATE TABLE _dever_job_schedule (identity TEXT PRIMARY KEY, fingerprint TEXT NOT NULL, last_minute BIGINT NOT NULL)", + "INSERT INTO _dever_jobs(id,tenant_id,target,schema,payload,dedupe_key,state,run_at,attempt,max_attempts,timeout_ms,claim_token,lease_until,error,updated_at) VALUES ('legacy',0,'content.article.job.publish','unit','null','business:legacy','pending',0,0,2,1000,'',0,'',0)", + ] { + db.execute(sql(statement), vec![]).await.unwrap(); + } + + store::initialize(db.clone()).await.unwrap(); + let migrated = db.query(sql("SELECT version,state,error,execution_kind FROM _dever_job_version CROSS JOIN _dever_jobs WHERE id='legacy'"), vec![]).await.unwrap(); + assert_eq!(migrated[0].get(0).unwrap(), &Value::Int(5)); + assert_eq!(migrated[0].get(1).unwrap(), &Value::Text("blocked".into())); + assert_eq!(migrated[0].get(2).unwrap(), &Value::Text("missing_execution_identity".into())); + assert_eq!(migrated[0].get(3).unwrap(), &Value::Text("legacy".into())); + + static SPEC: Spec = Spec { target: "content.article.job.publish", schema: "unit", attempts: 2, timeout_ms: 1000 }; + assert!(job::enqueue(db.clone(), None, &SPEC, Encoded::null(), "missing", 0).await.unwrap_err().to_string().contains("trusted Job execution identity")); + job::scope_system(None, job::enqueue(db.clone(), None, &SPEC, Encoded::null(), "system", 0)).await.unwrap(); + let user = ExecutionIdentity::User { + tenant_id: Some(7), + provider: "account".into(), + site: "admin".into(), + subject: "user:42".into(), + session: "session:9".into(), + tenant_claim: Some("tenant-seven".into()), + permission_key: "news.article.admin.publish".into(), + }; + job::scope_execution(user.clone(), job::enqueue(db.clone(), None, &SPEC, Encoded::null(), "user", 0)).await.unwrap(); + + let rows = db.query(sql("SELECT execution_kind,scope_tenant_id,auth_provider,auth_site,auth_subject,auth_session,auth_tenant,auth_permission FROM _dever_jobs WHERE dedupe_key='business:user'"), vec![]).await.unwrap(); + assert_eq!(rows[0].get(0).unwrap(), &Value::Text("user".into())); + assert_eq!(rows[0].get(1).unwrap(), &Value::Int(7)); + assert_eq!(rows[0].get(2).unwrap(), &Value::Text("account".into())); + assert_eq!(rows[0].get(3).unwrap(), &Value::Text("admin".into())); + assert_eq!(rows[0].get(4).unwrap(), &Value::Text("user:42".into())); + assert_eq!(rows[0].get(5).unwrap(), &Value::Text("session:9".into())); + assert_eq!(rows[0].get(6).unwrap(), &Value::Text("tenant-seven".into())); + assert_eq!(rows[0].get(7).unwrap(), &Value::Text("news.article.admin.publish".into())); + let claims = store::claim(db.clone(), &job::Clock::for_test(), 1000, 2).await.unwrap(); + assert!(claims.iter().any(|claim| claim.execution == ExecutionIdentity::System { tenant_id: None })); + assert!(claims.iter().any(|claim| claim.execution == user)); + + db.execute(sql("DROP TABLE _dever_jobs"), vec![]).await.unwrap(); + db.execute(sql("UPDATE _dever_job_version SET version=4 WHERE singleton=1"), vec![]).await.unwrap(); + db.execute(sql("CREATE TABLE _dever_jobs (id TEXT PRIMARY KEY, tenant_id BIGINT NOT NULL, scope_tenant_id BIGINT NOT NULL, execution_kind TEXT NOT NULL, auth_provider TEXT NOT NULL, auth_site TEXT NOT NULL, auth_subject TEXT NOT NULL, auth_session TEXT NOT NULL, auth_tenant TEXT NOT NULL, auth_capabilities TEXT NOT NULL, target TEXT NOT NULL, schema TEXT NOT NULL, payload TEXT NOT NULL, dedupe_key TEXT NOT NULL, state TEXT NOT NULL, run_at BIGINT NOT NULL, attempt BIGINT NOT NULL, max_attempts BIGINT NOT NULL, timeout_ms BIGINT NOT NULL, claim_token TEXT NOT NULL, lease_until BIGINT NOT NULL, error TEXT NOT NULL, updated_at BIGINT NOT NULL)"), vec![]).await.unwrap(); + for (id, execution_kind) in [("system-v4", "system"), ("user-v4", "user")] { + db.execute( + sql("INSERT INTO _dever_jobs(id,tenant_id,scope_tenant_id,execution_kind,auth_provider,auth_site,auth_subject,auth_session,auth_tenant,auth_capabilities,target,schema,payload,dedupe_key,state,run_at,attempt,max_attempts,timeout_ms,claim_token,lease_until,error,updated_at) VALUES (?1,0,0,?2,'account','admin','user:42','session:9','','[]','content.article.job.publish','unit','null',?3,'pending',0,0,2,1000,'',0,'',0)"), + vec![Value::Text(id.into()), Value::Text(execution_kind.into()), Value::Text(format!("v4:{id}"))], + ).await.unwrap(); + } + store::initialize(db.clone()).await.unwrap(); + let migrated = db.query(sql("SELECT version,id,state,error,auth_permission FROM _dever_job_version CROSS JOIN _dever_jobs ORDER BY id"), vec![]).await.unwrap(); + assert_eq!(migrated[0].get(0).unwrap(), &Value::Int(5)); + assert_eq!(migrated[0].get(1).unwrap(), &Value::Text("system-v4".into())); + assert_eq!(migrated[0].get(2).unwrap(), &Value::Text("pending".into())); + assert_eq!(migrated[0].get(4).unwrap(), &Value::Text(String::new())); + assert_eq!(migrated[1].get(1).unwrap(), &Value::Text("user-v4".into())); + assert_eq!(migrated[1].get(2).unwrap(), &Value::Text("blocked".into())); + assert_eq!(migrated[1].get(3).unwrap(), &Value::Text("authorization_contract_changed".into())); + assert_eq!(migrated[1].get(4).unwrap(), &Value::Text(String::new())); + let columns = db.query(sql("PRAGMA table_info(_dever_jobs)"), vec![]).await.unwrap(); + assert!(!columns.iter().any(|row| { + row.get(1).is_ok_and(|value| value == &Value::Text("auth_capabilities".into())) + })); + database::shutdown().await.unwrap(); + Ok(()) + }).unwrap(); +} + +#[cfg(all(feature = "sqlite", feature = "api", feature = "crypto", unix))] +#[test] +fn revoked_user_job_is_blocked_before_business_dispatch() { + use std::io::{Read, Write}; + use std::time::{Duration, Instant}; + + struct Child(std::process::Child); + impl Drop for Child { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } + } + + fn request( + address: std::net::SocketAddr, + path: &str, + body: &str, + cookie: Option<&str>, + ) -> String { + let mut stream = + std::net::TcpStream::connect_timeout(&address, Duration::from_secs(1)).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + let cookie = cookie + .map(|value| format!("Cookie: {value}\r\n")) + .unwrap_or_default(); + write!(stream, + "POST {path} HTTP/1.1\r\nHost: {address}\r\nOrigin: http://{address}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n{cookie}\r\n{body}", + body.len(), + ).unwrap(); + let mut response = String::new(); + stream.read_to_string(&mut response).unwrap(); + response + } + + fn sqlite_rows(path: std::path::PathBuf, sql: &'static str) -> Vec<dever_runtime::orm::Row> { + dever_runtime::task::run_entry(async move { + let database = dever_runtime::sqlite::Database::open(path, 1, 100) + .map_err(|error| error.to_string())?; + database + .query(sql, vec![]) + .await + .map_err(|error| error.to_string()) + }) + .unwrap() + } + + fn sqlite_execute(path: std::path::PathBuf, sql: &'static str) { + dever_runtime::task::run_entry(async move { + let database = dever_runtime::sqlite::Database::open(path, 1, 100) + .map_err(|error| error.to_string())?; + database + .execute(sql, vec![]) + .await + .map_err(|error| error.to_string())?; + Ok(()) + }) + .unwrap(); + } + + let project = temp::TemporaryDirectory::new(); + std::fs::create_dir(project.path().join("config")).unwrap(); + let reservation = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = reservation.local_addr().unwrap(); + let setting = |mode: &str| { + format!( + r#"{{ + "runtime":{{"mode":"{mode}","shutdown_ms":1000}}, + "http":{{"host":"127.0.0.1","port":{}}}, + "job":{{"workers":1,"poll_ms":10,"lease_ms":2000,"retry_base_ms":10,"retry_max_ms":100}}, + "database":{{"default":{{"type":"sqlite","path":"data/db/jobs.db","max_connections":4}}}}, + "auth":{{"providers":{{"session":{{ + "verify":"user.account.verify", + "jwtSecret":"0123456789abcdef0123456789abcdef", + "ttlSeconds":3600, + "cookie":"cms_session" + }}}}}}, + "sites":{{"front":{{ + "path":"front", + "auth":"session", + "hosts":["127.0.0.1"], + "origin":"http://{}" + }}}} + }}"#, + address.port(), + address + ) + }; + std::fs::write(project.path().join("config/setting.json"), setting("api")).unwrap(); + let settings = dever_runtime::config::Settings::load_project(project.path()).unwrap(); + + let mut application = SourceMap::default(); + application.add( + "user/account/model.dever", + "global type Account { session: Text unique\nactive: Bool default true }", + ); + application.add( + "user/account/model/effect.dever", + "global type Effect { value: Text }", + ); + application.add( + "user/account/app.dever", + r#" +type Identity { + id: Text + user_id: model.id? + tenant_id: model.id? +} +type Work { value: Text } +login(session: Text) (result: Text) { + stored = model.create({ session = session }) + discarded = stored + dever.auth.issue_cookie("account:1", session, null) + result = "ok" +} +verify(claims: dever.auth.Claims) (identity: Identity) { + stored = model.first({ where = session == claims.session and active == true }) + identity = verified(stored, claims) +} +verified(stored: model.Account, claims: dever.auth.Claims) (identity: Identity) { + identity = Identity { + id = claims.subject + user_id = stored.id + tenant_id = null + } +} +verified(stored: null, claims: dever.auth.Claims) (identity: Identity) { + fail(dever.api.Error.Unauthorized) +} +submit() (id: Id) { + id = dever.job.enqueue(job.deliver, Work { value = "executed" }, "revoked-user") +} +deliver(input: Work) () { + stored = model.effect.create({ value = input.value }) + discarded = stored +} +"#, + ); + application.add( + "user/account/api/front/session.dever", + "public post login = app.login\npost submit = app.submit", + ); + application.add("user/account/job.dever", "database default\njob deliver(input: app.Work) () retry(2) timeout(1000) { app.deliver(input) }"); + let program = + dever_core::check_with_settings(&application, &settings).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&application)) + .collect::<String>() + ) + }); + let rustc = std::env::var_os("RUSTC").unwrap_or_else(|| "rustc".into()); + let native = dever_core::native::compile_project( + &program, + &application, + &rustc, + dever_runtime::config::RuntimeProfile { + sqlite: true, + postgres: false, + }, + ) + .unwrap(); + let binary = project.path().join("application"); + native.save(&binary).unwrap(); + drop(reservation); + + let mut api = Child( + std::process::Command::new(&binary) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(), + ); + let deadline = Instant::now() + Duration::from_secs(15); + while std::net::TcpStream::connect_timeout(&address, Duration::from_millis(20)).is_err() { + if api.0.try_wait().unwrap().is_some() || Instant::now() >= deadline { + let mut stderr = String::new(); + if let Some(mut pipe) = api.0.stderr.take() { + pipe.read_to_string(&mut stderr).unwrap(); + } + panic!("API did not start: {stderr}"); + } + std::thread::sleep(Duration::from_millis(10)); + } + + let login = request( + address, + "/user/account/front/session/login", + r#"{"session":"session-1"}"#, + None, + ); + assert!(login.starts_with("HTTP/1.1 200"), "{login}"); + let cookie = login + .lines() + .find_map(|line| { + line.strip_prefix("set-cookie: ") + .or_else(|| line.strip_prefix("Set-Cookie: ")) + .map(|value| value.split(';').next().unwrap().to_owned()) + }) + .expect("login response Cookie"); + let database_path = project.path().join("data/db/jobs.db"); + for statement in [ + "INSERT INTO _dever_auth_role(id,site,name,all_permissions,enabled) VALUES ('submitter','front','Submitter',0,1)", + "INSERT INTO _dever_auth_role_permission(site,role_id,permission_key) VALUES ('front','submitter','user.account.front.submit')", + "INSERT INTO _dever_auth_user_role(user_id,site,role_id) VALUES (1,'front','submitter')", + ] { + sqlite_execute(database_path.clone(), statement); + } + let submit = request( + address, + "/user/account/front/session/submit", + "{}", + Some(&cookie), + ); + assert!(submit.starts_with("HTTP/1.1 200"), "{submit}"); + let _ = api.0.kill(); + let _ = api.0.wait(); + + sqlite_execute( + database_path.clone(), + "DELETE FROM _dever_auth_user_role WHERE user_id=1 AND site='front' AND role_id='submitter'", + ); + + std::fs::write( + project.path().join("config/setting.json"), + setting("worker"), + ) + .unwrap(); + let mut worker = Child( + std::process::Command::new(&binary) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(), + ); + let deadline = Instant::now() + Duration::from_secs(15); + let mut blocked = false; + while Instant::now() < deadline { + if worker.0.try_wait().unwrap().is_some() { + break; + } + let rows = sqlite_rows( + database_path.clone(), + "SELECT state,error FROM _dever_jobs WHERE dedupe_key='business:revoked-user'", + ); + blocked = rows.first().is_some_and(|row| { + row.get(0).ok() == Some(&dever_runtime::orm::Value::Text("blocked".into())) + && row.get(1).ok() + == Some(&dever_runtime::orm::Value::Text("identity_rejected".into())) + }); + if blocked { + break; + } + std::thread::sleep(Duration::from_millis(20)); + } + if !blocked { + let mut stderr = String::new(); + if let Some(mut pipe) = worker.0.stderr.take() { + pipe.read_to_string(&mut stderr).unwrap(); + } + panic!("revoked Job was not blocked: {stderr}"); + } + assert!( + sqlite_rows(database_path, "SELECT id FROM effect").is_empty(), + "revoked Job executed its business write" + ); + + assert!( + std::process::Command::new("kill") + .args(["-TERM", &worker.0.id().to_string()]) + .status() + .unwrap() + .success() + ); + while worker.0.try_wait().unwrap().is_none() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!( + worker + .0 + .try_wait() + .unwrap() + .is_some_and(|status| status.success()) + ); +} + +#[cfg(all(feature = "sqlite", unix))] +#[test] +fn one_native_application_gates_api_worker_and_all_modes() { + use std::time::{Duration, Instant}; + struct Child(std::process::Child); + impl Drop for Child { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } + } + let mut sources = sources( + "database default\njob publish() () retry(2) timeout(1000) { app.apply() }", + "submit() (id: Id) { id = dever.job.enqueue(job.publish, \"mode\") }\napply() () { article = model.create({ title = \"delivered\" }) }", + ); + sources.add( + "content/article/model.dever", + "global type Article { title: Text }", + ); + sources.add("content/article/api.dever", "cmd submit = app.submit"); + sources.add( + "monitor/health/app.dever", + "health() (response: Bool) { response = true }", + ); + sources.add("monitor/health/api.dever", "public get health = app.health"); + let program = checked(&sources); + let native = dever_core::native::compile_project( + &program, + &sources, + std::ffi::OsStr::new("rustc"), + dever_runtime::config::RuntimeProfile { + sqlite: true, + postgres: false, + }, + ) + .unwrap(); + for mode in ["api", "worker", "all"] { + let project = temp::TemporaryDirectory::new(); + std::fs::create_dir(project.path().join("config")).unwrap(); + let reservation = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = reservation.local_addr().unwrap(); + let http = if mode == "worker" { + String::new() + } else { + format!( + ",\"http\":{{\"host\":\"127.0.0.1\",\"port\":{}}}", + address.port() + ) + }; + std::fs::write(project.path().join("config/setting.json"), format!("{{\"database\":{{\"default\":{{\"type\":\"sqlite\",\"path\":\"data/db/jobs.db\",\"max_connections\":2}}}},\"runtime\":{{\"mode\":\"{mode}\",\"shutdown_ms\":1000}},\"job\":{{\"workers\":1,\"poll_ms\":10,\"lease_ms\":1500,\"retry_base_ms\":10,\"retry_max_ms\":100}}{http}}}")).unwrap(); + let binary = project.path().join("application"); + native.save(&binary).unwrap(); + let queued = std::process::Command::new(&binary) + .args(["content.article.submit", "{}"]) + .output() + .unwrap(); + assert!( + queued.status.success(), + "{}\n{}", + String::from_utf8_lossy(&queued.stdout), + String::from_utf8_lossy(&queued.stderr) + ); + let response: serde_json::Value = serde_json::from_slice(&queued.stdout).unwrap(); + assert_eq!(response["code"], 0); + drop(reservation); + let mut child = Child( + std::process::Command::new(binary) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(), + ); + let path = project.path().join("data/db/jobs.db"); + let expected = if mode == "api" { + "pending" + } else { + "succeeded" + }; + let deadline = Instant::now() + Duration::from_secs(15); + let mut ready = false; + while Instant::now() < deadline { + if child.0.try_wait().unwrap().is_some() { + break; + } + let listening = + std::net::TcpStream::connect_timeout(&address, Duration::from_millis(20)).is_ok(); + let state = if path.exists() { + let path = path.clone(); + dever_runtime::task::run_entry(async move { + let database = dever_runtime::sqlite::Database::open(path, 1, 100) + .map_err(|error| error.to_string())?; + let rows = database + .query( + "SELECT state FROM _dever_jobs WHERE dedupe_key='business:mode'", + vec![], + ) + .await + .map_err(|error| error.to_string())?; + Ok(rows.first().and_then(|row| row.get(0).ok()).cloned()) + }) + .ok() + .flatten() + } else { + None + }; + if listening == (mode != "worker") + && state == Some(dever_runtime::orm::Value::Text(expected.into())) + { + ready = true; + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + if !ready { + let _ = child.0.kill(); + let _ = child.0.wait(); + let mut stderr = String::new(); + if let Some(mut pipe) = child.0.stderr.take() { + use std::io::Read; + pipe.read_to_string(&mut stderr).unwrap(); + } + panic!("{mode} did not reach its service state: {stderr}"); + } + assert!( + std::process::Command::new("kill") + .args(["-TERM", &child.0.id().to_string()]) + .status() + .unwrap() + .success() + ); + while child.0.try_wait().unwrap().is_none() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!( + child + .0 + .try_wait() + .unwrap() + .is_some_and(|status| status.success()), + "{mode} did not drain successfully" + ); + } +} + +#[cfg(all(feature = "sqlite", unix))] +#[test] +fn worker_signal_stops_claiming_and_drains_or_cancels_before_database_close() { + use dever_runtime::{ + config, database, + job::{self, Clock, Spec, store, worker}, + wire::Encoded, + }; + use std::time::{Duration, Instant}; + let executable = std::env::current_exe().unwrap(); + let name = executable.file_stem().unwrap().to_str().unwrap(); + if !name.starts_with("durable-job-signal-") { + for scenario in ["grace", "timeout", "transaction"] { + let project = temp::TemporaryDirectory::new(); + std::fs::create_dir(project.path().join("config")).unwrap(); + let shutdown = if scenario == "timeout" { 50 } else { 2000 }; + std::fs::write(project.path().join("config/setting.json"), format!(r#"{{"runtime":{{"mode":"worker","shutdown_ms":{shutdown}}},"job":{{"workers":1,"poll_ms":5,"lease_ms":2400,"retry_base_ms":100,"retry_max_ms":1000}},"database":{{"default":{{"type":"sqlite","path":"data/db/jobs.db","max_connections":2}}}}}}"#)).unwrap(); + let child_path = project + .path() + .join(format!("durable-job-signal-{scenario}")); + std::fs::copy(&executable, &child_path).unwrap(); + let mut child = std::process::Command::new(child_path) + .args([ + "--exact", + "worker_signal_stops_claiming_and_drains_or_cancels_before_database_close", + "--nocapture", + ]) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(); + let deadline = Instant::now() + Duration::from_secs(15); + while !project.path().join("ready").exists() && Instant::now() < deadline { + if child.try_wait().unwrap().is_some() { + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + if !project.path().join("ready").exists() { + let _ = child.kill(); + let output = child.wait_with_output().unwrap(); + panic!( + "worker did not become ready: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + assert!( + std::process::Command::new("kill") + .args(["-TERM", &child.id().to_string()]) + .status() + .unwrap() + .success() + ); + while child.try_wait().unwrap().is_none() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + if child.try_wait().unwrap().is_none() { + let _ = child.kill(); + } + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + std::fs::read_to_string(project.path().join("closed")).unwrap(), + scenario + ); + } + return; + } + let directory = executable.parent().unwrap().to_path_buf(); + let graceful = !name.ends_with("timeout"); + let transactional = name.ends_with("transaction"); + config::bootstrap( + config::RuntimeProfile { + sqlite: true, + postgres: false, + }, + &[(Some("default"), "content")], + &[], + ) + .unwrap(); + dever_runtime::lifecycle::configure(false, true).unwrap(); + dever_runtime::task::run_entry(async move { + job::scope_system(None, async move { + let db = database::database_for(Some("default"), "content").unwrap(); + store::initialize(db.clone()).await.unwrap(); + static SPEC: Spec = Spec { target: "content.article.job.signal", schema: "unit", attempts: 3, timeout_ms: 2000 }; + config::settings().validate_job_connection(Some("default"), "content", SPEC.timeout_ms).unwrap(); + let old_policy = Spec { timeout_ms: 3000, ..SPEC }; + job::enqueue(db.clone(), None, &old_policy, Encoded::null(), "old-policy", -1).await.unwrap(); + // UUID tie-breaking is intentionally random within one millisecond; give + // this ordering assertion distinct due times instead of assuming key order. + for (run_at, key) in ["a", "b"].into_iter().enumerate() { + job::enqueue(db.clone(), None, &SPEC, Encoded::null(), key, run_at as i64).await.unwrap(); + } + let settings = config::settings().jobs().unwrap(); + let ready = directory.join("ready"); + let service_db = db.clone(); + let result = dever_runtime::lifecycle::run(async move { + worker::serve(vec![worker::Binding::fixed(service_db.clone(), &SPEC, None)], Clock::default(), settings, move |_| { + let ready = ready.clone(); + let db = service_db.clone(); + async move { + let transaction = if transactional { Some(db.begin().await.unwrap()) } else { None }; + std::fs::write(ready, "claimed").unwrap(); + tokio::time::sleep(Duration::from_millis(1000)).await; + if let Some(transaction) = transaction { transaction.commit().await.unwrap(); } + store::Outcome::Success + } + }).await + }).await; + assert_eq!(result.is_ok(), graceful, "{result:?}"); + let sql = database::Sql { sqlite: "SELECT state,error FROM _dever_jobs WHERE dedupe_key!='business:old-policy' ORDER BY dedupe_key", postgres: "SELECT state,error FROM _dever_jobs WHERE dedupe_key!='business:old-policy' ORDER BY dedupe_key" }; + let rows = db.query(sql, vec![]).await.unwrap(); + let expected = if graceful { "succeeded" } else { "running" }; + assert_eq!(rows[0].get(0).unwrap(), &dever_runtime::orm::Value::Text(expected.into()), "{:?}", rows[0].get(1)); + assert_eq!(rows[1].get(0).unwrap(), &dever_runtime::orm::Value::Text("pending".into())); + let policy = db.query(database::Sql { sqlite: "SELECT state,error FROM _dever_jobs WHERE dedupe_key='business:old-policy'", postgres: "SELECT state,error FROM _dever_jobs WHERE dedupe_key='business:old-policy'" }, vec![]).await.unwrap(); + assert_eq!(policy[0].get(0).unwrap(), &dever_runtime::orm::Value::Text("blocked".into())); + assert_eq!(policy[0].get(1).unwrap(), &dever_runtime::orm::Value::Text("invalid_policy".into())); + database::shutdown().await.unwrap(); + assert!(db.query(database::Sql { sqlite: "SELECT 1", postgres: "SELECT 1" }, vec![]).await.is_err()); + dever_runtime::log::flush(); + std::fs::write(directory.join("closed"), if transactional { "transaction" } else if graceful { "grace" } else { "timeout" }).unwrap(); + Ok(()) + }).await + }).unwrap(); +} + +#[cfg(feature = "sqlite")] +#[test] +fn tenant_jobs_share_the_business_database_and_restore_only_tenant_scope() { + use dever_runtime::{ + config, database, + job::{self, Clock, Spec, store, worker}, + orm::Value, + wire::Encoded, + }; + + let executable = std::env::current_exe().unwrap(); + if executable.file_stem().unwrap() != "tenant-job-case" { + let project = temp::TemporaryDirectory::new(); + std::fs::create_dir(project.path().join("config")).unwrap(); + std::fs::write(project.path().join("config/setting.json"), r#"{ + "tenant":{"database":"default","max_pools":2,"idle_timeout_ms":1000}, + "database":{"default":{"type":"sqlite","path":"data/platform.db","tenant_directory":"data/tenants","max_connections":2}} + }"#).unwrap(); + let child = project.path().join("tenant-job-case"); + std::fs::copy(executable, &child).unwrap(); + let output = std::process::Command::new(child) + .args([ + "--exact", + "tenant_jobs_share_the_business_database_and_restore_only_tenant_scope", + "--nocapture", + ]) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + return; + } + + config::bootstrap_scoped( + config::RuntimeProfile { + sqlite: true, + postgres: false, + }, + &[(Some("default"), "content", true)], + &[&[(Some("default"), "content", true)]], + ) + .unwrap(); + dever_runtime::task::run_entry(async { + static SPEC: Spec = Spec { + target: "content.article.job.publish", + schema: "unit", + attempts: 2, + timeout_ms: 1000, + }; + const FINGERPRINT: &str = "tenant-job-test-v1"; + dever_runtime::tenant::initialize(FINGERPRINT) + .await + .unwrap(); + + for tenant_id in [1, 2] { + let db = dever_runtime::tenant::begin_migration("default", tenant_id, FINGERPRINT) + .await + .unwrap(); + store::initialize(db.clone()).await.unwrap(); + db.execute( + database::Sql { + sqlite: "CREATE TABLE business (id BIGINT PRIMARY KEY)", + postgres: "CREATE TABLE business (id BIGINT PRIMARY KEY)", + }, + vec![], + ) + .await + .unwrap(); + dever_runtime::tenant::mark_database_ready(&db, tenant_id, FINGERPRINT) + .await + .unwrap(); + dever_runtime::tenant::finish_migration(tenant_id, FINGERPRINT) + .await + .unwrap(); + } + + let storage = Box::leak(Box::new(database::StorageBinding::new( + Some("default"), + "content", + database::ModelScope::Tenant, + ))); + let db1 = dever_runtime::tenant::scope(1, storage.database()) + .await + .unwrap(); + let rolled_back = db1.begin().await.unwrap(); + database::Executor::new(db1.clone(), Some(&rolled_back)) + .unwrap() + .execute( + database::Sql { + sqlite: "INSERT INTO business(id) VALUES (1)", + postgres: "INSERT INTO business(id) VALUES (1)", + }, + vec![], + ) + .await + .unwrap(); + job::scope_system( + Some(1), + job::enqueue( + db1.clone(), + Some(&rolled_back), + &SPEC, + Encoded::null(), + "same", + 0, + ), + ) + .await + .unwrap(); + rolled_back.rollback().await.unwrap(); + assert!( + db1.query( + database::Sql { + sqlite: "SELECT id FROM business", + postgres: "SELECT id FROM business" + }, + vec![] + ) + .await + .unwrap() + .is_empty() + ); + assert!( + db1.query( + database::Sql { + sqlite: "SELECT id FROM _dever_jobs", + postgres: "SELECT id FROM _dever_jobs" + }, + vec![] + ) + .await + .unwrap() + .is_empty() + ); + + let tenant1_job = dever_runtime::tenant::scope( + 1, + job::scope_system(Some(1), async { + let db = storage.database().await.unwrap(); + job::enqueue(db, None, &SPEC, Encoded::null(), "same", 0) + .await + .unwrap() + }), + ) + .await; + let tenant2_job = dever_runtime::tenant::scope( + 2, + job::scope_system(Some(2), async { + let db = storage.database().await.unwrap(); + job::enqueue(db, None, &SPEC, Encoded::null(), "same", 0) + .await + .unwrap() + }), + ) + .await; + assert_ne!(tenant1_job, tenant2_job); + + let binding = worker::Binding::scoped(storage, database::ModelScope::Tenant, &SPEC, None); + let drained = dever_runtime::tenant::scope( + 1, + worker::drain(vec![binding], Clock::for_test(), 1, |claim| async move { + assert_eq!(claim.tenant_id, Some(1)); + assert_eq!(dever_runtime::tenant::current_id().unwrap(), 1); + #[cfg(feature = "api")] + assert!(dever_runtime::auth::id().is_err()); + store::Outcome::Success + }), + ) + .await + .unwrap(); + assert_eq!(drained, 1); + + let db2 = dever_runtime::tenant::scope(2, storage.database()) + .await + .unwrap(); + let rows = db2 + .query( + database::Sql { + sqlite: "SELECT tenant_id FROM _dever_jobs WHERE state='pending'", + postgres: "SELECT tenant_id FROM _dever_jobs WHERE state='pending'", + }, + vec![], + ) + .await + .unwrap(); + assert_eq!(rows[0].get(0).unwrap(), &Value::Int(2)); + db2.execute( + database::Sql { + sqlite: "UPDATE _dever_jobs SET tenant_id=1 WHERE state='pending'", + postgres: "UPDATE _dever_jobs SET tenant_id=1 WHERE state='pending'", + }, + vec![], + ) + .await + .unwrap(); + assert!( + store::claim(db2, &Clock::for_test(), 1000, 1) + .await + .unwrap_err() + .to_string() + .contains("tenant ownership") + ); + + let Err(unknown) = dever_runtime::tenant::scope(3, storage.database()).await else { + panic!("unknown tenant must not resolve a database"); + }; + assert!(unknown.to_string().contains("not provisioned")); + database::shutdown().await.unwrap(); + Ok(()) + }) + .unwrap(); +} diff --git a/test/dever-tests/tests/error_effects.rs b/test/dever-tests/tests/error_effects.rs new file mode 100644 index 0000000..4726a78 --- /dev/null +++ b/test/dever-tests/tests/error_effects.rs @@ -0,0 +1,176 @@ +mod support; + +use support::{checked, rejected, sources}; + +const ERRORS: &str = r#"public type ComputeResult { + Done(value: Int) + error Invalid(value: Int) +} +compute(value: 0) (answer: Int) { + fail(ComputeResult.Invalid(value)) +} +compute(value: other) (answer: Int) { + answer = value + 1 +} +render(result: ComputeResult.Done(value)) (text: Text) recover("render handled compute errors") { + text = int.to_text(value) +} +render(result: ComputeResult.Invalid(value)) (text: Text) recover("render handled compute errors") { + text = "invalid " + int.to_text(value) +} +"#; + +#[test] +fn fail_and_result_have_stable_syntax_and_formatting() { + let source = format!("{ERRORS}main() (text: Text) {{ text = render(result(compute(0))) }}"); + let mut map = sources(&source); + let formatted = dever_core::format::format(map.get(map.files()[0].id())).unwrap(); + map = sources(&formatted); + let reformatted = dever_core::format::format(map.get(map.files()[0].id())).unwrap(); + assert_eq!(formatted, reformatted); + assert!(formatted.contains("fail(ComputeResult.Invalid(value))")); + assert!(formatted.contains("result(compute(0))")); +} + +#[test] +fn error_sets_propagate_and_are_visible_in_the_public_api() { + let source = format!( + "{ERRORS}forward(value: Int) (answer: Int) {{ answer = compute(value) }}\npublic main() (answer: Int) {{ answer = forward(1) }}" + ); + let program = checked(&sources(&source)); + let snapshot = program.api_snapshot(); + assert!( + snapshot.contains("function main.main() (answer: Int [])") + && snapshot.contains("errors=[main.ComputeResult.Invalid]"), + "{snapshot}" + ); +} + +#[test] +fn result_capture_requires_matching_success_and_error_variants() { + checked(&sources(&format!( + "{ERRORS}main() (text: Text) {{ text = render(result(compute(0))) }}" + ))); + rejected( + &format!( + "{ERRORS}type Wrong {{ Done(value: Int)\nerror Other(value: Int) }}\nwrong(result: Wrong.Done(value)) () recover(\"handled\") {{}}\nwrong(result: Wrong.Other(value)) () recover(\"handled\") {{}}\nmain() () {{ wrong(result(compute(0))) }}" + ), + "result capture errors do not match the call", + ); + rejected( + &format!( + "{ERRORS}type Wrong {{ Done\nerror Invalid(value: Int) }}\nwrong(result: Wrong) () recover(\"handled\") {{}}\nmain() () {{ wrong(result(compute(0))) }}" + ), + "success payload must match", + ); +} + +#[test] +fn result_capture_does_not_introduce_a_concurrency_effect() { + checked(&sources(&format!( + "{ERRORS}main() (result: ComputeResult) pure {{ result = result(compute(0)) }}" + ))); +} + +#[test] +fn fail_must_be_an_error_variant_and_end_its_clause() { + rejected( + "type Outcome { Done\nerror Failed }\npublic main() () { fail(Outcome.Done) }", + "fail requires an error choice variant", + ); + rejected( + "type Outcome { error Failed }\npublic main() () { fail(Outcome.Failed)\ndever.io.println(\"unreachable\") }", + "fail must be the final statement", + ); +} + +#[test] +fn native_default_propagation_reaches_the_cli_boundary_once() { + let output = support::run( + "type Outcome { error Failed(value: Int) }\nstep() () { fail(Outcome.Failed(7)) }\npublic main() () { step() }", + ); + assert!(!output.status.success()); + assert!(output.stdout.is_empty()); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert_eq!(stderr.lines().count(), 1, "{stderr}"); + assert!(stderr.contains("main.Outcome.Failed(7)"), "{stderr}"); + assert!(stderr.contains("main.dever:3:"), "{stderr}"); +} + +#[test] +fn native_result_capture_converts_success_and_business_failure_to_choice_values() { + for (value, expected) in [("0", "invalid 0"), ("4", "5")] { + let output = support::stdout(&format!( + "{ERRORS}main() (text: Text) {{ text = render(result(compute({value}))) }}" + )); + assert_eq!(output, format!("text = {expected}\n")); + } +} + +#[test] +fn task_wait_preserves_business_error_identity() { + let output = support::run( + "type Outcome { error Failed(value: Int) }\nstep() () { fail(Outcome.Failed(9)) }\npublic main() () { task = run(step())\nwait(task) }", + ); + assert!(!output.status.success()); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!(stderr.contains("main.Outcome.Failed(9)"), "{stderr}"); + assert!(stderr.contains("main.dever:4:"), "{stderr}"); +} + +#[test] +fn capture_wraps_a_shared_error_choice_without_losing_its_identity() { + let source = r#"public type Failure { + error Missing(message: Text) + error Conflict(message: Text) +} +public type Outcome { + Done(value: Int) + error Failed(error: Failure) +} +load(conflict: true) (value: Int) { fail(Failure.Conflict("duplicate")) } +load(conflict: false) (value: Int) { fail(Failure.Missing("absent")) } +public main() (outcome: Outcome) { outcome = result(load(false)) } +"#; + assert_eq!( + support::stdout(source), + "outcome = main.Outcome.Failed(main.Failure.Missing(absent))\n" + ); + #[cfg(feature = "reference")] + { + let sources = sources(source); + let result = dever_core::reference::evaluate(&checked(&sources), "main.main").unwrap(); + assert!(result.fault.is_none(), "{:?}", result.fault); + assert_eq!( + result.stdout, + "outcome = main.Outcome.Failed(main.Failure.Missing(absent))\n" + ); + } + rejected( + &source.replace( + "error Failed(error: Failure)", + "error Failed(error: Failure)\nerror Again(error: Failure)", + ), + "overlapping error wrappers", + ); + rejected( + &source.replace( + "outcome = result(load(false))", + "outcome = result(load(false))\noutcome = Outcome.Done(0)", + ), + "failure is not handled", + ); + let wrapped = source.replace( + "public main() (outcome: Outcome) { outcome = result(load(false)) }", + r#" +rewrap(captured: Outcome.Done(number)) (answer: Int) { answer = number } +rewrap(captured: Outcome.Failed(error)) (answer: Int) { fail(Outcome.Failed(error)) } +load_again() (answer: Int) { answer = rewrap(result(load(false))) } +public main() (outcome: Outcome) { outcome = result(load_again()) } +"#, + ); + assert_eq!( + support::stdout(&wrapped), + "outcome = main.Outcome.Failed(main.Failure.Missing(absent))\n" + ); +} diff --git a/test/dever-tests/tests/external_command.rs b/test/dever-tests/tests/external_command.rs new file mode 100644 index 0000000..b2b8030 --- /dev/null +++ b/test/dever-tests/tests/external_command.rs @@ -0,0 +1,452 @@ +use std::fs; +use std::path::Path; +use std::time::Duration; + +use dever_runtime::bytes::Bytes; +use dever_runtime::component::{self, Definition, Reply}; +use dever_runtime::external::{self, Resource}; +use dever_runtime::wire::Encoder; +use serde_json::json; +use sha2::{Digest, Sha256}; + +#[path = "support/sandbox.rs"] +mod sandbox; +#[path = "support/temp.rs"] +mod temp; + +const PORT: &str = "tools.command"; + +struct Fixture { + directory: temp::TemporaryDirectory, + resources: Vec<Resource<'static>>, +} + +impl Fixture { + fn new(timeout_ms: u64, output_limit: usize) -> Self { + Self::with_program( + timeout_ms, + output_limit, + fs::read(env!("CARGO_BIN_EXE_command-fixture")).unwrap(), + ) + } + + fn with_program(timeout_ms: u64, output_limit: usize, program: Vec<u8>) -> Self { + let directory = temp::TemporaryDirectory::new(); + fs::create_dir(directory.path().join("markers")).unwrap(); + fs::create_dir(directory.path().join("config")).unwrap(); + let settings = json!({"adapter": {PORT: { + "files": {"markers": {"path":"markers", "write":true}}, + "command": {"timeout_ms":timeout_ms, "output_limit":output_limit} + }}}); + fs::write( + directory.path().join("config/setting.json"), + settings.to_string(), + ) + .unwrap(); + let workspace = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let mut assets = sandbox::assets(&workspace); + assets.push(sandbox::Asset { + path: "commands/fixture/tool".into(), + bytes: program, + executable: true, + }); + assets.push(sandbox::Asset { + path: "tool.dever-command.json".into(), + bytes: json!({"format":"dever-command-launch-v1", "ecosystem":"command", "entry":"tool", + "executable":"commands/fixture/tool", "arguments":[], "working_directory":"commands/fixture"}) + .to_string().into_bytes(), + executable: false, + }); + let resources = assets + .into_iter() + .map(|asset| Resource { + path: Box::leak(asset.path.into_boxed_str()), + sha256: Box::leak(format!("{:x}", Sha256::digest(&asset.bytes)).into_boxed_str()), + bytes: Box::leak(asset.bytes.into_boxed_slice()), + executable: asset.executable, + }) + .collect(); + Self { + directory, + resources, + } + } + + async fn start(&self) -> Result<(), String> { + external::prepare( + self.directory.path(), + &external::bundle_digest(&self.resources), + &self.resources, + )?; + component::start(Definition { + key: PORT.into(), + ecosystem: "command".into(), + entry: "tool".into(), + port: PORT.into(), + adapter: "default".into(), + schema: "command-fixture-v1".into(), + capabilities: vec!["file".into(), "process".into()], + operations: vec!["execute".into()], + setting: None, + timeout_ms: 30_000, + }) + .await + } + + fn marker(&self, name: &str) -> std::path::PathBuf { + self.directory.path().join("markers").join(name) + } +} + +struct Output { + code: i64, + stdout: Vec<u8>, + stderr: Vec<u8>, +} + +async fn invoke(args: &[&str], stdin: Option<&[u8]>) -> Result<Output, String> { + let mut writer = Encoder::default(); + writer.begin_object()?; + writer.key("args")?; + writer.begin_array()?; + for arg in args { + writer.text(arg)?; + } + writer.end()?; + if let Some(stdin) = stdin { + writer.key("stdin")?; + writer.bytes(&Bytes::new(stdin.to_vec()))?; + } + writer.end()?; + let reply = component::call(PORT, "execute", &writer.finish()?).await?; + let Reply::Result(payload) = reply else { + return Err("unexpected business error".into()); + }; + let node = dever_runtime::wire::parse(&payload)?; + let output = node + .object()? + .get("output") + .ok_or("missing output")? + .object()?; + Ok(Output { + code: output.get("code").ok_or("missing code")?.int()?, + stdout: output + .get("stdout") + .ok_or("missing stdout")? + .bytes()? + .values() + .to_vec(), + stderr: output + .get("stderr") + .ok_or("missing stderr")? + .bytes()? + .values() + .to_vec(), + }) +} + +#[test] +fn ordinary_commands_preserve_argv_binary_io_and_nonzero_status() { + let fixture = Fixture::new(5000, 1024 * 1024); + dever_runtime::task::run_entry(async { + fixture.start().await?; + assert!( + !fixture.marker("starts").exists(), + "startup must not execute the tool" + ); + let output = invoke( + &[ + "echo", + "a b", + "\"quoted\"", + "$(literal);*", + "/data/markers", + "", + ], + None, + ) + .await?; + assert_eq!( + output.stdout, + b"a b\0\"quoted\"\0$(literal);*\0/data/markers\0\0" + ); + assert_eq!(output.code, 0); + let binary: Vec<u8> = (0..=255).cycle().take(131_072).collect(); + let output = invoke(&["stdin"], Some(&binary)).await?; + assert_eq!(output.stdout, binary); + assert_eq!(output.stderr, [0, 255, 128, 10]); + assert!( + invoke(&["stdin"], None).await?.stdout.is_empty(), + "omitted stdin must send EOF" + ); + let output = invoke(&["duplex", "262144"], Some(&binary)).await?; + assert_eq!(output.stdout, vec![254; 262_144]); + assert_eq!(output.stderr, vec![253; 262_144]); + assert_eq!(invoke(&["exit", "37"], None).await?.code, 37); + assert_eq!(invoke(&["exit", "1"], None).await?.code, 1); + assert_eq!(invoke(&["exit", "127"], None).await?.code, 127); + assert!( + invoke(&["fds"], None).await?.stdout.is_empty(), + "observation descriptors leaked into the command" + ); + // Early stdin close must not replace the program's ordinary result. + assert_eq!(invoke(&["exit", "13"], Some(&binary)).await?.code, 13); + let nul = invoke(&["echo", "bad\0arg"], None) + .await + .err() + .ok_or("NUL accepted")?; + assert!(nul.contains("NUL")); + let huge = "a".repeat(65_536); + assert!(invoke(&["echo", &huge], None).await.is_err()); + assert!( + invoke(&["stdin"], Some(&vec![0; 8 * 1024 * 1024 + 1])) + .await + .err() + .is_some_and(|error| error.contains("stdin exceeds")) + ); + for _ in 0..5 { + assert_eq!(invoke(&["exit", "0"], None).await?.code, 0); + } + component::shutdown().await + }) + .unwrap(); +} + +#[test] +fn combined_output_limit_closes_process_then_allows_the_next_call() { + let fixture = Fixture::new(5000, 262_144); + dever_runtime::task::run_entry(async { + fixture.start().await?; + let error = invoke(&["duplex", "200000"], Some(&vec![7; 131_072])) + .await + .err() + .ok_or("limit accepted")?; + assert!(error.contains("output_limit"), "{error}"); + assert_eq!(invoke(&["exit", "0"], None).await?.code, 0); + component::shutdown().await + }) + .unwrap(); +} + +async fn heartbeat(fixture: &Fixture) -> Result<(), String> { + // Verified-resource preparation precedes process startup; debug hashing is + // not the command deadline and may dominate on a busy author machine. + tokio::time::timeout(Duration::from_secs(15), async { + loop { + if fs::read_to_string(fixture.marker("heartbeat")) + .ok() + .and_then(|text| text.parse::<u32>().ok()) + .is_some_and(|count| count >= 2) + { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .map_err(|_| { + format!( + "descendant heartbeat did not start; starts={:?}; ready={}; error={:?}", + fs::read_to_string(fixture.marker("starts")), + fixture.marker("ready").exists(), + fs::read_to_string(fixture.marker("fixture-error")) + ) + }) +} + +#[test] +fn guard_exec_failure_is_a_fault_not_a_program_exit_code() { + let fixture = Fixture::with_program(5000, 1024 * 1024, b"not an ELF executable\n".to_vec()); + dever_runtime::task::run_entry(async { + fixture.start().await?; + let error = invoke(&[], None) + .await + .err() + .ok_or("exec failure was returned as an exit code")?; + assert!( + error.contains("startup failed") && error.contains("cannot execute"), + "{error}" + ); + component::shutdown().await + }) + .unwrap(); +} + +async fn assert_reaped(fixture: &Fixture) { + let lifetime = fs::OpenOptions::new() + .write(true) + .open(fixture.marker("lifetime.lock")) + .unwrap(); + lifetime + .try_lock() + .expect("descendant still holds its lifetime lock after cleanup returned"); + let before = fs::read(fixture.marker("heartbeat")).unwrap(); + tokio::time::sleep(Duration::from_millis(100)).await; + assert_eq!( + fs::read(fixture.marker("heartbeat")).unwrap(), + before, + "descendant survived cleanup" + ); + assert_eq!( + fs::read_to_string(fixture.marker("starts")).unwrap(), + "started\n", + "command was replayed" + ); +} + +#[test] +fn timeout_reaps_descendants_and_keeps_the_adapter_available() { + let fixture = Fixture::new(800, 1024 * 1024); + dever_runtime::task::run_entry(async { + fixture.start().await?; + let call = dever_runtime::task::run(async { + invoke(&["tree", "/data/markers"], None).await.map(|_| ()) + }) + .await?; + heartbeat(&fixture).await?; + let error = dever_runtime::task::wait(call) + .await + .expect_err("command did not time out"); + assert!(error.contains("timed out"), "{error}"); + assert_reaped(&fixture).await; + assert_eq!(invoke(&["exit", "0"], None).await?.code, 0); + component::shutdown().await + }) + .unwrap(); +} + +#[test] +fn cancellation_and_root_close_reap_descendants_without_replay() { + for root_close in [false, true] { + let fixture = Fixture::new(5000, 1024 * 1024); + dever_runtime::task::run_entry(async { + fixture.start().await?; + let call = dever_runtime::task::run(async { + invoke(&["tree", "/data/markers"], None).await.map(|_| ()) + }) + .await?; + heartbeat(&fixture).await?; + if root_close { + // Return while the call is active: the invocation closes its + // component owner before draining the structured child task. + return Ok::<(), String>(()); + } + dever_runtime::task::stop(call).await?; + assert_reaped(&fixture).await; + assert_eq!(invoke(&["exit", "0"], None).await?.code, 0); + component::shutdown().await + }) + .unwrap_or_else(|error| { + assert!(root_close && error.contains("stopped"), "{error}"); + }); + let before = fs::read(fixture.marker("heartbeat")).unwrap(); + let lifetime = fs::OpenOptions::new() + .write(true) + .open(fixture.marker("lifetime.lock")) + .unwrap(); + lifetime + .try_lock() + .expect("root close returned before descendant cleanup"); + std::thread::sleep(Duration::from_millis(100)); + assert_eq!(fs::read(fixture.marker("heartbeat")).unwrap(), before); + assert_eq!( + fs::read_to_string(fixture.marker("starts")).unwrap(), + "started\n" + ); + } +} + +#[test] +fn queued_cancellation_does_not_wait_for_an_unrelated_running_command() { + let fixture = Fixture::new(5000, 1024 * 1024); + dever_runtime::task::run_entry(async { + fixture.start().await?; + let active = dever_runtime::task::run(async { + invoke(&["tree", "/data/markers"], None).await.map(|_| ()) + }) + .await?; + heartbeat(&fixture).await?; + let (entered, admitted) = tokio::sync::oneshot::channel(); + let queued = dever_runtime::task::run(async move { + let _ = entered.send(()); + invoke(&["echo", "queued"], None).await.map(|_| ()) + }) + .await?; + admitted.await.map_err(|error| error.to_string())?; + tokio::time::sleep(Duration::from_millis(20)).await; + tokio::time::timeout( + Duration::from_millis(500), + dever_runtime::task::stop(queued), + ) + .await + .map_err(|_| "queued cancellation waited for the active command")??; + let before = fs::read(fixture.marker("heartbeat")).unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + assert_ne!(fs::read(fixture.marker("heartbeat")).unwrap(), before); + dever_runtime::task::stop(active).await?; + assert_reaped(&fixture).await; + component::shutdown().await + }) + .unwrap(); +} + +#[test] +fn task_deadline_waits_for_command_process_cleanup() { + let fixture = Fixture::new(5000, 1024 * 1024); + dever_runtime::task::run_entry(async { + fixture.start().await?; + let call = dever_runtime::task::run(async { + invoke(&["tree", "/data/markers"], None).await.map(|_| ()) + }) + .await?; + heartbeat(&fixture).await?; + assert_eq!(dever_runtime::task::wait_timeout(call, 10).await?, None); + assert_reaped(&fixture).await; + component::shutdown().await + }) + .unwrap(); +} + +#[test] +fn explicit_shutdown_drains_active_and_queued_commands() { + let fixture = Fixture::new(5000, 1024 * 1024); + dever_runtime::task::run_entry(async { + fixture.start().await?; + let active = dever_runtime::task::run(async { + invoke(&["tree", "/data/markers"], None).await.map(|_| ()) + }) + .await?; + heartbeat(&fixture).await?; + let queued = + dever_runtime::task::run(async { invoke(&["echo", "queued"], None).await.map(|_| ()) }) + .await?; + component::shutdown().await?; + assert_reaped(&fixture).await; + assert!(dever_runtime::task::wait(active).await.is_err()); + assert!(dever_runtime::task::wait(queued).await.is_err()); + Ok::<(), String>(()) + }) + .unwrap(); +} + +#[test] +fn command_settings_validate_limits_without_affecting_worker_selection() { + let directory = temp::TemporaryDirectory::new(); + fs::create_dir(directory.path().join("config")).unwrap(); + for command in [ + json!({"timeout_ms":0}), + json!({"timeout_ms":3_600_001}), + json!({"output_limit":0}), + json!({"output_limit":8_388_609}), + json!({"unknown":1}), + ] { + fs::write( + directory.path().join("config/setting.json"), + json!({"adapter":{PORT:{"command":command}}}).to_string(), + ) + .unwrap(); + let settings = dever_runtime::config::Settings::load_project(directory.path()).unwrap(); + assert!(settings.command_limits(PORT).is_err()); + assert!(settings.select_adapter(PORT, &["default"]).is_ok()); + } +} diff --git a/test/dever-tests/tests/external_component.rs b/test/dever-tests/tests/external_component.rs new file mode 100644 index 0000000..800313d --- /dev/null +++ b/test/dever-tests/tests/external_component.rs @@ -0,0 +1,326 @@ +use std::fs; +use std::path::PathBuf; + +use dever_runtime::component::{self, Definition, Reply}; +use dever_runtime::wire::{Encoded, Encoder}; +use sha2::{Digest, Sha256}; + +#[path = "support/sandbox.rs"] +mod sandbox; +#[path = "support/temp.rs"] +mod temp; + +const OPERATIONS: &[&str] = &[ + "echo", + "fail", + "wait", + "timeout", + "crash", + "duplicate", + "out_of_order", + "unknown", + "oversize", + "truncated", +]; + +#[test] +fn component_protocol_and_lifecycle_are_bounded() { + let fixture = install_fixture(); + let result = dever_runtime::task::run_entry(async { + fixture.prepare()?; + // One invocation keeps the same deployment grants even if settings are + // replaced between Worker startup, cancellation and restart. + fs::write( + fixture.directory.path().join("config/setting.json"), + "invalid", + ) + .map_err(|error| error.to_string())?; + normal_lifecycle(fixture.entry) + .await + .map_err(|error| format!("normal lifecycle: {error}"))?; + shutdown_with_queued_calls_is_bounded(fixture.entry) + .await + .map_err(|error| format!("queued shutdown: {error}"))?; + rejected_handshakes(fixture.entry) + .await + .map_err(|error| format!("handshake rejection: {error}"))?; + malformed_responses(fixture.entry) + .await + .map_err(|error| format!("malformed response: {error}"))?; + cancelled_lifecycle(&fixture) + .await + .map_err(|error| format!("cancelled lifecycle: {error}")) + }); + result.expect("external component protocol fixture"); + assert!(fixture.path.with_extension("shutdown").is_file()); + assert!(dever_runtime::task::run_entry(async { fixture.prepare() }).is_err()); +} + +async fn cancelled_lifecycle(fixture: &InstalledFixture) -> Result<(), String> { + let entry = fixture.entry; + let start = dever_runtime::task::run(async move { + component::start(definition("cancel_start", entry, "delayed_start", 2_000)).await + }) + .await?; + wait_for_marker(&fixture.path.with_extension("starting")).await?; + dever_runtime::task::stop(start).await?; + component::shutdown().await?; + + component::start(definition( + "cancel_shutdown", + entry, + "delayed_shutdown", + 2_000, + )) + .await?; + let stopping = dever_runtime::task::run(component::shutdown()).await?; + wait_for_marker(&fixture.path.with_extension("stopping")).await?; + dever_runtime::task::stop(stopping).await?; + // The cancelled caller must leave ownership available for a complete second drain. + component::shutdown().await?; + Ok(()) +} + +async fn wait_for_marker(path: &std::path::Path) -> Result<(), String> { + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while !path.is_file() { + tokio::time::sleep(std::time::Duration::from_millis(2)).await; + } + }) + .await + .map_err(|_| "fixture did not reach its lifecycle checkpoint".to_owned()) +} + +async fn shutdown_with_queued_calls_is_bounded(entry: &'static str) -> Result<(), String> { + component::start(definition("queued_shutdown", entry, "normal", 5_000)).await?; + let mut calls = Vec::new(); + for _ in 0..80 { + calls.push( + dever_runtime::task::run(async { + component::call("queued_shutdown", "wait", &payload(0)) + .await + .map(|_| ()) + }) + .await?, + ); + } + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + tokio::time::timeout(std::time::Duration::from_secs(1), component::shutdown()) + .await + .map_err(|_| "external Adapter shutdown did not stop queued calls".to_owned())??; + for call in calls { + let error = dever_runtime::task::wait(call) + .await + .expect_err("shutdown must stop every pending call"); + assert!( + error.contains("stopped"), + "unexpected shutdown error: {error}" + ); + } + Ok(()) +} + +async fn normal_lifecycle(entry: &'static str) -> Result<(), String> { + // A short-lived caller starts the shared Worker; ending its Scope must not stop it. + let started = dever_runtime::task::run(async move { + component::start(definition("normal", entry, "normal", 50)).await + }) + .await?; + dever_runtime::task::wait(started) + .await + .map_err(|error| format!("start: {error}"))?; + + for value in [1_i64, 2] { + match component::call("normal", "echo", &payload(value)) + .await + .map_err(|error| format!("echo {value}: {error}"))? + { + Reply::Result(body) => assert_eq!(body, format!(r#"{{"value":{value}}}"#)), + Reply::Error { .. } => return Err("echo returned a business error".into()), + } + } + match component::call("normal", "fail", &payload(0)) + .await + .map_err(|error| format!("business error: {error}"))? + { + Reply::Error { identity, payload } => { + assert_eq!(identity, "notification.delivery.Rejected"); + assert_eq!(payload, r#"{"reason":"fixture"}"#); + } + Reply::Result(_) => return Err("fail returned a result".into()), + } + + let wait = dever_runtime::task::run(async { + component::call("normal", "wait", &payload(0)) + .await + .map(|_| ()) + }) + .await?; + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + dever_runtime::task::stop(wait) + .await + .map_err(|error| format!("cancel wait: {error}"))?; + + let timeout = component::call("normal", "timeout", &payload(0)) + .await + .expect_err("timeout operation must reach the deadline"); + assert_eq!(timeout, "external Adapter call timed out"); + assert!(matches!( + component::call("normal", "echo", &payload(3)) + .await + .map_err(|error| format!("echo after timeout: {error}"))?, + Reply::Result(_) + )); + + let crash = component::call("normal", "crash", &payload(0)) + .await + .expect_err("crashed worker must fail its pending call"); + assert!( + crash.contains("early eof"), + "unexpected crash fault: {crash}" + ); + assert!(matches!( + component::call("normal", "echo", &payload(4)) + .await + .map_err(|error| format!("echo after crash: {error}"))?, + Reply::Result(_) + )); + component::shutdown() + .await + .map_err(|error| format!("shutdown: {error}"))?; + Ok(()) +} + +async fn rejected_handshakes(entry: &'static str) -> Result<(), String> { + for (key, mode, expected) in [ + ("duplicate_ready", "duplicate_ready", "duplicate JSON field"), + ( + "capability_mismatch", + "capability_mismatch", + "handshake does not match", + ), + ] { + let error = component::start(definition(key, entry, mode, 100)) + .await + .expect_err("invalid handshake must be rejected"); + assert!( + error.contains(expected), + "unexpected handshake error: {error}" + ); + } + Ok(()) +} + +async fn malformed_responses(entry: &'static str) -> Result<(), String> { + for operation in [ + "duplicate", + "out_of_order", + "unknown", + "oversize", + "truncated", + ] { + component::start(definition(operation, entry, "normal", 100)).await?; + let error = component::call(operation, operation, &payload(0)) + .await + .expect_err("invalid response must fail its pending call"); + let expected = match operation { + "duplicate" => "duplicate JSON field", + "out_of_order" => "out-of-order request id", + "unknown" => "unknown message kind", + "oversize" => "frame exceeds byte limit", + "truncated" => "cannot read complete component protocol frame", + _ => unreachable!(), + }; + assert!( + error.contains(expected), + "unexpected {operation} call fault: {error}" + ); + let cleanup = component::shutdown() + .await + .expect_err("protocol fault must survive cleanup"); + assert!( + cleanup.contains(expected), + "unexpected {operation} fault: {cleanup}" + ); + } + Ok(()) +} + +fn definition( + key: &'static str, + entry: &'static str, + mode: &'static str, + timeout_ms: u64, +) -> Definition { + Definition { + ecosystem: "exec".into(), + key: key.into(), + entry: entry.into(), + port: "notification.delivery".into(), + adapter: "notification.delivery.fixture".into(), + schema: "a19ca3e615ce5e71".into(), + capabilities: vec!["network".into(), "file".into()], + operations: OPERATIONS.iter().map(|name| (*name).into()).collect(), + setting: Some(format!( + r#"{{"mode":"{mode}","marker":"/data/lifecycle/worker"}}"# + )), + timeout_ms, + } +} + +fn payload(value: i64) -> Encoded { + let mut writer = Encoder::default(); + writer.begin_object().expect("payload object"); + writer.key("value").expect("payload field"); + writer.int(value).expect("payload value"); + writer.end().expect("payload end"); + writer.finish().expect("payload JSON") +} + +struct InstalledFixture { + directory: temp::TemporaryDirectory, + path: PathBuf, + entry: &'static str, + resources: Vec<dever_runtime::external::Resource<'static>>, +} + +impl InstalledFixture { + fn prepare(&self) -> Result<(), String> { + let digest = dever_runtime::external::bundle_digest(&self.resources); + dever_runtime::external::prepare(self.directory.path(), &digest, &self.resources) + } +} + +fn install_fixture() -> InstalledFixture { + let directory = temp::TemporaryDirectory::new(); + fs::create_dir(directory.path().join("lifecycle")).unwrap(); + fs::create_dir(directory.path().join("config")).unwrap(); + fs::write(directory.path().join("config/setting.json"), r#"{"adapter":{"notification.delivery":{"files":{"lifecycle":{"path":"lifecycle","write":true}}}}}"#).unwrap(); + let workspace = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let mut assets = sandbox::assets(&workspace); + assets.push(sandbox::Asset { + path: "workers/fixture/worker".into(), + bytes: fs::read(env!("CARGO_BIN_EXE_component-fixture")).unwrap(), + executable: true, + }); + assets.push(sandbox::Asset { + path: "worker.dever-worker.json".into(), + bytes: br#"{"format":"dever-worker-launch-v1","ecosystem":"exec","entry":"worker","executable":"workers/fixture/worker","arguments":[],"working_directory":"workers/fixture"}"#.to_vec(), executable: false, + }); + let resources = assets + .into_iter() + .map(|asset| dever_runtime::external::Resource { + path: Box::leak(asset.path.into_boxed_str()), + sha256: Box::leak(format!("{:x}", Sha256::digest(&asset.bytes)).into_boxed_str()), + bytes: Box::leak(asset.bytes.into_boxed_slice()), + executable: asset.executable, + }) + .collect(); + let path = directory.path().join("lifecycle/worker"); + InstalledFixture { + directory, + path, + entry: "worker", + resources, + } +} diff --git a/test/dever-tests/tests/external_resources.rs b/test/dever-tests/tests/external_resources.rs new file mode 100644 index 0000000..bfc5980 --- /dev/null +++ b/test/dever-tests/tests/external_resources.rs @@ -0,0 +1,494 @@ +#[path = "support/sandbox.rs"] +mod sandbox; +#[path = "support/temp.rs"] +mod temp; + +use std::fs; + +use dever_runtime::external::{ + Resource, bundle_digest, extract, verified_entry, verified_worker_launch, +}; +use sha2::{Digest, Sha256}; +use temp::TemporaryDirectory; + +fn digest(bytes: &[u8]) -> String { + Sha256::digest(bytes) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +#[test] +fn entry_resolution_is_bound_to_the_manifest_and_rechecks_bytes() { + let root = TemporaryDirectory::new(); + let bytes = b"verified worker"; + let hash = digest(bytes); + let resources = [Resource { + path: "module/mail/delivery/worker", + bytes, + sha256: &hash, + executable: true, + }]; + let bundle = bundle_digest(&resources); + let directory = extract(root.path(), &bundle, &resources).unwrap(); + let entry = verified_entry(&directory, &resources, resources[0].path).unwrap(); + assert!(entry.starts_with(&directory)); + assert!( + verified_entry(&directory, &resources, "other-worker") + .unwrap_err() + .contains("manifest") + ); + fs::write(&entry, b"changed worker").unwrap(); + assert!( + verified_entry(&directory, &resources, resources[0].path) + .unwrap_err() + .contains("integrity") + ); +} + +#[test] +fn large_resource_verification_checks_every_chunk_and_exact_length() { + let root = TemporaryDirectory::new(); + let bytes = vec![157; 262_147]; + let hash = digest(&bytes); + let resources = [Resource { + path: "runtime.bin", + bytes: &bytes, + sha256: &hash, + executable: true, + }]; + let directory = extract(root.path(), &bundle_digest(&resources), &resources).unwrap(); + let path = verified_entry(&directory, &resources, "runtime.bin").unwrap(); + let mut tampered = bytes.clone(); + tampered[128 * 1024] ^= 1; + for altered in [&tampered[..], &bytes[..bytes.len() - 1]] { + fs::write(&path, altered).unwrap(); + assert!( + verified_entry(&directory, &resources, "runtime.bin") + .unwrap_err() + .contains("integrity") + ); + } + let mut appended = bytes.clone(); + appended.push(1); + fs::write(&path, &appended).unwrap(); + assert!( + verified_entry(&directory, &resources, "runtime.bin") + .unwrap_err() + .contains("integrity") + ); + fs::write(&path, &bytes).unwrap(); + assert_eq!( + verified_entry(&directory, &resources, "runtime.bin").unwrap(), + path + ); +} + +#[test] +fn managed_launch_is_bound_to_its_checked_entry_and_private_execution_tree() { + let root = TemporaryDirectory::new(); + let entry = "module/mail/delivery/adapter/worker.py"; + let manifest_path = format!("{entry}.dever-worker.json"); + let manifest = serde_json::to_vec(&serde_json::json!({ + "format": "dever-worker-launch-v1", "ecosystem": "pip", "entry": entry, + "executable": "workers/fixture/runtime/bin/python3", + "arguments": [{"literal":"-I"},{"resource":"workers/fixture/runner.py"}], + "working_directory": "workers/fixture" + })) + .unwrap(); + let interpreter = b"managed interpreter bytes"; + let runner = b"generated checked runner"; + let hashes = [digest(&manifest), digest(interpreter), digest(runner)]; + let resources = [ + Resource { + path: &manifest_path, + bytes: &manifest, + sha256: &hashes[0], + executable: false, + }, + Resource { + path: "workers/fixture/runtime/bin/python3", + bytes: interpreter, + sha256: &hashes[1], + executable: true, + }, + Resource { + path: "workers/fixture/runner.py", + bytes: runner, + sha256: &hashes[2], + executable: false, + }, + ]; + let directory = extract(root.path(), &bundle_digest(&resources), &resources).unwrap(); + let launch = verified_worker_launch(&directory, &resources, entry, "pip").unwrap(); + assert_eq!(launch.executable, directory.join(resources[1].path)); + assert_eq!( + launch.arguments, + [ + std::ffi::OsString::from("-I"), + directory.join(resources[2].path).into_os_string() + ] + ); + assert_eq!(launch.working_directory, directory.join("workers/fixture")); + assert!( + verified_worker_launch(&directory, &resources, entry, "npm") + .unwrap_err() + .contains("identity") + ); + assert!( + verified_worker_launch(&directory, &resources, "other.py", "pip") + .unwrap_err() + .contains("missing") + ); + fs::write(&launch.executable, b"changed interpreter").unwrap(); + assert!( + verified_worker_launch(&directory, &resources, entry, "pip") + .unwrap_err() + .contains("integrity") + ); +} + +#[test] +fn managed_launch_rejects_cross_worker_paths_and_unbounded_arguments() { + let entry = "module/mail/delivery/adapter/worker.mjs"; + let manifest_path = format!("{entry}.dever-worker.json"); + let base = serde_json::json!({ + "format": "dever-worker-launch-v1", "ecosystem": "npm", "entry": entry, + "executable": "workers/fixture/node", + "arguments": [{"resource":"workers/fixture/runner.mjs"}], + "working_directory": "workers/fixture" + }); + let mut cases = Vec::new(); + for (field, value) in [ + ("working_directory", serde_json::json!("workers/../outside")), + ("executable", serde_json::json!("workers/neighbor/node")), + ( + "arguments", + serde_json::json!([{"resource":"workers/neighbor/runner.mjs"}]), + ), + ( + "arguments", + serde_json::json!([{"literal":"bad\u{0000}argument"}]), + ), + ( + "arguments", + serde_json::json!([{"resource":"/usr/bin/node"}]), + ), + ( + "arguments", + serde_json::json!([{"resource":"workers/fixture/runner.mjs","literal":"hidden"}]), + ), + ( + "arguments", + serde_json::json!( + (0..65) + .map(|_| serde_json::json!({"literal":"a"})) + .collect::<Vec<_>>() + ), + ), + ( + "arguments", + serde_json::json!([{"literal":"a".repeat(65_537)}]), + ), + ] { + let mut case = base.clone(); + case[field] = value; + cases.push(case); + } + for case in cases { + let root = TemporaryDirectory::new(); + let manifest = serde_json::to_vec(&case).unwrap(); + let hash = digest(&manifest); + let bytes = b"fixture"; + let bytes_hash = digest(bytes); + let resources = [ + Resource { + path: &manifest_path, + bytes: &manifest, + sha256: &hash, + executable: false, + }, + Resource { + path: "workers/fixture/node", + bytes, + sha256: &bytes_hash, + executable: true, + }, + Resource { + path: "workers/fixture/runner.mjs", + bytes, + sha256: &bytes_hash, + executable: false, + }, + Resource { + path: "workers/neighbor/node", + bytes, + sha256: &bytes_hash, + executable: true, + }, + Resource { + path: "workers/neighbor/runner.mjs", + bytes, + sha256: &bytes_hash, + executable: false, + }, + ]; + let directory = extract(root.path(), &bundle_digest(&resources), &resources).unwrap(); + assert!( + verified_worker_launch(&directory, &resources, entry, "npm").is_err(), + "accepted {case}" + ); + } +} + +#[test] +fn manifest_identity_includes_execute_permission_and_rejects_duplicates() { + let root = TemporaryDirectory::new(); + let bytes = b"worker"; + let hash = digest(bytes); + let mut resources = [Resource { + path: "worker", + bytes, + sha256: &hash, + executable: false, + }]; + let original = bundle_digest(&resources); + let directory = extract(root.path(), &original, &resources).unwrap(); + assert!( + verified_entry(&directory, &resources, "worker") + .unwrap_err() + .contains("not executable") + ); + resources[0].executable = true; + assert_ne!(original, bundle_digest(&resources)); + assert!( + extract(root.path(), &original, &resources) + .unwrap_err() + .contains("bundle identity") + ); + let duplicate = [resources[0], resources[0]]; + assert!( + extract(root.path(), &bundle_digest(&duplicate), &duplicate) + .unwrap_err() + .contains("duplicate") + ); +} + +#[test] +fn concurrent_first_extraction_publishes_one_complete_manifest() { + let root = TemporaryDirectory::new(); + let bytes = b"worker"; + let hash = digest(bytes); + let resources = [Resource { + path: "worker", + bytes, + sha256: &hash, + executable: true, + }]; + let bundle = bundle_digest(&resources); + let barrier = std::sync::Barrier::new(4); + std::thread::scope(|scope| { + let mut workers = Vec::new(); + for _ in 0..4 { + let root = root.path(); + let resources = &resources; + let bundle = &bundle; + let barrier = &barrier; + workers.push(scope.spawn(move || { + barrier.wait(); + let directory = extract(root, bundle, resources).unwrap(); + verified_entry(&directory, resources, "worker").unwrap() + })); + } + let paths = workers + .into_iter() + .map(|worker| worker.join().unwrap()) + .collect::<Vec<_>>(); + assert!(paths.iter().all(|path| path == &paths[0])); + }); + assert_eq!( + fs::read_dir(root.path().join("data/cache/lib")) + .unwrap() + .count(), + 1 + ); +} + +#[cfg(unix)] +#[test] +fn extraction_and_launch_reject_symlinks_in_every_path_segment() { + use std::os::unix::fs::symlink; + let root = TemporaryDirectory::new(); + let outside = TemporaryDirectory::new(); + let bytes = b"worker"; + let hash = digest(bytes); + let resources = [Resource { + path: "nested/worker", + bytes, + sha256: &hash, + executable: true, + }]; + let bundle = bundle_digest(&resources); + symlink(outside.path(), root.path().join("data")).unwrap(); + assert!( + extract(root.path(), &bundle, &resources) + .unwrap_err() + .contains("symbolic link") + ); + fs::remove_file(root.path().join("data")).unwrap(); + let directory = extract(root.path(), &bundle, &resources).unwrap(); + fs::rename(directory.join("nested"), outside.path().join("nested")).unwrap(); + symlink(outside.path().join("nested"), directory.join("nested")).unwrap(); + assert!( + verified_entry(&directory, &resources, "nested/worker") + .unwrap_err() + .contains("symbolic link") + ); +} + +#[cfg(unix)] +#[test] +fn existing_cache_permissions_are_checked_before_reuse_and_launch() { + use std::os::unix::fs::PermissionsExt; + let root = TemporaryDirectory::new(); + let bytes = b"worker"; + let hash = digest(bytes); + let resources = [Resource { + path: "worker", + bytes, + sha256: &hash, + executable: true, + }]; + let bundle = bundle_digest(&resources); + fs::create_dir(root.path().join("data")).unwrap(); + fs::set_permissions(root.path().join("data"), fs::Permissions::from_mode(0o777)).unwrap(); + assert!( + extract(root.path(), &bundle, &resources) + .unwrap_err() + .contains("permissions") + ); + fs::set_permissions(root.path().join("data"), fs::Permissions::from_mode(0o700)).unwrap(); + let directory = extract(root.path(), &bundle, &resources).unwrap(); + fs::set_permissions(&directory, fs::Permissions::from_mode(0o777)).unwrap(); + assert!( + extract(root.path(), &bundle, &resources) + .unwrap_err() + .contains("permissions") + ); + fs::set_permissions(&directory, fs::Permissions::from_mode(0o700)).unwrap(); + fs::set_permissions(directory.join("worker"), fs::Permissions::from_mode(0o770)).unwrap(); + assert!( + verified_entry(&directory, &resources, "worker") + .unwrap_err() + .contains("permissions") + ); +} + +#[test] +#[ignore = "requires explicitly prepared sandbox assets and the private Rust bootstrap compiler"] +fn packaged_exec_runs_without_source_worker_or_inherited_environment() { + let root = TemporaryDirectory::new(); + let mut sources = dever_core::source::SourceMap::default(); + sources.add( + "notification/mail/port.dever", + "read() (value: Int) fails app.ReadResult\n", + ); + sources.add("notification/mail/app.dever", "type ReadResult { error Unavailable(message: Text) }\nread() (value: Int) { value = port.read() }\n"); + sources.add( + "notification/mail/adapter.dever", + "setting {\n mode: Text\n marker: Text\n}\nexternal exec \"worker\" { allow file }\n", + ); + sources.add("notification/mail/api.dever", "cmd read = app.read\n"); + let program = dever_core::check(&sources).unwrap_or_else(|errors| panic!("{errors:?}")); + let worker = fs::read(env!("CARGO_BIN_EXE_component-fixture")).unwrap(); + let entry = program.external_worker_entries().pop().unwrap(); + let resource = dever_core::native::EmbeddedResource { + path: "workers/fixture/worker".into(), + sha256: digest(&worker), + bytes: worker, + executable: true, + }; + let mut copy = resource.clone(); + copy.path = "assets/worker-copy.bin".into(); + copy.executable = false; + let mut resources = vec![resource, copy]; + let manifest = serde_json::to_vec(&serde_json::json!({ + "format":"dever-worker-launch-v1", "ecosystem":"exec", "entry":entry, + "executable":"workers/fixture/worker", "arguments":[], + "working_directory":"workers/fixture" + })) + .unwrap(); + resources.push(dever_core::native::EmbeddedResource { + path: format!("{entry}.dever-worker.json"), + sha256: digest(&manifest), + bytes: manifest, + executable: false, + }); + for asset in sandbox::assets(&std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../..")) { + resources.push(dever_core::native::EmbeddedResource { + path: asset.path, + sha256: digest(&asset.bytes), + bytes: asset.bytes, + executable: asset.executable, + }); + } + let native = dever_core::native::compile_project_with_resources( + &program, + &sources, + std::ffi::OsStr::new("rustc"), + dever_runtime::config::RuntimeProfile::default(), + &resources, + ) + .unwrap(); + let executable = root.path().join("program"); + native.save(&executable).unwrap(); + fs::create_dir(root.path().join("config")).unwrap(); + fs::create_dir(root.path().join("lifecycle")).unwrap(); + fs::write( + root.path().join("config/setting.json"), + r#"{"adapter":{"notification.mail":{"use":"default","setting":{"mode":"normal","marker":"/data/lifecycle/worker"},"files":{"lifecycle":{"path":"lifecycle","write":true}}}}}"#, + ) + .unwrap(); + assert!(!root.path().join(&entry).exists()); + let shutdown = root.path().join("lifecycle/worker.shutdown"); + for _ in 0..2 { + let result = std::process::Command::new(&executable) + .env_clear() + .current_dir(root.path()) + .args(["notification.mail.read", "{}"]) + .output() + .unwrap(); + assert!( + result.status.success(), + "{}", + String::from_utf8_lossy(&result.stderr) + ); + let response: serde_json::Value = serde_json::from_slice(&result.stdout).unwrap(); + assert_eq!(response["code"], 0); + assert_eq!(response["data"], 7); + assert!( + shutdown.is_file(), + "entry cleanup must complete the Worker shutdown handshake" + ); + // 下一次执行必须留下新的确认，不能复用上一次的关机标记。 + fs::remove_file(&shutdown).unwrap(); + } + let bundle = fs::read_dir(root.path().join("data/cache/lib")) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + assert_eq!( + fs::read(bundle.join("assets/worker-copy.bin")).unwrap(), + resources[0].bytes + ); + fs::write(bundle.join("workers/fixture/worker"), b"tampered").unwrap(); + let result = std::process::Command::new(&executable) + .env_clear() + .args(["notification.mail.read", "{}"]) + .output() + .unwrap(); + assert!(!result.status.success()); + assert!(String::from_utf8_lossy(&result.stderr).contains("integrity verification")); +} diff --git a/test/dever-tests/tests/formatter.rs b/test/dever-tests/tests/formatter.rs new file mode 100644 index 0000000..6ee1ee2 --- /dev/null +++ b/test/dever-tests/tests/formatter.rs @@ -0,0 +1,640 @@ +use std::fs; +use std::path::{Path as FilePath, PathBuf}; +use std::process::ExitCode; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use dever_core::source::{SourceMap, Span}; +use dever_core::syntax::*; + +#[path = "../../../crates/dever-cli/src/format.rs"] +mod cli_format; + +fn parsed_at(path: &FilePath, text: &str) -> Package { + let mut sources = SourceMap::default(); + let id = sources.add(path, text); + dever_core::parse(sources.get(id)).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }) +} + +fn formatted_at(path: &FilePath, text: &str) -> String { + let mut sources = SourceMap::default(); + let id = sources.add(path, text); + dever_core::format::format(sources.get(id)).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }) +} + +fn assert_roundtrip(source: &str) -> String { + assert_roundtrip_at(FilePath::new("main.dever"), source) +} + +fn assert_roundtrip_at(path: &FilePath, source: &str) -> String { + let output = formatted_at(path, source); + assert_eq!( + shape(parsed_at(path, source)), + shape(parsed_at(path, &output)), + "{output}" + ); + assert_eq!( + formatted_at(path, &output), + output, + "formatter must be idempotent" + ); + output +} + +#[test] +fn canonical_layout_preserves_field_evaluation_order_and_groups() { + let source = "type Pair{a:Int\nb:Int}\npublic main( )(result:Pair){result=Pair{b=second()\na=first()}}\n"; + assert_eq!( + assert_roundtrip(source), + "type Pair {\n a: Int\n b: Int\n}\n\npublic main() (result: Pair) {\n result = Pair {\n b = second()\n a = first()\n }\n}\n" + ); +} + +#[test] +fn all_syntax_and_literal_forms_roundtrip_without_semantic_checking() { + let source = r#"type Empty {} +type Event {Ready(bytes: Bytes, code: Int) +End() +Failed(message: Text)} +public run(route: handler(value: Int) (answer: Int), stream: Stream<Event>, limit: Int >= -2 and < 8) (result: List<MapEntry<Text, Int?>>) { +values = [1, 2, -3, 1.20, 3e-2, 5.0] +mapping = {"z" = 1 +"a" = 2} +result = [] +empty = {} +truth = not false and true or false +maths = -(-2) + 3 * (4 - 5) / 6 // 7 % 8 +compare = (1 < 2) == (3 <= 4) != (5 > 6) == (7 >= 8) +literal = "引号\" 斜杠\\ 换行\n回车\r制表\t零\u{0}控制\u{1f}字符\u{4e2d}" +maybe = null +record = app.Pair {b = route(2) +a = route(1)} +record.a = route(3) +call = (route(1)).answer +each(consume, stream) +} +pick(event: Event.Ready(bytes, code)) (value: Int) {value=code} +pick(event: other) (value: Int) {value=-1} +constant(value: -2) (result: Int) {result=2} +empty_generic(value: List<>) () {} +"#; + let output = assert_roundtrip(source); + assert!(output.contains("\\u{0}")); + assert!(output.contains("b = route(2)\n a = route(1)")); + assert!(output.contains("End()")); +} + +#[test] +fn application_declarations_roundtrip_with_full_structure() { + for (path, source) in [ + ("system/health/api.dever", "cmd ping=app.ping\n"), + ("news/article/api.dever", "rest model\n"), + ( + "notification/mail/adapter/worker.dever", + "external exec \"worker\" {lib \"pip:fixture@1.0.0\"\nallow network}\n", + ), + ("user/session/job.dever", "schedule cleanup=\"0 * * * *\"\n"), + ] { + assert_roundtrip_at(FilePath::new(path), source); + } +} + +#[test] +fn comments_inside_expressions_signatures_and_eof_are_preserved() { + let source = r#"# 模块说明 +# 类型说明 +type Empty { # 空类型 +# 保留空类型内部注释 +} +public main( # 输入说明 +value: handler( # 回调输入 +value: Int # 数值 +) (answer: Int), # 回调输出 +values: List< # 元素类型 +Int +>) (result: Int) { # 正文 + result=( # 分组 + 1 # 第一项 + + # 运算符 + 2 + ) # 结果 + list=[ # 列表 + 1, # 成员 + # 下一成员 + 2 # 最后一项 + ] + call=f(1 # 逗号之前 + ,2) + mapping={ # 映射 + "key"=2 # 映射值 + # 结束映射 + } + # 结束正文 +} # 结束函数 +# 文件结尾 +# 无换行结尾"#; + let output = assert_roundtrip(source); + assert!(output.starts_with("# 模块说明\n# 类型说明\ntype Empty")); + assert!(output.ends_with("# 文件结尾\n# 无换行结尾\n")); +} + +#[test] +fn long_positional_lists_wrap_with_two_space_indentation() { + let source = "public main() () {send(\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\", \"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\")}\n"; + let output = assert_roundtrip(source); + assert!(output.contains(" send(\n \"aaaaaaaa")); + assert!(output.contains("\",\n \"bbbbbbbb")); + assert!(output.ends_with("\n )\n}\n")); +} + +#[test] +fn line_endings_comment_whitespace_and_unicode_are_canonical() { + for newline in ["\n", "\r\n", "\r"] { + let source = [ + "# 中文注释 ", + "", + "main() (value: Text) {value=\"# 😀\\u{2028}\\u{7f}\"} # 尾注释\t", + "", + ] + .join(newline); + let output = assert_roundtrip(&source); + assert!(!output.contains('\r')); + assert!(output.starts_with("# 中文注释\n")); + assert!(output.ends_with("} # 尾注释\n")); + } +} + +#[test] +fn current_examples_official_packages_and_complete_contract_roundtrip() { + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../.."); + for relative in ["library", "examples"] { + let sources = SourceMap::load(&root.join(relative)).unwrap(); + for source in sources.files() { + let path = source_root_relative(source.path()); + assert_roundtrip_at(&path, source.text()); + } + } + let prd = include_str!("../../../.trellis/tasks/09-04-dever-language-mvp/prd.md"); + let (_, example) = prd.split_once("## Complete Package Example\n").unwrap(); + let (_, fenced) = example.split_once("```dever\n").unwrap(); + assert_roundtrip(fenced.split_once("\n```").unwrap().0); +} + +fn source_root_relative(path: &FilePath) -> PathBuf { + let segments = path.iter().collect::<Vec<_>>(); + let start = segments + .iter() + .enumerate() + .filter_map(|(index, segment)| { + matches!(segment.to_str(), Some("module" | "test")).then_some(index + 1) + }) + .next_back() + .unwrap_or(0); + segments[start..].iter().collect() +} + +#[test] +fn invalid_source_never_produces_formatted_output() { + for text in ["package", "main() () {x =}", "# comment\n$"] { + let mut sources = SourceMap::default(); + let id = sources.add("broken.dever", text); + assert!(dever_core::format::format(sources.get(id)).is_err()); + } +} + +#[test] +fn cli_prepares_all_files_before_writes_and_check_is_read_only() { + let temporary = Temporary::new(); + let module = temporary.0.join("module"); + fs::create_dir(&module).unwrap(); + let valid = ""; + fs::write(module.join("main.dever"), valid).unwrap(); + fs::write(module.join("broken.dever"), "package").unwrap(); + fs::write(module.join("unrelated.txt"), "unchanged").unwrap(); + let sources = SourceMap::load_project(&module, &temporary.0.join("test")).unwrap(); + assert!(cli_format::execute(&temporary.0, &sources, false).is_err()); + assert_eq!( + fs::read_to_string(module.join("main.dever")).unwrap(), + valid + ); + fs::remove_file(module.join("broken.dever")).unwrap(); + let sources = SourceMap::load_project(&module, &temporary.0.join("test")).unwrap(); + assert_eq!( + cli_format::execute(&temporary.0, &sources, true).unwrap(), + ExitCode::FAILURE + ); + assert_eq!( + fs::read_to_string(module.join("main.dever")).unwrap(), + valid + ); + assert_eq!( + cli_format::execute(&temporary.0, &sources, false).unwrap(), + ExitCode::SUCCESS + ); + let updated = SourceMap::load_project(&module, &temporary.0.join("test")).unwrap(); + assert_eq!( + cli_format::execute(&temporary.0, &updated, true).unwrap(), + ExitCode::SUCCESS + ); + assert_eq!( + fs::read_to_string(module.join("unrelated.txt")).unwrap(), + "unchanged" + ); + assert_eq!(fs::read_dir(&module).unwrap().count(), 2); +} + +#[test] +fn source_path_set_changes_prevent_all_rewrites() { + let temporary = Temporary::new(); + let module = temporary.0.join("module"); + fs::create_dir(&module).unwrap(); + let original = ""; + let existing = module.join("main.dever"); + let added = module.join("added.dever"); + fs::write(&existing, original).unwrap(); + let snapshot = SourceMap::load_project(&module, &temporary.0.join("test")).unwrap(); + fs::write(&added, "package").unwrap(); + assert!(cli_format::execute(&temporary.0, &snapshot, false).is_err()); + assert_eq!(fs::read_to_string(&existing).unwrap(), original); + assert_eq!(fs::read_to_string(&added).unwrap(), "package"); + + fs::remove_file(&added).unwrap(); + fs::rename(&existing, &added).unwrap(); + assert!(cli_format::execute(&temporary.0, &snapshot, false).is_err()); + assert_eq!(fs::read_to_string(&added).unwrap(), original); + assert!(!existing.exists()); +} + +#[test] +fn changed_snapshot_and_read_only_files_prevent_all_rewrites() { + let temporary = Temporary::new(); + let module = temporary.0.join("module"); + fs::create_dir(&module).unwrap(); + let original = ""; + let first = module.join("first.dever"); + let second = module.join("second.dever"); + fs::write(&first, original).unwrap(); + fs::write(&second, original).unwrap(); + let sources = SourceMap::load_project(&module, &temporary.0.join("test")).unwrap(); + fs::write(&second, "edited").unwrap(); + assert!(cli_format::execute(&temporary.0, &sources, false).is_err()); + assert_eq!(fs::read_to_string(&first).unwrap(), original); + fs::write(&second, original).unwrap(); + let permissions = fs::metadata(&second).unwrap().permissions(); + let mut read_only = permissions.clone(); + read_only.set_readonly(true); + fs::set_permissions(&second, read_only).unwrap(); + let result = cli_format::execute(&temporary.0, &sources, false); + fs::set_permissions(&second, permissions).unwrap(); + assert!(result.is_err()); + assert_eq!(fs::read_to_string(&first).unwrap(), original); + assert_eq!(fs::read_dir(&module).unwrap().count(), 2); +} + +#[test] +fn module_and_test_sources_are_prepared_before_any_rewrite() { + let temporary = Temporary::new(); + let module = temporary.0.join("module"); + let tests = temporary.0.join("test/user/account"); + fs::create_dir(&module).unwrap(); + fs::create_dir_all(&tests).unwrap(); + let main = module.join("main.dever"); + let test = tests.join("register.dever"); + let unformatted_main = "main()(){ }"; + let unformatted_test = "register()(){assert(true)}"; + fs::write(&main, unformatted_main).unwrap(); + fs::write(&test, unformatted_test).unwrap(); + let sources = SourceMap::load_project(&module, &temporary.0.join("test")).unwrap(); + + let permissions = fs::metadata(&test).unwrap().permissions(); + let mut read_only = permissions.clone(); + read_only.set_readonly(true); + fs::set_permissions(&test, read_only).unwrap(); + let result = cli_format::execute(&temporary.0, &sources, false); + fs::set_permissions(&test, permissions).unwrap(); + assert!(result.is_err()); + assert_eq!(fs::read_to_string(&main).unwrap(), unformatted_main); + assert_eq!(fs::read_to_string(&test).unwrap(), unformatted_test); + + cli_format::execute(&temporary.0, &sources, false).unwrap(); + assert_eq!(fs::read_to_string(&main).unwrap(), "main() () {}\n"); + assert_eq!( + fs::read_to_string(&test).unwrap(), + "register() () {\n assert(true)\n}\n" + ); +} + +#[cfg(unix)] +#[test] +fn symbolic_links_are_rejected_and_permissions_are_preserved() { + use std::os::unix::fs::{PermissionsExt, symlink}; + let temporary = Temporary::new(); + let module = temporary.0.join("module"); + fs::create_dir(&module).unwrap(); + let original = ""; + let file = module.join("main.dever"); + fs::write(&file, original).unwrap(); + symlink(&file, module.join("linked.dever")).unwrap(); + assert!(SourceMap::load_project(&module, &temporary.0.join("test")).is_err()); + assert_eq!(fs::read_to_string(&file).unwrap(), original); + fs::remove_file(module.join("linked.dever")).unwrap(); + fs::set_permissions(&file, fs::Permissions::from_mode(0o640)).unwrap(); + let sources = SourceMap::load_project(&module, &temporary.0.join("test")).unwrap(); + cli_format::execute(&temporary.0, &sources, false).unwrap(); + assert_eq!( + fs::metadata(&file).unwrap().permissions().mode() & 0o777, + 0o640 + ); +} + +struct Temporary(PathBuf); + +impl Temporary { + fn new() -> Self { + static NEXT: AtomicUsize = AtomicUsize::new(0); + let path = std::env::temp_dir().join(format!( + "dever-formatter-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir(&path).unwrap(); + Self(path) + } +} + +impl Drop for Temporary { + fn drop(&mut self) { + fs::remove_dir_all(&self.0).unwrap(); + } +} + +// Compare full syntax structure and decoded literals independently of source positions. +fn shape(mut package: Package) -> Package { + let span = Span { + start: 0, + end: 0, + ..package.span + }; + package.span = span; + normalize_path(&mut package.name, span); + for comment in &mut package.comments { + comment.span = span; + comment.text = comment.text.trim_end().to_owned(); + } + for declaration in &mut package.declarations { + match declaration { + Declaration::Api(binding) => { + binding.span = span; + binding.action.span = span; + normalize_path(&mut binding.target, span); + } + Declaration::Rest(rest) => { + rest.span = span; + if let Some(model) = &mut rest.model { + normalize_path(model, span); + } + } + Declaration::External(external) => { + external.span = span; + external.entry_span = span; + for name in external.libs.iter_mut().chain(&mut external.capabilities) { + name.span = span; + } + } + Declaration::Type(declaration) => { + declaration.span = span; + declaration.name.span = span; + match &mut declaration.shape { + TypeShape::Record(fields) => normalize_fields(fields, span), + TypeShape::Choice(variants) => { + for variant in variants { + variant.span = span; + variant.name.span = span; + normalize_fields(&mut variant.payload, span); + } + } + } + } + Declaration::Function(function) => { + function.span = span; + function.name.span = span; + normalize_fields(&mut function.outputs, span); + if let Some(failure) = &mut function.fails { + normalize_type(failure, span); + } + for input in &mut function.inputs { + input.span = span; + input.name.span = span; + match &mut input.kind { + InputKind::Handler(signature) => { + signature.span = span; + normalize_fields(&mut signature.inputs, span); + normalize_fields(&mut signature.outputs, span); + } + InputKind::Value(pattern) => match pattern { + Pattern::Typed { ty, bounds } => { + normalize_type(ty, span); + for bound in bounds { + bound.span = span; + } + } + Pattern::Variant { + name, + arguments, + bindings, + span: pattern_span, + } => { + normalize_path(name, span); + for argument in arguments { + normalize_type(argument, span); + } + normalize_path(bindings, span); + *pattern_span = span; + } + Pattern::Literal { + span: pattern_span, .. + } + | Pattern::Other(pattern_span) => *pattern_span = span, + }, + } + } + for statement in &mut function.body { + statement.span = span; + match &mut statement.kind { + StatementKind::Assign { target, value } => { + normalize_path(target, span); + normalize_expression(value, span); + } + StatementKind::Call(call) => normalize_expression(call, span), + } + } + } + Declaration::Database(binding) => { + binding.span = span; + binding.name.span = span; + } + Declaration::Schedule(schedule) => { + schedule.span = span; + schedule.target.span = span; + } + Declaration::ModelIndex(index) => { + index.span = span; + normalize_path(&mut index.fields, span); + } + Declaration::Relation(relation) => { + relation.span = span; + relation.name.span = span; + normalize_path(&mut relation.field, span); + } + Declaration::Seed(seed) => { + seed.span = span; + for row in &mut seed.rows { + for field in row { + field.span = span; + field.name.span = span; + normalize_expression(&mut field.value, span); + } + } + } + Declaration::Migration(migration) => { + migration.span = span; + migration.name.span = span; + for operation in &mut migration.operations { + match operation { + MigrationOperation::Drop(field) => field.span = span, + MigrationOperation::Sql(sql) => { + sql.span = span; + sql.sqlite.span = span; + sql.postgres.span = span; + for parameter in &mut sql.parameters { + normalize_expression(parameter, span); + } + } + } + } + } + Declaration::ModelSql(sql) => { + sql.span = span; + sql.name.span = span; + normalize_fields(&mut sql.inputs, span); + normalize_fields(&mut sql.outputs, span); + sql.sqlite.span = span; + sql.postgres.span = span; + } + } + } + package +} + +fn normalize_path(path: &mut Path, span: Span) { + for name in path { + name.span = span; + } +} + +fn normalize_fields(fields: &mut [Field], span: Span) { + for field in fields { + field.span = span; + field.name.span = span; + normalize_type(&mut field.ty, span); + for bound in &mut field.bounds { + bound.span = span; + } + if let Some(value) = &mut field.storage.default { + normalize_expression(value, span); + } + if let Some(from) = &mut field.storage.from { + from.span = span; + } + for expression in [ + &mut field.storage.owner, + &mut field.storage.create, + &mut field.storage.replace, + &mut field.storage.search, + ] + .into_iter() + .flatten() + { + normalize_expression(expression, span); + } + } +} + +fn normalize_type(ty: &mut TypeRef, span: Span) { + ty.span = span; + normalize_path(&mut ty.name, span); + for argument in &mut ty.arguments { + normalize_type(argument, span); + } + for parameter in &mut ty.parameters { + parameter.span = span; + } +} + +fn normalize_expression(expression: &mut Expression, span: Span) { + expression.span = span; + match &mut expression.kind { + ExpressionKind::Literal(_) => {} + ExpressionKind::Name(path) => normalize_path(path, span), + ExpressionKind::Group(value) + | ExpressionKind::Fail(value) + | ExpressionKind::CaptureResult(value) + | ExpressionKind::Unary { value, .. } => normalize_expression(value, span), + ExpressionKind::Contextual(ContextualExpression::Channel { element, capacity }) => { + normalize_type(element, span); + normalize_expression(capacity, span); + } + ExpressionKind::Field { value, name } => { + normalize_expression(value, span); + name.span = span; + } + ExpressionKind::Call { + function, + arguments, + } => { + normalize_expression(function, span); + for argument in arguments { + normalize_expression(argument, span); + } + } + ExpressionKind::Record { name, fields } => { + normalize_path(name, span); + for field in fields { + field.span = span; + field.name.span = span; + normalize_expression(&mut field.value, span); + } + } + ExpressionKind::List(values) => { + for value in values { + normalize_expression(value, span); + } + } + ExpressionKind::Map(entries) => { + for entry in entries { + entry.span = span; + normalize_expression(&mut entry.key, span); + normalize_expression(&mut entry.value, span); + } + } + ExpressionKind::Binary { left, right, .. } => { + normalize_expression(left, span); + normalize_expression(right, span); + } + } +} diff --git a/test/dever-tests/tests/hello_native.rs b/test/dever-tests/tests/hello_native.rs new file mode 100644 index 0000000..493e7f6 --- /dev/null +++ b/test/dever-tests/tests/hello_native.rs @@ -0,0 +1,356 @@ +use std::ffi::OsString; +use std::fs; +use std::io::{self, Write}; +use std::process::Command; + +use dever_core::hir::Program; +use dever_core::source::SourceMap; + +const HELLO: &str = "greet() () {\n dever.io.println(\"hello\")\n}\n"; + +fn sources(files: &[(&str, &str)]) -> SourceMap { + let mut sources = SourceMap::default(); + for (path, text) in files { + sources.add(path, *text); + } + sources +} + +fn checked(sources: &SourceMap) -> Program { + dever_core::check(sources).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(sources)) + .collect::<String>() + ) + }) +} + +fn compile(sources: &SourceMap) -> dever_core::native::NativeProgram { + compile_entry(sources, "app.user.main") +} + +fn compile_entry(sources: &SourceMap, entry: &str) -> dever_core::native::NativeProgram { + let rustc = std::env::var_os("RUSTC").unwrap_or_else(|| OsString::from("rustc")); + dever_core::native::compile(&checked(sources), sources, entry, &rustc).unwrap() +} + +#[test] +fn hello_example_is_a_native_executable_with_exact_stdout_and_cleanup() { + let sources = sources(&[("app/user/app.dever", HELLO)]); + let native = compile_entry(&sources, "app.user.greet"); + let path = native.executable().to_owned(); + let output = Command::new(&path).output().unwrap(); + assert!(output.status.success(), "{:?}", output); + assert_eq!(output.stdout, b"hello\n"); + assert!(output.stderr.is_empty()); + + let saved = path.parent().unwrap().join("saved-program"); + native.save(&saved).unwrap(); + let original = fs::read(&saved).unwrap(); + assert_eq!( + native.save(&saved).unwrap_err().kind(), + io::ErrorKind::AlreadyExists + ); + assert_eq!(fs::read(&saved).unwrap(), original); + assert_eq!(Command::new(&saved).output().unwrap().stdout, b"hello\n"); + drop(native); + assert!(!path.parent().unwrap().exists()); +} + +#[test] +fn text_calls_resolve_forward_across_packages_and_preserve_value_semantics() { + let sources = sources(&[ + ( + "app/user/app.dever", + r#"main() () { + message = "first" + saved = message + message = "second" + show(message) + show(saved) + echo.change(message) + show(message) + self("quote: \" slash: \\ newline: \n tab: \t null: \u{0} unicode: \u{4e2d}") + show() +} +show(message: Text) () { + echo.say(message) +} +show() () { + echo.say("") +} +self(value: Text) () { + echo.say(value) +} +"#, + ), + ( + "app/echo/app.dever", + r#"say(message: Text) () { + dever.io.println(message) +} +change(message: Text) () { + message = "changed" + say(message) +} +"#, + ), + ]); + let native = compile(&sources); + let output = Command::new(native.executable()).output().unwrap(); + assert!(output.status.success(), "{:?}", output); + assert_eq!( + String::from_utf8(output.stdout).unwrap(), + "second\nfirst\nchanged\nsecond\nquote: \" slash: \\ newline: \n tab: \t null: \0 unicode: \u{4e2d}\n\n" + ); + assert!(output.stderr.is_empty()); +} + +#[test] +fn invalid_calls_and_expressions_fail_before_native_compilation() { + for (body, code, message) in [ + ("missing(\"hello\")", "C004", "unknown function"), + ("dever.io.println(1)", "C005", "expected Text"), + ("dever.io.println(true)", "C005", "expected Text"), + ("dever.io.println()", "C004", "unknown function"), + ("dever.io.println(\"a\", \"b\")", "C004", "unknown function"), + ("dever.io.println(message)", "C004", "uninitialized local"), + ("message = message", "C004", "uninitialized local"), + ("message = \"a\"\nmessage = 1", "C005", "expected Text"), + ( + "message = dever.io.println(\"a\")", + "C005", + "zero-output action cannot be assigned", + ), + ("dever.io.println(\"a\" + 1)", "C005", "numeric operands"), + ("message = \"a\"\nmessage()", "C005", "not callable"), + ("message.value = \"a\"", "C004", "uninitialized local"), + ] { + let source = format!("main() () {{\n {body}\n}}\n"); + let sources = sources(&[("app/user/app.dever", &source)]); + let errors = dever_core::check(&sources).unwrap_err(); + assert!( + errors + .iter() + .any(|error| error.code == code && error.message.contains(message)), + "{body}: {errors:?}" + ); + assert!(errors[0].render(&sources).contains("app/user/app.dever:")); + } +} + +#[test] +fn diagnostics_point_to_the_actual_invalid_argument() { + let sources = sources(&[( + "app/user/app.dever", + "main() () {\n dever.io.println(42)\n}\n", + )]); + let errors = dever_core::check(&sources).unwrap_err(); + assert_eq!(errors.len(), 1); + assert!( + errors[0] + .render(&sources) + .starts_with("app/user/app.dever:2:20: error[C005]: expected Text, found Int\n") + ); +} + +#[test] +fn invalid_declarations_are_rejected_even_when_unused() { + for (declarations, expected) in [ + ("unused() (result: Text) {}", "uninitialized"), + ("unused(value: Unknown) () {}", "unknown type"), + ("unused(value: null) () {}", "cannot infer"), + ("unused(value: \"a\") () {}", "do not cover"), + ("unused(value: Text, value: Text) () {}", "duplicate local"), + ("unused() () {}\nunused() () {}", "overlap"), + ("type User { name: Missing }", "unknown type"), + ("unused() () { missing() }", "unknown function"), + ] { + let source = format!("main() () {{}}\n{declarations}\n"); + let sources = sources(&[("app/user/app.dever", &source)]); + let errors = dever_core::check(&sources).unwrap_err(); + assert!( + errors.iter().any(|error| error.message.contains(expected)), + "{declarations}: {errors:?}" + ); + } +} + +#[test] +fn package_paths_namespaces_and_exposures_are_checked() { + for (path, source, message) in [ + ("dever/io/app.dever", "main() () {}", "reserved"), + ("text.dever", "main() () {}", "reserved"), + ( + "app/user/app.dever", + "exposes (main)\nmain() () {}", + "package/exposes declarations were removed", + ), + ( + "app/user/app.dever", + "public main() () {}", + "visibility is defined by the source role", + ), + ] { + let sources = sources(&[(path, source)]); + let errors = dever_core::check(&sources).unwrap_err(); + assert!( + errors.iter().any(|error| error.message.contains(message)), + "{errors:?}" + ); + } + let loose = sources(&[("user.dever", "main() () {}")]); + let application = + SourceMap::from_compilation_sources(&loose.compilation_sources().unwrap()).unwrap(); + assert!( + dever_core::check(&application) + .unwrap_err() + .iter() + .any(|error| error.message.contains("application source must be")) + ); + let sources = sources(&[("app/user/app.dever", ""), ("app/user/app.dever", "")]); + assert!( + dever_core::check(&sources) + .unwrap_err() + .iter() + .any(|error| error.message == "duplicate package") + ); +} + +#[test] +fn private_functions_cannot_be_called_from_another_package() { + let sources = sources(&[ + ( + "app/user/app.dever", + "main() () { app.echo.domain.hidden() }", + ), + ("app/echo/domain.dever", "hidden() () {}"), + ]); + assert!( + dever_core::check(&sources) + .unwrap_err() + .iter() + .any(|error| error.code == "C006") + ); +} + +#[test] +fn recursion_and_package_cycles_are_rejected() { + for body in ["main()", "next()"] { + let source = format!("main() () {{ {body} }}\nnext() () {{ main() }}"); + let sources = sources(&[("app/user/app.dever", &source)]); + assert!( + dever_core::check(&sources) + .unwrap_err() + .iter() + .any(|error| error.message.contains("recursive")) + ); + } + let sources = sources(&[ + ( + "app/first/app.dever", + "start() () { second.middle() }\nfinish() () {}", + ), + ("app/second/app.dever", "middle() () { first.finish() }"), + ]); + let errors = dever_core::check(&sources).unwrap_err(); + assert_eq!(errors.len(), 1); + assert!(errors[0].message.contains("cyclic package")); +} + +#[test] +fn entry_requires_a_public_zero_input_function() { + let sources = sources(&[ + ( + "app/user/app.dever", + "main() () {}\nwith_input(message: Text) () {}", + ), + ("app/user/domain.dever", "hidden() () {}"), + ]); + let program = checked(&sources); + assert!(dever_core::native::emit(&program, &sources, "app.user.main").is_ok()); + for (entry, message) in [ + ("app.user.domain.hidden", "must be public"), + ("app.user.with_input", "zero-input"), + ("missing.main", "zero-input"), + ] { + assert!( + dever_core::native::emit(&program, &sources, entry) + .unwrap_err() + .contains(message) + ); + } +} + +#[test] +fn missing_toolchain_reports_an_actionable_error() { + let sources = sources(&[("app/user/app.dever", "main() () {}")]); + let result = dever_core::native::compile( + &checked(&sources), + &sources, + "app.user.main", + std::ffi::OsStr::new("/nonexistent-dever-test/rustc"), + ); + let Err(error) = result else { + panic!("missing compiler succeeded") + }; + assert!(error.contains("set RUSTC")); +} + +struct FailingOutput { + fail_write: bool, +} + +impl Write for FailingOutput { + fn write(&mut self, buffer: &[u8]) -> io::Result<usize> { + if self.fail_write { + Err(io::ErrorKind::BrokenPipe.into()) + } else { + Ok(buffer.len()) + } + } + + fn flush(&mut self) -> io::Result<()> { + Err(io::ErrorKind::Other.into()) + } +} + +#[test] +fn runtime_preserves_utf8_and_propagates_write_and_flush_failures() { + let mut bytes = Vec::new(); + dever_runtime::write_line(&mut bytes, "hello \u{4e2d}\n").unwrap(); + assert_eq!(bytes, "hello \u{4e2d}\n\n".as_bytes()); + for (fail_write, expected) in [ + (true, io::ErrorKind::BrokenPipe), + (false, io::ErrorKind::Other), + ] { + assert_eq!( + dever_runtime::write_line(&mut FailingOutput { fail_write }, "hello") + .unwrap_err() + .kind(), + expected + ); + } +} + +#[cfg(target_os = "linux")] +#[test] +fn native_output_failure_has_dever_location_and_nonzero_exit() { + let sources = sources(&[("app/user/app.dever", HELLO)]); + let native = compile_entry(&sources, "app.user.greet"); + let full = fs::OpenOptions::new() + .write(true) + .open("/dev/full") + .unwrap(); + let output = Command::new(native.executable()) + .stdout(full) + .output() + .unwrap(); + assert!(!output.status.success()); + let error = String::from_utf8(output.stderr).unwrap(); + assert!(error.contains("app/user/app.dever:2:3:"), "{error}"); + assert!(error.contains("standard output failed"), "{error}"); +} diff --git a/test/dever-tests/tests/http2_client.rs b/test/dever-tests/tests/http2_client.rs new file mode 100644 index 0000000..6baa4fd --- /dev/null +++ b/test/dever-tests/tests/http2_client.rs @@ -0,0 +1,566 @@ +use std::future::{Future, poll_fn}; +use std::time::Duration; + +use dever_runtime::{ + async_stream::AsyncStream, bytes::Bytes, channel::Channel, http, net, task, tls, +}; +use tokio::sync::{mpsc, oneshot}; + +const ROOT: &[u8] = include_bytes!("../fixtures/tls/root.pem"); +const CERT: &[u8] = include_bytes!("../fixtures/tls/server.pem"); +const KEY: &[u8] = include_bytes!("../fixtures/tls/server-key.pem"); + +fn bytes(value: &[u8]) -> Bytes { + Bytes::new(value.to_vec()) +} + +fn limits(connections: i64, streams: i64) -> http::Limits { + http::Limits { + header_bytes: 8192, + body_bytes: 65536, + timeout_ms: 1000, + connections, + http2: Some(http::Http2Limits { + streams, + stream_window_bytes: 65535, + connection_window_bytes: 65535, + }), + } +} + +fn pool_limits() -> http::PoolLimits { + http::PoolLimits { + idle_ms: 1000, + chunk_bytes: 4, + read_ms: 500, + } +} + +fn request(target: &str) -> http::Request { + http::Request { + method: "GET".into(), + target: target.into(), + headers: vec![], + body: bytes(b""), + } +} + +fn trust() -> tls::ClientTls { + tls::client(&bytes(ROOT)).unwrap() +} + +fn identity() -> tls::ServerTls { + tls::server(&bytes(CERT), &bytes(KEY)).unwrap() +} + +fn run(future: impl Future<Output = Result<(), String>>) { + task::run_entry_with( + task::RuntimeConfig { + worker_threads: 1, + max_blocking_threads: 2, + task_capacity: 64, + }, + async { + tokio::time::timeout(Duration::from_secs(8), future) + .await + .map_err(|_| "HTTP/2 client test deadline exceeded".to_owned())? + }, + ) + .unwrap(); +} + +fn h2_error(error: impl std::fmt::Display) -> String { + error.to_string() +} + +async fn next_request( + requests: &mut mpsc::Receiver<( + hyper::Request<h2::RecvStream>, + h2::server::SendResponse<bytes::Bytes>, + )>, +) -> Result< + ( + hyper::Request<h2::RecvStream>, + h2::server::SendResponse<bytes::Bytes>, + ), + String, +> { + requests + .recv() + .await + .ok_or("HTTP/2 connection closed before request".into()) +} + +fn response() -> hyper::Response<()> { + hyper::Response::builder().status(200).body(()).unwrap() +} + +async fn start_peer( + socket: tokio::net::TcpStream, +) -> Result< + ( + mpsc::Receiver<( + hyper::Request<h2::RecvStream>, + h2::server::SendResponse<bytes::Bytes>, + )>, + oneshot::Sender<oneshot::Sender<()>>, + task::Task<()>, + ), + String, +> { + let mut connection = h2::server::handshake(socket).await.map_err(h2_error)?; + let (requests, receiver) = mpsc::channel(4); + let (shutdown, mut shutdown_request) = oneshot::channel::<oneshot::Sender<()>>(); + let driver = task::run(async move { + let mut graceful = false; + loop { + let request = if graceful { + connection.accept().await + } else { + tokio::select! { + request = connection.accept() => request, + command = &mut shutdown_request => { + if let Ok(acknowledge) = command { + connection.graceful_shutdown(); + graceful = true; + let _ = acknowledge.send(()); + continue; + } + connection.accept().await + } + } + }; + let Some(request) = request else { + return Ok(()); + }; + requests + .send(request.map_err(h2_error)?) + .await + .map_err(|_| "HTTP/2 peer request receiver closed".to_owned())?; + } + }) + .await?; + Ok((receiver, shutdown, driver)) +} + +#[test] +fn one_physical_connection_multiplexes_upload_and_streaming_download() { + run(async { + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .map_err(h2_error)?; + let port = i64::from(listener.local_addr().map_err(h2_error)?.port()); + let peer = task::run(async move { + let (socket, _) = listener.accept().await.map_err(h2_error)?; + let (mut requests, _shutdown, driver) = start_peer(socket).await?; + + let (first, mut first_reply) = next_request(&mut requests).await?; + assert_eq!(first.method(), hyper::Method::POST); + assert_eq!(first.uri().scheme_str(), Some("http")); + let authority = format!("localhost:{port}"); + assert_eq!( + first.uri().authority().map(|value| value.as_str()), + Some(authority.as_str()) + ); + assert_eq!(first.uri().path(), "/upload"); + let mut upload = first.into_body(); + let mut uploaded = Vec::new(); + while let Some(chunk) = upload.data().await { + let chunk = chunk.map_err(h2_error)?; + uploaded.extend_from_slice(&chunk); + } + assert_eq!(uploaded, b"abcdefgh"); + let mut first_body = first_reply + .send_response(response(), false) + .map_err(h2_error)?; + first_body + .send_data(bytes::Bytes::from_static(b"first"), false) + .map_err(h2_error)?; + + let (second, mut second_reply) = next_request(&mut requests).await?; + assert_eq!(second.uri().path(), "/second"); + let mut second_body = second_reply + .send_response(response(), false) + .map_err(h2_error)?; + second_body + .send_data(bytes::Bytes::from_static(b"second"), true) + .map_err(h2_error)?; + first_body + .send_data(bytes::Bytes::from_static(b"last"), true) + .map_err(h2_error)?; + + task::wait(driver).await?; + Ok(()) + }) + .await?; + + let client = http::client( + &format!("http://localhost:{port}"), + tls::system(), + limits(1, 2), + pool_limits(), + ) + .await?; + let first = http::open_stream( + &client, + http::StreamRequest { + method: "POST".into(), + target: "/upload".into(), + headers: vec![], + body: AsyncStream::from_values([Ok(bytes(b"abcd")), Ok(bytes(b"efgh"))]), + }, + ) + .await?; + assert_eq!(first.body.pull().await.unwrap()?.values(), b"firs"); + assert_eq!(first.body.pull().await.unwrap()?.values(), b"t"); + assert_eq!( + http::request(&client, request("/second")) + .await? + .body + .values(), + b"second" + ); + let mut tail = Vec::new(); + while let Some(chunk) = first.body.pull().await { + tail.extend_from_slice(chunk?.values()); + } + assert_eq!(tail, b"last"); + http::close_client(&client).await?; + task::wait(peer).await?; + Ok(()) + }); +} + +#[test] +fn canceling_one_response_resets_only_its_stream() { + run(async { + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .map_err(h2_error)?; + let port = i64::from(listener.local_addr().map_err(h2_error)?.port()); + let peer = task::run(async move { + let (socket, _) = listener.accept().await.map_err(h2_error)?; + let (mut requests, _shutdown, driver) = start_peer(socket).await?; + let (_, mut first_reply) = next_request(&mut requests).await?; + let mut first_body = first_reply + .send_response(response(), false) + .map_err(h2_error)?; + first_body + .send_data(bytes::Bytes::from_static(b"open"), false) + .map_err(h2_error)?; + + let (_, mut neighbor_reply) = next_request(&mut requests).await?; + let mut neighbor_body = neighbor_reply + .send_response(response(), false) + .map_err(h2_error)?; + neighbor_body + .send_data(bytes::Bytes::from_static(b"neighbor"), true) + .map_err(h2_error)?; + + let reason = poll_fn(|context| first_body.poll_reset(context)) + .await + .map_err(h2_error)?; + assert_eq!(reason, h2::Reason::CANCEL); + + let (_, mut final_reply) = next_request(&mut requests).await?; + let mut final_body = final_reply + .send_response(response(), false) + .map_err(h2_error)?; + final_body + .send_data(bytes::Bytes::from_static(b"still-open"), true) + .map_err(h2_error)?; + task::wait(driver).await?; + Ok(()) + }) + .await?; + + let client = http::client( + &format!("http://localhost:{port}"), + tls::system(), + limits(1, 2), + pool_limits(), + ) + .await?; + let first = http::open(&client, request("/cancel")).await?; + assert_eq!(first.body.pull().await.unwrap()?.values(), b"open"); + assert_eq!( + http::request(&client, request("/neighbor")) + .await? + .body + .values(), + b"neighbor" + ); + first.body.close(); + assert_eq!( + http::request(&client, request("/after-reset")) + .await? + .body + .values(), + b"still-open" + ); + http::close_client(&client).await?; + task::wait(peer).await?; + Ok(()) + }); +} + +#[test] +fn stream_limit_blocks_without_expiring_an_active_body_then_reaps_at_eof() { + run(async { + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .map_err(h2_error)?; + let port = i64::from(listener.local_addr().map_err(h2_error)?.port()); + let finish_first = Channel::new(1)?; + let finish = finish_first.clone(); + let first_connection_closed = Channel::new(1)?; + let closed = first_connection_closed.clone(); + let peer = task::run(async move { + let (first_socket, _) = listener.accept().await.map_err(h2_error)?; + let (mut first_requests, _shutdown, first_driver) = start_peer(first_socket).await?; + let (_, mut first_reply) = next_request(&mut first_requests).await?; + let mut first_body = first_reply + .send_response(response(), false) + .map_err(h2_error)?; + first_body + .send_data(bytes::Bytes::from_static(b"open"), false) + .map_err(h2_error)?; + finish.receive().await?; + assert!(matches!( + first_requests.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + )); + first_body + .send_data(bytes::Bytes::from_static(b"done"), true) + .map_err(h2_error)?; + task::wait(first_driver).await?; + closed.send(()).await?; + + let (second_socket, _) = listener.accept().await.map_err(h2_error)?; + let (mut second_requests, _shutdown, second_driver) = start_peer(second_socket).await?; + let (_, mut second_reply) = next_request(&mut second_requests).await?; + let mut second_body = second_reply + .send_response(response(), false) + .map_err(h2_error)?; + second_body + .send_data(bytes::Bytes::from_static(b"new"), true) + .map_err(h2_error)?; + task::wait(second_driver).await?; + Ok(()) + }) + .await?; + + let mut budget = limits(1, 1); + budget.timeout_ms = 60; + let mut pool = pool_limits(); + pool.idle_ms = 30; + let client = http::client( + &format!("http://localhost:{port}"), + tls::system(), + budget, + pool, + ) + .await?; + let first = http::open(&client, request("/active")).await?; + assert_eq!(first.body.pull().await.unwrap()?.values(), b"open"); + task::sleep(80).await?; + assert!( + http::request(&client, request("/blocked")) + .await + .unwrap_err() + .contains("timed out") + ); + finish_first.send(()).await?; + assert_eq!(first.body.pull().await.unwrap()?.values(), b"done"); + assert!(first.body.pull().await.is_none()); + first_connection_closed.receive().await?; + assert_eq!( + http::request(&client, request("/after-idle")) + .await? + .body + .values(), + b"new" + ); + http::close_client(&client).await?; + task::wait(peer).await?; + Ok(()) + }); +} + +#[test] +fn goaway_replaces_the_connection_within_the_physical_budget() { + run(async { + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .map_err(h2_error)?; + let port = i64::from(listener.local_addr().map_err(h2_error)?.port()); + let goaway = Channel::new(1)?; + let goaway_sent = goaway.clone(); + let finish_old_stream = Channel::new(1)?; + let finish = finish_old_stream.clone(); + let peer = task::run(async move { + let (first_socket, _) = listener.accept().await.map_err(h2_error)?; + let (mut first_requests, shutdown, first_driver) = start_peer(first_socket).await?; + let (_, mut first_reply) = next_request(&mut first_requests).await?; + let mut first_body = first_reply + .send_response(response(), false) + .map_err(h2_error)?; + first_body + .send_data(bytes::Bytes::from_static(b"open"), false) + .map_err(h2_error)?; + let (acknowledge, acknowledged) = oneshot::channel(); + shutdown + .send(acknowledge) + .map_err(|_| "HTTP/2 peer driver closed before GOAWAY".to_owned())?; + acknowledged + .await + .map_err(|_| "HTTP/2 peer did not begin GOAWAY".to_owned())?; + task::sleep(100).await?; + goaway_sent.send(()).await?; + finish.receive().await?; + assert!(matches!( + first_requests.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + )); + first_body + .send_data(bytes::Bytes::from_static(b"done"), true) + .map_err(h2_error)?; + task::wait(first_driver).await?; + + let (second_socket, _) = listener.accept().await.map_err(h2_error)?; + let (mut second_requests, _shutdown, second_driver) = start_peer(second_socket).await?; + let (_, mut second_reply) = next_request(&mut second_requests).await?; + let mut second_body = second_reply + .send_response(response(), false) + .map_err(h2_error)?; + second_body + .send_data(bytes::Bytes::from_static(b"second"), true) + .map_err(h2_error)?; + task::wait(second_driver).await?; + Ok(()) + }) + .await?; + + let mut budget = limits(1, 2); + budget.timeout_ms = 80; + let client = http::client( + &format!("http://localhost:{port}"), + tls::system(), + budget, + pool_limits(), + ) + .await?; + let first = http::open(&client, request("/before-goaway")).await?; + assert_eq!(first.body.pull().await.unwrap()?.values(), b"open"); + goaway.receive().await?; + let blocked = http::request(&client, request("/must-wait")) + .await + .unwrap_err(); + assert!(blocked.contains("timed out"), "{blocked}"); + finish_old_stream.send(()).await?; + assert_eq!(first.body.pull().await.unwrap()?.values(), b"done"); + assert!(first.body.pull().await.is_none()); + assert_eq!( + http::request(&client, request("/after-goaway")) + .await? + .body + .values(), + b"second" + ); + http::close_client(&client).await?; + task::wait(peer).await?; + Ok(()) + }); +} + +#[test] +fn https_requires_h2_alpn_without_protocol_fallback() { + run(async { + let listener = net::listen("127.0.0.1", 0).await?; + let port = net::port(&listener)?; + let mut http1 = limits(1, 1); + http1.http2 = None; + let server_listener = listener.clone(); + let server = task::run(async move { + http::serve_tls( + |_| async { + Ok(http::Response { + status: 200, + headers: vec![], + body: bytes(b"unexpected"), + }) + }, + server_listener, + http1, + identity(), + ) + .await + }) + .await?; + let client = http::client( + &format!("https://localhost:{port}"), + trust(), + limits(1, 1), + pool_limits(), + ) + .await?; + let error = http::request(&client, request("/alpn")).await.unwrap_err(); + assert!(error.contains("NoApplicationProtocol"), "{error}"); + http::close_client(&client).await?; + task::stop(server).await?; + listener.close()?; + Ok(()) + }); +} + +#[test] +fn invalid_http2_budgets_and_outgoing_headers_fail_before_connecting() { + run(async { + let base = limits(1, 2); + let mut cases = Vec::new(); + for streams in [0, 65537] { + let mut budget = base; + budget.http2.as_mut().unwrap().streams = streams; + cases.push((budget, "connections * streams")); + } + let mut multiplied = base; + multiplied.connections = 32769; + cases.push((multiplied, "connections * streams")); + for window in [0, i64::from(i32::MAX) + 1] { + let mut budget = base; + budget.http2.as_mut().unwrap().stream_window_bytes = window; + cases.push((budget, "stream_window_bytes")); + } + for window in [65534, i64::from(i32::MAX) + 1] { + let mut budget = base; + budget.http2.as_mut().unwrap().connection_window_bytes = window; + cases.push((budget, "connection_window_bytes")); + } + for (budget, expected) in cases { + let failure = http::client("http://127.0.0.1:1", tls::system(), budget, pool_limits()) + .await + .unwrap_err(); + assert!(failure.contains(expected), "{failure}"); + } + + let client = http::client("http://127.0.0.1:1", tls::system(), base, pool_limits()).await?; + for (name, value) in [ + ("Connection", "close"), + ("Keep-Alive", "timeout=5"), + ("TE", "gzip"), + ] { + let mut outgoing = request("/"); + outgoing.headers.push(http::Header { + name: name.into(), + value: bytes(value.as_bytes()), + }); + let failure = http::request(&client, outgoing).await.unwrap_err(); + assert!( + failure.contains("HTTP/2 forbids connection-specific header"), + "{failure}" + ); + } + http::close_client(&client).await?; + Ok(()) + }); +} diff --git a/test/dever-tests/tests/http2_server.rs b/test/dever-tests/tests/http2_server.rs new file mode 100644 index 0000000..c6e97a3 --- /dev/null +++ b/test/dever-tests/tests/http2_server.rs @@ -0,0 +1,706 @@ +use std::future::{Future, poll_fn}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::time::Duration; + +use dever_runtime::{bytes::Bytes, http, net, sse, task, tls}; +use rustls::RootCertStore; +use rustls::pki_types::{CertificateDer, ServerName, pem::PemObject}; +use tokio::io::AsyncReadExt; + +const ROOT: &[u8] = include_bytes!("../fixtures/tls/root.pem"); +const CERT: &[u8] = include_bytes!("../fixtures/tls/server.pem"); +const KEY: &[u8] = include_bytes!("../fixtures/tls/server-key.pem"); + +fn bytes(value: &[u8]) -> Bytes { + Bytes::new(value.to_vec()) +} + +fn limits(timeout_ms: i64) -> http::Limits { + http::Limits { + header_bytes: 8192, + body_bytes: 65536, + timeout_ms, + connections: 2, + http2: Some(http::Http2Limits { + streams: 4, + stream_window_bytes: 65535, + connection_window_bytes: 65535, + }), + } +} + +fn live() -> http::LiveLimits { + http::LiveLimits { + chunk_bytes: 1024, + idle_ms: 1000, + heartbeat_ms: 30, + } +} + +fn run(future: impl Future<Output = Result<(), String>>) { + task::run_entry_with( + task::RuntimeConfig { + worker_threads: 1, + max_blocking_threads: 2, + task_capacity: 64, + }, + async { + tokio::time::timeout(Duration::from_secs(8), future) + .await + .map_err(|_| "HTTP/2 server test deadline exceeded".to_owned())? + }, + ) + .unwrap(); +} + +fn error(error: impl std::fmt::Display) -> String { + error.to_string() +} + +fn request(port: i64, target: &str) -> hyper::Request<()> { + hyper::Request::builder() + .version(hyper::Version::HTTP_2) + .uri(format!("http://localhost:{port}{target}")) + .body(()) + .unwrap() +} + +async fn connect( + port: i64, +) -> Result<(h2::client::SendRequest<bytes::Bytes>, task::Task<()>), String> { + let socket = tokio::net::TcpStream::connect(("127.0.0.1", port as u16)) + .await + .map_err(error)?; + let (sender, connection) = h2::client::handshake(socket).await.map_err(error)?; + let driver = task::run(async move { connection.await.map_err(error) }).await?; + Ok((sender, driver)) +} + +async fn connect_with_window( + port: i64, + stream_window: u32, +) -> Result<(h2::client::SendRequest<bytes::Bytes>, task::Task<()>), String> { + let socket = tokio::net::TcpStream::connect(("127.0.0.1", port as u16)) + .await + .map_err(error)?; + let mut builder = h2::client::Builder::new(); + builder.initial_window_size(stream_window); + let (sender, connection) = builder + .handshake::<_, bytes::Bytes>(socket) + .await + .map_err(error)?; + let driver = task::run(async move { connection.await.map_err(error) }).await?; + Ok((sender, driver)) +} + +async fn start_request( + sender: h2::client::SendRequest<bytes::Bytes>, + request: hyper::Request<()>, +) -> Result< + ( + h2::client::SendRequest<bytes::Bytes>, + h2::client::ResponseFuture, + ), + String, +> { + let mut sender = sender.ready().await.map_err(error)?; + let (response, _) = sender.send_request(request, true).map_err(error)?; + Ok((sender, response)) +} + +async fn response_body(response: hyper::Response<h2::RecvStream>) -> Result<Vec<u8>, String> { + let mut body = response.into_body(); + let mut result = Vec::new(); + while let Some(chunk) = body.data().await { + result.extend_from_slice(&chunk.map_err(error)?); + } + Ok(result) +} + +async fn wait_until(mut condition: impl FnMut() -> bool) -> Result<(), String> { + tokio::time::timeout(Duration::from_secs(1), async { + while !condition() { + tokio::task::yield_now().await; + } + }) + .await + .map_err(|_| "condition was not reached".to_owned()) +} + +#[test] +fn buffered_server_multiplexes_and_preserves_http2_request_semantics() { + run(async { + let gate = Arc::new(tokio::sync::Semaphore::new(0)); + let invoked = Arc::new(AtomicUsize::new(0)); + let route_gate = gate.clone(); + let route_invoked = invoked.clone(); + let listener = net::listen("127.0.0.1", 0).await?; + let port = net::port(&listener)?; + let owner = listener.clone(); + let server = task::run(http::serve( + move |request| { + let gate = route_gate.clone(); + let invoked = route_invoked.clone(); + dever_runtime::api::scoped_request(request, move |request| async move { + invoked.fetch_add(1, Ordering::SeqCst); + let path = request.target.split('?').next().unwrap().to_owned(); + let request_id = dever_runtime::api::request_id()?; + assert_eq!( + dever_runtime::api::client_address()?, + Some("127.0.0.1".into()) + ); + assert!(request.headers.iter().any(|header| { + header.name == "host" + && header.value.values() == format!("localhost:{port}").as_bytes() + })); + if request.target.starts_with("/slow") { + let _ = gate.acquire().await.map_err(error)?; + } + assert_eq!(dever_runtime::api::path()?, path); + assert_eq!(dever_runtime::api::request_id()?, request_id); + dever_runtime::api::response_header("x-api-path", &path)?; + let response = http::Response { + status: 200, + headers: vec![], + body: Bytes::from_string(request.target), + }; + dever_runtime::api::commit_response_metadata()?; + Ok(response) + }) + }, + owner, + http::Limits { + body_bytes: 16, + ..limits(500) + }, + )) + .await?; + let (sender, driver) = connect(port).await?; + let (sender, slow) = start_request(sender, request(port, "/slow?value=1")).await?; + let (sender, fast) = start_request(sender, request(port, "/fast")).await?; + let fast = tokio::time::timeout(Duration::from_millis(200), fast) + .await + .map_err(|_| "HTTP/2 streams were serialized".to_owned())? + .map_err(error)?; + assert_eq!(fast.status(), hyper::StatusCode::OK); + assert_eq!(fast.headers().get("x-api-path").unwrap(), "/fast"); + assert_eq!(response_body(fast).await?, b"/fast"); + gate.add_permits(1); + let slow = slow.await.map_err(error)?; + assert_eq!(slow.status(), hyper::StatusCode::OK); + assert_eq!(slow.headers().get("x-api-path").unwrap(), "/slow"); + assert_eq!(response_body(slow).await?, b"/slow?value=1"); + + let mismatched = hyper::Request::builder() + .version(hyper::Version::HTTP_2) + .uri(format!("http://localhost:{port}/mismatch")) + .header(hyper::header::HOST, "other.example") + .body(()) + .unwrap(); + let (sender, rejected) = start_request(sender, mismatched).await?; + let rejected = rejected.await.map_err(error)?; + assert_eq!(rejected.status(), hyper::StatusCode::BAD_REQUEST); + assert!(!rejected.headers().contains_key(hyper::header::CONNECTION)); + assert_eq!(response_body(rejected).await?, b""); + assert_eq!(invoked.load(Ordering::SeqCst), 2); + + let mut sender = sender.ready().await.map_err(error)?; + let oversized = hyper::Request::builder() + .method(hyper::Method::POST) + .version(hyper::Version::HTTP_2) + .uri(format!("http://localhost:{port}/oversized")) + .body(()) + .unwrap(); + let (oversized, mut upload) = sender.send_request(oversized, false).map_err(error)?; + upload + .send_data(bytes::Bytes::from_static(b"12345678901234567"), true) + .map_err(error)?; + drop(upload); + let oversized = oversized.await.map_err(error)?; + assert_eq!(oversized.status(), hyper::StatusCode::PAYLOAD_TOO_LARGE); + assert_eq!(response_body(oversized).await?, b""); + assert_eq!(invoked.load(Ordering::SeqCst), 2); + + let (sender, neighbor) = start_request(sender, request(port, "/neighbor")).await?; + assert_eq!( + response_body(neighbor.await.map_err(error)?).await?, + b"/neighbor" + ); + assert_eq!(invoked.load(Ordering::SeqCst), 3); + drop(sender); + task::wait(driver).await?; + listener.close()?; + task::wait(server).await + }); +} + +struct Finished(Arc<AtomicUsize>); + +impl Drop for Finished { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } +} + +struct Marked(Arc<AtomicBool>); + +impl Drop for Marked { + fn drop(&mut self) { + self.0.store(true, Ordering::SeqCst); + } +} + +#[test] +fn resetting_one_live_stream_keeps_sse_neighbor_and_drains_blocking_work() { + run(async { + let finished = Arc::new(AtomicUsize::new(0)); + let blocking_started = Arc::new(AtomicBool::new(false)); + let blocking_done = Arc::new(AtomicBool::new(false)); + let route_finished = finished.clone(); + let route_blocking_started = blocking_started.clone(); + let route_blocking_done = blocking_done.clone(); + let stop_gate = Arc::new(tokio::sync::Semaphore::new(0)); + let stop_started = Arc::new(AtomicBool::new(false)); + let listener = net::listen("127.0.0.1", 0).await?; + let port = net::port(&listener)?; + let owner = listener.clone(); + let owner_stop_gate = stop_gate.clone(); + let owner_stop_started = stop_started.clone(); + let server_owner = task::run(async move { + let server = task::run(http::serve_live( + move |request, reply| { + let finished = route_finished.clone(); + let blocking_started = route_blocking_started.clone(); + let blocking_done = route_blocking_done.clone(); + async move { + let _finished = Finished(finished.clone()); + match request.target.as_str() { + "/pending" => { + task::parallel(move || { + blocking_started.store(true, Ordering::SeqCst); + std::thread::sleep(Duration::from_millis(500)); + blocking_done.store(true, Ordering::SeqCst); + Ok(()) + }) + .await?; + std::future::pending().await + } + "/events" => { + sse::start(&reply, vec![]).await?; + sse::send( + &reply, + sse::Event { + event: "update".into(), + data: "neighbor".into(), + id: None, + retry_ms: None, + }, + ) + .await?; + std::future::pending().await + } + _ => { + http::respond( + &reply, + http::Response { + status: 200, + headers: vec![], + body: bytes(b"still-open"), + }, + ) + .await + } + } + } + }, + owner, + limits(1000), + live(), + )) + .await?; + let _ = owner_stop_gate.acquire().await.map_err(error)?; + owner_stop_started.store(true, Ordering::SeqCst); + task::stop(server).await + }) + .await?; + let (sender, driver) = connect(port).await?; + let (sender, pending) = start_request(sender, request(port, "/pending")).await?; + wait_until(|| blocking_started.load(Ordering::SeqCst)).await?; + drop(pending); + + let (sender, events) = start_request(sender, request(port, "/events")).await?; + let mut events = events.await.map_err(error)?.into_body(); + assert_eq!( + events.data().await.unwrap().map_err(error)?, + b"event: update\ndata: neighbor\n\n"[..] + ); + + let (sender, final_response) = start_request(sender, request(port, "/after-reset")).await?; + assert_eq!( + response_body(final_response.await.map_err(error)?).await?, + b"still-open" + ); + drop(events); + stop_gate.add_permits(1); + wait_until(|| stop_started.load(Ordering::SeqCst)).await?; + task::stop(server_owner).await?; + assert!(blocking_done.load(Ordering::SeqCst)); + drop(sender); + let _ = task::wait(driver).await; + listener.close()?; + assert!(finished.load(Ordering::SeqCst) >= 3); + Ok(()) + }); +} + +#[test] +fn request_faults_return_500_without_stopping_the_http2_connection() { + run(async { + let listener = net::listen("127.0.0.1", 0).await?; + let port = net::port(&listener)?; + let owner = listener.clone(); + let server = task::run(http::serve( + |request| async move { + if request.target == "/fault" { + return Err("buffered handler fault".to_owned()); + } + Ok(http::Response { + status: 200, + headers: vec![], + body: bytes(b"still-open"), + }) + }, + owner, + limits(500), + )) + .await?; + let (sender, driver) = connect(port).await?; + let (sender, response) = start_request(sender, request(port, "/fault")).await?; + let response = response.await.map_err(error)?; + assert_eq!(response.status(), hyper::StatusCode::INTERNAL_SERVER_ERROR); + let (sender, response) = start_request(sender, request(port, "/after-fault")).await?; + assert_eq!( + response_body(response.await.map_err(error)?).await?, + b"still-open" + ); + drop(sender); + task::stop(server).await?; + let _ = task::wait(driver).await; + listener.close()?; + + let listener = net::listen("127.0.0.1", 0).await?; + let port = net::port(&listener)?; + let owner = listener.clone(); + let server = task::run(http::serve_live( + |request, reply| async move { + if request.target == "/fault" { + return Err("live handler fault".to_owned()); + } + http::respond( + &reply, + http::Response { + status: 200, + headers: vec![], + body: bytes(b"still-open"), + }, + ) + .await + }, + owner, + limits(500), + live(), + )) + .await?; + let (sender, driver) = connect(port).await?; + let (sender, response) = start_request(sender, request(port, "/fault")).await?; + let response = response.await.map_err(error)?; + assert_eq!(response.status(), hyper::StatusCode::INTERNAL_SERVER_ERROR); + let (sender, response) = start_request(sender, request(port, "/after-fault")).await?; + assert_eq!( + response_body(response.await.map_err(error)?).await?, + b"still-open" + ); + drop(sender); + task::stop(server).await?; + let _ = task::wait(driver).await; + listener.close()?; + + let listener = net::listen("127.0.0.1", 0).await?; + let port = net::port(&listener)?; + let owner = listener.clone(); + let server = task::run(http::serve( + |request| async move { + if request.target != "/invalid-response" { + return Ok(http::Response { + status: 200, + headers: vec![], + body: bytes(b"still-open"), + }); + } + Ok(http::Response { + status: 200, + headers: vec![http::Header { + name: "te".into(), + value: bytes(b"trailers"), + }], + body: bytes(b""), + }) + }, + owner, + limits(500), + )) + .await?; + let (sender, driver) = connect(port).await?; + let (sender, response) = start_request(sender, request(port, "/invalid-response")).await?; + let response = response.await.map_err(error)?; + assert_eq!(response.status(), hyper::StatusCode::INTERNAL_SERVER_ERROR); + let (sender, response) = start_request(sender, request(port, "/after-invalid")).await?; + assert_eq!( + response_body(response.await.map_err(error)?).await?, + b"still-open" + ); + drop(sender); + task::stop(server).await?; + let _ = task::wait(driver).await; + listener.close()?; + Ok(()) + }); +} + +#[test] +fn flow_controlled_stream_does_not_block_its_neighbor_and_resumes_on_window_update() { + run(async { + let body_finished = Arc::new(AtomicBool::new(false)); + let reset_dropped = Arc::new(AtomicBool::new(false)); + let reset_writes = Arc::new(AtomicUsize::new(0)); + let route_finished = body_finished.clone(); + let route_reset_dropped = reset_dropped.clone(); + let route_reset_writes = reset_writes.clone(); + let listener = net::listen("127.0.0.1", 0).await?; + let port = net::port(&listener)?; + let owner = listener.clone(); + let server = task::run(http::serve_live( + move |request, reply| { + let body_finished = route_finished.clone(); + let reset_dropped = route_reset_dropped.clone(); + let reset_writes = route_reset_writes.clone(); + async move { + match request.target.as_str() { + "/large" => { + http::start(&reply, 200, vec![]).await?; + for _ in 0..4 { + http::write(&reply, Bytes::new(vec![7; 16384])).await?; + } + http::finish(&reply).await?; + body_finished.store(true, Ordering::SeqCst); + Ok(()) + } + "/reset-write" => { + let _dropped = Marked(reset_dropped); + http::start(&reply, 200, vec![]).await?; + for _ in 0..16 { + reset_writes.fetch_add(1, Ordering::SeqCst); + http::write(&reply, Bytes::new(vec![8; 16384])).await?; + } + http::finish(&reply).await + } + _ => { + http::respond( + &reply, + http::Response { + status: 200, + headers: vec![], + body: bytes(b"neighbor"), + }, + ) + .await + } + } + } + }, + owner, + limits(500), + http::LiveLimits { + chunk_bytes: 16384, + ..live() + }, + )) + .await?; + let (sender, driver) = connect_with_window(port, 1024).await?; + let (sender, large) = start_request(sender, request(port, "/large")).await?; + let mut large = large.await.map_err(error)?.into_body(); + let (sender, neighbor) = start_request(sender, request(port, "/neighbor")).await?; + let neighbor = tokio::time::timeout(Duration::from_millis(200), neighbor) + .await + .map_err(|_| "flow-controlled stream blocked its neighbor".to_owned())? + .map_err(error)?; + assert_eq!(response_body(neighbor).await?, b"neighbor"); + assert!(!body_finished.load(Ordering::SeqCst)); + + let mut received = 0; + let mut flow = large.flow_control().clone(); + while let Some(chunk) = large.data().await { + let chunk = chunk.map_err(error)?; + received += chunk.len(); + flow.release_capacity(chunk.len()).map_err(error)?; + } + assert_eq!(received, 4 * 16384); + wait_until(|| body_finished.load(Ordering::SeqCst)).await?; + drop(flow); + drop(large); + + let (sender, reset) = start_request(sender, request(port, "/reset-write")).await?; + let reset = reset.await.map_err(error)?.into_body(); + wait_until(|| reset_writes.load(Ordering::SeqCst) >= 3).await?; + tokio::time::sleep(Duration::from_millis(30)).await; + assert!(reset_writes.load(Ordering::SeqCst) < 16); + drop(reset); + wait_until(|| reset_dropped.load(Ordering::SeqCst)).await?; + let (sender, neighbor) = start_request(sender, request(port, "/after-reset-write")).await?; + assert_eq!( + response_body(neighbor.await.map_err(error)?).await?, + b"neighbor" + ); + + drop(sender); + task::wait(driver).await?; + listener.close()?; + task::wait(server).await + }); +} + +#[test] +fn listener_close_sends_goaway_and_drains_an_active_stream() { + run(async { + let entered = Arc::new(AtomicBool::new(false)); + let gate = Arc::new(tokio::sync::Semaphore::new(0)); + let route_entered = entered.clone(); + let route_gate = gate.clone(); + let listener = net::listen("127.0.0.1", 0).await?; + let port = net::port(&listener)?; + let owner = listener.clone(); + let server = task::run(http::serve( + move |_| { + let entered = route_entered.clone(); + let gate = route_gate.clone(); + async move { + entered.store(true, Ordering::SeqCst); + let _ = gate.acquire().await.map_err(error)?; + Ok(http::Response { + status: 200, + headers: vec![], + body: bytes(b"drained"), + }) + } + }, + owner, + limits(500), + )) + .await?; + let (sender, driver) = connect(port).await?; + let (mut sender, response) = start_request(sender, request(port, "/active")).await?; + wait_until(|| entered.load(Ordering::SeqCst)).await?; + listener.close()?; + tokio::time::timeout(Duration::from_millis(500), async { + loop { + if poll_fn(|context| sender.poll_ready(context)).await.is_err() { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .map_err(|_| "client did not observe server GOAWAY".to_owned())?; + gate.add_permits(1); + assert_eq!( + response_body(response.await.map_err(error)?).await?, + b"drained" + ); + drop(sender); + task::wait(driver).await?; + task::wait(server).await + }); +} + +#[test] +fn silent_peer_times_out_and_tls_negotiates_h2() { + run(async { + let listener = net::listen("127.0.0.1", 0).await?; + let port = net::port(&listener)?; + let owner = listener.clone(); + let server = task::run(http::serve( + |_| async { + Ok(http::Response { + status: 200, + headers: vec![], + body: bytes(b"ok"), + }) + }, + owner, + limits(80), + )) + .await?; + let mut silent = tokio::net::TcpStream::connect(("127.0.0.1", port as u16)) + .await + .map_err(error)?; + tokio::time::sleep(Duration::from_millis(160)).await; + let mut received = Vec::new(); + tokio::time::timeout( + Duration::from_millis(500), + silent.read_to_end(&mut received), + ) + .await + .map_err(|_| "silent HTTP/2 peer was not closed after its deadline".to_owned())? + .map_err(error)?; + let (sender, driver) = connect(port).await?; + let (sender, response) = start_request(sender, request(port, "/after-timeout")).await?; + assert_eq!(response_body(response.await.map_err(error)?).await?, b"ok"); + drop(sender); + task::wait(driver).await?; + listener.close()?; + task::wait(server).await?; + + let listener = net::listen("127.0.0.1", 0).await?; + let port = net::port(&listener)?; + let owner = listener.clone(); + let server = task::run(http::serve_tls( + |_| async { + Ok(http::Response { + status: 200, + headers: vec![], + body: bytes(b"tls"), + }) + }, + owner, + limits(500), + tls::server(&bytes(CERT), &bytes(KEY))?, + )) + .await?; + let socket = tokio::net::TcpStream::connect(("127.0.0.1", port as u16)) + .await + .map_err(error)?; + let mut roots = RootCertStore::empty(); + for certificate in CertificateDer::pem_slice_iter(ROOT) { + roots.add(certificate.map_err(error)?).map_err(error)?; + } + let mut config = rustls::ClientConfig::builder() + .with_root_certificates(roots) + .with_no_client_auth(); + config.alpn_protocols = vec![b"h2".to_vec()]; + let name = ServerName::try_from("localhost".to_owned()).map_err(error)?; + let stream = tokio_rustls::TlsConnector::from(Arc::new(config)) + .connect(name, socket) + .await + .map_err(error)?; + assert_eq!(stream.get_ref().1.alpn_protocol(), Some(b"h2".as_slice())); + let (sender, connection) = h2::client::handshake(stream).await.map_err(error)?; + let driver = task::run(async move { connection.await.map_err(error) }).await?; + let (sender, response) = start_request(sender, request(port, "/tls")).await?; + assert_eq!(response_body(response.await.map_err(error)?).await?, b"tls"); + drop(sender); + task::wait(driver).await?; + listener.close()?; + task::wait(server).await + }); +} diff --git a/test/dever-tests/tests/http_engine.rs b/test/dever-tests/tests/http_engine.rs new file mode 100644 index 0000000..55e4b6d --- /dev/null +++ b/test/dever-tests/tests/http_engine.rs @@ -0,0 +1,621 @@ +use std::future::Future; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::time::Duration; + +use dever_runtime::{bytes::Bytes, channel::Channel, http, net, task}; + +fn limits() -> http::Limits { + http::Limits { + header_bytes: 8192, + body_bytes: 1024, + timeout_ms: 1000, + connections: 4, + http2: None, + } +} + +fn run_test(test: impl Future<Output = Result<(), String>>) { + task::run_entry_with( + task::RuntimeConfig { + worker_threads: 1, + max_blocking_threads: 2, + task_capacity: 32, + }, + async { + tokio::time::timeout(Duration::from_secs(8), test) + .await + .map_err(|_| "HTTP test deadline exceeded".to_owned())? + }, + ) + .unwrap(); +} + +fn header(name: &str, value: &str) -> http::Header { + http::Header { + name: name.into(), + value: Bytes::from_text(value), + } +} + +fn request() -> http::Request { + http::Request { + method: "GET".into(), + target: "/".into(), + headers: vec![header("host", "test")], + body: Bytes::new(vec![]), + } +} + +fn response(body: Bytes) -> http::Response { + http::Response { + status: 200, + headers: vec![], + body, + } +} + +async fn server<F, Fut>( + route: F, + limits: http::Limits, +) -> Result<(task::Task<()>, net::Listener, i64), String> +where + F: Fn(http::Request) -> Fut + Clone + Send + Sync + 'static, + Fut: Future<Output = Result<http::Response, String>> + Send + 'static, +{ + let listener = net::listen("127.0.0.1", 0).await?; + let port = net::port(&listener)?; + let owned = listener.clone(); + let task = task::run(async move { http::serve(route, owned, limits).await }).await?; + Ok((task, listener, port)) +} + +// 独立、只读的测试线协议观察器：按 Content-Length 保留下一条响应，不调用 Hyper。 +struct Peer { + socket: net::Socket, + buffered: Vec<u8>, +} + +impl Peer { + async fn connect(port: i64) -> Result<Self, String> { + let socket = net::connect_timeout("127.0.0.1", port, 2000).await?; + net::timeout(&socket, 2000)?; + Ok(Self { + socket, + buffered: vec![], + }) + } + + async fn write(&self, wire: &[u8]) -> Result<(), String> { + net::write(&self.socket, &Bytes::new(wire.to_vec())).await + } + + async fn read_more(&mut self) -> Result<(), String> { + let bytes = net::read(&self.socket, 4096) + .await? + .ok_or("unexpected HTTP EOF")?; + self.buffered.extend_from_slice(bytes.values()); + Ok(()) + } + + async fn response(&mut self, head_only: bool) -> Result<(String, Vec<u8>), String> { + let end = loop { + if let Some(end) = self + .buffered + .windows(4) + .position(|part| part == b"\r\n\r\n") + { + break end + 4; + } + self.read_more().await?; + }; + let headers = String::from_utf8(self.buffered[..end].to_vec()).unwrap(); + let length = if head_only { + 0 + } else { + headers + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::<usize>().unwrap()) + }) + .unwrap_or(0) + }; + while self.buffered.len() < end + length { + self.read_more().await?; + } + let body = self.buffered[end..end + length].to_vec(); + self.buffered.drain(..end + length); + Ok((headers.to_ascii_lowercase(), body)) + } +} + +#[test] +fn persistent_pipeline_preserves_chunked_binary_body_and_duplicate_headers() { + run_test(async { + let (server, listener, port) = server( + |request| async move { + if request.target == "/first" { + assert_eq!(request.body.values(), &[0, 255, b'x']); + let values: Vec<_> = request + .headers + .iter() + .filter(|header| header.name == "x-many") + .map(|header| header.value.values()) + .collect(); + assert_eq!(values, [b"one".as_slice(), &[0x80]]); + } + Ok(http::Response { + status: 200, + headers: vec![header("set-cookie", "a=1"), header("set-cookie", "b=2")], + body: request.body, + }) + }, + limits(), + ) + .await?; + let mut peer = Peer::connect(port).await?; + peer.write(b"POST /first HTTP/1.1\r\nHost: test\r\nX-Many: one\r\nX-Many: \x80\r\nTransfer-Encoding: chunked\r\n\r\n2\r\n\0\xff\r\n1\r\nx\r\n0\r\n\r\nPOST /next HTTP/1.1\r\nHost: test\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok").await?; + let (headers, body) = peer.response(false).await?; + assert!(headers.starts_with("http/1.1 200")); + assert_eq!(headers.matches("set-cookie:").count(), 2); + assert_eq!(body, [0, 255, b'x']); + assert_eq!(peer.response(false).await?.1, b"ok"); + assert_eq!(net::read(&peer.socket, 1).await?, None); + task::stop(server).await?; + listener.close()?; + Ok(()) + }); +} + +#[test] +fn head_and_no_content_responses_keep_next_message_aligned() { + run_test(async { + let (server, listener, port) = server( + |request| async move { + let mut result = response(Bytes::from_text("head representation")); + if request.target == "/empty" { + result.status = 204; + result.body = Bytes::new(vec![]); + } + if request.target == "/reset" { + result.status = 205; + result.body = Bytes::new(vec![]); + } + if request.target == "/cached" { + result.status = 304; + result.body = Bytes::new(vec![]); + } + Ok(result) + }, + limits(), + ) + .await?; + let mut peer = Peer::connect(port).await?; + for (wire, status, head) in [ + ( + b"HEAD / HTTP/1.1\r\nHost: test\r\n\r\n".as_slice(), + "200", + true, + ), + (b"GET /empty HTTP/1.1\r\nHost: test\r\n\r\n", "204", false), + (b"GET /reset HTTP/1.1\r\nHost: test\r\n\r\n", "205", false), + (b"GET /cached HTTP/1.1\r\nHost: test\r\n\r\n", "304", false), + ] { + peer.write(wire).await?; + let (headers, body) = peer.response(head).await?; + assert!( + headers.starts_with(&format!("http/1.1 {status}")), + "{headers}" + ); + assert!(body.is_empty()); + } + task::stop(server).await?; + listener.close()?; + Ok(()) + }); +} + +#[test] +fn invalid_peers_and_limits_do_not_reach_the_handler_or_stop_the_server() { + run_test(async { + let calls = Arc::new(AtomicUsize::new(0)); + let observed = calls.clone(); + let mut budget = limits(); + budget.body_bytes = 4; + let (server, listener, port) = server( + move |_| { + observed.fetch_add(1, Ordering::SeqCst); + async { Ok(response(Bytes::new(vec![]))) } + }, + budget, + ) + .await?; + let oversized = format!( + "GET / HTTP/1.1\r\nHost: test\r\nX-Large: {}\r\n\r\n", + "a".repeat(9000) + ); + for (wire, status) in [ + (b"POST / HTTP/1.1\r\nHost: test\r\nContent-Length: 1\r\nContent-Length: 2\r\n\r\nx".as_slice(), "400"), + (b"GET / HTTP/1.1\r\nHost: a\r\nHost: b\r\n\r\n", "400"), + (b"POST / HTTP/1.1\r\nHost: test\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nabcde\r\n0\r\n\r\n", "413"), + (b"POST / HTTP/1.1\r\nHost: test\r\nContent-Length: 5\r\n\r\n", "413"), + (b"POST / HTTP/1.1\r\nHost: test\r\nTransfer-Encoding: chunked\r\n\r\n0\r\nX-Trailer: yes\r\n\r\n", "501"), + (b"CONNECT test:80 HTTP/1.1\r\nHost: test\r\n\r\n", "501"), + (b"GET / HTTP/1.1\r\nHost: test\r\nConnection: upgrade\r\nUpgrade: websocket\r\n\r\n", "501"), + (oversized.as_bytes(), "431"), + ] { + let mut peer = Peer::connect(port).await?; + peer.write(wire).await?; + let (headers, _) = peer.response(false).await?; + assert!(headers.starts_with(&format!("http/1.1 {status}")), "{headers}"); + } + assert_eq!(calls.load(Ordering::SeqCst), 0); + assert_eq!( + http::send("127.0.0.1", port, request(), limits()) + .await? + .status, + 200 + ); + assert_eq!(calls.load(Ordering::SeqCst), 1); + task::stop(server).await?; + listener.close()?; + Ok(()) + }); +} + +struct Dropped(Arc<AtomicBool>); +impl Drop for Dropped { + fn drop(&mut self) { + self.0.store(true, Ordering::SeqCst); + } +} + +#[test] +fn stop_closes_active_connections_and_drains_handler_descendants() { + run_test(async { + let entered = Channel::new(1)?; + let notify = entered.clone(); + let dropped = Arc::new(AtomicBool::new(false)); + let observed = dropped.clone(); + let (server, listener, port) = server( + move |_| { + let notify = notify.clone(); + let marker = Dropped(observed.clone()); + async move { + let child = task::run(async move { + let _marker = marker; + notify.send(()).await?; + std::future::pending::<()>().await; + Ok(()) + }) + .await?; + task::wait(child).await?; + Ok(response(Bytes::new(vec![]))) + } + }, + limits(), + ) + .await?; + let peer = Peer::connect(port).await?; + peer.write(b"GET / HTTP/1.1\r\nHost: test\r\n\r\n").await?; + entered.receive().await?; + task::stop(server).await?; + assert!(dropped.load(Ordering::SeqCst)); + assert_eq!(net::read(&peer.socket, 1).await?, None); + // serve 只释放自己的 listener 别名，调用者仍可决定资源生命周期。 + assert_eq!(net::port(&listener)?, port); + listener.close()?; + Ok(()) + }); +} + +#[test] +fn handler_fault_returns_500_and_keeps_the_accept_loop_running() { + run_test(async { + let (server, listener, port) = server( + |request| async move { + if request.target == "/fault" { + return Err("route fault".into()); + } + Ok(response(Bytes::from_text("healthy"))) + }, + limits(), + ) + .await?; + let mut peer = Peer::connect(port).await?; + peer.write(b"GET /fault HTTP/1.1\r\nHost: test\r\n\r\n") + .await?; + let (headers, body) = peer.response(false).await?; + assert!(headers.starts_with("http/1.1 500"), "{headers}"); + assert!(body.is_empty()); + let healthy = http::send("127.0.0.1", port, request(), limits()).await?; + assert_eq!(healthy.status, 200); + assert_eq!(healthy.body.values(), b"healthy"); + task::stop(server).await?; + listener.close()?; + Ok(()) + }); +} + +#[test] +fn connection_capacity_backpressures_before_dispatching_another_request() { + run_test(async { + let entered = Channel::new(2)?; + let release = Channel::new(2)?; + let notify = entered.clone(); + let wait = release.clone(); + let calls = Arc::new(AtomicUsize::new(0)); + let observed = calls.clone(); + let mut budget = limits(); + budget.connections = 1; + let (server, listener, port) = server( + move |_| { + let notify = notify.clone(); + let wait = wait.clone(); + let observed = observed.clone(); + async move { + observed.fetch_add(1, Ordering::SeqCst); + notify.send(()).await?; + wait.receive().await?; + Ok(response(Bytes::new(vec![]))) + } + }, + budget, + ) + .await?; + let mut first = Peer::connect(port).await?; + first + .write(b"GET / HTTP/1.1\r\nHost: test\r\nConnection: close\r\n\r\n") + .await?; + entered.receive().await?; + let mut second = Peer::connect(port).await?; + second + .write(b"GET / HTTP/1.1\r\nHost: test\r\nConnection: close\r\n\r\n") + .await?; + tokio::task::yield_now().await; + assert_eq!(calls.load(Ordering::SeqCst), 1); + release.send(()).await?; + first.response(false).await?; + entered.receive().await?; + assert_eq!(calls.load(Ordering::SeqCst), 2); + release.send(()).await?; + second.response(false).await?; + task::stop(server).await?; + listener.close()?; + Ok(()) + }); +} + +#[test] +fn header_body_and_handler_deadlines_release_the_connection() { + run_test(async { + let mut budget = limits(); + budget.timeout_ms = 100; + let (server, listener, port) = server( + |_| async { + std::future::pending::<()>().await; + Ok(response(Bytes::new(vec![]))) + }, + budget, + ) + .await?; + let peer = Peer::connect(port).await?; + peer.write(b"GET / HTTP/1.1\r\nHost:").await?; + // Hyper 对未完成的请求头超时直接关闭连接，没有可交给 handler 的请求。 + assert_eq!(net::read(&peer.socket, 1).await?, None); + for (wire, expected) in [ + ( + b"POST / HTTP/1.1\r\nHost: test\r\nContent-Length: 2\r\n\r\nx".as_slice(), + "408", + ), + (b"GET / HTTP/1.1\r\nHost: test\r\n\r\n", "504"), + ] { + let mut peer = Peer::connect(port).await?; + peer.write(wire).await?; + let (headers, _) = peer.response(false).await?; + assert!( + headers.starts_with(&format!("http/1.1 {expected}")), + "{headers}" + ); + assert_eq!(net::read(&peer.socket, 1).await?, None); + } + task::stop(server).await?; + listener.close()?; + Ok(()) + }); +} + +#[test] +fn client_accepts_chunked_and_eof_bodies_but_rejects_truncation_and_oversize() { + run_test(async { + for (wire, expected) in [ + (b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nSet-Cookie: a=1\r\nSet-Cookie: b=2\r\n\r\n2\r\nok\r\n0\r\n\r\n".as_slice(), true), + (b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\nok", true), + (b"HTTP/1.1 200 OK\r\nContent-Length: 3\r\n\r\nok", false), + (b"HTTP/1.1 200 OK\r\nContent-Length: 2048\r\n\r\n", false), + ] { + let listener = net::listen("127.0.0.1", 0).await?; + let port = net::port(&listener)?; + let peer = task::run(async move { + let socket = net::accept(&listener).await?; + net::timeout(&socket, 1000)?; + let sent = net::read(&socket, 4096).await?.unwrap().to_text()?; + assert!(sent.starts_with("GET / HTTP/1.1\r\n")); + assert!(sent.contains("host: test\r\n")); + for piece in wire.chunks(7) { net::write(&socket, &Bytes::new(piece.to_vec())).await?; tokio::task::yield_now().await; } + Ok(()) + }).await?; + let received = http::send("127.0.0.1", port, request(), limits()).await; + if expected { assert_eq!(received?.body.values(), b"ok"); } else { assert!(received.is_err()); } + task::wait(peer).await?; + } + Ok(()) + }); +} + +#[test] +fn invalid_outgoing_messages_and_configuration_fail_before_connecting() { + run_test(async { + for invalid in [ + header("x-test", "ok\r\ninjected: yes"), + header("content-length", "0"), + header("transfer-encoding", "chunked"), + ] { + let mut request = request(); + request.headers.push(invalid); + let error = http::send("127.0.0.1", 0, request, limits()) + .await + .unwrap_err(); + assert!(!error.contains("Connection refused"), "{error}"); + } + let mut excessive = request(); + excessive.headers.extend(vec![header("x", ""); 32769]); + assert_eq!( + http::send("127.0.0.1", 0, excessive, limits()) + .await + .unwrap_err(), + "HTTP headers exceed header_bytes" + ); + for budget in [ + http::Limits { + header_bytes: 1, + ..limits() + }, + http::Limits { + body_bytes: -1, + ..limits() + }, + http::Limits { + timeout_ms: 0, + ..limits() + }, + http::Limits { + connections: 0, + ..limits() + }, + ] { + assert!( + http::send("127.0.0.1", 0, request(), budget) + .await + .unwrap_err() + .starts_with("HTTP") + ); + } + Ok(()) + }); +} + +#[test] +fn invalid_handler_responses_return_500_and_keep_the_accept_loop_running() { + run_test(async { + for status in [99, 204, 205, 304, 600] { + let (server, listener, port) = server( + move |request| async move { + if request.target != "/invalid" { + return Ok(response(Bytes::from_text("healthy"))); + } + Ok(http::Response { + status, + headers: vec![], + body: Bytes::from_text("invalid body"), + }) + }, + limits(), + ) + .await?; + let mut peer = Peer::connect(port).await?; + peer.write(b"GET /invalid HTTP/1.1\r\nHost: test\r\n\r\n") + .await?; + let (headers, body) = peer.response(false).await?; + assert_eq!(headers.split_whitespace().nth(1), Some("500"), "{headers}"); + assert!(headers.lines().any(|line| line == "connection: close")); + assert!(body.is_empty()); + assert!(peer.buffered.is_empty()); + assert_eq!(net::read(&peer.socket, 1).await?, None); + let healthy = http::send("127.0.0.1", port, request(), limits()).await?; + assert_eq!(healthy.status, 200); + assert_eq!(healthy.body.values(), b"healthy"); + listener.close()?; + task::wait(server).await?; + } + Ok(()) + }); +} + +#[test] +fn client_timeout_and_stop_release_the_owned_connection_driver() { + run_test(async { + for stop in [false, true] { + let listener = net::listen("127.0.0.1", 0).await?; + let port = net::port(&listener)?; + let received = Channel::new(1)?; + let notify = received.clone(); + let peer = task::run(async move { + let socket = net::accept(&listener).await?; + net::timeout(&socket, 2000)?; + assert!(net::read(&socket, 4096).await?.is_some()); + notify.send(()).await?; + assert_eq!(net::read(&socket, 1).await?, None); + Ok(()) + }) + .await?; + let mut budget = limits(); + budget.timeout_ms = if stop { 1000 } else { 100 }; + let client = task::run(async move { + let result = http::send("127.0.0.1", port, request(), budget).await; + assert_eq!(result.unwrap_err(), "HTTP request timed out"); + Ok(()) + }) + .await?; + received.receive().await?; + if stop { + task::stop(client).await?; + } else { + task::wait(client).await?; + } + task::wait(peer).await?; + } + Ok(()) + }); +} + +#[test] +fn a_stalled_writer_releases_its_connection_slot() { + run_test(async { + let mut budget = limits(); + budget.timeout_ms = 100; + budget.connections = 1; + budget.body_bytes = 16 * 1024 * 1024; + let entered = Channel::new(1)?; + let notify = entered.clone(); + let (server, listener, port) = server( + move |request| { + let notify = notify.clone(); + async move { + let body = if request.target == "/slow" { + notify.send(()).await?; + Bytes::new(vec![b'x'; 16 * 1024 * 1024]) + } else { + Bytes::from_text("next") + }; + Ok(response(body)) + } + }, + budget, + ) + .await?; + let slow = Peer::connect(port).await?; + slow.write(b"GET /slow HTTP/1.1\r\nHost: test\r\nConnection: close\r\n\r\n") + .await?; + entered.receive().await?; + let mut next = Peer::connect(port).await?; + next.write(b"GET /next HTTP/1.1\r\nHost: test\r\nConnection: close\r\n\r\n") + .await?; + assert_eq!(next.response(false).await?.1, b"next"); + task::stop(server).await?; + listener.close()?; + Ok(()) + }); +} diff --git a/test/dever-tests/tests/http_library.rs b/test/dever-tests/tests/http_library.rs new file mode 100644 index 0000000..5f60981 --- /dev/null +++ b/test/dever-tests/tests/http_library.rs @@ -0,0 +1,246 @@ +mod support; + +use std::ffi::OsString; +use std::io::{Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::process::Command; +use std::time::{Duration, Instant}; + +use support::{checked, rejected, sources, stdout}; + +const ROUTE: &str = r#" +route(request: dever.http.Request) (response: dever.http.Response) recover("route policy") { + response = dever.http.Response { status = 201 +headers = [dever.http.header("Set-Cookie", "a=1"), dever.http.header("Set-Cookie", "b=2")] +body = request.body } +} +"#; + +const LOCAL_SERVER: &str = r#" +exchange(result: dever.net.PortResult.Bound(port), listener: dever.system.Listener) (answer: Bool) recover("test reports network failures") { + server = run(dever.http.serve(route, listener, dever.http.default_limits())) + request = dever.http.Request { method = "POST" +target = "/echo" +headers = [dever.http.header("Host", "test")] +body = dever.bytes.from_text("hello") } + answer = received(result(dever.http.send("127.0.0.1", port, request, dever.http.default_limits()))) + stop(server) + dever.net.close_listener(listener) +} +exchange(result: dever.net.PortResult.Failed(message), listener: dever.system.Listener) (answer: Bool) recover("test reports network failures") { answer = false } +listening(result: dever.net.ListenResult.Listening(listener)) (answer: Bool) recover("test reports network failures") { + answer = exchange(result(dever.net.port(listener)), listener) +} +listening(result: dever.net.ListenResult.Failed(message)) (answer: Bool) recover("test reports network failures") { answer = false } +received(result: dever.http.ResponseResult.Read(response)) (answer: Bool) recover("test reports network failures") { + answer = response.status == 201 and response.body == dever.bytes.from_text("hello") and length(response.headers) >= 2 and cookie(dever.http.header_value(response.headers, "SET-COOKIE")) +} +received(result: dever.http.ResponseResult.Failed(message)) (answer: Bool) recover("test reports network failures") { answer = false } +cookie(value: null) (answer: Bool) { answer = false } +cookie(value: Bytes) (answer: Bool) { answer = value == dever.bytes.from_text("a=1") } +main() (answer: Bool) { answer = listening(result(dever.net.listen("127.0.0.1", 0))) } +"#; + +fn local_source(http2: bool) -> String { + let protocol = if http2 { + "dever.http.default_http2_limits()" + } else { + "null" + }; + let server = LOCAL_SERVER.replace("dever.http.default_limits()", "http_limits()"); + format!( + r#"{ROUTE} +{server} +http_limits() (limits: dever.http.Limits) {{ + limits = dever.http.default_limits() + limits.http2 = {protocol} +}} +"# + ) +} + +#[test] +fn native_http_uses_static_handlers_and_runs_under_structured_tasks() { + let source = local_source(false); + let map = sources(&source); + let program = checked(&map); + let generated = dever_core::native::emit(&program, &map, "main.main").unwrap(); + assert!(generated.contains("dever_runtime::http::serve(")); + assert!(generated.contains("dever_runtime::http::send(")); + assert!(!generated.contains("Box<dyn Fn")); + assert_eq!(stdout(&source), "answer = true\n"); +} + +#[test] +fn native_http2_buffered_server_and_single_use_client() { + assert_eq!(stdout(&local_source(true)), "answer = true\n"); +} + +#[test] +fn handler_contracts_and_forwarded_effects_reach_the_http_boundary() { + let shared = format!( + r#"{ROUTE} +public start(listener: dever.system.Listener) () {{ + dever.http.serve(route, listener, dever.http.default_limits()) +}} +"# + ); + let program = checked(&sources(&shared)); + let api = program.api_snapshot(); + let start = api + .lines() + .find(|line| line.starts_with("function main.start")) + .unwrap(); + assert!( + start.contains("network") + && start.contains("concurrency") + && start.contains("route policy"), + "{start}" + ); + rejected( + &shared + .replace( + "(response: dever.http.Response)", + "(wrong: dever.http.Response)", + ) + .replace(" response =", " wrong ="), + "handler signature", + ); + rejected( + &shared.replace( + "response = dever.http.Response", + "dever.io.println(\"blocking\")\nresponse = dever.http.Response", + ), + "cannot run inline", + ); +} + +#[test] +fn intrinsic_handlers_participate_in_recursion_checks() { + rejected( + r#"public route(request: dever.http.Request) (response: dever.http.Response) { + start(null) + response = dever.http.Response { status = 200 +headers = [] +body = request.body } +} +start(unused: Text?) () { + open(result(dever.net.listen("127.0.0.1", 0))) +} +open(result: dever.net.ListenResult.Listening(listener)) () recover("test observes failure") { dever.http.serve(route, listener, dever.http.default_limits()) } +open(result: dever.net.ListenResult.Failed(message)) () recover("test observes failure") {} +"#, + "recursive", + ); +} + +#[test] +fn header_helpers_preserve_duplicates_bytes_and_case_insensitive_lookup() { + assert_eq!( + stdout( + r#"public main() (found: Bytes?, missing: Bytes?, count: Int) { + headers = [dever.http.header("X-Key", "one"), dever.http.header("X-Key", "two")] + found = dever.http.header_value(headers, "x-KEY") + missing = dever.http.header_value(headers, "absent") + count = length(headers) +}"# + ), + "found = [111, 110, 101]\nmissing = null\ncount = 2\n" + ); +} + +fn compile(source: &str) -> dever_core::native::NativeProgram { + let sources = sources(source); + let rustc = std::env::var_os("RUSTC").unwrap_or_else(|| OsString::from("rustc")); + dever_core::native::compile(&checked(&sources), &sources, "main.main", &rustc).unwrap() +} + +fn accept_before(listener: &TcpListener) -> TcpStream { + let deadline = Instant::now() + Duration::from_secs(5); + loop { + match listener.accept() { + Ok((socket, _)) => return socket, + Err(error) + if error.kind() == std::io::ErrorKind::WouldBlock && Instant::now() < deadline => + { + std::thread::sleep(Duration::from_millis(5)) + } + Err(error) => panic!("test peer accept failed: {error}"), + } + } +} + +#[test] +fn native_client_matches_independent_wire_and_retains_repeated_response_headers() { + let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + listener.set_nonblocking(true).unwrap(); + let port = listener.local_addr().unwrap().port(); + let native = compile(&format!( + r#"received(result: dever.http.ResponseResult.Read(response)) (answer: Bool) recover("test reports wire failure") {{ + answer = response.status == 201 and response.body == dever.bytes.from_text("abc") and length(response.headers) == 3 +}} +received(result: dever.http.ResponseResult.Failed(message)) (answer: Bool) recover("test reports wire failure") {{ answer = false }} +public main() (answer: Bool) {{ + request = dever.http.Request {{ method = "POST" +target = "/echo" +headers = [dever.http.header("Host", "example.test")] +body = dever.bytes.from_text("x") }} + answer = received(result(dever.http.send("127.0.0.1", {port}, request, dever.http.default_limits()))) +}} +"# + )); + let peer = std::thread::spawn(move || { + let mut socket = accept_before(&listener); + socket + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + socket + .set_write_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let expected = b"POST /echo HTTP/1.1\r\nhost: example.test\r\ncontent-length: 1\r\n\r\nx"; + let mut wire = vec![0; expected.len()]; + socket.read_exact(&mut wire).unwrap(); + assert_eq!(wire, expected); + for piece in [ + b"HTTP/1.1 201 Created\r\nTransfer-Encoding: chunked\r\nSet-Cookie: a=1\r\n".as_slice(), + b"Set-Cookie: b=2\r\n\r\n1\r\na\r\n", + b"2\r\nbc\r\n0\r\n\r\n", + ] { + socket.write_all(piece).unwrap(); + std::thread::sleep(Duration::from_millis(5)); + } + }); + let output = Command::new(native.executable()).output().unwrap(); + peer.join().unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(String::from_utf8(output.stdout).unwrap(), "answer = true\n"); +} + +#[test] +fn http_and_server_examples_pass_checking_and_format_roundtrips() { + for (path, check_source) in [ + ("library/dever/http.dever", false), + ("examples/old/dever/http/module/main.dever", true), + ] { + let disk_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../..") + .join(path); + let source = std::fs::read_to_string(disk_path).unwrap(); + let mut map = dever_core::source::SourceMap::default(); + let id = map.add(path, source); + let first = dever_core::format::format(map.get(id)).unwrap(); + let mut formatted = dever_core::source::SourceMap::default(); + let id = formatted.add("main.dever", first.clone()); + assert_eq!( + dever_core::format::format(formatted.get(id)).unwrap(), + first + ); + if check_source { + checked(&formatted); + } + } +} diff --git a/test/dever-tests/tests/json_library.rs b/test/dever-tests/tests/json_library.rs new file mode 100644 index 0000000..9d9597d --- /dev/null +++ b/test/dever-tests/tests/json_library.rs @@ -0,0 +1,265 @@ +mod support; + +fn source_text(value: &str) -> String { + format!("{value:?}").replace("\\0", "\\u{0}") +} + +const ROUNDTRIP: &str = r#" +roundtrip(source: Text) (result: Text) { + result = parsed(result(dever.json.parse(source))) +} +parsed(result: dever.json.value.ParseResult.Parsed(document)) (text: Text) recover("test observes parsed failures in its assertions") { + text = written(result(dever.json.stringify(document))) +} +parsed(result: dever.json.value.ParseResult.Failed(position, message)) (text: Text) recover("test observes parsed failures in its assertions") { + text = "ERROR " + int.to_text(position) + ": " + message +} +written(result: dever.json.value.WriteResult.Written(value)) (text: Text) recover("test observes written failures in its assertions") { text = value } +written(result: dever.json.value.WriteResult.Failed(node, message)) (text: Text) recover("test observes written failures in its assertions") { + text = "INVALID " + int.to_text(node) + ": " + message +} +show(source: Text) () { dever.io.println(roundtrip(source)) } +"#; + +fn roundtrips(values: &[&str]) -> Vec<String> { + let literals = values + .iter() + .map(|value| source_text(value)) + .collect::<Vec<_>>() + .join(", "); + let source = format!("{ROUNDTRIP}\npublic main() () {{each(show, [{literals}])}}\n"); + support::stdout(&source) + .lines() + .map(str::to_owned) + .collect() +} + +#[test] +fn json_sources_pass_the_same_checker_as_application_packages() { + support::checked(&support::sources("public main() () {}\n")); +} + +#[test] +fn json_values_unicode_and_exact_numbers_roundtrip_natively() { + let cases = [ + ("null", "null"), + (" \t\r\ntrue\n", "true"), + ("false", "false"), + ("-0", "-0"), + ("1.2300e+40000", "1.2300e+40000"), + ( + "12345678901234567890123456789012345678901234567890", + "12345678901234567890123456789012345678901234567890", + ), + ( + r#""\"\\\/\b\f\n\r\t\u0000\u001f""#, + r#""\"\\/\b\f\n\r\t\u0000\u001f""#, + ), + (r#""\uD834\uDD1E\ud83d\ude00中文""#, "\"𝄞😀中文\""), + ( + r#" { "z": [ 1, null, { "empty": [] } ], "a": {}, "": "value" } "#, + r#"{"z":[1,null,{"empty":[]}],"a":{},"":"value"}"#, + ), + ( + "[[],{},true,false,null,0,-12.5E-7]", + "[[],{},true,false,null,0,-12.5E-7]", + ), + ]; + let output = roundtrips(&cases.iter().map(|(source, _)| *source).collect::<Vec<_>>()); + assert_eq!( + output, + cases + .iter() + .map(|(_, expected)| *expected) + .collect::<Vec<_>>() + ); + let second = roundtrips(&output.iter().map(String::as_str).collect::<Vec<_>>()); + assert_eq!(second, output); +} + +#[test] +fn malformed_json_is_rejected_without_default_values() { + let cases = [ + "", + " ", + "+1", + "01", + "-01", + "-", + ".1", + "1.", + "1e", + "1e+", + "NaN", + "Infinity", + "tru", + "True", + "nul", + "true false", + "nullx", + "[]x", + "[", + "{", + "[1,]", + "[,1]", + "[1 2]", + "{,}", + "{a:1}", + "{\"a\"}", + "{\"a\":}", + "{\"a\":1,}", + "{\"a\":1 \"b\":2}", + "[}", + "\"", + "\"\\", + "\"line\nbreak\"", + "\"\0\"", + r#""\x20""#, + r#""\u12""#, + r#""\uXY00""#, + r#""\ud800""#, + r#""\udc00""#, + r#""\ud800x""#, + r#""\ud800\n""#, + r#""\ud800\u0041""#, + r#"{"a":1,"a":2}"#, + r#"{"a":1,"\u0061":2}"#, + "\u{feff}null", + "[1\u{a0},2]", + ]; + let output = roundtrips(&cases); + for (source, result) in cases.iter().zip(&output) { + assert!( + result.starts_with("ERROR "), + "{source:?} unexpectedly produced {result:?}" + ); + } + assert!(output[output.len() - 4].contains("duplicate JSON object name")); + assert!(output[output.len() - 3].contains("duplicate JSON object name")); +} + +#[test] +fn nesting_limit_and_unicode_error_offsets_are_explicit() { + let allowed = format!("{}0{}", "[".repeat(256), "]".repeat(256)); + let exceeded = format!("{}0{}", "[".repeat(257), "]".repeat(257)); + let output = roundtrips(&[&allowed, &exceeded, "[\"😀\",]", "{\"a\":"]); + assert_eq!(output[0], allowed); + assert_eq!(output[1], "ERROR 256: JSON nesting exceeds 256 containers"); + assert_eq!(output[2], "ERROR 5: expected a JSON value"); + assert_eq!(output[3], "ERROR 5: unterminated JSON container"); +} + +#[test] +fn constructed_documents_are_validated_and_can_use_any_map_insertion_order() { + let source = format!( + r#"{ROUNDTRIP} +show_document(document: dever.json.value.Document) () {{ + dever.io.println(written(result(dever.json.stringify(document)))) +}} +public main() () {{ + show_document(dever.json.value.Document {{root = 1 + nodes = {{1 = dever.json.value.Value.Array([0]) + 0 = dever.json.value.Value.String("\"\\\n\u{{1}}😀")}}}}) + show_document(dever.json.value.Document {{root = 0 + nodes = {{0 = dever.json.value.Value.Array([0])}}}}) + show_document(dever.json.value.Document {{root = 0 + nodes = {{0 = dever.json.value.Value.Array([-1])}}}}) + show_document(dever.json.value.Document {{root = 0 + nodes = {{0 = dever.json.value.Value.Array([1]) + 1 = dever.json.value.Value.Null}}}}) + show_document(dever.json.value.Document {{root = 3 + nodes = {{3 = dever.json.value.Value.Null}}}}) + show_document(dever.json.value.Document {{root = -1 + nodes = {{0 = dever.json.value.Value.Null}}}}) + show_document(dever.json.value.Document {{root = 0 + nodes = {{0 = dever.json.value.Value.Number("01")}}}}) + show_document(dever.json.value.Document {{root = 0 + nodes = {{}}}}) +}} +"# + ); + let output = support::stdout(&source); + let rows = output.lines().collect::<Vec<_>>(); + assert_eq!(rows[0], r#"["\"\\\n\u0001😀"]"#); + assert_eq!( + rows[1], + "INVALID 0: JSON child references must precede their parent" + ); + assert_eq!(rows[2], rows[1]); + assert_eq!(rows[3], rows[1]); + assert_eq!( + rows[4], + "INVALID 0: JSON node IDs must be consecutive from zero" + ); + assert_eq!( + rows[5], + "INVALID -1: JSON root does not refer to a document node" + ); + assert_eq!(rows[6], "INVALID 0: invalid JSON number spelling"); + assert_eq!( + rows[7], + "INVALID 0: JSON root does not refer to a document node" + ); +} + +#[test] +fn parsed_document_accessors_preserve_order_and_distinguish_json_null_from_missing() { + let output = support::stdout( + r#"inspect(parsed: dever.json.value.ParseResult.Parsed(document)) (result: Bool) recover("test observes inspect failures in its assertions") { + array = dever.json.property(document, document.root, "values") + result = inspect_array(array, document) +} +inspect(parsed: other) (result: Bool) recover("test observes inspect failures in its assertions") { result = false } +inspect_array(id: Int, document: dever.json.value.Document) (result: Bool) { + first = dever.json.element(document, id, 0) + second = dever.json.element(document, id, 1) + missing = dever.json.element(document, id, 2) + negative = dever.json.element(document, id, -1) + result = equals(first, 0) and equals(second, 1) and missing == null and negative == null and is_null(dever.json.node(document, 0)) and dever.json.node(document, 99) == null and dever.json.property(document, document.root, "missing") == null and dever.json.element(document, document.root, 0) == null +} +inspect_array(id: null, document: dever.json.value.Document) (result: Bool) { result = false } +equals(id: Int, expected: Int) (result: Bool) { result = id == expected } +equals(id: null, expected: Int) (result: Bool) { result = false } +is_null(value: dever.json.value.Value.Null) (result: Bool) { result = true } +is_null(value: null) (result: Bool) { result = false } +is_null(value: other) (result: Bool) { result = false } +public main() (valid: Bool) { valid = inspect(result(dever.json.parse("{\"values\":[null,123]}"))) } +"#, + ); + assert_eq!(output, "valid = true\n"); +} + +#[test] +fn constructed_graph_limits_reject_depth_and_exponential_expansion() { + let nested = std::iter::repeat_n("0", 257).collect::<Vec<_>>().join(","); + let doubled = std::iter::repeat_n("0", 23).collect::<Vec<_>>().join(","); + let source = format!( + r#"{ROUNDTRIP} +type Growth {{document: dever.json.value.Document + duplicate: Bool}} +seed(duplicate: Bool) (result: Growth) {{ + result = Growth {{document = dever.json.value.Document {{root = 0 + nodes = {{0 = dever.json.value.Value.String("a")}}}} + duplicate = duplicate}} +}} +grow(unused: Int, state: Growth) (next: Growth) {{ + id = length(state.document.nodes) + document = state.document + document.nodes = put(document.nodes, id, dever.json.value.Value.Array(children(state.duplicate, id - 1))) + document.root = id + next = state + next.document = document +}} +children(duplicate: true, previous: Int) (result: List<Int>) {{result = [previous, previous]}} +children(duplicate: false, previous: Int) (result: List<Int>) {{result = [previous]}} +public main() (deep: Text, expanded: Text) {{ + deep = written(result(dever.json.stringify(reduce(grow, [{nested}], seed(false)).document))) + expanded = written(result(dever.json.stringify(reduce(grow, [{doubled}], seed(true)).document))) +}} +"# + ); + assert_eq!( + support::stdout(&source), + "deep = INVALID 257: JSON nesting exceeds 256 containers\nexpanded = INVALID 22: JSON output exceeds 16777216 Unicode scalars\n" + ); +} diff --git a/test/dever-tests/tests/live_library.rs b/test/dever-tests/tests/live_library.rs new file mode 100644 index 0000000..6b96465 --- /dev/null +++ b/test/dever-tests/tests/live_library.rs @@ -0,0 +1,103 @@ +mod support; +use support::{checked, rejected, sources, stdout}; + +const ROUTE: &str = r#" +route(request: dever.http.Request, reply: dever.system.HttpReply) () recover("connection policy") { + dever.sse.start(reply, []) + dever.sse.send(reply, dever.sse.Event { event = "tick" + data = "hello" + id = "1" + retry_ms = 1000 }) +} +public start(listener: dever.system.Listener) () { + dever.http.serve_live(route, listener, dever.http.default_limits(), dever.http.default_live_limits()) +} +"#; + +#[test] +fn live_handler_contract_and_transitive_effects_are_checked() { + let source = ROUTE; + let map = sources(source); + let program = checked(&map); + let api = program.api_snapshot(); + let start = api + .lines() + .find(|line| line.starts_with("function main.start")) + .unwrap(); + assert!( + start.contains("network") + && start.contains("concurrency") + && start.contains("connection policy"), + "{start}" + ); + rejected( + &source.replace("dever.system.HttpReply) ()", "dever.system.Socket) ()"), + "expected", + ); + rejected( + &source.replace( + " dever.sse.start", + " dever.io.println(\"blocked\")\n dever.sse.start", + ), + "cannot run inline", + ); +} + +#[test] +fn live_example_checks_and_formats_idempotently() { + let source = include_str!("../../../examples/old/dever/live_http/module/main.dever"); + let map = sources(source); + checked(&map); + let first = dever_core::format::format(&map.files()[0]).unwrap(); + let second = sources(&first); + assert_eq!( + dever_core::format::format(&second.files()[0]).unwrap(), + first + ); +} + +#[test] +fn native_live_server_and_client_cover_stream_sse_and_websocket_resources() { + let example = include_str!("../../../examples/old/dever/live_http/module/main.dever"); + let routes = example.split("\nmain() () {").next().unwrap(); + let program = format!( + r#"{routes} +checked_response(result: dever.http.ResponseResult.Read(response), expected: Bytes) (ok: Bool) recover("test reports response errors") {{ ok = response.status == 200 and response.body == expected }} +checked_response(result: dever.http.ResponseResult.Failed(message), expected: Bytes) (ok: Bool) recover("test reports response errors") {{ ok = false }} +fetch(port: Int, target: Text, expected: Bytes) (ok: Bool) {{ + request = dever.http.Request {{ method = "GET" + target = target + headers = [dever.http.header("Host", "test")] + body = dever.bytes.from_text("") }} + ok = checked_response(result(dever.http.send("127.0.0.1", port, request, dever.http.default_limits())), expected) +}} +received(result: dever.websocket.ReceiveResult.Done(state)) (ok: Bool) recover("test reports socket errors") {{ ok = received_state(state) }} +received(result: other) (ok: Bool) recover("test reports socket errors") {{ ok = false }} +received_state(state: dever.websocket.ReceiveState.Read(message)) (ok: Bool) {{ ok = text_message(message) }} +received_state(state: other) (ok: Bool) {{ ok = false }} +text_message(message: dever.websocket.Message.Text(value)) (ok: Bool) {{ ok = value == "native" }} +text_message(message: other) (ok: Bool) {{ ok = false }} +connected(result: dever.websocket.ConnectResult.Connected(socket)) (ok: Bool) recover("test reports socket errors") {{ + dever.websocket.send(socket, dever.websocket.Message.Text("native")) + ok = received(result(dever.websocket.receive(socket))) + dever.websocket.close(socket, 1000, "done") +}} +connected(result: dever.websocket.ConnectResult.Failed(message)) (ok: Bool) recover("test reports socket errors") {{ ok = false }} +bound(result: dever.net.PortResult.Bound(port), listener: dever.system.Listener) (ok: Bool) recover("test reports startup errors") {{ + server = run(dever.http.serve_live(route, listener, dever.http.default_limits(), dever.http.default_live_limits())) + normal = fetch(port, "/", dever.bytes.from_text("Dever: /stream, /events, /ws")) + stream = fetch(port, "/stream", dever.bytes.from_text("first\nsecond\n")) + events = fetch(port, "/events", dever.bytes.from_text("event: tick\nid: 1\nretry: 1000\ndata: 1\n\nevent: tick\nid: 2\nretry: 1000\ndata: 2\n\nevent: tick\nid: 3\nretry: 1000\ndata: 3\n\n")) + websocket = connected(result(dever.websocket.connect("127.0.0.1", port, "/ws", dever.websocket.default_limits()))) + ok = normal and stream and events and websocket + stop(server) + dever.net.close_listener(listener) +}} +bound(result: dever.net.PortResult.Failed(message), listener: dever.system.Listener) (ok: Bool) recover("test reports startup errors") {{ ok = false }} +listening(result: dever.net.ListenResult.Listening(listener)) (ok: Bool) recover("test reports startup errors") {{ ok = bound(result(dever.net.port(listener)), listener) }} +listening(result: dever.net.ListenResult.Failed(message)) (ok: Bool) recover("test reports startup errors") {{ ok = false }} +main() (ok: Bool) {{ ok = listening(result(dever.net.listen("127.0.0.1", 0))) }} +"# + ); + assert_eq!(stdout(&program), "ok = true\n"); +} diff --git a/test/dever-tests/tests/live_network.rs b/test/dever-tests/tests/live_network.rs new file mode 100644 index 0000000..8a3c847 --- /dev/null +++ b/test/dever-tests/tests/live_network.rs @@ -0,0 +1,767 @@ +mod support; +#[path = "support/wire_peer.rs"] +mod wire_peer; + +use dever_runtime::{bytes::Bytes, http, net, sse, task, websocket as ws}; +use std::future::Future; +use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, +}; +use std::time::Duration; +use wire_peer::Peer; + +fn limits() -> http::Limits { + http::Limits { + header_bytes: 8192, + body_bytes: 1024, + timeout_ms: 500, + connections: 4, + http2: None, + } +} +fn live() -> http::LiveLimits { + http::LiveLimits { + chunk_bytes: 256, + idle_ms: 1000, + heartbeat_ms: 30, + } +} +fn ws_limits() -> ws::Limits { + ws::Limits { + message_bytes: 1024, + idle_ms: 1000, + write_ms: 500, + } +} +fn run(test: impl Future<Output = Result<(), String>>) { + task::run_entry_with( + task::RuntimeConfig { + worker_threads: 1, + max_blocking_threads: 2, + task_capacity: 64, + }, + async { + tokio::time::timeout(Duration::from_secs(8), test) + .await + .map_err(|_| "live test deadline".to_owned())? + }, + ) + .unwrap(); +} +async fn server<F, Fut>( + route: F, + limits: http::Limits, + live: http::LiveLimits, +) -> Result<(task::Task<()>, net::Listener, i64), String> +where + F: Fn(http::Request, http::HttpReply) -> Fut + Clone + Send + Sync + 'static, + Fut: Future<Output = Result<(), String>> + Send + 'static, +{ + let listener = net::listen("127.0.0.1", 0).await?; + let port = net::port(&listener)?; + let owner = listener.clone(); + let server = + task::run(async move { http::serve_live(route, owner, limits, live).await }).await?; + Ok((server, listener, port)) +} +struct Dropped(Arc<AtomicUsize>); +impl Drop for Dropped { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } +} +struct Finished(Arc<tokio::sync::Semaphore>); +impl Drop for Finished { + fn drop(&mut self) { + self.0.add_permits(1); + } +} + +#[test] +fn sse_wire_preserves_empty_ids_multiline_data_and_exact_size_limit() { + let event = sse::Event { + event: "change".into(), + data: "一\r\n二\r\n\r".into(), + id: Some("".into()), + retry_ms: Some(0), + }; + let expected = "event: change\nid: \nretry: 0\ndata: 一\ndata: 二\ndata: \ndata: \n\n"; + assert_eq!( + sse::encode(event.clone(), expected.len()).unwrap().values(), + expected.as_bytes() + ); + assert!(sse::encode(event, expected.len() - 1).is_err()); + for event in [ + sse::Event { + event: "bad\rname".into(), + data: "".into(), + id: None, + retry_ms: None, + }, + sse::Event { + event: "".into(), + data: "".into(), + id: Some("bad\0id".into()), + retry_ms: None, + }, + sse::Event { + event: "".into(), + data: "".into(), + id: None, + retry_ms: Some(-1), + }, + ] { + assert!(sse::encode(event, 1024).is_err()); + } + assert_eq!( + sse::encode( + sse::Event { + event: "".into(), + data: "".into(), + id: None, + retry_ms: None + }, + 8 + ) + .unwrap() + .values(), + b"data: \n\n" + ); +} + +#[test] +fn streaming_first_chunk_arrives_before_handler_finishes() { + run(async { + let gate = Arc::new(tokio::sync::Semaphore::new(0)); + let handler_gate = gate.clone(); + let (server, listener, port) = server( + move |_, reply| { + let gate = handler_gate.clone(); + async move { + http::start(&reply, 200, vec![]).await?; + http::write(&reply, Bytes::from_text("first")).await?; + let _permit = gate.acquire().await.unwrap(); + http::write(&reply, Bytes::from_text("second")).await?; + http::finish(&reply).await + } + }, + limits(), + live(), + ) + .await?; + let mut peer = Peer::connect(port).await?; + peer.write(b"GET / HTTP/1.1\r\nHost: test\r\n\r\n").await?; + assert!( + peer.head() + .await? + .to_ascii_lowercase() + .contains("transfer-encoding: chunked") + ); + assert_eq!(peer.chunk().await?, b"first"); + gate.add_permits(1); + assert_eq!(peer.chunk().await?, b"second"); + assert!(peer.chunk().await?.is_empty()); + task::stop(server).await?; + listener.close()?; + Ok(()) + }); +} + +#[test] +fn sse_heartbeats_outlive_request_deadline_and_stop_drains_handler() { + run(async { + let dropped = Arc::new(AtomicUsize::new(0)); + let marker = dropped.clone(); + let (server, listener, port) = server( + move |request, reply| { + let marker = marker.clone(); + async move { + let _guard = Dropped(marker); + assert!( + request + .headers + .iter() + .any(|header| header.name == "last-event-id" + && header.value.values() == b"42") + ); + sse::start(&reply, vec![]).await?; + sse::send( + &reply, + sse::Event { + event: "update".into(), + data: "hello".into(), + id: Some("43".into()), + retry_ms: None, + }, + ) + .await?; + std::future::pending().await + } + }, + http::Limits { + timeout_ms: 80, + ..limits() + }, + live(), + ) + .await?; + let mut peer = Peer::connect(port).await?; + peer.write(b"GET / HTTP/1.1\r\nHost: test\r\nLast-Event-ID: 42\r\n\r\n") + .await?; + assert!(peer.head().await?.contains("text/event-stream")); + assert_eq!( + peer.chunk().await?, + b"event: update\nid: 43\ndata: hello\n\n" + ); + for _ in 0..5 { + assert_eq!(peer.chunk().await?, b":\n\n"); + } + task::stop(server).await?; + assert_eq!(dropped.load(Ordering::SeqCst), 1); + assert_eq!(net::read(&peer.socket, 1).await?, None); + listener.close()?; + Ok(()) + }); +} + +#[test] +fn head_body_drop_releases_stream_and_buffered_handlers_before_next_request() { + run(async { + let dropped = Arc::new(AtomicUsize::new(0)); + let marker = dropped.clone(); + let (server, listener, port) = server( + move |request, reply| { + let marker = marker.clone(); + async move { + let _guard = Dropped(marker); + if request.target == "/stream" { + http::start(&reply, 200, vec![]).await?; + } else { + http::respond( + &reply, + http::Response { + status: 200, + headers: vec![], + body: Bytes::from_text("ok"), + }, + ) + .await?; + } + std::future::pending().await + } + }, + limits(), + live(), + ) + .await?; + let mut peer = Peer::connect(port).await?; + peer.write(b"HEAD /stream HTTP/1.1\r\nHost: test\r\n\r\nHEAD /full HTTP/1.1\r\nHost: test\r\n\r\nGET /full HTTP/1.1\r\nHost: test\r\n\r\n").await?; + for _ in 0..3 { + assert!(peer.head().await?.starts_with("HTTP/1.1 200")); + } + assert_eq!(peer.take(2).await?, b"ok"); + task::stop(server).await?; + assert_eq!(dropped.load(Ordering::SeqCst), 3); + listener.close()?; + Ok(()) + }); +} + +#[test] +fn handler_fault_after_headers_truncates_only_that_response() { + run(async { + let gate = Arc::new(tokio::sync::Semaphore::new(0)); + let handler_gate = gate.clone(); + let (server, listener, port) = server( + move |_, reply| { + let gate = handler_gate.clone(); + async move { + http::start(&reply, 200, vec![]).await?; + http::write(&reply, Bytes::from_text("partial")).await?; + let _permit = gate.acquire().await.unwrap(); + Err("precise handler fault".into()) + } + }, + limits(), + live(), + ) + .await?; + let mut peer = Peer::connect(port).await?; + peer.write(b"GET / HTTP/1.1\r\nHost: test\r\n\r\n").await?; + peer.head().await?; + assert_eq!(peer.chunk().await?, b"partial"); + gate.add_permits(1); + assert!(peer.chunk().await.is_err()); + task::stop(server).await?; + listener.close()?; + Ok(()) + }); +} + +async fn echo(_request: http::Request, reply: http::HttpReply) -> Result<(), String> { + let socket = ws::accept(&reply, ws_limits()).await?; + while let Some(message) = ws::receive(&socket).await? { + if matches!(message, ws::Message::Text(_) | ws::Message::Binary(_)) { + ws::send(&socket, message).await?; + } + } + Ok(()) +} + +#[test] +fn slow_stream_peer_applies_backpressure_and_stop_releases_the_writer() { + run(async { + let produced = Arc::new(AtomicUsize::new(0)); + let counter = produced.clone(); + let finished = Arc::new(tokio::sync::Semaphore::new(0)); + let signal = finished.clone(); + let (server, listener, port) = server( + move |_, reply| { + let counter = counter.clone(); + let signal = signal.clone(); + async move { + let _finished = Finished(signal); + http::start(&reply, 200, vec![]).await?; + let chunk = Bytes::new(vec![1; 65536]); + for _ in 0..10000 { + if http::write(&reply, chunk.clone()).await.is_err() { + break; + } + counter.fetch_add(1, Ordering::SeqCst); + } + Ok(()) + } + }, + http::Limits { + timeout_ms: 100, + ..limits() + }, + http::LiveLimits { + chunk_bytes: 65536, + ..live() + }, + ) + .await?; + let mut peer = Peer::connect(port).await?; + peer.write(b"GET / HTTP/1.1\r\nHost: test\r\n\r\n").await?; + peer.head().await?; + // 客户端停止读取；写入只能占用有界通道与传输缓冲，不能把全部正文排队。 + let _permit = tokio::time::timeout(Duration::from_secs(2), finished.acquire()) + .await + .unwrap() + .unwrap(); + let count = produced.load(Ordering::SeqCst); + assert!( + count > 0 && count < 10000, + "produced {count} chunks without a consuming peer" + ); + task::stop(server).await?; + listener.close()?; + Ok(()) + }); +} + +#[test] +fn stream_idle_timeout_is_a_peer_failure_and_server_accepts_next_request() { + run(async { + let (server, listener, port) = server( + |request, reply| async move { + if request.target == "/idle" { + http::start(&reply, 200, vec![]).await?; + std::future::pending().await + } else { + http::respond( + &reply, + http::Response { + status: 200, + headers: vec![], + body: Bytes::from_text("ok"), + }, + ) + .await + } + }, + limits(), + http::LiveLimits { + idle_ms: 60, + heartbeat_ms: 20, + ..live() + }, + ) + .await?; + let mut peer = Peer::connect(port).await?; + peer.write(b"GET /idle HTTP/1.1\r\nHost: test\r\n\r\n") + .await?; + peer.head().await?; + assert!(peer.chunk().await.is_err()); + let mut peer = Peer::connect(port).await?; + peer.write(b"GET /ok HTTP/1.1\r\nHost: test\r\n\r\n") + .await?; + assert!(peer.head().await?.starts_with("HTTP/1.1 200")); + assert_eq!(peer.take(2).await?, b"ok"); + task::stop(server).await?; + listener.close()?; + Ok(()) + }); +} + +#[test] +fn websocket_rejects_invalid_mask_utf8_and_fragmented_message_size() { + run(async { + let failures = dever_runtime::channel::Channel::new(4)?; + let sent = failures.clone(); + let (server, listener, port) = server( + move |_, reply| { + let failures = sent.clone(); + async move { + let socket = ws::accept( + &reply, + ws::Limits { + message_bytes: 4, + ..ws_limits() + }, + ) + .await?; + let error = ws::receive(&socket).await.unwrap_err(); + failures.send(error).await + } + }, + limits(), + live(), + ) + .await?; + for case in 0..4 { + let mut peer = Peer::connect(port).await?; + peer.upgrade().await?; + match case { + 0 => peer.send_frame(0x81, false, b"bad").await?, + 1 => peer.send_frame(0x81, true, &[255]).await?, + 2 => peer.send_frame(0x82, true, b"large").await?, + _ => { + peer.send_frame(0x01, true, b"abc").await?; + peer.send_frame(0x80, true, b"def").await?; + } + } + let error = failures.receive().await?.unwrap(); + assert!(!error.is_empty()); + assert_eq!(net::read(&peer.socket, 1).await?, None); + } + task::stop(server).await?; + listener.close()?; + Ok(()) + }); +} + +#[test] +fn upgraded_sessions_hold_connection_capacity_and_parent_stop_closes_them() { + run(async { + let (server, listener, port) = server( + echo, + http::Limits { + connections: 1, + ..limits() + }, + live(), + ) + .await?; + let first = ws::connect("127.0.0.1", port, "/ws", ws_limits()).await?; + let second = ws::connect("127.0.0.1", port, "/ws", ws_limits()); + tokio::pin!(second); + assert!( + tokio::time::timeout(Duration::from_millis(30), &mut second) + .await + .is_err() + ); + ws::close(&first, 1000, "next").await?; + let second = second.await?; + let stream = ws::messages(second.clone()); + ws::send(&second, ws::Message::Binary(Bytes::from_text("next"))).await?; + assert!( + matches!(stream.pull().await.unwrap()?, ws::Message::Binary(bytes) if bytes.values() == b"next") + ); + task::stop(server).await?; + assert!(stream.pull().await.unwrap().is_err()); + assert!(stream.pull().await.is_none()); + listener.close()?; + Ok(()) + }); +} + +#[test] +fn websocket_client_rejects_wrong_handshake_accept() { + run(async { + let listener = net::listen("127.0.0.1", 0).await?; + let port = net::port(&listener)?; + let owner = listener.clone(); + let peer = task::run(async move { + let mut peer = Peer::new(net::accept(&owner).await?)?; + let head = peer.head().await?; + assert!(head.starts_with("GET /chat HTTP/1.1")); + assert!(head.to_ascii_lowercase().contains("sec-websocket-key:")); + peer.write(b"HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nSec-WebSocket-Accept: wrong\r\n\r\n").await?; + assert_eq!(net::read(&peer.socket, 1).await?, None); + Ok(()) + }).await?; + assert!( + ws::connect("127.0.0.1", port, "/chat", ws_limits()) + .await + .is_err() + ); + task::wait(peer).await?; + listener.close()?; + Ok(()) + }); +} + +#[test] +fn client_frames_are_masked_and_server_upgrade_retains_preread_frame_bytes() { + run(async { + let listener = net::listen("127.0.0.1", 0).await?; + let port = net::port(&listener)?; + let owner = listener.clone(); + let peer = task::run(async move { + let mut peer = Peer::new(net::accept(&owner).await?)?; + let head = peer.head().await?; + let key = head.lines().find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("sec-websocket-key").then(|| value.trim()) + }).unwrap(); + // 只复用握手摘要；帧头、掩码及负载由独立观察器核验。 + let accept = tokio_tungstenite::tungstenite::handshake::derive_accept_key(key.as_bytes()); + peer.write(format!("HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nSec-WebSocket-Accept: {accept}\r\n\r\n").as_bytes()).await?; + assert_eq!(peer.frame().await?, (0x81, true, b"masked".to_vec())); + peer.send_frame(0x88, false, &[3, 232]).await?; + assert_eq!(peer.frame().await?, (0x88, true, vec![3, 232])); + Ok(()) + }).await?; + let socket = ws::connect("127.0.0.1", port, "/", ws_limits()).await?; + ws::send(&socket, ws::Message::Text("masked".into())).await?; + assert!(ws::receive(&socket).await?.is_none()); + task::wait(peer).await?; + listener.close()?; + + let (server, listener, port) = server(echo, limits(), live()).await?; + let mut peer = Peer::connect(port).await?; + let mut request = b"GET /ws HTTP/1.1\r\nHost: test\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nSec-WebSocket-Version: 13\r\nSec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n\r\n".to_vec(); + request.extend_from_slice(&[0x81, 0x82, 0, 0, 0, 0, b'o', b'k']); + peer.write(&request).await?; + assert!(peer.head().await?.starts_with("HTTP/1.1 101")); + assert_eq!(peer.frame().await?, (0x81, false, b"ok".to_vec())); + task::stop(server).await?; + listener.close()?; + Ok(()) + }); +} + +#[test] +fn websocket_cancel_waiting_writer_keeps_active_frame_and_cancel_active_writer_closes() { + run(async { + let sockets = dever_runtime::channel::Channel::new(1)?; + let sender = sockets.clone(); + let (server, listener, port) = server( + move |_, reply| { + let sender = sender.clone(); + async move { + let socket = ws::accept( + &reply, + ws::Limits { + message_bytes: 8 * 1024 * 1024, + write_ms: 3000, + ..ws_limits() + }, + ) + .await?; + sender.send(socket).await?; + std::future::pending().await + } + }, + limits(), + live(), + ) + .await?; + let mut peer = Peer::connect(port).await?; + peer.upgrade().await?; + let socket = sockets.receive().await?.unwrap(); + let first = ws::send( + &socket, + ws::Message::Binary(Bytes::new(vec![42; 8 * 1024 * 1024])), + ); + tokio::pin!(first); + assert!( + tokio::time::timeout(Duration::from_millis(20), &mut first) + .await + .is_err() + ); + let waiting = socket.clone(); + let waiting = task::run(async move { + ws::send(&waiting, ws::Message::Text("must not appear".into())).await + }) + .await?; + tokio::task::yield_now().await; + task::stop(waiting).await?; + let reader = async { peer.frame().await }; + let (sent, frame) = tokio::join!(first, reader); + sent?; + let frame = frame?; + assert_eq!( + (frame.0, frame.1, frame.2.len()), + (0x82, false, 8 * 1024 * 1024) + ); + assert!(frame.2.iter().all(|byte| *byte == 42)); + let writing = socket.clone(); + let writing = task::run(async move { + ws::send( + &writing, + ws::Message::Binary(Bytes::new(vec![17; 8 * 1024 * 1024])), + ) + .await + }) + .await?; + tokio::time::sleep(Duration::from_millis(30)).await; + task::stop(writing).await?; + assert!( + ws::send(&socket, ws::Message::Text("closed".into())) + .await + .is_err() + ); + task::stop(server).await?; + listener.close()?; + Ok(()) + }); +} + +#[test] +fn stopping_live_service_drains_started_blocking_descendants() { + run(async { + let started = Arc::new(tokio::sync::Semaphore::new(0)); + let completed = Arc::new(AtomicUsize::new(0)); + let signal = started.clone(); + let count = completed.clone(); + let (server, listener, port) = server( + move |_, reply| { + let signal = signal.clone(); + let count = count.clone(); + async move { + sse::start(&reply, vec![]).await?; + let child = task::run(async move { + task::blocking(move || { + signal.add_permits(1); + std::thread::sleep(Duration::from_millis(80)); + count.fetch_add(1, Ordering::SeqCst); + Ok(()) + }) + .await + }) + .await?; + task::wait(child).await + } + }, + limits(), + live(), + ) + .await?; + let peer = Peer::connect(port).await?; + peer.write(b"GET / HTTP/1.1\r\nHost: test\r\n\r\n").await?; + let _permit = started.acquire().await.unwrap(); + task::stop(server).await?; + assert_eq!(completed.load(Ordering::SeqCst), 1); + listener.close()?; + Ok(()) + }); +} + +#[test] +fn rejected_reply_operations_leave_the_committed_stream_open() { + run(async { + let (server, listener, port) = server( + |_, reply| async move { + assert!( + http::write(&reply, Bytes::from_text("too early")) + .await + .is_err() + ); + assert!(http::start(&reply, 204, vec![]).await.is_err()); + http::start(&reply, 200, vec![]).await?; + assert!(http::start(&reply, 201, vec![]).await.is_err()); + assert!( + http::respond( + &reply, + http::Response { + status: 200, + headers: vec![], + body: Bytes::from_text("duplicate") + } + ) + .await + .is_err() + ); + assert!(sse::start(&reply, vec![]).await.is_err()); + assert!(ws::accept(&reply, ws_limits()).await.is_err()); + assert!(http::write(&reply, Bytes::new(vec![0; 257])).await.is_err()); + tokio::task::yield_now().await; + http::write(&reply, Bytes::from_text("still-open")).await?; + http::finish(&reply).await + }, + limits(), + live(), + ) + .await?; + let mut peer = Peer::connect(port).await?; + peer.write(b"GET / HTTP/1.1\r\nHost: test\r\n\r\n").await?; + assert!(peer.head().await?.starts_with("HTTP/1.1 200")); + assert_eq!(peer.chunk().await?, b"still-open"); + assert!(peer.chunk().await?.is_empty()); + task::stop(server).await?; + listener.close()?; + Ok(()) + }); +} + +#[test] +fn websocket_independent_frames_cover_fragmentation_ping_binary_and_close() { + run(async { + let (server, listener, port) = server(echo, limits(), live()).await?; + let mut peer = Peer::connect(port).await?; + peer.upgrade().await?; + peer.send_frame(0x01, true, &[0xe4]).await?; + peer.send_frame(0x89, true, b"alive").await?; + assert_eq!(peer.frame().await?, (0x8a, false, b"alive".to_vec())); + peer.send_frame(0x80, true, &[0xbd, 0xa0]).await?; + assert_eq!(peer.frame().await?, (0x81, false, "你".as_bytes().to_vec())); + peer.send_frame(0x82, true, &[0, 255, 1]).await?; + assert_eq!(peer.frame().await?, (0x82, false, vec![0, 255, 1])); + peer.send_frame(0x88, true, &[3, 232]).await?; + assert_eq!(peer.frame().await?, (0x88, false, vec![3, 232])); + assert_eq!(net::read(&peer.socket, 1).await?, None); + task::stop(server).await?; + listener.close()?; + Ok(()) + }); +} + +#[test] +fn websocket_client_is_full_duplex_and_close_wakes_pending_receiver() { + run(async { + let (server, listener, port) = server(echo, limits(), live()).await?; + let socket = ws::connect("127.0.0.1", port, "/ws", ws_limits()).await?; + let reader = socket.clone(); + let waiting = task::run(async move { ws::receive(&reader).await }).await?; + ws::send(&socket, ws::Message::Text("duplex".into())).await?; + assert!( + matches!(task::wait(waiting).await?, Some(ws::Message::Text(text)) if text == "duplex") + ); + let reader = socket.clone(); + let waiting = task::run(async move { Ok(ws::receive(&reader).await.is_err()) }).await?; + ws::close(&socket, 1000, "done").await?; + assert!(task::wait(waiting).await?); + assert!( + ws::send(&socket, ws::Message::Text("closed".into())) + .await + .is_err() + ); + task::stop(server).await?; + listener.close()?; + Ok(()) + }); +} diff --git a/test/dever-tests/tests/logging.rs b/test/dever-tests/tests/logging.rs new file mode 100644 index 0000000..1200f0a --- /dev/null +++ b/test/dever-tests/tests/logging.rs @@ -0,0 +1,14 @@ +mod support; + +#[test] +fn source_logging_emits_one_structured_record() { + let output = + support::run("public main() () { dever.log.info(\"ready\", {\"component\" = \"cms\"}) }"); + assert!(output.status.success()); + assert!(output.stdout.is_empty()); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert_eq!(stderr.lines().count(), 1, "{stderr}"); + assert!(stderr.contains("\"level\":\"info\""), "{stderr}"); + assert!(stderr.contains("\"message\":\"ready\""), "{stderr}"); + assert!(stderr.contains("\"component\":\"cms\""), "{stderr}"); +} diff --git a/test/dever-tests/tests/markdown_source.rs b/test/dever-tests/tests/markdown_source.rs new file mode 100644 index 0000000..f493fa7 --- /dev/null +++ b/test/dever-tests/tests/markdown_source.rs @@ -0,0 +1,938 @@ +mod support; + +use std::fs; +use std::path::Path; +use std::process::Command; + +use dever_core::diagnostic::Diagnostic; +use dever_core::source::SourceMap; +use support::checked; +use support::temp::TemporaryDirectory; + +const MAIN: &str = "main() (answer: Int) { answer = 42 }\n"; + +fn unstructured_document(blocks: &[&str]) -> String { + let mut text = String::from("# 程序说明\n\n标题不是 package，表格不是类型。\n"); + for block in blocks { + text.push_str("\n## 说明\n\n```dever\n"); + text.push_str(block); + text.push_str("```\n\n这一段不执行。\n"); + } + text +} + +fn documented_list(label: &str, values: &[&str], described: bool) -> String { + if values.is_empty() { + return format!("- {label}：无\n"); + } + let mut text = format!("- {label}：\n"); + for value in values { + text.push_str(&format!( + " - `{value}`{}\n", + if described { "：测试说明" } else { "" } + )); + } + text +} + +fn type_section(title: &str, name: &str, kind: &str, entries: &[&str], code: &str) -> String { + format!( + "## {title}\n\n测试类型的用途。\n\n- 类型：`{name}`\n{}\n```dever\n{code}```\n", + documented_list(kind, entries, true) + ) +} + +fn function_section( + title: &str, + name: &str, + inputs: &[&str], + outputs: &[&str], + code: &str, +) -> String { + format!( + "## {title}\n\n测试函数的用途。\n\n- 函数：`{name}`\n{}{}\n```dever\n{code}```\n", + documented_list("输入", inputs, true), + documented_list("输出", outputs, true) + ) +} + +fn package_document( + package: &str, + public_types: &[&str], + public_functions: &[&str], + usages: &[&str], + sections: &[String], +) -> String { + let mut text = format!( + "# 测试程序\n\n测试包的用途。\n\n- 包：`{package}`\n{}{}{}", + documented_list("公开类型", public_types, false), + documented_list("公开方法", public_functions, false), + documented_list("使用", usages, false), + ); + for section in sections { + text.push('\n'); + text.push_str(section); + } + text +} + +fn main_document() -> String { + package_document( + "main", + &[], + &[], + &[], + &[function_section( + "主函数", + "main", + &[], + &["answer: Int"], + MAIN, + )], + ) +} + +fn model_section(declaration: &str, code: &str, signature: Option<(&[&str], &[&str])>) -> String { + let signature = signature.map_or_else(String::new, |(inputs, outputs)| { + format!( + "{}{}", + documented_list("输入", inputs, true), + documented_list("输出", outputs, true), + ) + }); + format!( + "## 存储声明\n\n约束存储结构和初始化行为。\n\n- 声明：`{declaration}`\n{signature}\n```dever\n{code}\n```\n" + ) +} + +fn model_document() -> String { + package_document( + "app.item.model", + &["Item"], + &[], + &[], + &[ + model_section("database primary", "database primary", None), + type_section( + "项目", + "Item", + "字段", + &["name: Text(1, 64)", "cost: Decimal(5, 2)"], + "type Item { name: Text(1, 64) unique\n cost: Decimal(5, 2) default 0 }\n", + ), + model_section("index(cost, created_at)", "index(cost, created_at)", None), + model_section("unique(name, cost)", "unique(name, cost)", None), + model_section( + "relation children = app.child.model.item_id", + "relation children = app.child.model.item_id", + None, + ), + model_section("seed", "seed { { name = \"初始\"\n cost = 1.25 } }", None), + model_section( + "migrate remove_obsolete", + "migrate remove_obsolete { drop obsolete }", + None, + ), + model_section( + "sql by_name", + concat!( + "sql by_name(name: Text) (item: Item?) {\n", + " sqlite = \"SELECT id, created_at, name, cost FROM item WHERE name = ?1\"\n", + " postgres = \"SELECT id, created_at, name, cost FROM item WHERE name = $1\"\n}\n" + ), + Some((&["name: Text"], &["item: Item?"])), + ), + ], + ) +} + +fn model_sources(document: &str) -> SourceMap { + let mut sources = SourceMap::default(); + sources.add("app/item/model.dever.md", document); + sources.add( + "app/child/model.dever", + concat!("", "type Child { item_id: app.item.model.id }\n"), + ); + sources.add( + "app/item/app.dever", + "count() (total: Int) { total = model.count() }\n", + ); + sources.add( + "main.dever", + concat!("", "main() (total: Int) { total = app.item.count() }\n"), + ); + sources +} + +#[test] +fn markdown_models_preserve_all_storage_declarations_and_formatting() { + let document = model_document(); + let sources = model_sources(&document); + let program = dever_core::check(&sources) + .unwrap_or_else(|errors| panic!("{}", render(&errors, &sources))); + let snapshot = program.model_snapshot(); + assert!(snapshot.contains("model app.item.model name=Item")); + assert!(snapshot.contains("connection=explicit:primary")); + assert!(snapshot.contains("field cost Decimal(5,2)")); + assert!(snapshot.contains("relation to-many children = app.child.model.item_id")); + assert!(snapshot.contains("migrate remove_obsolete drop(obsolete)")); + assert!(snapshot.contains("sql by_name")); + let formatted = dever_core::format::format(&sources.files()[0]).unwrap(); + let reformatted_sources = model_sources(&formatted); + assert_eq!( + formatted, + dever_core::format::format(&reformatted_sources.files()[0]).unwrap(), + ); + assert_eq!( + snapshot, + dever_core::check(&reformatted_sources) + .unwrap() + .model_snapshot(), + ); + + let mut plain = SourceMap::default(); + let model = dever_core::parse(&sources.files()[0]).unwrap(); + let mut code = String::from(""); + for declaration in &model.declarations { + let span = match declaration { + dever_core::syntax::Declaration::Api(value) => value.span, + dever_core::syntax::Declaration::Rest(value) => value.span, + dever_core::syntax::Declaration::External(value) => value.span, + dever_core::syntax::Declaration::Type(value) => value.span, + dever_core::syntax::Declaration::Database(value) => value.span, + dever_core::syntax::Declaration::Schedule(value) => value.span, + dever_core::syntax::Declaration::ModelIndex(value) => value.span, + dever_core::syntax::Declaration::Relation(value) => value.span, + dever_core::syntax::Declaration::Seed(value) => value.span, + dever_core::syntax::Declaration::Migration(value) => value.span, + dever_core::syntax::Declaration::ModelSql(value) => value.span, + dever_core::syntax::Declaration::Function(value) => value.span, + }; + code.push_str(&document[span.start..span.end]); + code.push('\n'); + } + plain.add("app/item/model.dever", code); + for source in &sources.files()[1..] { + plain.add(source.path(), source.text()); + } + let plain_program = dever_core::check(&plain).unwrap(); + assert_eq!(snapshot, plain_program.model_snapshot()); + assert_eq!(program.api_snapshot(), plain_program.api_snapshot()); + dever_core::native::emit(&program, &sources, "main.main").unwrap(); +} + +#[test] +fn markdown_model_preserves_global_scope() { + let document = package_document( + "platform.tenant.model", + &["Tenant"], + &[], + &[], + &[type_section( + "租户", + "Tenant", + "字段", + &["name: Text(1, 64)"], + "global type Tenant { name: Text(1, 64) }\n", + )], + ); + let mut sources = SourceMap::default(); + let source = sources.add("platform/tenant/model.dever.md", document); + let program = dever_core::check(&sources) + .unwrap_or_else(|errors| panic!("{}", render(&errors, &sources))); + assert!( + program.model_snapshot().contains("scope=global"), + "{}", + program.model_snapshot(), + ); + let formatted = dever_core::format::format(sources.get(source)).unwrap(); + assert!(formatted.contains("global type Tenant"), "{formatted}"); +} + +#[test] +fn markdown_model_metadata_is_checked_and_cannot_skip_documentation() { + for (before, after, code) in [ + ( + "- 声明：`database primary`", + "- 声明：`database other`", + "M006", + ), + ( + "- 声明：`index(cost, created_at)`", + "- 声明：`index(created_at, cost)`", + "M006", + ), + ( + "- 声明：`unique(name, cost)`", + "- 声明：`unique(cost, name)`", + "M006", + ), + ( + "- 声明：`relation children = app.child.model.item_id`", + "- 声明：`relation children = app.child.model.id`", + "M006", + ), + ("- 声明：`seed`", "", "M005"), + ( + "- 声明：`migrate remove_obsolete`", + "- 声明：`migrate wrong`", + "M006", + ), + ("- 声明：`sql by_name`", "- 声明：`sql wrong`", "M006"), + ( + " - `item: Item?`：测试说明", + " - `item: Item`：测试说明", + "M008", + ), + ( + " - `name: Text`：测试说明", + " - `name: Int`：测试说明", + "M007", + ), + ("`cost: Decimal(5, 2)`", "`cost: Decimal(6, 2)`", "M008"), + ("约束存储结构和初始化行为。", "", "M005"), + ("default 0", "default 0.001", "C014"), + ("cost = 1.25", "cost = 0.001", "C014"), + ] { + let sources = model_sources(&model_document().replace(before, after)); + let errors = dever_core::check(&sources).unwrap_err(); + assert!( + errors.iter().any(|error| error.code == code), + "{}", + render(&errors, &sources), + ); + } +} + +fn markdown_sources(text: &str) -> SourceMap { + let mut sources = SourceMap::default(); + sources.add("main.dever.md", text); + sources +} + +fn formatted(text: &str) -> String { + let sources = markdown_sources(text); + dever_core::format::format(&sources.files()[0]).unwrap_or_else(|errors| { + panic!("{}", render(&errors, &sources)); + }) +} + +fn render(errors: &[Diagnostic], sources: &SourceMap) -> String { + errors.iter().map(|error| error.render(sources)).collect() +} + +fn assert_rejected(text: &str, code: &str) { + let sources = markdown_sources(text); + let errors = dever_core::parse(&sources.files()[0]).unwrap_err(); + assert!( + errors.iter().any(|error| error.code == code), + "{}", + render(&errors, &sources) + ); + assert!(dever_core::format::format(&sources.files()[0]).is_err()); +} + +fn assert_check_rejected(text: &str, code: &str) -> Vec<Diagnostic> { + let sources = markdown_sources(text); + let errors = dever_core::check(&sources).unwrap_err(); + assert!( + errors.iter().any(|error| error.code == code), + "{}", + render(&errors, &sources) + ); + errors +} + +fn execute(sources: &SourceMap) -> std::process::Output { + let rustc = std::env::var_os("RUSTC").unwrap_or_else(|| "rustc".into()); + let native = + dever_core::native::compile(&checked(sources), sources, "main.main", &rustc).unwrap(); + Command::new(native.executable()).output().unwrap() +} + +#[test] +fn markdown_and_plain_sources_have_identical_native_results_and_public_api() { + let blocks = [ + "", + "type Pair { original: Int\nchanged: Int }\n", + "choose(value: true) (answer: Int) { answer = 7 }\nchoose(value: false) (answer: Int) { answer = 9 }\n", + "main() (answer: Int, pair: Pair) {\nvalue = 2\nvalue = value + 1\nanswer = choose(true) * value\npair = Pair { original = 2\nchanged = value }\n}\n", + ]; + let markdown_text = package_document( + "main", + &[], + &[], + &[], + &[ + type_section( + "数值对", + "Pair", + "字段", + &["original: Int", "changed: Int"], + blocks[1], + ), + function_section( + "选择数值", + "choose", + &["value: Bool"], + &["answer: Int"], + blocks[2], + ), + function_section( + "主函数", + "main", + &[], + &["answer: Int", "pair: Pair"], + blocks[3], + ), + ], + ); + let markdown = markdown_sources(&markdown_text); + let plain = support::sources(&blocks.concat()); + let expected = b"answer = 21\npair = main.Pair { original = 2, changed = 3 }\n"; + let left = execute(&plain); + let right = execute(&markdown); + assert!(left.status.success() && right.status.success()); + assert!(left.stderr.is_empty() && right.stderr.is_empty()); + assert_eq!(left.stdout, right.stdout); + assert_eq!(right.stdout, expected); + assert_eq!( + checked(&plain).api_snapshot(), + checked(&markdown).api_snapshot() + ); +} + +#[test] +fn real_checkout_example_loads_typescript_highlighted_dever_blocks() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../examples/old/markdown"); + let sources = SourceMap::load(&root).unwrap(); + assert_eq!(sources.files().len(), 1); + let package = dever_core::parse(&sources.files()[0]).unwrap(); + assert_eq!(package.declarations.len(), 6); +} + +#[test] +fn markdown_contract_requires_one_code_declaration_per_documented_section() { + let package_and_function = "# 测试程序\n\n测试包的用途。\n\n- 包：`main`\n- 公开类型：无\n- 公开方法：无\n- 使用：无\n\n```dever\nmain() (answer: Int) { answer = 42 }\n```\n"; + assert_rejected(package_and_function, "M004"); + + let mixed_functions = package_document( + "main", + &[], + &[], + &[], + &[function_section( + "两个函数", + "helper", + &[], + &["answer: Int"], + "helper() (answer: Int) { answer = 1 }\nmain() (answer: Int) { answer = helper() }\n", + )], + ); + assert_rejected(&mixed_functions, "M005"); + + let split_clauses = package_document( + "main", + &[], + &[], + &[], + &[ + function_section( + "真值分句", + "choose", + &["value: Bool"], + &["answer: Int"], + "choose(value: true) (answer: Int) { answer = 1 }\n", + ), + function_section( + "假值分句", + "choose", + &["value: Bool"], + &["answer: Int"], + "choose(value: false) (answer: Int) { answer = 0 }\n", + ), + ], + ); + assert_rejected(&split_clauses, "M005"); +} + +#[test] +fn markdown_contract_accepts_all_clauses_of_one_function_in_one_section() { + let text = package_document( + "main", + &[], + &[], + &[], + &[function_section( + "选择数值", + "choose", + &["value: Bool"], + &["answer: Int"], + "choose(value: true) (answer: Int) { answer = 1 }\nchoose(value: false) (answer: Int) { answer = 0 }\n", + )], + ); + checked(&markdown_sources(&text)); + + let inconsistent_names = text.replace("choose(value: false)", "choose(flag: false)"); + let errors = assert_check_rejected(&inconsistent_names, "M007"); + assert!( + errors + .iter() + .find(|error| error.code == "M007") + .is_some_and(|error| !error.related.is_empty()) + ); +} + +#[test] +fn markdown_contract_checks_package_name_public_api_and_usage_in_order() { + for text in [ + main_document().replace("- 包：`main`", "- 包：`wrong`"), + main_document().replace("- 公开方法：无", "- 公开方法：\n - `main`"), + main_document().replace("- 使用：无", "- 使用：\n - `main.main()`"), + ] { + let errors = assert_check_rejected(&text, "M006"); + assert!( + errors + .iter() + .filter(|error| error.code == "M006") + .all(|error| !error.related.is_empty()) + ); + } +} + +#[test] +fn markdown_contract_checks_type_fields_variants_and_descriptions() { + let record = package_document( + "main", + &[], + &[], + &[], + &[type_section( + "数值对", + "Pair", + "字段", + &["left: Int", "right: Int >= 0"], + "type Pair {\nleft: Int\nright: Int >= 0\n}\n", + )], + ); + checked(&markdown_sources(&record)); + let stale_field = record.replace("`right: Int >= 0`", "`right: Int`"); + assert_check_rejected(&stale_field, "M008"); + let missing_description = record.replace("`left: Int`：测试说明", "`left: Int`："); + assert_rejected(&missing_description, "M005"); + + let choice = package_document( + "main", + &[], + &[], + &[], + &[type_section( + "处理结果", + "Result", + "分支", + &["Ok(value: Int)", "error Failed(message: Text)"], + "type Result {\nOk(value: Int)\nerror Failed(message: Text)\n}\n", + )], + ); + checked(&markdown_sources(&choice)); + assert_check_rejected( + &choice.replace("`Ok(value: Int)`", "`Ok(value: Text)`"), + "M008", + ); +} + +#[test] +fn markdown_contract_checks_resolved_function_inputs_and_outputs() { + let text = package_document( + "main", + &[], + &[], + &[], + &[function_section( + "选择数值", + "choose", + &["value: Bool"], + &["answer: Int"], + "choose(value: true) (answer: Int) { answer = 1 }\nchoose(value: false) (answer: Int) { answer = 0 }\n", + )], + ); + assert_check_rejected(&text.replace("`value: Bool`", "`value: Int`"), "M007"); + for stale_output in [ + text.replace("`answer: Int`", "`result: Int`"), + text.replace("`answer: Int`", "`answer: Text`"), + ] { + assert_check_rejected(&stale_output, "M008"); + } + + let missing_purpose = main_document().replace("测试函数的用途。\n\n- 函数", "- 函数"); + assert_rejected(&missing_purpose, "M005"); +} + +#[test] +fn markdown_contract_uses_source_syntax_for_related_types() { + let document = package_document( + "main", + &[], + &[], + &[], + &[function_section( + "关联名称", + "name", + &["value: Related<app.model.user.User>"], + &["result: Text"], + concat!( + "name(value: Related.Unloaded<app.model.user.User>()) (result: Text) { ", + "result = \"unloaded\" }\n", + "name(value: Related.Loaded<app.model.user.User>(user)) (result: Text) { ", + "result = user.name }\n", + ), + )], + ); + let mut sources = markdown_sources(&document); + sources.add( + "app/model/user.dever", + "public type User { name: Text(1, 64) }\n", + ); + checked(&sources); + + let stale = document.replace( + "value: Related<app.model.user.User>", + "value: app.model.user.User", + ); + let mut stale_sources = markdown_sources(&stale); + stale_sources.add( + "app/model/user.dever", + "public type User { name: Text(1, 64) }\n", + ); + let errors = dever_core::check(&stale_sources).unwrap_err(); + assert!(errors.iter().any(|error| error.code == "M007")); +} + +#[test] +fn loading_ignores_regular_markdown_and_checks_nested_and_duplicate_packages() { + let directory = TemporaryDirectory::new(); + fs::create_dir_all(directory.path().join("account/user")).unwrap(); + fs::write( + directory.path().join("README.md"), + "```dever\nnot a program\n```\n", + ) + .unwrap(); + fs::write(directory.path().join("example.dever.md.bak"), "bad").unwrap(); + fs::write( + directory.path().join("main.dever"), + "main() (answer: Int) { answer = account.user.value() }\n", + ) + .unwrap(); + fs::write( + directory.path().join("account/user/app.dever.md"), + package_document( + "account.user.app", + &[], + &["value"], + &["account.user.value()"], + &[function_section( + "读取数值", + "value", + &[], + &["answer: Int"], + "value() (answer: Int) { answer = 42 }\n", + )], + ), + ) + .unwrap(); + let sources = SourceMap::load(directory.path()).unwrap(); + assert_eq!(sources.files().len(), 2); + checked(&sources); + + fs::write(directory.path().join("main.dever.md"), main_document()).unwrap(); + let sources = SourceMap::load(directory.path()).unwrap(); + let errors = dever_core::check(&sources).unwrap_err(); + assert!( + errors + .iter() + .any(|error| error.code == "C002" && error.message.contains("package")), + "{}", + render(&errors, &sources) + ); + + let mut wrong_path = SourceMap::default(); + wrong_path.add("wrong.dever.md", main_document()); + assert!( + dever_core::check(&wrong_path) + .unwrap_err() + .iter() + .any(|error| error.code == "M006") + ); +} + +#[test] +fn type_coverage_and_failure_contracts_remain_mandatory() { + let cases = [ + ( + "C005", + package_document( + "main", + &[], + &[], + &[], + &[function_section( + "主函数", + "main", + &[], + &["answer: Int"], + "main() (answer: Int) { answer = true }\n", + )], + ), + ), + ( + "C009", + package_document( + "main", + &[], + &[], + &[], + &[function_section( + "主函数", + "main", + &["value: Bool"], + &["answer: Int"], + "main(value: true) (answer: Int) { answer = 1 }\n", + )], + ), + ), + ( + "C012", + package_document( + "main", + &[], + &[], + &[], + &[ + type_section( + "结果", + "Result", + "分支", + &["error Failed"], + "type Result { error Failed }\n", + ), + function_section( + "主函数", + "main", + &["value: Result"], + &[], + "main(value: Result) () { discarded = value }\n", + ), + ], + ), + ), + ]; + for (code, text) in cases { + let sources = markdown_sources(&text); + let errors = dever_core::check(&sources).unwrap_err(); + assert!( + errors.iter().any(|error| error.code == code), + "{}", + render(&errors, &sources) + ); + } +} + +#[test] +fn source_locations_preserve_unicode_and_all_line_endings() { + for newline in ["\n", "\r\n", "\r"] { + let text = package_document( + "main", + &[], + &[], + &[], + &[function_section( + "函数", + "main", + &[], + &["answer: Int"], + "main() (answer: Int) {\n label = \"中文\"\n answer = missing(label)\n}\n", + )], + ) + .replace('\n', newline); + let sources = markdown_sources(&text); + let source = &sources.files()[0]; + assert_eq!(source.text(), text); + let errors = dever_core::check(&sources).unwrap_err(); + let error = errors + .iter() + .find(|error| error.code == "C004") + .unwrap_or_else(|| panic!("newline {newline:?}: {}", render(&errors, &sources))); + assert_eq!(error.primary.start, text.find("missing").unwrap()); + assert_eq!(source.position(error.primary.start), (22, 12)); + assert!(error.render(&sources).contains("main.dever.md:")); + + let invalid = text.replace("missing(label)", "\"中文\" @"); + let sources = markdown_sources(&invalid); + let errors = dever_core::parse(&sources.files()[0]).unwrap_err(); + let error = errors.iter().find(|error| error.code == "L001").unwrap(); + assert_eq!(error.primary.start, invalid.find('@').unwrap()); + assert_eq!(sources.files()[0].position(error.primary.start), (22, 17)); + } +} + +#[test] +fn native_fault_points_to_original_markdown_line() { + let text = package_document( + "main", + &[], + &[], + &[], + &[ + function_section( + "除法", + "divide", + &["left: Int", "right: Int"], + &["answer: Int"], + "divide(left: Int, right: Int) (answer: Int) {\n answer = left // right\n}\n", + ), + function_section( + "主函数", + "main", + &[], + &["answer: Int"], + "main() (answer: Int) { answer = divide(1, 0) }\n", + ), + ], + ) + .replace('\n', "\r\n"); + let output = execute(&markdown_sources(&text)); + assert!(!output.status.success()); + let error = String::from_utf8(output.stderr).unwrap(); + assert!(error.contains("main.dever.md:"), "{error}"); + assert!(error.contains("Int division by zero"), "{error}"); +} + +#[test] +fn showcase_fences_quotes_lists_html_and_other_languages_never_execute() { + let examples = [ + "````markdown\n```dever\ninvalid @ program\n```\n````\n", + "> ```dever\n> invalid @ program\n> ```\n", + "- Example:\n\n ```dever\n invalid @ program\n ```\n", + "<pre>\n```dever\ninvalid @ program\n```\n</pre>\n", + "<!--\n```dever\ninvalid @ program\n```\n-->\n", + "<div>\n```dever\ninvalid @ program\n```\n</div>\n", + "```rust\ninvalid @ program\n```\n", + "```rust dever\ninvalid @ program\n```\n", + "```\ninvalid @ program\n```\n", + " ```dever\n invalid @ program\n ```\n", + "```dever example\ninvalid @ program\n```\n", + "```dever\ninvalid @ program\n```\n", + "~~~dever\\\ninvalid @ program\n~~~\n", + ]; + for example in examples { + let text = format!("{example}\n{}", main_document()); + let sources = markdown_sources(&text); + checked(&sources); + assert!(formatted(&text).starts_with(example), "{example}"); + } +} + +#[test] +fn fences_accept_commonmark_indentation_tildes_and_longer_closures() { + // After a Markdown list, CommonMark treats two or three spaces as list nesting. + for indentation in ["", " "] { + for marker in ["```", "~~~~"] { + for newline in ["\n", "\r\n", "\r"] { + let text = format!( + "# 说明\n\n测试包的用途。\n\n- 包：`main`\n- 公开类型：无\n- 公开方法：无\n- 使用：无\n\n## 主函数\n\n测试函数的用途。\n\n- 函数：`main`\n- 输入：无\n- 输出：\n - `answer: Int`：测试说明\n\n\n{indentation}{marker} dever \t\n{MAIN}{indentation}{marker}{} \n尾文", + &marker[..1] + ) + .replace('\n', newline); + let sources = markdown_sources(&text); + if let Err(errors) = dever_core::check(&sources) { + panic!( + "indentation={indentation:?}, marker={marker:?}, newline={newline:?}: {}", + render(&errors, &sources) + ); + } + let output = formatted(&text); + assert_eq!(formatted(&output), output); + assert!(output.ends_with(&format!( + "{indentation}{marker}{} {newline}尾文", + &marker[..1] + ))); + assert!(output.contains(&format!("{indentation} answer = 42{newline}"))); + } + } + } + for indentation in [" ", " "] { + let text = format!("# 说明\n\n说明。\n\n{indentation}```dever\n{indentation}```\n"); + let sources = markdown_sources(&text); + let errors = dever_core::parse(&sources.files()[0]).unwrap_err(); + assert!( + errors.iter().any(|error| error.code == "M004"), + "{}", + render(&errors, &sources) + ); + assert!(!errors.iter().any(|error| error.code == "M001")); + } +} + +#[test] +fn no_program_block_and_unclosed_program_fences_are_explicit_errors() { + for text in ["", "# 只有说明\n", "```text\nexample\n```\n"] { + assert_rejected(text, "M001"); + } + for text in [ + "```dever", + "```dever\n", + "```dever\n", + "```dever\n~~~\n", + "````dever\n```\n", + "```dever\n``` extra", + "```dever\n```\t\n", + "```dever\n```\t", + "```dever\n ```\n", + ] { + assert_rejected(text, "M002"); + } + assert_rejected( + "# 空包\n\n用于测试空包。\n\n- 包：`main`\n- 公开类型：无\n- 公开方法：无\n- 使用：无\n\n```dever\n```", + "M004", + ); + assert_rejected("```dever\n```\n", "M004"); +} + +#[test] +fn declarations_cannot_straddle_program_blocks() { + for blocks in [ + vec!["main() (answer: Int) {\n", "answer = 42\n}\n"], + vec!["type Pair {\n", "value: Int\n}\n", MAIN], + vec!["main(\n", ") (answer: Int) { answer = 42 }\n"], + ] { + assert_rejected(&unstructured_document(&blocks), "M003"); + } +} + +#[test] +fn lexical_failures_stay_inside_their_own_program_block() { + let text = + unstructured_document(&["main() (answer: Text) { answer = \"unfinished\n", "\"\n}\n"]); + assert_rejected(&text, "L005"); + let sources = markdown_sources(&text); + for error in dever_core::parse(&sources.files()[0]).unwrap_err() { + assert!( + !sources.files()[0] + .line(sources.files()[0].position(error.primary.start).0) + .starts_with("这一段") + ); + } +} + +#[test] +fn formatter_preserves_prose_fences_and_block_local_comments() { + let text = "# 不改标题 \r\n\r\n测试包的用途。\r\n\r\n- 包：`main`\r\n- 公开类型：无\r\n- 公开方法：无\r\n- 使用：无\r\n\r\n## 主函数\r\n\r\n测试函数的用途。\r\n\r\n- 函数：`main`\r\n- 输入：无\r\n- 输出：\r\n - `answer: Int`：返回值\r\n\r\n~~~dever \t\r\n# 函数前\r\nmain( )(answer:Int){answer=42 # 赋值\r\n# 函数内尾\r\n}\r\n# 函数后\r\n~~~~ \r\n\r\n```rust\r\nleave( this );\r\n```\r\n尾文不加换行"; + let output = formatted(text); + assert_eq!(formatted(&output), output); + let expected = text.replace( + "main( )(answer:Int){answer=42 # 赋值\r\n# 函数内尾\r\n}", + "main() (answer: Int) {\r\n answer = 42 # 赋值\r\n# 函数内尾\r\n}", + ); + assert_eq!(output, expected); + checked(&markdown_sources(&output)); +} diff --git a/test/dever-tests/tests/model_syntax.rs b/test/dever-tests/tests/model_syntax.rs new file mode 100644 index 0000000..5582f32 --- /dev/null +++ b/test/dever-tests/tests/model_syntax.rs @@ -0,0 +1,386 @@ +use dever_core::source::SourceMap; + +#[path = "support/temp.rs"] +mod temp; + +const USER: &str = r#" +database report + +public type UserStatus { + Active = "启用" + Disabled = "禁用" +} + +public type User { + uuid: Uuid generated unique + external_uuid: Uuid default "018f08d8-59b0-7cc2-98c4-dc0c0c07398f" + email: Text(254) unique + display_name: Text(1, 64) from nickname + status: UserStatus default UserStatus.Active index + enabled: Bool default true + score: Float? + balance: Decimal(20, 4) default 0 + avatar: Bytes(1048576)? + birthday: Date? + wake_at: Time? + active_for: Duration default 0 + profile: Json? +} + +index(status, created_at) + +relation articles = news.model.news.author_id + +seed { + { + email = "admin@example.com" + display_name = "Admin" + status = UserStatus.Active + enabled = true + balance = 0 + active_for = 0 + } +} + +migrate remove_legacy_code { + drop legacy_code +} + +public status_options() (options: Map<UserStatus, Text>) pure { + options = UserStatus.options +} +"#; + +const NEWS: &str = r#" +public type News { + author_id: user.model.user.id + title: Text(1, 160) + published_at: DateTime? +} +"#; + +fn sources(user: &str) -> SourceMap { + let mut sources = SourceMap::default(); + sources.add("user/model/user.dever", user); + sources.add("news/model/news.dever", NEWS); + sources +} + +fn errors(user: &str) -> String { + let sources = sources(user); + match dever_core::check(&sources) { + Ok(_) => panic!("Model source must be rejected"), + Err(errors) => errors.iter().map(|error| error.render(&sources)).collect(), + } +} + +#[test] +fn model_syntax_formats_idempotently_and_builds_a_stable_schema() { + let mut source_map = SourceMap::default(); + let id = source_map.add("user/model/user.dever", USER); + let formatted = dever_core::format::format(source_map.get(id)).unwrap(); + let mut reformatted_map = SourceMap::default(); + let reformatted_id = reformatted_map.add("user/model/user.dever", &formatted); + assert_eq!( + dever_core::format::format(reformatted_map.get(reformatted_id)).unwrap(), + formatted + ); + + let program = dever_core::check(&sources(USER)).unwrap(); + let snapshot = program.model_snapshot(); + assert!( + snapshot.contains("model user.model.user name=User table=user connection=explicit:report") + ); + assert!(snapshot.contains("field id id:user.model.user nullable=false generated=true")); + assert!(snapshot.contains("field created_at DateTime nullable=false generated=true")); + assert!(snapshot.contains( + "field display_name Text(1,64) nullable=false generated=false default=- from=nickname" + )); + assert!(snapshot.contains("field balance Decimal(20,4)")); + assert!(snapshot.contains("field external_uuid Uuid nullable=false generated=false")); + assert!( + snapshot.contains("choice user.model.user.UserStatus [Active=\"启用\",Disabled=\"禁用\"]") + ); + assert!(snapshot.contains("index unique (email) implicit=false")); + assert!(snapshot.contains("index plain (author_id) implicit=true")); + assert!(snapshot.contains("relation to-many articles = news.model.news.author_id")); + assert!(snapshot.contains("migrate remove_legacy_code drop(legacy_code)")); + + let native = + dever_core::native::emit(&program, &sources(USER), "user.model.user.status_options") + .unwrap(); + assert!(native.contains("Map::<T")); + assert!(native.contains("from_unique")); + + let formatted_program = dever_core::check(&sources(&formatted)).unwrap(); + assert_eq!(program.model_snapshot(), formatted_program.model_snapshot()); +} + +#[test] +fn model_rest_binding_modifiers_parse_and_format_in_source_order() { + let mut sources = SourceMap::default(); + let source = sources.add( + "user/profile/model.dever", + "type Profile {\nname:Text create=text.trim(input) replace=text.lower(input) search=input\n}\n", + ); + let formatted = dever_core::format::format(sources.get(source)).unwrap(); + assert!( + formatted.contains( + "name: Text create = text.trim(input) replace = text.lower(input) search = input" + ), + "{formatted}" + ); + let checked = dever_core::check(&sources).unwrap(); + assert!( + checked + .model_snapshot() + .contains("index plain (name) implicit=true") + ); +} + +#[test] +fn global_type_is_model_only_and_part_of_the_schema_identity() { + let mut sources = SourceMap::default(); + let global = sources.add( + "platform/tenant/model.dever", + "global type Tenant { name: Text(1, 64) }\n", + ); + sources.add( + "news/article/model.dever", + "type Article { tenant_id: platform.tenant.model.id\n tenant_ref: platform.tenant.model.id\n title: Text(1, 160) }\n", + ); + let formatted = dever_core::format::format(sources.get(global)).unwrap(); + assert!(formatted.starts_with("global type Tenant"), "{formatted}"); + + let program = dever_core::check(&sources).unwrap(); + let snapshot = program.model_snapshot(); + assert!( + snapshot.contains("model platform.tenant.model name=Tenant table=tenant connection=package:platform scope=global"), + "{snapshot}", + ); + assert!( + snapshot.contains("model news.article.model name=Article table=article connection=package:news scope=tenant"), + "{snapshot}", + ); + assert!( + snapshot.contains("relation logical to-one tenant = platform.tenant.model.tenant_id"), + "{snapshot}", + ); + assert!( + snapshot.contains("field tenant_ref id:platform.tenant.model nullable=false generated=false default=- from=- reference=logical"), + "{snapshot}", + ); + + sources.add( + "news/article/app.dever", + "list() (page: model.Page) { page = model.list({ with = [tenant] }) }\n", + ); + let errors = dever_core::check(&sources).unwrap_err(); + assert!(errors.iter().any(|error| { + error + .message + .contains("logical cross-scope relation 'tenant' cannot be loaded") + })); + + let mut regular = SourceMap::default(); + regular.add( + "user/account/app.dever", + "global type Identity { value: Text }\n", + ); + let errors = dever_core::check(®ular).unwrap_err(); + assert!(errors.iter().any(|error| { + error + .message + .contains("allowed only for the primary record") + })); + + let mut reverse = SourceMap::default(); + reverse.add( + "platform/tenant/model.dever", + "global type Tenant { article: news.article.model.id }\n", + ); + reverse.add("news/article/model.dever", "type Article { title: Text }\n"); + let errors = dever_core::check(&reverse).unwrap_err(); + assert!( + errors + .iter() + .any(|error| error.message.contains("cannot reference tenant Model")) + ); +} + +#[test] +fn authentication_verify_can_read_only_global_models() { + let root = temp::TemporaryDirectory::new(); + std::fs::create_dir(root.path().join("config")).unwrap(); + std::fs::write( + root.path().join("config/setting.json"), + r#"{ + "auth": {"providers": {"session": { + "verify": "user.account.verify", + "jwtSecret": "0123456789abcdef0123456789abcdef" + }}}, + "sites": {"admin": {"path": "admin", "auth": "session"}} + }"#, + ) + .unwrap(); + let settings = dever_runtime::config::Settings::load_project(root.path()).unwrap(); + let app = r#"type Identity { + id: Text + user_id: user.account.model.id? + tenant_id: platform.tenant.model.id? +} +verify(claims: dever.auth.Claims) (identity: Identity) { + identity = Identity { + id = claims.subject + ":" + int.to_text(model.count()) + user_id = null + tenant_id = null + } +} +profile() (value: Text) { value = "ok" } +"#; + let checked = |account: &str| { + let mut sources = SourceMap::default(); + sources.add("user/account/model.dever", account); + sources.add( + "platform/tenant/model.dever", + "global type Tenant { name: Text }\n", + ); + sources.add("user/account/app.dever", app); + sources.add( + "user/account/api/admin/profile.dever", + "get profile = app.profile\n", + ); + ( + dever_core::check_with_settings(&sources, &settings), + sources, + ) + }; + + let (result, sources) = checked("type Account { email: Text }\n"); + let errors = result.unwrap_err(); + let rendered = errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>(); + assert!( + rendered.contains("authentication verify may access only global Models"), + "{rendered}" + ); + + let (result, sources) = checked("global type Account { email: Text }\n"); + result.unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }); +} + +#[test] +fn model_contracts_reject_ambiguous_or_unsafe_declarations() { + let regular = { + let mut sources = SourceMap::default(); + sources.add("main.dever", "public type Value { name: Text(64) }"); + sources + }; + let regular_errors = dever_core::check(®ular).unwrap_err(); + assert!( + regular_errors + .iter() + .any(|error| error.message.contains("only allowed on Model fields")) + ); + + let invalid_generated = USER.replace( + "uuid: Uuid generated unique", + "uuid: Uuid? generated unique", + ); + assert!(errors(&invalid_generated).contains("requires a non-null Uuid field")); + + let invalid_uuid = USER.replace("018f08d8-59b0-7cc2-98c4-dc0c0c07398f", "not-a-uuid"); + assert!(errors(&invalid_uuid).contains("canonical 8-4-4-4-12 form")); + + let missing_key = USER.replace("email = \"admin@example.com\"\n", ""); + assert!(errors(&missing_key).contains("unique key")); + + let nullable_key = USER + .replace("email: Text(254) unique", "email: Text(254)? unique") + .replace("email = \"admin@example.com\"", "email = null"); + assert!(errors(&nullable_key).contains("non-null values")); + + let wrong_name = USER + .replace("type User {", "type Account {") + .replace(" User\n", " Account\n"); + assert!(errors(&wrong_name).contains("must be named 'User'")); + + let fake_id = USER.replace( + "status_options() (options: Map<UserStatus, Text>) pure {", + "fake_id() (value: user.model.user.id) {\n value = user.model.user.id {}\n}\n\nstatus_options() (options: Map<UserStatus, Text>) pure {", + ); + assert!(errors(&fake_id).contains("created only by database operations")); +} + +#[test] +fn model_defaults_and_seeds_validate_storage_constants_before_emission() { + let cases = [ + ("Int", "9223372036854775807", true), + ("Int", "-9223372036854775808", true), + ("Int", "1.5", false), + ("Int", "9223372036854775808", false), + ("Int", "-9223372036854775809", false), + ("Int", "1e2", false), + ("Float", "1e308", true), + ("Float", "-1e308", true), + ("Float", "1e309", false), + ("Float", "-1e309", false), + ("Decimal(3, 2)", "9.99", true), + ("Decimal(3, 2)", "-9.99", true), + ("Decimal(3, 2)", "1.2300", true), + ("Decimal(3, 2)", "10", false), + ("Decimal(3, 2)", "0.001", false), + ("Decimal(3, 2)", "1234.5678", false), + ("Decimal(34, 0)", "9999999999999999999999999999999999", true), + ("Text(1, 3)", "\"中😀文\"", true), + ("Text(1, 3)", "\"\"", false), + ("Text(1, 3)", "\"中😀文字\"", false), + ("DateTime", "1.5", false), + ("Date", "9223372036854775808", false), + ("Time", "1e2", false), + ("Duration", "-9223372036854775808", true), + ]; + for (ty, value, valid) in cases { + for seed in [false, true] { + let field = if seed { + ty.to_owned() + } else { + format!("{ty} default {value}") + }; + let seed = if seed { + format!("seed {{ {{ key = \"one\"\n value = {value} }} }}") + } else { + String::new() + }; + let text = format!( + "public type Item {{ key: Text unique\n value: {field} }}\n{seed}\npublic total_count() (total: Int) {{ total = app.model.item.count() }}\n" + ); + let mut sources = SourceMap::default(); + sources.add("app/model/item.dever", text); + match dever_core::check(&sources) { + Ok(program) => { + assert!(valid, "accepted {ty} {value}, {seed}"); + dever_core::native::emit(&program, &sources, "app.model.item.total_count") + .unwrap(); + } + Err(errors) => { + assert!(!valid, "rejected {ty} {value}: {errors:?}"); + assert!( + errors.iter().any(|error| error.code == "C014"), + "{errors:?}", + ); + } + } + } + } +} diff --git a/test/dever-tests/tests/native_cache.rs b/test/dever-tests/tests/native_cache.rs new file mode 100644 index 0000000..8ad2db2 --- /dev/null +++ b/test/dever-tests/tests/native_cache.rs @@ -0,0 +1,593 @@ +mod support; + +#[path = "../../../crates/dever-core/src/native/build/cache.rs"] +#[allow(dead_code)] +mod cache; +#[path = "../../../crates/dever-cli/src/api.rs"] +mod cli_api; + +use std::ffi::{OsStr, OsString}; +use std::fs; +use std::path::Path; +use std::process::Command; +use std::time::Instant; + +use dever_core::native::{NativeProgram, compile_with_cache}; +use support::temp::TemporaryDirectory; + +fn rustc() -> OsString { + std::env::var_os("RUSTC").unwrap_or_else(|| "rustc".into()) +} + +fn compile(source: &str, root: &Path, compiler: &OsStr) -> NativeProgram { + let sources = support::sources(source); + compile_with_cache( + &support::checked(&sources), + &sources, + "main.main", + compiler, + root, + ) + .unwrap() +} + +const HELLO: &str = "main() () { dever.io.println(\"cache hello\") }\n"; + +#[test] +fn identified_artifacts_share_owned_cache_and_reject_modified_bytes() { + use dever_core::native::compile_artifact; + + let directory = TemporaryDirectory::new(); + let root = directory.path().join("cache"); + let first = compile_artifact(&root, "llvm-source-and-pack-1", |_, output| { + fs::write(output, b"complete artifact").map_err(|error| error.to_string()) + }) + .unwrap(); + assert!(!first.cache_hit()); + let second = compile_artifact(&root, "llvm-source-and-pack-1", |_, _| { + panic!("an exact cache hit must not invoke the compiler") + }) + .unwrap(); + assert!(second.cache_hit()); + assert_ne!(first.executable(), second.executable()); + let output = directory.path().join("published"); + second.save(&output).unwrap(); + assert_eq!( + second.save(&output).unwrap_err().kind(), + std::io::ErrorKind::AlreadyExists + ); + let temporary = first.executable().parent().unwrap().to_path_buf(); + drop(first); + assert!(!temporary.exists()); + assert_eq!(fs::read(&output).unwrap(), b"complete artifact"); + + let stored = fs::read_dir(&root).unwrap().next().unwrap().unwrap().path(); + fs::write(stored.join("program"), b"tampered").unwrap(); + let result = compile_artifact(&root, "llvm-source-and-pack-1", |_, _| { + panic!("corruption must fail before compilation") + }); + assert!(result.err().unwrap().contains("incomplete or modified")); + let changed = compile_artifact(&root, "llvm-source-and-pack-2", |_, output| { + fs::write(output, b"changed input").map_err(|error| error.to_string()) + }) + .unwrap(); + assert!(!changed.cache_hit()); +} + +#[test] +fn identified_artifact_failure_releases_directory_without_publication() { + use dever_core::native::compile_artifact; + + let directory = TemporaryDirectory::new(); + let root = directory.path().join("cache"); + let mut temporary = None; + let result = compile_artifact(&root, "failed-link", |build, output| { + temporary = Some(build.to_path_buf()); + fs::write(output, b"incomplete").unwrap(); + Err("link rejected".into()) + }); + assert_eq!(result.err().unwrap(), "link rejected"); + assert!(!temporary.unwrap().exists()); + assert!(!root.exists()); +} + +#[test] +fn workspace_profiles_keep_independent_tests_without_debug_or_incremental_artifacts() { + let workspace = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let cargo = fs::read_to_string(workspace.join("Cargo.toml")).unwrap(); + for profile in ["dev", "test"] { + let section = cargo + .split(&format!("[profile.{profile}]")) + .nth(1) + .unwrap_or_else(|| panic!("missing profile.{profile}")) + .split("\n[") + .next() + .unwrap(); + assert!( + section.contains("debug = 0"), + "profile.{profile} enables debuginfo" + ); + assert!( + section.contains("incremental = false"), + "profile.{profile} enables incremental" + ); + } + let release = cargo + .split("[profile.release]") + .nth(1) + .unwrap() + .split("\n[") + .next() + .unwrap(); + assert!(release.contains("codegen-units = 1")); + assert!(release.contains("lto = \"fat\"")); + + let tests = fs::read_to_string(workspace.join("test/dever-tests/Cargo.toml")).unwrap(); + for target in [ + "native_cache", + "api_routes", + "sqlite_orm", + "postgres_orm", + "external_component", + ] { + assert!( + tests.contains(&format!("name = \"{target}\"")), + "missing test target {target}" + ); + } + assert!( + tests.matches("[[test]]").count() >= 60, + "test targets were unexpectedly merged" + ); +} + +#[test] +fn repeated_build_reuses_native_code_and_keeps_owned_output_semantics() { + let directory = TemporaryDirectory::new(); + let root = directory.path().join("cache"); + let started = Instant::now(); + let first = compile(HELLO, &root, &rustc()); + let first_time = started.elapsed(); + assert!(!first.cache_hit()); + let started = Instant::now(); + let second = compile(HELLO, &root, &rustc()); + eprintln!( + "native cache: cold_ms={}, hit_ms={}", + first_time.as_millis(), + started.elapsed().as_millis() + ); + assert!(second.cache_hit()); + assert_ne!(first.executable(), second.executable()); + assert_eq!( + Command::new(second.executable()).output().unwrap().stdout, + b"cache hello\n" + ); + let output = directory.path().join("saved"); + second.save(&output).unwrap(); + let original = fs::read(&output).unwrap(); + assert_eq!( + second.save(&output).unwrap_err().kind(), + std::io::ErrorKind::AlreadyExists + ); + assert_eq!(fs::read(&output).unwrap(), original); + let temporary = second.executable().parent().unwrap().to_owned(); + drop(second); + assert!(!temporary.exists()); + for entry in fs::read_dir(&root).unwrap() { + let mut files: Vec<_> = fs::read_dir(entry.unwrap().path()) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect(); + files.sort(); + assert_eq!(files, ["executable-fingerprint", "identity", "program"]); + } + let modified = HELLO.replace("cache hello", "changed source"); + let changed = compile(&modified, &root, &rustc()); + assert!(!changed.cache_hit()); + assert_eq!( + Command::new(changed.executable()).output().unwrap().stdout, + b"changed source\n" + ); +} + +#[test] +fn modified_cached_executables_are_never_run() { + let directory = TemporaryDirectory::new(); + let root = directory.path().join("cache"); + let native = compile(HELLO, &root, &rustc()); + let entry = fs::read_dir(&root).unwrap().next().unwrap().unwrap().path(); + fs::write(entry.join("program"), "incomplete executable").unwrap(); + let sources = support::sources(HELLO); + let result = compile_with_cache( + &support::checked(&sources), + &sources, + "main.main", + &rustc(), + &root, + ); + let Err(error) = result else { + panic!("modified executable must not be reused") + }; + assert!(error.contains("incomplete or modified"), "{error}"); + assert_eq!( + Command::new(native.executable()).output().unwrap().stdout, + b"cache hello\n" + ); +} + +#[test] +fn concurrent_publish_never_exposes_partial_artifacts() { + let directory = TemporaryDirectory::new(); + let root = directory.path().join("cache"); + let natives = std::thread::scope(|scope| { + let first = scope.spawn(|| compile(HELLO, &root, &rustc())); + let second = scope.spawn(|| compile(HELLO, &root, &rustc())); + [first.join().unwrap(), second.join().unwrap()] + }); + for native in natives { + assert_eq!( + Command::new(native.executable()).output().unwrap().stdout, + b"cache hello\n" + ); + } + assert!(compile(HELLO, &root, &rustc()).cache_hit()); + assert_eq!(fs::read_dir(root).unwrap().count(), 1); +} + +#[cfg(unix)] +#[test] +fn changing_compiler_bytes_invalidates_even_when_version_is_unchanged() { + use std::os::unix::fs::{PermissionsExt, symlink}; + let directory = TemporaryDirectory::new(); + let root = directory.path().join("cache"); + let sysroot = Command::new(rustc()) + .args(["--print", "sysroot"]) + .output() + .unwrap(); + assert!(sysroot.status.success()); + let bin = Path::new(std::str::from_utf8(&sysroot.stdout).unwrap().trim()).join("bin"); + let wrapper = directory.path().join("rustc"); + let quoted = bin + .join("rustc") + .display() + .to_string() + .replace('\'', "'\\''"); + let script = format!("#!/bin/sh\nexec '{quoted}' \"$@\"\n"); + fs::write(&wrapper, &script).unwrap(); + fs::set_permissions(&wrapper, fs::Permissions::from_mode(0o700)).unwrap(); + symlink(bin.join("cargo"), directory.path().join("cargo")).unwrap(); + assert!(!compile(HELLO, &root, wrapper.as_os_str()).cache_hit()); + assert!(compile(HELLO, &root, wrapper.as_os_str()).cache_hit()); + fs::write(&wrapper, format!("{script}# changed wrapper\n")).unwrap(); + assert!(!compile(HELLO, &root, wrapper.as_os_str()).cache_hit()); +} + +#[test] +fn runtime_dependencies_toolchain_version_and_options_are_cache_inputs() { + let directory = TemporaryDirectory::new(); + let runtime = directory.path().join("runtime"); + fs::create_dir(&runtime).unwrap(); + fs::create_dir(runtime.join("deps")).unwrap(); + fs::write(runtime.join("libdever_runtime.rlib"), "runtime version one").unwrap(); + fs::write( + runtime.join("deps/libdependency.rlib"), + "dependency version one", + ) + .unwrap(); + let root = directory.path().join("cache"); + fs::create_dir(&root).unwrap(); + let program = directory.path().join("program"); + fs::write(&program, "test executable bytes").unwrap(); + let inputs = cache::RuntimeInputs::read(&runtime).unwrap(); + let memoized_inputs = cache::RuntimeInputs::read(&runtime).unwrap(); + let original = cache::Cache::new( + &root, + "generated", + &rustc(), + b"version one", + &inputs, + &["opt-level=3"], + ) + .unwrap(); + let staging = directory.path().join("staging"); + fs::create_dir(&staging).unwrap(); + original.publish(&program, &staging).unwrap(); + assert!( + original + .restore(&directory.path().join("restored")) + .unwrap() + ); + let memoized = cache::Cache::new( + &root, + "generated", + &rustc(), + b"version one", + &memoized_inputs, + &["opt-level=3"], + ) + .unwrap(); + assert!( + memoized + .restore(&directory.path().join("memoized")) + .unwrap() + ); + for (version, options) in [ + (b"version two".as_slice(), "opt-level=3"), + (b"version one".as_slice(), "opt-level=2"), + ] { + let changed = + cache::Cache::new(&root, "generated", &rustc(), version, &inputs, &[options]).unwrap(); + assert!(!changed.restore(&directory.path().join("absent")).unwrap()); + } + for path in [ + runtime.join("libdever_runtime.rlib"), + runtime.join("deps/libdependency.rlib"), + ] { + fs::write(path, "changed bytes").unwrap(); + let changed_inputs = cache::RuntimeInputs::read(&runtime).unwrap(); + let changed = cache::Cache::new( + &root, + "generated", + &rustc(), + b"version one", + &changed_inputs, + &["opt-level=3"], + ) + .unwrap(); + assert!(!changed.restore(&directory.path().join("absent")).unwrap()); + } + let snapshot = directory.path().join("snapshot"); + inputs.write(&snapshot).unwrap(); + assert_eq!( + fs::read_to_string(snapshot.join("libdever_runtime.rlib")).unwrap(), + "runtime version one" + ); + assert_eq!( + fs::read_dir(&runtime) + .unwrap() + .filter_map(Result::ok) + .filter(|entry| { + entry + .file_name() + .to_string_lossy() + .starts_with(".dever-runtime-identity-") + }) + .count(), + 1 + ); +} + +#[test] +fn runtime_input_views_are_exact_reused_and_concurrently_published() { + let directory = TemporaryDirectory::new(); + let source = directory.path().join("cargo/release/deps"); + fs::create_dir_all(&source).unwrap(); + let runtime = directory.path().join("cargo/release/libdever_runtime.rlib"); + fs::write(&runtime, "runtime one").unwrap(); + let first_dependency = source.join("libfirst-1.rlib"); + let second_dependency = source.join("libsecond-2.rlib"); + fs::write(&first_dependency, "first").unwrap(); + fs::write(&second_dependency, "second").unwrap(); + let inputs = directory.path().join("inputs"); + + let first = + cache::RuntimeInputs::publish(&inputs, &runtime, std::slice::from_ref(&first_dependency)) + .unwrap(); + fs::remove_file(&runtime).unwrap(); + fs::write(&runtime, "runtime two").unwrap(); + let second = std::thread::scope(|scope| { + let left = scope.spawn(|| { + cache::RuntimeInputs::publish( + &inputs, + &runtime, + std::slice::from_ref(&second_dependency), + ) + .unwrap() + }); + let right = scope.spawn(|| { + cache::RuntimeInputs::publish( + &inputs, + &runtime, + std::slice::from_ref(&second_dependency), + ) + .unwrap() + }); + [left.join().unwrap(), right.join().unwrap()] + }); + assert_eq!(fs::read_dir(&inputs).unwrap().count(), 2); + + let first_output = directory.path().join("first-output"); + first.write(&first_output).unwrap(); + assert!(first_output.join("deps/libfirst-1.rlib").is_file()); + assert!(!first_output.join("deps/libsecond-2.rlib").exists()); + for (index, input) in second.into_iter().enumerate() { + let output = directory.path().join(format!("second-output-{index}")); + input.write(&output).unwrap(); + assert_eq!( + fs::read_to_string(output.join("libdever_runtime.rlib")).unwrap(), + "runtime two" + ); + assert!(output.join("deps/libsecond-2.rlib").is_file()); + assert!(!output.join("deps/libfirst-1.rlib").exists()); + } +} + +#[cfg(unix)] +#[test] +fn runtime_input_views_reject_unknown_files_and_symbolic_links() { + use std::os::unix::fs::symlink; + + let directory = TemporaryDirectory::new(); + let runtime = directory.path().join("runtime"); + fs::create_dir(&runtime).unwrap(); + fs::create_dir(runtime.join("deps")).unwrap(); + fs::write(runtime.join("libdever_runtime.rlib"), "runtime").unwrap(); + fs::write(runtime.join("deps/libdependency.rlib"), "dependency").unwrap(); + + fs::write(runtime.join("deps/unknown.txt"), "unknown").unwrap(); + assert!(cache::RuntimeInputs::read(&runtime).is_err()); + fs::remove_file(runtime.join("deps/unknown.txt")).unwrap(); + + fs::remove_file(runtime.join("deps/libdependency.rlib")).unwrap(); + let outside = directory.path().join("outside.rlib"); + fs::write(&outside, "outside").unwrap(); + symlink(&outside, runtime.join("deps/libdependency.rlib")).unwrap(); + assert!(cache::RuntimeInputs::read(&runtime).is_err()); +} + +#[test] +fn checked_program_cache_preserves_source_identity_and_rechecks_api_baselines() { + let original = support::sources(HELLO); + let started = Instant::now(); + let program = support::checked(&original); + let cold = started.elapsed(); + let started = Instant::now(); + let cached = support::checked(&original); + eprintln!( + "frontend cache: cold_us={}, hit_us={}", + cold.as_micros(), + started.elapsed().as_micros() + ); + assert_eq!(program.api_snapshot(), cached.api_snapshot()); + let directory = TemporaryDirectory::new(); + let baseline = directory.path().join("dever.api"); + cli_api::write(&program, Some(&baseline)).unwrap(); + cli_api::check_baseline(&cached, directory.path()).unwrap(); + fs::write(&baseline, "changed baseline").unwrap(); + assert!(cli_api::check_baseline(&support::checked(&original), directory.path()).is_err()); + + let mut wrong_path = dever_core::source::SourceMap::default(); + wrong_path.add("wrong.dever", HELLO); + let wrong_program = support::checked(&wrong_path); + assert!(dever_core::native::emit(&wrong_program, &wrong_path, "main.main").is_err()); + let changed = support::sources(&HELLO.replace("dever.io.println", "unknown")); + let errors = dever_core::check(&changed).unwrap_err(); + let repeated_errors = dever_core::check(&changed).unwrap_err(); + assert_eq!( + errors[0].render(&changed), + repeated_errors[0].render(&changed) + ); + assert!(errors.iter().any(|error| error.code == "C004")); + + let mut additional = dever_core::source::SourceMap::default(); + additional.add("main.dever", HELLO); + additional.add("user/account/app.dever", "extra() () {}\n"); + fs::write(&baseline, program.api_snapshot()).unwrap(); + assert!(cli_api::check_baseline(&support::checked(&additional), directory.path()).is_err()); +} + +#[test] +#[ignore = "requires a freshly built local dever-cli binary"] +fn actual_cli_checks_baseline_on_check_run_and_cached_build() { + let compiler = std::env::current_exe() + .unwrap() + .parent() + .unwrap() + .parent() + .unwrap() + .join(format!("dever{}", std::env::consts::EXE_SUFFIX)); + assert!( + compiler.is_file(), + "build dever-cli before running this test" + ); + let directory = TemporaryDirectory::new(); + let domain = directory.path().join("module/sample/cache"); + fs::create_dir_all(&domain).unwrap(); + fs::create_dir(directory.path().join("config")).unwrap(); + fs::write( + domain.join("app.dever"), + "hello() (message: Text) { message = \"cache hello\" }\n", + ) + .unwrap(); + fs::write(domain.join("api.dever"), "cmd hello = app.hello\n").unwrap(); + fs::write(directory.path().join("config/setting.json"), "{}\n").unwrap(); + let baseline = directory.path().join("dever.api"); + let api = Command::new(&compiler) + .arg("api") + .arg(directory.path()) + .arg("--output") + .arg(&baseline) + .output() + .unwrap(); + assert!( + api.status.success(), + "{}", + String::from_utf8_lossy(&api.stderr) + ); + let check = Command::new(&compiler) + .arg("check") + .arg(directory.path()) + .output() + .unwrap(); + assert!( + check.status.success(), + "{}", + String::from_utf8_lossy(&check.stderr) + ); + let run = Command::new(&compiler) + .arg("run") + .arg(directory.path()) + .args(["--", "sample.cache.hello", "{}"]) + .output() + .unwrap(); + assert!( + run.status.success(), + "{}", + String::from_utf8_lossy(&run.stderr) + ); + let expected = serde_json::json!({"code": 0, "message": "ok", "data": "cache hello"}); + assert_eq!( + serde_json::from_slice::<serde_json::Value>(&run.stdout).unwrap(), + expected + ); + let saved = directory.path().join("saved"); + let build = Command::new(&compiler) + .arg("build") + .arg(directory.path()) + .arg("--output") + .arg(&saved) + .output() + .unwrap(); + assert!( + build.status.success(), + "{}", + String::from_utf8_lossy(&build.stderr) + ); + let standalone = Command::new(saved) + .current_dir(directory.path()) + .args(["sample.cache.hello", "{}"]) + .output() + .unwrap(); + assert!( + standalone.status.success(), + "{}", + String::from_utf8_lossy(&standalone.stderr) + ); + assert_eq!( + serde_json::from_slice::<serde_json::Value>(&standalone.stdout).unwrap(), + expected + ); + fs::write(baseline, "changed baseline").unwrap(); + let blocked_output = directory.path().join("blocked"); + for action in ["check", "run", "build"] { + let mut command = Command::new(&compiler); + command.arg(action).arg(directory.path()); + if action == "build" { + command.arg("--output").arg(&blocked_output); + } else if action == "run" { + command.args(["--", "sample.cache.hello", "{}"]); + } + let output = command.output().unwrap(); + assert!( + !output.status.success(), + "{action} skipped changed baseline" + ); + assert!( + String::from_utf8_lossy(&output.stderr).contains("API baseline"), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!(output.stdout.is_empty()); + } + assert!(!blocked_output.exists()); +} diff --git a/test/dever-tests/tests/native_entry.rs b/test/dever-tests/tests/native_entry.rs new file mode 100644 index 0000000..e04a488 --- /dev/null +++ b/test/dever-tests/tests/native_entry.rs @@ -0,0 +1,793 @@ +use dever_core::source::SourceMap; +use dever_runtime::api::{Inputs, QueryValue}; +use dever_runtime::bytes::Bytes; +use dever_runtime::config::RuntimeProfile; +use dever_runtime::http::{Header, Request}; +use std::fs; +use std::process::Command; + +#[path = "support/temp.rs"] +mod temp; + +fn source(files: &[(&str, &str)]) -> SourceMap { + let mut sources = SourceMap::default(); + for (path, text) in files { + sources.add(*path, *text); + } + sources +} + +#[test] +fn project_entry_collects_api_and_command_without_main() { + let sources = source(&[ + ( + "user/account/app.dever", + "greet(name: Text) (value: Text) { value = text.trim(name) }", + ), + ( + "user/account/api.dever", + "get greeting = app.greet\ncmd greeting = app.greet", + ), + ]); + let program = dever_core::check(&sources).unwrap(); + let code = + dever_core::native::emit_project(&program, &sources, RuntimeProfile::default()).unwrap(); + assert!(code.contains("dever_runtime::api::serve(api_route).await")); + assert!(code.contains("\"user.account.greeting\" => {")); + assert!(code.contains("let service_start = args.is_empty()")); + assert!(code.contains("cmd_dispatch(&args[0], &args[1]).await")); + assert!(!code.contains("main.main")); +} + +#[test] +fn project_entry_can_embed_verified_external_resources() { + let sources = source(&[ + ( + "user/account/app.dever", + "greet() (value: Text) { value = \"ok\" }", + ), + ("user/account/api.dever", "cmd greeting = app.greet"), + ]); + let program = dever_core::check(&sources).unwrap(); + let code = dever_core::native::emit_project_with_resources( + &program, + &sources, + RuntimeProfile { + sqlite: true, + postgres: false, + }, + &[dever_core::native::EmbeddedResource { + path: "worker.bin".into(), + bytes: b"worker".to_vec(), + sha256: "87eba76e7f3164534045ba922e7770fb58bbd14ad732bbf5ba6f11cc56989e6e".into(), + executable: true, + }], + ) + .unwrap(); + assert!(code.contains("dever_runtime::external::prepare")); + assert!(code.contains("DEVER_EXTERNAL_RESOURCES")); + assert!(code.contains("include_bytes!(\"external-resource-0.bin\")")); +} + +#[test] +fn large_external_resources_do_not_expand_generated_source_and_reject_forged_digests() { + use sha2::{Digest, Sha256}; + let sources = source(&[ + ( + "user/account/app.dever", + "greet() (value: Text) { value = \"ok\" }", + ), + ("user/account/api.dever", "cmd greeting = app.greet"), + ]); + let program = dever_core::check(&sources).unwrap(); + let baseline = + dever_core::native::emit_project(&program, &sources, RuntimeProfile::default()).unwrap(); + let bytes = vec![157; 1024 * 1024]; + let mut resource = dever_core::native::EmbeddedResource { + path: "runtime.bin".into(), + sha256: Sha256::digest(&bytes) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(), + bytes, + executable: true, + }; + let code = dever_core::native::emit_project_with_resources( + &program, + &sources, + RuntimeProfile::default(), + std::slice::from_ref(&resource), + ) + .unwrap(); + assert!( + code.len().saturating_sub(baseline.len()) < 8 * 1024, + "resource bytes must not become Rust literals" + ); + assert!(code.contains("include_bytes!(\"external-resource-0.bin\")")); + let mut copy = resource.clone(); + copy.path = "runtime-copy.bin".into(); + copy.executable = false; + let shared = dever_core::native::emit_project_with_resources( + &program, + &sources, + RuntimeProfile::default(), + &[resource.clone(), copy], + ) + .unwrap(); + assert_eq!(shared.matches("include_bytes!(").count(), 1); + assert!(!shared.contains("external-resource-1.bin")); + assert!(shared.contains("runtime-copy.bin")); + assert_eq!(shared.matches("bytes: DEVER_EXTERNAL_BYTES_0").count(), 2); + resource.bytes[512 * 1024] ^= 1; + assert!( + dever_core::native::emit_project_with_resources( + &program, + &sources, + RuntimeProfile::default(), + &[resource], + ) + .unwrap_err() + .contains("invalid digest") + ); +} + +#[test] +fn api_and_command_inputs_share_strict_json_validation() { + assert!(Inputs::from_json_object("{\"profile\":{\"name\":\"a\",\"name\":\"b\"}}").is_err()); + let mut inputs = Inputs::from_json_object("{}").unwrap(); + assert!(inputs.raw_json("required").is_err()); + assert_eq!(inputs.optional_raw_json("optional").unwrap(), None); + assert!(inputs.json("required", QueryValue::Text).is_err()); + assert_eq!( + inputs.optional_json("optional", QueryValue::Text).unwrap(), + "null" + ); + let request = Request { + method: "PUT".into(), + target: "/user/account/profile".into(), + headers: vec![Header { + name: "content-type".into(), + value: Bytes::from_text("application/json"), + }], + body: Bytes::from_text("{\"count\":12}"), + }; + let mut inputs = Inputs::from_request(&request).unwrap(); + assert_eq!(inputs.json("count", QueryValue::Int).unwrap(), "12"); + inputs.finish().unwrap(); +} + +#[test] +fn generated_write_entry_rejects_detached_concurrency() { + let sources = source(&[ + ( + "content/article/model.dever", + "type Article { title: Text(1, 32) }", + ), + ( + "content/article/app.dever", + "write() () { article = model.create({ title = \"published\" }) }\npublish() (value: Text) { task = run(write())\nwait(task)\nvalue = \"done\" }", + ), + ("content/article/api.dever", "post publish = app.publish"), + ]); + let errors = dever_core::check(&sources).unwrap_err(); + let rendered = errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>(); + assert!( + rendered.contains("transaction cannot start concurrent work"), + "{rendered}" + ); +} + +#[test] +fn generated_write_entry_allows_structured_blocking_work() { + let sources = source(&[ + ( + "user/account/model.dever", + "type Account { email: Text(1, 254)\nprivate password_hash: Text(255) }", + ), + ( + "user/account/app.dever", + r#"register(password: Secret) (id: model.id) { + hash = blocking(dever.crypto.password_hash(password)) + id = persist(hash) +} +transaction persist(hash: Text) (id: model.id) { + stored = model.create({ + email = "reader@example.com" + password_hash = hash + }) + id = stored.id +}"#, + ), + ("user/account/api.dever", "post register = app.register"), + ]); + let program = dever_core::check(&sources).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }); + let code = + dever_core::native::emit_project(&program, &sources, RuntimeProfile::default()).unwrap(); + assert!( + code.contains("transaction_database.begin().await"), + "{code}" + ); + assert!(code.contains("task::blocking_typed"), "{code}"); +} + +#[test] +fn http_request_capabilities_are_not_available_to_commands() { + let sources = source(&[ + ( + "user/account/app.dever", + "current() (value: Text) { value = dever.api.request_id() }", + ), + ("user/account/api.dever", "cmd current = app.current"), + ]); + let errors = dever_core::check(&sources).unwrap_err(); + let rendered = errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>(); + assert!( + rendered.contains("HTTP request capabilities are not available from CMD or Job"), + "{rendered}" + ); +} + +#[test] +fn http_request_capabilities_cannot_escape_into_detached_tasks() { + let sources = source(&[ + ( + "user/account/app.dever", + "read() (value: Text) { value = dever.api.request_id() }\ncurrent() (value: Text) { task = run(read())\nvalue = wait(task) }", + ), + ("user/account/api.dever", "get current = app.current"), + ]); + let errors = dever_core::check(&sources).unwrap_err(); + let rendered = errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>(); + assert!(rendered.contains("HTTP request, identity, and site capabilities cannot be used inside detached or parallel work"), "{rendered}"); +} + +#[test] +fn generated_http_context_capabilities_compile_without_a_source_context_value() { + let sources = source(&[ + ( + "user/account/app.dever", + "current() (value: Text) { value = dever.api.request_id()\ndever.api.set_header(\"x-request-id\", value)\noptions = dever.api.secure_cookie_options()\ndever.api.set_cookie(\"theme\", \"dark\", options) }", + ), + ("user/account/api.dever", "get current = app.current"), + ]); + let program = dever_core::check(&sources).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }); + let rustc = std::env::var_os("RUSTC").unwrap_or_else(|| "rustc".into()); + let native = + dever_core::native::compile_project(&program, &sources, &rustc, RuntimeProfile::default()) + .unwrap(); + assert!(native.executable().is_file()); +} + +#[test] +fn generated_authenticated_route_compiles_with_safe_standard_error_mapping() { + let directory = temp::TemporaryDirectory::new(); + fs::create_dir(directory.path().join("config")).unwrap(); + fs::write( + directory.path().join("config/setting.json"), + r#"{ + "auth": { + "providers": { + "session": { + "verify": "user.account.verify", + "jwtSecret": "0123456789abcdef0123456789abcdef" + } + } + }, + "sites": { + "admin": {"path": "admin", "auth": "session"} + } + }"#, + ) + .unwrap(); + let settings = dever_runtime::config::Settings::load_project(directory.path()).unwrap(); + let sources = source(&[ + ( + "user/account/model.dever", + "global type Account { email: Text(1, 254) }", + ), + ( + "platform/tenant/model.dever", + "global type Tenant { name: Text(1, 64) }", + ), + ( + "user/account/app.dever", + r#"type Identity { + id: Text + user_id: user.account.model.id? + tenant_id: platform.tenant.model.id? +} +verify(claims: dever.auth.Claims) (identity: Identity) { + identity = Identity { + id = claims.subject + user_id = null + tenant_id = null + } +} +profile() (response: Text) { + fail(dever.api.Error.Forbidden) +} +permissions() (response: List<dever.auth.Permission>) { + response = dever.auth.permissions() +} +manage_role() (ok: Bool) { + dever.auth.save_role("reader", "Reader", false, ["user.account.admin.profile"]) + dever.auth.grant_role(1, "reader") + dever.auth.revoke_role(1, "reader") + dever.auth.disable_role("reader") + ok = true +} +type Work { value: Text } +submit() (id: Id) { + id = dever.job.enqueue(job.deliver, Work { value = "queued" }, "profile") +} +deliver(input: Work) () { + news.article.deliver(input.value) +}"#, + ), + ( + "user/account/api/admin/profile.dever", + "get profile = app.profile\nget permissions = app.permissions\npost manage_role = app.manage_role\npost submit = app.submit", + ), + ( + "user/account/job.dever", + "database default\njob deliver(input: app.Work) () retry(2) timeout(1000) { app.deliver(input) }", + ), + ( + "news/article/model.dever", + "type Article { title: Text(1, 128) }", + ), + ( + "news/article/app.dever", + "deliver(title: Text) () { stored = model.create({ title = title }) }", + ), + ( + "gateway/session/app.dever", + "login() (response: Text) { token = dever.auth.issue(\"account:1\", \"session:1\", null)\noptions = dever.api.secure_cookie_options()\ndever.api.set_secret_cookie(\"cms_session\", token, options)\nresponse = \"ok\" }", + ), + ( + "gateway/session/api/admin/login.dever", + "public post login = app.login", + ), + ]); + let program = dever_core::check_with_settings(&sources, &settings).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }); + let generated = dever_core::native::emit_project( + &program, + &sources, + RuntimeProfile { + sqlite: true, + postgres: false, + }, + ) + .unwrap(); + assert!(generated.contains("dever_runtime::api::standard_error(403)")); + assert!(generated.contains("dever_runtime::auth::validate_job_source(&provider, &site)")); + assert!(generated.contains( + "dever_runtime::auth::validate_job_identity(&verified_claims, &verified_identity)" + )); + assert!(generated.contains("dever_runtime::auth::store::authorize")); + assert!(generated.contains("AUTHORIZATION_PERMISSIONS.iter().any")); + assert!(generated.contains("permission_key")); + assert!(generated.contains("let auth_claims =")); + assert!(generated.contains("job_dispatch_verified(claim).await")); + let dispatch = generated.split_once("async fn job_dispatch(").unwrap().1; + let verify = dispatch.find("let identity:").unwrap(); + let permission = dispatch + .find("AUTHORIZATION_PERMISSIONS.iter().any") + .unwrap(); + let components = permission + + dispatch[permission..] + .find("job_require_components(&claim).await") + .unwrap(); + let role = components + + dispatch[components..] + .find("dever_runtime::auth::store::authorize") + .unwrap(); + assert!(verify < permission && permission < components && components < role); + let rustc = std::env::var_os("RUSTC").unwrap_or_else(|| "rustc".into()); + let native = dever_core::native::compile_project( + &program, + &sources, + &rustc, + RuntimeProfile { + sqlite: true, + postgres: false, + }, + ) + .unwrap(); + assert!(native.executable().is_file()); +} + +#[test] +fn command_only_entry_requires_explicit_selection() { + let sources = source(&[ + ( + "user/account/app.dever", + "greet() (value: Text) { value = \"ok\" }", + ), + ("user/account/api.dever", "cmd greeting = app.greet"), + ]); + let program = dever_core::check(&sources).unwrap(); + let code = + dever_core::native::emit_project(&program, &sources, RuntimeProfile::default()).unwrap(); + assert!(code.contains("application has no service entry; select a CMD and JSON input")); + assert!(code.contains("\"user.account.greeting\" => Some(false)")); + assert!(!code.contains("dever_runtime::api::serve(api_route)")); +} + +#[test] +fn application_build_rejects_unreachable_app_and_private_forwarder() { + let sources = source(&[ + ( + "user/account/domain.dever", + "base(value: Text) (result: Text) pure { result = text.trim(value) }\nalias(value: Text) (result: Text) pure { result = base(value) }", + ), + ( + "user/account/app.dever", + "greet(value: Text) (result: Text) { result = domain.alias(value) }\nunused() (result: Text) { result = \"unused\" }", + ), + ("user/account/api.dever", "get greeting = app.greet"), + ]); + let program = dever_core::check(&sources).unwrap(); + let errors = program.application_errors().unwrap(); + let rendered = errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>(); + assert!( + rendered.contains("unreachable from API, CMD, or Job"), + "{rendered}" + ); + assert!( + rendered.contains("only forwards unchanged inputs"), + "{rendered}" + ); +} + +#[test] +fn generated_api_and_command_binary_compiles() { + let sources = source(&[ + ( + "user/account/app.dever", + "greet(name: Text) (value: Text) { value = text.trim(name) }", + ), + ( + "user/account/api.dever", + "get greeting = app.greet\npost greeting = app.greet\ncmd greeting = app.greet", + ), + ]); + let program = dever_core::check(&sources).unwrap(); + assert!(program.application_errors().unwrap().is_empty()); + let rustc = std::env::var_os("RUSTC").unwrap_or_else(|| "rustc".into()); + let binary = + dever_core::native::compile_project(&program, &sources, &rustc, RuntimeProfile::default()) + .unwrap(); + assert!(binary.executable().is_file()); + let directory = temp::TemporaryDirectory::new(); + fs::create_dir(directory.path().join("config")).unwrap(); + fs::write(directory.path().join("config/setting.json"), "{}\n").unwrap(); + let executable = directory.path().join("app"); + binary.save(&executable).unwrap(); + let run = Command::new(&executable) + .args(["user.account.greeting", r#"{"name":" Ada "}"#]) + .output() + .unwrap(); + assert!( + run.status.success(), + "{}", + String::from_utf8_lossy(&run.stderr) + ); + assert_eq!( + String::from_utf8(run.stdout).unwrap().trim(), + r#"{"code":0,"message":"ok","data":"Ada"}"# + ); +} + +#[test] +fn generated_job_entry_wraps_model_writes_in_transaction() { + let sources = source(&[ + ( + "content/article/model.dever", + "type Article { title: Text(1, 32) }", + ), + ( + "content/article/app.dever", + "apply() () { article = model.create({ title = \"published\" }) }", + ), + ( + "content/article/job.dever", + "database default\njob publish() () retry(2) timeout(1000) { app.apply() }", + ), + ]); + let program = dever_core::check(&sources).unwrap(); + assert!(program.application_errors().unwrap().is_empty()); + let code = dever_core::native::emit_project( + &program, + &sources, + RuntimeProfile { + sqlite: true, + postgres: false, + }, + ) + .unwrap(); + assert!(code.contains("job_serve().await")); + assert!(code.contains("Some(&transaction)")); + assert!(code.contains("transaction.commit().await")); + assert!(code.contains("transaction.rollback().await")); + let rustc = std::env::var_os("RUSTC").unwrap_or_else(|| "rustc".into()); + let binary = dever_core::native::compile_project( + &program, + &sources, + &rustc, + RuntimeProfile { + sqlite: true, + postgres: false, + }, + ) + .unwrap(); + assert!(binary.executable().is_file()); +} + +#[test] +fn postgres_profile_compiles_without_connecting() { + let sources = source(&[ + ( + "content/article/model.dever", + "global type Article { title: Text(1, 32) }", + ), + ( + "content/article/app.dever", + "count() (total: Int) { total = model.count() }", + ), + ("content/article/api.dever", "cmd count = app.count"), + ]); + let program = dever_core::check(&sources).unwrap(); + assert!(program.application_errors().unwrap().is_empty()); + let rustc = std::env::var_os("RUSTC").unwrap_or_else(|| "rustc".into()); + let binary = dever_core::native::compile_project( + &program, + &sources, + &rustc, + RuntimeProfile { + sqlite: false, + postgres: true, + }, + ) + .unwrap(); + assert!(binary.executable().is_file()); +} + +#[cfg(feature = "sqlite")] +#[test] +fn generated_tenant_migration_is_explicit_idempotent_and_never_falls_back() { + use dever_runtime::orm::Value; + use dever_runtime::sqlite::Database; + + let directory = temp::TemporaryDirectory::new(); + fs::create_dir(directory.path().join("config")).unwrap(); + fs::write( + directory.path().join("config/setting.json"), + r#"{ + "tenant": { + "database": "default", + "max_pools": 4, + "idle_timeout_ms": 60000 + }, + "database": { + "default": { + "type": "sqlite", + "path": "data/platform.db", + "tenant_directory": "data/tenants", + "max_connections": 2 + } + } + }"#, + ) + .unwrap(); + let settings = dever_runtime::config::Settings::load_project(directory.path()).unwrap(); + let sources = source(&[ + ( + "content/article/model.dever", + "type Article { title: Text(1, 160) }", + ), + ( + "content/article/app.dever", + "count() (total: Int) { total = model.count() }", + ), + ("content/article/api.dever", "cmd count = app.count"), + ]); + let program = dever_core::check_with_settings(&sources, &settings).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }); + assert!(program.application_errors().unwrap().is_empty()); + let rustc = std::env::var_os("RUSTC").unwrap_or_else(|| "rustc".into()); + let binary = dever_core::native::compile_project( + &program, + &sources, + &rustc, + RuntimeProfile { + sqlite: true, + postgres: false, + }, + ) + .unwrap(); + let executable = directory.path().join("app"); + binary.save(&executable).unwrap(); + let tenant_database = directory.path().join("data/tenants/tenant_7.db"); + + let unscoped = Command::new(&executable) + .args(["content.article.count", "{}"]) + .output() + .unwrap(); + assert!(!unscoped.status.success()); + assert!( + String::from_utf8_lossy(&unscoped.stderr).contains("requires an explicit tenant"), + "{}", + String::from_utf8_lossy(&unscoped.stderr), + ); + assert!(!tenant_database.exists()); + + for _ in 0..2 { + let migrated = Command::new(&executable) + .args(["--dever-tenant-migrate", "7"]) + .output() + .unwrap(); + assert!( + migrated.status.success(), + "{}", + String::from_utf8_lossy(&migrated.stderr), + ); + } + assert!(tenant_database.is_file()); + + let scoped = Command::new(&executable) + .args(["--dever-tenant-cmd", "7", "content.article.count", "{}"]) + .output() + .unwrap(); + assert!( + scoped.status.success(), + "{}", + String::from_utf8_lossy(&scoped.stderr) + ); + assert_eq!( + String::from_utf8(scoped.stdout).unwrap().trim(), + r#"{"code":0,"message":"ok","data":0}"#, + ); + + let disabled = Command::new(&executable) + .args(["--dever-tenant-component", "7", "disable", "content"]) + .output() + .unwrap(); + assert!( + disabled.status.success(), + "{}", + String::from_utf8_lossy(&disabled.stderr) + ); + let blocked = Command::new(&executable) + .args(["--dever-tenant-cmd", "7", "content.article.count", "{}"]) + .output() + .unwrap(); + assert!(!blocked.status.success()); + assert!( + String::from_utf8_lossy(&blocked.stderr).contains("tenant component 'content' is disabled"), + "{}", + String::from_utf8_lossy(&blocked.stderr), + ); + let enabled = Command::new(&executable) + .args(["--dever-tenant-component", "7", "enable", "content"]) + .output() + .unwrap(); + assert!( + enabled.status.success(), + "{}", + String::from_utf8_lossy(&enabled.stderr) + ); + let restored = Command::new(&executable) + .args(["--dever-tenant-cmd", "7", "content.article.count", "{}"]) + .output() + .unwrap(); + assert!( + restored.status.success(), + "{}", + String::from_utf8_lossy(&restored.stderr) + ); + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + runtime.block_on(async { + let tenant = Database::open(&tenant_database, 1, 100).unwrap(); + let marker = tenant + .query( + "SELECT tenant_id FROM _dever_tenant_ready WHERE marker = 1", + Vec::new(), + ) + .await + .unwrap(); + assert_eq!(marker[0].get(0).unwrap(), &Value::Int(7)); + tenant + .query( + "SELECT id, title, created_at FROM article LIMIT 1", + Vec::new(), + ) + .await + .unwrap(); + + let platform = Database::open(directory.path().join("data/platform.db"), 1, 100).unwrap(); + assert!( + platform + .query( + "SELECT id, title, created_at FROM article LIMIT 1", + Vec::new() + ) + .await + .is_err() + ); + }); +} + +#[test] +fn generated_command_rolls_back_when_output_encoding_fails() { + let sources = source(&[ + ( + "content/article/model.dever", + "type Article { title: Text(1, 32) }", + ), + ( + "content/article/app.dever", + "create() (article: model.Article) { article = model.create({ title = \"published\" }) }", + ), + ("content/article/api.dever", "cmd create = app.create"), + ]); + let program = dever_core::check(&sources).unwrap(); + let code = dever_core::native::emit_project( + &program, + &sources, + RuntimeProfile { + sqlite: true, + postgres: false, + }, + ) + .unwrap(); + assert!(code.contains("let encoded = match cmd_output_0_encode(&value)")); + assert!(code.contains("Err(error) => { let rollback = transaction.rollback().await;")); +} diff --git a/test/dever-tests/tests/native_ownership.rs b/test/dever-tests/tests/native_ownership.rs new file mode 100644 index 0000000..05c70cf --- /dev/null +++ b/test/dever-tests/tests/native_ownership.rs @@ -0,0 +1,309 @@ +mod support; + +#[test] +fn empty_byte_concatenation_reuses_storage_without_changing_value_aliases() { + use dever_runtime::bytes::Bytes; + let original = Bytes::from_text("abc"); + let empty = Bytes::new(Vec::new()); + let prefixed = empty.clone().concat(&original); + let suffixed = original.clone().concat(&empty); + assert_eq!(prefixed.values().as_ptr(), original.values().as_ptr()); + assert_eq!(suffixed.values().as_ptr(), original.values().as_ptr()); + let changed = prefixed.concat(&Bytes::from_text("d")); + assert_eq!(changed.values(), b"abcd"); + assert_eq!(original.values(), b"abc"); + assert_eq!(suffixed.values(), b"abc"); +} + +#[test] +fn owned_bytes_transfer_text_storage_in_both_directions() { + use dever_runtime::bytes::Bytes; + + let text = String::from("transfer"); + let allocation = text.as_ptr(); + let bytes = Bytes::from_string(text); + assert_eq!(bytes.values().as_ptr(), allocation); + let text = bytes.into_text().unwrap(); + assert_eq!(text.as_ptr(), allocation); + assert_eq!(text, "transfer"); + + let shared = Bytes::from_text("shared"); + assert_eq!(shared.clone().into_text().unwrap(), "shared"); + assert_eq!(shared.values(), b"shared"); +} + +#[test] +fn byte_slices_share_storage_and_consuming_iteration_preserves_aliases() { + use dever_runtime::bytes::Bytes; + let original = Bytes::from_text("abcdef"); + let slice = original.slice(2, 5).unwrap(); + assert_eq!(slice.values().as_ptr(), original.values()[2..].as_ptr()); + assert_eq!(slice.clone().into_values().collect::<Vec<_>>(), b"cde"); + assert_eq!(slice.concat(&Bytes::from_text("x")).values(), b"cdex"); + assert_eq!(original.values(), b"abcdef"); + assert_eq!( + Bytes::from_text("xyz").into_values().collect::<Vec<_>>(), + b"xyz" + ); +} + +#[test] +fn scalar_reads_after_record_transfer_preserve_old_values_and_reassignments() { + let source = r#"type State { text: Text +count: Int } +append_text(state: State) (next: State) { next = state +next.text = state.text + "x" +next.count = 99 } +step(value: Int, state: State) (next: State) { + next = append_text(state) + next.count = state.count + 1 +} +public main() (text: Text, count: Int, changed: Int) { + state = State { text = "a" +count = 0 } + final = reduce(step, [1, 2, 3], state) + text = final.text + count = final.count + copied = append_text(state) + state.count = 7 + changed = state.count +} +"#; + assert_eq!( + support::stdout(source), + "text = axxx\ncount = 3\nchanged = 7\n" + ); +} + +#[test] +fn total_collection_stages_fuse_but_faulting_stages_keep_eager_order() { + let source = r#"positive(value: Float) (result: Bool) { result = value > 0e0 } +twice(value: Float) (result: Float) { result = value * 2e0 } +public main() (result: Float) { + mapped = each(twice, [1e0, -2e0, 3e0]) + selected = filter(positive, mapped) + result = sum(selected) +} +"#; + let sources = support::sources(source); + let generated = + dever_core::native::emit(&support::checked(&sources), &sources, "main.main").unwrap(); + assert!(generated.contains("checked total handler")); + assert_eq!(support::stdout(source), "result = 8\n"); + + let faulting = r#"increment(value: Int) (result: Int) pure { result = value + 1 } +observe(value: Int) () { dever.io.println(int.to_text(value)) } +public main() () { each(observe, each(increment, [1, 9223372036854775807])) } +"#; + let output = support::run(faulting); + assert!(!output.status.success()); + assert!( + output.stdout.is_empty(), + "mapping must finish before any observation" + ); + assert!(String::from_utf8_lossy(&output.stderr).contains("overflow")); +} + +#[test] +fn record_updates_preserve_old_fields_aliases_and_expression_order() { + let source = r#"type Pair { first: Text +second: Text } +grow(value: Int, state: Pair) (next: Pair) { + next = state + next.first = state.first + "x" + next.second = state.second + "y" +} + +old_field(state: Pair) (next: Pair) { + next = state + next.first = "changed" + next.second = state.first +} +echo(value: Text) (result: Text) { dever.io.println(value) +result = value } +public main() (grown: Text, original: Text, old: Text, repeated: Text, siblings: Text) { + pair = Pair { first = "a" +second = "b" } + alias = pair + updated = reduce(grow, [1, 2, 3], pair) + grown = updated.first + updated.second + original = alias.first + alias.second + changed = old_field(alias) + old = changed.first + changed.second + pair.first = echo(pair.first) + echo(pair.first) + repeated = pair.first + pair.first = pair.second + pair.first + siblings = pair.first + pair.second +} +"#; + assert_eq!( + support::stdout(source), + "a\na\ngrown = axxxbyyy\noriginal = ab\nold = changeda\nrepeated = aa\nsiblings = baab\n" + ); +} + +#[test] +fn consumed_lists_move_unique_elements_and_clone_shared_elements_only_on_pull() { + use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }; + struct Counted(Arc<AtomicUsize>); + impl Clone for Counted { + fn clone(&self) -> Self { + self.0.fetch_add(1, Ordering::SeqCst); + Self(self.0.clone()) + } + } + let copies = Arc::new(AtomicUsize::new(0)); + let make = + || dever_runtime::collections::List::new((0..8).map(|_| Counted(copies.clone())).collect()); + let owned = make(); + assert_eq!(owned.into_values().count(), 8); + assert_eq!(copies.load(Ordering::SeqCst), 0); + let shared = make(); + let mut consumed = shared.clone().into_values(); + assert!(consumed.next().is_some()); + drop(consumed); + assert_eq!(copies.load(Ordering::SeqCst), 1); + assert_eq!(shared.values().len(), 8); +} + +#[test] +fn consuming_collection_lookups_move_unique_values_and_preserve_aliases() { + use dever_runtime::collections::{List, Map}; + use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }; + + struct Counted(Arc<AtomicUsize>); + impl Clone for Counted { + fn clone(&self) -> Self { + self.0.fetch_add(1, Ordering::SeqCst); + Self(Arc::clone(&self.0)) + } + } + + let copies = Arc::new(AtomicUsize::new(0)); + let counted = || Counted(Arc::clone(&copies)); + + assert!(List::new(vec![counted()]).into_first().is_some()); + assert_eq!(copies.load(Ordering::SeqCst), 0); + let shared_list = List::new(vec![counted()]); + assert!(shared_list.clone().into_first().is_some()); + assert_eq!(copies.load(Ordering::SeqCst), 1); + assert_eq!(shared_list.values().len(), 1); + + let key = "key".to_owned(); + let owned_map = Map::new(vec![(key.clone(), counted())]).unwrap(); + assert!(owned_map.into_get(&key).is_some()); + assert_eq!(copies.load(Ordering::SeqCst), 1); + let shared_map = Map::new(vec![(key.clone(), counted())]).unwrap(); + assert!(shared_map.clone().into_get(&key).is_some()); + assert_eq!(copies.load(Ordering::SeqCst), 2); + assert!(shared_map.get(&key).is_some()); + assert_eq!(copies.load(Ordering::SeqCst), 3); + + let entries = Map::new(vec![(key, counted())]).unwrap().into_entries(); + assert_eq!(entries.values().len(), 1); + assert_eq!(copies.load(Ordering::SeqCst), 3); +} + +#[test] +fn direct_split_consumption_preserves_unicode_empty_pieces_and_early_stop() { + let source = r#"join(value: Text, state: Text) (next: Text) { next = state + "[" + value + "]" } +stop(value: Text, state: Text) (next: Text, stop: Bool) { + dever.io.println(value) + next = state + value + stop = value == "好" +} +public main() (unicode: Text, empty: Text, separated: Text, stopped: Text, alias: List<Text>) { + unicode = reduce(join, text.split("你好🙂", ""), "") + empty = reduce(join, text.split("", ""), "seed") + separated = reduce(join, text.split(",a,,", ","), "") + stopped = reduce_until(stop, text.split("你好🙂", ""), "") + parts = text.split("ab", "") + consumed = reduce(join, parts, "") + alias = parts +} +"#; + let sources = support::sources(source); + let generated = + dever_core::native::emit(&support::checked(&sources), &sources, "main.main").unwrap(); + assert!(generated.contains("dever_runtime::text::pieces(")); + assert_eq!( + support::stdout(source), + "你\n好\nunicode = [你][好][🙂]\nempty = seed\nseparated = [][a][][]\nstopped = 你好\nalias = [a, b]\n" + ); +} + +#[test] +fn complete_decimal_domain_needs_no_runtime_range_comparison() { + let sources = support::sources( + "add(value: Int, total: Decimal) (next: Decimal) { next = total + value }\npublic main() (result: Decimal) { result = reduce(add, dever.bytes.from_text(\"abc\"), 0.0) }", + ); + let generated = + dever_core::native::emit(&support::checked(&sources), &sources, "main.main").unwrap(); + assert!( + !generated.contains(">= DecimalValue") && !generated.contains("<= DecimalValue"), + "{generated}" + ); + assert_eq!(support::stdout(sources.files()[0].text()), "result = 294\n"); +} + +#[test] +fn moved_inputs_keep_live_aliases_and_partial_assignments_independent() { + assert_eq!( + support::stdout( + r#"type Pair { + first: Text + second: Text +} +copy(value: Text) (answer: Text) { answer = value } +grow(value: Int, state: List<Int>) (next: List<Int>) { next = append(state, value) } +join(left: Text, right: Text) (answer: Text) { answer = left + right } +public main() (original: Text, changed: Text, repeated: Text, before: List<Int>, after: List<Int>, preserved: Text) { + pair = Pair { + first = copy("a") + second = copy("b") + } + alias = pair + pair.first = pair.second + pair.first + original = alias.first + alias.second + changed = pair.first + pair.second + value = copy("x") + repeated = join(value, value) + initial = [0] + after = reduce(grow, [1, 2, 3], initial) + before = initial + record = Pair { + first = copy("f") + second = copy("s") + } + field = record.first + preserved = record.second + field +} +"# + ), + "original = ab\nchanged = bab\nrepeated = xx\nbefore = [0]\nafter = [0, 1, 2, 3]\npreserved = sf\n" + ); +} + +#[test] +fn reduction_state_is_transferred_and_unique_values_are_not_cloned() { + let sources = support::sources( + r#"grow(value: Int, state: List<Int>) (next: List<Int>) { next = append(state, value) } +public main() (answer: List<Int>) { answer = reduce(grow, [1, 2, 3], []) } +"#, + ); + let generated = + dever_core::native::emit(&support::checked(&sources), &sources, "main.main").unwrap(); + assert!(!generated.contains("state.clone()"), "{generated}"); + assert!(!generated.contains("_p1.clone()"), "{generated}"); + assert!(!generated.contains("_v1.clone()"), "{generated}"); + assert_eq!( + support::stdout(sources.files()[0].text()), + "answer = [1, 2, 3]\n" + ); +} diff --git a/test/dever-tests/tests/orm_connections.rs b/test/dever-tests/tests/orm_connections.rs new file mode 100644 index 0000000..836f93d --- /dev/null +++ b/test/dever-tests/tests/orm_connections.rs @@ -0,0 +1,448 @@ +mod support; + +use std::ffi::OsString; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; + +use dever_core::source::SourceMap; +use dever_runtime::database::{Field, Model, Schema, SchemaSql}; +use dever_runtime::orm::{Row, Value}; +use dever_runtime::sqlite::Database; + +const GUARD_MODEL: &str = r#" +database z + +public type Guard { + code: Text(16) +} +"#; + +#[derive(Debug, PartialEq)] +struct ConnectionState { + tables: Vec<Row>, + history: Vec<Row>, + data: Vec<Vec<Row>>, +} + +#[derive(Debug, PartialEq)] +struct CommittedConnections { + a: ConnectionState, + default: ConnectionState, + report: ConnectionState, + root: ConnectionState, +} + +fn model_source(name: &str, database: Option<&str>, seed: &str) -> String { + let database = database + .map(|name| format!("database {name}\n\n")) + .unwrap_or_default(); + format!( + "{database}public type {name} {{\n code: Text(16) unique\n}}\n\nseed {{\n {{\n code = \"{seed}\"\n }}\n}}\n" + ) +} + +fn sources() -> SourceMap { + let mut sources = SourceMap::default(); + // The z-bound Model is deliberately first. Runtime migration order must not follow this order. + sources.add("zfirst/model/guard.dever", GUARD_MODEL); + sources.add( + "later/model/omega.dever", + model_source("Omega", Some("a"), "omega"), + ); + sources.add( + "earlier/model/beta.dever", + model_source("Beta", Some("a"), "beta"), + ); + sources.add( + "fallback/model/defaulted.dever", + model_source("Defaulted", None, "default"), + ); + sources.add( + "root/model/rooted.dever", + model_source("Rooted", None, "root"), + ); + sources.add( + "analytics/model/reported.dever", + model_source("Reported", Some("report"), "report"), + ); + sources.add("main.dever", "public main() () {}\n"); + sources +} + +#[test] +fn named_connection_models_are_valid_without_opening_a_database() { + let sources = sources(); + let program = dever_core::check(&sources).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }); + assert!( + program + .model_snapshot() + .contains("model zfirst.model.guard name=Guard") + ); +} + +fn compile() -> (dever_core::native::NativeProgram, String, String) { + let sources = sources(); + let program = dever_core::check(&sources).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }); + let snapshot = program.model_snapshot(); + let guard = snapshot + .lines() + .find(|line| line.starts_with("model zfirst.model.guard ")) + .expect("guard Model snapshot"); + let revision = snapshot_value(guard, "revision"); + let seed_revision = snapshot_value(guard, "seed"); + let rustc = std::env::var_os("RUSTC").unwrap_or_else(|| OsString::from("rustc")); + let native = dever_core::native::compile_application( + &program, + &sources, + "main.main", + &rustc, + dever_runtime::config::RuntimeProfile { + sqlite: true, + postgres: false, + }, + ) + .unwrap(); + (native, revision, seed_revision) +} + +fn snapshot_value(line: &str, name: &str) -> String { + line.split_whitespace() + .find_map(|part| part.strip_prefix(&format!("{name}="))) + .expect("Model snapshot value") + .to_owned() +} + +fn configure(root: &Path) { + fs::create_dir_all(root.join("config")).unwrap(); + fs::create_dir_all(root.join("data/db")).unwrap(); + fs::write( + root.join("config/setting.json"), + r#"{ + "database": { + "default": {"type":"sqlite","path":"data/db/default.db","max_connections":1,"max_page_size":20}, + "a": {"type":"sqlite","path":"data/db/a.db","max_connections":1,"max_page_size":20}, + "analytics": {"type":"sqlite","path":"data/db/analytics.db","max_connections":1,"max_page_size":20}, + "report": {"type":"sqlite","path":"data/db/report.db","max_connections":1,"max_page_size":20}, + "root": {"type":"sqlite","path":"data/db/root.db","max_connections":1,"max_page_size":20}, + "z": {"type":"sqlite","path":"data/db/z.db","max_connections":1,"max_page_size":20} + } +}"#, + ) + .unwrap(); +} + +fn database_path(root: &Path, name: &str) -> PathBuf { + root.join("data/db").join(format!("{name}.db")) +} + +fn run(native: &dever_core::native::NativeProgram) -> Output { + Command::new(native.executable()).output().unwrap() +} + +async fn execute(path: &Path, sql: String) { + let database = Database::open(path, 1, 20).unwrap(); + database.execute_batch(sql).await.unwrap(); +} + +fn guard_table(table: &str) -> String { + format!( + "CREATE TABLE \"{table}\" (\n \"id\" INTEGER PRIMARY KEY AUTOINCREMENT,\n \"code\" TEXT NOT NULL CHECK (length(\"code\") <= 16),\n \"created_at\" INTEGER NOT NULL DEFAULT (unixepoch() * 1000)\n)" + ) +} + +async fn prepare_guard_catalog_drift(path: &Path, revision: String, seed_revision: String) { + let database = Database::open(path, 1, 20).unwrap(); + database + .migrate(Model { + schema: Schema { + model: "zfirst.model.guard".into(), + table: "guard".into(), + revision, + seed_revision, + fields: vec![ + Field { + name: "id".into(), + ty: "id:zfirst.model.guard".into(), + nullable: false, + generated: true, + default: None, + rename_from: None, + }, + Field { + name: "code".into(), + ty: "Text(16)".into(), + nullable: false, + generated: false, + default: None, + rename_from: None, + }, + Field { + name: "created_at".into(), + ty: "DateTime".into(), + nullable: false, + generated: true, + default: Some("generated:created_at".into()), + rename_from: None, + }, + ], + indexes: vec![], + migrations: vec![], + }, + create_table: SchemaSql { + sqlite: guard_table("guard"), + postgres: String::new(), + }, + create_temporary_table: guard_table("_dever_guard_test"), + create_indexes: vec![], + postgres_columns: vec![], + postgres_constraints: vec![], + seeds: vec![], + data_migrations: vec![], + }) + .await + .unwrap(); + database + .execute_batch("ALTER TABLE guard ADD COLUMN rogue TEXT".into()) + .await + .unwrap(); +} + +async fn connection_state(path: &Path, queries: &[&'static str]) -> ConnectionState { + if !path.exists() { + return ConnectionState { + tables: vec![], + history: vec![], + data: vec![], + }; + } + let database = Database::open(path, 1, 20).unwrap(); + let tables = database + .query( + "SELECT name, sql FROM sqlite_schema WHERE type = 'table' AND name NOT LIKE 'sqlite_%' AND name != '_dever_history' ORDER BY name", + vec![], + ) + .await + .unwrap(); + let has_history = database + .query( + "SELECT name FROM sqlite_schema WHERE type = 'table' AND name = '_dever_history'", + vec![], + ) + .await + .unwrap(); + let history = if has_history.is_empty() { + vec![] + } else { + database + .query( + "SELECT model, kind, name, revision, definition FROM _dever_history ORDER BY model, kind, name", + vec![], + ) + .await + .unwrap() + }; + let mut data = Vec::new(); + for query in queries { + data.push(database.query(query, vec![]).await.unwrap()); + } + ConnectionState { + tables, + history, + data, + } +} + +async fn committed_connections( + a: &Path, + default: &Path, + report: &Path, + root: &Path, +) -> CommittedConnections { + CommittedConnections { + a: connection_state( + a, + &[ + "SELECT code FROM omega ORDER BY id", + "SELECT code FROM beta ORDER BY id", + ], + ) + .await, + default: connection_state(default, &["SELECT code FROM defaulted ORDER BY id"]).await, + report: connection_state(report, &["SELECT code FROM reported ORDER BY id"]).await, + root: connection_state(root, &["SELECT code FROM rooted ORDER BY id"]).await, + } +} + +fn text(row: &Row, index: usize) -> &str { + match row.get(index).unwrap() { + Value::Text(value) => value, + value => panic!("expected text at column {index}, got {value:?}"), + } +} + +fn table_names(state: &ConnectionState) -> Vec<&str> { + state.tables.iter().map(|row| text(row, 0)).collect() +} + +fn history_models(state: &ConnectionState) -> Vec<&str> { + state + .history + .iter() + .filter(|row| text(row, 1) == "schema") + .map(|row| text(row, 0)) + .collect() +} + +fn assert_complete_history(state: &ConnectionState, models: &[&str]) { + let expected = models + .iter() + .flat_map(|model| ["catalog", "schema", "seed"].map(|kind| (*model, kind, "current"))) + .collect::<Vec<_>>(); + let actual = state + .history + .iter() + .map(|row| (text(row, 0), text(row, 1), text(row, 2))) + .collect::<Vec<_>>(); + assert_eq!(actual, expected); +} + +#[tokio::test] +async fn native_sqlite_connections_bind_and_recover_in_stable_order() { + let (native, guard_revision, guard_seed_revision) = compile(); + let root = native.executable().parent().unwrap(); + configure(root); + let a = database_path(root, "a"); + let default = database_path(root, "default"); + let analytics = database_path(root, "analytics"); + let report = database_path(root, "report"); + let root_database = database_path(root, "root"); + let z = database_path(root, "z"); + + execute( + &a, + "CREATE TABLE beta (id INTEGER PRIMARY KEY, code INTEGER NOT NULL)".into(), + ) + .await; + prepare_guard_catalog_drift(&z, guard_revision, guard_seed_revision).await; + + let first = run(&native); + assert!(!first.status.success()); + let first_error = String::from_utf8(first.stderr).unwrap(); + assert!( + first_error.contains("table 'beta' exists without Dever schema history"), + "{first_error}" + ); + let first_a = connection_state(&a, &["SELECT code FROM omega ORDER BY id"]).await; + assert_eq!(table_names(&first_a), ["beta", "omega"]); + assert_eq!(history_models(&first_a), ["later.model.omega"]); + assert_complete_history(&first_a, &["later.model.omega"]); + assert_eq!(text(&first_a.data[0][0], 0), "omega"); + for pending in [&default, &report, &root_database] { + assert_eq!( + connection_state(pending, &[]).await, + ConnectionState { + tables: vec![], + history: vec![], + data: vec![], + } + ); + } + + execute(&a, "DROP TABLE beta".into()).await; + let second = run(&native); + assert!(!second.status.success()); + let second_error = String::from_utf8(second.stderr).unwrap(); + assert!( + second_error.contains("table 'guard' does not match its recorded Dever schema"), + "{second_error}" + ); + + let second_connections = committed_connections(&a, &default, &report, &root_database).await; + assert_eq!(table_names(&second_connections.a), ["beta", "omega"]); + assert_eq!( + history_models(&second_connections.a), + ["earlier.model.beta", "later.model.omega"] + ); + assert_complete_history( + &second_connections.a, + &["earlier.model.beta", "later.model.omega"], + ); + assert_eq!(table_names(&second_connections.default), ["defaulted"]); + assert_eq!( + history_models(&second_connections.default), + ["fallback.model.defaulted"] + ); + assert_complete_history(&second_connections.default, &["fallback.model.defaulted"]); + assert_eq!(table_names(&second_connections.report), ["reported"]); + assert_eq!( + history_models(&second_connections.report), + ["analytics.model.reported"] + ); + assert_complete_history(&second_connections.report, &["analytics.model.reported"]); + assert_eq!(table_names(&second_connections.root), ["rooted"]); + assert_eq!( + history_models(&second_connections.root), + ["root.model.rooted"] + ); + assert_complete_history(&second_connections.root, &["root.model.rooted"]); + assert_eq!(text(&second_connections.a.data[0][0], 0), "omega"); + assert_eq!(text(&second_connections.a.data[1][0], 0), "beta"); + assert_eq!(text(&second_connections.default.data[0][0], 0), "default"); + assert_eq!(text(&second_connections.report.data[0][0], 0), "report"); + assert_eq!(text(&second_connections.root.data[0][0], 0), "root"); + assert_eq!( + connection_state(&analytics, &[]).await, + ConnectionState { + tables: vec![], + history: vec![], + data: vec![], + } + ); + + execute(&z, format!("DROP TABLE guard; {}", guard_table("guard"))).await; + let third = run(&native); + assert!( + third.status.success(), + "{}", + String::from_utf8_lossy(&third.stderr) + ); + let third_connections = committed_connections(&a, &default, &report, &root_database).await; + let third_z = connection_state(&z, &["SELECT code FROM guard ORDER BY id"]).await; + assert_eq!(third_connections, second_connections); + assert_eq!(table_names(&third_z), ["guard"]); + assert_eq!(history_models(&third_z), ["zfirst.model.guard"]); + assert_complete_history(&third_z, &["zfirst.model.guard"]); + assert!(third_z.data[0].is_empty()); + + let fourth = run(&native); + assert!( + fourth.status.success(), + "{}", + String::from_utf8_lossy(&fourth.stderr) + ); + assert_eq!( + committed_connections(&a, &default, &report, &root_database).await, + third_connections + ); + assert_eq!( + connection_state(&z, &["SELECT code FROM guard ORDER BY id"]).await, + third_z + ); +} diff --git a/test/dever-tests/tests/orm_data_migration.rs b/test/dever-tests/tests/orm_data_migration.rs new file mode 100644 index 0000000..49378f9 --- /dev/null +++ b/test/dever-tests/tests/orm_data_migration.rs @@ -0,0 +1,442 @@ +#[path = "support/orm_sqlite.rs"] +mod orm_sqlite; +mod support; + +use std::fs; +use std::process::Command; + +use dever_core::source::SourceMap; +use dever_runtime::database::{DataMigration, MigrationPhase, Model, Sql}; +use dever_runtime::orm::{ErrorKind, Value}; +use dever_runtime::sqlite::Database; +use orm_sqlite::{created_at, database_state, id, migration, model, seed, text, text_field}; + +const SOURCE: &str = r#"type User { name: Text(1, 8) unique } +migrate normalize_names { + before { + sqlite = "UPDATE user SET name = ?1 WHERE name = ?2" + postgres = "UPDATE user SET name = $1 WHERE name = $2" + parameters = ["short", "too-long-name"] + } + drop legacy + after { + parameters = ["final"] + postgres = "UPDATE user SET name = $1" + sqlite = "UPDATE user SET name = ?1" + } +} +"#; + +fn sources(source: &str) -> SourceMap { + let mut sources = SourceMap::default(); + sources.add("app/user/model.dever", source); + sources.add( + "app/user/app.dever", + "main() (total: Int) { total = model.count() }\n", + ); + sources +} + +fn snapshot(source: &str) -> String { + let sources = sources(source); + support::checked(&sources).model_snapshot() +} + +#[test] +fn migration_sql_formats_and_hashes_all_execution_inputs() { + let source_map = sources(SOURCE); + let formatted = dever_core::format::format(&source_map.files()[0]).unwrap(); + let formatted_map = sources(&formatted); + assert_eq!( + formatted, + dever_core::format::format(&formatted_map.files()[0]).unwrap() + ); + let original = snapshot(SOURCE); + assert_eq!(original, snapshot(&formatted)); + for (before, after) in [ + ( + "[\"short\", \"too-long-name\"]", + "[\"other\", \"too-long-name\"]", + ), + ( + "UPDATE user SET name = ?1 WHERE name = ?2", + "UPDATE user SET name = ?1 WHERE name != ?2", + ), + ( + "UPDATE user SET name = $1 WHERE name = $2", + "UPDATE user SET name = $1 WHERE name != $2", + ), + ("before {", "after {"), + ] { + assert_ne!(original, snapshot(&SOURCE.replace(before, after))); + } +} + +#[test] +fn migration_sql_rejects_unbound_nonliteral_and_non_dml_operations() { + for sql in [ + "COMMIT", + "PRAGMA foreign_keys = OFF", + "DROP TABLE user", + "SELECT 1", + "UPDATE user SET name = ?1; DELETE FROM user WHERE name = ?2", + "UPDATE user SET name = ?1; /* hidden boundary */ COMMIT", + "UPDATE user SET name = ?3 WHERE name = ?2", + "UPDATE user SET name = $1 WHERE name = $2", + "-- only a comment", + "UPDATE _dever_history SET revision = ?1 WHERE name = ?2", + "DELETE FROM main.[_dever_history] WHERE name IN (?1, ?2)", + "UPDATE '_dever_history' SET revision = ?1 WHERE name = ?2", + "UPDATE `_Dever_history` SET revision = ?1 WHERE name = ?2", + "UPDATE sqlite_master SET name = ?1 WHERE name = ?2", + "UPDATE user SET name = '你好", + "UPDATE user SET name = '", + "UPDATE [你好 SET name = ?1 WHERE name = ?2", + ] { + let source = SOURCE.replacen("UPDATE user SET name = ?1 WHERE name = ?2", sql, 1); + let errors = dever_core::check(&sources(&source)).unwrap_err(); + assert!( + errors.iter().any(|error| error.code == "C014"), + "{sql}: {errors:?}" + ); + } + for sql in [ + r#"UPDATE \"_dever_history\" SET revision = $1 WHERE name = $2"#, + r#"UPDATE U&\"\\005fdever_history\" SET revision = $1 WHERE name = $2"#, + "UPDATE pg_catalog.pg_class SET relname = $1 WHERE relname = $2", + ] { + let source = SOURCE.replacen("UPDATE user SET name = $1 WHERE name = $2", sql, 1); + assert!( + dever_core::check(&sources(&source)) + .unwrap_err() + .iter() + .any(|error| error.code == "C014") + ); + } + for parameter in ["1 + 2", "name", "1.25", "9223372036854775808", "1e999"] { + let source = SOURCE.replacen( + "[\"short\", \"too-long-name\"]", + &format!("[{parameter}, \"old\"]"), + 1, + ); + assert!( + dever_core::check(&sources(&source)) + .unwrap_err() + .iter() + .any(|error| error.code == "C014") + ); + } + for omitted in [ + " parameters = [\"short\", \"too-long-name\"]\n", + " sqlite = \"UPDATE user SET name = ?1 WHERE name = ?2\"\n", + " postgres = \"UPDATE user SET name = $1 WHERE name = $2\"\n", + ] { + assert!( + dever_core::check(&sources(&SOURCE.replacen(omitted, "", 1))) + .unwrap_err() + .iter() + .any(|error| error.code == "P001") + ); + } +} + +#[test] +fn migration_sql_scanner_preserves_quoted_and_commented_boundaries() { + let source = SOURCE.replace( + "UPDATE user SET name = ?1 WHERE name = ?2", + "/* prefix */ UPDATE user SET name = ?1 WHERE name = ?2 AND '; ?90' = '; ?90'; -- tail", + ).replace( + "UPDATE user SET name = $1 WHERE name = $2", + "/* prefix /* nested */ */ UPDATE user SET name = $1 WHERE name = $2 AND $quoted$; $90$quoted$ = $quoted$; $90$quoted$; -- tail", + ); + snapshot(&source); + snapshot(&SOURCE.replace("UPDATE user SET", "UPDATE my_pg_notes SET")); +} + +#[test] +fn migration_sql_rejects_unclosed_postgres_dollar_quotes_without_panicking() { + for sql in [ + "UPDATE user SET name = $tag", + "UPDATE user SET name = $tag$unfinished", + "UPDATE user SET name = $$unfinished", + "UPDATE user SET name = $tag$你好", + ] { + let source = format!( + "type User {{ name: Text }}\n\ + migrate change_name {{ after {{\n\ + sqlite = \"UPDATE user SET name = 'updated'\"\n\ + postgres = \"{sql}\"\nparameters = []\n}} }}\n" + ); + let errors = dever_core::check(&sources(&source)).unwrap_err(); + assert!( + errors.iter().any(|error| error.code == "C014"), + "{sql}: {errors:?}" + ); + } +} + +fn initial() -> Model { + model( + "a000000000000001", + "empty", + vec![ + id(), + text_field("name", 64, false), + text_field("legacy", 64, true), + created_at(), + ], + vec![], + vec![], + vec![], + ) +} + +fn statement(phase: MigrationPhase, sql: &'static str, parameters: Vec<Value>) -> DataMigration { + DataMigration { + name: "normalize_names".into(), + phase, + sql: Sql { + sqlite: sql, + postgres: "unused SQLite test dialect", + }, + parameters, + } +} + +fn upgraded(after: &'static str) -> Model { + let mut model = model( + "a000000000000002", + "with-seed", + vec![id(), text_field("name", 8, false), created_at()], + vec![], + vec![migration("normalize_names", "transform-v1", &["legacy"])], + vec![seed( + "INSERT INTO user (name) VALUES (?1)", + vec![Value::Text("seed".into())], + )], + ); + model.data_migrations = vec![ + statement( + MigrationPhase::Before, + "UPDATE user SET name = ?1 WHERE name = ?2", + vec![ + Value::Text("short".into()), + Value::Text("too-long-name".into()), + ], + ), + statement( + MigrationPhase::After, + after, + vec![Value::Text("final".into())], + ), + ]; + model +} + +#[tokio::test] +async fn data_migrations_surround_ddl_run_once_and_skip_fresh_databases() { + let root = support::temp::TemporaryDirectory::new(); + let database = Database::open(root.path().join("existing.db"), 1, 50).unwrap(); + database.migrate(initial()).await.unwrap(); + database + .execute_batch("INSERT INTO user (name, legacy) VALUES ('too-long-name', 'old')".into()) + .await + .unwrap(); + database + .migrate(upgraded("UPDATE user SET name = ?1")) + .await + .unwrap(); + let state = database_state(&database).await; + assert_eq!(text(&state.data[0], 1), "final"); + assert_eq!(text(&state.data[1], 1), "seed"); + assert_eq!( + state + .columns + .iter() + .map(|row| text(row, 0)) + .collect::<Vec<_>>(), + ["id", "name", "created_at"] + ); + database + .migrate(upgraded("UPDATE user SET name = ?1")) + .await + .unwrap(); + assert_eq!(database_state(&database).await, state); + let mut changed = upgraded("UPDATE user SET name = ?1"); + changed.schema.migrations[0].revision = "transform-v2".into(); + assert_eq!( + database.migrate(changed).await.unwrap_err().kind(), + ErrorKind::Migration + ); + assert_eq!(database_state(&database).await, state); + let mut removed = upgraded("UPDATE user SET name = ?1"); + removed.schema.migrations.clear(); + assert_eq!( + database.migrate(removed).await.unwrap_err().kind(), + ErrorKind::Migration + ); + assert_eq!(database_state(&database).await, state); + + let fresh = Database::open(root.path().join("fresh.db"), 1, 50).unwrap(); + fresh + .migrate(upgraded("UPDATE missing_historical_table SET name = ?1")) + .await + .unwrap(); + assert_eq!(text(&database_state(&fresh).await.data[0], 1), "seed"); +} + +#[tokio::test] +async fn data_ddl_seed_and_history_roll_back_together_and_catalog_is_checked_first() { + let root = support::temp::TemporaryDirectory::new(); + let database = Database::open(root.path().join("rollback.db"), 1, 50).unwrap(); + database.migrate(initial()).await.unwrap(); + database + .execute_batch("INSERT INTO user (name, legacy) VALUES ('too-long-name', 'old')".into()) + .await + .unwrap(); + let before = database_state(&database).await; + assert!( + database + .migrate(upgraded("UPDATE missing_table SET name = ?1")) + .await + .is_err() + ); + assert_eq!(database_state(&database).await, before); + let mut bad_seed = upgraded("UPDATE user SET name = ?1"); + bad_seed.seeds[0].parameters[0] = Value::Text("long-seed-name".into()); + assert_eq!( + database.migrate(bad_seed).await.unwrap_err().kind(), + ErrorKind::Constraint + ); + assert_eq!(database_state(&database).await, before); + database + .execute_batch("ALTER TABLE user ADD COLUMN rogue TEXT".into()) + .await + .unwrap(); + drop(database); + let database = Database::open(root.path().join("rollback.db"), 1, 50).unwrap(); + let drifted = database_state(&database).await; + assert_eq!( + database + .migrate(upgraded("UPDATE user SET name = ?1")) + .await + .unwrap_err() + .kind(), + ErrorKind::Migration + ); + assert_eq!(database_state(&database).await, drifted); +} + +#[tokio::test] +async fn data_only_steps_keep_source_order_and_applied_history_is_immutable() { + let root = support::temp::TemporaryDirectory::new(); + let database = Database::open(root.path().join("data-only.db"), 1, 50).unwrap(); + database.migrate(initial()).await.unwrap(); + database + .execute_batch("INSERT INTO user (name) VALUES ('')".into()) + .await + .unwrap(); + let upgrade = || { + let mut model = initial(); + model.schema.revision = "a000000000000003".into(); + model.schema.migrations = vec![ + migration("normalize_names", "data-only-v1", &[]), + migration("keep_order", "data-only-v2", &[]), + ]; + model.data_migrations = vec![ + statement( + MigrationPhase::Before, + "UPDATE user SET name = name || ?1", + vec![Value::Text("first';--".into())], + ), + statement( + MigrationPhase::Before, + "UPDATE user SET name = name || ?1", + vec![Value::Text("second".into())], + ), + statement( + MigrationPhase::After, + "UPDATE user SET name = name || ?1", + vec![Value::Text("after".into())], + ), + ]; + model + }; + database.migrate(upgrade()).await.unwrap(); + assert_eq!( + text(&database_state(&database).await.data[0], 1), + "first';--secondafter" + ); + database + .execute_batch("UPDATE _dever_history SET applied_at = 1 WHERE kind = 'migration'".into()) + .await + .unwrap(); + database.migrate(upgrade()).await.unwrap(); + let history = database + .query( + "SELECT applied_at FROM _dever_history WHERE kind = 'migration'", + vec![], + ) + .await + .unwrap(); + assert!( + history + .iter() + .all(|row| row.get(0).unwrap() == &Value::Int(1)) + ); + let before = database_state(&database).await; + let mut reordered = upgrade(); + reordered.schema.migrations.reverse(); + assert_eq!( + database.migrate(reordered).await.unwrap_err().kind(), + ErrorKind::Migration + ); + assert_eq!(database_state(&database).await, before); +} + +#[test] +fn native_data_migration_executes_bound_values_during_upgrade() { + let root = support::temp::TemporaryDirectory::new(); + fs::create_dir(root.path().join("config")).unwrap(); + fs::write( + root.path().join("config/setting.json"), + r#"{"database":{"default":{"type":"sqlite","path":"data/app.db","max_connections":1}}}"#, + ) + .unwrap(); + let original = "type User { name: Text(64) unique\n legacy: Text? }\nseed { { name = \"too-long-name\"\n legacy = \"old\" } }\n"; + for (name, source) in [("initial", original), ("upgraded", SOURCE)] { + let sources = sources(source); + let program = support::checked(&sources); + let compiler = std::env::var_os("RUSTC").unwrap_or_else(|| "rustc".into()); + let native = dever_core::native::compile_application( + &program, + &sources, + "app.user.main", + &compiler, + dever_runtime::config::RuntimeProfile { + sqlite: true, + postgres: false, + }, + ) + .unwrap(); + let executable = root.path().join(name); + fs::copy(native.executable(), &executable).unwrap(); + for _ in 0..2 { + let output = Command::new(&executable).output().unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(output.stdout, b"total = 1\n"); + } + } + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + runtime.block_on(async { + let database = Database::open(root.path().join("data/app.db"), 1, 50).unwrap(); + assert_eq!(text(&database_state(&database).await.data[0], 2), "final"); + }); +} diff --git a/test/dever-tests/tests/orm_migration.rs b/test/dever-tests/tests/orm_migration.rs new file mode 100644 index 0000000..814baad --- /dev/null +++ b/test/dever-tests/tests/orm_migration.rs @@ -0,0 +1,570 @@ +#[path = "support/orm_sqlite.rs"] +mod orm_sqlite; +mod support; + +use dever_runtime::orm::{ErrorKind, Value}; +use dever_runtime::sqlite::Database; +use orm_sqlite::{ + TestField, bool_default, bytes_field, created_at, database_state, id, index, migration, model, + text, text_field, +}; + +fn text_model( + revision: &str, + text_maximum: usize, + bytes_maximum: usize, +) -> dever_runtime::database::Model { + model( + revision, + "seed-empty", + vec![ + id(), + text_field("name", text_maximum, false), + bytes_field("payload", bytes_maximum, false), + created_at(), + ], + vec![], + vec![], + vec![], + ) +} + +#[tokio::test] +async fn new_database_uses_only_the_final_schema_and_records_real_catalog() { + let root = support::temp::TemporaryDirectory::new(); + let database = Database::open(root.path().join("new.db"), 1, 50).unwrap(); + let final_model = model( + "1000000000000001", + "seed-empty", + vec![ + id(), + text_field("display_name", 128, false).renamed_from("nickname"), + bytes_field("payload", 16, true), + bool_default("enabled"), + created_at(), + ], + vec![index( + "_dever_idx_user_display_name", + &["display_name"], + true, + )], + vec![], + vec![], + ); + + database.migrate(final_model).await.unwrap(); + let state = database_state(&database).await; + + let columns = state + .columns + .iter() + .map(|row| text(row, 0)) + .collect::<Vec<_>>(); + assert_eq!( + columns, + ["id", "display_name", "payload", "enabled", "created_at"] + ); + assert!(!columns.contains(&"nickname")); + assert_eq!(state.history.len(), 3); + assert!( + state + .history + .iter() + .any(|row| { text(row, 0) == "schema" && text(row, 2) == "1000000000000001" }) + ); + assert!(state.catalog.iter().any(|row| { + text(row, 0) == "table" + && text(row, 2).contains("length(\"display_name\") <= 128") + && text(row, 2).contains("DEFAULT 1") + })); + assert!( + database + .query( + "SELECT name FROM sqlite_schema WHERE type = 'table' AND name LIKE '_dever_user_%'", + vec![], + ) + .await + .unwrap() + .is_empty() + ); +} + +#[tokio::test] +async fn rename_widen_and_add_fields_preserve_existing_data_and_are_idempotent() { + let root = support::temp::TemporaryDirectory::new(); + let database = Database::open(root.path().join("automatic.db"), 1, 50).unwrap(); + let initial = model( + "2000000000000001", + "seed-empty", + vec![ + id(), + text_field("nickname", 64, false), + bytes_field("payload", 8, false), + created_at(), + ], + vec![], + vec![], + vec![], + ); + database.migrate(initial).await.unwrap(); + database + .execute_batch( + "INSERT INTO user (nickname, payload) VALUES ('Administrator', X'0102')".into(), + ) + .await + .unwrap(); + + let upgraded = || { + model( + "2000000000000002", + "seed-empty", + vec![ + id(), + text_field("display_name", 128, false).renamed_from("nickname"), + bytes_field("payload", 16, false), + text_field("bio", 32, true), + bool_default("enabled"), + created_at(), + ], + vec![], + vec![], + vec![], + ) + }; + database.migrate(upgraded()).await.unwrap(); + let once = database_state(&database).await; + database.migrate(upgraded()).await.unwrap(); + assert_eq!(database_state(&database).await, once); + + let rows = database + .query( + "SELECT display_name, payload, bio, enabled FROM user", + vec![], + ) + .await + .unwrap(); + assert_eq!( + rows[0].get(0).unwrap(), + &Value::Text("Administrator".into()) + ); + assert_eq!(rows[0].get(1).unwrap(), &Value::Bytes(vec![1, 2])); + assert_eq!(rows[0].get(2).unwrap(), &Value::Null); + assert_eq!(rows[0].get(3).unwrap(), &Value::Int(1)); +} + +#[tokio::test] +async fn text_and_bytes_tightening_succeeds_or_rolls_back_as_one_unit() { + let root = support::temp::TemporaryDirectory::new(); + let valid = Database::open(root.path().join("narrow-valid.db"), 1, 50).unwrap(); + valid + .migrate(text_model("3000000000000001", 16, 8)) + .await + .unwrap(); + valid + .execute_batch("INSERT INTO user (name, payload) VALUES ('short', X'0102')".into()) + .await + .unwrap(); + valid + .migrate(text_model("3000000000000002", 5, 2)) + .await + .unwrap(); + let valid_state = database_state(&valid).await; + assert!(valid_state.catalog.iter().any(|row| { + text(row, 0) == "table" + && text(row, 2).contains("length(\"name\") <= 5") + && text(row, 2).contains("length(\"payload\") <= 2") + })); + + let invalid = Database::open(root.path().join("narrow-invalid.db"), 1, 50).unwrap(); + invalid + .migrate(text_model("3000000000000001", 16, 8)) + .await + .unwrap(); + invalid + .execute_batch("INSERT INTO user (name, payload) VALUES ('too-long', X'010203')".into()) + .await + .unwrap(); + let before = database_state(&invalid).await; + assert_eq!( + invalid + .migrate(text_model("3000000000000002", 5, 2)) + .await + .unwrap_err() + .kind(), + ErrorKind::Constraint + ); + assert_eq!(database_state(&invalid).await, before); +} + +#[tokio::test] +async fn not_null_and_unique_changes_validate_existing_rows_before_commit() { + let root = support::temp::TemporaryDirectory::new(); + + let nullable = |revision: &str, required: bool| { + model( + revision, + "seed-empty", + vec![ + id(), + text_field("code", 16, false), + text_field("note", 32, !required), + created_at(), + ], + vec![], + vec![], + vec![], + ) + }; + let nullable_valid = Database::open(root.path().join("nullable-valid.db"), 1, 50).unwrap(); + nullable_valid + .migrate(nullable("4000000000000001", false)) + .await + .unwrap(); + nullable_valid + .execute_batch("INSERT INTO user (code, note) VALUES ('one', 'value')".into()) + .await + .unwrap(); + nullable_valid + .migrate(nullable("4000000000000002", true)) + .await + .unwrap(); + assert_eq!( + orm_sqlite::integer(&database_state(&nullable_valid).await.columns[2], 2), + 1 + ); + + let nullable_invalid = Database::open(root.path().join("nullable-invalid.db"), 1, 50).unwrap(); + nullable_invalid + .migrate(nullable("4000000000000001", false)) + .await + .unwrap(); + nullable_invalid + .execute_batch("INSERT INTO user (code, note) VALUES ('one', NULL)".into()) + .await + .unwrap(); + let before = database_state(&nullable_invalid).await; + assert_eq!( + nullable_invalid + .migrate(nullable("4000000000000002", true)) + .await + .unwrap_err() + .kind(), + ErrorKind::Constraint + ); + assert_eq!(database_state(&nullable_invalid).await, before); + + let unique = |revision: &str, enabled: bool| { + model( + revision, + "seed-empty", + vec![id(), text_field("code", 16, false), created_at()], + enabled + .then(|| index("_dever_idx_user_code", &["code"], true)) + .into_iter() + .collect(), + vec![], + vec![], + ) + }; + let unique_valid = Database::open(root.path().join("unique-valid.db"), 1, 50).unwrap(); + unique_valid + .migrate(unique("4000000000000011", false)) + .await + .unwrap(); + unique_valid + .execute_batch("INSERT INTO user (code) VALUES ('one'), ('two')".into()) + .await + .unwrap(); + unique_valid + .migrate(unique("4000000000000012", true)) + .await + .unwrap(); + assert!( + database_state(&unique_valid) + .await + .catalog + .iter() + .any(|row| { + text(row, 1) == "_dever_idx_user_code" + && text(row, 2).starts_with("CREATE UNIQUE INDEX") + }) + ); + + let unique_invalid = Database::open(root.path().join("unique-invalid.db"), 1, 50).unwrap(); + unique_invalid + .migrate(unique("4000000000000011", false)) + .await + .unwrap(); + unique_invalid + .execute_batch("INSERT INTO user (code) VALUES ('same'), ('same')".into()) + .await + .unwrap(); + let before = database_state(&unique_invalid).await; + assert_eq!( + unique_invalid + .migrate(unique("4000000000000012", true)) + .await + .unwrap_err() + .kind(), + ErrorKind::Constraint + ); + assert_eq!(database_state(&unique_invalid).await, before); +} + +#[tokio::test] +async fn adding_a_foreign_key_accepts_valid_rows_and_rolls_back_orphans() { + let root = support::temp::TemporaryDirectory::new(); + let schema = |revision: &str, foreign_key: bool| { + let owner = if foreign_key { + TestField::new( + "owner_id", + "id:app.model.team", + false, + "\"owner_id\" INTEGER NOT NULL REFERENCES \"team\" (\"id\")", + ) + } else { + TestField::new("owner_id", "Int", false, "\"owner_id\" INTEGER NOT NULL") + }; + model( + revision, + "seed-empty", + vec![id(), owner, created_at()], + vec![], + vec![], + vec![], + ) + }; + + let valid = Database::open(root.path().join("foreign-key-valid.db"), 1, 50).unwrap(); + valid + .execute_batch( + "CREATE TABLE team (id INTEGER PRIMARY KEY); INSERT INTO team (id) VALUES (1)".into(), + ) + .await + .unwrap(); + valid + .migrate(schema("5000000000000001", false)) + .await + .unwrap(); + valid + .execute_batch("INSERT INTO user (owner_id) VALUES (1)".into()) + .await + .unwrap(); + valid + .migrate(schema("5000000000000002", true)) + .await + .unwrap(); + let foreign_keys = valid + .query( + "SELECT \"table\", \"from\", \"to\" FROM pragma_foreign_key_list('user')", + vec![], + ) + .await + .unwrap(); + assert_eq!(foreign_keys.len(), 1); + assert_eq!(text(&foreign_keys[0], 0), "team"); + assert_eq!(text(&foreign_keys[0], 1), "owner_id"); + assert_eq!(text(&foreign_keys[0], 2), "id"); + drop(valid); + + let invalid = Database::open(root.path().join("foreign-key-invalid.db"), 1, 50).unwrap(); + invalid + .execute_batch("CREATE TABLE team (id INTEGER PRIMARY KEY)".into()) + .await + .unwrap(); + invalid + .migrate(schema("5000000000000001", false)) + .await + .unwrap(); + invalid + .execute_batch("INSERT INTO user (owner_id) VALUES (99)".into()) + .await + .unwrap(); + let before = database_state(&invalid).await; + assert_eq!( + invalid + .migrate(schema("5000000000000002", true)) + .await + .unwrap_err() + .kind(), + ErrorKind::Migration + ); + assert_eq!(database_state(&invalid).await, before); + drop(invalid); + + let cross_family = Database::open(root.path().join("foreign-key-type.db"), 1, 50).unwrap(); + cross_family + .execute_batch("CREATE TABLE team (id INTEGER PRIMARY KEY)".into()) + .await + .unwrap(); + cross_family + .migrate(model( + "5000000000000011", + "seed-empty", + vec![ + id(), + TestField::new("owner_id", "Float", false, "\"owner_id\" REAL NOT NULL"), + created_at(), + ], + vec![], + vec![], + vec![], + )) + .await + .unwrap(); + cross_family + .execute_batch("INSERT INTO user (owner_id) VALUES (1.0)".into()) + .await + .unwrap(); + let before = database_state(&cross_family).await; + assert_eq!( + cross_family + .migrate(schema("5000000000000012", true)) + .await + .unwrap_err() + .kind(), + ErrorKind::Migration + ); + assert_eq!(database_state(&cross_family).await, before); +} + +#[tokio::test] +async fn named_drop_runs_once_and_applied_content_cannot_change_or_disappear() { + let root = support::temp::TemporaryDirectory::new(); + let database = Database::open(root.path().join("named-drop.db"), 1, 50).unwrap(); + let initial = || { + model( + "6000000000000001", + "seed-empty", + vec![ + id(), + text_field("code", 16, false), + text_field("legacy", 32, true), + created_at(), + ], + vec![], + vec![], + vec![], + ) + }; + database.migrate(initial()).await.unwrap(); + database + .execute_batch("INSERT INTO user (code, legacy) VALUES ('one', 'old')".into()) + .await + .unwrap(); + + let dropped = |migration_revision: &str, include_migration: bool| { + model( + "6000000000000002", + "seed-empty", + vec![id(), text_field("code", 16, false), created_at()], + vec![], + include_migration + .then(|| migration("remove_legacy", migration_revision, &["legacy"])) + .into_iter() + .collect(), + vec![], + ) + }; + let before_drop = database_state(&database).await; + assert_eq!( + database + .migrate(dropped("remove-legacy-v1", false)) + .await + .unwrap_err() + .kind(), + ErrorKind::Migration + ); + assert_eq!(database_state(&database).await, before_drop); + + database + .migrate(dropped("remove-legacy-v1", true)) + .await + .unwrap(); + let applied = database_state(&database).await; + assert_eq!( + applied + .columns + .iter() + .map(|row| text(row, 0)) + .collect::<Vec<_>>(), + ["id", "code", "created_at"] + ); + assert!(applied.history.iter().any(|row| { + text(row, 0) == "migration" + && text(row, 1) == "remove_legacy" + && text(row, 2) == "remove-legacy-v1" + })); + + database + .migrate(dropped("remove-legacy-v1", true)) + .await + .unwrap(); + assert_eq!(database_state(&database).await, applied); + assert_eq!( + database + .migrate(dropped("remove-legacy-v2", true)) + .await + .unwrap_err() + .kind(), + ErrorKind::Migration + ); + assert_eq!(database_state(&database).await, applied); + assert_eq!( + database + .migrate(dropped("remove-legacy-v1", false)) + .await + .unwrap_err() + .kind(), + ErrorKind::Migration + ); + assert_eq!(database_state(&database).await, applied); +} + +#[tokio::test] +async fn catalog_drift_is_rejected_without_rewriting_history_or_data() { + let root = support::temp::TemporaryDirectory::new(); + + let indexed = || { + model( + "8000000000000001", + "seed-empty", + vec![id(), text_field("code", 16, false), created_at()], + vec![index("_dever_idx_user_code", &["code"], true)], + vec![], + vec![], + ) + }; + let index_drift = Database::open(root.path().join("index-drift.db"), 1, 50).unwrap(); + index_drift.migrate(indexed()).await.unwrap(); + index_drift + .execute_batch( + "INSERT INTO user (code) VALUES ('one'); \ + DROP INDEX _dever_idx_user_code; \ + CREATE INDEX _dever_idx_user_code ON user (code)" + .into(), + ) + .await + .unwrap(); + let before = database_state(&index_drift).await; + assert_eq!( + index_drift.migrate(indexed()).await.unwrap_err().kind(), + ErrorKind::Migration + ); + assert_eq!(database_state(&index_drift).await, before); + + let table_drift = Database::open(root.path().join("table-drift.db"), 1, 50).unwrap(); + table_drift.migrate(indexed()).await.unwrap(); + table_drift + .execute_batch( + "INSERT INTO user (code) VALUES ('one'); ALTER TABLE user ADD COLUMN rogue TEXT".into(), + ) + .await + .unwrap(); + let before = database_state(&table_drift).await; + assert_eq!( + table_drift.migrate(indexed()).await.unwrap_err().kind(), + ErrorKind::Migration + ); + assert_eq!(database_state(&table_drift).await, before); +} diff --git a/test/dever-tests/tests/orm_native_sql.rs b/test/dever-tests/tests/orm_native_sql.rs new file mode 100644 index 0000000..adc0067 --- /dev/null +++ b/test/dever-tests/tests/orm_native_sql.rs @@ -0,0 +1,432 @@ +use std::ffi::OsString; +use std::fs; +use std::process::Command; + +use dever_core::source::SourceMap; + +const MODEL: &str = r#" +type Status { + Active + Disabled +} + +type UserSummary { + status: Status + total: Int +} + +type User { + email: Text(254) unique + status: Status default Status.Active +} + +sql by_status(status: Status) (users: List<User>) { + sqlite = "SELECT \"id\", \"created_at\", \"email\", \"status\" FROM \"user\" WHERE \"status\" = ?1 ORDER BY \"id\"" + postgres = "SELECT \"id\", \"created_at\", \"email\", \"status\" FROM \"user\" WHERE \"status\" = $1 ORDER BY \"id\"" +} + +sql summary(status: Status) (summary: UserSummary) { + sqlite = "SELECT \"status\", COUNT(*) FROM \"user\" WHERE \"status\" = ?1 GROUP BY \"status\"" + postgres = "SELECT \"status\", COUNT(*) FROM \"user\" WHERE \"status\" = $1 GROUP BY \"status\"" +} + +sql by_email(email: Text) (user: User?) { + sqlite = "SELECT \"id\", \"created_at\", \"email\", \"status\" FROM \"user\" WHERE \"email\" = ?1" + postgres = "SELECT \"id\", \"created_at\", \"email\", \"status\" FROM \"user\" WHERE \"email\" = $1" +} + +sql required_by_email(email: Text) (user: User) { + sqlite = "SELECT \"id\", \"created_at\", \"email\", \"status\" FROM \"user\" WHERE \"email\" = ?1" + postgres = "SELECT \"id\", \"created_at\", \"email\", \"status\" FROM \"user\" WHERE \"email\" = $1" +} +"#; + +const APP: &str = r#" +transaction create_and_read() (user: app.user.model.User) { + created = model.create({ + email = "transaction@example.com" + }) + user = model.required_by_email(created.email) +} + +email_or_none(user: null) (email: Text) { + email = "none" +} + +email_or_none(user: app.user.model.User) (email: Text) { + email = user.email +} + +main() () { + first = model.create({ + email = "first@example.com" + }) + second = model.create({ + email = "second@example.com" + }) + disabled = model.create({ + email = "disabled@example.com" + status = app.user.model.Status.Disabled + }) + + active = model.by_status(app.user.model.Status.Active) + blocking(dever.io.println(int.to_text(length(active)))) + summary = model.summary(app.user.model.Status.Active) + blocking(dever.io.println(int.to_text(summary.total))) + missing = model.by_email("missing@example.com") + blocking(dever.io.println(email_or_none(missing))) + found = model.by_email(first.email) + blocking(dever.io.println(email_or_none(found))) + transaction_user = create_and_read() + blocking(dever.io.println(transaction_user.email)) +} +"#; + +fn sources(model: &str, app: &str) -> SourceMap { + let mut sources = SourceMap::default(); + sources.add("app/user/model.dever", model); + sources.add("app/user/app.dever", app); + sources +} + +fn checked(model: &str, app: &str) -> (SourceMap, dever_core::hir::Program) { + let sources = sources(model, app); + let program = dever_core::check(&sources).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }); + (sources, program) +} + +fn errors(model: &str) -> String { + source_errors(&sources(model, APP)) +} + +fn source_errors(sources: &SourceMap) -> String { + dever_core::check(sources) + .unwrap_err() + .iter() + .map(|error| error.render(sources)) + .collect() +} + +fn compile(model: &str, app: &str) -> dever_core::native::NativeProgram { + let (sources, program) = checked(model, app); + let rustc = std::env::var_os("RUSTC").unwrap_or_else(|| OsString::from("rustc")); + dever_core::native::compile_application( + &program, + &sources, + "app.user.main", + &rustc, + dever_runtime::config::RuntimeProfile { + sqlite: true, + postgres: false, + }, + ) + .unwrap() +} + +fn configure(native: &dever_core::native::NativeProgram) { + let root = native.executable().parent().unwrap(); + fs::create_dir_all(root.join("config")).unwrap(); + fs::write( + root.join("config/setting.json"), + r#"{ + "database": { + "default": { + "type": "sqlite", + "path": "data/db/app.db", + "max_connections": 1, + "max_page_size": 10 + } + } +}"#, + ) + .unwrap(); +} + +#[test] +fn typed_sql_is_static_formatted_and_lowers_to_existing_model_executor() { + let (sources, program) = checked(MODEL, APP); + let formatted = dever_core::format::format(&sources.files()[0]).unwrap(); + checked(&formatted, APP); + + let snapshot = program.model_snapshot(); + assert_eq!( + snapshot.matches("model app.user.model ").count(), + 1, + "{snapshot}" + ); + assert!(snapshot.contains("sql by_status"), "{snapshot}"); + assert!(snapshot.contains("Many:app.user.model.User"), "{snapshot}"); + assert!( + snapshot.contains("One:app.user.model.UserSummary"), + "{snapshot}" + ); + + let native = dever_core::native::emit(&program, &sources, "app.user.main").unwrap(); + assert!(native.contains("sqlite: \"SELECT \\\"id\\\""), "{native}"); + assert!(native.contains("postgres: \"SELECT \\\"id\\\""), "{native}"); + assert!(native.contains("\\\"status\\\" = ?1"), "{native}"); + assert!(native.contains("\\\"status\\\" = $1"), "{native}"); + assert!(native.contains("decode_sql_record_"), "{native}"); + assert!(native.contains("database.query_bounded("), "{native}"); + assert!(native.contains("database.max_page_size()"), "{native}"); + assert!(!native.contains("query_owned"), "{native}"); +} + +#[test] +fn typed_sql_rejects_dynamic_text_invalid_shapes_and_unowned_records() { + let dynamic = MODEL.replace("sqlite = \"SELECT", "sqlite = query\n query = \"SELECT"); + let rendered = errors(&dynamic); + assert!( + rendered.contains("SQL text must be a static Text literal"), + "{rendered}" + ); + + let wrong_dialect = MODEL.replace( + "WHERE \\\"status\\\" = $1 ORDER BY", + "WHERE \\\"status\\\" = ?1 ORDER BY", + ); + assert!(errors(&wrong_dialect).contains("other dialect's parameter marker")); + + let missing_parameter = MODEL.replace( + "WHERE \\\"email\\\" = $1\"\n}\n\nsql required_by_email", + "WHERE \\\"email\\\" = 'fixed'\"\n}\n\nsql required_by_email", + ); + assert!(errors(&missing_parameter).contains("declared but not bound")); + + let map_result = MODEL.replace("(summary: UserSummary)", "(summary: Map<Text, Int>)"); + assert!(errors(&map_result).contains("concrete Model or record type")); + + let unused_record = MODEL + .replace("type UserSummary", "type Unused") + .replace("(summary: UserSummary)", "(summary: User)"); + assert!(errors(&unused_record).contains("auxiliary record in a Model file must be used")); + + let public_record = MODEL.replace("type UserSummary", "public type UserSummary"); + assert!(errors(&public_record).contains("visibility is defined by the source role")); + + let unowned_record = MODEL + .replace( + "type UserSummary {\n status: Status\n total: Int\n}\n\n", + "", + ) + .replace( + "(summary: UserSummary)", + "(summary: reporting.user.UserSummary)", + ); + let mut unowned_sources = sources(&unowned_record, APP); + unowned_sources.add( + "reporting/user/app.dever", + "type UserSummary {\n status: app.user.model.Status\n total: Int\n}\n", + ); + let rendered = source_errors(&unowned_sources); + assert!( + rendered.contains("SQL result record must be private to its Model file"), + "{rendered}" + ); + + let wrong_argument = APP.replace( + "model.by_status(app.user.model.Status.Active)", + "model.by_status(1)", + ); + assert!(dever_core::check(&sources(MODEL, &wrong_argument)).is_err()); +} + +#[test] +fn typed_sql_can_project_into_an_owning_app_view_without_exporting_private_model_data() { + let model = r#" +type User { + name: Text + private password_hash: Text +} +sql summaries() (users: List<app.UserView>) { + sqlite = "SELECT name FROM user" + postgres = "SELECT name FROM user" +} +"#; + let app = r#" +type UserView { + name: Text +} +list() (users: List<UserView>) { + users = model.summaries() +} +"#; + for result_name in ["app.UserView", "app.user.UserView"] { + let mut sources = sources(&model.replace("app.UserView", result_name), app); + sources.add("app/user/api.dever", "cmd list = app.list\n"); + let program = dever_core::check(&sources).unwrap(); + assert!(program.model_snapshot().contains("Many:app.user.UserView")); + dever_core::native::emit(&program, &sources, "app.user.list").unwrap(); + dever_core::llvm::emit_application(&program, &sources).unwrap(); + } + + // App View 只接收显式存储标量，不能借投影导出私有字段或整个 Model。 + for field in [ + "private name: Text", + "name: Secret", + "name: model.User", + "name: Int >= 1", + ] { + let rendered = source_errors(&sources(model, &app.replace("name: Text", field))); + assert!( + rendered.contains("SQL result field 'name' must be visible"), + "{rendered}" + ); + } + let domain_result = model.replace("app.UserView", "app.user.domain.UserView"); + let mut invalid = sources(&domain_result, app); + invalid.add( + "app/user/domain.dever", + "type UserView {\n name: Text\n}\n", + ); + assert!(source_errors(&invalid).contains("owning-domain App View")); + assert!(errors(&MODEL.replace("total: Int", "total: Int >= 1")).contains("without bounds")); +} + +#[test] +fn typed_sql_placeholder_validation_ignores_sql_lexical_regions() { + let model = format!( + r#"{MODEL} + +sql lexical(first: Text, second: Text) (users: List<User>) {{ + sqlite = "SELECT \"id\", \"created_at\", \"email\", \"status\" FROM \"user\" WHERE (?2 = ?1 OR ?2 = ?1) AND '?9 $8' = '?7 $6' AND \"name?5$4\" = \"name?3$2\" AND `name?15$14` = [name?13$12] /* ?11 $10 */ -- ?17 $16\n" + postgres = "SELECT \"id\", \"created_at\", \"email\", \"status\" FROM \"user\" WHERE ($2 = $1 OR $2 = $1) AND '$9 ?8' = '$7 ?6' AND \"name$5?4\" = \"name$3?2\" AND $$ $11 ?10 $$ = $$ $9 ?8 $$ AND $tag$ $7 ?6 $tag$ = $tag$ $5 ?4 $tag$ /* $13 ?12 */ -- $15 ?14\n" +}} +"# + ); + checked(&model, APP); + checked(&model.replace("$tag$", "$标签$"), APP); + checked( + &model.replace("($2 = $1 OR $2 = $1)", "(name$tag$ = $2 OR name$tag$ = $1)"), + APP, + ); +} + +#[test] +fn sqlite_executes_model_projection_custom_projection_and_transaction_sql() { + let native = compile(MODEL, APP); + configure(&native); + let output = Command::new(native.executable()).output().unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + String::from_utf8(output.stdout).unwrap(), + "2\n2\nnone\nfirst@example.com\ntransaction@example.com\n" + ); + assert!(output.stderr.is_empty()); +} + +#[test] +fn sqlite_reports_missing_extra_and_mismatched_result_columns() { + let missing = MODEL.replace( + "SELECT \\\"id\\\", \\\"created_at\\\", \\\"email\\\", \\\"status\\\" FROM \\\"user\\\" WHERE \\\"email\\\" = ?1\"\n postgres", + "SELECT \\\"id\\\" FROM \\\"user\\\" WHERE \\\"email\\\" = ?1\"\n postgres", + ); + let extra = MODEL.replace( + "SELECT \\\"id\\\", \\\"created_at\\\", \\\"email\\\", \\\"status\\\" FROM \\\"user\\\" WHERE \\\"email\\\" = ?1\"\n postgres", + "SELECT \\\"id\\\", \\\"created_at\\\", \\\"email\\\", \\\"status\\\", 1 FROM \\\"user\\\" WHERE \\\"email\\\" = ?1\"\n postgres", + ); + let mismatched = MODEL.replace("COUNT(*)", "'not-an-int'"); + + for (model, message) in [ + (missing, "database row is missing column"), + (extra, "has extra columns"), + (mismatched, "expected a database Int"), + ] { + let native = compile(&model, APP); + configure(&native); + let output = Command::new(native.executable()).output().unwrap(); + assert!(!output.status.success()); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!(stderr.contains(message), "{stderr}"); + } +} + +#[test] +fn singular_sql_result_cardinality_is_strict() { + use dever_runtime::orm::{Row, bounded_rows, optional_row, required_row}; + + assert!( + required_row(Vec::new(), "required") + .unwrap_err() + .to_string() + .contains("no rows") + ); + assert!( + required_row(vec![Row::new(Vec::new()), Row::new(Vec::new())], "required") + .unwrap_err() + .to_string() + .contains("more than one row") + ); + assert!(optional_row(Vec::new(), "optional").unwrap().is_none()); + assert!( + optional_row(vec![Row::new(Vec::new()), Row::new(Vec::new())], "optional") + .unwrap_err() + .to_string() + .contains("more than one row") + ); + assert_eq!( + bounded_rows(vec![Row::new(Vec::new()), Row::new(Vec::new())], 1) + .unwrap_err() + .to_string(), + "database query returned more than one row" + ); + assert_eq!( + bounded_rows((0..11).map(|_| Row::new(Vec::new())).collect(), 10,) + .unwrap_err() + .to_string(), + "database query returned more than 10 rows" + ); +} + +#[test] +fn sqlite_bounds_native_sql_rows_before_decoding() { + let multiple = MODEL.replace( + "WHERE \\\"email\\\" = ?1\"\n postgres", + "WHERE ?1 = ?1\"\n postgres", + ); + let many = r#" +main() () { + one = model.create({email = "one@example.com"}) + two = model.create({email = "two@example.com"}) + three = model.create({email = "three@example.com"}) + four = model.create({email = "four@example.com"}) + five = model.create({email = "five@example.com"}) + six = model.create({email = "six@example.com"}) + seven = model.create({email = "seven@example.com"}) + eight = model.create({email = "eight@example.com"}) + nine = model.create({email = "nine@example.com"}) + ten = model.create({email = "ten@example.com"}) + eleven = model.create({email = "eleven@example.com"}) + users = model.by_status(app.user.model.Status.Active) + blocking(dever.io.println(int.to_text(length(users)))) +} +"#; + + for (model, app, message) in [ + ( + multiple.as_str(), + APP, + "database query returned more than one row", + ), + (MODEL, many, "database query returned more than 10 rows"), + ] { + let native = compile(model, app); + configure(&native); + let output = Command::new(native.executable()).output().unwrap(); + assert!(!output.status.success()); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!(stderr.contains(message), "{stderr}"); + } +} diff --git a/test/dever-tests/tests/orm_performance_fixture.rs b/test/dever-tests/tests/orm_performance_fixture.rs new file mode 100644 index 0000000..d98046d --- /dev/null +++ b/test/dever-tests/tests/orm_performance_fixture.rs @@ -0,0 +1,183 @@ +use std::path::Path; + +use dever_core::source::SourceMap; + +#[path = "../examples/network_bench/response.rs"] +mod benchmark_response; + +const CONFIG: &str = r#" +lifetime() (value: Int) { value = 1000 } +idle_sample_ms() (value: Int) { value = 10 } +pending() (value: Int) { value = 4 } +channel_capacity() (value: Int) { value = 4 } +input() (value: Bytes) { value = dever.bytes.from_text("xxxx") } +pending_input() (value: Bytes) { value = dever.bytes.from_text("xxxx") } +orm_input() (value: Bytes) { value = dever.bytes.from_text("xxxx") } +orm_rows() (value: Bytes) { value = dever.bytes.from_text("rrrr") } +orm_concurrency() (value: Int) { value = 2 } +orm_cancel_work() (value: Int) { value = 1000 } +certificate() (value: Bytes) { value = dever.bytes.from_text("") } +key() (value: Bytes) { value = dever.bytes.from_text("") } +limits() (value: dever.http.Limits) { value = dever.http.default_limits() } +http2_limits() (value: dever.http.Limits) { value = dever.http.default_limits() } +large_body() (value: Bytes) { value = dever.bytes.from_text("xxxxxxxxxxxxxxxx") } +"#; + +fn checked_fixture(name: &str) -> (SourceMap, dever_core::hir::Program) { + let root = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../performance/fixtures") + .join(name); + let loaded = SourceMap::load(&root).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(ToString::to_string) + .collect::<Vec<_>>() + .join("\n") + ) + }); + let mut sources = SourceMap::default(); + for source in loaded.files() { + let source_path = source.path().to_str().unwrap(); + let path = match (name, source_path) { + ("orm", "app.dever") => "benchmark/item/app.dever".to_owned(), + ("orm", "app/model/item.dever") => "benchmark/item/model.dever".to_owned(), + ("orm", "app/api.dever") => "benchmark/item/api.dever".to_owned(), + (_, "app.dever") => match name { + "runtime" => "benchmark/runtime/app.dever".to_owned(), + "http" => "benchmark/http/app.dever".to_owned(), + "live" => "benchmark/live/app.dever".to_owned(), + _ => unreachable!("unknown performance fixture"), + }, + ("runtime" | "http" | "live", "port.dever" | "adapter.dever" | "api.dever") => { + format!("benchmark/{name}/{source_path}") + } + _ => unreachable!("unknown performance fixture source"), + }; + sources.add(path, source.text()); + } + sources.add( + "benchmark/bench/app.dever", + include_str!("../../performance/fixtures/bench.dever"), + ); + sources.add("benchmark/config/app.dever", CONFIG); + let program = dever_core::check(&sources).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }); + (sources, program) +} + +#[test] +fn orm_performance_fixture_checks_every_packaged_entry() { + let (sources, program) = checked_fixture("orm"); + for entry in [ + "orm_idle", + "orm_crud", + "orm_list", + "orm_cursor", + "orm_stream", + "orm_pool", + "orm_seed", + ] { + program + .application_warnings(&format!("benchmark.item.{entry}")) + .unwrap_or_else(|error| panic!("benchmark.item.{entry}: {error}")); + } + let snapshot = program.api_snapshot(); + assert!(snapshot.contains("function benchmark.item.count()")); + assert!(snapshot.contains("function benchmark.item.cancel()")); + let generated = dever_core::native::emit_project( + &program, + &sources, + dever_runtime::config::RuntimeProfile { + sqlite: true, + postgres: false, + }, + ) + .unwrap(); + assert!(generated.contains("(\"GET\", \"/benchmark/item/count\")")); + assert!(generated.contains("(\"POST\", \"/benchmark/item/cancel\")")); + assert!(generated.contains("dever_runtime::api::serve(api_route).await")); +} + +#[test] +fn cms_performance_fixture_checks_stock_source_parity() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../examples/cms"); + let checked = |format| { + let project = root.join(format); + let sources = + SourceMap::load_project(&project.join("module"), &project.join("test")).unwrap(); + dever_core::check(&sources).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }) + }; + let plain = checked("dever"); + let markdown = checked("md"); + assert_eq!(plain.api_snapshot(), markdown.api_snapshot()); + assert_eq!(plain.model_snapshot(), markdown.model_snapshot()); +} + +#[test] +fn orm_http_client_contract_uses_configured_row_count() { + let _ = benchmark_response::HyperResponses::new().response("/json"); + let _ = benchmark_response::RuntimeResponses::new().response("/json"); + let first = benchmark_response::ExpectedResponse::for_path( + benchmark_response::ORM_COUNT_PATH, + Some(17), + ) + .unwrap(); + assert_eq!( + first.body.as_ref(), + br#"{"code":0,"message":"ok","data":17}"# + ); + assert_eq!(first.content_type, "application/json; charset=utf-8"); + let second = benchmark_response::ExpectedResponse::for_path( + benchmark_response::ORM_COUNT_PATH, + Some(23), + ) + .unwrap(); + assert_ne!(first.body, second.body); + assert!( + benchmark_response::ExpectedResponse::for_path(benchmark_response::ORM_COUNT_PATH, None,) + .is_err() + ); +} + +#[test] +fn profile_free_performance_fixtures_still_check() { + for (fixture, entries) in [ + ( + "runtime", + &[ + "sync_idle", + "async_idle", + "tasks", + "tasks_on_worker", + "channels", + "cancel", + ][..], + ), + ("http", &["http", "https", "http2", "https2"]), + ("live", &["tcp", "ws", "sse"]), + ] { + let (_sources, program) = checked_fixture(fixture); + for entry in entries { + program + .application_warnings(&format!("benchmark.{fixture}.{entry}")) + .unwrap_or_else(|error| panic!("{fixture} benchmark.{fixture}.{entry}: {error}")); + } + } +} diff --git a/test/dever-tests/tests/orm_query.rs b/test/dever-tests/tests/orm_query.rs new file mode 100644 index 0000000..b9e7422 --- /dev/null +++ b/test/dever-tests/tests/orm_query.rs @@ -0,0 +1,549 @@ +use dever_core::source::SourceMap; +use dever_runtime::config::RuntimeProfile; + +const MODEL: &str = r#" +public type UserStatus { + Active = "启用" + Disabled = "禁用" +} + +public type User { + external_uuid: Uuid default "018f08d8-59b0-7cc2-98c4-dc0c0c07398f" + email: Text(254) unique + display_name: Text(1, 64) + status: UserStatus default UserStatus.Active index + enabled: Bool default true + age: Int default 0 + alias: Text? +} +"#; + +const SERVICE: &str = r#" +public register() (user: user.model.user.User) { + user = user.create({ + external_uuid = "018f08d8-59b0-7cc2-98c4-dc0c0c07398f" + email = "reader@example.com" + display_name = "Reader" + }) +} + +public active() (users: user.model.user.Page) { + users = user.list({ + where = status == user.model.user.UserStatus.Active and enabled == true + order = [created_at.desc] + page = 2 + size = 10 + }) +} + +public find(id: user.model.user.id) (user: user.model.user.User) { + user = user.get(id) +} + +public disable(id: user.model.user.id) (changed: Int) { + changed = user.update(id, { + status = user.model.user.UserStatus.Disabled + enabled = false + }) +} + +public remove_disabled() (changed: Int) { + changed = user.delete({ + where = status == user.model.user.UserStatus.Disabled + }) +} + +public bulk() (changed: Int) { + changed = user.create_many([ + { + email = "one@example.com" + display_name = "One" + }, + { + email = "two@example.com" + display_name = "Two" + } + ]) +} + +public advanced() (users: user.model.user.Page) { + users = user.list({ + where = not contains(display_name, "bot") and in(status, [user.model.user.UserStatus.Active, user.model.user.UserStatus.Disabled]) and between(age, 18, 65) and starts_with(email, "reader") and ends_with(email, ".com") and alias == null + }) +} + +public page_after() (users: user.model.user.Cursor) { + users = user.cursor({ + after = null + order = [created_at.desc, id.desc] + size = 10 + }) +} + +show(user: user.model.user.User) () {} + +public scan() () { + users = user.stream({ size = 4 }) + each(show, users) +} + +public save() (saved: user.model.user.User) { + saved = user.upsert( + { email = "reader@example.com" }, + { display_name = "Reader" }, + { display_name = "Updated" } + ) +} +"#; + +fn sources(service: &str) -> SourceMap { + let mut sources = SourceMap::default(); + sources.add("user/model/user.dever", MODEL); + sources.add("user/service/account.dever", service); + sources +} + +fn errors(service: &str) -> String { + let sources = sources(service); + dever_core::check(&sources) + .unwrap_err() + .iter() + .map(|error| error.render(&sources)) + .collect() +} + +#[test] +fn model_operations_build_static_query_plans_and_parameterized_sql() { + let sources = sources(SERVICE); + let program = dever_core::check(&sources).unwrap(); + let api = program.api_snapshot(); + assert!(api.contains("function user.service.account.register")); + assert!(api.contains("effects=[database]"), "{api}"); + + let create = + dever_core::native::emit(&program, &sources, "user.service.account.register").unwrap(); + assert!(create.contains("INSERT INTO \\\"user\\\""), "{create}"); + assert!(create.contains("VALUES (?1, ?2, ?3)"), "{create}"); + assert!( + create.contains("dever_runtime::orm::Uuid::from_bytes"), + "{create}" + ); + assert!( + create.contains("dever_runtime::orm::Value::Uuid"), + "{create}" + ); + assert!(create.contains("$1::uuid"), "{create}"); + assert!(create.contains("decode_model_"), "{create}"); + + let list = dever_core::native::emit(&program, &sources, "user.service.account.active").unwrap(); + assert!(list.contains("SELECT COUNT(*) FROM \\\"user\\\""), "{list}"); + assert!( + list.contains("ORDER BY \\\"created_at\\\" DESC, \\\"id\\\" DESC"), + "{list}" + ); + assert!(list.contains("LIMIT ?3 OFFSET ?4"), "{list}"); + + let advanced = + dever_core::native::emit(&program, &sources, "user.service.account.advanced").unwrap(); + assert!( + advanced.contains("NOT (instr(\\\"display_name\\\", ?1) > 0)"), + "{advanced}" + ); + assert!( + advanced.contains("\\\"status\\\" IN (?2, ?3)"), + "{advanced}" + ); + assert!( + advanced.contains("\\\"age\\\" BETWEEN ?4 AND ?5"), + "{advanced}" + ); + assert!(advanced.contains("\\\"alias\\\" IS NULL"), "{advanced}"); + + let cursor = + dever_core::native::emit(&program, &sources, "user.service.account.page_after").unwrap(); + assert!( + cursor.contains("(CAST(?1 AS BIGINT) IS NULL OR ((\\\"created_at\\\" < ?1) OR (\\\"created_at\\\" IS ?1 AND \\\"id\\\" < ?2)))"), + "{cursor}" + ); + assert!( + cursor.contains("(CAST($1 AS BIGINT) IS NULL OR ((\\\"created_at\\\" < $1) OR (\\\"created_at\\\" IS NOT DISTINCT FROM $1 AND \\\"id\\\" < $2)))"), + "{cursor}" + ); + assert!(cursor.contains("LIMIT ?3"), "{cursor}"); + assert!(!cursor.contains("COUNT(*)"), "{cursor}"); + + let bulk = dever_core::native::emit(&program, &sources, "user.service.account.bulk").unwrap(); + assert!(bulk.contains("VALUES (?1, ?2), (?3, ?4)"), "{bulk}"); + + let stream = dever_core::native::emit(&program, &sources, "user.service.account.scan").unwrap(); + assert!(stream.contains("database.stream("), "{stream}"); + assert!(stream.contains("ORDER BY \\\"id\\\" DESC"), "{stream}"); + assert!( + !stream.contains("collect::<Result<Vec<_>, _>>()"), + "{stream}" + ); + + let upsert = dever_core::native::emit(&program, &sources, "user.service.account.save").unwrap(); + assert!( + upsert.contains("ON CONFLICT (\\\"email\\\") DO UPDATE"), + "{upsert}" + ); +} + +#[test] +fn model_operations_reject_unknown_fields_and_invalid_query_shapes() { + let unknown = SERVICE.replace( + "email = \"reader@example.com\"", + "unknown = \"reader@example.com\"", + ); + assert!(errors(&unknown).contains("unknown Model field 'unknown'")); + + let generated = SERVICE.replace( + "email = \"reader@example.com\"", + "id = \"reader@example.com\"", + ); + assert!(errors(&generated).contains("generated Model field 'id'")); + + let unknown_option = SERVICE.replace("page = 2", "offset = 2"); + assert!(errors(&unknown_option).contains("unknown query option 'offset'")); + + let invalid_in = SERVICE.replace( + "in(status, [user.model.user.UserStatus.Active, user.model.user.UserStatus.Disabled])", + "in(status, [])", + ); + assert!(errors(&invalid_in).contains("between 1 and 100 values")); + + let invalid_text = SERVICE.replace("contains(display_name, \"bot\")", "contains(age, \"bot\")"); + assert!(errors(&invalid_text).contains("requires a Text field")); + + let invalid_arity = + SERVICE.replace("contains(display_name, \"bot\")", "contains(display_name)"); + assert!(errors(&invalid_arity).contains("where 'contains' expects 2 arguments")); + + let invalid_field = SERVICE.replace( + "contains(display_name, \"bot\")", + "contains(\"bot\", display_name)", + ); + assert!(errors(&invalid_field).contains("the first 'contains' argument must be a Model field")); + + let outside_where = SERVICE.replace( + "user = user.get(id)", + "value = contains(\"reader\", \"read\")\n user = user.get(id)", + ); + let outside_where_errors = errors(&outside_where); + assert!( + outside_where_errors.contains("unknown function 'contains' with 2 argument(s)"), + "{outside_where_errors}" + ); + + let invalid_cursor = SERVICE.replace("order = [created_at.desc, id.desc]", "order = alias.asc"); + assert!(errors(&invalid_cursor).contains("does not support nullable Model fields")); +} + +#[test] +fn empty_create_many_uses_one_valid_statement_per_dialect() { + let mut sources = SourceMap::default(); + sources.add("app/event/model.dever", "type Event {}"); + sources.add( + "app/event/app.dever", + "create() (created: Int) { created = model.create_many([{}, {}]) }", + ); + let program = dever_core::check(&sources).unwrap(); + let native = dever_core::native::emit(&program, &sources, "app.event.create").unwrap(); + assert!( + native.contains("sqlite: \"INSERT INTO \\\"event\\\" (\\\"id\\\") VALUES (NULL), (NULL)\""), + "{native}" + ); + assert!( + native.contains("postgres: \"INSERT INTO \\\"event\\\" (\\\"created_at\\\") VALUES (DEFAULT), (DEFAULT)\""), + "{native}" + ); +} + +#[test] +fn transaction_functions_generate_one_shared_database_boundary() { + let service = SERVICE.replacen("register()", "transaction register()", 1); + let sources = sources(&service); + let program = dever_core::check(&sources).unwrap(); + let native = + dever_core::native::emit(&program, &sources, "user.service.account.register").unwrap(); + + assert!( + native.contains("transaction_database.begin().await"), + "{native}" + ); + assert!(native.contains("transaction.commit().await"), "{native}"); + assert!(native.contains("transaction.rollback().await"), "{native}"); + assert!(native.contains("database::Executor::new"), "{native}"); + assert!( + native.contains("postgres: \"INSERT INTO \\\"user\\\""), + "{native}" + ); +} + +#[test] +fn database_effects_flow_through_helpers_and_nested_transactions() { + let helper = r#" +create_user() (user: user.model.user.User) { + user = user.create({ + email = "reader@example.com" + display_name = "Reader" + }) +} + +public transaction register() (user: user.model.user.User) { + user = create_user() +} +"#; + let helper_sources = sources(helper); + let helper_program = dever_core::check(&helper_sources).unwrap(); + let helper_native = dever_core::native::emit( + &helper_program, + &helper_sources, + "user.service.account.register", + ) + .unwrap(); + assert!( + helper_native + .matches("Option<&dever_runtime::database::Transaction>") + .count() + >= 2, + "{helper_native}" + ); + assert!(helper_native.contains("(None).await"), "{helper_native}"); + assert!( + helper_native.contains("(_database).await"), + "{helper_native}" + ); + + let nested = helper.replacen( + "create_user() (user: user.model.user.User)", + "transaction create_user() (user: user.model.user.User)", + 1, + ); + let nested_sources = sources(&nested); + let nested_program = dever_core::check(&nested_sources).unwrap(); + let nested_native = dever_core::native::emit( + &nested_program, + &nested_sources, + "user.service.account.register", + ) + .unwrap(); + assert_eq!( + nested_native + .matches("transaction_database.begin().await") + .count(), + 2, + "{nested_native}" + ); + assert!( + nested_native.contains("(_database).await"), + "{nested_native}" + ); +} + +#[test] +fn transaction_functions_require_database_work_and_reject_concurrency() { + let mut empty = SourceMap::default(); + empty.add("main.dever", "public transaction save() () {}"); + let empty_error = dever_core::check(&empty) + .unwrap_err() + .iter() + .map(|error| error.render(&empty)) + .collect::<String>(); + assert!( + empty_error.contains("must perform a database operation"), + "{empty_error}" + ); + + let concurrent = r#"work() () {} +public transaction register() (user: user.model.user.User) { + task = run(work()) + user = user.create({ + email = "reader@example.com" + display_name = "Reader" + }) + wait(task) +} +"#; + assert!(errors(concurrent).contains("transaction functions cannot start concurrent work")); + + let streaming = r#"public transaction scan() () { + users = user.stream({ size = 1 }) + close(users) +} +"#; + assert!(errors(streaming).contains("transaction functions cannot start concurrent work")); + + let structured_blocking = r#"hash(password: Secret) (value: Text) { + value = blocking(dever.crypto.password_hash(password)) +} + +public transaction register(password: Secret) (user: user.model.user.User) { + password_hash = hash(password) + user = user.create({ + email = password_hash + display_name = "Reader" + }) +} +"#; + assert!( + errors(structured_blocking) + .contains("transaction functions cannot perform password hashing or verification") + ); + + let short_transaction = r#"hash(password: Secret) (value: Text) { + value = blocking(dever.crypto.password_hash(password)) +} + +transaction create_user(email: Text) (user: user.model.user.User) { + user = user.create({ + email = email + display_name = "Reader" + }) +} + +public register(password: Secret) (user: user.model.user.User) { + password_hash = hash(password) + user = create_user(password_hash) +} +"#; + let sources = sources(short_transaction); + dever_core::check(&sources).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }); +} + +#[test] +fn nullable_comparisons_use_null_safe_database_operators() { + let service = r#" +find_alias(value: Text?) (users: user.model.user.Page) { + users = user.list({ where = alias == value }) +} + +public main() (users: user.model.user.Page) { + users = find_alias(null) +} +"#; + let sources = sources(service); + let program = dever_core::check(&sources).unwrap(); + let native = dever_core::native::emit(&program, &sources, "user.service.account.main").unwrap(); + + assert!(native.contains("\\\"alias\\\" IS ?1"), "{native}"); + assert!( + native.contains("\\\"alias\\\" IS NOT DISTINCT FROM $1"), + "{native}" + ); +} + +#[test] +fn database_effects_follow_bound_handlers_inside_transactions() { + let service = r#" +apply(route: handler() (user: user.model.user.User)) (user: user.model.user.User) { + user = route() +} + +forward(route: handler() (user: user.model.user.User)) (user: user.model.user.User) { + user = apply(route) +} + +create_user() (user: user.model.user.User) { + user = user.create({ + email = "reader@example.com" + display_name = "Reader" + }) +} + +public transaction register() (user: user.model.user.User) { + user = forward(create_user) +} +"#; + let sources = sources(service); + let program = dever_core::check(&sources).unwrap(); + let native = + dever_core::native::emit(&program, &sources, "user.service.account.register").unwrap(); + + assert!( + native.contains("transaction_database.begin().await"), + "{native}" + ); + assert!(native.contains("(_database).await"), "{native}"); + + let concurrent = service.replace( + "create_user() (user: user.model.user.User) {", + "work() () {}\n\ncreate_user() (user: user.model.user.User) {\n task = run(work())\n wait(task)", + ); + assert!(errors(&concurrent).contains("transaction functions cannot start concurrent work")); +} + +#[test] +fn project_models_generate_scoped_storage_and_controlled_tenant_migration() { + let mut sources = SourceMap::default(); + sources.add( + "platform/role/model.dever", + "global type Role { name: Text(1, 32) }", + ); + sources.add( + "user/profile/model.dever", + "type Profile {\n role_id: platform.role.model.id\n display_name: Text(1, 64)\n}", + ); + sources.add( + "user/profile/app.dever", + "count() (value: Int) { value = model.count() }", + ); + sources.add("user/profile/api.dever", "cmd count = app.count"); + + let program = dever_core::check(&sources).unwrap(); + let native = dever_core::native::emit_project( + &program, + &sources, + RuntimeProfile { + sqlite: true, + postgres: false, + }, + ) + .unwrap(); + + assert!(!native.contains("MODEL_DATABASE_"), "{native}"); + assert!(native.contains("MODEL_STORAGE_"), "{native}"); + assert!( + native.contains("OnceLock<dever_runtime::database::StorageBinding>"), + "{native}" + ); + assert!( + native.contains("dever_runtime::database::ModelScope::Global"), + "{native}" + ); + assert!( + native.contains("dever_runtime::database::ModelScope::Tenant"), + "{native}" + ); + assert!( + native.contains("dever_runtime::config::bootstrap_scoped"), + "{native}" + ); + assert!( + native.contains("async fn tenant_migrate(tenant_id: i64)"), + "{native}" + ); + assert!(native.contains("--dever-tenant-migrate"), "{native}"); + assert!( + native.contains("dever_runtime::tenant::begin_migration"), + "{native}" + ); + assert!( + native.contains("dever_runtime::tenant::mark_database_ready"), + "{native}" + ); + assert!( + native.contains("dever_runtime::tenant::finish_migration"), + "{native}" + ); + assert!(!native.contains("REFERENCES \\\"role\\\""), "{native}"); +} diff --git a/test/dever-tests/tests/orm_relations.rs b/test/dever-tests/tests/orm_relations.rs new file mode 100644 index 0000000..40d46c7 --- /dev/null +++ b/test/dever-tests/tests/orm_relations.rs @@ -0,0 +1,380 @@ +use std::ffi::OsString; +use std::fs; +use std::process::Command; + +use dever_core::source::SourceMap; + +const ORGANIZATION: &str = r#" +type Organization { + name: Text(1, 64) +} + +relation users = app.people.model.user.organization_id +"#; + +const USER: &str = r#" +type User { + name: Text(1, 64) + organization_id: app.people.model.organization.id? + manager_id: app.people.model.user.id? +} + +relation posts = app.people.model.post.author_id +relation other_posts = app.people.model.post.author_id +relation unloaded_posts = app.people.model.post.author_id +"#; + +const POST: &str = r#" +type Post { + author_id: app.people.model.user.id + title: Text(1, 64) +} +"#; + +const APP: &str = r#" +organization_name(value: Related.Unloaded<app.people.model.organization.Organization?>()) (name: Text) { + name = "unloaded" +} + +organization_name(value: Related.Loaded<app.people.model.organization.Organization?>(organization)) (name: Text) { + name = nullable_organization_name(organization) +} + +nullable_organization_name(organization: null) (name: Text) { + name = "none" +} + +nullable_organization_name(organization: app.people.model.organization.Organization) (name: Text) { + name = organization.name +} + +manager_name(value: Related.Unloaded<app.people.model.user.User?>()) (name: Text) { + name = "unloaded" +} + +manager_name(value: Related.Loaded<app.people.model.user.User?>(manager)) (name: Text) { + name = nullable_manager_name(manager) +} + +nullable_manager_name(manager: null) (name: Text) { + name = "none" +} + +nullable_manager_name(manager: app.people.model.user.User) (name: Text) { + name = manager.name +} + +post_count(value: Related.Unloaded<List<app.people.model.post.Post>>()) (count: Int) { + count = -1 +} + +post_count(value: Related.Loaded<List<app.people.model.post.Post>>(posts)) (count: Int) { + count = length(posts) +} + +show(user: app.people.model.user.User) () { + blocking(dever.io.println(organization_name(user.organization))) + blocking(dever.io.println(manager_name(user.manager))) + blocking(dever.io.println(int.to_text(post_count(user.posts)))) + blocking(dever.io.println(int.to_text(post_count(user.other_posts)))) + blocking(dever.io.println(int.to_text(post_count(user.unloaded_posts)))) +} + +users() (page: app.people.model.user.Page) { + page = model.user.list({ + order = id.asc + size = 10 + with = [organization, manager, posts, other_posts] + }) +} + +transaction transaction_users() (page: app.people.model.user.Page) { + page = users() +} + +related_first() (item: app.people.model.user.User?) { + item = model.user.first({ + order = id.asc + with = [organization, manager, posts] + }) +} + +related_cursor() (page: app.people.model.user.Cursor) { + page = model.user.cursor({ + order = id.asc + size = 10 + with = [organization, manager, posts] + }) +} + +main() () { + organization = model.organization.create({ name = "Acme" }) + first_user = model.user.create({ + name = "First" + organization_id = organization.id + }) + second_user = model.user.create({ + name = "Second" + manager_id = first_user.id + }) + first_post = model.post.create({ + author_id = first_user.id + title = "One" + }) + second_post = model.post.create({ + author_id = first_user.id + title = "Two" + }) + + unloaded = model.user.get(first_user.id) + blocking(dever.io.println(organization_name(unloaded.organization))) + + page = transaction_users() + each(show, page.items) +} +"#; + +fn sources(app: &str) -> SourceMap { + let mut sources = SourceMap::default(); + sources.add("app/people/model/organization.dever", ORGANIZATION); + sources.add("app/people/model/user.dever", USER); + sources.add("app/people/model/post.dever", POST); + sources.add("app/people/app.dever", app); + sources +} + +fn checked(app: &str) -> (SourceMap, dever_core::hir::Program) { + let sources = sources(app); + let program = dever_core::check(&sources).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }); + (sources, program) +} + +fn errors(app: &str) -> String { + let sources = sources(app); + dever_core::check(&sources) + .unwrap_err() + .iter() + .map(|error| error.render(&sources)) + .collect() +} + +#[test] +fn logical_cross_scope_relations_cannot_be_loaded_implicitly() { + let mut sources = SourceMap::default(); + sources.add( + "platform/tenant/model.dever", + "global type Tenant { name: Text }\n", + ); + sources.add( + "news/article/model.dever", + "type Article { tenant_id: platform.tenant.model.id\n title: Text }\n", + ); + sources.add( + "news/article/app.dever", + "list() (page: model.Page) { page = model.list({ with = [tenant] }) }\n", + ); + let errors = dever_core::check(&sources).unwrap_err(); + let rendered = errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>(); + assert!( + rendered.contains("logical cross-scope relation 'tenant' cannot be loaded"), + "{rendered}", + ); +} + +fn configure(native: &dever_core::native::NativeProgram) { + let root = native.executable().parent().unwrap(); + fs::create_dir_all(root.join("config")).unwrap(); + fs::write( + root.join("config/setting.json"), + r#"{ + "database": { + "default": { + "type": "sqlite", + "path": "data/db/app.db", + "max_connections": 1, + "max_page_size": 10 + } + } +}"#, + ) + .unwrap(); +} + +#[test] +fn with_is_static_and_generates_join_and_one_batched_to_many_shape() { + let source_map = sources(APP); + let formatted = dever_core::format::format(&source_map.files()[3]).unwrap(); + assert!(formatted.contains("Related.Loaded<app.people.model.user.User?>")); + checked(&formatted); + + let (sources, program) = checked(APP); + let snapshot = program.model_snapshot(); + assert!(!snapshot.contains("field organization "), "{snapshot}"); + assert!(!snapshot.contains("field posts "), "{snapshot}"); + let api = program.api_snapshot(); + assert!( + api.contains("organization: Related<app.people.model.organization.Organization?>"), + "{api}" + ); + assert!( + api.contains("posts: Related<List<app.people.model.post.Post>>"), + "{api}" + ); + + let native = dever_core::native::emit(&program, &sources, "app.people.users").unwrap(); + assert!( + native.contains("LEFT JOIN \\\"organization\\\""), + "{native}" + ); + assert!( + native.contains("LEFT JOIN \\\"user\\\" AS \\\"_dever_relation_1\\\""), + "{native}" + ); + assert!( + native + .contains("ROW_NUMBER() OVER (PARTITION BY \\\"author_id\\\" ORDER BY \\\"id\\\" ASC)"), + "{native}" + ); + assert!(native.contains(".chunks(998)"), "{native}"); + assert_eq!( + native.matches("database.query_owned(").count(), + 2, + "{native}" + ); + assert!( + native.contains( + "sqlite: \"SELECT COUNT(*) FROM \\\"user\\\"\", postgres: \"SELECT COUNT(*) FROM \\\"user\\\"\"" + ), + "{native}" + ); + + let first = dever_core::native::emit(&program, &sources, "app.people.related_first").unwrap(); + assert!(first.contains("LEFT JOIN \\\"organization\\\""), "{first}"); + assert_eq!(first.matches("database.query_owned(").count(), 1, "{first}"); + + let cursor = dever_core::native::emit(&program, &sources, "app.people.related_cursor").unwrap(); + assert!( + cursor.contains("LEFT JOIN \\\"organization\\\""), + "{cursor}" + ); + assert_eq!( + cursor.matches("database.query_owned(").count(), + 1, + "{cursor}" + ); + assert!(!cursor.contains("COUNT(*)"), "{cursor}"); +} + +#[test] +fn relation_batch_sql_reserves_one_parameter_for_the_per_parent_limit() { + let sql = dever_runtime::database::relation_sql( + "SELECT * FROM child WHERE parent_id IN (", + ") AND rank <= ", + " ORDER BY parent_id, id", + 998, + ); + assert_eq!(sql.sqlite.matches('?').count(), 999, "{}", sql.sqlite); + assert!( + sql.sqlite.contains("?998) AND rank <= ?999"), + "{}", + sql.sqlite + ); + assert_eq!(sql.postgres.matches('$').count(), 999, "{}", sql.postgres); + assert!( + sql.postgres.contains("$998) AND rank <= $999"), + "{}", + sql.postgres + ); +} + +#[test] +fn with_rejects_unknown_duplicate_and_stream_relations() { + let unknown = APP.replace("with = [organization, manager, posts]", "with = [missing]"); + assert!(errors(&unknown).contains("unknown Model relation 'missing'")); + + let duplicate = APP.replace( + "with = [organization, manager, posts]", + "with = [organization, organization]", + ); + assert!(errors(&duplicate).contains("with cannot repeat relation 'organization'")); + + let stream = APP.replace("page = model.user.list({", "page = model.user.stream({"); + assert!(errors(&stream).contains("unknown query option 'with'")); +} + +#[test] +fn related_patterns_are_typed_exhaustive_and_do_not_relax_regular_cycles() { + let invalid_payload = APP.replace( + "main() () {", + "invalid(value: Related<Int>) () {\n}\n\nmain() () {", + ); + assert!( + errors(&invalid_payload) + .contains("Related payload must be a Model, nullable Model or List<Model>") + ); + let nullable_list = APP.replace( + "main() () {", + "invalid(value: Related<List<app.people.model.user.User>?>) () {\n}\n\nmain() () {", + ); + assert!( + errors(&nullable_list) + .contains("Related payload must be a Model, nullable Model or List<Model>") + ); + + let missing_unloaded = APP.replace( + concat!( + "organization_name(value: Related.Unloaded<", + "app.people.model.organization.Organization?>()) (name: Text) {\n", + " name = \"unloaded\"\n}\n\n", + ), + "", + ); + assert!(errors(&missing_unloaded).contains("do not cover every input combination")); + + let recursive = r#" +type Node { + child: Node? +} +"#; + assert!(errors(recursive).contains("recursive type definitions are not allowed")); +} + +#[test] +fn sqlite_native_loads_requested_relations_and_exposes_related_matching() { + let (sources, program) = checked(APP); + let rustc = std::env::var_os("RUSTC").unwrap_or_else(|| OsString::from("rustc")); + let native = dever_core::native::compile_application( + &program, + &sources, + "app.people.main", + &rustc, + dever_runtime::config::RuntimeProfile { + sqlite: true, + postgres: false, + }, + ) + .unwrap(); + configure(&native); + let output = Command::new(native.executable()).output().unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + String::from_utf8(output.stdout).unwrap(), + "unloaded\nAcme\nnone\n2\n2\n-1\nnone\nFirst\n0\n0\n-1\n" + ); + assert!(output.stderr.is_empty()); +} diff --git a/test/dever-tests/tests/orm_seed.rs b/test/dever-tests/tests/orm_seed.rs new file mode 100644 index 0000000..6b98ffe --- /dev/null +++ b/test/dever-tests/tests/orm_seed.rs @@ -0,0 +1,212 @@ +#[path = "support/orm_sqlite.rs"] +mod orm_sqlite; +mod support; + +use dever_runtime::orm::{ErrorKind, Value}; +use dever_runtime::sqlite::Database; +use orm_sqlite::{TestField, created_at, database_state, id, index, model, seed, text, text_field}; + +const INSERT_SEED: &str = "INSERT INTO \"user\" (\"code\", \"email\", \"label\") VALUES (?1, ?2, ?3) ON CONFLICT (\"code\") DO NOTHING"; + +fn seed_row(code: &str, email: &str, label: &str) -> dever_runtime::database::Seed { + seed( + INSERT_SEED, + vec![ + Value::Text(code.into()), + Value::Text(email.into()), + Value::Text(label.into()), + ], + ) +} + +fn seed_model(seed_revision: &str, rows: &[(&str, &str, &str)]) -> dever_runtime::database::Model { + model( + "7000000000000001", + seed_revision, + vec![ + id(), + text_field("code", 32, false), + text_field("email", 128, false), + text_field("label", 64, false), + created_at(), + ], + vec![ + index("_dever_idx_user_code", &["code"], true), + index("_dever_idx_user_email", &["email"], true), + ], + vec![], + rows.iter() + .map(|(code, email, label)| seed_row(code, email, label)) + .collect(), + ) +} + +fn initial_seed() -> dever_runtime::database::Model { + seed_model("seed-1", &[("pending", "pending@seed.test", "Pending")]) +} + +fn expanded_seed() -> dever_runtime::database::Model { + seed_model( + "seed-2", + &[ + ("pending", "pending@changed.test", "Changed by source"), + ("done", "done@seed.test", "Done"), + ], + ) +} + +#[tokio::test] +async fn changed_seed_appends_missing_rows_preserves_business_values_and_fast_skips() { + let root = support::temp::TemporaryDirectory::new(); + let database = Database::open(root.path().join("seed.db"), 1, 50).unwrap(); + database.migrate(initial_seed()).await.unwrap(); + database + .execute_batch( + "UPDATE user SET email = 'business@test.local', label = 'Business value' WHERE code = 'pending'" + .into(), + ) + .await + .unwrap(); + + database.migrate(expanded_seed()).await.unwrap(); + let rows = database + .query("SELECT code, email, label FROM user ORDER BY code", vec![]) + .await + .unwrap(); + assert_eq!(rows.len(), 2); + assert_eq!(text(&rows[0], 0), "done"); + assert_eq!(text(&rows[0], 1), "done@seed.test"); + assert_eq!(text(&rows[1], 0), "pending"); + assert_eq!(text(&rows[1], 1), "business@test.local"); + assert_eq!(text(&rows[1], 2), "Business value"); + + database + .execute_batch( + "CREATE TABLE seed_probe (attempt INTEGER NOT NULL); \ + CREATE TRIGGER observe_seed BEFORE INSERT ON user BEGIN \ + INSERT INTO seed_probe (attempt) VALUES (1); \ + END" + .into(), + ) + .await + .unwrap(); + let before = database_state(&database).await; + database.migrate(expanded_seed()).await.unwrap(); + assert_eq!(database_state(&database).await, before); + let attempts = database + .query("SELECT COUNT(*) FROM seed_probe", vec![]) + .await + .unwrap(); + assert_eq!(attempts[0].get(0).unwrap(), &Value::Int(0)); + assert!( + before + .history + .iter() + .any(|row| { text(row, 0) == "seed" && text(row, 2) == "seed-2" }) + ); +} + +#[tokio::test] +async fn non_target_seed_constraint_failure_rolls_back_rows_and_history() { + let root = support::temp::TemporaryDirectory::new(); + let database = Database::open(root.path().join("seed-rollback.db"), 1, 50).unwrap(); + database.migrate(expanded_seed()).await.unwrap(); + database + .execute_batch("UPDATE user SET label = 'Business done' WHERE code = 'done'".into()) + .await + .unwrap(); + let before = database_state(&database).await; + + let invalid = seed_model( + "seed-3", + &[ + ("pending", "pending@changed.test", "Pending"), + ("done", "done@seed.test", "Done"), + ("archived", "archived@seed.test", "Archived"), + ("queued", "done@seed.test", "Queued"), + ], + ); + assert_eq!( + database.migrate(invalid).await.unwrap_err().kind(), + ErrorKind::Constraint + ); + assert_eq!(database_state(&database).await, before); + assert!( + before + .history + .iter() + .any(|row| { text(row, 0) == "seed" && text(row, 2) == "seed-2" }) + ); + assert!(!before.data.iter().any(|row| text(row, 1) == "archived")); + assert!( + before + .data + .iter() + .any(|row| { text(row, 1) == "done" && text(row, 3) == "Business done" }) + ); +} + +#[tokio::test] +async fn deferred_constraint_commit_failure_rolls_back_and_releases_the_connection() { + let root = support::temp::TemporaryDirectory::new(); + let database = Database::open(root.path().join("seed-commit-rollback.db"), 1, 50).unwrap(); + database + .execute_batch( + "CREATE TABLE team (id INTEGER PRIMARY KEY); INSERT INTO team (id) VALUES (1)".into(), + ) + .await + .unwrap(); + + let seeded = |revision: &str, owner_id: i64| { + model( + "7100000000000001", + revision, + vec![ + id(), + TestField::new( + "owner_id", + "id:app.model.team", + false, + "\"owner_id\" INTEGER NOT NULL REFERENCES \"team\" (\"id\") DEFERRABLE INITIALLY DEFERRED", + ), + created_at(), + ], + vec![index("_dever_idx_user_owner_id", &["owner_id"], true)], + vec![], + vec![seed( + "INSERT INTO \"user\" (\"owner_id\") VALUES (?1) ON CONFLICT (\"owner_id\") DO NOTHING", + vec![Value::Int(owner_id)], + )], + ) + }; + + assert_eq!( + database + .migrate(seeded("seed-invalid", 99)) + .await + .unwrap_err() + .kind(), + ErrorKind::Constraint + ); + assert!( + database + .query( + "SELECT name FROM sqlite_schema WHERE name IN ('user', '_dever_history', '_dever_idx_user_owner_id')", + vec![], + ) + .await + .unwrap() + .is_empty() + ); + + database.migrate(seeded("seed-valid", 1)).await.unwrap(); + let state = database_state(&database).await; + assert_eq!(state.data.len(), 1); + assert_eq!(state.data[0].get(1).unwrap(), &Value::Int(1)); + assert!( + state + .history + .iter() + .any(|row| { text(row, 0) == "seed" && text(row, 2) == "seed-valid" }) + ); +} diff --git a/test/dever-tests/tests/performance.rs b/test/dever-tests/tests/performance.rs new file mode 100644 index 0000000..dd99154 --- /dev/null +++ b/test/dever-tests/tests/performance.rs @@ -0,0 +1,288 @@ +mod support; + +use std::path::Path; +use std::process::Command; +use std::time::Instant; +use support::{checked, sources, temp::TemporaryDirectory}; + +struct Kernel { + name: &'static str, + bytes: usize, + dever: &'static str, + rust: &'static str, + decimal_initial: bool, + expected: String, +} + +const SAMPLES: usize = 9; +const PROCESSES: usize = 25; +const MAX_RATIO: f64 = 1.15; + +#[test] +#[ignore = "explicit bounded record/list/JSON/large-map benchmark"] +fn allocation_sensitive_source_scenarios() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("fixtures/performance"); + let sources = dever_core::source::SourceMap::load(&root).unwrap(); + let program = checked(&sources); + let compiler = std::env::var_os("RUSTC").unwrap_or_else(|| "rustc".into()); + let executable = + dever_core::native::compile(&program, &sources, "main.main", &compiler).unwrap(); + for process in 0..3 { + let output = Command::new(executable.executable()).output().unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!(output.stderr.is_empty()); + let stdout = String::from_utf8(output.stdout).unwrap(); + let mut counts = std::collections::BTreeMap::<&str, usize>::new(); + for line in stdout.lines() { + let fields: Vec<_> = line.split('|').collect(); + assert_eq!(fields.len(), 3, "{line}"); + let expected = match fields[0] { + "text" | "list" | "map" => "16384", + "json" => "1", + _ => panic!("unexpected result: {line}"), + }; + assert_eq!(fields[1], expected); + assert!(fields[2].parse::<u64>().unwrap() > 0); + *counts.entry(fields[0]).or_default() += 1; + println!("process={process} {line}"); + } + assert_eq!( + counts, + std::collections::BTreeMap::from([("text", 6), ("list", 6), ("json", 6), ("map", 1)]) + ); + } +} + +#[test] +#[ignore = "explicit bounded native hot-path benchmark; never part of default checks"] +fn native_hot_paths_stay_within_the_matching_runtime_baseline() { + let kernels = [ + Kernel { + name: "checked_int_reduce", + bytes: 1_048_576, + dever: "result = int.to_text(reduce(add_int, bytes, 0))", + rust: "let mut total = 0i64; for value in bytes.values() { total = number::int_add(total, i64::from(*value)).unwrap(); } total.to_string()", + expected: "133693440".into(), + decimal_initial: false, + }, + Kernel { + name: "float_reduce", + bytes: 1_048_576, + dever: "result = float.to_text(reduce(add_float, bytes, float.from_int(0)))", + rust: "let mut total = 0.0f64; for value in bytes.values() { total += f64::from(*value); } total.to_string()", + expected: "133693440".into(), + decimal_initial: false, + }, + Kernel { + name: "list_each_filter_sum", + bytes: 262_144, + dever: "values = each(identity, bytes)\nselected = filter(even, values)\nmapped = each(triple, selected)\nresult = int.to_text(sum(mapped))", + rust: "let values = List::new(bytes.values().iter().map(|value| i64::from(*value)).collect()); let selected = List::new(values.values().iter().copied().filter(|value| number::int_rem(*value, 2).unwrap() == 0).collect()); let mapped = List::new(selected.values().iter().map(|value| number::int_mul(*value, 3).unwrap()).collect()); let mut total = 0i64; for value in mapped.values() { total = number::int_add(total, *value).unwrap(); } total.to_string()", + expected: "49938432".into(), + decimal_initial: false, + }, + Kernel { + name: "decimal_reduce", + bytes: 16_384, + dever: "result = decimal.to_text(reduce(add_decimal, bytes, 0.0))", + rust: "let mut total = decimal_initial; for value in bytes.values() { total = total.checked_add(DecimalValue::from_int(i64::from(*value))).unwrap(); } total.to_string()", + expected: "2.08896E+6".into(), + decimal_initial: true, + }, + Kernel { + name: "ordered_map_reduce", + bytes: 65_536, + dever: "values = reduce(map_put, bytes, {})\nresult = int.to_text(length(values))", + rust: "let mut values = Map::new(Vec::new()).unwrap(); for value in bytes.values() { let value = i64::from(*value); values = values.put(value, number::int_mul(value, 3).unwrap()); } values.pairs().count().to_string()", + expected: "256".into(), + decimal_initial: false, + }, + ]; + let directory = TemporaryDirectory::new(); + let compiler = std::env::var_os("RUSTC").unwrap_or_else(|| "rustc".into()); + let mut failures = Vec::new(); + for kernel in kernels { + let input = directory.path().join(format!("{}.bin", kernel.name)); + std::fs::write( + &input, + (0..kernel.bytes) + .map(|index| index as u8) + .collect::<Vec<_>>(), + ) + .unwrap(); + let source = dever_benchmark(&kernel, &input); + let sources = sources(&source); + let program = checked(&sources); + let native_started = Instant::now(); + let native = + dever_core::native::compile(&program, &sources, "main.main", &compiler).unwrap(); + let native_build_ms = native_started.elapsed().as_millis(); + let baseline_source = directory.path().join(format!("{}.rs", kernel.name)); + let baseline = directory.path().join(kernel.name); + std::fs::write(&baseline_source, rust_benchmark(&kernel, &input)).unwrap(); + let runtime = native.executable().parent().unwrap().join("runtime"); + let baseline_started = Instant::now(); + let compiled = Command::new(&compiler) + .arg("--edition=2024") + .args(dever_core::native::OPTIMIZATION_ARGS) + .arg("--extern") + .arg(format!( + "dever_runtime={}", + runtime.join("libdever_runtime.rlib").display() + )) + .arg("-L") + .arg(format!("dependency={}", runtime.join("deps").display())) + .arg(&baseline_source) + .arg("-o") + .arg(&baseline) + .output() + .unwrap(); + let baseline_build_ms = baseline_started.elapsed().as_millis(); + assert!( + compiled.status.success(), + "{}", + String::from_utf8_lossy(&compiled.stderr) + ); + // 分别测量进程内部热路径；构建、文件读取和进程启动不计入时间。 + let (native_nanos, baseline_nanos) = medians( + kernel.name, + native.executable(), + &baseline, + &kernel.expected, + ); + let ratio = native_nanos as f64 / baseline_nanos as f64; + println!( + "{}: native={}ns baseline={}ns ratio={ratio:.3} budget={MAX_RATIO:.2} native_build={native_build_ms}ms baseline_build={baseline_build_ms}ms", + kernel.name, native_nanos, baseline_nanos + ); + if ratio > MAX_RATIO { + failures.push(format!("{}: {ratio:.3}", kernel.name)); + } + } + assert!( + failures.is_empty(), + "hot-path budget exceeded: {}", + failures.join(", ") + ); +} + +fn medians(kernel: &str, native: &Path, baseline: &Path, expected: &str) -> (u64, u64) { + let executables = [native, baseline]; + let mut measurements = [Vec::new(), Vec::new()]; + // 同一 ELF 在共享宿主上也会出现不同耗时档；交错足够多的独立进程， + // 减少五组采样偶然偏向一档的风险，并保留分组证据供后续诊断。 + for process in 0..PROCESSES { + let order = if process % 2 == 0 { [0, 1] } else { [1, 0] }; + for index in order { + let samples = measure(executables[index], expected); + println!( + "{kernel} process={process} side={} samples={samples:?}", + ["native", "baseline"][index] + ); + measurements[index].extend(samples); + } + } + let [native, baseline] = measurements.map(|mut values| { + values.sort_unstable(); + values[values.len() / 2] + }); + (native, baseline) +} + +fn measure(executable: &Path, expected: &str) -> Vec<u64> { + let output = Command::new(executable).output().unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!( + output.stderr.is_empty(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let stdout = String::from_utf8(output.stdout).unwrap(); + let mut samples = Vec::new(); + for line in stdout.lines() { + let (result, nanos) = line + .split_once('|') + .unwrap_or_else(|| panic!("invalid benchmark output: {line}")); + assert_eq!(result, expected); + samples.push(nanos.parse::<u64>().unwrap()); + } + assert_eq!(samples.len(), SAMPLES + 1); + samples.remove(0); // 同样舍弃首个预热样本。 + assert!(samples.iter().all(|sample| *sample > 0)); + samples +} + +fn dever_benchmark(kernel: &Kernel, input: &Path) -> String { + format!( + r#"public main() () {{ opened(result(dever.io.open({input:?}))) }} +opened(result: dever.io.OpenResult.Opened(file)) () recover("benchmark reports IO failures") {{ + read(result(dever.io.read(file, {bytes}))) + dever.io.close(file) +}} +opened(result: dever.io.OpenResult.Failed(message)) () recover("benchmark reports IO failures") {{ dever.io.println("FAILED:" + message) }} +read(result: dever.io.ReadResult.Done(state)) () recover("benchmark reports IO failures") {{ read_state(state) }} +read(result: other) () recover("benchmark reports IO failures") {{ dever.io.println("FAILED:input") }} +read_state(state: dever.io.ReadState.Read(bytes)) () {{ each(sample, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], bytes) }} +read_state(state: other) () {{ dever.io.println("FAILED:input") }} +clock(result: dever.time.ClockResult.Read(value)) (answer: Int) recover("benchmark reports clock failures") {{ answer = value }} +clock(result: dever.time.ClockResult.Failed(message)) (answer: Int) recover("benchmark reports clock failures") {{ dever.io.println("FAILED:" + message) +answer = -1 }} +sample(index: Int, bytes: Bytes) () {{ + start = clock(result(dever.time.monotonic_nanos())) + result = work(bytes) + elapsed = clock(result(dever.time.monotonic_nanos())) - start + dever.io.println(result + "|" + int.to_text(elapsed)) +}} +work(bytes: Bytes) (result: Text) {{ {body} }} +add_int(value: Int, total: Int) (next: Int) {{ next = total + value }} +identity(value: Int) (result: Int) {{ result = value }} +even(value: Int) (result: Bool) {{ result = value % 2 == 0 }} +triple(value: Int) (result: Int) {{ result = value * 3 }} +add_decimal(value: Int, total: Decimal) (next: Decimal) {{ next = total + value }} +add_float(value: Int, total: Float) (next: Float) {{ next = total + float.from_int(value) }} +map_put(value: Int, values: Map<Int, Int>) (next: Map<Int, Int>) {{ next = put(values, value, value * 3) }} +"#, + input = input.to_str().unwrap(), + bytes = kernel.bytes, + body = kernel.dever + ) +} + +fn rust_benchmark(kernel: &Kernel, input: &Path) -> String { + // Decimal 的 quantum 属于表示：0.0 与 ZERO 不可作为同成本输入比较。 + // 只给 Decimal 内核传入一次性解析的初值，解析不进入计时区间。 + let (parameters, setup, arguments) = if kernel.decimal_initial { + ( + ", decimal_initial: DecimalValue", + "let decimal_initial = DecimalValue::parse(\"0.0\").unwrap();", + ", decimal_initial", + ) + } else { + ("", "", "") + }; + format!( + r#"use dever_runtime::{{bytes::Bytes, collections::{{List, Map}}, number::{{self, DecimalValue}}, time}}; +fn work(bytes: &Bytes{parameters}) -> String {{ {body} }} +fn main() {{ + let bytes = Bytes::new(std::fs::read({input:?}).unwrap()); + {setup} + for _ in 0..10 {{ + let start = time::monotonic_nanos().unwrap(); + let result = work(std::hint::black_box(&bytes){arguments}); + let elapsed = time::monotonic_nanos().unwrap() - start; + println!("{{result}}|{{elapsed}}"); + }} +}} +"#, + input = input.to_str().unwrap(), + body = kernel.rust + ) +} diff --git a/test/dever-tests/tests/performance_port_fixture.rs b/test/dever-tests/tests/performance_port_fixture.rs new file mode 100644 index 0000000..593324a --- /dev/null +++ b/test/dever-tests/tests/performance_port_fixture.rs @@ -0,0 +1,99 @@ +use dever_core::source::SourceMap; +use std::path::Path; + +const RUNTIME_CONFIG: &str = r#" +lifetime() (value: Int) { value = 1000 } +idle_sample_ms() (value: Int) { value = 10 } +pending() (value: Int) { value = 4 } +channel_capacity() (value: Int) { value = 4 } +input() (value: Bytes) { value = dever.bytes.from_text("xxxx") } +pending_input() (value: Bytes) { value = dever.bytes.from_text("xxxx") } +"#; + +const HTTP_CONFIG: &str = r#" +lifetime() (value: Int) { value = 1000 } +certificate() (value: Bytes) { value = dever.bytes.from_text("") } +key() (value: Bytes) { value = dever.bytes.from_text("") } +limits() (value: dever.http.Limits) { value = dever.http.default_limits() } +http2_limits() (value: dever.http.Limits) { value = dever.http.default_limits() } +large_body() (value: Bytes) { value = dever.bytes.from_text("xxxxxxxxxxxxxxxx") } +"#; + +fn check_fixture(fixture: &str, entries: &[&str]) { + let root = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../performance/fixtures") + .join(fixture); + let loaded = SourceMap::load(&root).expect("load performance fixture"); + let mut sources = SourceMap::default(); + for source in loaded.files() { + sources.add( + format!("benchmark/{fixture}/{}", source.path().display()), + source.text(), + ); + } + match fixture { + "runtime" => { + sources.add( + "benchmark/bench/app.dever", + include_str!("../../performance/fixtures/bench.dever"), + ); + sources.add("benchmark/config/app.dever", RUNTIME_CONFIG); + } + "http" => { + sources.add("benchmark/config/app.dever", HTTP_CONFIG); + } + "live" => {} + _ => unreachable!("unknown performance fixture"), + } + let program = dever_core::check(&sources).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }); + for entry in entries { + program + .application_warnings(&format!("benchmark.{fixture}.{entry}")) + .unwrap_or_else(|error| panic!("{fixture}.{entry}: {error}")); + } + let generated = dever_core::native::emit_project( + &program, + &sources, + dever_runtime::config::RuntimeProfile::default(), + ) + .unwrap_or_else(|error| panic!("{fixture} native emission: {error}")); + for entry in entries { + assert!( + generated.contains(&format!("\"benchmark.{fixture}.{entry}\" => {{")), + "missing production CMD root benchmark.{fixture}.{entry}" + ); + } +} + +#[test] +fn runtime_port_fixture_checks_entries() { + check_fixture( + "runtime", + &[ + "sync_idle", + "async_idle", + "tasks", + "tasks_on_worker", + "channels", + "cancel", + ], + ); +} + +#[test] +fn http_port_fixture_checks_entries() { + check_fixture("http", &["http", "https", "http2", "https2"]); +} + +#[test] +fn live_port_fixture_checks_entries() { + check_fixture("live", &["tcp", "ws", "sse"]); +} diff --git a/test/dever-tests/tests/pooled_library.rs b/test/dever-tests/tests/pooled_library.rs new file mode 100644 index 0000000..5afedd6 --- /dev/null +++ b/test/dever-tests/tests/pooled_library.rs @@ -0,0 +1,260 @@ +mod support; + +use support::{checked, rejected, sources, stdout}; + +const CONTEXT: &str = r#"public type Context { name: Text + client: dever.system.HttpClient + events: Channel<Int> } +route(request: dever.http.Request, context: Context) (response: dever.http.Response) { + send(context.events, 1) + response = dever.http.Response { status = 200 + headers = [] + body = dever.bytes.from_text(context.name) } +} +public serve(listener: dever.system.Listener, context: Context) () { + dever.system.http_serve(route, listener, dever.http.default_limits(), context) +} +"#; + +#[test] +fn typed_shared_context_and_async_effects_are_checked_before_emission() { + let map = sources(CONTEXT); + let api = checked(&map).api_snapshot(); + let serve = api + .lines() + .find(|line| line.starts_with("function main.serve")) + .unwrap(); + assert!( + serve.contains("network") && serve.contains("concurrency"), + "{serve}" + ); + rejected( + &CONTEXT.replace("context: Context) (response", "context: Text) (response"), + "signature", + ); + rejected( + &CONTEXT.replace("client: dever.system.HttpClient", "client: Stream<Int>"), + "transferable", + ); + rejected( + &CONTEXT.replace( + " send(context.events, 1)", + " dever.io.println(context.name)", + ), + "cannot run inline", + ); +} + +#[test] +fn stream_construction_does_not_hide_unhandled_failures() { + rejected( + "type Result { Ok\nerror Failed(message: Text) }\nfail_work() () { fail(Result.Failed(\"bad\")) }\nwork() (value: Result) { value = result(fail_work()) }\nmain() () { values = stream([work()])\nclose(values) }", + "failure", + ); + rejected( + "work() (value: dever.time.SleepResult) { value = result(dever.task.sleep(1)) }\nfallback() (value: dever.time.SleepResult) { value = dever.time.SleepResult.Done }\nmain() () { job = run(work())\nvalue = timeout(job, 10, fallback) }", + "failure", + ); +} + +fn native_source(http2: bool) -> String { + let root = include_str!("../fixtures/tls/root.pem"); + let cert = include_str!("../fixtures/tls/server.pem"); + let key = include_str!("../fixtures/tls/server-key.pem"); + let program = r#"type Context { prefix: Text + client: dever.system.HttpClient + hits: Channel<Int> } + +request(target: Text) (value: dever.http.Request) { + value = dever.http.Request { method = "GET" + target = target + headers = [] + body = dever.bytes.from_text("") } +} +serve(listener: dever.system.Listener, tls: dever.system.ServerTls, context: Context) () { + limits = http_limits() + # HTTP/1 的两条池连接之外，还需要一个独立 WSS 连接名额。 + limits.connections = 3 + dever.system.http_serve_live_tls(route, listener, limits, dever.http.default_live_limits(), tls, context) +} +copy_body(body: Bytes) (result: Bytes) { result = body } +route(request: dever.http.Request, reply: dever.system.HttpReply, context: Context) () { + child = run(copy_body(request.body)) + request.body = wait(child) + send(context.hits, 1) + path(request.target, request.body, reply, context) +} +path(target: "/proxy", body: Bytes, reply: dever.system.HttpReply, context: Context) () recover("test route reports failures") { + proxy(result(dever.http.request(context.client, request("/value"))), reply) +} +path(target: "/events", body: Bytes, reply: dever.system.HttpReply, context: Context) () recover("test route reports failures") { + dever.sse.start(reply, []) + dever.sse.send(reply, dever.sse.Event { event = "" + data = context.prefix + id = null + retry_ms = null }) +} +path(target: "/upload", body: Bytes, reply: dever.system.HttpReply, context: Context) () recover("test route reports failures") { + dever.http.respond(reply, dever.http.Response { status = 200 + headers = [] + body = body }) +} +path(target: "/ws", body: Bytes, reply: dever.system.HttpReply, context: Context) () recover("test route reports failures") { + upgraded(result(dever.websocket.accept(reply, dever.websocket.default_limits()))) +} +path(target: other, body: Bytes, reply: dever.system.HttpReply, context: Context) () recover("test route reports failures") { + dever.http.respond(reply, dever.http.Response { status = 200 + headers = [] + body = dever.bytes.from_text(context.prefix) }) +} +proxy(result: dever.http.ResponseResult.Read(response), reply: dever.system.HttpReply) () recover("test proxy reports failure") { + response.headers = [] + dever.http.respond(reply, response) +} +proxy(result: dever.http.ResponseResult.Failed(message), reply: dever.system.HttpReply) () recover("test proxy reports failure") { + dever.http.respond(reply, dever.http.Response { status = 502 + headers = [] + body = dever.bytes.from_text(message) }) +} +upgraded(result: dever.websocket.ConnectResult.Connected(socket)) () recover("test WebSocket policy") { + each(echo, dever.websocket.messages(socket), socket) +} +upgraded(result: dever.websocket.ConnectResult.Failed(message)) () recover("test WebSocket policy") {} +echo(event: dever.websocket.MessageEvent.Read(message), socket: dever.system.WebSocket) () recover("test WebSocket policy") { + dever.websocket.send(socket, message) +} +echo(event: dever.websocket.MessageEvent.Failed(message), socket: dever.system.WebSocket) () recover("test WebSocket policy") {} +received(result: dever.websocket.ReceiveResult.Done(state)) (ok: Bool) recover("test WebSocket receive") { ok = received_state(state) } +received(result: other) (ok: Bool) recover("test WebSocket receive") { ok = false } +received_state(state: dever.websocket.ReceiveState.Read(message)) (ok: Bool) { ok = message_ok(message) } +received_state(state: other) (ok: Bool) { ok = false } +message_ok(message: dever.websocket.Message.Text(text)) (ok: Bool) { ok = text == "native" } +message_ok(message: other) (ok: Bool) { ok = false } +connected(result: dever.websocket.ConnectResult.Connected(socket)) (ok: Bool) recover("test WebSocket client") { + dever.websocket.send(socket, dever.websocket.Message.Text("native")) + ok = received(result(dever.websocket.receive(socket))) + dever.websocket.close(socket, 1000, "done") +} +connected(result: dever.websocket.ConnectResult.Failed(message)) (ok: Bool) recover("test WebSocket client") { ok = false } +buffered(result: dever.http.ResponseResult.Read(response), expected: Text) (ok: Bool) recover("test buffered client") { + ok = response.status == 200 and response.body == dever.bytes.from_text(expected) +} +buffered(result: dever.http.ResponseResult.Failed(message), expected: Text) (ok: Bool) recover("test buffered client") { ok = false } +chunk(event: dever.io.ReadEvent.Chunk(bytes), state: Bytes) (result: Bytes) recover("test stream failure") { result = dever.bytes.concat(state, bytes) } +chunk(event: dever.io.ReadEvent.Failed(message), state: Bytes) (result: Bytes) recover("test stream failure") { result = dever.bytes.from_text("stream failed") } +streamed(result: dever.http.StreamResponseResult.Read(response), expected: Text) (ok: Bool) recover("test streamed client") { + body = reduce(chunk, response.body, dever.bytes.from_text("")) + ok = response.status == 200 and body == dever.bytes.from_text(expected) +} +streamed(result: dever.http.StreamResponseResult.Failed(message), expected: Text) (ok: Bool) recover("test streamed client") { ok = false } +add(value: Int, state: Int) (sum: Int) { sum = state + value } +stopped() () {} + +ready(result: dever.http.ClientResult.Ready(client), listener: dever.system.Listener, identity: dever.system.ServerTls, trust: dever.system.ClientTls, port: Int) (ok: Bool) recover("test client lifecycle") { + hits = channel(Int, 16) + context = Context { prefix = "native" + client = client + hits = hits } + server = run(serve(listener, identity, context)) + proxy = buffered(result(dever.http.request(client, request("/proxy"))), "native") + direct = buffered(result(dever.http.request(client, request("/value"))), "native") + upload = dever.http.StreamRequest { method = "POST" + target = "/upload" + headers = [] + body = stream([dever.io.ReadEvent.Chunk(dever.bytes.from_text("ab")), dever.io.ReadEvent.Chunk(dever.bytes.from_text("cd"))]) } + uploaded = streamed(result(dever.http.open_stream(client, upload)), "abcd") + events = streamed(result(dever.http.open(client, request("/events"))), "data: native\n\n") + socket = connected(result(dever.websocket.open("wss://localhost:" + int.to_text(port) + "/ws", trust, dever.websocket.default_limits()))) + dever.net.close_listener(listener) + timeout(server, 1000, stopped) + dever.http.close_client(client) + close(hits) + count = reduce(add, stream(hits), 0) + ok = proxy and direct and uploaded and events and socket and count == 6 +} +ready(result: dever.http.ClientResult.Failed(message), listener: dever.system.Listener, identity: dever.system.ServerTls, trust: dever.system.ClientTls, port: Int) (ok: Bool) recover("test client lifecycle") { ok = false } +bound(result: dever.net.PortResult.Bound(port), listener: dever.system.Listener, identity: dever.system.ServerTls, trust: dever.system.ClientTls) (ok: Bool) recover("test startup") { + limits = http_limits() + pool = dever.http.default_pool_limits() + pool.chunk_bytes = 2 + ok = ready(result(dever.http.client("https://localhost:" + int.to_text(port), trust, limits, pool)), listener, identity, trust, port) +} +bound(result: dever.net.PortResult.Failed(message), listener: dever.system.Listener, identity: dever.system.ServerTls, trust: dever.system.ClientTls) (ok: Bool) recover("test startup") { ok = false } +listening(result: dever.net.ListenResult.Listening(listener), identity: dever.system.ServerTls, trust: dever.system.ClientTls) (ok: Bool) recover("test startup") { + ok = bound(result(dever.net.port(listener)), listener, identity, trust) +} +listening(result: dever.net.ListenResult.Failed(message), identity: dever.system.ServerTls, trust: dever.system.ClientTls) (ok: Bool) recover("test startup") { ok = false } +identity(result: dever.tls.ServerResult.Ready(config), trust: dever.system.ClientTls) (ok: Bool) recover("test TLS identity") { + ok = listening(result(dever.net.listen("127.0.0.1", 0)), config, trust) +} +identity(result: dever.tls.ServerResult.Failed(message), trust: dever.system.ClientTls) (ok: Bool) recover("test TLS identity") { ok = false } +trusted(result: dever.tls.ClientResult.Ready(config)) (ok: Bool) recover("test TLS trust") { + ok = identity(result(dever.tls.server(dever.bytes.from_text(CERT_PEM), dever.bytes.from_text(KEY_PEM))), config) +} +trusted(result: dever.tls.ClientResult.Failed(message)) (ok: Bool) recover("test TLS trust") { ok = false } +public main() (ok: Bool) { ok = trusted(result(dever.tls.client(dever.bytes.from_text(ROOT_PEM)))) } +"#; + let mut program = program + .replace("CERT_PEM", &format!("{cert:?}")) + .replace("KEY_PEM", &format!("{key:?}")) + .replace("ROOT_PEM", &format!("{root:?}")); + let (connections, protocol) = if http2 { + (1, "dever.http.default_http2_limits()") + } else { + (2, "null") + }; + program.push_str(&format!( + r#" +http_limits() (limits: dever.http.Limits) {{ + limits = dever.http.default_limits() + limits.connections = {connections} + limits.http2 = {protocol} +}} +"# + )); + if http2 { + // 嵌套 /proxy -> /value 必须复用唯一连接；普通 WSS 另由 HTTP/1 变体覆盖。 + program = program.replace( + "socket = connected(result(dever.websocket.open(\"wss://localhost:\" + int.to_text(port) + \"/ws\", trust, dever.websocket.default_limits())))", + "socket = true", + ).replace("count == 6", "count == 5"); + } + program +} + +#[test] +fn native_source_checks_and_formats_with_tls_resources_and_streams() { + for http2 in [false, true] { + let map = sources(&native_source(http2)); + checked(&map); + let formatted = dever_core::format::format(&map.files()[0]).unwrap(); + let map = sources(&formatted); + checked(&map); + assert_eq!( + dever_core::format::format(&map.files()[0]).unwrap(), + formatted + ); + } +} + +#[test] +fn native_tls_pool_context_upload_sse_websocket_and_graceful_shutdown() { + assert_eq!(stdout(&native_source(false)), "ok = true\n"); +} + +#[test] +fn native_http2_tls_multiplexed_proxy_upload_sse_and_child_tasks() { + assert_eq!(stdout(&native_source(true)), "ok = true\n"); +} + +#[test] +fn backend_example_checks_and_formats() { + let map = sources(include_str!( + "../../../examples/old/dever/backend/module/main.dever" + )); + checked(&map); + let first = dever_core::format::format(&map.files()[0]).unwrap(); + let map = sources(&first); + checked(&map); + assert_eq!(dever_core::format::format(&map.files()[0]).unwrap(), first); +} diff --git a/test/dever-tests/tests/pooled_network.rs b/test/dever-tests/tests/pooled_network.rs new file mode 100644 index 0000000..2a7db06 --- /dev/null +++ b/test/dever-tests/tests/pooled_network.rs @@ -0,0 +1,691 @@ +#[path = "support/wire_peer.rs"] +mod wire_peer; + +use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, +}; +use std::time::Duration; + +use dever_runtime::{ + async_stream::AsyncStream, bytes::Bytes, channel::Channel, http, net, task, tls, websocket, +}; +use wire_peer::Peer; + +const ROOT: &[u8] = include_bytes!("../fixtures/tls/root.pem"); +const CERT: &[u8] = include_bytes!("../fixtures/tls/server.pem"); +const KEY: &[u8] = include_bytes!("../fixtures/tls/server-key.pem"); + +fn bytes(value: &[u8]) -> Bytes { + Bytes::new(value.to_vec()) +} +fn limits() -> http::Limits { + http::Limits { + header_bytes: 8192, + body_bytes: 65536, + timeout_ms: 1000, + connections: 2, + http2: None, + } +} +fn pool_limits() -> http::PoolLimits { + http::PoolLimits { + idle_ms: 1000, + chunk_bytes: 4, + read_ms: 500, + } +} +fn request(target: &str) -> http::Request { + http::Request { + method: "GET".into(), + target: target.into(), + headers: vec![], + body: bytes(b""), + } +} +fn trust() -> tls::ClientTls { + tls::client(&bytes(ROOT)).unwrap() +} +fn identity() -> tls::ServerTls { + tls::server(&bytes(CERT), &bytes(KEY)).unwrap() +} +fn run(future: impl std::future::Future<Output = Result<(), String>>) { + task::run_entry_with( + task::RuntimeConfig { + worker_threads: 1, + max_blocking_threads: 2, + task_capacity: 64, + }, + async { + tokio::time::timeout(Duration::from_secs(8), future) + .await + .expect("bounded owned peers") + }, + ) + .unwrap(); +} +async fn client( + port: i64, + limits: http::Limits, + pool: http::PoolLimits, +) -> Result<http::HttpClient, String> { + http::client( + &format!("http://localhost:{port}"), + tls::system(), + limits, + pool, + ) + .await +} +async fn accept(listener: &net::Listener) -> Result<Peer, String> { + Peer::new(net::accept(listener).await?) +} + +#[test] +fn pooled_requests_reuse_one_transport_and_preserve_wire_contracts() { + run(async { + let listener = net::listen("127.0.0.1", 0).await?; + let port = net::port(&listener)?; + let peer = task::run(async move { + let mut peer = accept(&listener).await?; + for index in 0..20 { + let head = peer.head().await?; + assert!(head.starts_with(&format!("GET /{index} HTTP/1.1\r\n")), "{head}"); + assert!(head.to_ascii_lowercase().contains(&format!("host: localhost:{port}\r\n"))); + peer.write(b"HTTP/1.1 200 OK\r\nContent-Length: 6\r\nSet-Cookie: a=1\r\nSet-Cookie: b=2\r\n\r\nabcdef").await?; + } + assert!(net::read(&peer.socket, 1).await?.is_none()); + Ok(()) + }).await?; + let client = client(port, limits(), pool_limits()).await?; + for index in 0..20 { + let response = http::request(&client, request(&format!("/{index}"))).await?; + assert_eq!(response.body.values(), b"abcdef"); + assert_eq!( + response + .headers + .iter() + .filter(|header| header.name == "set-cookie") + .count(), + 2 + ); + } + http::close_client(&client).await?; + task::wait(peer).await?; + Ok(()) + }); +} + +#[test] +fn streamed_response_delivers_early_chunks_past_the_header_deadline() { + run(async { + let listener = net::listen("127.0.0.1", 0).await?; + let port = net::port(&listener)?; + let gate = Channel::new(1).unwrap(); + let signal = gate.clone(); + let peer = task::run(async move { + let mut peer = accept(&listener).await?; + peer.head().await?; + peer.write(b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nContent-Type: text/event-stream\r\n\r\n6\r\nhello!\r\n").await?; + signal.receive().await?; + task::sleep(80).await?; + peer.write(b"6\r\nworld!\r\n0\r\n\r\n").await?; + assert!(net::read(&peer.socket, 1).await?.is_none()); + Ok(()) + }).await?; + let mut bounded = limits(); + bounded.timeout_ms = 40; + let client = client(port, bounded, pool_limits()).await?; + let response = http::open(&client, request("/events")).await?; + assert_eq!(response.body.pull().await.unwrap()?.values(), b"hell"); + assert_eq!(response.body.pull().await.unwrap()?.values(), b"o!"); + gate.send(()).await?; + let mut tail = Vec::new(); + while let Some(chunk) = response.body.pull().await { + let chunk = chunk?; + assert!(chunk.values().len() <= 4); + tail.extend_from_slice(chunk.values()); + } + assert_eq!(tail, b"world!"); + http::close_client(&client).await?; + task::wait(peer).await?; + Ok(()) + }); +} + +#[test] +fn abandoned_response_releases_capacity_and_closes_incomplete_connection() { + run(async { + let listener = net::listen("127.0.0.1", 0).await?; + let port = net::port(&listener)?; + let peer = task::run(async move { + let mut first = accept(&listener).await?; + first.head().await?; + first + .write(b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\na") + .await?; + assert!(net::read(&first.socket, 1).await?.is_none()); + let mut second = accept(&listener).await?; + second.head().await?; + second + .write(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") + .await?; + assert!(net::read(&second.socket, 1).await?.is_none()); + Ok(()) + }) + .await?; + let mut bounded = limits(); + bounded.connections = 1; + let client = client(port, bounded, pool_limits()).await?; + let first = http::open(&client, request("/first")).await?; + let pending_client = client.clone(); + let pending = + task::run(async move { http::request(&pending_client, request("/second")).await }) + .await?; + task::sleep(20).await?; + first.body.close(); + assert_eq!(task::wait(pending).await?.body.values(), b"ok"); + http::close_client(&client).await?; + task::wait(peer).await?; + Ok(()) + }); +} + +#[test] +fn closing_pool_wakes_head_waiters_queue_waiters_and_body_readers() { + run(async { + let listener = net::listen("127.0.0.1", 0).await?; + let port = net::port(&listener)?; + let accepted = Channel::new(1).unwrap(); + let signal = accepted.clone(); + let peer = task::run(async move { + let mut peer = accept(&listener).await?; + peer.head().await?; + signal.send(()).await?; + assert!(net::read(&peer.socket, 1).await?.is_none()); + Ok(()) + }) + .await?; + let mut bounded = limits(); + bounded.connections = 1; + let client = client(port, bounded, pool_limits()).await?; + let a = client.clone(); + let first = + task::run(async move { Ok(http::open(&a, request("/first")).await.is_err()) }).await?; + accepted.receive().await?; + let b = client.clone(); + let queued = + task::run(async move { Ok(http::open(&b, request("/queued")).await.is_err()) }).await?; + http::close_client(&client).await?; + http::close_client(&client).await?; + assert!(task::wait(first).await? && task::wait(queued).await?); + task::wait(peer).await?; + + let listener = net::listen("127.0.0.1", 0).await?; + let port = net::port(&listener)?; + let peer = task::run(async move { + let mut peer = accept(&listener).await?; + peer.head().await?; + peer.write(b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n") + .await?; + assert!(net::read(&peer.socket, 1).await?.is_none()); + Ok(()) + }) + .await?; + let client = self::client(port, limits(), pool_limits()).await?; + let body = http::open(&client, request("/body")).await?.body; + let reader = body.clone(); + let pending = task::run(async move { Ok(reader.pull().await.unwrap().is_err()) }).await?; + http::close_client(&client).await?; + assert!(task::wait(pending).await?); + assert!(body.pull().await.is_none()); + task::wait(peer).await?; + Ok(()) + }); +} + +#[test] +fn idle_expiry_closes_transport_without_another_request() { + run(async { + let listener = net::listen("127.0.0.1", 0).await?; + let port = net::port(&listener)?; + let peer = task::run(async move { + let mut peer = accept(&listener).await?; + peer.head().await?; + peer.write(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") + .await?; + assert!(net::read(&peer.socket, 1).await?.is_none()); + Ok(()) + }) + .await?; + let mut pool = pool_limits(); + pool.idle_ms = 30; + let client = client(port, limits(), pool).await?; + http::request(&client, request("/")).await?; + tokio::time::timeout(Duration::from_millis(300), task::wait(peer)) + .await + .unwrap()?; + http::close_client(&client).await?; + Ok(()) + }); +} + +#[test] +fn upload_is_chunked_and_pull_driven() { + run(async { + let listener = net::listen("127.0.0.1", 0).await?; + let port = net::port(&listener)?; + let accepted = Channel::new(1).unwrap(); + let signal = accepted.clone(); + let peer = task::run(async move { + let mut peer = accept(&listener).await?; + let head = peer.head().await?.to_ascii_lowercase(); + assert!(head.starts_with("post /upload http/1.1\r\n")); + assert!(head.contains("transfer-encoding: chunked\r\n")); + assert_eq!(peer.chunk().await?, b"abc"); + signal.send(()).await?; + assert_eq!(peer.chunk().await?, b"def"); + assert!(peer.chunk().await?.is_empty()); + peer.write(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") + .await?; + assert!(net::read(&peer.socket, 1).await?.is_none()); + Ok(()) + }) + .await?; + let client = client(port, limits(), pool_limits()).await?; + let input = Channel::new(1).unwrap(); + input.send(Ok(bytes(b"abc"))).await?; + let source = input.clone().stream(); + let sender_client = client.clone(); + let upload = task::run(async move { + http::open_stream( + &sender_client, + http::StreamRequest { + method: "POST".into(), + target: "/upload".into(), + headers: vec![], + body: source, + }, + ) + .await + }) + .await?; + accepted.receive().await?; + input.send(Ok(bytes(b"def"))).await?; + input.close().await?; + let response = task::wait(upload).await?; + assert_eq!(response.body.pull().await.unwrap()?.values(), b"ok"); + assert!(response.body.pull().await.is_none()); + http::close_client(&client).await?; + task::wait(peer).await?; + Ok(()) + }); +} + +#[test] +fn truncated_and_oversized_bodies_are_errors_and_stream_fault_is_terminal() { + run(async { + for wire in [ + b"HTTP/1.1 200 OK\r\nContent-Length: 9\r\n\r\na".as_slice(), + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n1\r\na\r\n0\r\nX-Trailer: bad\r\n\r\n".as_slice(), + ] { + let listener = net::listen("127.0.0.1", 0).await?; + let port = net::port(&listener)?; + let peer = task::run(async move { + let mut peer = accept(&listener).await?; + peer.head().await?; + peer.write(wire).await?; + peer.socket.close()?; + Ok(()) + }).await?; + let client = client(port, limits(), pool_limits()).await?; + let body = http::open(&client, request("/")).await?.body; + let mut failures = 0; + while let Some(chunk) = body.pull().await { if chunk.is_err() { failures += 1; } } + assert_eq!(failures, 1); + assert!(body.pull().await.is_none()); + http::close_client(&client).await?; + task::wait(peer).await?; + } + let listener = net::listen("127.0.0.1", 0).await?; + let port = net::port(&listener)?; + let peer = task::run(async move { + let mut peer = accept(&listener).await?; + peer.head().await?; + peer.write(b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\n1234") + .await?; + assert!(net::read(&peer.socket, 1).await?.is_none()); + Ok(()) + }) + .await?; + let mut bounded = limits(); + bounded.body_bytes = 3; + let client = client(port, bounded, pool_limits()).await?; + assert!( + http::request(&client, request("/")) + .await + .unwrap_err() + .contains("body_bytes") + ); + http::close_client(&client).await?; + task::wait(peer).await?; + Ok(()) + }); +} + +#[test] +fn upload_rejects_oversized_chunks_and_source_failure() { + run(async { + for chunk in [Ok(bytes(b"too large")), Err("source failed".to_owned())] { + let listener = net::listen("127.0.0.1", 0).await?; + let port = net::port(&listener)?; + let peer = task::run(async move { + let peer = accept(&listener).await?; + while net::read(&peer.socket, 4096).await?.is_some() {} + Ok(()) + }) + .await?; + let client = client(port, limits(), pool_limits()).await?; + let response = http::open_stream( + &client, + http::StreamRequest { + method: "POST".into(), + target: "/".into(), + headers: vec![], + body: AsyncStream::from_values([chunk]), + }, + ) + .await; + assert!(response.is_err()); + http::close_client(&client).await?; + task::wait(peer).await?; + } + Ok(()) + }); +} + +#[test] +fn https_verifies_ca_and_hostname_and_keeps_server_alive_after_rejection() { + run(async { + let listener = net::listen("127.0.0.1", 0).await?; + let port = net::port(&listener)?; + let server = task::run(http::serve_tls( + |request| async move { + Ok(http::Response { + status: 200, + headers: vec![], + body: Bytes::from_string(request.target), + }) + }, + listener, + limits(), + identity(), + )) + .await?; + let untrusted = http::client( + &format!("https://localhost:{port}"), + tls::system(), + limits(), + pool_limits(), + ) + .await?; + assert!( + http::request(&untrusted, request("/untrusted")) + .await + .is_err() + ); + http::close_client(&untrusted).await?; + let wrong_name = http::client( + &format!("https://127.0.0.1:{port}"), + trust(), + limits(), + pool_limits(), + ) + .await?; + assert!( + http::request(&wrong_name, request("/wrong-name")) + .await + .is_err() + ); + http::close_client(&wrong_name).await?; + let trusted = http::client( + &format!("https://localhost:{port}"), + trust(), + limits(), + pool_limits(), + ) + .await?; + for path in ["/one", "/two"] { + assert_eq!( + http::request(&trusted, request(path)).await?.body.values(), + path.as_bytes() + ); + } + http::close_client(&trusted).await?; + task::stop(server).await?; + assert!(tls::client(&bytes(b"not a certificate")).is_err()); + assert!(tls::server(&bytes(CERT), &bytes(ROOT)).is_err()); + Ok(()) + }); +} + +#[test] +fn wss_upgrade_duplex_and_close_share_the_tls_transport() { + run(async { + let listener = net::listen("127.0.0.1", 0).await?; + let port = net::port(&listener)?; + let ws_limits = websocket::Limits { + message_bytes: 1024, + idle_ms: 1000, + write_ms: 1000, + }; + let live = http::LiveLimits { + chunk_bytes: 1024, + idle_ms: 1000, + heartbeat_ms: 100, + }; + let server = task::run(http::serve_live_tls( + move |_, reply| async move { + let socket = websocket::accept(&reply, ws_limits).await?; + if let Some(message) = websocket::receive(&socket).await? { + websocket::send(&socket, message).await?; + } + assert!(websocket::receive(&socket).await?.is_none()); + Ok(()) + }, + listener, + limits(), + live, + identity(), + )) + .await?; + let socket = + websocket::open(&format!("wss://localhost:{port}/ws"), trust(), ws_limits).await?; + websocket::send(&socket, websocket::Message::Text("中文 TLS".into())).await?; + assert!( + matches!(websocket::receive(&socket).await?, Some(websocket::Message::Text(value)) if value == "中文 TLS") + ); + websocket::close(&socket, 1000, "done").await?; + task::stop(server).await?; + Ok(()) + }); +} + +#[test] +fn graceful_shutdown_stops_accepting_and_finishes_active_response() { + run(async { + let listener = net::listen("127.0.0.1", 0).await?; + let port = net::port(&listener)?; + let gate = Channel::new(1).unwrap(); + let release = gate.clone(); + let returned = Arc::new(AtomicBool::new(false)); + let completed = returned.clone(); + let live = http::LiveLimits { + chunk_bytes: 1024, + idle_ms: 1000, + heartbeat_ms: 100, + }; + let server = task::run(http::serve_live( + move |_, reply| { + let gate = gate.clone(); + let returned = returned.clone(); + async move { + http::start(&reply, 200, vec![]).await?; + http::write(&reply, bytes(b"first")).await?; + gate.receive().await?; + http::write(&reply, bytes(b"last")).await?; + returned.store(true, Ordering::Release); + Ok(()) + } + }, + listener.clone(), + limits(), + live, + )) + .await?; + let mut peer = Peer::connect(port).await?; + peer.write(b"GET / HTTP/1.1\r\nHost: test\r\n\r\n").await?; + peer.head().await?; + assert_eq!(peer.chunk().await?, b"first"); + listener.close()?; + assert!(!completed.load(Ordering::Acquire)); + release.send(()).await?; + assert_eq!(peer.chunk().await?, b"last"); + assert!(peer.chunk().await?.is_empty()); + task::wait(server).await?; + assert!(completed.load(Ordering::Acquire)); + assert!(net::connect_timeout("127.0.0.1", port, 100).await.is_err()); + Ok(()) + }); +} + +#[test] +fn task_scope_closes_an_escaped_pool_instead_of_detaching_drivers() { + run(async { + let child = task::run(async { + http::client("http://localhost:1", tls::system(), limits(), pool_limits()).await + }) + .await?; + let pool = task::wait(child).await?; + assert_eq!( + http::open(&pool, request("/")).await.unwrap_err(), + "HTTP client is closed" + ); + http::close_client(&pool).await?; + Ok(()) + }); +} + +#[test] +fn stopping_an_upload_closes_partial_request_and_pool_remains_usable() { + run(async { + let listener = net::listen("127.0.0.1", 0).await?; + let port = net::port(&listener)?; + let accepted = Channel::new(1).unwrap(); + let signal = accepted.clone(); + let peer = task::run(async move { + let mut first = accept(&listener).await?; + first.head().await?; + assert_eq!(first.chunk().await?, b"abc"); + signal.send(()).await?; + assert!(net::read(&first.socket, 1).await?.is_none()); + let mut next = accept(&listener).await?; + assert!(next.head().await?.starts_with("GET /next HTTP/1.1")); + next.write(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") + .await?; + assert!(net::read(&next.socket, 1).await?.is_none()); + Ok(()) + }) + .await?; + let mut bounded = limits(); + bounded.connections = 1; + let client = client(port, bounded, pool_limits()).await?; + let chunks = Channel::new(1).unwrap(); + chunks.send(Ok(bytes(b"abc"))).await?; + let stream = chunks.clone().stream(); + let source = stream.clone(); + let sender_client = client.clone(); + let sender = task::run(async move { + let _response = http::open_stream( + &sender_client, + http::StreamRequest { + method: "POST".into(), + target: "/upload".into(), + headers: vec![], + body: source, + }, + ) + .await?; + Ok(()) + }) + .await?; + accepted.receive().await?; + task::stop(sender).await?; + assert!(stream.pull().await.is_none()); + assert_eq!( + http::request(&client, request("/next")) + .await? + .body + .values(), + b"ok" + ); + http::close_client(&client).await?; + task::wait(peer).await?; + Ok(()) + }); +} + +#[test] +fn pool_queue_and_body_deadlines_release_only_their_own_operations() { + run(async { + let listener = net::listen("127.0.0.1", 0).await?; + let port = net::port(&listener)?; + let peer = task::run(async move { + let mut first = accept(&listener).await?; + first.head().await?; + first + .write(b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\na") + .await?; + assert!(net::read(&first.socket, 1).await?.is_none()); + let mut next = accept(&listener).await?; + next.head().await?; + next.write(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") + .await?; + assert!(net::read(&next.socket, 1).await?.is_none()); + Ok(()) + }) + .await?; + let mut bounded = limits(); + bounded.connections = 1; + bounded.timeout_ms = 30; + let mut pool = pool_limits(); + pool.read_ms = 30; + let client = client(port, bounded, pool).await?; + let body = http::open(&client, request("/first")).await?.body; + assert!( + http::request(&client, request("/queued")) + .await + .unwrap_err() + .contains("timed out") + ); + assert_eq!(body.pull().await.unwrap()?.values(), b"a"); + assert!( + body.pull() + .await + .unwrap() + .unwrap_err() + .contains("read timed out") + ); + assert!(body.pull().await.is_none()); + assert_eq!( + http::request(&client, request("/next")) + .await? + .body + .values(), + b"ok" + ); + http::close_client(&client).await?; + task::wait(peer).await?; + Ok(()) + }); +} diff --git a/test/dever-tests/tests/port_adapter.rs b/test/dever-tests/tests/port_adapter.rs new file mode 100644 index 0000000..002ba80 --- /dev/null +++ b/test/dever-tests/tests/port_adapter.rs @@ -0,0 +1,984 @@ +#[path = "support/temp.rs"] +mod temp; + +use dever_core::source::SourceMap; +use std::{fs, process::Command}; +use temp::TemporaryDirectory; + +const APP: &str = "type ReadResult {\n Read(value: Int)\n error Unavailable(message: Text)\n}\nread() (value: Int) { value = port.read() }\n"; +const PORT: &str = "read() (value: Int) fails app.ReadResult\n"; +const ADAPTER: &str = "port.read() (value: Int) { value = 7 }"; +const COMMAND_PORT: &str = "execute(args: List<Text>, stdin: Bytes?) (output: dever.process.Output) fails dever.process.Error"; +const COMMAND_APP: &str = "read() (value: Int) { output = port.execute([\"argument with spaces\"], null)\n value = output.code }"; + +fn command_sources(port: &str, adapter: &str, app: &str) -> SourceMap { + sources(port, adapter, &[("notification/mail/app.dever", app)]) +} + +#[test] +fn command_contract_selects_only_its_bytes_policy_and_both_typed_backends() { + use dever_core::wire::Policy; + + let mut source = command_sources( + COMMAND_PORT, + "external command \"bin/tool\" {}", + COMMAND_APP, + ); + source.add("notification/mail/api.dever", "cmd read = app.read"); + let program = checked(&source); + assert!(program.has_external_adapters()); + assert!(program.external_worker_contracts().is_empty()); + assert!(program.external_worker_entries().is_empty()); + assert!(program.external_lib_requests().is_empty()); + let contracts = program.external_command_contracts(); + assert_eq!(contracts.len(), 1); + let contract = &contracts[0]; + assert_eq!(contract.ecosystem, "command"); + assert_eq!(contract.operations, ["execute"]); + assert_eq!(contract.entry, "module/notification/mail/bin/tool"); + assert_eq!( + contract.inputs["execute"]["fields"]["stdin"]["value"], + serde_json::json!({"type":"bytes","encoding":"base64"}) + ); + assert_eq!( + contract.outputs["execute"]["fields"]["output"]["name"], + "dever.process.Output" + ); + assert_eq!( + contract.errors.keys().collect::<Vec<_>>(), + [&"dever.process.Error.Failed".to_owned()] + ); + assert!( + program + .wire_schema("dever.process.Output", Policy::Command) + .is_ok() + ); + for policy in [ + Policy::Output, + Policy::ApiInput, + Policy::SettingInput, + Policy::Job, + ] { + assert!( + program + .wire_schema("dever.process.Output", policy) + .unwrap_err() + .contains("Bytes") + ); + } + let native = dever_core::native::emit(&program, &source, "notification.mail.read").unwrap(); + assert!(native.contains("node.bytes()")); + assert!(native.contains("writer.bytes(value)")); + let llvm = dever_core::llvm::emit_application(&program, &source).unwrap(); + assert!(llvm.contains("call i32 @dever_rt_v1_wire_node_bytes(")); + assert!(llvm.contains("call i32 @dever_rt_v1_wire_encoder_bytes(")); + + checked(&command_sources( + &COMMAND_PORT + .replace(", stdin: Bytes?", "") + .replace("execute", "convert"), + "external command \"bin/tool\" { allow file }", + "read() (value: Int) { output = port.convert([])\n value = output.code }", + )); +} + +#[test] +fn command_contract_rejects_signature_drift_worker_options_and_paths() { + for (port, message) in [ + ( + COMMAND_PORT.replace("args:", "arguments:"), + "inputs must be", + ), + (COMMAND_PORT.replace("List<Text>", "Text"), "inputs must be"), + (COMMAND_PORT.replace("stdin:", "input:"), "inputs must be"), + (COMMAND_PORT.replace("Bytes?", "Bytes"), "inputs must be"), + ( + COMMAND_PORT.replace("Bytes?", "Bytes?, extra: Int"), + "inputs must be", + ), + ( + COMMAND_PORT.replace("output:", "result:"), + "must return output", + ), + ( + COMMAND_PORT.replace("dever.process.Output", "Int"), + "must return output", + ), + ( + COMMAND_PORT.replace("dever.process.Error", "dever.process.ArgumentsResult"), + "must declare fails", + ), + ( + format!( + "{COMMAND_PORT}\n{}", + COMMAND_PORT.replace("execute", "convert") + ), + "exactly one Port operation", + ), + ] { + rejected( + &command_sources( + &port, + "external command \"bin/tool\" {}", + "read() (value: Int) { value = 0 }", + ), + message, + ); + } + for (adapter, message) in [ + ( + "setting { value: Int }\nexternal command \"bin/tool\" {}", + "does not accept setting or lib", + ), + ( + "external command \"bin/tool\" { lib \"pip:example@1.0\" }", + "does not accept setting or lib", + ), + ("external command \"../tool\" {}", "portable relative path"), + ("external command \"/tool\" {}", "portable relative path"), + ( + "external command \"bin/\\u{0}tool\" {}", + "portable relative path", + ), + ( + "external command \"bin/tool\" { allow unknown }", + "unknown external capability", + ), + ] { + rejected( + &command_sources(COMMAND_PORT, adapter, COMMAND_APP), + message, + ); + } + rejected( + &command_sources(COMMAND_PORT, "external exec \"bin/tool\" {}", COMMAND_APP), + "unsupported wire type Bytes", + ); + let mut api = command_sources( + COMMAND_PORT, + "external command \"bin/tool\" {}", + "read() (output: dever.process.Output) { output = port.execute([], null) }", + ); + api.add("notification/mail/api.dever", "cmd read = app.read"); + rejected(&api, "Bytes"); +} + +#[test] +fn command_adapter_formats_plain_and_markdown_sources_idempotently() { + let adapter = "external command \"bin/tool\"{allow file\nallow process}"; + let source = command_sources(COMMAND_PORT, adapter, COMMAND_APP); + checked(&source); + let plain = source + .files() + .iter() + .find(|file| file.path().ends_with("adapter.dever")) + .unwrap(); + let formatted = dever_core::format::format(plain).unwrap(); + assert_eq!( + formatted, + "external command \"bin/tool\" {\n allow file\n allow process\n}\n" + ); + checked(&command_sources(COMMAND_PORT, &formatted, COMMAND_APP)); + + let markdown = format!( + "# 命令适配器\n\n执行普通二进制。\n\n- 包：`notification.mail.adapter`\n- 公开类型：无\n- 公开方法：无\n- 使用：无\n\n## 外部程序\n\n传递参数和标准输入。\n\n- 声明：`external command bin/tool`\n\n```dever\n{adapter}\n```\n" + ); + let mut source = SourceMap::default(); + source.add("notification/mail/app.dever", COMMAND_APP); + source.add("notification/mail/port.dever", COMMAND_PORT); + let id = source.add("notification/mail/adapter.dever.md", &markdown); + checked(&source); + let formatted = dever_core::format::format(source.get(id)).unwrap(); + assert!(formatted.contains("传递参数和标准输入。")); + let mut round_trip = SourceMap::default(); + let id = round_trip.add("notification/mail/adapter.dever.md", formatted.clone()); + assert_eq!( + dever_core::format::format(round_trip.get(id)).unwrap(), + formatted + ); +} + +#[test] +fn command_effects_and_test_fakes_reuse_the_port_boundary() { + rejected( + &command_sources( + COMMAND_PORT, + "external command \"bin/tool\" {}", + &COMMAND_APP.replace("(value: Int)", "(value: Int) pure"), + ), + "pure", + ); + let root = TemporaryDirectory::new(); + for (path, text) in [ + ("module/notification/mail/app.dever", COMMAND_APP), + ("module/notification/mail/port.dever", COMMAND_PORT), + ( + "module/notification/mail/adapter.dever", + "external command \"bin/missing-program\" { allow process }", + ), + ( + "test/notification/mail/first.dever", + "first() () { assert_eq(app.read(), 23) }\nport.execute(args: List<Text>, stdin: Bytes?) (output: dever.process.Output) { output = dever.process.Output { code = 23\n stdout = dever.bytes.from_text(\"fake\")\n stderr = dever.bytes.from_text(\"\") } }", + ), + ] { + let path = root.path().join(path); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, text).unwrap(); + } + let source = + SourceMap::load_project(&root.path().join("module"), &root.path().join("test")).unwrap(); + let program = checked(&source); + let llvm = dever_core::llvm::emit_test_suite(&program, &source).unwrap(); + assert!(!llvm.contains("@dever_rt_v1_external_start")); + assert!(!llvm.contains("bin/missing-program")); +} + +#[cfg(feature = "component")] +#[test] +fn command_bytes_wire_is_lossless_strict_and_bounded() { + use dever_runtime::{bytes::Bytes, wire}; + for (bytes, json) in [(vec![], "\"\""), (vec![0, 255, 128, 10], "\"AP+ACg==\"")] { + let bytes = Bytes::new(bytes); + let mut writer = wire::Encoder::default(); + writer.bytes(&bytes).unwrap(); + let encoded = writer.finish().unwrap(); + assert_eq!(encoded.as_str(), json); + assert_eq!(wire::parse(json).unwrap().bytes().unwrap(), bytes); + } + for json in [ + "null", + "[0,255]", + "\"_w==\"", + "\"/w\"", + "\"/x==\"", + "\" /w==\"", + ] { + assert!(wire::parse(json).unwrap().bytes().is_err(), "{json}"); + } + let oversized = Bytes::new(vec![0; wire::MAX_BYTES * 3 / 4]); + assert!(wire::Encoder::default().bytes(&oversized).is_err()); +} + +fn sources(port: &str, adapter: &str, extra: &[(&str, &str)]) -> SourceMap { + let mut sources = SourceMap::default(); + sources.add("notification/mail/port.dever", port); + sources.add("notification/mail/adapter.dever", adapter); + if !extra + .iter() + .any(|(path, _)| *path == "notification/mail/app.dever") + { + sources.add("notification/mail/app.dever", APP); + } + for (path, text) in extra { + sources.add(*path, *text); + } + sources +} + +fn checked(sources: &SourceMap) -> dever_core::hir::Program { + dever_core::check(sources).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(sources)) + .collect::<Vec<_>>() + .join("\n") + ) + }) +} + +fn generated_external_schema(port: &str, app: &str, adapter: &str) -> String { + let source = sources( + port, + adapter, + &[ + ("notification/mail/app.dever", app), + ( + "main.dever", + "main() (value: Int) { value = notification.mail.read() }", + ), + ], + ); + let program = checked(&source); + let generated = dever_core::native::emit(&program, &source, "main.main").unwrap(); + let definition = generated + .split("dever_runtime::component::Definition {") + .nth(1) + .expect("external component definition"); + let schema = definition + .split("schema: \"") + .nth(1) + .expect("external schema literal"); + schema + .split('"') + .next() + .expect("external schema value") + .to_owned() +} + +fn rejected(sources: &SourceMap, message: &str) { + let errors = dever_core::check(sources).expect_err("must reject invalid Port boundary"); + assert!( + errors.iter().any(|error| error.message.contains(message)), + "{errors:?}" + ); +} + +#[test] +fn checks_bodyless_contract_implementation_and_setting_schema() { + checked(&sources(PORT, ADAPTER, &[])); + checked(&sources( + PORT, + "setting { value: Int }\nport.read() (value: Int) { value = setting.value }", + &[], + )); + rejected( + &sources( + &PORT.replace("fails app.ReadResult", "{ value = 1 }"), + ADAPTER, + &[], + ), + "bodyless", + ); + rejected( + &sources(PORT, "port.read() (value: Text) { value = \"wrong\" }", &[]), + "preserve Port", + ); + rejected( + &sources( + PORT, + "type Other { error Broken(message: Text) }\nport.read() (value: Int) { fail(Other.Broken(\"failed\")) }", + &[], + ), + "failure set exceeds", + ); + rejected( + &sources( + PORT, + "setting { value: Int }\nport.read() (value: Int) {\n setting.value = 1\n value = 7\n}", + &[], + ), + "read-only", + ); + checked(&sources( + PORT, + ADAPTER, + &[( + "notification/mail/app.dever", + &format!("{APP}\nsetting() (value: Int) {{ value = 1 }}"), + )], + )); +} + +#[test] +fn port_contract_types_include_nested_app_dtos() { + let port = "type Message { recipient: app.Recipient }\nread(message: Message) (value: Int) fails app.ReadResult"; + let adapter = "port.read(message: port.Message) (value: Int) { value = message.recipient.id }"; + let app = "type Recipient { id: Int }\ntype Unrelated { value: Int }\ntype ReadResult { error Unavailable(message: Text) }\nread() (value: Int) { value = port.read(port.Message { recipient = Recipient { id = 1 } }) }"; + let extra = [("notification/mail/app.dever", app)]; + checked(&sources(port, adapter, &extra)); + rejected( + &sources( + port, + &format!("{adapter}\ntype Private {{ unrelated: app.Unrelated }}"), + &extra, + ), + "outside the current domain boundary", + ); +} + +#[test] +fn rejects_role_bypasses_and_private_port_failure_exports() { + rejected( + &sources( + "type Error { error Failed(message: Text) }\nread() (value: Int) fails Error", + ADAPTER, + &[], + ), + "public App error choice", + ); + rejected( + &sources( + PORT, + "read() (value: Int) { value = 1 }\nport.read() (value: Int) { value = read() }", + &[( + "notification/mail/app.dever", + "type ReadResult { error Unavailable(message: Text) }\nread() (value: Int) { value = adapter.read() }", + )], + ), + "not callable", + ); + rejected( + &sources( + PORT, + ADAPTER, + &[( + "notification/mail/domain.dever", + "read() (value: Int) { value = port.read() }", + )], + ), + "not callable", + ); + rejected( + &sources(PORT, "port.read() (value: Int) { value = app.read() }", &[]), + "not callable", + ); +} + +#[test] +fn formats_port_and_qualified_adapter_without_changing_contracts() { + for (path, text) in [ + ("notification/mail/port.dever", PORT), + ( + "notification/mail/adapter.dever", + "setting { value: Int }\nport.read() (value: Int) { value = setting.value }", + ), + ] { + let mut sources = SourceMap::default(); + let id = sources.add(path, text); + let formatted = dever_core::format::format(sources.get(id)).unwrap(); + let mut formatted_sources = SourceMap::default(); + let id = formatted_sources.add(path, &formatted); + assert_eq!( + formatted, + dever_core::format::format(formatted_sources.get(id)).unwrap() + ); + } +} + +#[test] +fn checks_and_formats_external_exec_adapter_contract() { + let adapter = "external exec \"worker/mail\" {\n allow network\n allow gpu\n}"; + let source = sources(PORT, adapter, &[]); + checked(&source); + let id = source + .files() + .iter() + .find(|file| file.path().to_string_lossy().contains("adapter")) + .expect("Adapter source") + .id(); + let formatted = dever_core::format::format(source.get(id)).unwrap(); + assert_eq!( + formatted, + "external exec \"worker/mail\" {\n allow network\n allow gpu\n}\n" + ); + let mut round_trip = SourceMap::default(); + round_trip.add("notification/mail/adapter.dever", &formatted); + round_trip.add("notification/mail/app.dever", APP); + round_trip.add("notification/mail/port.dever", PORT); + checked(&round_trip); + + checked(&sources( + PORT, + "setting {\n endpoint: Text\n token: Secret\n}\nexternal exec \"worker/mail\" { allow network }", + &[], + )); + + rejected( + &sources(PORT, "external exec \"../worker\" {}", &[]), + "portable relative path", + ); + rejected( + &sources(PORT, "external exec \"worker\" { allow unknown }", &[]), + "unknown external capability", + ); + rejected( + &sources( + PORT, + "external exec \"worker\" {\n allow file\n allow file\n}", + &[], + ), + "duplicate external capability", + ); + rejected( + &sources( + PORT, + "external exec \"worker\" {}\nport.read() (value: Int) { value = 1 }", + &[], + ), + "may contain only", + ); + + let mut missing = SourceMap::default(); + missing.add( + "notification/mail/adapter.dever", + "external exec \"worker\" {}", + ); + rejected(&missing, "requires one Port identity"); + + let mut ambiguous = SourceMap::default(); + ambiguous.add("notification/mail/app.dever", APP); + ambiguous.add("notification/mail/port/primary.dever", PORT); + ambiguous.add( + "notification/mail/port/secondary.dever", + "send() () fails app.ReadResult", + ); + ambiguous.add( + "notification/mail/adapter.dever", + "external exec \"worker\" {}", + ); + rejected(&ambiguous, "multiple Port identities"); + + let mut wrong_role = SourceMap::default(); + let id = wrong_role.add("notification/mail/app.dever", "external exec \"worker\" {}"); + assert!( + dever_core::parse(wrong_role.get(id)).is_err(), + "external must stay contextual to Adapter sources" + ); +} + +#[test] +fn external_adapter_exposes_exact_lib_requests_to_the_checked_program() { + let source = sources( + PORT, + "external exec \"worker/mail\" {\n lib \"pip:fixture@1.0.0\"\n lib \"npm:fixture@1.0.0\"\n allow network\n}", + &[( + "main.dever", + "main() (value: Int) { value = notification.mail.read() }", + )], + ); + let program = checked(&source); + assert_eq!( + program.external_lib_requests(), + vec![ + "npm:fixture@1.0.0".to_owned(), + "pip:fixture@1.0.0".to_owned(), + ] + ); + let adapter = source + .files() + .iter() + .find(|file| file.path().to_string_lossy().contains("adapter")) + .unwrap(); + let formatted = dever_core::format::format(source.get(adapter.id())).unwrap(); + assert!(formatted.contains("lib \"pip:fixture@1.0.0\"")); + assert!(formatted.contains("lib \"npm:fixture@1.0.0\"")); +} + +#[test] +fn external_schema_binds_setting_and_top_level_wire_names() { + let adapter = |field: &str| { + format!( + "setting {{ endpoint: {field} }}\nexternal exec \"worker/mail\" {{ allow network }}" + ) + }; + let base = generated_external_schema(PORT, APP, &adapter("Text")); + let changed_setting = generated_external_schema(PORT, APP, &adapter("Int")); + let renamed_output = generated_external_schema( + "read() (count: Int) fails app.ReadResult\n", + "type ReadResult {\n Read(value: Int)\n error Unavailable(message: Text)\n}\nread() (count: Int) { count = port.read() }\n", + &adapter("Text"), + ); + assert_ne!( + base, changed_setting, + "setting schema must participate in the protocol identity" + ); + assert_ne!( + base, renamed_output, + "top-level wire field names must participate in the protocol identity" + ); +} + +#[test] +fn managed_external_adapters_expose_checked_typed_worker_contracts() { + let port = "type Address { city: Text }\ntype Message {\n recipient: Address\n tags: List<Text>\n}\nread(message: Message) (value: Int?) fails app.ReadResult"; + let app = "type ReadResult { error Unavailable(message: Text, retry_after: Int?) }"; + for (ecosystem, entry) in [ + ("pip", "python/main.py"), + ("npm", "javascript/main.mjs"), + ("go", "go/main.go"), + ] { + let adapter = format!( + "setting {{ token: Secret }}\nexternal {ecosystem} \"{entry}\" {{\n lib \"sample@1.2.3\"\n allow network\n}}" + ); + let source = sources(port, &adapter, &[("notification/mail/app.dever", app)]); + let contracts = checked(&source).external_worker_contracts(); + let [contract] = contracts.as_slice() else { + panic!("one checked Worker") + }; + assert_eq!(contract.ecosystem, ecosystem); + assert_eq!(contract.libs, [format!("{ecosystem}:sample@1.2.3")]); + assert_eq!( + contract.setting.as_ref().unwrap()["fields"]["token"]["type"], + "secret" + ); + assert_eq!( + contract.inputs["read"]["fields"]["message"]["fields"]["recipient"]["fields"]["city"]["type"], + "text" + ); + assert_eq!( + contract.inputs["read"]["fields"]["message"]["fields"]["tags"]["type"], + "list" + ); + assert_eq!( + contract.outputs["read"]["fields"]["value"]["type"], + "nullable" + ); + let errors = contract.errors.iter().collect::<Vec<_>>(); + let [(identity, error)] = errors.as_slice() else { + panic!("one declared error") + }; + assert!(identity.ends_with("ReadResult.Unavailable")); + assert_eq!(error["fields"]["retry_after"]["type"], "nullable"); + assert_eq!(contract.sdk_manifest()["schema"], contract.schema); + let adapter_source = source + .files() + .iter() + .find(|file| file.path().to_string_lossy().contains("adapter")) + .unwrap(); + let formatted = dever_core::format::format(source.get(adapter_source.id())).unwrap(); + assert!(formatted.contains(&format!("external {ecosystem} \"{entry}\""))); + } + rejected( + &sources( + PORT, + "external pip \"python/main.py\" { lib \"npm:other@1.0\" }", + &[], + ), + "exact name@version", + ); + + let source = sources( + PORT, + "external pip \"python/main.py\" { lib \"sample@1.2.3\" }", + &[( + "main.dever", + "main() (value: Int) { value = notification.mail.read() }", + )], + ); + let generated = dever_core::native::emit(&checked(&source), &source, "main.main").unwrap(); + assert!(generated.contains("ecosystem: \"pip\"")); +} + +#[test] +fn markdown_contract_accepts_external_exec_declaration() { + let mut source = SourceMap::default(); + source.add("notification/mail/app.dever", APP); + source.add("notification/mail/port.dever", PORT); + let adapter = "# 外部邮件适配器\n\n通过受管 Worker 实现邮件边界。\n\n- 包：`notification.mail.adapter`\n- 公开类型：无\n- 公开方法：无\n- 使用：无\n\n## Worker\n\n声明外部 Worker。\n\n- 声明：`external exec worker/mail`\n\n```dever\nexternal exec \"worker/mail\" {\n allow network\n}\n```\n"; + source.add("notification/mail/adapter.dever.md", adapter); + checked(&source); + + let mut invalid = SourceMap::default(); + invalid.add("notification/mail/app.dever", APP); + invalid.add("notification/mail/port.dever", PORT); + invalid.add( + "notification/mail/adapter.dever.md", + adapter.replace("external exec worker/mail`", "external exec worker/other`"), + ); + rejected(&invalid, "documented Model declaration"); +} + +#[test] +fn external_exec_adapter_emits_dispatch_and_shutdown_cleanup() { + let source = sources( + PORT, + "external exec \"worker/mail\" { allow network }", + &[( + "main.dever", + "main() (value: Int) { value = notification.mail.read() }", + )], + ); + let program = checked(&source); + let generated = dever_core::native::emit(&program, &source, "main.main").unwrap(); + assert!(generated.contains("dever_runtime::component::call(")); + assert!(generated.contains("dever_runtime::component::shutdown().await")); + assert!(generated.contains("match (application, cleanup)")); +} + +#[test] +fn markdown_contract_includes_declared_failures_and_qualified_implementation() { + let mut source = SourceMap::default(); + source.add("notification/mail/app.dever", APP); + let port = "# 邮件边界\n\n读取邮件计数。\n\n- 包：`notification.mail.port`\n- 公开类型：无\n- 公开方法：无\n- 使用：无\n\n## 读取计数\n\n声明读取合同。\n\n- 函数：`read`\n- 输入：无\n- 输出：\n - `value: Int`：计数。\n- 允许失败：`app.ReadResult`\n\n```dever\nread() (value: Int) fails app.ReadResult\n```\n"; + let adapter = "# 邮件实现\n\n提供固定计数。\n\n- 包：`notification.mail.adapter`\n- 公开类型：无\n- 公开方法：无\n- 使用：无\n\n## 读取计数\n\n实现读取合同。\n\n- 函数：`port.read`\n- 输入：无\n- 输出：\n - `value: Int`：计数。\n\n```dever\nport.read() (value: Int) { value = 7 }\n```\n"; + source.add("notification/mail/port.dever.md", port); + source.add("notification/mail/adapter.dever.md", adapter); + checked(&source); + let mut invalid = SourceMap::default(); + invalid.add("notification/mail/app.dever", APP); + invalid.add( + "notification/mail/port.dever.md", + port.replace("允许失败：`app.ReadResult`", "允许失败：`app.Other`"), + ); + invalid.add("notification/mail/adapter.dever", ADAPTER); + rejected(&invalid, "documented Port failure choice"); +} + +#[test] +fn effects_union_and_direct_io_boundaries_are_conservative() { + let mut source = SourceMap::default(); + source.add("notification/mail/app.dever", "pause() () { port.pause() }"); + source.add( + "notification/mail/port.dever", + "pause() () fails dever.time.SleepResult", + ); + source.add( + "notification/mail/adapter/blocking.dever", + "port.pause() () { dever.time.sleep(1) }", + ); + source.add( + "notification/mail/adapter/suspending.dever", + "port.pause() () { dever.task.sleep(1) }", + ); + rejected(&source, "blocking"); + + let mut source = SourceMap::default(); + source.add( + "notification/mail/app.dever", + "open() (file: dever.system.File) { file = dever.io.open(\"not-opened-by-check\") }", + ); + rejected(&source, "I/O must pass through a Port"); + + let source = sources( + PORT, + "port.read() (value: Int) { fail(app.ReadResult.Unavailable(\"offline\")) }", + &[], + ); + let program = checked(&source); + assert!(program.api_snapshot().contains("Unavailable")); + let source = sources( + PORT, + ADAPTER, + &[( + "notification/mail/app.dever", + "type ReadResult {\n Read(value: Int)\n error Unavailable(message: Text)\n}\ncapture() (response: ReadResult) { response = result(port.read()) }", + )], + ); + checked(&source); +} + +#[test] +fn native_selection_decodes_only_selected_setting_and_rejects_invalid_config() { + let mut source = SourceMap::default(); + source.add( + "main.dever", + "main() (value: Int) { value = notification.mail.read() }", + ); + source.add("notification/mail/app.dever", APP); + source.add("notification/mail/port.dever", PORT); + source.add("notification/mail/adapter/fixed.dever", ADAPTER); + source.add("notification/mail/adapter/configured.dever", "setting {\n value: Int\n token: Secret\n}\nport.read() (value: Int) { value = setting.value }"); + let program = checked(&source); + let native = dever_core::native::compile( + &program, + &source, + "main.main", + std::ffi::OsStr::new("rustc"), + ) + .unwrap(); + let root = TemporaryDirectory::new(); + let executable = root.path().join("program"); + native.save(&executable).unwrap(); + fs::create_dir(root.path().join("config")).unwrap(); + let config = root.path().join("config/setting.json"); + for (text, expected) in [ + ( + r#"{"adapter":{"notification.mail":{"use":"fixed"}}}"#, + Some("value = 7\n"), + ), + ( + r#"{"adapter":{"notification.mail":{"use":"configured","setting":{"value":19,"token":"never-print-this-secret"}}}}"#, + Some("value = 19\n"), + ), + ( + r#"{"adapter":{"notification.mail":{"use":"unknown"}}}"#, + None, + ), + (r#"{"adapter":{"notification.mail":{}}}"#, None), + ( + r#"{"adapter":{"notification.mail":{"use":"configured"}}}"#, + None, + ), + ( + r#"{"adapter":{"notification.mail":{"use":"configured","setting":{"value":19,"extra":1}}}}"#, + None, + ), + ( + r#"{"adapter":{"notification.mail":{"use":"configured","setting":{"value":"never-print-this-secret"}}}}"#, + None, + ), + ( + r#"{"adapter":{"notification.mail":{"use":"fixed","use":"configured"}}}"#, + None, + ), + (r#"{"adapter":{"unknown.port":{"use":"fixed"}}}"#, None), + (r#"{"adapter":null}"#, None), + ] { + fs::write(&config, text).unwrap(); + let result = Command::new(&executable).output().unwrap(); + assert_eq!( + result.status.success(), + expected.is_some(), + "{}", + String::from_utf8_lossy(&result.stderr) + ); + if let Some(expected) = expected { + assert_eq!(String::from_utf8_lossy(&result.stdout), expected); + } + assert!(!String::from_utf8_lossy(&result.stderr).contains("never-print-this-secret")); + } +} + +#[test] +fn single_adapter_executes_without_settings() { + let source = sources( + PORT, + ADAPTER, + &[( + "main.dever", + "main() (value: Int) { value = notification.mail.read() }", + )], + ); + let program = checked(&source); + let native = dever_core::native::compile( + &program, + &source, + "main.main", + std::ffi::OsStr::new("rustc"), + ) + .unwrap(); + let root = TemporaryDirectory::new(); + let executable = root.path().join("program"); + native.save(&executable).unwrap(); + let result = Command::new(&executable).output().unwrap(); + assert!( + result.status.success(), + "{}", + String::from_utf8_lossy(&result.stderr) + ); + assert_eq!(String::from_utf8_lossy(&result.stdout), "value = 7\n"); + fs::create_dir(root.path().join("config")).unwrap(); + fs::write( + root.path().join("config/setting.json"), + r#"{"adapter":null}"#, + ) + .unwrap(); + let result = Command::new(&executable).output().unwrap(); + assert!( + !result.status.success(), + "explicit null must not become an omitted binding" + ); +} + +#[test] +fn test_suite_uses_each_case_fake_without_production_settings() { + let root = TemporaryDirectory::new(); + for (path, text) in [ + ("module/main.dever", "main() () {}"), + ("module/notification/mail/app.dever", APP), + ("module/notification/mail/port.dever", PORT), + ( + "module/notification/mail/adapter.dever", + "setting { value: Int }\nport.read() (value: Int) { value = setting.value }", + ), + ( + "test/notification/mail/first.dever", + "first() () { assert_eq(app.read(), 11) }\nport.read() (value: Int) { value = 11 }", + ), + ( + "test/notification/mail/second.dever", + "second() () { assert_eq(app.read(), 23) }\nport.read() (value: Int) { value = 23 }", + ), + ( + "config/setting.json", + "invalid production setting must not be read", + ), + ] { + let path = root.path().join(path); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, text).unwrap(); + } + let source = + SourceMap::load_project(&root.path().join("module"), &root.path().join("test")).unwrap(); + let program = checked(&source); + let native = + dever_core::native::compile_test_suite(&program, &source, std::ffi::OsStr::new("rustc")) + .unwrap(); + for index in 0..2 { + let result = Command::new(native.executable()) + .arg(index.to_string()) + .output() + .unwrap(); + assert!( + result.status.success(), + "{}", + String::from_utf8_lossy(&result.stderr) + ); + } + fs::write( + root.path().join("test/notification/mail/first.dever"), + "first() () { assert_eq(app.read(), 11) }", + ) + .unwrap(); + let source = + SourceMap::load_project(&root.path().join("module"), &root.path().join("test")).unwrap(); + rejected(&source, "requires a fake"); +} + +#[test] +fn test_suite_keeps_blocking_and_suspending_fakes_case_local() { + let root = TemporaryDirectory::new(); + for (path, text) in [ + ("module/main.dever", "main() () {}"), + ( + "module/notification/mail/app.dever", + "pause() () { port.pause() }", + ), + ( + "module/notification/mail/port.dever", + "pause() () fails dever.time.SleepResult", + ), + ( + "module/notification/mail/adapter.dever", + "port.pause() () {}", + ), + ( + "test/notification/mail/first.dever", + "first() () {\n dever.time.sleep(1)\n app.pause()\n}\nport.pause() () { dever.time.sleep(1) }", + ), + ( + "test/notification/mail/second.dever", + "second() () { app.pause() }\nport.pause() () { dever.task.sleep(1) }", + ), + ] { + let path = root.path().join(path); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, text).unwrap(); + } + let source = + SourceMap::load_project(&root.path().join("module"), &root.path().join("test")).unwrap(); + let program = checked(&source); + let native = + dever_core::native::compile_test_suite(&program, &source, std::ffi::OsStr::new("rustc")) + .unwrap(); + for index in 0..2 { + let result = Command::new(native.executable()) + .arg(index.to_string()) + .output() + .unwrap(); + assert!( + result.status.success(), + "{}", + String::from_utf8_lossy(&result.stderr) + ); + } + let second = root.path().join("test/notification/mail/second.dever"); + fs::write(&second, "second() () {\n dever.time.sleep(1)\n app.pause()\n}\nport.pause() () { dever.task.sleep(1) }").unwrap(); + let source = + SourceMap::load_project(&root.path().join("module"), &root.path().join("test")).unwrap(); + rejected(&source, "blocking"); + fs::write(&second, "type Other { error Broken(message: Text) }\nsecond() () {}\nport.pause() () { fail(Other.Broken(\"unused fake still must match\")) }").unwrap(); + let source = + SourceMap::load_project(&root.path().join("module"), &root.path().join("test")).unwrap(); + rejected(&source, "failure set exceeds"); + fs::write( + &second, + "second() () {}\nunused(action: handler() ()) () pure { dever.time.sleep(1) }", + ) + .unwrap(); + let source = + SourceMap::load_project(&root.path().join("module"), &root.path().join("test")).unwrap(); + rejected(&source, "pure function"); +} diff --git a/test/dever-tests/tests/postgres_api.rs b/test/dever-tests/tests/postgres_api.rs new file mode 100644 index 0000000..424e9b4 --- /dev/null +++ b/test/dever-tests/tests/postgres_api.rs @@ -0,0 +1,305 @@ +mod support; + +use std::fs::{self, OpenOptions}; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::{SystemTime, UNIX_EPOCH}; + +use dever_core::source::SourceMap; +use dever_runtime::config::Settings; +use serde_json::{Value, json}; + +fn copy_directory(source: &Path, destination: &Path) { + fs::create_dir(destination).unwrap(); + for entry in fs::read_dir(source).unwrap() { + let entry = entry.unwrap(); + let path = entry.path(); + let target = destination.join(entry.file_name()); + let kind = entry.file_type().unwrap(); + if kind.is_dir() { + copy_directory(&path, &target); + } else if kind.is_file() { + fs::copy(path, target).unwrap(); + } else { + panic!("CMS source must contain only regular files and directories"); + } + } +} + +fn append_fixture(path: &Path, source: &str) { + let mut file = OpenOptions::new().append(true).open(path).unwrap(); + file.write_all(b"\n").unwrap(); + file.write_all(source.as_bytes()).unwrap(); +} + +fn copy_cms_project(workspace: &Path, project: &Path) { + let source = workspace.join("examples/cms/dever"); + fs::create_dir(project).unwrap(); + for directory in ["config", "module", "test"] { + copy_directory(&source.join(directory), &project.join(directory)); + } + append_fixture( + &project.join("module/user/account/app.dever"), + include_str!("../fixtures/postgres_api/account_app.dever"), + ); + append_fixture( + &project.join("module/user/membership/app.dever"), + include_str!("../fixtures/postgres_api/membership_app.dever"), + ); + append_fixture( + &project.join("module/user/account/api.dever"), + include_str!("../fixtures/postgres_api/account_api.dever"), + ); +} + +#[test] +fn postgres_member_fixture_checks_without_a_database() { + let workspace = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let temporary = support::temp::TemporaryDirectory::new(); + let project = temporary.path().join("project"); + copy_cms_project(&workspace, &project); + let sources = SourceMap::load_project(&project.join("module"), &project.join("test")).unwrap(); + let settings = Settings::load_project(&project).unwrap(); + let program = dever_core::check_with_settings(&sources, &settings).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }); + let application_errors = program.application_errors().unwrap(); + assert!(application_errors.is_empty(), "{application_errors:?}"); + assert!(program.api_snapshot().contains("fixture_member")); +} + +fn save_report(workspace: &Path, report: &Value) -> Result<PathBuf, String> { + let directory = workspace.join("target/performance/postgres-http-09-30-v1"); + fs::create_dir_all(&directory).map_err(|error| error.to_string())?; + let path = if directory.join("report.json").exists() { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|error| error.to_string())? + .as_nanos(); + directory.join(format!("report-{}-{nanos}.json", std::process::id())) + } else { + directory.join("report.json") + }; + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(&path) + .map_err(|error| error.to_string())?; + let document = serde_json::to_vec_pretty(report).map_err(|error| error.to_string())?; + file.write_all(&document) + .map_err(|error| error.to_string())?; + Ok(path) +} + +fn result_directory(workspace: &Path) -> PathBuf { + let parent = workspace.join("target/performance/postgres-http-09-30-v1"); + fs::create_dir_all(&parent).unwrap(); + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let run = parent.join(format!("run-{}-{nanos}", std::process::id())); + let mut builder = fs::DirBuilder::new(); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + builder.create(&run).unwrap(); + run.join("result") +} + +#[test] +#[ignore = "requires an isolated PostgreSQL instance configured in config/setting.json"] +fn postgres_cms_http_isolates_two_tenant_databases() { + let summary = support::postgres::run_in_isolated_databases("postgres_api", |setting| { + let workspace = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../..") + .canonicalize() + .unwrap(); + let compiler = workspace.join("target/debug/dever"); + assert!( + compiler.is_file(), + "build the current dever before PostgreSQL acceptance" + ); + let temporary = support::temp::TemporaryDirectory::new(); + let project = temporary.path().join("project"); + copy_cms_project(&workspace, &project); + let settings_path = project.join("config/setting.json"); + let mut settings: Value = + serde_json::from_slice(&fs::read(&settings_path).unwrap()).unwrap(); + settings["database"] = json!({"default": setting.database_setting()}); + fs::write( + &settings_path, + serde_json::to_vec_pretty(&settings).unwrap(), + ) + .unwrap(); + + let result = result_directory(&workspace); + let output = Command::new("python3") + .arg("-B") + .arg(workspace.join("test/performance/cms.py")) + .arg("postgres-case") + .arg("--project") + .arg(&project) + .arg("--output") + .arg(&result) + .arg("--compiler") + .arg(&compiler) + .arg("--articles") + .arg("2") + .arg("--rbac-fixture") + .output() + .map_err(|error| format!("cannot start PostgreSQL CMS acceptance: {error}"))?; + if !output.status.success() { + return Err(format!( + "PostgreSQL CMS acceptance failed; owned logs: {}\n{}\n{}", + result.display(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + )); + } + let response: Value = serde_json::from_slice(&output.stdout) + .map_err(|error| format!("invalid PostgreSQL CMS result: {error}"))?; + if response["status"] != "passed" { + return Err(format!( + "PostgreSQL CMS acceptance did not pass: {response}" + )); + } + let report_path = response["report"] + .as_str() + .ok_or_else(|| "PostgreSQL CMS result has no report path".to_owned())?; + let report_path = Path::new(report_path); + if !report_path.starts_with(&result) { + return Err("PostgreSQL CMS report is outside the owned result directory".into()); + } + let report: Value = + serde_json::from_slice(&fs::read(report_path).map_err(|error| error.to_string())?) + .map_err(|error| error.to_string())?; + if report["status"] != "passed" { + return Err(format!("PostgreSQL CMS report did not pass: {report}")); + } + let tenant_a = &report["tenants"]["tenant_a"]; + let tenant_b = &report["tenants"]["tenant_b"]; + let tenant_a_id = tenant_a["tenant_id"] + .as_i64() + .filter(|id| *id > 0) + .ok_or_else(|| "tenant A has no positive ID".to_owned())?; + let tenant_b_id = tenant_b["tenant_id"] + .as_i64() + .filter(|id| *id > 0) + .ok_or_else(|| "tenant B has no positive ID".to_owned())?; + let user_a_id = tenant_a["user_id"] + .as_i64() + .filter(|id| *id > 0) + .ok_or_else(|| "tenant A has no positive user ID".to_owned())?; + let user_b_id = tenant_b["user_id"] + .as_i64() + .filter(|id| *id > 0) + .ok_or_else(|| "tenant B has no positive user ID".to_owned())?; + if tenant_a_id == tenant_b_id || user_a_id == user_b_id { + return Err("tenant A and B resolved to the same identity".into()); + } + let titles_a = tenant_a["contract"]["titles"] + .as_array() + .ok_or_else(|| "tenant A has no title list".to_owned())?; + let titles_b = tenant_b["contract"]["titles"] + .as_array() + .ok_or_else(|| "tenant B has no title list".to_owned())?; + if titles_a.len() != 2 + || titles_b.len() != 2 + || titles_a == titles_b + || tenant_a["contract"]["articles"] != 2 + || tenant_b["contract"]["articles"] != 2 + || tenant_a["contract"]["duplicate_publish_status"] != 409 + || tenant_b["contract"]["duplicate_publish_status"] != 409 + || tenant_a["slugs"] != json!(["bench-0", "bench-1"]) + || tenant_b["slugs"] != json!(["bench-0", "bench-1"]) + || report["cross_site_cookie_rejected"] != true + || report["cross_tenant_login_rejected"] != true + { + return Err("PostgreSQL CMS tenant or HTTP contract is incomplete".into()); + } + let rbac = &report["rbac"]; + let member_user_id = rbac["member_user_id"] + .as_i64() + .filter(|id| *id > 0) + .ok_or_else(|| "RBAC fixture has no positive non-Owner user ID".to_owned())?; + if member_user_id == user_a_id || member_user_id == user_b_id { + return Err("RBAC fixture selected an Owner identity".into()); + } + let permission_names = ["admin_read", "admin_permissions", "front_list"]; + let mut permission_keys = Vec::new(); + for name in permission_names { + let key = rbac["permissions"][name] + .as_str() + .filter(|key| !key.is_empty()) + .ok_or_else(|| format!("RBAC report has no {name} permission key"))?; + if permission_keys.contains(&key) { + return Err("RBAC report reused one permission key for separate routes".into()); + } + permission_keys.push(key); + } + let checks = [ + "before_grant_denied", + "both_admin_roles_allowed", + "revoked_role_denied", + "second_role_still_allowed", + "front_same_named_role_still_allowed", + ]; + for check in checks { + if rbac[check] != true { + return Err(format!("RBAC HTTP check did not pass: {check}")); + } + } + let summary = json!({ + "status": "passed", + "source": report["source"], + "binary": report["binary"], + "ready_seconds": report["ready_seconds"], + "scope": report["scope"], + "tenants": { + "tenant_a": { + "tenant_id": tenant_a_id, "user_id": user_a_id, + "articles": 2, "titles": titles_a, "slugs": tenant_a["slugs"], + "duplicate_publish_status": 409, + }, + "tenant_b": { + "tenant_id": tenant_b_id, "user_id": user_b_id, + "articles": 2, "titles": titles_b, "slugs": tenant_b["slugs"], + "duplicate_publish_status": 409, + }, + }, + "cross_site_cookie_rejected": true, + "cross_tenant_login_rejected": true, + "rbac": { + "member_user_id": member_user_id, + "permissions": { + "admin_read": permission_keys[0], + "admin_permissions": permission_keys[1], + "front_list": permission_keys[2], + }, + "before_grant_denied": true, + "both_admin_roles_allowed": true, + "revoked_role_denied": true, + "second_role_still_allowed": true, + "front_same_named_role_still_allowed": true, + }, + }); + Ok(summary) + }); + let workspace = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../..") + .canonicalize() + .unwrap(); + let saved = save_report(&workspace, &summary).unwrap(); + println!("PostgreSQL CMS acceptance: {}", saved.display()); + println!("{summary}"); +} diff --git a/test/dever-tests/tests/postgres_orm.rs b/test/dever-tests/tests/postgres_orm.rs new file mode 100644 index 0000000..acdc5a3 --- /dev/null +++ b/test/dever-tests/tests/postgres_orm.rs @@ -0,0 +1,260 @@ +#[path = "support/orm_fixture.rs"] +mod orm_fixture; +#[path = "support/orm_postgres_migration.rs"] +mod orm_postgres_migration; +mod support; + +use std::sync::Arc; +use std::time::Duration; + +use dever_runtime::config::{PostgresTls, RuntimeProfile}; +use dever_runtime::orm::{ErrorKind, Row, Value}; +use dever_runtime::postgres::Database; +use orm_fixture::DatabaseState; +use rustls::ServerConfig; +use rustls::pki_types::{CertificateDer, PrivateKeyDer, pem::PemObject}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpListener; +use tokio_rustls::TlsAcceptor; + +use support::postgres::open_database as database; + +const CERTIFICATE: &[u8] = include_bytes!("../fixtures/tls/server.pem"); +const PRIVATE_KEY: &[u8] = include_bytes!("../fixtures/tls/server-key.pem"); +const FAST_WAIT: Duration = Duration::from_millis(100); +const FAST_IO: Duration = Duration::from_millis(100); +const NORMAL_IO: Duration = Duration::from_secs(5); + +#[test] +fn shared_orm_fixture_compiles_for_postgres_without_a_database() { + orm_fixture::compile( + "app.user.main", + RuntimeProfile { + sqlite: false, + postgres: true, + }, + ); +} + +fn integer(row: &Row, index: usize) -> i64 { + match row.get(index).unwrap() { + Value::Int(value) => *value, + value => panic!("expected Int at column {index}, got {value:?}"), + } +} + +async fn database_state(database: &Database) -> DatabaseState { + let columns = database + .query( + "SELECT column_name FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = 'user' ORDER BY ordinal_position", + vec![], + ) + .await + .unwrap(); + let managed_index_count = database + .query( + "SELECT COUNT(*) FROM pg_indexes WHERE schemaname = current_schema() AND tablename = 'user' AND indexname LIKE '_dever_idx_%'", + vec![], + ) + .await + .unwrap(); + let history = database + .query( + "SELECT kind, revision FROM _dever_history WHERE model = 'app.user.model' AND kind IN ('schema', 'seed') ORDER BY kind", + vec![], + ) + .await + .unwrap(); + let users = database + .query( + "SELECT email, display_name, status, age, alias FROM \"user\" ORDER BY email", + vec![], + ) + .await + .unwrap(); + orm_fixture::database_state(columns, managed_index_count, history, users) +} + +async fn rejects_untrusted_system_tls() { + let listener = TcpListener::bind(("127.0.0.1", 0)).await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let certificates = CertificateDer::pem_slice_iter(CERTIFICATE) + .collect::<Result<Vec<_>, _>>() + .unwrap(); + let key = PrivateKeyDer::from_pem_slice(PRIVATE_KEY).unwrap(); + let config = Arc::new( + ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(certificates, key) + .unwrap(), + ); + let peer = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = [0; 8]; + socket.read_exact(&mut request).await.unwrap(); + assert_eq!(request, [0, 0, 0, 8, 4, 210, 22, 47]); + socket.write_all(b"S").await.unwrap(); + assert!(TlsAcceptor::from(config).accept(socket).await.is_err()); + }); + let database = database( + &format!("postgres://dever@127.0.0.1:{port}/dever"), + PostgresTls::System, + 1, + FAST_WAIT, + Duration::from_secs(1), + ); + let error = database.query("SELECT 1", vec![]).await.unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Connection, "{error}"); + peer.await.unwrap(); +} + +async fn assert_pool_timeout_cancel_disconnect_and_shutdown(url: &str, tls: PostgresTls) { + let pool = Arc::new(database(url, tls, 1, FAST_WAIT, NORMAL_IO)); + let transaction = pool.begin().await.unwrap(); + assert_eq!( + pool.query("SELECT 1", vec![]).await.unwrap_err().kind(), + ErrorKind::PoolExhausted + ); + transaction.rollback().await.unwrap(); + assert_eq!( + integer(&pool.query("SELECT 1", vec![]).await.unwrap()[0], 0), + 1 + ); + + let timeout_pool = database(url, tls, 1, FAST_WAIT, FAST_IO); + assert_eq!( + timeout_pool + .query("SELECT pg_sleep(1)", vec![]) + .await + .unwrap_err() + .kind(), + ErrorKind::Timeout + ); + assert_eq!( + integer(&timeout_pool.query("SELECT 1", vec![]).await.unwrap()[0], 0), + 1 + ); + timeout_pool.close().await.unwrap(); + + let cancel_pool = Arc::new(database(url, tls, 1, FAST_WAIT, NORMAL_IO)); + let pending = { + let database = Arc::clone(&cancel_pool); + tokio::spawn(async move { database.query("SELECT pg_sleep(5)", vec![]).await }) + }; + tokio::time::sleep(Duration::from_millis(50)).await; + pending.abort(); + assert!(pending.await.unwrap_err().is_cancelled()); + assert_eq!( + integer(&cancel_pool.query("SELECT 1", vec![]).await.unwrap()[0], 0), + 1 + ); + cancel_pool.close().await.unwrap(); + + let disconnected = Arc::new(database(url, tls, 1, FAST_WAIT, NORMAL_IO)); + let backend = integer( + &disconnected + .query("SELECT pg_backend_pid()", vec![]) + .await + .unwrap()[0], + 0, + ); + let pending = { + let database = Arc::clone(&disconnected); + tokio::spawn(async move { database.query("SELECT pg_sleep(5)", vec![]).await }) + }; + tokio::time::sleep(Duration::from_millis(50)).await; + let administrator = database(url, tls, 1, FAST_WAIT, NORMAL_IO); + administrator + .query( + "SELECT pg_terminate_backend($1::bigint::integer)", + vec![Value::Int(backend)], + ) + .await + .unwrap(); + assert_eq!( + pending.await.unwrap().unwrap_err().kind(), + ErrorKind::Connection + ); + assert_eq!( + integer(&disconnected.query("SELECT 1", vec![]).await.unwrap()[0], 0), + 1 + ); + administrator.close().await.unwrap(); + disconnected.close().await.unwrap(); + + let draining = Arc::clone(&pool); + let transaction = pool.begin().await.unwrap(); + let shutdown = tokio::spawn(async move { draining.close().await }); + tokio::time::sleep(Duration::from_millis(20)).await; + assert!(!shutdown.is_finished()); + transaction.rollback().await.unwrap(); + shutdown.await.unwrap().unwrap(); + assert_eq!( + pool.query("SELECT 1", vec![]).await.unwrap_err().kind(), + ErrorKind::Connection + ); +} + +#[test] +#[ignore = "requires an isolated database configured in config/setting.json"] +fn postgres_orm_uses_the_configured_isolated_database() { + support::postgres::run_in_isolated_schema("postgres_orm", |setting| async move { + let scoped = setting.url(); + let tls = setting.tls(); + let profile = RuntimeProfile { + sqlite: false, + postgres: true, + }; + let main = orm_fixture::compile("app.user.main", profile); + let rollback = orm_fixture::compile("app.user.rollback", profile); + assert_eq!(main.revisions, rollback.revisions); + let application_setting = setting.settings_json(); + orm_fixture::configure(main.native.executable(), &application_setting); + orm_fixture::configure(rollback.native.executable(), &application_setting); + orm_fixture::assert_main_output(orm_fixture::run(main.native.executable())); + + let application = setting.open(2, Duration::from_secs(1), NORMAL_IO); + let committed = database_state(&application).await; + orm_fixture::assert_committed_state(&committed, &main.revisions); + orm_postgres_migration::assert_named_data_migration(&application).await; + + let rollback_output = orm_fixture::run(rollback.native.executable()); + assert!(!rollback_output.status.success()); + let rollback_error = String::from_utf8(rollback_output.stderr).unwrap(); + assert!( + rollback_error.contains("duplicate key value violates unique constraint"), + "{rollback_error}" + ); + assert_eq!(database_state(&application).await, committed); + + let stream_pool = setting.open(1, FAST_WAIT, NORMAL_IO); + let stream_backend = integer( + &stream_pool + .query("SELECT pg_backend_pid()", vec![]) + .await + .unwrap()[0], + 0, + ); + let stream = stream_pool + .stream("SELECT generate_series(1, 20)", vec![], 2) + .await + .unwrap(); + assert_eq!(integer(&stream.pull().await.unwrap().unwrap(), 0), 1); + stream.close(); + let replacement_backend = integer( + &stream_pool + .query("SELECT pg_backend_pid()", vec![]) + .await + .unwrap()[0], + 0, + ); + assert_ne!(replacement_backend, stream_backend); + stream_pool.close().await.unwrap(); + application.close().await.unwrap(); + + assert_pool_timeout_cancel_disconnect_and_shutdown(scoped, tls).await; + rejects_untrusted_system_tls().await; + + Ok(()) + }); +} diff --git a/test/dever-tests/tests/rest_model.rs b/test/dever-tests/tests/rest_model.rs new file mode 100644 index 0000000..bbf5180 --- /dev/null +++ b/test/dever-tests/tests/rest_model.rs @@ -0,0 +1,478 @@ +use dever_core::source::SourceMap; + +#[path = "support/temp.rs"] +mod temp; + +use temp::TemporaryDirectory; + +fn sources(model: &str) -> SourceMap { + let mut sources = SourceMap::default(); + sources.add("user/profile/model.dever", model); + sources.add("user/profile/api.dever", "rest\n"); + sources +} + +fn diagnostics(sources: &SourceMap) -> String { + dever_core::check(sources) + .expect_err("source must fail checking") + .iter() + .map(|error| error.render(sources)) + .collect() +} + +// Code-generation assertions may inspect unbound HIR; an executable API must +// use the same configured site/auth contract as the CLI. +fn compile_authenticated_rest(mut sources: SourceMap) { + sources.add("access/user/model.dever", "global type User { name: Text }"); + sources.add( + "access/tenant/model.dever", + "global type Tenant { name: Text }", + ); + sources.add( + "access/session/app.dever", + r#"type Identity { + id: Text + user_id: access.user.model.id? + tenant_id: access.tenant.model.id? +} +verify(claims: dever.auth.Claims) (identity: Identity) { + identity = Identity { id = claims.subject + user_id = null + tenant_id = null } +}"#, + ); + let directory = TemporaryDirectory::new(); + std::fs::create_dir(directory.path().join("config")).unwrap(); + std::fs::write( + directory.path().join("config/setting.json"), + r#"{ + "auth": {"providers": {"session": { + "verify": "access.session.verify", + "jwtSecret": "0123456789abcdef0123456789abcdef" + }}}, + "sites": {"admin": {"path": "", "auth": "session"}}, + "database": {"default": {"type": "sqlite", "path": "data/rest.db"}} + }"#, + ) + .unwrap(); + let settings = dever_runtime::config::Settings::load_project(directory.path()).unwrap(); + let program = dever_core::check_with_settings(&sources, &settings).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }); + let native = dever_core::native::compile_project( + &program, + &sources, + std::ffi::OsStr::new("rustc"), + dever_runtime::config::RuntimeProfile { + sqlite: true, + postgres: false, + }, + ) + .unwrap(); + assert!(native.executable().is_file()); +} + +#[test] +fn rest_derives_the_public_model_without_an_app_wrapper() { + let sources = sources( + "type Profile {\n name: Text(1, 64)\n bio: Text?\n score: Decimal(12, 2)?\n uid: Uuid generated\n private token: Text?\n}\n", + ); + let program = dever_core::check(&sources).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }); + assert!( + program + .model_snapshot() + .contains("model user.profile.model") + ); + let generated = dever_core::native::emit_project( + &program, + &sources, + dever_runtime::config::RuntimeProfile { + sqlite: true, + postgres: false, + }, + ) + .unwrap(); + assert!(generated.contains("transaction.commit().await")); + assert!(generated.contains("rest_rollback(transaction).await")); + assert!(generated.contains("dever_runtime::api::text_bounds(\"name\"")); + compile_authenticated_rest(sources); +} + +#[test] +fn rest_rejects_models_with_no_public_writable_field() { + let sources = sources("type Profile {\n private token: Text?\n}\n"); + assert!(diagnostics(&sources).contains("no writable public fields")); +} + +#[test] +fn rest_rejects_private_required_fields_without_a_source() { + let sources = sources("type Profile {\n name: Text\n private token: Text\n}\n"); + assert!(diagnostics(&sources).contains("private and required")); +} + +#[test] +fn rest_rejects_fields_without_a_shared_wire_codec() { + let sources = sources("type Profile {\n name: Text\n photo: Bytes\n}\n"); + assert!(diagnostics(&sources).contains("cannot be serialized")); +} + +#[test] +fn rest_owner_is_server_bound_and_scopes_every_database_statement() { + let mut sources = SourceMap::default(); + sources.add( + "user/account/model.dever", + "global type Account { email: Text(1, 254) unique }", + ); + sources.add( + "user/profile/model.dever", + r#"type Profile { + private owner user_id: user.account.model.id = dever.auth.user_id() + name: Text(1, 64) + bio: Text? +} +"#, + ); + sources.add("user/profile/api.dever", "rest\n"); + let program = dever_core::check(&sources).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }); + let snapshot = program.model_snapshot(); + assert!(snapshot.contains("owner=auth.user_id"), "{snapshot}"); + let generated = dever_core::native::emit_project( + &program, + &sources, + dever_runtime::config::RuntimeProfile { + sqlite: true, + postgres: false, + }, + ) + .unwrap(); + assert!( + generated.contains("dever_runtime::auth::user_id()"), + "{generated}" + ); + assert!( + !generated.contains("inputs.raw_json(\"user_id\")"), + "{generated}" + ); + for sql in [ + "SELECT COUNT(*) FROM \\\"profile\\\" WHERE \\\"user_id\\\" = ?1", + "WHERE \\\"id\\\" = ?1 AND \\\"user_id\\\" = ?2 LIMIT 1", + "INSERT INTO \\\"profile\\\" (\\\"name\\\", \\\"bio\\\", \\\"user_id\\\")", + "AND \\\"user_id\\\" = ?4 RETURNING", + "DELETE FROM \\\"profile\\\" WHERE \\\"id\\\" = ?1 AND \\\"user_id\\\" = ?2", + ] { + assert!(generated.contains(sql), "missing {sql}: {generated}"); + } + compile_authenticated_rest(sources); +} + +#[test] +fn rest_owner_accepts_only_one_non_null_integer_identity_source() { + for (model, expected) in [ + ( + "type Profile { owner user_id: Int? = dever.auth.user_id()\n name: Text }", + "must be non-null", + ), + ( + "type Profile { owner user_id: Text = dever.auth.user_id()\n name: Text }", + "must use Int or a Model id type", + ), + ( + "type Profile { owner user_id: Int default 1 = dever.auth.user_id()\n name: Text }", + "cannot also be generated or declare a default", + ), + ( + "type Profile { owner user_id: Int = dever.api.request_id()\n name: Text }", + "must be dever.auth.user_id() or dever.auth.tenant_id()", + ), + ( + "type Profile { owner user_id: Int = dever.auth.user_id()\n owner tenant_id: Int = dever.auth.tenant_id()\n name: Text }", + "only one REST owner field", + ), + ] { + let sources = sources(model); + let rendered = diagnostics(&sources); + assert!( + rendered.contains(expected), + "expected {expected}: {rendered}" + ); + } +} + +#[test] +fn rest_owner_syntax_formats_idempotently() { + let mut sources = SourceMap::default(); + sources.add( + "user/profile/model.dever", + "type Profile {\nprivate owner user_id:Int=dever.auth.user_id()\nname:Text index\n}\n", + ); + let source = &sources.files()[0]; + let formatted = dever_core::format::format(source).unwrap(); + assert!( + formatted.contains("private owner user_id: Int = dever.auth.user_id()"), + "{formatted}" + ); + let mut roundtrip = SourceMap::default(); + roundtrip.add("user/profile/model.dever", formatted.clone()); + assert_eq!( + dever_core::format::format(&roundtrip.files()[0]).unwrap(), + formatted + ); +} + +#[test] +fn authenticated_rest_uses_stable_operation_permissions() { + let directory = TemporaryDirectory::new(); + std::fs::create_dir(directory.path().join("config")).unwrap(); + std::fs::write( + directory.path().join("config/setting.json"), + r#"{ + "auth": {"providers": {"session": { + "verify": "user.profile.verify", + "jwtSecret": "0123456789abcdef0123456789abcdef" + }}}, + "sites": {"admin": {"path": "admin", "auth": "session"}} + }"#, + ) + .unwrap(); + let mut sources = SourceMap::default(); + sources.add( + "platform/tenant/model.dever", + "global type Tenant { name: Text }", + ); + sources.add( + "user/profile/model.dever", + "global type Profile { name: Text }", + ); + sources.add( + "user/profile/app.dever", + r#"type Identity { + id: Text + user_id: model.id? + tenant_id: platform.tenant.model.id? +} + +verify(claims: dever.auth.Claims) (identity: Identity) { + identity = Identity { + id = claims.subject + user_id = null + tenant_id = null + } +}"#, + ); + sources.add("user/profile/api/admin/manage.dever", "rest model"); + let settings = dever_runtime::config::Settings::load_project(directory.path()).unwrap(); + let program = dever_core::check_with_settings(&sources, &settings).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }); + let generated = dever_core::native::emit_project( + &program, + &sources, + dever_runtime::config::RuntimeProfile { + sqlite: true, + postgres: false, + }, + ) + .unwrap(); + for permission in [ + "user.profile.admin.read", + "user.profile.admin.create", + "user.profile.admin.replace", + "user.profile.admin.delete", + ] { + assert!( + generated.contains(&format!("api_authorize(\"{permission}\", \"admin\")")), + "{generated}" + ); + } +} + +#[test] +fn rest_field_bindings_are_checked_and_emit_static_search_sql() { + let mut sources = SourceMap::default(); + sources.add( + "user/account/model.dever", + "global type Account { email: Text(1, 254) unique }", + ); + sources.add( + "user/profile/domain.dever", + "normalize(value: Text) (result: Text) pure { result = text.lower(text.trim(value)) }", + ); + sources.add("user/profile/model.dever", r#"type Profile { + private owner user_id: user.account.model.id = dever.auth.user_id() + name: Text(1, 64) create = domain.normalize(input) replace = domain.normalize(input) search = domain.normalize(input) + private source: Text? create = dever.api.header("x-source") replace = dever.api.header("x-source") + private direct: Text? create = dever.system.api_header("x-direct") replace = dever.system.api_header("x-direct") +}"#); + sources.add("user/profile/api.dever", "rest model\n"); + let program = dever_core::check(&sources).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }); + assert!(program.application_errors().unwrap().is_empty()); + assert!( + program + .model_snapshot() + .contains("index plain (user_id,name) implicit=true"), + "{}", + program.model_snapshot() + ); + let generated = dever_core::native::emit_project( + &program, + &sources, + dever_runtime::config::RuntimeProfile { + sqlite: true, + postgres: false, + }, + ) + .unwrap(); + assert!( + generated.contains("inputs.raw_json(\"name\")"), + "{generated}" + ); + assert!( + !generated.contains("inputs.raw_json(\"source\")"), + "{generated}" + ); + assert!( + generated.contains("inputs.optional_raw_json(\"name\")"), + "{generated}" + ); + assert!( + generated.contains("WHERE \\\"user_id\\\" = ?1 AND \\\"name\\\" = ?2"), + "{generated}" + ); + assert!( + generated.contains("SELECT COUNT(*) FROM \\\"profile\\\" WHERE \\\"user_id\\\" = ?1"), + "{generated}" + ); + compile_authenticated_rest(sources); +} + +#[test] +fn rest_field_bindings_reject_unsafe_or_ambiguous_sources() { + for (model, domain, expected) in [ + ( + "type Profile { private token: Text create = input\n name: Text }", + "", + "private REST create binding cannot use request input", + ), + ( + "type Profile { name: Text search = \"fixed\" }", + "", + "search binding must use input", + ), + ( + "type Profile { name: Text search = input\n status: Int search = input }", + "", + "at most one search field", + ), + ( + "type Profile { name: Text create = domain.normalize(input) }", + "normalize(value: Text) (result: Text) { result = value }", + "pure same-domain Domain", + ), + ( + "type Profile { created: DateTime create = dever.time.now() }", + "", + "pure same-domain Domain", + ), + ( + "type Profile { name: Text create = domain.mutate(input) }", + "mutate(value: Text) (result: Text) pure {\n dever.api.set_header(\"x-test\", value)\n result = value\n}", + "pure same-domain Domain", + ), + ( + "type Profile { count: Int create = model.count() }", + "", + "Model operations are private to their owning domain App", + ), + ] { + let mut sources = sources(model); + if !domain.is_empty() { + sources.add("user/profile/domain.dever", domain); + } + let rendered = diagnostics(&sources); + assert!( + rendered.contains(expected), + "expected {expected}: {rendered}" + ); + } +} + +#[test] +fn rest_binding_modifiers_do_not_change_app_model_operations() { + let mut sources = SourceMap::default(); + sources.add( + "user/profile/model.dever", + "type Profile { name: Text create = text.trim(input) replace = text.lower(input) }", + ); + sources.add( + "user/profile/app.dever", + "create(name: Text) (profile: model.Profile) { profile = model.create({ name = name }) }", + ); + sources.add("user/profile/api.dever", "post create = app.create"); + let program = dever_core::check(&sources).unwrap(); + let native = dever_core::native::emit_project( + &program, + &sources, + dever_runtime::config::RuntimeProfile { + sqlite: true, + postgres: false, + }, + ) + .unwrap(); + assert!(!native.contains("text_trim"), "{native}"); + assert!(!native.contains("text_lower"), "{native}"); +} + +#[test] +fn rest_binding_syntax_formats_idempotently() { + let mut sources = SourceMap::default(); + sources.add("user/profile/model.dever", "type Profile {\nname:Text create=text.trim(input) replace=text.lower(input) search=input\n}\n"); + let formatted = dever_core::format::format(&sources.files()[0]).unwrap(); + assert!( + formatted.contains( + "name: Text create = text.trim(input) replace = text.lower(input) search = input" + ), + "{formatted}" + ); + let mut roundtrip = SourceMap::default(); + roundtrip.add("user/profile/model.dever", formatted.clone()); + assert_eq!( + dever_core::format::format(&roundtrip.files()[0]).unwrap(), + formatted + ); +} diff --git a/test/dever-tests/tests/runtime_foundations.rs b/test/dever-tests/tests/runtime_foundations.rs new file mode 100644 index 0000000..57f6e71 --- /dev/null +++ b/test/dever-tests/tests/runtime_foundations.rs @@ -0,0 +1,169 @@ +use dever_runtime::bytes::Bytes; +use dever_runtime::collections::{List, Map}; +use dever_runtime::number::{self, DecimalValue as Decimal}; +use dever_runtime::render::Render; +use dever_runtime::resource; +use std::cell::Cell; +use std::rc::Rc; + +#[path = "support/temp.rs"] +mod temp; + +#[test] +fn decimal128_is_exact_at_literals_and_half_even_at_operations() { + let parse = |text| Decimal::parse(text).unwrap(); + assert_eq!( + parse("0.1").checked_add(parse("0.2")).unwrap(), + parse("0.3") + ); + assert_eq!( + parse("1").checked_div(parse("3")).unwrap().to_string(), + "0.3333333333333333333333333333333333" + ); + assert_eq!(parse("2.345").round(2).unwrap(), parse("2.34")); + assert_eq!(parse("2.355").round(2).unwrap(), parse("2.36")); + assert_eq!(parse("-2.345").round(2).unwrap(), parse("-2.34")); + assert_eq!(parse("1250").round(-2).unwrap(), parse("1200")); + assert!(Decimal::parse("0.12345678901234567890123456789012345").is_err()); + assert!(parse("1").checked_div(Decimal::ZERO).is_err()); + assert!(Decimal::maximum().checked_mul(parse("10")).is_err()); + assert_eq!(parse("1").next_up().unwrap().next_down(), Some(parse("1"))); + assert!(Decimal::maximum().next_up().is_none()); + assert!(Decimal::minimum().next_down().is_none()); + assert_eq!( + Decimal::from_bytes(parse("2.5").to_bytes()).unwrap(), + parse("2.5") + ); + assert_eq!(number::int_div(-5, 2).unwrap(), -2); + assert_eq!(number::int_rem(-5, 2).unwrap(), -1); + assert!(number::int_rem(i64::MIN, -1).is_err()); +} + +#[test] +fn ordered_collections_preserve_values_and_nan_equality() { + let list = List::new(vec![1_i64, 2]); + let changed = list.clone().append(3); + assert_eq!(list.values(), &[1, 2]); + assert_eq!(changed.values(), &[1, 2, 3]); + let nan = List::new(vec![f64::NAN]); + assert_ne!(nan, nan.clone()); + let map = Map::new(vec![("a".to_string(), 1_i64), ("b".to_string(), 2)]).unwrap(); + let replaced = map.clone().put("a".into(), 3); + assert_eq!(replaced.render(), "{a = 3, b = 2}"); + assert_eq!(map.render(), "{a = 1, b = 2}"); + let reordered = map.clone().remove(&"a".into()).put("a".into(), 1); + assert_ne!(map, reordered); + assert_eq!(reordered.render(), "{b = 2, a = 1}"); + assert_eq!( + List::new(vec![map.clone(), replaced]).render(), + "[{a = 1, b = 2}, {a = 3, b = 2}]" + ); + assert!(Map::new(vec![(true, 1), (true, 2)]).is_err()); + assert_eq!(map.entries().values()[0].key, "a"); +} + +#[test] +fn bytes_validate_utf8_indices_and_value_semantics() { + let original = Bytes::from_text("abc"); + let combined = original.clone().concat(&Bytes::from_text("def")); + assert_eq!(original.to_text().unwrap(), "abc"); + assert_eq!(combined.slice(1, 4).unwrap().to_text().unwrap(), "bcd"); + assert_eq!(original.at(-1), None); + assert_eq!(original.at(3), None); + assert!(original.slice(2, 1).is_err()); + assert!(original.slice(0, 4).is_err()); + assert!(Bytes::from_ints(&[256]).is_err()); + assert!(Bytes::from_ints(&[255]).unwrap().to_text().is_err()); +} + +#[test] +fn file_handles_close_all_aliases_and_creation_never_overwrites() { + let directory = temp::TemporaryDirectory::new(); + let path = directory.path().join("sample.txt"); + let path = path.to_str().unwrap(); + let file = resource::file_create(path).unwrap(); + let alias = file.clone(); + resource::write(&file, &Bytes::from_text("hello")).unwrap(); + assert!(resource::file_create(path).is_err()); + file.close().unwrap(); + assert!(alias.close().unwrap_err().contains("closed")); + assert!( + resource::write(&alias, &Bytes::from_text("changed")) + .unwrap_err() + .contains("closed") + ); + let reader = resource::file_open(path).unwrap(); + assert!(resource::read(&reader, 0).is_err()); + assert_eq!( + resource::read(&reader, 64) + .unwrap() + .unwrap() + .to_text() + .unwrap(), + "hello" + ); + assert_eq!(resource::read(&reader, 64).unwrap(), None); + reader.close().unwrap(); + assert!(resource::read(&reader, 64).is_err()); + assert_eq!(std::fs::read_to_string(path).unwrap(), "hello"); +} + +#[test] +fn streams_share_progress_close_idempotently_and_release_only_their_alias() { + let directory = temp::TemporaryDirectory::new(); + let path = directory.path().join("stream.txt"); + std::fs::write(&path, "abcdef").unwrap(); + let file = resource::file_open(path.to_str().unwrap()).unwrap(); + let stream = resource::read_chunks(file.clone(), 2).unwrap(); + let alias = stream.clone(); + assert_eq!(stream.pull().unwrap().unwrap().to_text().unwrap(), "ab"); + assert_eq!(alias.pull().unwrap().unwrap().to_text().unwrap(), "cd"); + stream.close(); + stream.close(); + assert!(alias.pull().is_none()); + assert_eq!( + resource::read(&file, 2) + .unwrap() + .unwrap() + .to_text() + .unwrap(), + "ef" + ); +} + +#[test] +fn stream_creation_and_terminal_failures_are_explicit_once() { + let directory = temp::TemporaryDirectory::new(); + let path = directory.path().join("failure.txt"); + std::fs::write(&path, "x").unwrap(); + let file = resource::file_open(path.to_str().unwrap()).unwrap(); + assert!(resource::read_chunks(file.clone(), 0).is_err()); + let stream = resource::read_chunks(file.clone(), 1).unwrap(); + file.close().unwrap(); + assert!(stream.pull().unwrap().is_err()); + assert!(stream.pull().is_none()); + stream.close(); +} + +#[test] +fn mapped_streams_preserve_terminal_items_and_release_their_producer() { + struct DropMarker(Rc<Cell<bool>>); + + impl Drop for DropMarker { + fn drop(&mut self) { + self.0.set(true); + } + } + + let transform_dropped = Rc::new(Cell::new(false)); + let marker = DropMarker(Rc::clone(&transform_dropped)); + let source = resource::Stream::new(|| resource::Pull::Last(1_i64)); + let mapped = source.map(move |value| { + let _keep_alive = ▮ + value + 1 + }); + + assert_eq!(mapped.pull(), Some(2)); + assert!(transform_dropped.get()); + assert_eq!(mapped.pull(), None); +} diff --git a/test/dever-tests/tests/secure_contracts.rs b/test/dever-tests/tests/secure_contracts.rs new file mode 100644 index 0000000..9958ac0 --- /dev/null +++ b/test/dever-tests/tests/secure_contracts.rs @@ -0,0 +1,576 @@ +mod support; + +use dever_core::source::SourceMap; +use dever_runtime::{bytes::Bytes, crypto, secret::Secret}; + +fn sources(files: &[(&str, &str)]) -> SourceMap { + let mut sources = SourceMap::default(); + for (path, text) in files { + sources.add(*path, *text); + } + sources +} + +fn rejected(files: &[(&str, &str)], message: &str) { + let sources = sources(files); + let errors = dever_core::check(&sources).expect_err("must reject unsafe source"); + assert!( + errors.iter().any(|error| error.message.contains(message)), + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ); +} + +const ACCOUNT: &str = "type Account { email: Text\nprivate password_hash: Text(255) }"; + +fn checked_platform(sources: &SourceMap) -> dever_core::hir::Program { + let root = support::temp::TemporaryDirectory::new(); + std::fs::create_dir(root.path().join("config")).unwrap(); + std::fs::write( + root.path().join("config/setting.json"), + r#"{"database":{ + "default":{"type":"sqlite","path":"data/default.db"}, + "platform":{"type":"sqlite","path":"data/platform.db"} + }}"#, + ) + .unwrap(); + let settings = dever_runtime::config::Settings::load_project(root.path()).unwrap(); + let program = support::checked(sources); + program.validate_database_settings(&settings).unwrap(); + program +} + +#[test] +fn password_crypto_api_requires_a_short_explicit_write_transaction() { + let account = format!("database platform\nglobal {ACCOUNT}"); + let sources = sources(&[ + ("user/account/model.dever", &account), + ( + "user/account/app.dever", + r#" +login(credential_id: model.id, password: Secret) (id: model.id) { + stored = model.get(credential_id) + valid = blocking(dever.crypto.password_verify(password, stored.password_hash)) + dever.api.set_header("x-login", "accepted") + id = store(credential_id) +} +transaction store(credential_id: model.id) (id: model.id) { + stored = model.get(credential_id) + created = model.create({ email = stored.email + password_hash = stored.password_hash }) + id = created.id +} +"#, + ), + ( + "user/account/api/front/account.dever", + "public post login = app.login", + ), + ]); + let program = checked_platform(&sources); + let generated = dever_core::native::emit_project( + &program, + &sources, + dever_runtime::config::RuntimeProfile { + sqlite: true, + postgres: false, + }, + ) + .unwrap(); + assert!(!generated.contains("begin_deferred"), "{generated}"); + assert_eq!( + generated + .matches("transaction_database.begin().await") + .count(), + 1, + "{generated}" + ); + + rejected( + &[ + ("user/account/model.dever", &account), + ( + "user/account/app.dever", + r#" +login(credential_id: model.id, password: Secret) (id: model.id) { + stored = model.get(credential_id) + valid = blocking(dever.crypto.password_verify(password, stored.password_hash)) + created = model.create({ email = stored.email + password_hash = stored.password_hash }) + id = created.id +} +"#, + ), + ( + "user/account/api/front/account.dever", + "public post login = app.login", + ), + ], + "explicit transaction function", + ); + + rejected( + &[ + ("user/account/model.dever", &account), + ( + "user/account/app.dever", + r#" +transaction login(credential_id: model.id, password: Secret) (id: model.id) { + stored = model.get(credential_id) + valid = blocking(dever.crypto.password_verify(password, stored.password_hash)) + id = stored.id +} +"#, + ), + ], + "cannot perform password hashing or verification", + ); +} + +#[test] +fn ordinary_write_api_commits_before_publishing_response_metadata() { + let sources = sources(&[ + ("user/account/model.dever", ACCOUNT), + ( + "user/account/app.dever", + r#" +register(email: Text, password_hash: Text) (id: model.id) { + dever.api.set_header("x-register", "accepted") + stored = model.create({ email = email + password_hash = password_hash }) + id = stored.id +} +"#, + ), + ("user/account/api.dever", "post register = app.register"), + ]); + let program = support::checked(&sources); + let generated = dever_core::native::emit_project( + &program, + &sources, + dever_runtime::config::RuntimeProfile { + sqlite: true, + postgres: false, + }, + ) + .unwrap(); + let commit = generated + .find("if let Err(error) = transaction.commit().await") + .unwrap(); + let metadata = generated + .find("dever_runtime::api::commit_response_metadata()") + .unwrap(); + assert!( + commit < metadata, + "response metadata must be published only after the database commits" + ); +} + +#[cfg(feature = "sqlite")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn password_work_window_does_not_hold_the_sqlite_writer_lock() { + use std::sync::Arc; + use std::time::Duration; + + use dever_runtime::orm::Value; + use dever_runtime::sqlite::Database; + + let root = support::temp::TemporaryDirectory::new(); + let database = Arc::new(Database::open(root.path().join("login.db"), 2, 20).unwrap()); + database + .execute_batch( + "CREATE TABLE account(id INTEGER PRIMARY KEY,status TEXT NOT NULL);\ + CREATE TABLE session(id INTEGER PRIMARY KEY,account_id INTEGER NOT NULL);\ + CREATE TABLE platform_write(id INTEGER PRIMARY KEY,value INTEGER NOT NULL);\ + INSERT INTO account(id,status) VALUES(1,'active');\ + INSERT INTO platform_write(id,value) VALUES(1,0);" + .into(), + ) + .await + .unwrap(); + + let account = database + .query( + "SELECT status FROM account WHERE id=?1", + vec![Value::Int(1)], + ) + .await + .unwrap(); + assert_eq!(account.len(), 1); + + let (started, waiting) = tokio::sync::oneshot::channel(); + let password_work = tokio::task::spawn_blocking(move || { + let _ = started.send(()); + std::thread::sleep(Duration::from_millis(500)); + crypto::password_hash(&Secret::from_input(b"correct password".to_vec())).unwrap() + }); + waiting.await.unwrap(); + + tokio::time::timeout(Duration::from_millis(250), async { + let transaction = database.begin().await.unwrap(); + transaction + .execute( + "UPDATE platform_write SET value=value+1 WHERE id=?1", + vec![Value::Int(1)], + ) + .await + .unwrap(); + transaction.commit().await.unwrap(); + }) + .await + .expect("unrelated write must not wait for password work"); + assert!( + !password_work.is_finished(), + "the write must finish during the password work window" + ); + let _hash = password_work.await.unwrap(); + + let transaction = database.begin().await.unwrap(); + let active = transaction + .query( + "SELECT status FROM account WHERE id=?1 AND status='active'", + vec![Value::Int(1)], + ) + .await + .unwrap(); + assert_eq!( + active.len(), + 1, + "mutable account state is rechecked inside the short transaction" + ); + transaction + .execute( + "INSERT INTO session(id,account_id) VALUES(?1,?2)", + vec![Value::Int(1), Value::Int(1)], + ) + .await + .unwrap(); + transaction.commit().await.unwrap(); + + assert_eq!( + database + .query("SELECT id FROM session", vec![]) + .await + .unwrap() + .len(), + 1 + ); +} + +#[test] +fn private_model_fields_are_app_owned_and_do_not_change_storage_schema() { + let plain = sources(&[("user/account/model.dever", &ACCOUNT.replace("private ", ""))]); + let private = sources(&[("user/account/model.dever", ACCOUNT)]); + assert_eq!( + support::checked(&plain).model_snapshot(), + support::checked(&private).model_snapshot() + ); + support::checked(&sources(&[ + ("user/account/model.dever", ACCOUNT), + ( + "user/account/app.dever", + r#" +register(password: Secret) (id: model.id) { + hash = blocking(dever.crypto.password_hash(password)) + stored = model.create({ email = "a@example.com" + password_hash = hash }) + changed = model.update(stored.id, { password_hash = hash }) + valid = blocking(dever.crypto.password_verify(password, stored.password_hash)) + id = stored.id +} +"#, + ), + ])); + for caller in ["utility.dever", "other.dever"] { + rejected( + &[ + ("user/account/model.dever", ACCOUNT), + ( + caller, + "read(stored: user.account.model.Account) (hash: Text) { hash = stored.password_hash }", + ), + ], + "private field", + ); + } + rejected( + &[ + ("user/account/model.dever", ACCOUNT), + ( + "utility.dever", + "rewrite(stored: user.account.model.Account) () { stored.password_hash = \"replacement\" }", + ), + ], + "private field", + ); +} + +#[test] +fn private_models_cannot_cross_app_boundaries_or_render_nested_errors() { + for declaration in [ + "leak() (stored: model.Account) { stored = model.get(1) }", + "accept(stored: model.Account) () {}", + "type Envelope { values: List<model.Account?> }\naccept(value: Envelope) () {}", + "type Failure { error Failed(value: model.Account) }", + "accept(page: model.Page) () {}", + "accept(values: Related<List<model.Account>>) () {}", + ] { + rejected( + &[ + ("user/account/model.dever", ACCOUNT), + ("user/account/app.dever", declaration), + ], + "private Model", + ); + } +} + +#[test] +fn secrets_are_non_observable_at_every_container_and_error_boundary() { + for ty in [ + "Secret", + "Secret?", + "List<Secret>", + "Map<Text, Secret>", + "Envelope", + ] { + let source = format!( + "public type Envelope {{ value: Secret }}\npublic expose(value: {ty}) (out: {ty}) {{ out = value }}" + ); + rejected(&[("utility.dever", &source)], "output Secret"); + } + for source in [ + "type Failure { error Failed(value: Secret) }", + "type Nested { value: List<Secret> }\ntype Failure { error Failed(value: Nested) }", + ] { + rejected(&[("utility.dever", source)], "error payload"); + } + for (source, expected) in [ + ( + "compare(a: Secret, b: Secret) (same: Bool) { same = a == b }", + "comparable", + ), + ( + "join(a: Secret) (value: Text) { value = a + \"x\" }", + "numeric", + ), + ("bad(value: Map<Secret, Text>) () {}", "Map key"), + ("main() () { value = secret(\"password\") }", "unknown"), + ( + "log(value: Secret) () { dever.log.info(value, {}) }", + "expected Text", + ), + ] { + rejected(&[("utility.dever", source)], expected); + } + rejected( + &[( + "user/account/model.dever", + "type Account { password: Secret }", + )], + "Model field", + ); +} + +#[test] +fn public_post_does_not_guess_database_not_found_authentication_semantics() { + let account = format!("database platform\nglobal {ACCOUNT}"); + let sources = sources(&[ + ("user/account/model.dever", &account), + ( + "user/account/app.dever", + "lookup(id: model.id) (email: Text) { stored = model.get(id)\nemail = stored.email }", + ), + ( + "user/account/api/front/account.dever", + "public post lookup = app.lookup", + ), + ]); + let program = checked_platform(&sources); + let generated = dever_core::native::emit_project( + &program, + &sources, + dever_runtime::config::RuntimeProfile { + sqlite: true, + postgres: false, + }, + ) + .unwrap(); + assert!( + !generated.contains("api_anonymous_post_failure_response"), + "{generated}" + ); +} + +#[test] +fn removed_process_configuration_has_no_source_entry() { + for function in [ + "dever.process.environment", + "dever.system.process_environment", + ] { + support::rejected( + &format!("public main() () {{ ignored = {function}(\"setting\") }}"), + "unknown", + ); + } +} + +fn test_sources(test: &str) -> (support::temp::TemporaryDirectory, SourceMap) { + let root = support::temp::TemporaryDirectory::new(); + std::fs::create_dir_all(root.path().join("module")).unwrap(); + std::fs::create_dir_all(root.path().join("test/user/account")).unwrap(); + std::fs::write(root.path().join("module/main.dever"), "main() () {}").unwrap(); + std::fs::write( + root.path().join("test/user/account/credentials.dever"), + test, + ) + .unwrap(); + let sources = + SourceMap::load_project(&root.path().join("module"), &root.path().join("test")).unwrap(); + (root, sources) +} + +#[test] +fn secret_fixture_authority_and_native_rendering_are_checked() { + let (_root, sources) = test_sources( + r#" +credentials() () { + password = secret("test password") + hash = dever.crypto.password_hash(password) + assert(dever.crypto.password_verify(password, hash)) + assert(not dever.crypto.password_verify(secret("wrong"), hash)) +} +"#, + ); + let program = support::checked(&sources); + let rustc = std::env::var_os("RUSTC").unwrap_or_else(|| "rustc".into()); + let native = dever_core::native::compile_test_suite(&program, &sources, &rustc).unwrap(); + let output = std::process::Command::new(native.executable()) + .arg("0") + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!(output.stdout.is_empty()); + for test in [ + "credentials() () { assert_eq(secret(\"a\"), secret(\"a\")) }", + "credentials() () {}\nsecret(value: Text) (out: Text) { out = value }", + ] { + let (_root, sources) = test_sources(test); + assert!(dever_core::check(&sources).is_err()); + } + let (_root, mut sources) = test_sources( + "credentials() () {}\nread(stored: model.Account) (hash: Text) { hash = stored.password_hash }", + ); + sources.add("user/account/model.dever", ACCOUNT); + let errors = dever_core::check(&sources).expect_err("Test cannot read private Model fields"); + assert!( + errors + .iter() + .any(|error| error.message.contains("private field")) + ); +} + +#[test] +fn model_privacy_follows_related_cycles_without_affecting_layout_validation() { + rejected( + &[ + ( + "user/account/model.dever", + "type Account { private password_hash: Text\nfriend_id: user.account.model.id? }", + ), + ( + "user/account/app.dever", + "accept(value: Related<model.Account>) () {}", + ), + ], + "private Model", + ); +} + +#[test] +fn crypto_known_answers_and_password_cost_limits() { + let password = Secret::from_input(b"correct horse battery staple".to_vec()); + let hash = crypto::password_hash(&password).unwrap(); + assert!(hash.starts_with("$argon2id$v=19$m=19456,t=2,p=1$")); + assert!(crypto::password_verify(&password, &hash).unwrap()); + assert!(!crypto::password_verify(&Secret::from_input(b"wrong".to_vec()), &hash).unwrap()); + for bad in [ + "malformed".to_owned(), + hash.replace("m=19456", "m=19457"), + hash.replace("t=2", "t=3"), + hash.replace("p=1", "p=2"), + hash.replace("argon2id", "argon2i"), + hash.replace("v=19", "v=16"), + ] { + assert_eq!( + crypto::password_verify(&password, &bad).unwrap_err(), + "invalid or unsupported password hash" + ); + } + assert_eq!(format!("{password:?}"), "Secret([REDACTED])"); + let abc = Bytes::from_text("abc"); + assert_eq!( + crypto::sha256(&abc).values(), + &[ + 0xba, 0x78, 0x16, 0xbf, 0x8f, 0x01, 0xcf, 0xea, 0x41, 0x41, 0x40, 0xde, 0x5d, 0xae, + 0x22, 0x23, 0xb0, 0x03, 0x61, 0xa3, 0x96, 0x17, 0x7a, 0x9c, 0xb4, 0x10, 0xff, 0x61, + 0xf2, 0x00, 0x15, 0xad + ] + ); + // RFC 4231 test case 1. + let digest = crypto::hmac_sha256( + &Secret::from_input(vec![0x0b; 20]), + &Bytes::from_text("Hi There"), + ); + assert_eq!( + digest.values(), + &[ + 0xb0, 0x34, 0x4c, 0x61, 0xd8, 0xdb, 0x38, 0x53, 0x5c, 0xa8, 0xaf, 0xce, 0xaf, 0x0b, + 0xf1, 0x2b, 0x88, 0x1d, 0xc2, 0x00, 0xc9, 0x83, 0x3d, 0xa7, 0x26, 0xe9, 0x37, 0x6c, + 0x2e, 0x32, 0xcf, 0xf7 + ] + ); + assert!(crypto::constant_time_eq(&digest, &digest)); + assert!(!crypto::constant_time_eq(&digest, &abc)); + let first = crypto::token(32).unwrap(); + let second = crypto::token(32).unwrap(); + assert!(!crypto::constant_time_eq( + &crypto::hmac_sha256(&first, &abc), + &crypto::hmac_sha256(&second, &abc) + )); + assert!(crypto::token(0).is_err()); + assert!(crypto::token(1025).is_err()); +} + +#[test] +fn native_crypto_and_direct_system_intrinsics_select_crypto_runtime() { + let source = r#"public main() (length: Int, valid: Bool) { + key = dever.crypto.token(32) + hash = dever.crypto.password_hash(key) + valid = dever.crypto.password_verify(key, hash) + bytes = dever.system.crypto_sha256(dever.bytes.from_text("abc")) + length = dever.bytes.length(bytes) +}"#; + assert_eq!(support::stdout(source), "length = 32\nvalid = true\n"); + #[cfg(feature = "reference")] + { + let sources = support::sources(source); + let program = support::checked(&sources); + let output = dever_core::reference::evaluate(&program, "main.main").unwrap(); + assert!(output.fault.is_none()); + assert_eq!(output.stdout, "length = 32\nvalid = true\n"); + } + assert_eq!( + support::stdout( + "public main() (length: Int) { length = dever.bytes.length(dever.system.crypto_sha256(dever.bytes.from_text(\"abc\"))) }" + ), + "length = 32\n" + ); +} diff --git a/test/dever-tests/tests/source_architecture.rs b/test/dever-tests/tests/source_architecture.rs new file mode 100644 index 0000000..a125d7c --- /dev/null +++ b/test/dever-tests/tests/source_architecture.rs @@ -0,0 +1,350 @@ +#[path = "support/temp.rs"] +mod temp; + +use std::fs; + +use dever_core::diagnostic::Diagnostic; +use dever_core::source::SourceMap; +use temp::TemporaryDirectory; + +fn sources(files: &[(&str, &str)]) -> SourceMap { + let mut sources = SourceMap::default(); + for (path, text) in files { + sources.add(*path, *text); + } + sources +} + +fn checked(files: &[(&str, &str)]) -> dever_core::hir::Program { + let sources = sources(files); + dever_core::check(&sources).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<Vec<_>>() + .join("\n") + ) + }) +} + +fn rejected(files: &[(&str, &str)], code: &str, message: &str) -> Vec<Diagnostic> { + let sources = sources(files); + let errors = dever_core::check(&sources).expect_err("source must be rejected"); + assert!( + errors + .iter() + .any(|error| error.code == code && error.message.contains(message)), + "expected {code}: {message}; got {errors:?}" + ); + errors +} + +#[test] +fn app_role_exports_domain_capabilities_without_exposing_topic_files() { + let program = checked(&[ + ( + "main.dever", + "main() (response: user.account.Greeting) { response = user.account.greet() }", + ), + ( + "user/account/app/commands.dever", + "type Greeting { message: Text }\ngreet() (response: Greeting) { response = Greeting { message = int.to_text(count()) } }", + ), + ( + "user/account/app/queries.dever", + "count() (response: Int) { response = 1 }", + ), + ]); + let snapshot = program.api_snapshot(); + assert!(snapshot.contains("function user.account.greet")); + assert!(snapshot.contains("function user.account.count")); + assert!(snapshot.contains("type user.account.Greeting")); + assert!(!snapshot.contains("commands")); + assert!(!snapshot.contains("queries")); + let warnings = program.application_warnings("main.main").unwrap(); + assert!(warnings.iter().any(|warning| { + warning.code == "W007" && warning.message.contains("user.account.count") + })); +} + +#[test] +fn source_roles_replace_public_and_enforce_cohesive_layouts() { + rejected( + &[( + "user/account/app.dever", + "public greet() (response: Text) { response = \"hello\" }", + )], + "P001", + "source role", + ); + rejected( + &[( + "user/account/app/commands.dever", + "greet() (response: Text) { response = \"hello\" }", + )], + "C003", + "at least two cohesive topic files", + ); + rejected( + &[ + ( + "user/account/app.dever", + "greet() (response: Text) { response = \"hello\" }", + ), + ( + "user/account/app/queries.dever", + "count() (response: Int) { response = 1 }", + ), + ], + "C003", + "mutually exclusive", + ); + rejected( + &[( + "user/common/app.dever", + "greet() (response: Text) { response = \"hello\" }", + )], + "C003", + "generic source bucket", + ); +} + +#[test] +fn model_root_and_topic_keep_independent_persisted_identities() { + let root = ("user/account/model.dever", "type Account { name: Text }"); + let topic = ( + "user/account/model/session.dever", + "type Session { token: Text }", + ); + let root_schema = checked(&[root]).model_snapshot(); + let topic_schema = checked(&[topic]).model_snapshot(); + let combined_schema = checked(&[root, topic]).model_snapshot(); + + assert!(root_schema.contains("model user.account.model name=Account table=account ")); + assert!(topic_schema.contains("model user.account.model.session name=Session table=session ")); + assert_eq!( + combined_schema, + format!( + "{root_schema}{}", + topic_schema.strip_prefix("dever-model 1\n").unwrap() + ) + ); +} + +#[test] +fn api_paths_allow_one_topic_with_or_without_a_root_file() { + // Empty declarations isolate layout checks from API declaration syntax. + let root = ("user/account/api.dever", ""); + let topic = ("user/account/api/session/history.dever", ""); + checked(&[topic]); + checked(&[root, topic]); +} + +#[test] +fn business_role_topics_still_require_cohesive_exclusive_directories() { + for role in ["app", "domain", "port", "adapter", "job"] { + let root = format!("user/account/{role}.dever"); + let topic = format!("user/account/{role}/registration.dever"); + rejected(&[(&topic, "")], "C003", "at least two cohesive topic files"); + let errors = rejected(&[(&root, ""), (&topic, "")], "C003", "mutually exclusive"); + let conflict = errors + .iter() + .find(|error| error.message.contains("mutually exclusive")) + .unwrap(); + assert_eq!(conflict.related.len(), 1); + assert_eq!(conflict.related[0].message, "role file declared here"); + } +} + +#[test] +fn model_and_api_exceptions_preserve_topic_name_and_depth_checks() { + rejected( + &[("user/account/model/session/history.dever", "")], + "C003", + "topic directories are flat", + ); + for role in ["model", "api"] { + let topic = format!("user/account/{role}/helpers.dever"); + rejected(&[(&topic, "")], "C003", "generic source bucket"); + } +} + +#[test] +fn strict_project_loading_rejects_the_removed_service_layout() { + checked(&[ + ("main.dever", "main() () {}"), + ( + "user/service/greeting.dever", + "greet() (response: Text) { response = \"hello\" }", + ), + ]); + let root = TemporaryDirectory::new(); + fs::create_dir_all(root.path().join("user/service")).unwrap(); + fs::write(root.path().join("main.dever"), "main() () {}\n").unwrap(); + fs::write( + root.path().join("user/service/greeting.dever"), + "greet() (response: Text) { response = \"hello\" }\n", + ) + .unwrap(); + let sources = SourceMap::load(root.path()).unwrap(); + let errors = dever_core::check(&sources).unwrap_err(); + assert!(errors.iter().any(|error| { + error.code == "C003" && error.message.contains("application source must be") + })); +} + +#[test] +fn only_app_crosses_domain_boundaries() { + checked(&[ + ( + "user/account/domain.dever", + "normalize(value: Text) (response: Text) { response = text.trim(value) }", + ), + ( + "user/account/app.dever", + "normalize(value: Text) (response: Text) { response = domain.normalize(value) }", + ), + ( + "news/article/app.dever", + "author(value: Text) (response: Text) { response = user.account.normalize(value) }", + ), + ( + "user/profile/app.dever", + "label(value: Text) (response: Text) { response = account.normalize(value) }", + ), + ( + "main.dever", + "main() (response: Text) { response = news.article.author(\" editor \" ) }", + ), + ]); + rejected( + &[ + ( + "user/account/domain.dever", + "normalize(value: Text) (response: Text) { response = text.trim(value) }", + ), + ( + "news/article/app.dever", + "steal(value: Text) (response: Text) { response = user.account.domain.normalize(value) }", + ), + ], + "C006", + "not callable from this source role", + ); +} + +#[test] +fn app_failures_must_use_app_owned_error_types() { + rejected( + &[ + ( + "user/account/domain.dever", + "type RegistrationError { error InvalidName }\nreject() () { fail(RegistrationError.InvalidName) }", + ), + ( + "user/account/app.dever", + "register() () { domain.reject() }", + ), + ], + "C006", + "cannot expose private error type", + ); + checked(&[ + ( + "user/account/app.dever", + "type RegistrationError { error InvalidName }\nregister() () { domain.reject() }", + ), + ( + "user/account/domain.dever", + "reject() () { fail(app.RegistrationError.InvalidName) }", + ), + ]); +} + +#[test] +fn model_operations_belong_to_the_owning_domain_app() { + checked(&[ + ("user/account/model.dever", "type Account { name: Text }"), + ( + "user/account/app.dever", + "count() (total: Int) { total = model.count() }", + ), + ( + "main.dever", + "main() (total: Int) { total = user.account.count() }", + ), + ]); + rejected( + &[ + ("user/account/model.dever", "type Account { name: Text }"), + ( + "main.dever", + "main() (total: Int) { total = user.account.model.count() }", + ), + ], + "C006", + "Model operations are private", + ); +} + +#[test] +fn api_is_an_http_entry_and_not_a_source_capability() { + let sources = sources(&[ + ( + "user/account/app.dever", + "greet(name: Text) (response: Text) { response = \"hello, \" + name }", + ), + ("user/account/api.dever", "get greeting = app.greet"), + ]); + let program = dever_core::check(&sources).unwrap(); + let generated = dever_core::native::emit_project( + &program, + &sources, + dever_runtime::config::RuntimeProfile::default(), + ) + .unwrap(); + assert!(generated.contains("(\"GET\", \"/user/account/greeting\")")); + + rejected( + &[ + ("user/account/api.dever", "get greeting = app.greet"), + ( + "user/account/app.dever", + "greet() (response: Text) { response = \"hello\" }", + ), + ( + "news/article/app.dever", + "steal() (response: Text) { response = user.account.api.greeting() }", + ), + ], + "C004", + "unknown", + ); +} + +#[test] +fn main_contains_only_the_boot_function_and_port_rejects_business_bodies() { + let root = TemporaryDirectory::new(); + fs::write( + root.path().join("main.dever"), + "main() () {}\nhelper() () {}\n", + ) + .unwrap(); + let sources = SourceMap::load(root.path()).unwrap(); + let errors = dever_core::check(&sources).unwrap_err(); + assert!( + errors.iter().any(|error| { + error.code == "C005" && error.message.contains("only the ordinary main") + }) + ); + rejected( + &[( + "user/account/port.dever", + "load() (response: Text) { response = \"value\" }", + )], + "C005", + "bodyless", + ); +} diff --git a/test/dever-tests/tests/source_visibility.rs b/test/dever-tests/tests/source_visibility.rs new file mode 100644 index 0000000..f6b68ac --- /dev/null +++ b/test/dever-tests/tests/source_visibility.rs @@ -0,0 +1,277 @@ +use dever_core::diagnostic::Diagnostic; +use dever_core::source::SourceMap; +use dever_core::syntax::Declaration; + +fn sources(files: &[(&str, &str)]) -> SourceMap { + let mut sources = SourceMap::default(); + for (path, text) in files { + sources.add(*path, *text); + } + sources +} + +fn checked(files: &[(&str, &str)]) -> dever_core::hir::Program { + let sources = sources(files); + dever_core::check(&sources).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<Vec<_>>() + .join("\n") + ) + }) +} + +fn rejected(files: &[(&str, &str)], code: &str, message: &str) -> Vec<Diagnostic> { + let sources = sources(files); + let errors = dever_core::check(&sources).expect_err("source must be rejected"); + assert!( + errors + .iter() + .any(|error| error.code == code && error.message.contains(message)), + "expected {code}: {message}; got {errors:?}" + ); + errors +} + +fn markdown(package: &str, code: &str) -> String { + format!( + "# 问候\n\n提供问候文本。\n\n- 包：`{package}`\n- 公开类型：无\n- 公开方法：\n - `greet`\n- 使用：\n - `{package}.greet()`\n\n## 问候方法\n\n返回问候。\n\n- 函数：`greet`\n- 输入：无\n- 输出：\n - `response: Text`：问候文本。\n\n```dever\n{code}\n```\n" + ) +} + +#[test] +fn source_paths_define_packages_in_both_formats() { + let document = markdown( + "user.service.greeting", + "public greet() (response: Text) { response = \"你好\" }", + ); + for (path, text) in [ + ( + "user/service/greeting.dever", + "public greet() (response: Text) { response = \"你好\" }", + ), + ("user/service/greeting.dever.md", document.as_str()), + ] { + let sources = sources(&[(path, text)]); + let package = dever_core::parse(&sources.files()[0]).unwrap(); + assert_eq!( + package + .name + .iter() + .map(|name| name.text.as_str()) + .collect::<Vec<_>>(), + ["user", "service", "greeting"] + ); + checked(&[(path, text)]); + } + let sources = sources(&[("main.dever", "main() () {}")]); + assert_eq!( + dever_core::parse(&sources.files()[0]).unwrap().name[0].text, + "main" + ); +} + +#[test] +fn private_helpers_are_local_and_public_declarations_are_cross_package() { + let service = "public type Greeting { message: Text }\npublic greet() (response: Greeting) { response = Greeting { message = helper() } }\nhelper() (response: Text) { response = \"hello\" }\n"; + let caller = "main() (response: user.service.greeting.Greeting) { response = user.service.greeting.greet() }"; + let program = checked(&[ + ("user/service/greeting.dever", service), + ("main.dever", caller), + ]); + let snapshot = program.api_snapshot(); + assert!(snapshot.contains("function user.service.greeting.greet")); + assert!(snapshot.contains("type user.service.greeting.Greeting")); + assert!(!snapshot.contains("helper")); + assert!(!snapshot.contains("function main.main")); + rejected( + &[ + ("user/service/greeting.dever", service), + ( + "main.dever", + "main() (response: Text) { response = user.service.greeting.helper() }", + ), + ], + "C006", + "private", + ); + rejected( + &[ + ("secret.dever", "type Hidden { value: Int }"), + ("main.dever", "main(value: secret.Hidden) () {}"), + ], + "C006", + "private", + ); +} + +#[test] +fn public_visibility_must_match_every_clause_of_one_signature() { + rejected( + &[( + "sample.dever", + "public choose(value: true) (response: Int) { response = 1 }\nchoose(value: false) (response: Int) { response = 0 }", + )], + "C006", + "same public visibility", + ); + checked(&[( + "sample.dever", + "public choose(value: true) (response: Int) { response = 1 }\npublic choose(value: false) (response: Int) { response = 0 }", + )]); +} + +#[test] +fn public_boundaries_cannot_publish_private_types() { + for declaration in [ + "public identity(value: Hidden) (response: Hidden) { response = value }", + "public identity(value: List<Hidden>) (response: List<Hidden>) { response = value }", + "public apply(action: handler(value: Hidden) (response: Hidden), value: Hidden) (response: Hidden) { response = action(value) }", + "public type Published { value: Hidden }", + ] { + let source = format!("type Hidden {{ value: Int }}\n{declaration}"); + rejected(&[("sample.dever", &source)], "C006", "private type"); + } + let program = checked(&[( + "sample.dever", + "type Hidden { value: Int }\npublic type Published { private hidden: Hidden }", + )]); + assert!(!program.api_snapshot().contains("sample.Hidden")); +} + +#[test] +fn model_records_require_public_but_sql_result_records_remain_private() { + rejected( + &[("news/model/article.dever", "type Article { title: Text }")], + "C014", + "declared public", + ); + checked(&[( + "news/model/article.dever", + "public type Article { title: Text }", + )]); + rejected( + &[( + "news/model/article.dever", + "public type Article { title: Text }\npublic type Summary { count: Int }", + )], + "C014", + "SQL result record must be private", + ); +} + +#[test] +fn legacy_headers_and_non_declaration_public_modifiers_are_rejected() { + for text in [ + "package app exposes (main)\nmain() () {}", + "exposes(main)\nmain() () {}", + ] { + rejected( + &[("app.dever", text)], + "P001", + "package/exposes declarations were removed", + ); + } + rejected( + &[( + "news/model/article.dever", + "public database default\npublic type Article { title: Text }", + )], + "P001", + "only on types and functions", + ); +} + +#[test] +fn source_roots_and_standard_namespaces_cannot_be_spoofed() { + for path in ["../outside.dever", "/outside.dever", "source.txt"] { + rejected(&[(path, "main() () {}")], "C003", "relative .dever"); + } + rejected( + &[("dever/spoof.dever", "main() () {}")], + "C003", + "reserved standard", + ); + rejected( + &[("WrongName.dever", "main() () {}")], + "C010", + "lower_snake_case", + ); + let document = markdown( + "sample", + "public greet() (response: Text) { response = \"hi\" }", + ); + rejected( + &[("sample.dever", ""), ("sample.dever.md", &document)], + "C002", + "duplicate package", + ); +} + +#[test] +fn formatter_preserves_public_transaction_and_comment_positions() { + let code = "# 公共接口\npublic type Result{private value:Int}\npublic transaction write(value:Int)(response:Int){response=value}\nhelper()(){ }\n"; + let sources = sources(&[("sample.dever", code)]); + let formatted = dever_core::format::format(&sources.files()[0]).unwrap(); + assert!(formatted.starts_with("# 公共接口\npublic type Result")); + assert!(formatted.contains("public transaction write(value: Int) (response: Int)")); + assert!(!formatted.contains("package ")); + let mut second = SourceMap::default(); + second.add("sample.dever", &formatted); + assert_eq!( + dever_core::format::format(&second.files()[0]).unwrap(), + formatted + ); + let package = dever_core::parse(&second.files()[0]).unwrap(); + assert!(matches!(&package.declarations[0], Declaration::Type(ty) if ty.public)); + assert!(matches!(&package.declarations[1], Declaration::Function(function) if function.public)); + assert!( + matches!(&package.declarations[2], Declaration::Function(function) if !function.public) + ); +} + +#[test] +fn markdown_public_contract_and_source_locations_follow_headerless_code() { + let document = markdown("sample", "public greet()(response:Text){response=\"你好\"}"); + checked(&[("sample.dever.md", &document)]); + let sources = sources(&[("sample.dever.md", &document)]); + let formatted = dever_core::format::format(&sources.files()[0]).unwrap(); + let prefix = document.split("```dever").next().unwrap(); + assert!(formatted.starts_with(prefix)); + checked(&[("sample.dever.md", &formatted)]); + let private = document.replace("public greet", "greet"); + rejected(&[("sample.dever.md", &private)], "M006", "public methods"); + let invalid = document.replace("\"你好\"", "unknown"); + let errors = rejected(&[("sample.dever.md", &invalid)], "C004", "unknown"); + assert!( + errors + .iter() + .any(|error| &invalid[error.primary.start..error.primary.end] == "unknown") + ); + let h1_code = document.replace( + "## 问候方法", + "```dever\n# 不允许在 H1 下写代码\n```\n\n## 问候方法", + ); + rejected(&[("sample.dever.md", &h1_code)], "M004", "0 top-level"); +} + +#[test] +fn markdown_cannot_split_a_public_declaration_across_blocks() { + let document = markdown( + "sample", + "public\n```\n\n```dever\ngreet() (response: Text) { response = \"hi\" }", + ); + rejected(&[("sample.dever.md", &document)], "P001", "identifier"); + let document = markdown( + "sample", + "public greet() (response: Text) {\n```\n\n```dever\nresponse = \"hi\"\n}", + ); + rejected( + &[("sample.dever.md", &document)], + "M003", + "complete within one", + ); +} diff --git a/test/dever-tests/tests/sqlite_orm.rs b/test/dever-tests/tests/sqlite_orm.rs new file mode 100644 index 0000000..cc1cc6f --- /dev/null +++ b/test/dever-tests/tests/sqlite_orm.rs @@ -0,0 +1,175 @@ +#[path = "support/orm_fixture.rs"] +mod orm_fixture; +mod support; + +use std::fs; + +use dever_runtime::config::RuntimeProfile; +use dever_runtime::sqlite::Database; +use orm_fixture::DatabaseState; + +#[test] +fn shared_orm_fixture_remains_valid_without_opening_a_database() { + let sources = orm_fixture::sources(); + let program = dever_core::check(&sources).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }); + assert!( + program + .model_snapshot() + .contains("model app.user.model name=User") + ); +} + +async fn database_state(database: &Database) -> DatabaseState { + let columns = database + .query( + "SELECT name FROM pragma_table_info('user') ORDER BY cid", + vec![], + ) + .await + .unwrap(); + let managed_index_count = database + .query( + "SELECT COUNT(*) FROM sqlite_schema WHERE type = 'index' AND tbl_name = 'user' AND name LIKE '_dever_idx_%'", + vec![], + ) + .await + .unwrap(); + let history = database + .query( + "SELECT kind, revision FROM _dever_history WHERE model = 'app.user.model' AND kind IN ('schema', 'seed') ORDER BY kind", + vec![], + ) + .await + .unwrap(); + let users = database + .query( + "SELECT email, display_name, status, age, alias FROM user ORDER BY email", + vec![], + ) + .await + .unwrap(); + orm_fixture::database_state(columns, managed_index_count, history, users) +} + +#[tokio::test] +async fn sqlite_executes_the_shared_orm_fixture() { + let profile = RuntimeProfile { + sqlite: true, + postgres: false, + }; + let main = orm_fixture::compile("app.user.main", profile); + let rollback = orm_fixture::compile("app.user.rollback", profile); + assert_eq!(main.revisions, rollback.revisions); + + let root = main.native.executable().parent().unwrap(); + let setting = r#"{ + "database": { + "default": { + "type": "sqlite", + "path": "data/db/app.db", + "max_connections": 1, + "max_page_size": 10 + } + } +}"#; + orm_fixture::configure(main.native.executable(), setting); + orm_fixture::assert_main_output(orm_fixture::run(main.native.executable())); + + let database = Database::open(root.join("data/db/app.db"), 1, 10).unwrap(); + let committed = database_state(&database).await; + orm_fixture::assert_committed_state(&committed, &main.revisions); + drop(database); + + let rollback_executable = root.join("rollback"); + fs::copy(rollback.native.executable(), &rollback_executable).unwrap(); + let output = orm_fixture::run(&rollback_executable); + assert!(!output.status.success()); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!(stderr.contains("UNIQUE constraint failed"), "{stderr}"); + + let database = Database::open(root.join("data/db/app.db"), 1, 10).unwrap(); + assert_eq!(database_state(&database).await, committed); +} + +#[test] +fn database_failures_are_captured_after_transaction_rollback_and_task_wait() { + let mut sources = dever_core::source::SourceMap::default(); + sources.add( + "app/item/model.dever", + r#"type Item { name: Text unique } +"#, + ); + sources.add( + "app/item/app.dever", + r#"type ReadResult { + Found(value: app.item.model.Item) + error Failed(error: dever.database.Error) +} +transaction missing() (value: app.item.model.Item) { + saved = model.create({ name = "temporary" }) + removed = model.delete(saved.id) + value = model.get(saved.id) +} +transaction create_duplicate() (value: app.item.model.Item) { + saved = model.create({ name = "duplicate" }) + value = model.create({ name = "duplicate" }) +} +task_missing() (value: app.item.model.Item) { + task = run(missing()) + value = wait(task) +} +main() (missing: ReadResult, duplicate: ReadResult, count: Int) { + missing = result(task_missing()) + duplicate = result(create_duplicate()) + count = model.count() +} +"#, + ); + let program = support::checked(&sources); + let snapshot = program.api_snapshot(); + assert!( + snapshot.contains("dever.database.Error.NotFound"), + "{snapshot}" + ); + let native = dever_core::native::compile_application( + &program, + &sources, + "app.item.main", + &std::ffi::OsString::from("rustc"), + RuntimeProfile { + sqlite: true, + postgres: false, + }, + ) + .unwrap(); + orm_fixture::configure( + native.executable(), + r#"{ + "database": { "default": { "type": "sqlite", "path": "data/db/errors.db", "max_connections": 1 } } + }"#, + ); + let result = orm_fixture::run(native.executable()); + assert!( + result.status.success(), + "{}", + String::from_utf8_lossy(&result.stderr) + ); + let output = String::from_utf8(result.stdout).unwrap(); + assert!( + output.contains("app.item.ReadResult.Failed(dever.database.Error.NotFound("), + "{output}" + ); + assert!( + output.contains("app.item.ReadResult.Failed(dever.database.Error.Constraint("), + "{output}" + ); + assert!(output.ends_with("count = 0\n"), "{output}"); +} diff --git a/test/dever-tests/tests/sqlite_runtime.rs b/test/dever-tests/tests/sqlite_runtime.rs new file mode 100644 index 0000000..0703466 --- /dev/null +++ b/test/dever-tests/tests/sqlite_runtime.rs @@ -0,0 +1,314 @@ +mod support; + +use dever_runtime::orm::{ErrorKind, Uuid, Value, decimal, decimal_storage, uuid}; +use dever_runtime::sqlite::Database; + +#[test] +fn sqlite_invalid_text_preserves_the_column_and_error_kind_in_all_decoders() { + dever_runtime::task::run_entry(async { + let root = support::temp::TemporaryDirectory::new(); + let database = Database::open(root.path().join("invalid-text.db"), 1, 4).unwrap(); + let check = |error: dever_runtime::orm::Error| { + assert_eq!(error.kind(), ErrorKind::Database); + assert!(error.to_string().contains("index 1")); + }; + const INVALID_TEXT: &str = "SELECT 7, CAST(x'ff' AS TEXT)"; + check(database.query(INVALID_TEXT, vec![]).await.unwrap_err()); + check( + database + .query_bounded(INVALID_TEXT, vec![], 1) + .await + .unwrap_err(), + ); + check( + database + .query_owned(INVALID_TEXT.into(), vec![]) + .await + .unwrap_err(), + ); + let stream = database.stream(INVALID_TEXT, vec![], 1).await.unwrap(); + check(stream.pull().await.unwrap_err()); + stream.close(); + let rows = database.query("SELECT 1", vec![]).await.unwrap(); + assert_eq!(rows[0].get(0).unwrap(), &Value::Int(1)); + Ok(()) + }) + .unwrap(); +} + +#[tokio::test] +async fn sqlite_pool_executes_parameterized_queries_and_classifies_constraints() { + let root = support::temp::TemporaryDirectory::new(); + let database = Database::open(root.path().join("app.db"), 1, 50).unwrap(); + assert_eq!(database.max_page_size(), 50); + + database + .execute_batch( + "CREATE TABLE user (\ + id INTEGER PRIMARY KEY AUTOINCREMENT, \ + email TEXT NOT NULL UNIQUE, \ + enabled INTEGER NOT NULL\ + )" + .to_owned(), + ) + .await + .unwrap(); + database + .execute( + "INSERT INTO user (email, enabled) VALUES (?1, ?2)", + vec![Value::Text("admin@example.com".into()), Value::Bool(true)], + ) + .await + .unwrap(); + + let rows = database + .query( + "SELECT id, email, enabled FROM user WHERE email = ?1", + vec![Value::Text("admin@example.com".into())], + ) + .await + .unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].get(0).unwrap(), &Value::Int(1)); + assert_eq!( + rows[0].get(1).unwrap(), + &Value::Text("admin@example.com".into()) + ); + assert_eq!(rows[0].get(2).unwrap(), &Value::Int(1)); + + let duplicate = database + .execute( + "INSERT INTO user (email, enabled) VALUES (?1, ?2)", + vec![Value::Text("admin@example.com".into()), Value::Bool(false)], + ) + .await + .unwrap_err(); + assert_eq!(duplicate.kind(), ErrorKind::Constraint); + + let expected_uuid = Uuid::parse("018f08d8-59b0-7cc2-98c4-dc0c0c07398f").unwrap(); + database + .execute_batch( + "CREATE TABLE uuid_value (value BLOB NOT NULL CHECK (length(value) = 16))".to_owned(), + ) + .await + .unwrap(); + database + .execute( + "INSERT INTO uuid_value (value) VALUES (?1)", + vec![Value::Uuid(expected_uuid)], + ) + .await + .unwrap(); + let mut uuid_rows = database + .query("SELECT value FROM uuid_value", vec![]) + .await + .unwrap(); + let stored_uuid = uuid(uuid_rows.remove(0).into_values().next().unwrap()).unwrap(); + assert_eq!(stored_uuid, expected_uuid); +} + +#[tokio::test] +async fn sqlite_transactions_commit_or_rollback_on_the_owned_connection() { + let root = support::temp::TemporaryDirectory::new(); + let database = Database::open(root.path().join("transaction.db"), 1, 50).unwrap(); + database + .execute_batch("CREATE TABLE event (id INTEGER PRIMARY KEY, name TEXT NOT NULL)".to_owned()) + .await + .unwrap(); + + let transaction = database.begin().await.unwrap(); + transaction + .execute( + "INSERT INTO event (id, name) VALUES (?1, ?2)", + vec![Value::Int(1), Value::Text("rolled back".into())], + ) + .await + .unwrap(); + transaction.rollback().await.unwrap(); + assert!( + database + .query("SELECT id FROM event", vec![]) + .await + .unwrap() + .is_empty() + ); + + let transaction = database.begin().await.unwrap(); + transaction + .execute( + "INSERT INTO event (id, name) VALUES (?1, ?2)", + vec![Value::Int(2), Value::Text("committed".into())], + ) + .await + .unwrap(); + transaction.commit().await.unwrap(); + let rows = database + .query("SELECT id, name FROM event", vec![]) + .await + .unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].get(0).unwrap(), &Value::Int(2)); + assert_eq!(rows[0].get(1).unwrap(), &Value::Text("committed".into())); + + let abandoned = database.begin().await.unwrap(); + abandoned + .execute( + "INSERT INTO event (id, name) VALUES (?1, ?2)", + vec![Value::Int(3), Value::Text("abandoned".into())], + ) + .await + .unwrap(); + drop(abandoned); + let rows = database + .query("SELECT id FROM event WHERE id = ?1", vec![Value::Int(3)]) + .await + .unwrap(); + assert!(rows.is_empty()); +} + +#[test] +fn decimal_and_uuid_values_have_stable_storage_round_trips() { + let decimal_values = ["-10", "-2", "-0.01", "0", "0.01", "2", "10"]; + let encoded = decimal_values + .iter() + .map(|value| decimal_storage(value, 8, 2).unwrap()) + .collect::<Vec<_>>(); + let mut sorted = encoded.clone(); + sorted.sort(); + assert_eq!(sorted, encoded); + for (expected, encoded) in decimal_values.into_iter().zip(encoded) { + assert_eq!( + decimal(Value::Decimal(encoded)).unwrap(), + decimal(Value::Text(expected.to_owned())).unwrap() + ); + } + assert!(decimal_storage("1.234", 8, 2).is_err()); + assert!(decimal_storage("1000", 3, 0).is_err()); + assert!(decimal_storage("1e2147483647", 8, 2).is_err()); + assert_eq!( + decimal_storage("0e2147483647", 8, 2).unwrap(), + "D100000000:02" + ); + + let text = "018f08d8-59b0-7cc2-98c4-dc0c0c07398f"; + let value = Uuid::parse(text).unwrap(); + assert_eq!(value.to_string(), text); + assert_eq!(uuid(Value::Uuid(value)).unwrap(), value); + assert!(Uuid::parse("not-a-uuid").is_err()); + + let generated = Uuid::new_v7().unwrap(); + let generated_bytes = generated.into_bytes(); + assert_eq!(generated_bytes[6] >> 4, 7); + assert_eq!(generated_bytes[8] & 0xc0, 0x80); + assert_eq!(Uuid::parse(&generated.to_string()).unwrap(), generated); +} + +#[tokio::test] +async fn cancelled_sqlite_work_does_not_return_an_unknown_connection_to_the_pool() { + let root = support::temp::TemporaryDirectory::new(); + let database = + std::sync::Arc::new(Database::open(root.path().join("cancel.db"), 1, 50).unwrap()); + let query_database = database.clone(); + let query = tokio::spawn(async move { + query_database + .query( + "WITH RECURSIVE values_(value) AS (\ + VALUES(0) UNION ALL SELECT value + 1 FROM values_ WHERE value < 100000000\ + ) SELECT sum(value) FROM values_", + vec![], + ) + .await + }); + tokio::task::yield_now().await; + query.abort(); + let _ = query.await; + + let rows = tokio::time::timeout( + std::time::Duration::from_secs(2), + database.query("SELECT 1", vec![]), + ) + .await + .expect("the pool must replace the cancelled connection") + .unwrap(); + assert_eq!(rows[0].get(0).unwrap(), &Value::Int(1)); +} + +#[test] +fn sqlite_row_stream_is_bounded_and_releases_its_connection_when_closed_early_or_dropped() { + dever_runtime::task::run_entry(async { + let root = support::temp::TemporaryDirectory::new(); + let database = Database::open(root.path().join("stream.db"), 1, 4).unwrap(); + database + .execute_batch( + "CREATE TABLE event (id INTEGER PRIMARY KEY, name TEXT NOT NULL);\ + INSERT INTO event (id, name) VALUES \ + (1, 'one'), (2, 'two'), (3, 'three'), (4, 'four'), (5, 'five')" + .to_owned(), + ) + .await + .unwrap(); + + let stream = database + .stream("SELECT id, name FROM event ORDER BY id", vec![], 2) + .await + .unwrap(); + let first = stream.pull().await.unwrap().unwrap(); + assert_eq!(first.get(0).unwrap(), &Value::Int(1)); + + let waiting_for_connection = tokio::time::timeout( + std::time::Duration::from_millis(50), + database.query("SELECT count(*) FROM event", vec![]), + ) + .await; + assert!( + waiting_for_connection.is_err(), + "the stream must retain the only connection until it is closed" + ); + stream.close(); + + let rows = tokio::time::timeout( + std::time::Duration::from_secs(2), + database.query("SELECT count(*) FROM event", vec![]), + ) + .await + .expect("closing the row stream must return its only connection") + .unwrap(); + assert_eq!(rows[0].get(0).unwrap(), &Value::Int(5)); + + let stream = database + .stream("SELECT id FROM event ORDER BY id", vec![], 1) + .await + .unwrap(); + let mut ids = Vec::new(); + while let Some(row) = stream.pull().await.unwrap() { + ids.push(row.get(0).unwrap().clone()); + } + assert_eq!( + ids, + vec![ + Value::Int(1), + Value::Int(2), + Value::Int(3), + Value::Int(4), + Value::Int(5), + ] + ); + + let stream = database + .stream("SELECT id FROM event ORDER BY id", vec![], 1) + .await + .unwrap(); + assert!(stream.pull().await.unwrap().is_some()); + drop(stream); + let rows = tokio::time::timeout( + std::time::Duration::from_secs(2), + database.query("SELECT count(*) FROM event", vec![]), + ) + .await + .expect("dropping the row stream must return its only connection") + .unwrap(); + assert_eq!(rows[0].get(0).unwrap(), &Value::Int(5)); + Ok(()) + }) + .unwrap(); +} diff --git a/test/dever-tests/tests/standard_library.rs b/test/dever-tests/tests/standard_library.rs new file mode 100644 index 0000000..aa2d30f --- /dev/null +++ b/test/dever-tests/tests/standard_library.rs @@ -0,0 +1,124 @@ +mod support; + +use dever_runtime::{number, text, time}; +use std::process::Command; + +#[test] +fn unicode_text_uses_scalar_positions_and_explicit_invalid_ranges() { + let value = "a中😀e\u{301}"; + assert_eq!(text::codepoint("😀"), Some(0x1f600)); + assert_eq!(text::codepoint("ab"), None); + assert_eq!(text::from_codepoint(0x1f600).as_deref(), Some("😀")); + assert_eq!(text::from_codepoint(0xd800), None); + assert_eq!(text::from_codepoint(i64::MAX), None); + assert_eq!(text::at(value, 2).as_deref(), Some("😀")); + assert_eq!(text::at(value, 5), None); + assert_eq!(text::at(value, -1), None); + assert_eq!(text::slice(value, 1, 3).as_deref(), Some("中😀")); + assert_eq!(text::slice(value, 5, 5).as_deref(), Some("")); + assert_eq!(text::slice(value, 3, 2), None); + assert_eq!(text::slice(value, 5, 6), None); + assert_eq!(text::index_of(value, "e"), Some(3)); + assert_eq!(text::index_of(value, "x"), None); + assert_eq!(text::split("a中", "").values(), &["a", "中"]); + assert!(text::split("", "").values().is_empty()); + assert_eq!(text::split(",a,,", ",").values(), &["", "a", "", ""]); +} + +#[test] +fn numeric_parsing_preserves_precision_boundaries_and_ieee_values() { + assert_eq!(number::parse_int("-9223372036854775808"), Some(i64::MIN)); + assert_eq!(number::parse_int("9223372036854775808"), None); + assert_eq!(number::parse_int(" 2"), None); + assert_eq!( + number::parse_decimal("0.10000000000000000000000000000000001"), + None + ); + assert_eq!(number::parse_decimal("1.25e2").unwrap().to_string(), "125"); + assert_eq!(number::parse_decimal("NaN"), None); + assert_eq!(number::parse_decimal(" 1.2"), None); + assert_eq!( + number::parse_float("-0").unwrap().to_bits(), + (-0.0f64).to_bits() + ); + assert!(number::parse_float("NaN").unwrap().is_nan()); + assert_eq!(number::parse_float("-inf"), Some(f64::NEG_INFINITY)); + assert_eq!(number::parse_float("1_000"), None); +} + +#[test] +fn official_text_and_number_sources_execute_as_normal_packages() { + let output = support::stdout( + r#"public main() (joined: Text, empty: Text, character: Text?, slice: Text?, position: Int?, absent: Text?, pieces: List<Text>, replaced: Text, predicates: Bool, number: Int?, invalid: Decimal?, exact: Text, converted: Text, floating: Text) { + joined = text.join(["", "中", ""], "|") + empty = text.join([], "|") + character = text.at("a😀中", 1) + slice = text.slice("a😀中", 1, 3) + position = text.index_of("a😀中", "中") + absent = text.slice("a", 0, 2) + pieces = text.split("a,,b", ",") + replaced = text.replace("中a中", "中", "字") + predicates = text.starts_with("abc", "a") and text.ends_with("abc", "c") and text.contains("abc", "b") and text.is_empty("") + number = int.parse("-17") + invalid = decimal.parse("bad") + exact = decimal.to_text(1.25) + converted = int.to_text(-17) + floating = float.to_text(float.from_int(8)) +} +"#, + ); + assert_eq!( + output, + "joined = |中|\nempty = \ncharacter = 😀\nslice = 😀中\nposition = 2\nabsent = null\npieces = [a, , b]\nreplaced = 字a字\npredicates = true\nnumber = -17\ninvalid = null\nexact = 1.25\nconverted = -17\nfloating = 8\n" + ); +} + +#[test] +fn process_inputs_are_explicit_and_do_not_require_build_tools() { + let sources = support::sources( + r#"public main() (args: dever.process.ArgumentsResult) { + args = result(dever.process.arguments()) +} +"#, + ); + let program = support::checked(&sources); + let compiler = std::env::var_os("RUSTC").unwrap_or_else(|| "rustc".into()); + let native = dever_core::native::compile(&program, &sources, "main.main", &compiler).unwrap(); + let output = Command::new(native.executable()) + .env_clear() + .args(["first", "arg with spaces"]) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let output = String::from_utf8(output.stdout).unwrap(); + assert!( + output.contains("Read([first, arg with spaces])"), + "{output}" + ); +} + +#[test] +fn clock_failures_are_results() { + let before = time::monotonic_nanos().unwrap(); + time::sleep(0).unwrap(); + assert!(time::monotonic_nanos().unwrap() >= before); + assert!(time::unix_millis().unwrap() > 0); + assert!(time::sleep(-1).is_err()); + assert_eq!( + support::stdout( + r#"valid(result: dever.time.ClockResult.Read(value)) (answer: Bool) recover("test observes valid failures in its assertions") { answer = value >= 0 } +valid(result: other) (answer: Bool) recover("test observes valid failures in its assertions") { answer = false } +public main() (wall: Bool, elapsed: Bool, invalid: dever.time.SleepResult) { + wall = valid(result(dever.time.unix_millis())) + elapsed = valid(result(dever.time.monotonic_nanos())) + invalid = result(dever.time.sleep(-1)) +} +"# + ), + "wall = true\nelapsed = true\ninvalid = dever.time.SleepResult.Failed(sleep duration must not be negative)\n" + ); +} diff --git a/test/dever-tests/tests/structured_concurrency.rs b/test/dever-tests/tests/structured_concurrency.rs new file mode 100644 index 0000000..4e1dae6 --- /dev/null +++ b/test/dever-tests/tests/structured_concurrency.rs @@ -0,0 +1,351 @@ +mod support; + +use dever_core::source::SourceMap; +use dever_core::syntax::{Declaration, ExpressionKind, InputKind}; +use support::{checked, rejected, sources}; + +fn formatted(source: &str) -> String { + let sources = sources(source); + dever_core::format::format(&sources.files()[0]).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }) +} + +#[test] +fn calls_and_handlers_need_no_async_syntax() { + let source = + "public main(route: handler(value: Int) (answer: Int)) (answer: Int) { answer = route(1) }"; + let package = dever_core::parse(&sources(source).files()[0]).unwrap(); + let Declaration::Function(function) = &package.declarations[0] else { + panic!("expected function"); + }; + assert!(matches!(function.inputs[0].kind, InputKind::Handler(_))); + let dever_core::syntax::StatementKind::Assign { value, .. } = &function.body[0].kind else { + panic!("expected assignment"); + }; + assert!(matches!(value.kind, ExpressionKind::Call { .. })); + + let once = formatted(source); + assert!(!once.contains("async ")); + assert!(!once.contains("await(")); + assert_eq!(formatted(&once), once); +} + +#[test] +fn run_wait_groups_parallel_blocking_and_channels_check_together() { + let source = r#"pure_double(value: Int) (answer: Int) pure { answer = value * 2 } +pause() (result: dever.time.SleepResult) { result = result(dever.time.sleep(1)) } +value(value: Int) (answer: Int) { answer = value } +work() () {} +public main() (answer: Int, received: Int?, slept: dever.time.SleepResult) { + direct = value(20) + task = run(value(21)) + answer = wait(task) + parallel(pure_double(direct)) + tasks = group(2) + run(tasks, work()) + wait(tasks) + channel = channel(Int, 2) + send(channel, answer) + received = receive(channel) + close(channel) + slept = blocking(pause()) +}"#; + let snapshot = checked(&sources(source)).api_snapshot(); + assert!(snapshot.contains("function main.main")); + assert!(!snapshot.contains("async=")); + assert!( + snapshot.contains("effects=[concurrency, time]"), + "{snapshot}" + ); +} + +#[test] +fn suspension_is_inferred_through_direct_calls() { + let source = r#"sleep_ok(result: dever.time.SleepResult.Done) () recover("timer failure handled") {} +sleep_ok(result: dever.time.SleepResult.Failed(message)) () recover("timer failure handled") {} +work() () { sleep_ok(result(dever.task.sleep(0))) } +public main() () { work() }"#; + let snapshot = checked(&sources(source)).api_snapshot(); + assert!( + snapshot.contains("effects=[concurrency, time]"), + "{snapshot}" + ); + assert!(!snapshot.contains("async=")); + + rejected("async main() () {}", "'(' after a function name"); + rejected( + "work() () {}\nmain() () { await(work()) }", + "unknown function 'await'", + ); +} + +#[test] +fn handler_suspension_reaches_the_call_graph() { + let source = r#"sleep_ok(result: dever.time.SleepResult.Done) () recover("timer failure handled") {} +sleep_ok(result: dever.time.SleepResult.Failed(message)) () recover("timer failure handled") {} +route(value: Int) (answer: Int) { + sleep_ok(result(dever.task.sleep(0))) + answer = value +} +apply(route: handler(value: Int) (answer: Int)) (answer: Int) { answer = route(1) } +public main() (answer: Int) { answer = apply(route) }"#; + let snapshot = checked(&sources(source)).api_snapshot(); + assert!( + snapshot.contains("effects=[concurrency, time]"), + "{snapshot}" + ); + + rejected( + "route(value: Text) (answer: Int) { answer = 1 }\napply(route: handler(value: Int) (answer: Int)) (answer: Int) { answer = route(1) }\nmain() (answer: Int) { answer = apply(route) }", + "handler signature does not match", + ); + rejected("cycle() () { cycle() }", "recursive function"); +} + +#[test] +fn tasks_and_groups_are_affine() { + checked(&sources( + "work() () {}\npublic main() () { task = run(work())\ntasks = group(1)\nwork()\nstop(task)\nstop(tasks) }", + )); + for (body, message) in [ + ("task = run(work())", "must be consumed"), + ( + "task = run(work())\nwait(task)\nwait(task)", + "already been consumed", + ), + ( + "task = run(work())\ntask = run(work())\nstop(task)", + "consumed before it can be overwritten", + ), + ("task = run(work())\nvalues = [task]", "is affine"), + ( + "tasks = [run(work())]\nvalue = first(tasks)\nstop(value)", + "cannot be stored in another value", + ), + ] { + rejected(&format!("work() () {{}}\nmain() () {{ {body} }}"), message); + } + rejected( + "value() (answer: Int) { answer = 1 }\nmain() () { task = run(value())\nstop(task) }", + "only a zero-output Task", + ); + rejected( + "value() (answer: Int) { answer = 1 }\nmain() () { tasks = group(1)\nrun(tasks, value())\nwait(tasks) }", + "only zero-output calls", + ); +} + +#[test] +fn non_transferable_locals_cannot_cross_suspension_points() { + for (suspension, expression) in [ + ("work()", "work()"), + ( + "parallel_each(visit, [1], 1)", + "parallel_each(visit, [1], 1)", + ), + ( + "sleep_ok(result(dever.task.sleep(0)))", + "result(dever.task.sleep(0))", + ), + ] { + let source = format!( + r#"observe(value: Stream<Int>) () {{}} +visit(value: Int) () {{}} +sleep_ok(result: dever.time.SleepResult.Done) () recover("timer failure handled") {{}} +sleep_ok(result: dever.time.SleepResult.Failed(message)) () recover("timer failure handled") {{}} +work() () {{ sleep_ok(result(dever.task.sleep(0))) }} +main(stream: Stream<Int>) () {{ + {suspension} + observe(stream) +}}"# + ); + let source_map = sources(&source); + let errors = dever_core::check(&source_map).expect_err("non-transferable live stream"); + let start = source.rfind(expression).unwrap(); + assert!( + errors.iter().any(|error| { + error.message == "Stream<Int> local cannot remain live across a suspension point" + && error.primary.start == start + && error.primary.end == start + expression.len() + }), + "suspension {suspension}: {errors:?}" + ); + } + checked(&sources( + r#"type Outcome { Done +error Failed } +fail_work() () { fail(Outcome.Failed) } +observe(value: Stream<Int>) () {} +main(stream: Stream<Int>) (outcome: Outcome) { + outcome = result(fail_work()) + observe(stream) +}"#, + )); +} + +#[test] +fn channel_types_capacity_and_operations_are_checked() { + checked(&sources( + "public main() (received: Int?) { channel = channel(Int, 1)\nsend(channel, 7)\nreceived = receive(channel)\nclose(channel)\nclose(channel) }", + )); + checked(&sources( + "main(channel: Channel<Int>) () { close(channel) }", + )); + for (source, message) in [ + ( + "main() () { channel = channel(Stream<Int>, 1)\nclose(channel) }", + "item type must be transferable", + ), + ( + "main() () { channel = channel(Int, 0)\nclose(channel) }", + "channel capacity must be between", + ), + ( + "main() () { channel = channel(Int, 65537)\nclose(channel) }", + "channel capacity must be between", + ), + ( + "main() () { channel = channel(Int, 1)\nsend(channel, \"wrong\")\nclose(channel) }", + "expected Int", + ), + ] { + rejected(source, message); + } +} + +#[test] +fn group_limits_parallel_purity_and_blocking_effects_are_checked() { + for limit in ["0", "65537"] { + rejected( + &format!("main() () {{ tasks = group({limit})\nwait(tasks) }}"), + "group limit must be between", + ); + } + rejected( + "noisy() (answer: Int) { dever.io.println(\"x\")\nanswer = 1 }\nmain() (answer: Int) { answer = parallel(noisy()) }", + "pure function call", + ); + rejected( + r#"visit(value: Int) () {} +pause_all(values: List<Int>) (result: dever.time.SleepResult) { + parallel_each(visit, values, 2) + result = result(dever.time.sleep(0)) +} +main() (result: dever.time.SleepResult) { result = blocking(pause_all([1])) }"#, + "blocking cannot wrap a call that uses parallel_each", + ); + rejected( + "plain() (answer: Int) pure { answer = 1 }\nmain() (answer: Int) { answer = blocking(plain()) }", + "transitive blocking system effect", + ); +} + +#[test] +fn collection_handlers_preserve_suspension_and_worker_boundaries() { + let source = r#"sleep_ok(result: dever.time.SleepResult.Done) () recover("timer failure handled") {} +sleep_ok(result: dever.time.SleepResult.Failed(message)) () recover("timer failure handled") {} +visit(value: Int) () { sleep_ok(result(dever.task.sleep(value))) } +public main() () { each(visit, [1]) }"#; + let snapshot = checked(&sources(source)).api_snapshot(); + assert!( + snapshot.contains("effects=[concurrency, time]"), + "{snapshot}" + ); + + checked(&sources("apply(route: handler() ()) () { route() }")); + checked(&sources( + "apply(route: handler(value: Int) ()) () { each(route, [1]) }", + )); + checked(&sources( + "main() () { dever.io.println(\"blocking output\") }", + )); + for blocking_call in [ + "dever.io.println(\"blocking output\")", + "sleep_ok(result(dever.time.sleep(0)))", + ] { + let source = format!( + r#"sleep_ok(result: dever.time.SleepResult.Done) () recover("timer failure handled") {{}} +sleep_ok(result: dever.time.SleepResult.Failed(message)) () recover("timer failure handled") {{}} +main() () {{ + sleep_ok(result(dever.task.sleep(0))) + {blocking_call} +}}"# + ); + rejected(&source, "cannot run inline inside a suspending function"); + } + checked(&sources( + r#"sleep_ok(result: dever.time.SleepResult.Done) () recover("timer failure handled") {} +sleep_ok(result: dever.time.SleepResult.Failed(message)) () recover("timer failure handled") {} +main() () { + sleep_ok(result(dever.task.sleep(0))) + blocking(dever.io.println("bounded output")) +}"#, + )); +} + +#[test] +fn task_and_channel_boundaries_preserve_failure_obligations() { + let declarations = "type Result { Ready\nerror Failed }\nfail_work() () { fail(Result.Failed) }\nproduce_failure() (result: Result) { result = result(fail_work()) }\n"; + rejected( + &format!( + "{declarations}main() () {{ task = run(produce_failure())\nresult = wait(task) }}" + ), + "failure is not handled", + ); + checked(&sources(&format!( + "{declarations}main() (result: Result) {{ task = run(produce_failure())\nresult = wait(task) }}" + ))); + rejected( + &format!( + "{declarations}main() () {{ channel = channel(Result, 1)\nsend(channel, produce_failure())\nreceived = receive(channel)\nclose(channel) }}" + ), + "failure is not handled", + ); +} + +#[test] +fn api_and_markdown_contracts_hide_internal_suspension() { + let plain = checked(&sources( + "public apply(route: handler(value: Int) (answer: Int), value: Int) (answer: Int) { answer = route(value) }", + )); + let snapshot = plain.api_snapshot(); + assert!(!snapshot.contains("async="), "{snapshot}"); + assert!(!snapshot.contains("async handler("), "{snapshot}"); + + let document = r#"# Application package + +Documents a package. + +- 包：`main` +- 公开类型：无 +- 公开方法： + - `main` +- 使用： + - `main.main()` + +## Main + +Documents the entry. + +- 函数：`main` +- 输入：无 +- 输出：无 + +```dever +public main() () {} +``` +"#; + let mut markdown = SourceMap::default(); + markdown.add("main.dever.md", document); + let program = checked(&markdown); + assert!(!program.api_snapshot().contains("async=")); + let formatted = dever_core::format::format(&markdown.files()[0]).unwrap(); + assert!(formatted.contains("main() () {}")); + assert!(!formatted.contains("async main")); +} diff --git a/test/dever-tests/tests/support/authorization.rs b/test/dever-tests/tests/support/authorization.rs new file mode 100644 index 0000000..6d90e39 --- /dev/null +++ b/test/dever-tests/tests/support/authorization.rs @@ -0,0 +1,338 @@ +use dever_runtime::auth::store::{self, Permission, Role}; +use dever_runtime::{database, orm}; + +static PERMISSIONS: &[Permission] = &[ + Permission { + key: "news.article.admin.publish", + component: "news", + domain: "article", + site: "admin", + action: "publish", + method: "POST", + }, + Permission { + key: "news.article.admin.delete", + component: "news", + domain: "article", + site: "admin", + action: "delete", + method: "DELETE", + }, + Permission { + key: "news.article.front.read", + component: "news", + domain: "article", + site: "front", + action: "read", + method: "GET", + }, +]; + +pub async fn assert_contract(catalog: database::Database, authorization: database::Database) { + store::sync_catalog(catalog.clone(), PERMISSIONS) + .await + .unwrap(); + create_legacy_role_schema(&authorization).await; + store::initialize_roles(authorization.clone()) + .await + .unwrap(); + + assert!( + store::authorize( + authorization.clone(), + 9, + "admin", + "news.article.admin.publish", + ) + .await + .unwrap() + ); + let version = authorization + .query( + database::Sql { + sqlite: "SELECT version FROM _dever_auth_version", + postgres: "SELECT version FROM _dever_auth_version", + }, + vec![], + ) + .await + .unwrap(); + assert_eq!(version[0].get(0).unwrap(), &orm::Value::Int(2)); + + let listed = store::list_permissions(catalog.clone()).await.unwrap(); + assert_eq!(listed.len(), 3); + assert_eq!(listed[0].key, "news.article.admin.delete"); + assert_eq!(listed[0].method, "DELETE"); + assert_eq!(listed[2].key, "news.article.front.read"); + + let publisher = Role { + id: "publisher".into(), + site: "admin".into(), + name: "Publisher".into(), + all_permissions: false, + }; + let deleter = Role { + id: "deleter".into(), + site: "admin".into(), + name: "Deleter".into(), + all_permissions: false, + }; + let reader = Role { + id: "publisher".into(), + site: "front".into(), + name: "Reader".into(), + all_permissions: false, + }; + save_role( + &catalog, + &authorization, + &publisher, + "news.article.admin.publish", + ) + .await; + save_role( + &catalog, + &authorization, + &deleter, + "news.article.admin.delete", + ) + .await; + save_role(&catalog, &authorization, &reader, "news.article.front.read").await; + assert!( + store::grant_role(authorization.clone(), 4, "front", "deleter") + .await + .is_err() + ); + + assert!( + !store::authorize( + authorization.clone(), + 1, + "admin", + "news.article.admin.publish", + ) + .await + .unwrap() + ); + store::grant_role(authorization.clone(), 1, "admin", "publisher") + .await + .unwrap(); + assert!( + !store::authorize(authorization.clone(), 1, "front", "news.article.front.read",) + .await + .unwrap() + ); + store::grant_role(authorization.clone(), 1, "front", "publisher") + .await + .unwrap(); + assert!( + store::authorize( + authorization.clone(), + 1, + "admin", + "news.article.admin.publish", + ) + .await + .unwrap() + ); + assert!( + !store::authorize( + authorization.clone(), + 1, + "admin", + "news.article.admin.delete", + ) + .await + .unwrap() + ); + assert!( + store::authorize(authorization.clone(), 1, "front", "news.article.front.read",) + .await + .unwrap() + ); + + store::grant_role(authorization.clone(), 2, "admin", "publisher") + .await + .unwrap(); + store::grant_role(authorization.clone(), 2, "admin", "deleter") + .await + .unwrap(); + for permission in ["news.article.admin.publish", "news.article.admin.delete"] { + assert!( + store::authorize(authorization.clone(), 2, "admin", permission) + .await + .unwrap() + ); + } + + store::disable_role(authorization.clone(), "admin", "deleter") + .await + .unwrap(); + assert!( + store::authorize( + authorization.clone(), + 2, + "admin", + "news.article.admin.publish", + ) + .await + .unwrap() + ); + assert!( + !store::authorize( + authorization.clone(), + 2, + "admin", + "news.article.admin.delete", + ) + .await + .unwrap() + ); + + let owner_role = store::provision_owner(authorization.clone(), 3, "admin") + .await + .unwrap(); + assert_eq!(owner_role, "_dever_owner:admin"); + let reserved = Role { + id: owner_role.clone(), + site: "admin".into(), + name: "Not Owner".into(), + all_permissions: true, + }; + assert!( + store::save_role(catalog.clone(), authorization.clone(), &reserved, &[]) + .await + .is_err() + ); + assert!( + store::grant_role(authorization.clone(), 4, "admin", &owner_role) + .await + .is_err() + ); + assert!( + store::revoke_role(authorization.clone(), 3, "admin", &owner_role) + .await + .is_err() + ); + assert!( + store::disable_role(authorization.clone(), "admin", &owner_role) + .await + .is_err() + ); + assert!( + store::authorize( + authorization.clone(), + 3, + "admin", + "news.article.admin.publish", + ) + .await + .unwrap() + ); + assert!( + store::authorize( + authorization.clone(), + 3, + "admin", + "news.article.admin.delete", + ) + .await + .unwrap() + ); + assert!( + !store::authorize(authorization.clone(), 3, "front", "news.article.front.read",) + .await + .unwrap() + ); + + store::revoke_role(authorization.clone(), 1, "admin", "publisher") + .await + .unwrap(); + assert!( + !store::authorize( + authorization.clone(), + 1, + "admin", + "news.article.admin.publish", + ) + .await + .unwrap() + ); + assert!( + store::authorize(authorization.clone(), 1, "front", "news.article.front.read",) + .await + .unwrap() + ); + assert!( + store::authorize(catalog.clone(), 1, "admin", "news.article.front.read",) + .await + .is_err() + ); + + store::sync_catalog(catalog.clone(), &[PERMISSIONS[0], PERMISSIONS[2]]) + .await + .unwrap(); + let listed = store::list_permissions(catalog.clone()).await.unwrap(); + assert_eq!(listed.len(), 2); + let inactive = catalog + .query( + database::Sql { + sqlite: "SELECT active FROM _dever_permission WHERE key='news.article.admin.delete'", + postgres: "SELECT active FROM _dever_permission WHERE key='news.article.admin.delete'", + }, + vec![], + ) + .await + .unwrap(); + assert_eq!(inactive[0].get(0).unwrap(), &orm::Value::Int(0)); + assert!( + store::save_role( + catalog, + authorization, + &deleter, + &["news.article.admin.delete".into()], + ) + .await + .is_err() + ); +} + +async fn create_legacy_role_schema(database: &database::Database) { + for sql in [ + "CREATE TABLE _dever_auth_role (id TEXT PRIMARY KEY, site TEXT NOT NULL, name TEXT NOT NULL, all_permissions BIGINT NOT NULL CHECK(all_permissions IN (0,1)), enabled BIGINT NOT NULL CHECK(enabled IN (0,1)))", + "CREATE TABLE _dever_auth_role_permission (role_id TEXT NOT NULL REFERENCES _dever_auth_role(id) ON DELETE CASCADE, permission_key TEXT NOT NULL, PRIMARY KEY(role_id,permission_key))", + "CREATE TABLE _dever_auth_user_role (user_id BIGINT NOT NULL, role_id TEXT NOT NULL REFERENCES _dever_auth_role(id) ON DELETE CASCADE, PRIMARY KEY(user_id,role_id))", + "CREATE INDEX _dever_auth_role_site ON _dever_auth_role(site,enabled)", + "CREATE INDEX _dever_auth_user_role_user ON _dever_auth_user_role(user_id,role_id)", + "INSERT INTO _dever_auth_role(id,site,name,all_permissions,enabled) VALUES ('publisher','admin','Legacy Publisher',0,1)", + "INSERT INTO _dever_auth_role_permission(role_id,permission_key) VALUES ('publisher','news.article.admin.publish')", + "INSERT INTO _dever_auth_user_role(user_id,role_id) VALUES (9,'publisher')", + ] { + database + .execute( + database::Sql { + sqlite: sql, + postgres: sql, + }, + vec![], + ) + .await + .unwrap(); + } +} + +async fn save_role( + catalog: &database::Database, + authorization: &database::Database, + role: &Role, + permission: &str, +) { + store::save_role( + catalog.clone(), + authorization.clone(), + role, + &[permission.into()], + ) + .await + .unwrap(); +} diff --git a/test/dever-tests/tests/support/llvm.rs b/test/dever-tests/tests/support/llvm.rs new file mode 100644 index 0000000..277990c --- /dev/null +++ b/test/dever-tests/tests/support/llvm.rs @@ -0,0 +1,17 @@ +use dever_core::{check, llvm::emit_kernel, source::SourceMap}; + +/// Native acceptance always starts with checked source, not hand-written IR. +pub fn lower(source: &str) -> String { + let mut sources = SourceMap::default(); + sources.add("main.dever", source); + let program = check(&sources).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }); + emit_kernel(&program, &sources, "main.main").unwrap() +} diff --git a/test/dever-tests/tests/support/llvm_managed.rs b/test/dever-tests/tests/support/llvm_managed.rs new file mode 100644 index 0000000..e6bfc0a --- /dev/null +++ b/test/dever-tests/tests/support/llvm_managed.rs @@ -0,0 +1,68 @@ +use crate::llvm; +use std::process::Output; +use std::time::Duration; + +#[path = "llvm_module.rs"] +mod module; + +pub fn execute(source: &str, output_type: &str, checks: &str) { + execute_bounded(source, output_type, checks, 64, Duration::from_secs(5), &[]); +} + +/// Expensive protocol/crypto fixtures retain allocation checks with explicit bounds. +pub fn execute_bounded( + source: &str, + output_type: &str, + checks: &str, + iterations: usize, + timeout: Duration, + arguments: &[&str], +) -> Output { + let mut ir = llvm::lower(source); + // The emitter supplies the actual nominal and fault layouts. + ir.push_str(&format!( + "\ndefine i32 @dever_test_run() {{\nentry:\n %out = alloca {output_type}\n %fault = alloca %dever.fault\n %status = call i32 @dever_entry(ptr %out, ptr %fault)\n{checks}\n call void @dever_outputs_release(ptr %out)\n call void @dever_fault_release(ptr %fault)\n %exit = select i1 %passed, i32 0, i32 1\n ret i32 %exit\n}}\n" + )); + module::execute(&ir, source, iterations, timeout, arguments, None) +} + +pub fn boolean(source: &str) { + boolean_bounded(source, 64, Duration::from_secs(5), &[]); +} + +pub fn boolean_bounded( + source: &str, + iterations: usize, + timeout: Duration, + arguments: &[&str], +) -> Output { + // Dever equality requires T? on both sides, not implicit T-to-T? coercion. + let source = format!( + "maybe_int(value: Int) (answer: Int?) {{ answer = value }} +maybe_text(value: Text) (answer: Text?) {{ answer = value }} +maybe_decimal(value: Decimal) (answer: Decimal?) {{ answer = value }} +maybe_float(value: Float) (answer: Float?) {{ answer = value }} +maybe_list(value: List<Text>) (answer: List<Text>?) {{ answer = value }} +{source}" + ); + execute_bounded( + &source, + "{ i1 }", + " %okay = icmp eq i32 %status, 0 + br i1 %okay, label %success, label %failed +success: + %answer = load i1, ptr %out + br label %done +failed: + br label %done +done: + %passed = phi i1 [ %answer, %success ], [ false, %failed ]", + iterations, + timeout, + arguments, + ) +} + +pub fn runtime_fault(source: &str, message: &str) { + execute(source, "{ i1 }", &module::fault_check(3, Some(message))); +} diff --git a/test/dever-tests/tests/support/llvm_module.rs b/test/dever-tests/tests/support/llvm_module.rs new file mode 100644 index 0000000..d8fbfc7 --- /dev/null +++ b/test/dever-tests/tests/support/llvm_module.rs @@ -0,0 +1,175 @@ +use crate::{process, temp}; +use dever_backend_bridge::{BinaryResource, Target, emit_object_with_resources}; +use std::fmt::Write; +use std::fs; +use std::path::Path; +use std::process::{Command, Output, Stdio}; +use std::time::Duration; + +/// Kernel and application probes share the real archive and atomic ledger. +/// Deployment settings belong only to this owned test directory. +pub fn execute( + ir: &str, + fixture: &str, + iterations: usize, + timeout: Duration, + arguments: &[&str], + setting: Option<&str>, +) -> Output { + let directory = temp::TemporaryDirectory::new(); + if let Some(setting) = setting { + fs::create_dir(directory.path().join("config")).unwrap(); + fs::write(directory.path().join("config/setting.json"), setting).unwrap(); + } + execute_in(&directory, ir, fixture, iterations, timeout, arguments) +} + +/// Successive schema revisions share only this owned executable/data directory. +pub fn execute_in( + directory: &temp::TemporaryDirectory, + ir: &str, + fixture: &str, + iterations: usize, + timeout: Duration, + arguments: &[&str], +) -> Output { + let program = link_in(directory, ir, fixture, iterations); + execute_program(&program, fixture, timeout, arguments) +} + +/// Relocated executables use the same empty environment and bounded process owner. +pub fn execute_program( + program: &Path, + fixture: &str, + timeout: Duration, + arguments: &[&str], +) -> Output { + let directory = program.parent().expect("fixture executable directory"); + let stdout = directory.join("stdout"); + let stderr = directory.join("stderr"); + let status = process::status( + Command::new(program) + .env_clear() + .args(arguments) + .stdin(Stdio::null()) + .stdout(fs::File::create(&stdout).unwrap()) + .stderr(fs::File::create(&stderr).unwrap()), + timeout, + ) + .unwrap_or_else(|error| { + let completed = fs::read_to_string(&stdout) + .unwrap_or_default() + .lines() + .count(); + panic!("{fixture}: {error}; completed output lines: {completed}"); + }); + let stdout = fs::read(stdout).unwrap(); + let stderr = fs::read(stderr).unwrap(); + assert!( + status.success(), + "LLVM fixture failed: {status}\n{}\n{fixture}", + String::from_utf8_lossy(&stderr) + ); + Output { + status, + stdout, + stderr, + } +} + +/// HTTP probes control their own child after linking the same ledger driver. +pub fn link_in( + directory: &temp::TemporaryDirectory, + ir: &str, + fixture: &str, + iterations: usize, +) -> std::path::PathBuf { + link_with_driver(directory, ir, fixture, iterations, "managed-driver.c") +} + +/// Suite dispatch needs a case-index driver; object/archive/ledger linkage stays shared. +pub fn link_with_driver( + directory: &temp::TemporaryDirectory, + ir: &str, + fixture: &str, + iterations: usize, + driver: &str, +) -> std::path::PathBuf { + link_with_resources(directory, ir, fixture, iterations, driver, &[]) +} + +pub fn link_with_resources( + directory: &temp::TemporaryDirectory, + ir: &str, + fixture: &str, + iterations: usize, + driver: &str, + resources: &[BinaryResource<'_>], +) -> std::path::PathBuf { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let archive = root.join("target/native-runtime-abi/debug/libdever_backend_bridge.a"); + assert!( + archive.is_file(), + "missing explicit runtime ABI fixture archive" + ); + let object = directory.path().join("entry.o"); + fs::write(directory.path().join("entry.ll"), ir).unwrap(); + fs::write( + &object, + emit_object_with_resources(ir, Target::LinuxX86_64, resources).unwrap(), + ) + .unwrap(); + let program = directory.path().join("managed-kernel"); + let mut compiler = Command::new("/usr/bin/cc"); + compiler + .env_clear() + .args(["-std=c11", "-Wall", "-Wextra", "-Werror", "-O2"]) + .arg(format!("-DDEVER_TEST_ITERATIONS={iterations}")) + .arg(root.join("test/native-runtime-abi").join(driver)) + .arg(object) + .arg(archive) + .args([ + "-Wl,--wrap=malloc", + "-Wl,--wrap=calloc", + "-Wl,--wrap=realloc", + "-Wl,--wrap=free", + "-Wl,--wrap=posix_memalign", + "-Wl,--wrap=realpath", + "-ldl", + "-lpthread", + "-lm", + "-o", + ]) + .arg(&program) + // Author tools are explicit, not a product PATH fallback. + .arg("-B/usr/bin/") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()); + assert!( + process::status(&mut compiler, Duration::from_secs(30)) + .unwrap() + .success(), + "could not link LLVM fixture\n{fixture}" + ); + program +} + +/// Exact status/kind/message oracle for the common generated fault layout. +pub fn fault_check(code: u32, message: Option<&str>) -> String { + let mut checks = format!( + " %failed = icmp eq i32 %status, 1\n %code_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 0\n %code = load i32, ptr %code_ptr\n %kind = icmp eq i32 %code, {code}\n %status_ok = and i1 %failed, %kind" + ); + let Some(message) = message else { + checks.push_str("\n %passed = and i1 %status_ok, true"); + return checks; + }; + write!(checks, "\n %message_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 4, i32 0\n %message = load ptr, ptr %message_ptr\n %length_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 4, i32 1\n %length = load i64, ptr %length_ptr\n %length_ok = icmp eq i64 %length, {}\n %has_message = icmp ne ptr %message, null\n %buffer_ok = and i1 %length_ok, %has_message\n %header_ok = and i1 %status_ok, %buffer_ok\n br i1 %header_ok, label %content, label %wrong\ncontent:\n", message.len()).unwrap(); + let mut previous = "true".to_owned(); + for (index, byte) in message.bytes().enumerate() { + writeln!(checks, " %byte_ptr{index} = getelementptr i8, ptr %message, i64 {index}\n %byte{index} = load i8, ptr %byte_ptr{index}\n %byte_ok{index} = icmp eq i8 %byte{index}, {byte}\n %content_ok{index} = and i1 {previous}, %byte_ok{index}").unwrap(); + previous = format!("%content_ok{index}"); + } + write!(checks, " br label %done\nwrong:\n br label %done\ndone:\n %passed = phi i1 [ {previous}, %content ], [ false, %wrong ]").unwrap(); + checks +} diff --git a/test/dever-tests/tests/support/mod.rs b/test/dever-tests/tests/support/mod.rs new file mode 100644 index 0000000..ea1f0f1 --- /dev/null +++ b/test/dever-tests/tests/support/mod.rs @@ -0,0 +1,73 @@ +// Each independent test target imports only the helpers relevant to its layer. +#![allow(dead_code)] + +use std::ffi::OsString; +use std::process::Command; + +use dever_core::hir::Program; +use dever_core::source::SourceMap; + +pub mod temp; + +#[cfg(all(feature = "api", any(feature = "sqlite", feature = "postgres")))] +pub mod authorization; + +#[cfg(feature = "postgres")] +pub mod postgres; + +pub fn sources(source: &str) -> SourceMap { + let mut sources = SourceMap::default(); + sources.add("main.dever", source); + sources +} + +pub fn checked(sources: &SourceMap) -> Program { + dever_core::check(sources).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(sources)) + .collect::<String>() + ) + }) +} + +pub fn rejected(source: &str, message: &str) { + let sources = sources(source); + let errors = match dever_core::check(&sources) { + Err(errors) => errors, + Ok(_) => panic!("source must fail semantic checking:\n{source}"), + }; + assert!( + errors.iter().any(|error| error.message.contains(message)), + "expected {message:?}: {}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ); +} + +pub fn run(source: &str) -> std::process::Output { + let sources = sources(source); + let rustc = std::env::var_os("RUSTC").unwrap_or_else(|| OsString::from("rustc")); + let native = + dever_core::native::compile(&checked(&sources), &sources, "main.main", &rustc).unwrap(); + Command::new(native.executable()).output().unwrap() +} + +pub fn stdout(source: &str) -> String { + let output = run(source); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!( + output.stderr.is_empty(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).unwrap() +} diff --git a/test/dever-tests/tests/support/orm_fixture.rs b/test/dever-tests/tests/support/orm_fixture.rs new file mode 100644 index 0000000..fa12064 --- /dev/null +++ b/test/dever-tests/tests/support/orm_fixture.rs @@ -0,0 +1,352 @@ +#![allow(dead_code)] + +use std::ffi::OsString; +use std::fs; +use std::path::Path; +use std::process::{Command, Output}; + +use dever_core::source::SourceMap; +use dever_runtime::config::RuntimeProfile; +use dever_runtime::orm::{Row, Value}; + +const MODEL: &str = r#"type Status { + Active + Disabled +} + +type User { + uuid: Uuid generated unique + email: Text(254) unique + display_name: Text(1, 64) + status: Status default Status.Active index + age: Int default 0 + alias: Text? +} + +seed { + { + email = "seed@example.com" + display_name = "Seed User" + age = 10 + } +} +"#; + +const APP: &str = r#"show(user: app.user.model.User) () { + blocking(dever.io.println(user.email)) +} + +transaction create_transaction_user() (saved: app.user.model.User) { + created = model.create({ + email = "transaction@example.com" + display_name = "Transaction User" + age = 40 + }) + changed = model.update(created.id, { + alias = "committed" + }) + saved = model.get(created.id) +} + +transaction insert_then_fail() () { + inserted = model.create({ + email = "rollback@example.com" + display_name = "Rollback User" + }) + duplicate = model.create({ + email = "seed@example.com" + display_name = "Duplicate Seed" + }) +} + +main() () { + transaction_user = create_transaction_user() + blocking(dever.io.println(transaction_user.display_name)) + + created = model.create({ + email = "one@example.com" + display_name = "One" + age = 20 + }) + changed = model.update(created.id, { + display_name = "Updated One" + }) + loaded = model.get(created.id) + blocking(dever.io.println(loaded.display_name)) + + inserted = model.create_many([ + { + email = "two@example.com" + display_name = "Two" + age = 30 + }, + { + email = "three@example.com" + display_name = "Three" + age = 30 + } + ]) + blocking(dever.io.println(int.to_text(inserted))) + + saved = model.upsert( + { email = "two@example.com" }, + { + display_name = "Ignored" + age = 30 + }, + { display_name = "Upserted Two" } + ) + blocking(dever.io.println(saved.display_name)) + + total = model.count({ + where = not contains(display_name, "missing") and in(status, [app.user.model.Status.Active]) and between(age, 18, 50) and starts_with(email, "t") and ends_with(email, ".com") and alias == null + }) + blocking(dever.io.println(int.to_text(total))) + + page = model.list({ + order = id.asc + page = 1 + size = 2 + }) + blocking(dever.io.println(int.to_text(page.total))) + + cursor = model.cursor({ + order = [age.desc, id.desc] + size = 2 + }) + blocking(dever.io.println(int.to_text(length(cursor.items)))) + each(show, cursor.items) + next = model.cursor({ + after = cursor.next + order = [age.desc, id.desc] + size = 2 + }) + blocking(dever.io.println(int.to_text(length(next.items)))) + each(show, next.items) + + users = model.stream({ + order = id.asc + size = 1 + }) + each(show, users) + + removed = model.delete(created.id) + blocking(dever.io.println(int.to_text(removed))) +} + +rollback() () { + insert_then_fail() +} +"#; + +pub const MAIN_STDOUT: &str = concat!( + "Transaction User\n", + "Updated One\n", + "2\n", + "Upserted Two\n", + "2\n", + "5\n", + "2\n", + "transaction@example.com\n", + "three@example.com\n", + "2\n", + "two@example.com\n", + "one@example.com\n", + "seed@example.com\n", + "transaction@example.com\n", + "one@example.com\n", + "two@example.com\n", + "three@example.com\n", + "1\n", +); + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Revisions { + pub schema: String, + pub seed: String, +} + +#[derive(Debug, Eq, PartialEq)] +pub struct UserRow { + pub email: String, + pub display_name: String, + pub status: String, + pub age: i64, + pub alias: Option<String>, +} + +#[derive(Debug, Eq, PartialEq)] +pub struct DatabaseState { + pub columns: Vec<String>, + pub managed_index_count: usize, + pub schema_revision: String, + pub seed_revision: String, + pub users: Vec<UserRow>, +} + +pub struct CompiledFixture { + pub native: dever_core::native::NativeProgram, + pub revisions: Revisions, +} + +pub fn compile(entry: &str, profile: RuntimeProfile) -> CompiledFixture { + let sources = sources(); + let program = dever_core::check(&sources).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }); + let revisions = revisions(&program.model_snapshot()); + let rustc = std::env::var_os("RUSTC").unwrap_or_else(|| OsString::from("rustc")); + let native = + dever_core::native::compile_application(&program, &sources, entry, &rustc, profile) + .unwrap(); + CompiledFixture { native, revisions } +} + +pub fn configure(executable: &Path, setting: &str) { + let root = executable.parent().unwrap(); + fs::create_dir_all(root.join("config")).unwrap(); + fs::write(root.join("config/setting.json"), setting).unwrap(); +} + +pub fn run(executable: &Path) -> Output { + Command::new(executable).output().unwrap() +} + +pub fn assert_main_output(output: Output) { + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(String::from_utf8(output.stdout).unwrap(), MAIN_STDOUT); + assert!(output.stderr.is_empty()); +} + +pub fn assert_committed_state(state: &DatabaseState, revisions: &Revisions) { + assert_eq!( + state.columns, + [ + "id", + "created_at", + "uuid", + "email", + "display_name", + "status", + "age", + "alias" + ] + ); + assert_eq!(state.managed_index_count, 3); + assert_eq!(state.schema_revision, revisions.schema); + assert_eq!(state.seed_revision, revisions.seed); + assert_eq!( + state.users, + [ + UserRow { + email: "seed@example.com".into(), + display_name: "Seed User".into(), + status: "Active".into(), + age: 10, + alias: None, + }, + UserRow { + email: "three@example.com".into(), + display_name: "Three".into(), + status: "Active".into(), + age: 30, + alias: None, + }, + UserRow { + email: "transaction@example.com".into(), + display_name: "Transaction User".into(), + status: "Active".into(), + age: 40, + alias: Some("committed".into()), + }, + UserRow { + email: "two@example.com".into(), + display_name: "Upserted Two".into(), + status: "Active".into(), + age: 30, + alias: None, + }, + ] + ); +} + +pub fn database_state( + columns: Vec<Row>, + managed_index_count: Vec<Row>, + history: Vec<Row>, + users: Vec<Row>, +) -> DatabaseState { + DatabaseState { + columns: columns.iter().map(|row| text(row, 0)).collect(), + managed_index_count: integer(&managed_index_count[0], 0) as usize, + schema_revision: text(&history[0], 1), + seed_revision: text(&history[1], 1), + users: users + .iter() + .map(|row| UserRow { + email: text(row, 0), + display_name: text(row, 1), + status: text(row, 2), + age: integer(row, 3), + alias: nullable_text(row, 4), + }) + .collect(), + } +} + +pub fn sources() -> SourceMap { + let mut sources = SourceMap::default(); + sources.add("app/user/model.dever", MODEL); + sources.add("app/user/app.dever", APP); + sources +} + +fn revisions(snapshot: &str) -> Revisions { + let model = snapshot + .lines() + .find(|line| line.starts_with("model app.user.model ")) + .expect("shared ORM fixture Model snapshot"); + Revisions { + schema: snapshot_value(model, "revision"), + seed: snapshot_value(model, "seed"), + } +} + +fn snapshot_value(line: &str, name: &str) -> String { + line.split_whitespace() + .find_map(|part| part.strip_prefix(&format!("{name}="))) + .unwrap_or_else(|| panic!("missing {name} in Model snapshot: {line}")) + .to_owned() +} + +fn text(row: &Row, index: usize) -> String { + match row.get(index).unwrap() { + Value::Text(value) => value.clone(), + value => panic!("expected Text at column {index}, got {value:?}"), + } +} + +fn integer(row: &Row, index: usize) -> i64 { + match row.get(index).unwrap() { + Value::Int(value) => *value, + value => panic!("expected Int at column {index}, got {value:?}"), + } +} + +fn nullable_text(row: &Row, index: usize) -> Option<String> { + match row.get(index).unwrap() { + Value::Null => None, + Value::Text(value) => Some(value.clone()), + value => panic!("expected nullable Text at column {index}, got {value:?}"), + } +} diff --git a/test/dever-tests/tests/support/orm_postgres_migration.rs b/test/dever-tests/tests/support/orm_postgres_migration.rs new file mode 100644 index 0000000..98cdd68 --- /dev/null +++ b/test/dever-tests/tests/support/orm_postgres_migration.rs @@ -0,0 +1,163 @@ +use dever_runtime::database::{ + DataMigration, Field, Migration, MigrationPhase, Model, PostgresColumn, Schema, SchemaSql, Sql, +}; +use dever_runtime::orm::{ErrorKind, Value}; +use dever_runtime::postgres::Database; + +const MODEL: &str = "app.model.migration_probe"; +const TABLE: &str = "migration_probe"; +const BEFORE: &str = "UPDATE migration_probe SET name = $1 WHERE name = $2"; +const AFTER: &str = "UPDATE migration_probe SET tag = $1"; + +fn field(name: &str, nullable: bool) -> Field { + Field { + name: name.into(), + ty: "Text".into(), + nullable, + generated: false, + default: None, + rename_from: None, + } +} + +fn column(name: &str, nullable: bool) -> PostgresColumn { + PostgresColumn { + name: name.into(), + definition: format!("\"{name}\" TEXT{}", if nullable { "" } else { " NOT NULL" }), + sql_type: "TEXT".into(), + default: None, + } +} + +fn statement(phase: MigrationPhase, sql: &'static str, parameters: Vec<Value>) -> DataMigration { + DataMigration { + name: "backfill_tag".into(), + phase, + sql: Sql { + sqlite: "", + postgres: sql, + }, + parameters, + } +} + +fn model(upgraded: bool, after_sql: &'static str) -> Model { + let mut fields = vec![field("name", false)]; + let mut columns = vec![column("name", false)]; + if upgraded { + fields.push(field("tag", true)); + columns.push(column("tag", true)); + } + let data_migrations = if upgraded { + vec![ + statement( + MigrationPhase::Before, + BEFORE, + vec![ + Value::Text("short".into()), + Value::Text("legacy-name".into()), + ], + ), + statement( + MigrationPhase::After, + after_sql, + vec![Value::Text("tagged".into())], + ), + ] + } else { + vec![] + }; + Model { + schema: Schema { + model: MODEL.into(), + table: TABLE.into(), + revision: if upgraded { "probe-v2" } else { "probe-v1" }.into(), + seed_revision: "probe-seed".into(), + fields, + indexes: vec![], + migrations: if upgraded { + vec![Migration { + name: "backfill_tag".into(), + revision: "probe-v1".into(), + drops: vec![], + }] + } else { + vec![] + }, + }, + create_table: SchemaSql { + sqlite: String::new(), + postgres: if upgraded { + "CREATE TABLE migration_probe (\"name\" TEXT NOT NULL, \"tag\" TEXT)" + } else { + "CREATE TABLE migration_probe (\"name\" TEXT NOT NULL)" + } + .into(), + }, + create_temporary_table: String::new(), + create_indexes: vec![], + postgres_columns: columns, + postgres_constraints: vec![], + seeds: vec![], + data_migrations, + } +} + +pub async fn assert_named_data_migration(database: &Database) { + database.migrate(model(false, AFTER)).await.unwrap(); + database + .execute( + "INSERT INTO migration_probe (name) VALUES ($1)", + vec![Value::Text("legacy-name".into())], + ) + .await + .unwrap(); + + let failure = database + .migrate(model(true, "UPDATE missing_probe SET tag = $1")) + .await + .unwrap_err(); + assert_eq!(failure.kind(), ErrorKind::Database, "{failure}"); + let names = database + .query("SELECT name FROM migration_probe", vec![]) + .await + .unwrap(); + assert_eq!(names[0].get(0).unwrap(), &Value::Text("legacy-name".into())); + let columns = database + .query( + "SELECT column_name FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = 'migration_probe' ORDER BY ordinal_position", + vec![], + ) + .await + .unwrap(); + assert_eq!(columns.len(), 1); + + database.migrate(model(true, AFTER)).await.unwrap(); + let rows = database + .query("SELECT name, tag FROM migration_probe", vec![]) + .await + .unwrap(); + assert_eq!(rows[0].get(0).unwrap(), &Value::Text("short".into())); + assert_eq!(rows[0].get(1).unwrap(), &Value::Text("tagged".into())); + database + .execute( + "UPDATE migration_probe SET tag = $1", + vec![Value::Text("hand-edited".into())], + ) + .await + .unwrap(); + database.migrate(model(true, AFTER)).await.unwrap(); + let rows = database + .query("SELECT tag FROM migration_probe", vec![]) + .await + .unwrap(); + assert_eq!(rows[0].get(0).unwrap(), &Value::Text("hand-edited".into())); + let history = database + .query( + "SELECT COUNT(*) FROM _dever_history WHERE model = $1 AND kind = 'migration' AND name = 'backfill_tag'", + vec![Value::Text(MODEL.into())], + ) + .await + .unwrap(); + assert_eq!(history[0].get(0).unwrap(), &Value::Int(1)); +} diff --git a/test/dever-tests/tests/support/orm_sqlite.rs b/test/dever-tests/tests/support/orm_sqlite.rs new file mode 100644 index 0000000..0cb8110 --- /dev/null +++ b/test/dever-tests/tests/support/orm_sqlite.rs @@ -0,0 +1,232 @@ +#![allow(dead_code)] + +use dever_runtime::database::{Field, Index, Migration, Model, Schema, SchemaSql, Seed, Sql}; +use dever_runtime::orm::{Row, Value}; +use dever_runtime::sqlite::Database; + +pub const MODEL_NAME: &str = "app.model.user"; +pub const TABLE_NAME: &str = "user"; + +#[derive(Clone)] +pub struct TestField { + schema: Field, + sqlite: String, +} + +impl TestField { + pub fn new(name: &str, ty: &str, nullable: bool, sqlite: &str) -> Self { + Self { + schema: Field { + name: name.into(), + ty: ty.into(), + nullable, + generated: name == "id" || name == "created_at", + default: (name == "created_at").then(|| "generated:created_at".into()), + rename_from: None, + }, + sqlite: sqlite.into(), + } + } + + pub fn renamed_from(mut self, source: &str) -> Self { + self.schema.rename_from = Some(source.into()); + self + } + + pub fn default(mut self, value: &str) -> Self { + self.schema.default = Some(value.into()); + self + } +} + +pub fn id() -> TestField { + TestField::new( + "id", + "Int", + false, + "\"id\" INTEGER PRIMARY KEY AUTOINCREMENT", + ) +} + +pub fn created_at() -> TestField { + TestField::new( + "created_at", + "DateTime", + false, + "\"created_at\" INTEGER NOT NULL DEFAULT (unixepoch() * 1000)", + ) +} + +pub fn text_field(name: &str, maximum: usize, nullable: bool) -> TestField { + TestField::new( + name, + &format!("Text({maximum})"), + nullable, + &format!( + "\"{name}\" TEXT {}CHECK (length(\"{name}\") <= {maximum})", + if nullable { "" } else { "NOT NULL " } + ), + ) +} + +pub fn bytes_field(name: &str, maximum: usize, nullable: bool) -> TestField { + TestField::new( + name, + &format!("Bytes({maximum})"), + nullable, + &format!( + "\"{name}\" BLOB {}CHECK (length(\"{name}\") <= {maximum})", + if nullable { "" } else { "NOT NULL " } + ), + ) +} + +pub fn bool_default(name: &str) -> TestField { + TestField::new( + name, + "Bool", + false, + &format!("\"{name}\" INTEGER NOT NULL DEFAULT 1 CHECK (\"{name}\" IN (0, 1))"), + ) + .default("true") +} + +pub fn index(name: &str, fields: &[&str], unique: bool) -> Index { + Index { + name: name.into(), + fields: fields.iter().map(|field| (*field).into()).collect(), + unique, + } +} + +pub fn migration(name: &str, revision: &str, drops: &[&str]) -> Migration { + Migration { + name: name.into(), + revision: revision.into(), + drops: drops.iter().map(|field| (*field).into()).collect(), + } +} + +pub fn seed(sqlite: &'static str, parameters: Vec<Value>) -> Seed { + Seed { + sql: Sql { + sqlite, + postgres: "", + }, + parameters, + } +} + +pub fn model( + revision: &str, + seed_revision: &str, + fields: Vec<TestField>, + indexes: Vec<Index>, + migrations: Vec<Migration>, + seeds: Vec<Seed>, +) -> Model { + let temporary_table = format!("_dever_user_{}", &revision[..8]); + let column_sql = fields + .iter() + .map(|field| field.sqlite.as_str()) + .collect::<Vec<_>>() + .join(", "); + let create_indexes = indexes + .iter() + .map(|index| { + let fields = index + .fields + .iter() + .map(|field| quote(field)) + .collect::<Vec<_>>() + .join(", "); + SchemaSql { + sqlite: format!( + "CREATE {}INDEX IF NOT EXISTS {} ON {} ({fields})", + if index.unique { "UNIQUE " } else { "" }, + quote(&index.name), + quote(TABLE_NAME), + ), + postgres: String::new(), + } + }) + .collect(); + + Model { + schema: Schema { + model: MODEL_NAME.into(), + table: TABLE_NAME.into(), + revision: revision.into(), + seed_revision: seed_revision.into(), + fields: fields.into_iter().map(|field| field.schema).collect(), + indexes, + migrations, + }, + create_table: SchemaSql { + sqlite: format!("CREATE TABLE {} ({column_sql})", quote(TABLE_NAME)), + postgres: String::new(), + }, + create_temporary_table: format!("CREATE TABLE {} ({column_sql})", quote(&temporary_table)), + create_indexes, + postgres_columns: vec![], + postgres_constraints: vec![], + seeds, + data_migrations: vec![], + } +} + +#[derive(Debug, PartialEq)] +pub struct DatabaseState { + pub catalog: Vec<Row>, + pub columns: Vec<Row>, + pub history: Vec<Row>, + pub data: Vec<Row>, +} + +pub async fn database_state(database: &Database) -> DatabaseState { + DatabaseState { + catalog: database + .query( + "SELECT type, name, sql FROM sqlite_schema WHERE (type = 'table' AND name = 'user') OR (type = 'index' AND tbl_name = 'user') ORDER BY type, name", + vec![], + ) + .await + .unwrap(), + columns: database + .query( + "SELECT name, type, \"notnull\", dflt_value, pk FROM pragma_table_info('user') ORDER BY cid", + vec![], + ) + .await + .unwrap(), + history: database + .query( + "SELECT kind, name, revision, definition FROM _dever_history WHERE model = 'app.model.user' ORDER BY kind, name", + vec![], + ) + .await + .unwrap(), + data: database + .query("SELECT * FROM user ORDER BY id", vec![]) + .await + .unwrap(), + } +} + +pub fn text(row: &Row, index: usize) -> &str { + match row.get(index).unwrap() { + Value::Text(value) => value, + value => panic!("expected Text at column {index}, got {value:?}"), + } +} + +pub fn integer(row: &Row, index: usize) -> i64 { + match row.get(index).unwrap() { + Value::Int(value) => *value, + value => panic!("expected Int at column {index}, got {value:?}"), + } +} + +fn quote(identifier: &str) -> String { + format!("\"{}\"", identifier.replace('"', "\"\"")) +} diff --git a/test/dever-tests/tests/support/postgres.rs b/test/dever-tests/tests/support/postgres.rs new file mode 100644 index 0000000..60dc1aa --- /dev/null +++ b/test/dever-tests/tests/support/postgres.rs @@ -0,0 +1,520 @@ +use std::future::Future; +use std::panic::{AssertUnwindSafe, catch_unwind, resume_unwind}; +use std::path::Path; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use dever_runtime::config::{Database as DatabaseSetting, PostgresTls, Settings}; +use dever_runtime::orm::Value; +use dever_runtime::postgres::{ConnectionOptions, Database}; + +const POSTGRES_TEST_CONNECTION: &str = "postgres_test"; +const MAX_PAGE_SIZE: usize = 100; + +static NEXT_SCHEMA: AtomicU64 = AtomicU64::new(0); + +pub struct IsolatedDatabase { + document: serde_json::Value, + url: String, + tls: PostgresTls, +} + +impl IsolatedDatabase { + pub fn url(&self) -> &str { + &self.url + } + + pub fn tls(&self) -> PostgresTls { + self.tls + } + + pub fn open( + &self, + max_connections: usize, + wait_timeout: Duration, + io_timeout: Duration, + ) -> Database { + open_database( + &self.url, + self.tls, + max_connections, + wait_timeout, + io_timeout, + ) + } + + pub fn settings_json(&self) -> String { + let mut connection = self.document["database"][POSTGRES_TEST_CONNECTION].clone(); + connection["url"] = serde_json::Value::String(self.url.clone()); + let mut document = serde_json::Map::new(); + for name in ["http", "log"] { + if let Some(value) = self.document.get(name) { + document.insert(name.to_owned(), value.clone()); + } + } + if let Some(value) = self.document.get("tenant") { + let mut tenant = value.clone(); + tenant["database"] = serde_json::Value::String("default".into()); + document.insert("tenant".into(), tenant); + } + document.insert( + "database".into(), + serde_json::json!({"default": connection}), + ); + serde_json::to_string_pretty(&document).unwrap() + "\n" + } +} + +pub struct IsolatedTenantDatabases { + database_setting: serde_json::Value, +} + +impl IsolatedTenantDatabases { + pub fn database_setting(&self) -> serde_json::Value { + self.database_setting.clone() + } +} + +pub fn run_in_isolated_databases<F, T>(case_name: &str, test: F) -> T +where + F: FnOnce(IsolatedTenantDatabases) -> Result<T, String>, +{ + let setting = load_setting(case_name); + let suffix = unique_namespace(); + let control_case = format!("{case_name}_{suffix}"); + let template = setting.document["database"][POSTGRES_TEST_CONNECTION]["url"] + .as_str() + .unwrap(); + let control_url = case_url(template, &control_case); + let control_name = database_name(&control_url); + let tenant_prefix = format!("dever_t_{suffix}"); + assert!(control_name.len() <= 63 && valid_case_name(&control_name)); + assert!(tenant_prefix.len() <= 43 && valid_case_name(&tenant_prefix)); + + let mut cleanup_required = false; + let setup = catch_unwind(AssertUnwindSafe(|| { + dever_runtime::task::run_entry(async { + let administrator = open_database( + &setting.url, + setting.tls, + 1, + Duration::from_secs(1), + Duration::from_secs(5), + ); + let result = async { + let existing = database_names(&administrator).await?; + if existing.iter().any(|name| { + name == &control_name || name.starts_with(&format!("{tenant_prefix}_")) + }) { + return Err("isolated PostgreSQL database names already exist".into()); + } + cleanup_required = true; + administrator + .query_owned(format!("CREATE DATABASE \"{control_name}\""), vec![]) + .await + .map_err(|error| error.to_string())?; + Ok(()) + } + .await; + let close = administrator + .close() + .await + .map_err(|error| error.to_string()); + result.and(close) + }) + })); + + let mut database_setting = setting.document["database"][POSTGRES_TEST_CONNECTION].clone(); + database_setting["url"] = serde_json::Value::String(control_url); + database_setting["tenant_database_prefix"] = serde_json::Value::String(tenant_prefix.clone()); + let result = match setup { + Ok(Ok(())) => catch_unwind(AssertUnwindSafe(|| { + test(IsolatedTenantDatabases { database_setting }) + })), + Ok(Err(error)) => Ok(Err(error)), + Err(panic) => Err(panic), + }; + let cleanup = cleanup_required.then(|| { + catch_unwind(AssertUnwindSafe(|| { + dever_runtime::task::run_entry(cleanup_databases( + &setting.url, + setting.tls, + &control_name, + &tenant_prefix, + )) + })) + }); + let cleanup_error = match cleanup { + None | Some(Ok(Ok(()))) => None, + Some(Ok(Err(error))) => Some(error), + Some(Err(_)) => Some("PostgreSQL database cleanup panicked".into()), + }; + match result { + Ok(Ok(value)) => { + if let Some(error) = cleanup_error { + panic!("PostgreSQL database cleanup failed: {error}"); + } + value + } + Ok(Err(error)) => match cleanup_error { + Some(cleanup_error) => { + panic!("{error}; PostgreSQL database cleanup failed: {cleanup_error}") + } + None => panic!("{error}"), + }, + Err(panic) => { + if let Some(error) = cleanup_error { + eprintln!("PostgreSQL database cleanup failed after panic: {error}"); + } + resume_unwind(panic); + } + } +} + +async fn database_names(administrator: &Database) -> Result<Vec<String>, String> { + let rows = administrator + .query("SELECT datname::text FROM pg_database", vec![]) + .await + .map_err(|error| error.to_string())?; + rows.into_iter() + .map(|row| match row.get(0).map_err(|error| error.to_string())? { + Value::Text(name) => Ok(name.clone()), + _ => Err("PostgreSQL database name was not Text".into()), + }) + .collect() +} + +async fn cleanup_databases( + administrator_url: &str, + tls: PostgresTls, + control_name: &str, + tenant_prefix: &str, +) -> Result<(), String> { + let administrator = open_database( + administrator_url, + tls, + 1, + Duration::from_secs(1), + Duration::from_secs(5), + ); + let result = async { + let mut failures = Vec::new(); + match database_names(&administrator).await { + Ok(names) => { + for name in owned_tenant_database_names(names, tenant_prefix) { + if let Err(error) = administrator + .query_owned(format!("DROP DATABASE \"{name}\""), vec![]) + .await + { + failures.push(format!("cannot drop owned tenant database {name}: {error}")); + } + } + } + Err(error) => failures.push(format!("cannot list owned tenant databases: {error}")), + } + if let Err(error) = administrator + .query_owned( + format!("DROP DATABASE IF EXISTS \"{control_name}\""), + vec![], + ) + .await + { + failures.push(format!( + "cannot drop owned control database {control_name}: {error}" + )); + } + if failures.is_empty() { + Ok(()) + } else { + Err(failures.join("; ")) + } + } + .await; + let close = administrator + .close() + .await + .map_err(|error| error.to_string()); + result.and(close) +} + +fn owned_tenant_database_names(names: Vec<String>, prefix: &str) -> Vec<String> { + let marker = format!("{prefix}_"); + names + .into_iter() + .filter(|name| { + name.strip_prefix(&marker) + .and_then(|id| { + id.parse::<u64>() + .ok() + .filter(|value| *value > 0) + .map(|value| value.to_string() == id) + }) + .unwrap_or(false) + }) + .collect() +} + +pub fn run_in_isolated_schema<F, Fut>(case_name: &str, test: F) +where + F: FnOnce(IsolatedDatabase) -> Fut, + Fut: Future<Output = Result<(), String>>, +{ + let setting = load_setting(case_name); + let schema = unique_schema(); + let cleanup_url = setting.url.clone(); + let cleanup_schema = schema.clone(); + let tls = setting.tls; + let result = catch_unwind(AssertUnwindSafe(|| { + dever_runtime::task::run_entry(async move { + let administrator = open_database( + &setting.url, + tls, + 2, + Duration::from_secs(1), + Duration::from_secs(5), + ); + administrator + .query_owned(format!("CREATE SCHEMA \"{schema}\""), vec![]) + .await + .map_err(|error| error.to_string())?; + administrator + .close() + .await + .map_err(|error| error.to_string())?; + test(IsolatedDatabase { + document: setting.document, + url: scoped_url(&setting.url, &schema), + tls, + }) + .await + }) + })); + let cleanup = catch_unwind(AssertUnwindSafe(|| { + dever_runtime::task::run_entry(async move { + let administrator = open_database( + &cleanup_url, + tls, + 1, + Duration::from_secs(1), + Duration::from_secs(5), + ); + let drop_result = administrator + .query_owned( + format!("DROP SCHEMA IF EXISTS \"{cleanup_schema}\" CASCADE"), + vec![], + ) + .await; + let close_result = administrator.close().await; + drop_result.map_err(|error| error.to_string())?; + close_result.map_err(|error| error.to_string())?; + Ok(()) + }) + })); + match result { + Ok(result) => { + cleanup.unwrap().unwrap(); + result.unwrap(); + } + Err(panic) => { + let _ = cleanup; + resume_unwind(panic); + } + } +} + +struct PostgresTestSetting { + document: serde_json::Value, + url: String, + tls: PostgresTls, +} + +fn load_setting(case_name: &str) -> PostgresTestSetting { + assert!( + valid_case_name(case_name), + "PostgreSQL acceptance case names must use lower_snake_case" + ); + let workspace = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let settings = Settings::load_project(&workspace).unwrap_or_else(|error| { + panic!( + "PostgreSQL acceptance requires '{}': {error}", + workspace.join("config/setting.json").display() + ) + }); + let (url, tls) = match settings.database(POSTGRES_TEST_CONNECTION) { + Some(DatabaseSetting::Postgres { url, tls, .. }) => (case_url(url, case_name), *tls), + Some(DatabaseSetting::Sqlite { .. }) => { + panic!("database.{POSTGRES_TEST_CONNECTION} must use type 'postgres'") + } + None => panic!("database.{POSTGRES_TEST_CONNECTION} is required"), + }; + let document = serde_json::from_str( + &std::fs::read_to_string(workspace.join("config/setting.json")).unwrap(), + ) + .unwrap(); + PostgresTestSetting { document, url, tls } +} + +fn case_url(template: &str, case_name: &str) -> String { + let (url, query) = template + .split_once('?') + .map_or((template, ""), |(url, query)| (url, query)); + let database = url.rsplit_once('/').map_or("", |(_, database)| database); + assert!( + matches!( + template.split_once("://"), + Some(("postgres" | "postgresql", _)) + ) && template.matches("{case}").count() == 1 + && database.matches("{case}").count() == 1 + && !query_overrides_database(query), + concat!( + "config/setting.json database.postgres_test.url must contain exactly one ", + "{{case}} placeholder in its database name and must not override it with dbname" + ) + ); + template.replace("{case}", case_name) +} + +fn query_overrides_database(query: &str) -> bool { + query.split('&').any(|parameter| { + let key = parameter.split_once('=').map_or(parameter, |(key, _)| key); + percent_decode(key).as_deref() == Some(b"dbname") + }) +} + +fn percent_decode(value: &str) -> Option<Vec<u8>> { + let bytes = value.as_bytes(); + let mut decoded = Vec::with_capacity(bytes.len()); + let mut index = 0; + while index < bytes.len() { + if bytes[index] != b'%' { + decoded.push(bytes[index]); + index += 1; + continue; + } + let high = hex_digit(*bytes.get(index + 1)?)?; + let low = hex_digit(*bytes.get(index + 2)?)?; + decoded.push(high * 16 + low); + index += 3; + } + Some(decoded) +} + +fn hex_digit(byte: u8) -> Option<u8> { + match byte { + b'0'..=b'9' => Some(byte - b'0'), + b'a'..=b'f' => Some(byte - b'a' + 10), + b'A'..=b'F' => Some(byte - b'A' + 10), + _ => None, + } +} + +fn valid_case_name(case_name: &str) -> bool { + !case_name.is_empty() + && case_name.starts_with(|character: char| character.is_ascii_lowercase()) + && case_name.split('_').all(|part| { + !part.is_empty() + && part + .chars() + .all(|character| character.is_ascii_lowercase() || character.is_ascii_digit()) + }) +} + +fn unique_schema() -> String { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + format!( + "dever_t_{:x}_{:x}_{:x}", + std::process::id(), + nanos, + NEXT_SCHEMA.fetch_add(1, Ordering::Relaxed) + ) +} + +fn unique_namespace() -> String { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + format!( + "{:x}_{:x}_{:x}", + std::process::id(), + nanos, + NEXT_SCHEMA.fetch_add(1, Ordering::Relaxed) + ) +} + +fn database_name(url: &str) -> String { + let path = url.split_once('?').map_or(url, |(path, _)| path); + path.rsplit_once('/').unwrap().1.to_owned() +} + +fn scoped_url(url: &str, schema: &str) -> String { + let separator = if url.contains('?') { '&' } else { '?' }; + format!("{url}{separator}options=-csearch_path%3D{schema}") +} + +pub(crate) fn open_database( + url: &str, + tls: PostgresTls, + max_connections: usize, + wait_timeout: Duration, + io_timeout: Duration, +) -> Database { + Database::open(ConnectionOptions { + url, + database_name: None, + tls, + min_connections: 0, + max_connections, + max_page_size: MAX_PAGE_SIZE, + wait_timeout, + io_timeout, + }) + .unwrap() +} + +#[cfg(test)] +mod tests { + use super::{case_url, owned_tenant_database_names}; + + #[test] + fn case_placeholder_must_select_the_effective_database() { + assert_eq!( + case_url( + "postgresql://dever@localhost/dever_{case}?sslmode=disable", + "authorization_postgres", + ), + "postgresql://dever@localhost/dever_authorization_postgres?sslmode=disable" + ); + for url in [ + "postgresql://dever@localhost/dever_{case}?dbname=production", + "postgresql://dever@localhost/dever_{case}?%64bname=production", + "postgresql://dever@localhost/dever?application_name={case}", + ] { + assert!( + std::panic::catch_unwind(|| case_url(url, "postgres_orm")).is_err(), + "URL must be rejected: {url}" + ); + } + } + + #[test] + fn cleanup_only_selects_this_run_and_canonical_positive_tenant_ids() { + let names = [ + "dever_t_run_1", + "dever_t_run_2", + "dever_t_run_0", + "dever_t_run_01", + "dever_t_run_-1", + "dever_t_run_1_extra", + "dever_t_run2_1", + "dever_t_run", + ]; + assert_eq!( + owned_tenant_database_names(names.map(str::to_owned).to_vec(), "dever_t_run"), + ["dever_t_run_1", "dever_t_run_2"], + ); + } +} diff --git a/test/dever-tests/tests/support/process.rs b/test/dever-tests/tests/support/process.rs new file mode 100644 index 0000000..50b0355 --- /dev/null +++ b/test/dever-tests/tests/support/process.rs @@ -0,0 +1,40 @@ +use std::io; +use std::process::{Child, Command, ExitStatus}; +use std::time::{Duration, Instant}; + +/// Run only a test-owned process, and reap it on success, failure or timeout. +pub fn status(command: &mut Command, timeout: Duration) -> io::Result<ExitStatus> { + let mut child = command.spawn()?; + wait(&mut child, timeout) +} + +/// Wait for a child whose caller owns concurrent interaction and panic cleanup. +pub fn wait(child: &mut Child, timeout: Duration) -> io::Result<ExitStatus> { + let deadline = Instant::now() + timeout; + loop { + match child.try_wait() { + Ok(Some(status)) => return Ok(status), + Ok(None) => {} + Err(error) => { + terminate(child)?; + return Err(error); + } + } + if Instant::now() >= deadline { + terminate(child)?; + return Err(io::Error::new( + io::ErrorKind::TimedOut, + "test-owned process exceeded its deadline", + )); + } + std::thread::sleep(Duration::from_millis(5)); + } +} + +pub fn terminate(child: &mut Child) -> io::Result<()> { + let killed = child.kill(); + let reaped = child.wait(); + // Reap even if kill failed (for example, it raced with normal exit). + reaped?; + killed +} diff --git a/test/dever-tests/tests/support/sandbox.rs b/test/dever-tests/tests/support/sandbox.rs new file mode 100644 index 0000000..b0901de --- /dev/null +++ b/test/dever-tests/tests/support/sandbox.rs @@ -0,0 +1,40 @@ +//! Explicit private fixture assets; never consult PATH or download a tool. + +use std::fs; +use std::path::Path; + +pub struct Asset { + pub path: String, + pub bytes: Vec<u8>, + pub executable: bool, +} + +pub fn assets(workspace: &Path) -> Vec<Asset> { + let prepared = workspace.join("target/sandbox-inputs/assets"); + let mut assets = Vec::new(); + for directory in ["bin", "lib"] { + for entry in fs::read_dir(prepared.join(directory)) + .expect("prepare target/sandbox-inputs assets explicitly") + { + let entry = entry.unwrap(); + assert!(entry.file_type().unwrap().is_file()); + let name = entry.file_name().into_string().unwrap(); + if directory == "bin" && name == "guard" { + continue; + } + assets.push(Asset { + path: format!("sandbox/{directory}/{name}"), + bytes: fs::read(entry.path()).unwrap(), + executable: directory == "bin" || name.starts_with("ld-linux"), + }); + } + } + assets.push(Asset { + path: "sandbox/bin/guard".into(), + bytes: fs::read(workspace.join("target/debug/dever-sandbox-guard")) + .expect("build the current dever-sandbox-guard fixture explicitly"), + executable: true, + }); + assets.sort_by(|left, right| left.path.cmp(&right.path)); + assets +} diff --git a/test/dever-tests/tests/support/sqlite_inspect.rs b/test/dever-tests/tests/support/sqlite_inspect.rs new file mode 100644 index 0000000..2485cc1 --- /dev/null +++ b/test/dever-tests/tests/support/sqlite_inspect.rs @@ -0,0 +1,32 @@ +//! Inspect only an explicitly supplied, test-owned SQLite file between roots. +use dever_runtime::orm::Value; +use std::path::Path; + +pub fn query(path: &Path, sql: &'static str) -> Vec<Vec<Value>> { + dever_runtime::task::run_entry(async { + let database = dever_runtime::sqlite::Database::open(path, 1, 100) + .map_err(|error| error.to_string())?; + let rows = database + .query(sql, vec![]) + .await + .map_err(|error| error.to_string())?; + Ok(rows + .into_iter() + .map(|row| row.into_values().collect()) + .collect()) + }) + .unwrap() +} + +pub fn execute(path: &Path, sql: &'static str) { + dever_runtime::task::run_entry(async { + let database = dever_runtime::sqlite::Database::open(path, 1, 100) + .map_err(|error| error.to_string())?; + database + .execute(sql, vec![]) + .await + .map_err(|error| error.to_string())?; + Ok(()) + }) + .unwrap(); +} diff --git a/test/dever-tests/tests/support/temp.rs b/test/dever-tests/tests/support/temp.rs new file mode 100644 index 0000000..40125cd --- /dev/null +++ b/test/dever-tests/tests/support/temp.rs @@ -0,0 +1,44 @@ +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +pub struct TemporaryDirectory(PathBuf); + +impl TemporaryDirectory { + pub fn new() -> Self { + Self::new_in(&std::env::temp_dir()) + } + + pub fn new_in(parent: &Path) -> Self { + static NEXT: AtomicU64 = AtomicU64::new(0); + loop { + let path = parent.join(format!( + "dever-core-test-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let mut builder = fs::DirBuilder::new(); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + match builder.create(&path) { + Ok(()) => return Self(path), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => panic!("cannot create test directory: {error}"), + } + } + } + + pub fn path(&self) -> &Path { + &self.0 + } +} + +impl Drop for TemporaryDirectory { + fn drop(&mut self) { + // This object owns only the directory it exclusively created above. + let _ = fs::remove_dir_all(&self.0); + } +} diff --git a/test/dever-tests/tests/support/wire_peer.rs b/test/dever-tests/tests/support/wire_peer.rs new file mode 100644 index 0000000..0723b96 --- /dev/null +++ b/test/dever-tests/tests/support/wire_peer.rs @@ -0,0 +1,115 @@ +// Independent test targets use different subsets of the shared HTTP/WS wire tools. +#![allow(dead_code)] + +use dever_runtime::{bytes::Bytes, net}; + +// 测试独立观察 HTTP 分块和 WebSocket 帧，不使用被测协议引擎。 +pub struct Peer { + pub socket: net::Socket, + buffered: Vec<u8>, +} + +impl Peer { + pub async fn connect(port: i64) -> Result<Self, String> { + Self::new(net::connect_timeout("127.0.0.1", port, 1000).await?) + } + + pub fn new(socket: net::Socket) -> Result<Self, String> { + net::timeout(&socket, 1500)?; + Ok(Self { + socket, + buffered: Vec::new(), + }) + } + + pub async fn write(&self, bytes: &[u8]) -> Result<(), String> { + net::write(&self.socket, &Bytes::new(bytes.to_vec())).await + } + + async fn more(&mut self) -> Result<(), String> { + let bytes = net::read(&self.socket, 4096) + .await? + .ok_or("unexpected peer EOF")?; + self.buffered.extend_from_slice(bytes.values()); + Ok(()) + } + + pub async fn take(&mut self, count: usize) -> Result<Vec<u8>, String> { + while self.buffered.len() < count { + self.more().await?; + } + Ok(self.buffered.drain(..count).collect()) + } + + async fn through(&mut self, delimiter: &[u8]) -> Result<Vec<u8>, String> { + loop { + if let Some(index) = self + .buffered + .windows(delimiter.len()) + .position(|part| part == delimiter) + { + return self.take(index + delimiter.len()).await; + } + self.more().await?; + } + } + + pub async fn head(&mut self) -> Result<String, String> { + String::from_utf8(self.through(b"\r\n\r\n").await?).map_err(|error| error.to_string()) + } + + pub async fn chunk(&mut self) -> Result<Vec<u8>, String> { + let line = self.through(b"\r\n").await?; + let length = + usize::from_str_radix(std::str::from_utf8(&line[..line.len() - 2]).unwrap(), 16) + .unwrap(); + let bytes = self.take(length).await?; + assert_eq!(self.take(2).await?, b"\r\n"); + Ok(bytes) + } + + pub async fn frame(&mut self) -> Result<(u8, bool, Vec<u8>), String> { + let head = self.take(2).await?; + let masked = head[1] & 128 != 0; + let length = match head[1] & 127 { + 126 => u16::from_be_bytes(self.take(2).await?.try_into().unwrap()) as usize, + 127 => u64::from_be_bytes(self.take(8).await?.try_into().unwrap()) as usize, + value => value as usize, + }; + assert!(length <= 8 * 1024 * 1024, "bounded test frame"); + let mask = if masked { + self.take(4).await? + } else { + vec![0; 4] + }; + let mut bytes = self.take(length).await?; + for (index, byte) in bytes.iter_mut().enumerate() { + *byte ^= mask[index % 4]; + } + Ok((head[0], masked, bytes)) + } + + pub async fn send_frame(&self, opcode: u8, masked: bool, payload: &[u8]) -> Result<(), String> { + assert!(payload.len() < 126, "small independent test frame"); + let mask = [17, 23, 31, 47]; + let mut wire = vec![opcode, payload.len() as u8 | if masked { 128 } else { 0 }]; + if masked { + wire.extend_from_slice(&mask); + } + wire.extend( + payload + .iter() + .enumerate() + .map(|(index, byte)| byte ^ if masked { mask[index % 4] } else { 0 }), + ); + self.write(&wire).await + } + + pub async fn upgrade(&mut self) -> Result<(), String> { + self.write(b"GET /ws HTTP/1.1\r\nHost: test\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nSec-WebSocket-Version: 13\r\nSec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n\r\n").await?; + let headers = self.head().await?.to_ascii_lowercase(); + assert!(headers.starts_with("http/1.1 101"), "{headers}"); + assert!(headers.contains("sec-websocket-accept: s3pplmbitxaq9kygzzhzrbk+xoo=")); + Ok(()) + } +} diff --git a/test/dever-tests/tests/syntax_and_format.rs b/test/dever-tests/tests/syntax_and_format.rs new file mode 100644 index 0000000..671fd7f --- /dev/null +++ b/test/dever-tests/tests/syntax_and_format.rs @@ -0,0 +1,501 @@ +use std::fs; +use std::path::PathBuf; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use dever_core::diagnostic::{Diagnostic, Label}; +use dever_core::source::SourceMap; +use dever_core::syntax::{ + BinaryOperator as Op, Declaration, Expression, ExpressionKind as Expr, InputKind, Literal, + Package, Pattern, StatementKind, TypeShape, UnaryOperator, +}; + +fn parsed(text: &str) -> Package { + let mut sources = SourceMap::default(); + let id = sources.add("account/user.dever", text); + dever_core::parse(sources.get(id)).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }) +} + +fn rejected(text: &str) -> (SourceMap, Vec<Diagnostic>) { + let mut sources = SourceMap::default(); + let id = sources.add("broken.dever", text); + let errors = dever_core::parse(sources.get(id)).expect_err("invalid syntax must be rejected"); + assert!(!errors.is_empty()); + (sources, errors) +} + +fn assigned(expression: &str) -> Expression { + let package = parsed(&format!( + "public main() (result: Int) {{\n result = {expression}\n}}" + )); + let Declaration::Function(function) = package.declarations.into_iter().next().unwrap() else { + panic!("expected function") + }; + let StatementKind::Assign { value, .. } = function.body.into_iter().next().unwrap().kind else { + panic!("expected assignment") + }; + value +} + +#[test] +fn parses_the_complete_package_from_the_product_contract() { + let prd = include_str!("../../../.trellis/tasks/09-04-dever-language-mvp/prd.md"); + let (_, example_section) = prd.split_once("## Complete Package Example\n").unwrap(); + let (_, fenced) = example_section.split_once("```dever\n").unwrap(); + let (source, _) = fenced.split_once("\n```").unwrap(); + let package = parsed(source); + assert_eq!( + package + .name + .iter() + .map(|name| name.text.as_str()) + .collect::<Vec<_>>(), + ["account", "user"] + ); + assert!(package.declarations.iter().any(|declaration| matches!( + declaration, + Declaration::Function(function) + if function.name.text == "total_score_bonus" && function.public + ))); + assert_eq!( + package + .declarations + .iter() + .filter(|declaration| matches!(declaration, Declaration::Type(_))) + .count(), + 5 + ); + assert_eq!( + package + .declarations + .iter() + .filter(|declaration| matches!(declaration, Declaration::Function(_))) + .count(), + 22 + ); + let Declaration::Type(status) = &package.declarations[0] else { + panic!("expected UserStatus") + }; + let TypeShape::Choice(variants) = &status.shape else { + panic!("expected choice") + }; + assert_eq!(variants[2].name.text, "Disabled"); + assert_eq!(variants[2].payload[0].name.text, "reason"); +} + +#[test] +fn arithmetic_precedence_and_associativity_follow_the_language_contract() { + let Expr::Binary { + left, + operator: Op::Subtract, + right, + } = assigned("10 - 4 - 2 * 3").kind + else { + panic!("expected subtraction") + }; + assert!(matches!( + left.kind, + Expr::Binary { + operator: Op::Subtract, + .. + } + )); + assert!(matches!( + right.kind, + Expr::Binary { + operator: Op::Multiply, + .. + } + )); + let Expr::Binary { + left, + operator: Op::Or, + right, + } = assigned("not enabled and score >= 60 or override").kind + else { + panic!("expected or") + }; + let Expr::Binary { + left: negated, + operator: Op::And, + right: compared, + } = left.kind + else { + panic!("expected and") + }; + assert!(matches!( + negated.kind, + Expr::Unary { + operator: UnaryOperator::Not, + .. + } + )); + assert!(matches!( + compared.kind, + Expr::Binary { + operator: Op::GreaterEqual, + .. + } + )); + assert!(matches!(right.kind, Expr::Name(_))); +} + +#[test] +fn integer_division_is_never_a_comment() { + let Expr::Binary { + operator, right, .. + } = assigned("-5 // 2 # actual comment").kind + else { + panic!("expected division") + }; + assert_eq!(operator, Op::IntegerDivide); + assert!(matches!(right.kind, Expr::Literal(Literal::Number(ref number)) if number == "2")); +} + +#[test] +fn numeric_spelling_survives_parsing_without_rounding() { + for number in [ + "9223372036854775808", + "0.12345678901234567890123456789012345", + "1.25e-6", + ] { + assert!( + matches!(assigned(number).kind, Expr::Literal(Literal::Number(ref actual)) if actual == number) + ); + } + let Expr::Unary { value, .. } = assigned("-9223372036854775808").kind else { + panic!("expected unary minus") + }; + assert!( + matches!(value.kind, Expr::Literal(Literal::Number(ref number)) if number == "9223372036854775808") + ); +} + +#[test] +fn text_escapes_and_comments_preserve_unicode_and_spans() { + let source = "# 中文注释\npublic main() (text: Text) {\n text = \"中文\\n\\u{1f600}\\t\\\\\\\"\" # trailing\n}\n# final\n"; + let package = parsed(source); + assert_eq!(package.comments.len(), 3); + for comment in &package.comments { + assert_eq!(&source[comment.span.start..comment.span.end], comment.text); + } + let Declaration::Function(function) = &package.declarations[0] else { + panic!("expected function") + }; + let StatementKind::Assign { value, .. } = &function.body[0].kind else { + panic!("expected assignment") + }; + assert!( + matches!(&value.kind, Expr::Literal(Literal::Text(text)) if text == "中文\n\u{1f600}\t\\\"") + ); +} + +#[test] +fn diagnostic_points_to_the_bad_escape_after_multibyte_text() { + let source = "main() (text: Text) {\n text = \"中\\q\"\n}"; + let (sources, errors) = rejected(source); + assert_eq!(errors[0].code, "L006"); + assert_eq!( + &source[errors[0].primary.start..errors[0].primary.end], + "\\q" + ); + let rendered = errors[0].render(&sources); + assert!( + rendered.starts_with("broken.dever:2:12: error[L006]: invalid Text escape\n"), + "{rendered}" + ); + assert_eq!(rendered.lines().nth(1), Some(" | text = \"中\\q\"")); + assert_eq!(rendered.lines().count(), 2); + assert!(!rendered.contains("Token") && !rendered.contains("rust")); +} + +#[test] +fn bad_literals_and_non_ascii_identifiers_are_rejected() { + for value in [ + "1e", + "2e+", + "3suffix", + "\"\\u{d800}\"", + "\"\\u{110000}\"", + "\"\\u{}\"", + "\"\\x41\"", + "\"unterminated", + ] { + rejected(&format!("main() (result: Text) {{ result = {value} }}")); + } + let (_, errors) = rejected("package 示例 exposes ()"); + assert_eq!(errors[0].code, "L001"); +} + +#[test] +fn lexical_diagnostics_are_sorted_by_source_location() { + let (_, errors) = rejected("main() (text: Text) { text = \"\\q"); + assert_eq!(errors.len(), 2); + assert_eq!(errors[0].code, "L005"); + assert_eq!(errors[1].code, "L006"); + assert!(errors[0].primary.start < errors[1].primary.start); +} + +#[test] +fn related_diagnostic_labels_resolve_their_own_source_file() { + let mut sources = SourceMap::default(); + let first = sources.add("first.dever", "main() () { 1 }"); + let second = sources.add("second.dever", "main() () { 2 }"); + let mut primary = dever_core::parse(sources.get(first)).unwrap_err().remove(0); + let related = dever_core::parse(sources.get(second)) + .unwrap_err() + .remove(0); + primary.related.push(Label { + span: related.primary, + message: "related declaration".into(), + }); + let rendered = primary.render(&sources); + assert!(rendered.starts_with("first.dever:1:13: error[P002]")); + assert!(rendered.contains("second.dever:1:13: note: related declaration\n | main() () { 2 }")); +} + +#[test] +fn patterns_cover_ranges_choices_null_and_complement() { + let package = parsed( + "grade(score: Int >= 60 and < 90) (result: Text) { result = \"pass\" }\ngrade(score: Int < 60) (result: Text) { result = \"fail\" }\nselect(value: Result.Created(user, _)) (result: Text) { result = user.name }\nselect(value: null) (result: Text) { result = \"none\" }\nselect(value: other) (result: Text) { result = \"other\" }", + ); + let Declaration::Function(grade) = &package.declarations[0] else { + panic!("expected function") + }; + let InputKind::Value(Pattern::Typed { ty, bounds }) = &grade.inputs[0].kind else { + panic!("expected range") + }; + assert_eq!(ty.name[0].text, "Int"); + assert_eq!(bounds[0].comparison, Op::GreaterEqual); + assert_eq!(bounds[1].number, "90"); + let Declaration::Function(select) = &package.declarations[2] else { + panic!("expected function") + }; + let InputKind::Value(Pattern::Variant { bindings, .. }) = &select.inputs[0].kind else { + panic!("expected variant") + }; + assert_eq!(bindings[1].text, "_"); +} + +#[test] +fn generic_types_and_all_collection_literal_shapes_parse() { + let empty = parsed("type Empty {}\nmake() (value: Empty) { value = Empty {} }"); + assert!( + matches!(&empty.declarations[0], Declaration::Type(declaration) if matches!(&declaration.shape, TypeShape::Record(fields) if fields.is_empty())) + ); + parsed( + "main() (users: List<User>, labels: Map<Text, List<Int?>>) {\n users = [User { name = \"a\" }, User { name = \"b\" }]\n labels = {\n \"a\" = [1, null]\n \"b\" = []\n }\n labels = {}\n}", + ); + let Expr::Map(entries) = assigned("{\n\"first\" = 1\n\"second\" = 2\n}").kind else { + panic!("expected Map") + }; + assert_eq!(entries.len(), 2); + assert!(matches!(&entries[0].key.kind, Expr::Literal(Literal::Text(text)) if text == "first")); +} + +#[test] +fn handler_inputs_and_stream_types_have_dedicated_syntax() { + let package = parsed( + "serve(route: handler(request: Text) (response: Text), events: Stream<Text>) (result: Text) { result = route(\"ok\") }", + ); + let Declaration::Function(function) = &package.declarations[0] else { + panic!("expected function") + }; + let InputKind::Handler(signature) = &function.inputs[0].kind else { + panic!("expected handler input") + }; + assert_eq!(signature.inputs[0].name.text, "request"); + assert_eq!(signature.outputs[0].name.text, "response"); + let InputKind::Value(Pattern::Typed { ty, .. }) = &function.inputs[1].kind else { + panic!("expected Stream input") + }; + assert_eq!(ty.name[0].text, "Stream"); + assert_eq!(ty.arguments[0].name[0].text, "Text"); +} + +#[test] +fn handler_signature_syntax_is_rejected_outside_function_inputs() { + for source in [ + "type Box { route: handler(value: Int) (answer: Int) }", + "main() (route: handler(value: Int) (answer: Int)) {}", + ] { + rejected(source); + } +} + +#[test] +fn postfix_calls_grouping_and_multiline_arguments_preserve_structure() { + let Expr::Field { value, name } = assigned("split_name(\n \"a\" +\n \"b\"\n).first").kind + else { + panic!("expected field access") + }; + assert_eq!(name.text, "first"); + let Expr::Call { arguments, .. } = value.kind else { + panic!("expected call") + }; + assert!(matches!( + arguments[0].kind, + Expr::Binary { + operator: Op::Add, + .. + } + )); + let Expr::Binary { left, .. } = assigned("(1 + 2) * 3").kind else { + panic!("expected multiplication") + }; + assert!(matches!(left.kind, Expr::Group(_))); +} + +#[test] +fn excluded_constructs_cannot_silently_become_valid_syntax() { + for body in [ + "if true {}", + "return 1", + "result = null ?? 0", + "result += 1", + "result = value?.name", + "result = values[0]", + "result = 1;", + "1 + 2", + "make().field = 1", + "result = other", + ] { + rejected(&format!("main() (result: Int) {{\n {body}\n}}")); + } + for source in [ + "import account.user", + "package account.user exposes (User)\ntype User { name: Text }", + "type Mixed {\n name: Text\n Pending\n}", + "main() (value: Int??) { value = null }", + "main() { }", + "main(value: Int >= 0 and <= 5 and < 4) () {}", + ] { + rejected(source); + } +} + +#[test] +fn a_statement_requires_a_line_or_structural_boundary() { + parsed("main() () {\n emit(1)\n emit(2)\n}"); + rejected("main() () { emit(1) emit(2) }"); + rejected("main() (x: Int) { x = 1\n + 2 }"); +} + +#[test] +fn recovery_reports_independent_declaration_failures_in_source_order() { + let source = "first() (x: Int) { x = }\nsecond() (x: Int) { x = }"; + let (_, errors) = rejected(source); + assert_eq!(errors.len(), 2); + assert!(errors[0].primary.start < errors[1].primary.start); +} + +#[test] +fn excessive_nesting_is_a_diagnostic_instead_of_a_stack_overflow() { + let source = format!( + "main() (x: Int) {{ x = {}1{} }}", + "(".repeat(160), + ")".repeat(160) + ); + let (_, errors) = rejected(&source); + assert_eq!(errors[0].code, "P009"); + for expression in [ + format!("{}1", "1 + ".repeat(160)), + format!("f{}", "()".repeat(160)), + ] { + let (_, errors) = rejected(&format!("main() (x: Int) {{ x = {expression} }}")); + assert_eq!(errors[0].code, "P009"); + } +} + +#[test] +fn source_positions_support_crlf_unicode_and_eof() { + let mut sources = SourceMap::default(); + let id = sources.add("p.dever", "# 中\r\n\r\n"); + let source = sources.get(id); + assert_eq!(source.position(7), (2, 1)); + assert_eq!(source.position(source.text().len()), (3, 1)); + assert_eq!(source.line(1), "# 中"); + assert_eq!(source.line(3), ""); + let id = sources.add("cr.dever", "# comment\r\r"); + let source = sources.get(id); + assert_eq!(source.position(10), (2, 1)); + assert_eq!(source.line(1), "# comment"); + assert_eq!(source.position(source.text().len()), (3, 1)); + dever_core::parse(source).unwrap(); +} + +struct SourceDirectory(PathBuf); + +impl SourceDirectory { + fn create() -> Self { + static NEXT: AtomicUsize = AtomicUsize::new(0); + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let path = std::env::temp_dir().join(format!( + "dever-source-{}-{nonce}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir(&path).unwrap(); + Self(path) + } +} + +impl Drop for SourceDirectory { + fn drop(&mut self) { + fs::remove_dir_all(&self.0).expect("remove the test-owned source directory"); + } +} + +#[test] +fn source_loading_is_sorted_relative_and_ignores_other_file_formats() { + let directory = SourceDirectory::create(); + fs::create_dir(directory.0.join("account")).unwrap(); + fs::write(directory.0.join("z.dever"), "").unwrap(); + fs::write(directory.0.join("account/user.dever"), "").unwrap(); + fs::write(directory.0.join("ignored.json"), [0xff]).unwrap(); + let sources = SourceMap::load(&directory.0).unwrap(); + assert_eq!( + sources + .files() + .iter() + .map(|source| source.path().to_str().unwrap()) + .collect::<Vec<_>>(), + ["account/user.dever", "z.dever"] + ); + for source in sources.files() { + dever_core::parse(source).unwrap(); + } +} + +#[test] +fn source_loading_reports_invalid_utf8_and_missing_roots() { + let directory = SourceDirectory::create(); + fs::write(directory.0.join("broken.dever"), [b'#', 0xff]).unwrap(); + let errors = SourceMap::load(&directory.0).unwrap_err(); + assert_eq!(errors.len(), 1); + assert_eq!(errors[0].message, "invalid UTF-8 at byte 1"); + assert!(SourceMap::load(&directory.0.join("missing")).is_err()); + assert!(SourceMap::load(&directory.0.join("broken.dever")).is_err()); +} + +#[cfg(unix)] +#[test] +fn source_loading_does_not_follow_symbolic_links_outside_the_root() { + let outside = SourceDirectory::create(); + let directory = SourceDirectory::create(); + fs::write(outside.0.join("outside.dever"), "").unwrap(); + std::os::unix::fs::symlink(&outside.0, directory.0.join("escape")).unwrap(); + let errors = SourceMap::load(&directory.0).unwrap_err(); + assert!(errors[0].message.contains("symbolic links")); +} diff --git a/test/dever-tests/tests/tenant_database.rs b/test/dever-tests/tests/tenant_database.rs new file mode 100644 index 0000000..be479cf --- /dev/null +++ b/test/dever-tests/tests/tenant_database.rs @@ -0,0 +1,261 @@ +mod support; + +use std::fs; +use std::process::Command; +use std::time::Duration; + +use dever_runtime::config::{self, RuntimeProfile}; +use dever_runtime::database::{self, Executor, ModelScope, Sql, StorageBinding}; +use dever_runtime::orm::{self, ErrorKind, Value}; + +const SCHEMA: &str = "tenant-schema-v1"; + +#[test] +fn sqlite_tenant_databases_are_ready_bounded_and_transaction_isolated() { + let executable = std::env::current_exe().unwrap(); + let directory = executable.parent().unwrap(); + if !directory.join("config/setting.json").is_file() { + let project = support::temp::TemporaryDirectory::new(); + fs::create_dir(project.path().join("config")).unwrap(); + fs::write( + project.path().join("config/setting.json"), + r#"{ + "tenant": { + "database": "default", + "max_pools": 2, + "idle_timeout_ms": 1000 + }, + "database": { + "default": { + "type": "sqlite", + "path": "data/platform.db", + "tenant_directory": "data/tenants", + "max_connections": 2 + } + } + }"#, + ) + .unwrap(); + let child = project.path().join("tenant-database-case"); + fs::copy(executable, &child).unwrap(); + let output = Command::new(child) + .args([ + "--exact", + "sqlite_tenant_databases_are_ready_bounded_and_transaction_isolated", + "--nocapture", + ]) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + return; + } + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + runtime.block_on(async { + config::bootstrap( + RuntimeProfile { + sqlite: true, + postgres: false, + }, + &[], + &[], + ) + .unwrap(); + database::prepare().await.unwrap(); + dever_runtime::tenant::initialize(SCHEMA).await.unwrap(); + + let tenant_binding = StorageBinding::new(None, "content", ModelScope::Tenant); + let missing = match dever_runtime::tenant::scope(1, tenant_binding.database()).await { + Ok(_) => panic!("an unprovisioned tenant must not resolve a database"), + Err(error) => error, + }; + assert_eq!(missing.kind(), ErrorKind::Migration); + assert!(!directory.join("data/tenants/tenant_1.db").exists()); + + for tenant_id in [1, 2, 3] { + let database = dever_runtime::tenant::begin_migration("default", tenant_id, SCHEMA) + .await + .unwrap(); + database.execute(Sql { + sqlite: "CREATE TABLE profile (id INTEGER PRIMARY KEY, name TEXT NOT NULL)", + postgres: "CREATE TABLE profile (id BIGINT PRIMARY KEY, name TEXT NOT NULL)", + }, Vec::new()).await.unwrap(); + dever_runtime::tenant::mark_database_ready(&database, tenant_id, SCHEMA) + .await + .unwrap(); + dever_runtime::tenant::finish_migration(tenant_id, SCHEMA) + .await + .unwrap(); + } + + let tenant_components = ["media", "news"]; + dever_runtime::tenant::require_components(1, &["news"], &tenant_components) + .await + .unwrap(); + dever_runtime::tenant::disable_component(1, "news", &tenant_components) + .await + .unwrap(); + let disabled = dever_runtime::tenant::require_components( + 1, + &["media", "news", "news"], + &tenant_components, + ) + .await + .unwrap_err(); + assert_eq!(disabled.kind(), ErrorKind::InvalidData); + assert!( + disabled + .to_string() + .contains("tenant component 'news' is disabled") + ); + dever_runtime::tenant::require_components(1, &["media"], &tenant_components) + .await + .unwrap(); + + for component in ["platform", "unknown"] { + let error = dever_runtime::tenant::disable_component(1, component, &tenant_components) + .await + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::InvalidData); + assert!(error.to_string().contains("is not a tenant component")); + } + + let tenant_one = dever_runtime::tenant::scope(1, tenant_binding.database()) + .await + .unwrap(); + tenant_one + .execute( + Sql { + sqlite: "INSERT INTO profile (id,name) VALUES (?1,?2)", + postgres: "INSERT INTO profile (id,name) VALUES ($1,$2)", + }, + vec![Value::Int(1), Value::Text("one".into())], + ) + .await + .unwrap(); + + let tenant_two = dever_runtime::tenant::scope(2, tenant_binding.database()) + .await + .unwrap(); + tenant_two + .execute( + Sql { + sqlite: "INSERT INTO profile (id,name) VALUES (?1,?2)", + postgres: "INSERT INTO profile (id,name) VALUES ($1,$2)", + }, + vec![Value::Int(1), Value::Text("two".into())], + ) + .await + .unwrap(); + let rows = tenant_two + .query( + Sql { + sqlite: "SELECT name FROM profile WHERE id = ?1", + postgres: "SELECT name FROM profile WHERE id = $1", + }, + vec![Value::Int(1)], + ) + .await + .unwrap(); + assert_eq!(orm::text(rows[0].get(0).unwrap().clone()).unwrap(), "two"); + + let rows = tenant_one + .query( + Sql { + sqlite: "SELECT name FROM profile WHERE id = ?1", + postgres: "SELECT name FROM profile WHERE id = $1", + }, + vec![Value::Int(1)], + ) + .await + .unwrap(); + assert_eq!(orm::text(rows[0].get(0).unwrap().clone()).unwrap(), "one"); + assert!( + tenant_one + .query( + Sql { + sqlite: "SELECT component FROM _dever_tenant_component", + postgres: "SELECT component FROM _dever_tenant_component", + }, + Vec::new() + ) + .await + .is_err() + ); + + dever_runtime::tenant::enable_component(1, "news", &tenant_components) + .await + .unwrap(); + dever_runtime::tenant::require_components(1, &["news"], &tenant_components) + .await + .unwrap(); + + let transaction = tenant_one.begin().await.unwrap(); + let cross_tenant = match Executor::new(tenant_two.clone(), Some(&transaction)) { + Ok(_) => panic!("one transaction must not cross tenant databases"), + Err(error) => error, + }; + assert_eq!(cross_tenant.kind(), ErrorKind::Database); + transaction.rollback().await.unwrap(); + + let pool_error = match dever_runtime::tenant::scope(3, tenant_binding.database()).await { + Ok(_) => panic!("the tenant pool cap must be enforced"), + Err(error) => error, + }; + assert_eq!(pool_error.kind(), ErrorKind::PoolExhausted); + drop(tenant_one); + tokio::time::sleep(Duration::from_millis(1_050)).await; + let tenant_three = dever_runtime::tenant::scope(3, tenant_binding.database()) + .await + .unwrap(); + drop(tenant_three); + + let global = StorageBinding::new(None, "content", ModelScope::Global) + .database() + .await + .unwrap(); + global.execute(Sql { + sqlite: "CREATE TABLE platform_flag (id INTEGER PRIMARY KEY, name TEXT NOT NULL)", + postgres: "CREATE TABLE platform_flag (id BIGINT PRIMARY KEY, name TEXT NOT NULL)", + }, Vec::new()).await.unwrap(); + global + .execute( + Sql { + sqlite: "INSERT INTO platform_flag (id,name) VALUES (?1,?2)", + postgres: "INSERT INTO platform_flag (id,name) VALUES ($1,$2)", + }, + vec![Value::Int(1), Value::Text("shared".into())], + ) + .await + .unwrap(); + let global_from_other_tenant = dever_runtime::tenant::scope( + 2, + StorageBinding::new(None, "content", ModelScope::Global).database(), + ) + .await + .unwrap(); + let rows = global_from_other_tenant + .query( + Sql { + sqlite: "SELECT name FROM platform_flag WHERE id = ?1", + postgres: "SELECT name FROM platform_flag WHERE id = $1", + }, + vec![Value::Int(1)], + ) + .await + .unwrap(); + assert_eq!( + orm::text(rows[0].get(0).unwrap().clone()).unwrap(), + "shared" + ); + + database::shutdown().await.unwrap(); + }); +} diff --git a/test/dever-tests/tests/time_codec.rs b/test/dever-tests/tests/time_codec.rs new file mode 100644 index 0000000..4eea86e --- /dev/null +++ b/test/dever-tests/tests/time_codec.rs @@ -0,0 +1,489 @@ +mod support; + +use std::fs; +use std::process::Command; + +use dever_core::wire::Policy; +use dever_runtime::{time, wire}; + +#[test] +fn time_calendar_precision_and_checked_arithmetic() { + for (text, expected) in [ + ("1970-01-01T00:00:00Z", 0), + ("1969-12-31T23:59:59.999Z", -1), + ("1970-01-01T08:00:00+08:00", 0), + ("2000-02-29T00:00:00Z", 951_782_400_000), + ] { + let value = time::parse_datetime(text).unwrap(); + assert_eq!(value, expected); + assert_eq!( + time::parse_datetime(&time::format_datetime(value).unwrap()).unwrap(), + value + ); + } + assert_eq!(time::format_datetime(0).unwrap(), "1970-01-01T00:00:00Z"); + assert_eq!( + time::format_datetime(-1).unwrap(), + "1969-12-31T23:59:59.999Z" + ); + for text in [ + "1900-02-29T00:00:00Z", + "2000-02-29", + "2016-12-31T23:59:60Z", + "1970-01-01T00:00:00.0001Z", + "1970-01-01T00:00:00.0000000001Z", + ] { + assert!(time::parse_datetime(text).is_err(), "{text}"); + } + for text in ["0000-01-01", "1969-12-31", "2000-02-29", "9999-12-31"] { + assert_eq!( + time::format_date(time::parse_date(text).unwrap()).unwrap(), + text + ); + } + for text in ["1900-02-29", "2024-2-01", "2024-02-30", "+2024-02-01"] { + assert!(time::parse_date(text).is_err()); + } + for text in ["00:00:00", "23:59:59.999", "12:30:05.001"] { + assert_eq!( + time::format_time(time::parse_time(text).unwrap()).unwrap(), + text + ); + } + for text in ["24:00:00", "23:59:60", "12:00:00.1", "12:00:00.0001"] { + assert!(time::parse_time(text).is_err()); + } + assert!(time::format_time(86_400_000).is_err()); + assert!(time::format_time(-1).is_err()); + assert!(time::format_date(1).is_err()); + assert!(time::format_datetime(i64::MAX).is_err()); + assert!(time::format_datetime(i64::MIN).is_err()); + assert!(time::add(i64::MAX, 1).is_err()); + assert!(time::subtract(i64::MIN, 1).is_err()); + assert!(time::difference(i64::MAX, -1).is_err()); + assert_eq!(time::add(-1, time::duration(1)).unwrap(), 0); + assert_eq!(time::difference(-1, 0).unwrap(), -1); +} + +#[test] +fn parser_rejects_ambiguous_and_unbounded_input_without_losing_json_numbers() { + for text in [ + r#"{"a":{"x":1,"x":2}}"#, + r#"[{"x":1,"x":2}]"#, + "true false", + "NaN", + "Infinity", + "[1,]", + r#""\ud800""#, + ] { + assert!(wire::parse(text).is_err(), "{text}"); + } + let precise = + r#"{"big":1234567890123456789012345678901234567890,"tiny":1.234567890123456789e-900}"#; + assert_eq!(wire::parse(precise).unwrap().raw(), precise); + assert_eq!( + wire::parse("-9223372036854775808").unwrap().int().unwrap(), + i64::MIN + ); + for text in ["9223372036854775808", "1.0", "1e0", "true", "\"1\""] { + assert!(wire::parse(text).unwrap().int().is_err()); + } + assert!(wire::parse("1e900").unwrap().float().is_err()); + let at_depth = format!( + "{}0{}", + "[".repeat(wire::MAX_DEPTH), + "]".repeat(wire::MAX_DEPTH) + ); + assert!(wire::parse(&at_depth).is_ok()); + assert!(wire::parse(&format!("[{at_depth}]")).is_err()); + assert!(wire::parse(&format!("\"{}\"", "x".repeat(wire::MAX_BYTES))).is_err()); + assert!(wire::parse(&format!("[{}]", vec!["0"; wire::MAX_ELEMENTS].join(","))).is_err()); +} + +#[test] +fn encoder_enforces_limits_and_preserves_raw_json_in_api_envelope() { + let raw = "1234567890123456789012345678901234567890"; + let mut writer = wire::Encoder::default(); + writer.begin_object().unwrap(); + writer.key("raw").unwrap(); + writer.json(raw).unwrap(); + writer.key("when").unwrap(); + writer.text(&time::format_datetime(-1).unwrap()).unwrap(); + writer.end().unwrap(); + let response = dever_runtime::api::success(writer.finish().unwrap()); + let text = std::str::from_utf8(response.body.values()).unwrap(); + assert_eq!(response.status, 200); + assert_eq!( + text, + format!( + "{{\"code\":0,\"message\":\"ok\",\"data\":{{\"raw\":{raw},\"when\":\"1969-12-31T23:59:59.999Z\"}}}}" + ) + ); + for value in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { + assert!(wire::Encoder::default().float(value).is_err()); + } + let mut writer = wire::Encoder::default(); + writer.begin_array().unwrap(); + for _ in 1..wire::MAX_ELEMENTS { + writer.null().unwrap(); + } + assert!(writer.null().is_err()); + assert!(wire::Encoder::default().json(r#"{"x":1,"x":2}"#).is_err()); + + let mut writer = wire::Encoder::default(); + writer.begin_object().unwrap(); + writer.key("missing").unwrap(); + assert!(writer.end().is_err()); + assert!(writer.finish().is_err()); + + let mut writer = wire::Encoder::default(); + writer.begin_array().unwrap(); + writer.text(&"x".repeat(wire::MAX_BYTES - 3)).unwrap(); + assert!(writer.end().is_err()); + assert!(writer.finish().is_err()); + + let mut writer = wire::Encoder::default(); + writer.begin_array().unwrap(); + writer.text(&"x".repeat(wire::MAX_BYTES - 4)).unwrap(); + assert!(writer.int(1).is_err()); + assert!(writer.end().is_err()); + assert!(writer.finish().is_err()); +} + +const TYPES: &str = r#" +public type Entry { label: Text + count: Int? } +public type Payload { + active: Bool + count: Int + ratio: Float + amount: Decimal + label: Text + key: Id + uuid: Uuid + instant: DateTime + date: Date + clock: Time + elapsed: Duration + raw: Json + entries: List<Entry> +} +public type Credentials { password: Secret } +public main() () {} +"#; + +#[test] +fn schema_policy_and_field_contracts_have_one_owner() { + let program = support::checked(&support::sources(TYPES)); + let output = program.wire_schema("main.Payload", Policy::Output).unwrap(); + let input = program + .wire_schema("main.Payload", Policy::ApiInput) + .unwrap(); + let job = program.wire_schema("main.Payload", Policy::Job).unwrap(); + assert_ne!(output.fingerprint(), input.fingerprint()); + assert_ne!(output.fingerprint(), job.fingerprint()); + let shifted = support::checked(&support::sources(&format!( + "public type Unrelated {{ value: Text }}\n{TYPES}" + ))); + assert_eq!( + output.fingerprint(), + shifted + .wire_schema("main.Payload", Policy::Output) + .unwrap() + .fingerprint() + ); + for policy in [Policy::ApiInput, Policy::SettingInput] { + let schema = program.wire_schema("main.Credentials", policy).unwrap(); + let code = dever_core::native::wire::emit(&schema, 0); + assert!(code.contains("Secret::from_input")); + assert!(!code.contains("_encode")); + } + for policy in [Policy::Output, Policy::Job] { + assert!(program.wire_schema("main.Credentials", policy).is_err()); + } + for declaration in [ + "public type Bad { private name: Text }", + "public type Bad { port: Int >= 1 and <= 65535 }", + "public type Bad { stream: Stream<Int> }", + "public type Bad { values: Map<Text, Int> }", + "public type Bad { One\nTwo }", + ] { + let program = support::checked(&support::sources(&format!( + "{declaration}\npublic main() () {{}}" + ))); + for policy in [ + Policy::Output, + Policy::ApiInput, + Policy::SettingInput, + Policy::Job, + ] { + assert!( + program.wire_schema("main.Bad", policy).is_err(), + "{declaration}" + ); + } + } + let mut bounded = support::sources("main() () {}"); + bounded.add( + "counter/api/value.dever", + "get_value() (response: Int >= 0) { response = 1 }", + ); + assert!( + dever_core::check(&bounded) + .unwrap_err() + .iter() + .any(|error| error + .message + .contains("wire fields must be public and unconstrained")) + ); +} + +#[test] +fn wire_schemas_do_not_change_model_time_storage_identity() { + let mut sources = support::sources("main() () { calendar.event.save() }"); + sources.add( + "calendar/event/model.dever", + "type Event { instant: DateTime\ndate: Date\ntime: Time\nelapsed: Duration }", + ); + sources.add( + "calendar/event/app.dever", + r#"type ClockView { instant: DateTime + date: Date + time: Time + elapsed: Duration } + save() () { + stored = model.create({ + instant = dever.time.parse_datetime("1969-12-31T23:59:59.999Z") + date = dever.time.parse_date("2000-02-29") + time = dever.time.parse_time("23:59:59.999") + elapsed = dever.time.duration(-7) + }) + }"#, + ); + let program = support::checked(&sources); + let before = program.model_snapshot(); + for (name, ty) in [ + ("instant", "DateTime"), + ("date", "Date"), + ("time", "Time"), + ("elapsed", "Duration"), + ] { + assert!( + before.contains(&format!("field {name} {ty} nullable=false")), + "{before}" + ); + } + for policy in [ + Policy::Output, + Policy::ApiInput, + Policy::SettingInput, + Policy::Job, + ] { + program + .wire_schema("calendar.event.ClockView", policy) + .unwrap(); + let id = program + .wire_schema("calendar.event.model.id", policy) + .unwrap(); + assert!( + id.fingerprint() + .contains("ModelId(\"calendar.event.model.id\")") + ); + let model = program + .wire_schema("calendar.event.model.Event", policy) + .unwrap(); + assert!( + model + .fingerprint() + .contains("ModelId(\"calendar.event.model.id\")") + ); + } + assert_eq!(before, program.model_snapshot()); + let generated = dever_core::native::emit(&program, &sources, "main.main").unwrap(); + assert!(generated.contains("f0: i64,\nf1: i64,\nf2: i64,\nf3: i64,")); + assert_eq!( + generated + .matches("dever_runtime::orm::Value::Int(p") + .count(), + 4 + ); +} + +#[test] +fn official_time_bridge_is_pure_and_matches_reference() { + let source = r#"public main() (stamp: Text, date: Text, clock: Text, delta: Duration, invalid: dever.time.DateResult) pure { + instant = dever.time.parse_datetime("1969-12-31T23:59:59.999Z") + stamp = dever.time.format_datetime(dever.time.add(instant, dever.time.duration(1))) + date = dever.time.format_date(dever.time.parse_date("2000-02-29")) + clock = dever.time.format_time(dever.time.parse_time("23:59:59.999")) + delta = dever.time.difference(dever.time.subtract(instant, dever.time.duration(1)), instant) + invalid = result(dever.time.parse_date("1900-02-29")) +}"#; + let sources = support::sources(source); + let program = support::checked(&sources); + let expected = "stamp = 1970-01-01T00:00:00Z\ndate = 2000-02-29\nclock = 23:59:59.999\ndelta = -1\ninvalid = dever.time.DateResult.Failed(invalid calendar Date)\n"; + assert_eq!(support::stdout(source), expected); + let reference = dever_core::reference::evaluate(&program, "main.main").unwrap(); + assert_eq!(reference.stdout, expected); + support::rejected( + "public main() (value: DateTime) pure { value = dever.time.now() }", + "pure", + ); +} + +#[test] +fn time_api_has_equal_plain_and_markdown_contracts() { + let code = "main() (value: Text) pure { value = dever.time.format_datetime(dever.time.parse_datetime(\"2000-02-29T08:00:00+08:00\")) }"; + let plain = support::sources(code); + let mut markdown = dever_core::source::SourceMap::default(); + markdown.add("main.dever.md", format!("# 时间转换\n\n验证 UTC 毫秒合同。\n\n- 包：`main`\n- 公开类型：无\n- 公开方法：无\n- 使用：无\n\n## 入口\n\n转换日期时间。\n\n- 函数：`main`\n- 输入：无\n- 输出：\n - `value: Text`：UTC 时间。\n\n```dever\n{code}\n```\n")); + let plain_program = support::checked(&plain); + let markdown_program = support::checked(&markdown); + assert_eq!( + plain_program.api_snapshot(), + markdown_program.api_snapshot() + ); + let expected = "value = 2000-02-29T00:00:00Z\n"; + for program in [&plain_program, &markdown_program] { + let output = dever_core::reference::evaluate(program, "main.main").unwrap(); + assert!(output.fault.is_none()); + assert_eq!(output.stdout, expected); + } +} + +#[test] +fn generated_api_uses_shared_time_codec_without_a_listener() { + let mut sources = support::sources("main() () { dever.api.serve() }"); + sources.add("calendar/api/time.dever", "get_now() (response: DateTime) { response = dever.time.parse_datetime(\"1970-01-01T08:00:00+08:00\") }\nget_again() (response: DateTime) { response = dever.time.parse_datetime(\"1969-12-31T23:59:59.999Z\") }"); + let program = support::checked(&sources); + let mut code = dever_core::native::emit(&program, &sources, "main.main") + .unwrap() + .replace("fn main()", "fn unused_main()"); + assert_eq!(code.matches("fn wire_0_encode").count(), 1); + assert!(!code.contains("fn wire_1_encode")); + code.push_str(r#" +fn main() { + let request = dever_runtime::http::Request { + method: "GET".into(), target: "/calendar/time/now".into(), headers: vec![], body: Bytes::from_text(""), + }; + let response = task::run_entry(dever_runtime::api::scoped_request(request, api_route)).unwrap(); + assert_eq!(response.status, 200); + assert_eq!(std::str::from_utf8(response.body.values()).unwrap(), "{\"code\":0,\"message\":\"ok\",\"data\":\"1970-01-01T00:00:00Z\"}"); +} +"#); + compile_and_run(&code); +} + +#[test] +fn generated_concrete_codec_roundtrips_and_rejects_bad_records() { + let sources = support::sources(TYPES); + let program = support::checked(&sources); + let mut code = dever_core::native::emit(&program, &sources, "main.main") + .unwrap() + .replace("fn main()", "fn unused_main()"); + code.push_str(&dever_core::native::wire::emit( + &program.wire_schema("main.Payload", Policy::Output).unwrap(), + 0, + )); + code.push_str(&dever_core::native::wire::emit( + &program + .wire_schema("main.Credentials", Policy::SettingInput) + .unwrap(), + 1, + )); + code.push_str(r###" +fn main() { + let raw = r#"{"active":true,"count":-9223372036854775808,"ratio":-0.0,"amount":"1234567890123456789012345678901234","label":"你好","key":"9007199254740993","uuid":"12345678-1234-5678-9012-123456789abc","instant":"1969-12-31T23:59:59.999Z","date":"2000-02-29","clock":"23:59:59.999","elapsed":-7,"raw":1234567890123456789012345678901234567890,"entries":[{"label":"entry"}]}"#; + let decoded = wire_0_decode(raw).unwrap(); + assert_eq!(decoded.f1, i64::MIN); + assert_eq!(decoded.f2.to_bits(), (-0.0f64).to_bits()); + assert_eq!(decoded.f3, DecimalValue::parse("1234567890123456789012345678901234").unwrap()); + assert_eq!(decoded.f4, "你好"); + assert_eq!(decoded.f5.0, "9007199254740993"); + assert_eq!(decoded.f6.to_string(), "12345678-1234-5678-9012-123456789abc"); + assert_eq!(decoded.f7, -1); + assert_eq!(decoded.f8, 951782400000); + assert_eq!(decoded.f9, 86399999); + assert_eq!(decoded.f10, -7); + assert!(decoded.f12.values()[0].f1.is_none()); + let encoded = wire_0_encode(&decoded).unwrap(); + assert!(encoded.as_str().contains("1234567890123456789012345678901234567890")); + assert!(encoded.as_str().contains("1969-12-31T23:59:59.999Z")); + let again = wire_0_decode(encoded.as_str()).unwrap(); + assert_eq!(encoded.as_str(), wire_0_encode(&again).unwrap().as_str()); + assert!(wire_0_decode(&raw.replace("\"count\":-9223372036854775808,", "")).is_err()); + assert!(wire_0_decode(&raw.replace("\"active\":true", "\"extra\":true,\"active\":true")).is_err()); + assert!(wire_0_decode(&raw.replace("\"label\":\"entry\"", "\"label\":\"entry\",\"label\":\"again\"")).is_err()); + assert!(wire_0_decode(&raw.replace("\"ratio\":-0.0", "\"ratio\":1e900")).is_err()); + assert!(wire_1_decode(r#"{"password":"secret"}"#).is_ok()); + assert!(wire_1_decode(r#"{"password":42}"#).is_err()); +} +"###); + compile_and_run(&code); +} + +fn compile_and_run(code: &str) { + let workspace = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let built = Command::new("cargo") + .current_dir(&workspace) + .args([ + "build", + "--locked", + "--offline", + "-p", + "dever-runtime", + "--features", + "api", + "--config", + "profile.dev.debug=0", + "--message-format=json", + ]) + .output() + .unwrap(); + assert!( + built.status.success(), + "{}", + String::from_utf8_lossy(&built.stderr) + ); + let artifact = String::from_utf8(built.stdout) + .unwrap() + .lines() + .filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok()) + .find(|value| { + value["reason"] == "compiler-artifact" && value["target"]["name"] == "dever_runtime" + }) + .unwrap(); + let library = artifact["filenames"] + .as_array() + .unwrap() + .iter() + .filter_map(|value| value.as_str()) + .find(|name| name.ends_with(".rlib")) + .unwrap(); + let directory = support::temp::TemporaryDirectory::new(); + let source = directory.path().join("codec.rs"); + let executable = directory.path().join("codec"); + fs::write(&source, code).unwrap(); + let dependencies = std::path::Path::new(library).parent().unwrap().join("deps"); + let built = Command::new("rustc") + .args(["--edition=2024", "-C", "debuginfo=0", "--extern"]) + .arg(format!("dever_runtime={library}")) + .arg("-L") + .arg(format!("dependency={}", dependencies.display())) + .arg(&source) + .arg("-o") + .arg(&executable) + .output() + .unwrap(); + assert!( + built.status.success(), + "{}", + String::from_utf8_lossy(&built.stderr) + ); + let result = Command::new(executable).output().unwrap(); + assert!( + result.status.success(), + "{}", + String::from_utf8_lossy(&result.stderr) + ); +} diff --git a/test/dever-tests/tests/upload_boundary.rs b/test/dever-tests/tests/upload_boundary.rs new file mode 100644 index 0000000..afe067e --- /dev/null +++ b/test/dever-tests/tests/upload_boundary.rs @@ -0,0 +1,337 @@ +#[path = "support/temp.rs"] +mod temp; + +use bytes::Bytes; +use dever_runtime::api::upload; +use dever_runtime::config::UploadLimits; +use http_body_util::StreamBody; +use hyper::body::Frame; +use std::convert::Infallible; + +fn body( + bytes: &[u8], + chunk: usize, +) -> impl hyper::body::Body<Data = Bytes, Error = Infallible> + Unpin { + StreamBody::new(futures_util::stream::iter( + bytes + .chunks(chunk) + .map(|chunk| Ok(Frame::data(Bytes::copy_from_slice(chunk)))) + .collect::<Vec<Result<Frame<Bytes>, Infallible>>>(), + )) +} + +fn multipart(filename: &str, bytes: &[u8]) -> Vec<u8> { + let mut body = format!("--boundary\r\nContent-Disposition: form-data; name=\"caption\"\r\n\r\nhello\r\n--boundary\r\nContent-Disposition: form-data; name=\"file\"; filename=\"{filename}\"\r\nContent-Type: image/png\r\n\r\n").into_bytes(); + body.extend_from_slice(bytes); + body.extend_from_slice(b"\r\n--boundary--\r\n"); + body +} + +#[tokio::test] +async fn streaming_split_boundaries_preserve_bytes_and_remove_owned_files() { + let directory = temp::TemporaryDirectory::new(); + let mut payload = vec![b'x'; 170_001]; + payload.extend_from_slice(b"\r\n--boundaryX is file content\r\n--boundary--X\r\n--boundary-"); + for chunk in [1, 7, 16_384, 250_000] { + let bytes = multipart("photo.png", &payload); + let mut parts = upload::decode( + body(&bytes, chunk), + "multipart/form-data; boundary=boundary", + directory.path(), + UploadLimits::default(), + ) + .await + .unwrap(); + assert_eq!( + parts + .json("caption", dever_runtime::api::QueryValue::Text) + .unwrap(), + "\"hello\"" + ); + let file = parts.take("file").unwrap(); + assert_eq!(file.filename(), "photo.png"); + assert_eq!(file.content_type(), "image/png"); + assert_eq!(file.size(), payload.len() as i64); + let files = std::fs::read_dir(directory.path()) + .unwrap() + .collect::<Result<Vec<_>, _>>() + .unwrap(); + assert_eq!(files.len(), 1); + assert_eq!(std::fs::read(files[0].path()).unwrap(), payload); + parts.finish().unwrap(); + file.close(); + assert_eq!(std::fs::read_dir(directory.path()).unwrap().count(), 0); + } +} + +#[tokio::test] +async fn malformed_limits_and_duplicate_fields_cleanup_partial_files() { + let directory = temp::TemporaryDirectory::new(); + for bytes in [ + multipart("../escape", b"hello"), + multipart("x", b"hello")[..150].to_vec(), + multipart("x", b"hello").repeat(2), + ] { + assert!( + upload::decode( + body(&bytes, 3), + "multipart/form-data; boundary=boundary", + directory.path(), + UploadLimits::default() + ) + .await + .is_err() + ); + assert_eq!(std::fs::read_dir(directory.path()).unwrap().count(), 0); + } + let bytes = multipart("x", &[0; 40_000]); + for limits in [ + UploadLimits { + file_bytes: 100, + ..Default::default() + }, + UploadLimits { + total_bytes: 256, + file_bytes: 100, + ..Default::default() + }, + UploadLimits { + header_bytes: 10, + ..Default::default() + }, + ] { + assert!( + upload::decode( + body(&bytes, 1024), + "multipart/form-data; boundary=boundary", + directory.path(), + limits + ) + .await + .is_err() + ); + assert_eq!(std::fs::read_dir(directory.path()).unwrap().count(), 0); + } + let duplicate = b"--boundary\r\nContent-Disposition: form-data; name=\"file\"; filename=\"first\"\r\n\r\nfirst\r\n--boundary\r\nContent-Disposition: form-data; name=\"file\"; filename=\"second\"\r\n\r\nsecond\r\n--boundary--\r\n"; + assert!( + upload::decode( + body(duplicate, 1), + "multipart/form-data; boundary=boundary", + directory.path(), + UploadLimits::default() + ) + .await + .is_err() + ); + assert_eq!(std::fs::read_dir(directory.path()).unwrap().count(), 0); +} + +#[tokio::test] +async fn cancellation_drops_partial_upload_owner() { + let directory = temp::TemporaryDirectory::new(); + let bytes = multipart("x", &[0; 40_000]); + let frames = futures_util::stream::iter(vec![Ok::<_, Infallible>(Frame::data( + Bytes::copy_from_slice(&bytes[..30_000]), + ))]); + let stalled = futures_util::StreamExt::chain(frames, futures_util::stream::pending()); + let mut decoding = Box::pin(upload::decode( + StreamBody::new(stalled), + "multipart/form-data; boundary=boundary", + directory.path(), + UploadLimits::default(), + )); + let writing = async { + for _ in 0..100 { + if std::fs::read_dir(directory.path()) + .unwrap() + .any(|file| file.unwrap().metadata().unwrap().len() > 1000) + { + return; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + panic!("decoder did not spool the initial body frames"); + }; + tokio::select! { + _ = writing => {}, + _ = &mut decoding => panic!("incomplete decoder unexpectedly finished"), + } + drop(decoding); + for _ in 0..100 { + if std::fs::read_dir(directory.path()).unwrap().count() == 0 { + return; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + panic!("cancelled upload retained a temporary file"); +} + +fn sources(app: &str, route: &str) -> dever_core::source::SourceMap { + let mut sources = dever_core::source::SourceMap::default(); + sources.add("media/image/app.dever", app); + sources.add("media/image/api.dever", route); + sources +} + +fn storage_sources() -> dever_core::source::SourceMap { + let mut sources = sources( + "accept(file: Upload) (key: Uuid) { key = port.store(file) }", + "post upload = app.accept", + ); + sources.add( + "media/image/port.dever", + "store(file: Upload) (key: Uuid) fails dever.storage.PutResult", + ); + sources.add( + "media/image/adapter.dever", + "port.store(file: Upload) (key: Uuid) { key = dever.storage.put(file) }", + ); + sources +} + +#[test] +fn upload_affinity_and_route_contract_are_checked() { + for (app, route) in [ + ( + "accept(file: Upload) (size: Int) { size = dever.api.upload_size(file) }", + "post upload = app.accept", + ), + ( + "accept(file: Upload) (size: Int) { dever.api.close_upload(file)\nsize = dever.api.upload_size(file) }", + "post upload = app.accept", + ), + ( + "accept(file: Upload) (size: Int) { size = dever.api.upload_size(file)\ndever.api.close_upload(file) }", + "get upload = app.accept", + ), + ( + "accept(file: Upload) (size: Int) { size = dever.api.upload_size(file)\ndever.api.close_upload(file) }", + "cmd upload = app.accept", + ), + ( + "type Payload { file: Upload }\naccept(file: Payload) (size: Int) { size = 1 }", + "post upload = app.accept", + ), + ( + "accept(file: Upload) (key: Uuid) { key = dever.storage.put(file) }", + "post upload = app.accept", + ), + ( + "accept(file: Upload) (size: Int) { size = helper(file) }\nhelper(file: Upload) (size: Int) { size = dever.api.upload_size(file)\ndever.api.close_upload(file) }", + "post upload = app.accept", + ), + ] { + assert!( + dever_core::check(&sources(app, route)).is_err(), + "unexpected accepted: {app}" + ); + } + let sources = sources( + "accept(file: Upload, caption: Text) (size: Int) { size = dever.api.upload_size(file)\nname = dever.api.upload_filename(file)\ndever.api.close_upload(file) }", + "post upload = app.accept", + ); + dever_core::check(&sources).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }); + let sources = storage_sources(); + dever_core::check(&sources).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }); +} + +#[test] +fn upload_route_native_compiles_with_owned_resource() { + use std::io::{Read, Write}; + let sources = storage_sources(); + let program = dever_core::check(&sources).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }); + let native = dever_core::native::compile_project( + &program, + &sources, + std::ffi::OsStr::new("/root/.cargo/bin/rustc"), + dever_runtime::config::RuntimeProfile::default(), + ) + .unwrap(); + assert!(native.executable().is_file()); + let directory = temp::TemporaryDirectory::new(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + drop(listener); + std::fs::create_dir(directory.path().join("config")).unwrap(); + std::fs::write( + directory.path().join("config/setting.json"), + format!(r#"{{"http":{{"host":"127.0.0.1","port":{port},"timeout_ms":1000}}}}"#), + ) + .unwrap(); + let executable = directory.path().join("upload-test"); + native.save(&executable).unwrap(); + let mut child = Child( + std::process::Command::new(executable) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(), + ); + let started = std::time::Instant::now(); + let mut peer = loop { + assert!( + started.elapsed() < std::time::Duration::from_secs(5), + "upload listener startup timeout" + ); + if let Some(status) = child.0.try_wait().unwrap() { + panic!("upload application exited: {status}"); + } + match std::net::TcpStream::connect(("127.0.0.1", port)) { + Ok(peer) => break peer, + Err(_) => std::thread::sleep(std::time::Duration::from_millis(10)), + } + }; + peer.set_read_timeout(Some(std::time::Duration::from_secs(3))) + .unwrap(); + let bytes = b"--boundary\r\nContent-Disposition: form-data; name=\"file\"; filename=\"hello.txt\"\r\n\r\nhello\r\n--boundary--\r\n"; + write!(peer, "POST /media/image/upload HTTP/1.1\r\nHost: localhost\r\nContent-Type: multipart/form-data; boundary=boundary\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", bytes.len()).unwrap(); + peer.write_all(bytes).unwrap(); + let mut response = String::new(); + peer.read_to_string(&mut response).unwrap(); + assert!(response.starts_with("HTTP/1.1 200"), "{response}"); + let envelope: serde_json::Value = + serde_json::from_str(response.split_once("\r\n\r\n").unwrap().1).unwrap(); + let key = envelope["data"].as_str().unwrap(); + assert!(dever_runtime::orm::Uuid::parse(key).is_ok()); + assert_eq!( + std::fs::read(directory.path().join("data/upload").join(key)).unwrap(), + b"hello" + ); + assert_eq!( + std::fs::read_dir(directory.path().join("data/tmp")) + .unwrap() + .count(), + 0 + ); +} + +struct Child(std::process::Child); +impl Drop for Child { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} diff --git a/test/ecosystem-release/arm64/bootstrap.sh b/test/ecosystem-release/arm64/bootstrap.sh new file mode 100644 index 0000000..e27f7b2 --- /dev/null +++ b/test/ecosystem-release/arm64/bootstrap.sh @@ -0,0 +1,8 @@ +#!/bin/sh +ip link set lo up +echo DEVER_ARM_GUEST_BEGIN +sh /payload/run.sh +result=$? +echo DEVER_ARM_GUEST_EXIT=$result +sync +poweroff -f diff --git a/test/ecosystem-release/arm64/guest.sh b/test/ecosystem-release/arm64/guest.sh new file mode 100644 index 0000000..4d33c3e --- /dev/null +++ b/test/ecosystem-release/arm64/guest.sh @@ -0,0 +1,65 @@ +#!/bin/sh +set -eu +uname -m +test "$(uname -m)" = aarch64 +test -e /proc/self/ns/user +echo ARM_KERNEL_READY +check_cleanup() { + for process in /proc/[0-9]*/exe; do + executable=$(readlink "$process" || true) + case "$executable" in + /worker/*|*/workers/*|*/sandbox/bin/*|*/sandbox/lib/*) + echo "ERROR: Worker process leaked: $executable" + exit 1 + ;; + esac + done +} +mkdir -p /tmp/apps +for ecosystem in pip npm go; do + cp -a "/payload/apps/$ecosystem" "/tmp/apps/$ecosystem" + program="/tmp/apps/$ecosystem/program" + if ! "$program" sample.worker.probe '{}' > "/tmp/$ecosystem-first.json" 2> "/tmp/$ecosystem-first.log"; then + cat "/tmp/$ecosystem-first.json" "/tmp/$ecosystem-first.log" + exit 1 + fi + grep -q '"data":true' "/tmp/$ecosystem-first.json" + check_cleanup + echo "ARM_WORKER_CALL_PASS=$ecosystem" + if "$program" sample.worker.reject '{}' > "/tmp/$ecosystem-reject.json" 2> "/tmp/$ecosystem-reject.log"; then + echo "ERROR: $ecosystem Worker failure unexpectedly succeeded" + exit 1 + fi + grep -Fq '"level":"error"' "/tmp/$ecosystem-reject.log" + grep -Fq 'sample/worker/app.dever:3:' "/tmp/$ecosystem-reject.log" + grep -Fq 'cannot read component protocol frame header: early eof' "/tmp/$ecosystem-reject.log" + check_cleanup + echo "ARM_WORKER_ERROR_PASS=$ecosystem" + "$program" sample.worker.probe '{}' > "/tmp/$ecosystem-repeat.json" + grep -q '"data":true' "/tmp/$ecosystem-repeat.json" + check_cleanup + cat "/tmp/$ecosystem-first.json" + echo "ARM_WORKER_PASS=$ecosystem" + # 只清理当前 guest 用例的可写副本和缓存；原产物保留在只读 payload。 + rm -r /tmp/apps + mkdir -p /tmp/apps +done +cp -a /payload/apps/http /tmp/apps/http +/tmp/apps/http/application > /tmp/http.log 2>&1 & +server=$! +trap 'kill "$server" 2>/dev/null || true; wait "$server" 2>/dev/null || true' EXIT +attempt=0 +until wget -q -O /tmp/http.json http://127.0.0.1:18080/sample/http/hello; do + attempt=$((attempt + 1)) + if test "$attempt" -ge 20; then + cat /tmp/http.log + exit 1 + fi + sleep 1 +done +grep -q '"data":"arm-http"' /tmp/http.json +cat /tmp/http.json +kill "$server" +wait "$server" +trap - EXIT +echo ARM_HTTP_PASS diff --git a/test/ecosystem-release/arm64/init.sh b/test/ecosystem-release/arm64/init.sh new file mode 100644 index 0000000..17bd6ba --- /dev/null +++ b/test/ecosystem-release/arm64/init.sh @@ -0,0 +1,27 @@ +#!/bin/busybox sh +set -eu +/bin/busybox --install -s /bin +mount -t proc proc /proc +mount -t sysfs sysfs /sys +mount -t devtmpfs devtmpfs /dev +mkdir -p /dev/pts /dev/shm /tmp /payload /run +mount -t devpts devpts /dev/pts +mount -t tmpfs tmpfs /tmp +mount -t tmpfs tmpfs /dev/shm +chmod 1777 /tmp /dev/shm +modprobe virtio_mmio +modprobe virtio_blk +modprobe ext4 +mount -o ro /dev/vda /payload +# initramfs 的 rootfs 不支持 pivot_root；先进入正常根挂载再验收嵌套沙箱。 +mkdir -p /newroot +mount -t tmpfs tmpfs /newroot +cp -a /bin /sbin /lib /etc /newroot/ +cp /init-stage /newroot/init-stage +mkdir -p /newroot/proc /newroot/sys /newroot/dev /newroot/tmp /newroot/payload /newroot/run +mount -o move /proc /newroot/proc +mount -o move /sys /newroot/sys +mount -o move /dev /newroot/dev +mount -o move /tmp /newroot/tmp +mount -o move /payload /newroot/payload +exec /bin/busybox switch_root /newroot /bin/sh /init-stage diff --git a/test/ecosystem-release/arm64/probe.go b/test/ecosystem-release/arm64/probe.go new file mode 100644 index 0000000..7905e8b --- /dev/null +++ b/test/ecosystem-release/arm64/probe.go @@ -0,0 +1,49 @@ +package main + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "errors" + "io" + "runtime" + + "github.com/google/uuid" +) + +func Probe(_ context.Context, _ any, _ any) (any, error) { + identity, err := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + if err != nil { + return nil, err + } + if runtime.GOARCH != "arm64" || identity.Version() != 1 { + return nil, errors.New("ARM target or locked UUID library differs") + } + message := []byte("ARM 独立依赖") + var encoded bytes.Buffer + writer := gzip.NewWriter(&encoded) + if _, err := writer.Write(message); err != nil { + return nil, err + } + if err := writer.Close(); err != nil { + return nil, err + } + reader, err := gzip.NewReader(&encoded) + if err != nil { + return nil, err + } + decoded, err := io.ReadAll(reader) + if err != nil { + return nil, err + } + if err := reader.Close(); err != nil { + return nil, err + } + digest := sha256.Sum256(message) + return map[string]any{"okay": bytes.Equal(decoded, message) && len(digest) == 32}, nil +} + +func Reject(_ context.Context, _ any, _ any) (any, error) { + return nil, errors.New("expected ARM Worker failure") +} diff --git a/test/ecosystem-release/arm64/probe.mjs b/test/ecosystem-release/arm64/probe.mjs new file mode 100644 index 0000000..38d2259 --- /dev/null +++ b/test/ecosystem-release/arm64/probe.mjs @@ -0,0 +1,27 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { createRequire } from 'node:module'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { gzipSync, gunzipSync } from 'node:zlib'; + +const require = createRequire(import.meta.url); +const isNumber = require('is-number'); + +export async function probe() { + assert.equal(process.arch, 'arm64'); + const bundle = resolve(dirname(fileURLToPath(import.meta.url)), '..'); + assert.equal(process.execPath, resolve(bundle, 'runtime/bin/node')); + assert.equal(require('is-number/package.json').version, '7.0.0'); + assert.equal(isNumber('42'), true); + assert.equal(isNumber('invalid'), false); + const message = Buffer.from('ARM 独立依赖'); + assert.deepEqual(gunzipSync(gzipSync(message)), message); + assert.equal(createHash('sha256').update(message).digest().length, 32); + await Promise.resolve(); + return { okay: true }; +} + +export async function reject() { + throw new Error('expected ARM Worker failure'); +} diff --git a/test/ecosystem-release/arm64/probe.py b/test/ecosystem-release/arm64/probe.py new file mode 100644 index 0000000..11caafd --- /dev/null +++ b/test/ecosystem-release/arm64/probe.py @@ -0,0 +1,31 @@ +import asyncio +import gzip +import hashlib +from pathlib import Path +import platform +import sqlite3 +import sys + +import six + + +async def probe(payload, setting): + assert platform.machine() == "aarch64" + assert six.__version__ == "1.17.0" and six.PY3 + assert list(six.iteritems({"value": 42})) == [("value", 42)] + runtime = Path(sys.executable).resolve().parents[1] + bundle = runtime.parent + assert Path(sys.prefix).resolve() == runtime + assert all(Path(path).resolve().is_relative_to(bundle) for path in sys.path) + assert Path(six.__file__).resolve().is_relative_to(bundle) + message = "ARM 独立依赖".encode() + assert gzip.decompress(gzip.compress(message)) == message + assert len(hashlib.sha256(message).digest()) == 32 + with sqlite3.connect(":memory:") as connection: + assert connection.execute("select 6 * 7").fetchone() == (42,) + await asyncio.sleep(0) + return {"okay": True} + + +async def reject(payload, setting): + raise RuntimeError("expected ARM Worker failure") diff --git a/test/ecosystem-release/arm64/simulate.py b/test/ecosystem-release/arm64/simulate.py new file mode 100644 index 0000000..e145901 --- /dev/null +++ b/test/ecosystem-release/arm64/simulate.py @@ -0,0 +1,88 @@ +"""用显式准备的私有 ARM 内核运行独立应用，不改主机服务或沙箱策略。""" +from pathlib import Path +import json +import shutil +import subprocess +import time + +SOURCE = Path(__file__).resolve().parent +ROOT = SOURCE.parents[2] / "target/arm64-cross" + + +def prepare_images(): + init = ROOT / "initramfs" + for source, destination in [ + ("init.sh", init / "init-arm"), + ("bootstrap.sh", init / "init-stage"), + ("guest.sh", ROOT / "guest/run.sh"), + ]: + shutil.copyfile(SOURCE / source, destination) + destination.chmod(0o755) + archive = ROOT / "guest-initramfs.gz" + with archive.open("wb") as output: + with subprocess.Popen( + ["/usr/bin/find", ".", "-print0"], cwd=init, stdout=subprocess.PIPE + ) as find: + try: + with subprocess.Popen( + ["/usr/bin/cpio", "--null", "-o", "--format=newc", "--quiet"], + cwd=init, stdin=find.stdout, stdout=subprocess.PIPE, + ) as cpio: + find.stdout.close() + try: + subprocess.run(["/usr/bin/gzip", "-1"], stdin=cpio.stdout, stdout=output, check=True) + finally: + cpio.stdout.close() + finally: + find.stdout.close() + if find.returncode or cpio.returncode: + raise RuntimeError("initramfs creation failed") + disk = ROOT / "guest.ext4" + with disk.open("wb") as output: + output.truncate(1024 * 1024 * 1024) + subprocess.run( + ["/usr/sbin/mke2fs", "-q", "-t", "ext4", "-F", "-d", str(ROOT / "guest"), str(disk)], + check=True, + ) + return archive, disk + + +def main(): + archive, disk = prepare_images() + command = [ + "/lib64/ld-linux-x86-64.so.2", "--library-path", + str(ROOT / "tools/usr/lib/x86_64-linux-gnu"), + str(ROOT / "tools/usr/bin/qemu-system-aarch64"), + "-machine", "virt", "-cpu", "cortex-a72", "-smp", "2", "-m", "768", + "-nographic", "-no-reboot", "-nic", "none", + "-kernel", str(ROOT / "iso/boot/vmlinuz-virt"), "-initrd", str(archive), + "-append", "console=ttyAMA0 rdinit=/init-arm panic=-1", + "-drive", f"file={disk},format=raw,if=none,readonly=on,id=payload", + "-device", "virtio-blk-device,drive=payload", + ] + log = ROOT / "logs" / f"guest-{time.time_ns()}.log" + started = time.monotonic() + timed_out = False + with log.open("xb") as output: + try: + result = subprocess.run(command, stdout=output, stderr=subprocess.STDOUT, timeout=600) + exit_code = result.returncode + except subprocess.TimeoutExpired: + timed_out = True + exit_code = None + transcript = log.read_text(errors="replace") + required = ["DEVER_ARM_GUEST_EXIT=0", "ARM_HTTP_PASS"] + required.extend(f"ARM_WORKER_PASS={ecosystem}" for ecosystem in ("pip", "npm", "go")) + passed = exit_code == 0 and all(marker in transcript for marker in required) + record = { + "command": command, "exit_code": exit_code, "timed_out": timed_out, "passed": passed, + "seconds": time.monotonic() - started, "log": str(log), + } + log.with_suffix(".json").write_text(json.dumps(record, indent=2) + "\n") + print(json.dumps(record), flush=True) + if not passed: + raise RuntimeError("ARM guest acceptance failed; see serial log") + + +if __name__ == "__main__": + main() diff --git a/test/ecosystem-release/build_inputs.py b/test/ecosystem-release/build_inputs.py new file mode 100644 index 0000000..7e734d9 --- /dev/null +++ b/test/ecosystem-release/build_inputs.py @@ -0,0 +1,142 @@ +"""Explicit Linux author SDK leaves for offline Python/npm build acceptance. + +This prepares test inputs, not a trusted public distribution. System files are +declared here and hashed; fixed Python/Node headers and npm come from the same +verified upstream archives used by prepare.py. No PATH lookup or downloads. +""" + +import argparse +import json +import os +from pathlib import Path +import re +import shutil + +from prepare import SOURCES, digest, materialize + + +TARGET = "linux-x86_64" +MULTIARCH = "x86_64-linux-gnu" +GCC_VERSION = "13" +NPM_NODE_GYP_SHIM = "npm/frontend/node_modules/npm/node_modules/@npmcli/run-script/lib/node-gyp-bin/node-gyp" +TOOLS = { + "sh": "dash", "cc": "x86_64-linux-gnu-gcc-13", "gcc": "x86_64-linux-gnu-gcc-13", + "c++": "x86_64-linux-gnu-g++-13", "g++": "x86_64-linux-gnu-g++-13", + "as": "x86_64-linux-gnu-as", "ld": "x86_64-linux-gnu-ld.bfd", + "ar": "x86_64-linux-gnu-ar", "ranlib": "x86_64-linux-gnu-ranlib", + "nm": "x86_64-linux-gnu-nm", "strip": "x86_64-linux-gnu-strip", + **{name: name for name in ( + "make", "env", "basename", "dirname", "cat", "cp", "mv", "rm", "mkdir", + "rmdir", "ln", "touch", "chmod", "pwd", "printf", "test", "true", "false", + "sed", "grep", "find", "sort", "tr", "cut", "head", "tail", "wc", "uname", + )}, +} +LIBRARIES = ( + "libc.so.6", "libm.so.6", "libdl.so.2", "libpthread.so.0", "librt.so.1", + "libgcc_s.so.1", "libstdc++.so.6", "libisl.so.23", "libmpc.so.3", + "libmpfr.so.6", "libgmp.so.10", "libz.so.1", "libzstd.so.1", + "libbfd-2.42-system.so", "libctf.so.0", "libjansson.so.4", + "libsframe.so.1", "libpcre2-8.so.0", "libselinux.so.1", "libacl.so.1", + "libattr.so.1", "libtinfo.so.6", "libcrypto.so.3", "libssl.so.3", + "libc.so", "libc_nonshared.a", "libm.so", "libm-2.39.a", "libmvec.so.1", + "libpthread.a", "libdl.a", "librt.a", "libutil.a", + "crt1.o", "Scrt1.o", "crti.o", "crtn.o", +) + + +def signed_library_paths(content): + # 仅迁移以 /lib 开头的绝对 token，不能再次改写已有的 /usr/lib。 + return re.sub(r"(?<!\S)/lib/" + re.escape(MULTIARCH) + "/", + f"/usr/lib/{MULTIARCH}/", content) + + +def sdk_leaf(source, destination, changes): + original = source + source = source.resolve(strict=True) + if not source.is_file() or not source.is_relative_to("/usr"): + raise ValueError(f"SDK leaf must resolve to an explicit /usr file: {original}") + destination.parent.mkdir(parents=True, exist_ok=True) + if original.name in ("libc.so", "libm.so", "libgcc_s.so"): + content = source.read_text() + rewritten = signed_library_paths(content) + if rewritten != content: + destination.write_text(rewritten) + changes.append({"source": str(original), "source_sha256": digest(source), + "transform": "linker script uses signed /usr/lib multiarch leaves"}) + return + # These are read-only author inputs. Never chmod a hardlink to a host file. + os.link(source, destination) + + +def sdk_tree(source, destination, changes): + for current, directories, names in os.walk(source, followlinks=False): + current = Path(current) + if any((current / name).is_symlink() for name in directories): + raise ValueError(f"unexpected directory alias in explicit SDK input: {current}") + for name in sorted(names): + sdk_leaf(current / name, destination / (current / name).relative_to(source), changes) + + +def leaves(root, sources): + entries = [] + executables = [] + for source, prefix in sources: + for path in sorted(source.rglob("*")): + if not path.is_file(): + continue + name = (Path(prefix) / path.relative_to(source)).as_posix() + entries.append({"source": path.relative_to(root).as_posix(), + "path": name, "sha256": digest(path)}) + if path.stat().st_mode & 0o111: + executables.append(name) + if len(entries) > 65_536 or sum((root / entry["source"]).stat().st_size for entry in entries) > 512 * 1024 * 1024: + raise ValueError("author SDK exceeds the build-only pack budget") + return {"files": entries, "executables": executables} + + +def prepare(output): + output.mkdir(parents=True, exist_ok=False) + changes = [] + try: + rootfs = output / "rootfs" + for name, source in TOOLS.items(): + sdk_leaf(Path("/usr/bin") / source, rootfs / "usr/bin" / name, changes) + for source in (Path("/usr/include"), + Path(f"/usr/lib/gcc/{MULTIARCH}/{GCC_VERSION}"), + Path(f"/usr/libexec/gcc/{MULTIARCH}/{GCC_VERSION}")): + sdk_tree(source, rootfs / source.relative_to("/"), changes) + for name in LIBRARIES: + source = Path("/usr/lib") / MULTIARCH / name + sdk_leaf(source, rootfs / source.relative_to("/"), changes) + python = output / "python" + node = output / "node" + materialize("pip", python, selector=lambda name: name.startswith("include/")) + materialize("npm", node, selector=lambda name: name.startswith(("include/node/", "lib/node_modules/npm/"))) + npm = leaves(output, ((rootfs, "rootfs"), (node / "include/node", "npm/include/node"), + (node / "lib/node_modules/npm", "npm/frontend/node_modules/npm"))) + shim = next(entry for entry in npm["files"] if entry["path"] == NPM_NODE_GYP_SHIM) + if not (output / shim["source"]).read_bytes().startswith(b"#!/usr/bin/env sh\n"): + raise ValueError("pinned npm node-gyp shim changed its interpreter") + # run-script executes this upstream shell entry; the remaining frontend + # JavaScript is loaded by the exact packaged Node interpreter. + npm["executables"] = sorted(set(npm["executables"]) | {NPM_NODE_GYP_SHIM}) + manifest = { + "format": "dever-build-author-leaves-v1", "target": TARGET, + "provenance": {"kind": "private Ubuntu GNU author SDK", + "gcc": GCC_VERSION, "system_linker_script_changes": changes, + "upstream_archives": {name: {"path": str(value[0]), "sha256": value[1]} + for name, value in SOURCES.items() if name != "go"}}, + "pip": leaves(output, ((rootfs, "rootfs"), (python / "include", "runtime/include"))), + "npm": npm, + } + (output / "inputs.json").write_text(json.dumps(manifest, indent=2) + "\n") + except BaseException: + shutil.rmtree(output) + raise + print(output / "inputs.json") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output", type=Path) + prepare(parser.parse_args().output.absolute()) diff --git a/test/ecosystem-release/prepare.py b/test/ecosystem-release/prepare.py new file mode 100644 index 0000000..f3bad9a --- /dev/null +++ b/test/ecosystem-release/prepare.py @@ -0,0 +1,243 @@ +"""Prepare pinned, offline upstream inputs for the explicit release acceptance. + +This test author tool never discovers interpreters or downloads anything. It +materializes selected distribution files and resolves only in-archive aliases. +The production release maker subsequently verifies every prepared input again. +""" + +import argparse +import gzip +import hashlib +import json +import posixpath +import shutil +import tarfile +from pathlib import Path, PurePosixPath + + +WORKSPACE = Path(__file__).resolve().parents[2] +INPUTS = WORKSPACE / "target/ecosystem-release-inputs" +SOURCES = { + "pip": ( + INPUTS / "downloads/cpython-3.12.14.tar.gz", + "72748da13197c1fb161e3afeef20a6a385ff24f2165e6e2758e47008e7faba4c", + "python/", + ), + "npm": ( + INPUTS / "downloads/node-v24.15.0-linux-x64.tar.xz", + "472655581fb851559730c48763e0c9d3bc25975c59d518003fc0849d3e4ba0f6", + "node-v24.15.0-linux-x64/", + ), + "go": ( + WORKSPACE / "target/go-managed/runtime.pack", + "8818a1eccb510948a55f63088c01678391a5e0c2236e6eacadfe45d0a5edbb43", + "", + ), +} + + +def digest(path): + with path.open("rb") as source: + return hashlib.file_digest(source, "sha256").hexdigest() + + +def relative_path(name): + path = PurePosixPath(name) + if path.is_absolute() or path.as_posix() != name or ".." in path.parts: + raise ValueError(f"unsafe upstream archive path: {name}") + return name + + +def selected(ecosystem, path): + if ecosystem == "npm": + return path in ("bin/node", "LICENSE") + if ecosystem == "pip": + # Keep the complete standard library and its native/data dependencies. + # pip, user site-packages, development headers and command aliases are + # not needed by the isolated runtime (-I -S -B). + return path == "bin/python3.12" or ( + path.startswith(("lib/", "share/")) + and not path.startswith("lib/python3.12/site-packages/") + ) + return True + + +def materialize(ecosystem, destination, *, selector=None, source_descriptor=None): + source, expected, prefix = source_descriptor or SOURCES[ecosystem] + if digest(source) != expected: + raise ValueError(f"{ecosystem} upstream input digest differs") + aliases = {} + seen = set() + with tarfile.open(source, "r:*") as archive: + for member in archive: + name = relative_path(member.name.rstrip("/")) + if not name.startswith(prefix) or member.isdir(): + continue + path = name[len(prefix):] + if not (selector(path) if selector is not None else selected(ecosystem, path)): + continue + if path in seen: + raise ValueError(f"duplicate upstream path: {path}") + seen.add(path) + output = destination / path + output.parent.mkdir(parents=True, exist_ok=True) + if member.issym(): + target = posixpath.normpath(posixpath.join(posixpath.dirname(path), member.linkname)) + aliases[path] = relative_path(target) + elif member.isfile(): + with archive.extractfile(member) as payload, output.open("xb") as target: + shutil.copyfileobj(payload, target) + output.chmod(0o644) + else: + raise ValueError(f"unsupported upstream entry: {path}") + while aliases: + resolved = [] + for path, target in aliases.items(): + if (destination / target).is_file(): + shutil.copyfile(destination / target, destination / path) + resolved.append(path) + if not resolved: + raise ValueError("upstream aliases escape the selected tree or contain a cycle") + for path in resolved: + del aliases[path] + return {"sha256": expected, "bytes": source.stat().st_size} + + +def runtime_settings(root, ecosystem, *, target="linux-x86_64"): + architecture = {"linux-x86_64": "x86_64", "linux-aarch64": "aarch64"}[target] + destination = root / ecosystem + versions = {"pip": ("cpython", "3.12.14"), "npm": ("node", "24.15.0"), "go": ("go", "1.26.3")} + name, version = versions[ecosystem] + python_markers = { + "implementation_name": "cpython", "implementation_version": version, + "os_name": "posix", "platform_machine": architecture, + "platform_python_implementation": "CPython", "platform_release": "", + "platform_system": "Linux", "platform_version": "", + "python_full_version": version, "python_version": "3.12", "sys_platform": "linux", + } if ecosystem == "pip" else None + # This fixed CPython 3.12 author fixture uses a conservative glibc 2.36 + # compatibility ceiling. Its separately signed Ubuntu OS ABI pack currently + # supplies glibc 2.39; admission checks actual ELF version requirements too. + oldest_glibc = 4 if architecture == "x86_64" else 16 + platforms = [f"manylinux_2_{minor}_{architecture}" for minor in range(36, oldest_glibc, -1)] + platforms += [f"manylinux2014_{architecture}"] + if architecture == "x86_64": + platforms += ["manylinux2010_x86_64", "manylinux1_x86_64"] + platforms += [f"linux_{architecture}"] + python_tags = [f"cp312-cp312-{platform}" for platform in platforms] + python_tags += [f"cp3{minor}-abi3-{platform}" for minor in range(12, 1, -1) for platform in platforms] + python_tags += [f"py3-none-{platform}" for platform in platforms] + python_tags += ["py312-none-any", "py3-none-any"] + if ecosystem != "go": + manifest = { + "format": "dever-worker-runtime-v1", + "ecosystem": ecosystem, + "target": target, + "executable": "bin/python3.12" if ecosystem == "pip" else "bin/node", + "arguments": ["-I", "-S", "-B"] if ecosystem == "pip" else [], + } + if ecosystem == "pip": + manifest["python_wheel_tags"] = python_tags + manifest["python_extension_suffixes"] = [f".cpython-312-{architecture}-linux-gnu.so", ".abi3.so", ".so"] + manifest["python_markers"] = python_markers + (destination / "dever-runtime.json").write_text(json.dumps(manifest), encoding="utf-8") + files = [ + {"source": path.relative_to(root).as_posix(), "path": path.relative_to(destination).as_posix(), "sha256": digest(path)} + for path in sorted(destination.rglob("*")) if path.is_file() + ] + settings = {"name": name, "version": version, "files": files} + if ecosystem == "npm": + settings["npm_libc"] = "glibc" + if ecosystem == "pip": + settings["python_markers"] = python_markers + settings["python_wheel_tags"] = python_tags + return settings + + +def worker_runtime(root, output, ecosystem): + """Make one explicit fixture pack from verified leaves, without another tree copy.""" + settings = json.loads((root / "config/setting.json").read_text(encoding="utf-8"))["runtimes"][ecosystem] + inputs = sorted(settings["files"], key=lambda entry: entry["path"]) + for entry in inputs: + source = root / relative_path(entry["source"]) + if source.is_symlink() or not source.is_file() or digest(source) != entry["sha256"]: + raise ValueError(f"existing {ecosystem} author input changed: {entry['source']}") + descriptor = next(entry for entry in inputs if entry["path"] == "dever-runtime.json") + manifest = json.loads((root / descriptor["source"]).read_text(encoding="utf-8")) + if manifest["ecosystem"] != ecosystem: + raise ValueError("runtime ecosystem differs from the selected author input") + if ecosystem == "pip" and (manifest["python_wheel_tags"] != settings["python_wheel_tags"] or manifest["python_markers"] != settings["python_markers"]): + raise ValueError("Python runtime and registry descriptors disagree") + output.mkdir(parents=True, exist_ok=False) + try: + pack = output / "runtime.pack" + with pack.open("xb") as target, gzip.GzipFile(filename="", mode="wb", fileobj=target, mtime=0) as compressed, tarfile.open(fileobj=compressed, mode="w|") as archive: + for entry in inputs: + source = root / entry["source"] + info = tarfile.TarInfo(relative_path(entry["path"])) + info.size = source.stat().st_size + info.mode = 0o755 if entry["path"] == manifest["executable"] else 0o644 + with source.open("rb") as payload: + archive.addfile(info, payload) + if digest(source) != entry["sha256"]: + raise ValueError(f"{ecosystem} author input changed during archive creation: {entry['source']}") + runtime = { + "pack": {"name": settings["name"], "version": settings["version"], "sha256": digest(pack)}, + "target": manifest["target"], + "python_markers": None, + "python_wheel_tags": [], + } + metadata = ("python_markers", "python_wheel_tags") if ecosystem == "pip" else ("npm_libc",) + runtime.update({field: settings[field] for field in metadata}) + (output / "config").mkdir() + (output / "config/setting.json").write_text(json.dumps({"runtime": runtime}, indent=2) + "\n", encoding="utf-8") + except BaseException: + shutil.rmtree(output) + raise + print(output) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output", type=Path) + action = parser.add_mutually_exclusive_group() + action.add_argument("--refresh", action="store_true", help="refresh descriptors of existing verified inputs without extracting a second copy") + action.add_argument("--python-worker", type=Path, help="write a fresh explicit Python Worker fixture pack from already prepared inputs") + action.add_argument("--npm-worker", type=Path, help="write a fresh explicit Node Worker fixture pack from already prepared inputs") + arguments = parser.parse_args() + root = arguments.output.absolute() + for ecosystem, destination in (("pip", arguments.python_worker), ("npm", arguments.npm_worker)): + if destination is not None: + worker_runtime(root, destination.absolute(), ecosystem) + return + if arguments.refresh: + setting_path = root / "config/setting.json" + settings = json.loads(setting_path.read_text(encoding="utf-8")) + for runtime in settings["runtimes"].values(): + for entry in runtime["files"]: + if digest(root / relative_path(entry["source"])) != entry["sha256"]: + raise ValueError(f"existing author input changed: {entry['source']}") + settings["runtimes"] = {ecosystem: runtime_settings(root, ecosystem) for ecosystem in SOURCES} + setting_path.write_text(json.dumps(settings, indent=2) + "\n", encoding="utf-8") + print(root) + return + root.mkdir(parents=True, exist_ok=False) + try: + provenance = {} + runtimes = {} + for ecosystem in SOURCES: + destination = root / ecosystem + destination.mkdir() + provenance[ecosystem] = materialize(ecosystem, destination) + runtimes[ecosystem] = runtime_settings(root, ecosystem) + (root / "config").mkdir() + (root / "config/setting.json").write_text(json.dumps({"runtimes": runtimes}, indent=2) + "\n", encoding="utf-8") + (root / "provenance.json").write_text(json.dumps(provenance, indent=2) + "\n", encoding="utf-8") + except BaseException: + shutil.rmtree(root) + raise + print(root) + + +if __name__ == "__main__": + main() diff --git a/test/ecosystem-release/probe.go b/test/ecosystem-release/probe.go new file mode 100644 index 0000000..68fd6e5 --- /dev/null +++ b/test/ecosystem-release/probe.go @@ -0,0 +1,40 @@ +package main + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "errors" + "io" + "math/big" +) + +func Probe(_ context.Context, _ any, _ any) (any, error) { + message := []byte("独立标准库") + var encoded bytes.Buffer + writer := gzip.NewWriter(&encoded) + if _, err := writer.Write(message); err != nil { + return nil, err + } + if err := writer.Close(); err != nil { + return nil, err + } + reader, err := gzip.NewReader(&encoded) + if err != nil { + return nil, err + } + decoded, err := io.ReadAll(reader) + if err != nil { + return nil, err + } + if err := reader.Close(); err != nil { + return nil, err + } + value, okay := new(big.Int).SetString("9007199254740993", 10) + if !okay || value.Add(value, big.NewInt(1)).String() != "9007199254740994" || !bytes.Equal(decoded, message) { + return nil, errors.New("packaged standard library result differs") + } + digest := sha256.Sum256(message) + return map[string]any{"okay": len(digest) == 32}, nil +} diff --git a/test/ecosystem-release/probe.mjs b/test/ecosystem-release/probe.mjs new file mode 100644 index 0000000..2645f1c --- /dev/null +++ b/test/ecosystem-release/probe.mjs @@ -0,0 +1,25 @@ +import { createHash } from 'node:crypto'; +import { gzipSync, gunzipSync } from 'node:zlib'; +import assert from 'node:assert/strict'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { createRequire } from 'node:module'; + +const addon = createRequire(import.meta.url)('bufferutil/build/Release/bufferutil.node'); + +export async function probe() { + const bundle = resolve(dirname(fileURLToPath(import.meta.url)), '..'); + assert.equal(process.execPath, resolve(bundle, 'runtime/bin/node')); + const message = Buffer.from('独立标准库'); + const mask = Buffer.from([1, 2, 3, 4]); + const masked = Buffer.alloc(message.length); + addon.mask(message, mask, masked, 0, message.length); + assert.notDeepEqual(masked, message); + addon.unmask(masked, mask); + assert.deepEqual(masked, message); + assert.deepEqual(gunzipSync(gzipSync(message)), message); + assert.equal(createHash('sha256').update(message).digest().length, 32); + assert.equal(9007199254740993n + 1n, 9007199254740994n); + await Promise.resolve(); + return { okay: true }; +} diff --git a/test/ecosystem-release/probe.py b/test/ecosystem-release/probe.py new file mode 100644 index 0000000..14418d7 --- /dev/null +++ b/test/ecosystem-release/probe.py @@ -0,0 +1,38 @@ +import asyncio +import bz2 +import ctypes +import decimal +import gzip +import hashlib +import lzma +from pathlib import Path +import sqlite3 +import ssl +import sys +import simplejson +from simplejson import _speedups + + +async def probe(payload, setting): + assert _speedups.__file__.endswith(".so") + assert _speedups.scanstring('"native"', 1) == ("native", 8) + assert simplejson.loads(simplejson.dumps({"native": 42})) == {"native": 42} + runtime = Path(sys.executable).resolve().parents[1] + bundle = runtime.parent + assert Path(sys.prefix).resolve() == runtime + assert all(Path(path).resolve().is_relative_to(bundle) for path in sys.path) + for module in tuple(sys.modules.values()): + path = getattr(module, "__file__", None) + if path and not path.startswith("<"): + assert Path(path).resolve().is_relative_to(bundle), path + await asyncio.sleep(0) + message = "独立标准库".encode() + for compress, decompress in ((gzip.compress, gzip.decompress), (bz2.compress, bz2.decompress), (lzma.compress, lzma.decompress)): + assert decompress(compress(message)) == message + assert len(hashlib.sha256(message).digest()) == 32 + assert ssl.OPENSSL_VERSION.startswith("OpenSSL ") + assert ctypes.sizeof(ctypes.c_uint64) == 8 + assert decimal.Decimal("0.1") + decimal.Decimal("0.2") == decimal.Decimal("0.3") + with sqlite3.connect(":memory:") as connection: + assert connection.execute("select 6 * 7").fetchone() == (42,) + return {"okay": True} diff --git a/test/ecosystem-release/test_build_inputs.py b/test/ecosystem-release/test_build_inputs.py new file mode 100644 index 0000000..e5f0630 --- /dev/null +++ b/test/ecosystem-release/test_build_inputs.py @@ -0,0 +1,23 @@ +"""作者工具的 GNU 链接脚本迁移合同；不读取或修改宿主工具。""" + +import unittest + +from build_inputs import MULTIARCH, signed_library_paths + + +class LinkerScriptTests(unittest.TestCase): + def test_rewrites_only_root_library_tokens(self): + original = (f"GROUP ( /lib/{MULTIARCH}/libc.so.6 " + f"/usr/lib/{MULTIARCH}/libc_nonshared.a " + "AS_NEEDED ( /lib64/ld-linux-x86-64.so.2 ) )") + expected = original.replace(f"( /lib/{MULTIARCH}/", f"( /usr/lib/{MULTIARCH}/") + self.assertEqual(signed_library_paths(original), expected) + self.assertEqual(signed_library_paths(expected), expected) + + def test_keeps_relative_and_other_target_paths(self): + original = "GROUP ( libgcc_s.so.1 -lgcc /lib/other-target/libc.so )" + self.assertEqual(signed_library_paths(original), original) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/external-sdk/check.py b/test/external-sdk/check.py new file mode 100644 index 0000000..d820083 --- /dev/null +++ b/test/external-sdk/check.py @@ -0,0 +1,282 @@ +"""Opt-in host SDK contract check. Pass only absolute runtime paths.""" + +import argparse +import json +from pathlib import Path +import struct +import subprocess +import tempfile +import threading +import time + + +ROOT = Path(__file__).resolve().parents[2] +HELLO = { + "kind": "hello", "version": "dever-component-1", "port": "example.Text", + "schema": "fixture-schema-v1", "adapter": "example.TextAdapter", + "capabilities": [], "operations": ["text.render", "text.fail", "number.echo", "text.stubborn"], + "setting": {"prefix": "hello "}, +} + + +def frame(message): + body = json.dumps(message, separators=(",", ":")).encode() + return struct.pack(">I", len(body)) + body + + +def read_exact(stream, size): + result = bytearray() + while len(result) < size: + part = stream.read(size - len(result)) + if not part: + raise AssertionError("worker closed protocol stream") + result.extend(part) + return bytes(result) + + +def receive(process): + deadline = threading.Timer(3, process.kill) + deadline.start() + try: + size = struct.unpack(">I", read_exact(process.stdout, 4))[0] + assert 0 < size <= 16 * 1024 * 1024 + return json.loads(read_exact(process.stdout, size)) + finally: + deadline.cancel() + + +def send(process, message): + process.stdin.write(frame(message)) + process.stdin.flush() + + +def run_case(command, *, keep_input_open=False): + with subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=0) as process: + try: + send(process, HELLO) + ready = receive(process) + assert ready == {key: value for key, value in HELLO.items() if key != "setting"} | {"kind": "ready"}, ready + send(process, {"kind": "health", "id": 0}) + assert receive(process) == {"kind": "health", "id": 0} + send(process, {"kind": "call", "id": 1, "operation": "text.render", "payload": {"text": "world", "delay_ms": 0}}) + assert receive(process) == {"kind": "result", "id": 1, "payload": {"value": "hello world"}} + send(process, {"kind": "call", "id": 2, "operation": "text.fail", "payload": {"text": "x", "delay_ms": 0}}) + assert receive(process) == {"kind": "error", "id": 2, "error": "example.rejected", "payload": {"reason": "rejected"}} + send(process, {"kind": "call", "id": 3, "operation": "text.render", "payload": {"text": "late", "delay_ms": 500}}) + send(process, {"kind": "cancel", "id": 3}) + assert receive(process) == {"kind": "error", "id": 3, "error": "dever.cancelled", "payload": None} + time.sleep(0.6) + send(process, {"kind": "call", "id": 4, "operation": "text.render", "payload": {"text": "again", "delay_ms": 0}}) + assert receive(process) == {"kind": "result", "id": 4, "payload": {"value": "hello again"}} + for request_id, number in enumerate((9007199254740993, 9223372036854775807, -9223372036854775808), 5): + send(process, {"kind": "call", "id": request_id, "operation": "number.echo", "payload": {"number": number}}) + assert receive(process) == {"kind": "result", "id": request_id, "payload": {"number": number}} + for request_id in (8, 9): + send(process, {"kind": "call", "id": request_id, "operation": "text.render", "payload": {"text": "cancel", "delay_ms": 500}}) + send(process, {"kind": "cancel", "id": request_id}) + assert receive(process) == {"kind": "error", "id": request_id, "error": "dever.cancelled", "payload": None} + send(process, {"kind": "shutdown"}) + assert receive(process) == {"kind": "shutdown"} + # Python must finalize its reader without relying on the host's EOF timing. + if not keep_input_open: + process.stdin.close() + assert process.wait(timeout=3) == 0, process.stderr.read().decode() + except Exception as error: + if process.poll() is None: + process.kill() + process.wait(timeout=3) + raise AssertionError(f"worker case failed: {error}; stderr={process.stderr.read().decode()}") from error + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=3) + + +def run_queued_shutdown(command): + with subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=0) as process: + try: + send(process, HELLO) + assert receive(process)["kind"] == "ready" + messages = [ + {"kind": "call", "id": 1, "operation": "text.render", "payload": {"text": "cancel", "delay_ms": 500}}, + {"kind": "shutdown"}, + *[{"kind": "health", "id": 0} for _ in range(4)], + ] + process.stdin.write(b"".join(frame(message) for message in messages)) + process.stdin.flush() + assert receive(process) == {"kind": "shutdown"} + assert process.wait(timeout=3) == 0, process.stderr.read().decode() + assert process.stderr.read() == b"", "shutdown left task/transport diagnostics" + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=3) + + +def run_typed_case(command, *, keep_input_open=False): + hello = { + "kind": "hello", "version": "dever-component-1", "port": "example.Typed", + "schema": "typed-fixture-v1", "adapter": "example.Managed", + "capabilities": [], "operations": ["render"], "setting": {"prefix": "hello "}, + } + with subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=0) as process: + try: + send(process, hello) + assert receive(process)["kind"] == "ready" + message = {"name": "Ada", "number": 9223372036854775807, "labels": ["a", "b"]} + send(process, {"kind": "call", "id": 1, "operation": "render", "payload": {"message": message}}) + assert receive(process) == {"kind": "result", "id": 1, "payload": {"text": "hello Ada9223372036854775807"}} + message["name"] = "reject" + send(process, {"kind": "call", "id": 2, "operation": "render", "payload": {"message": message}}) + assert receive(process) == {"kind": "error", "id": 2, "error": "example.Rejected", "payload": {"reason": "rejected"}} + send(process, {"kind": "shutdown"}) + assert receive(process) == {"kind": "shutdown"} + if not keep_input_open: + process.stdin.close() + assert process.wait(timeout=3) == 0, process.stderr.read().decode() + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=3) + + with subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=0) as process: + try: + send(process, hello) + assert receive(process)["kind"] == "ready" + send(process, {"kind": "call", "id": 1, "operation": "render", "payload": { + "message": {"name": "Ada", "number": 9223372036854775808, "labels": []}, + }}) + process.stdin.close() + assert process.wait(timeout=3) != 0 + assert process.stdout.read() == b"" + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=3) + + +def run_rejection(command, body): + with subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as process: + try: + process.stdin.write(struct.pack(">I", len(body)) + body) + process.stdin.flush() + process.stdin.close() + assert process.wait(timeout=3) != 0 + assert process.stdout.read() == b"" + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=3) + + +def run_protocol_rejection(command, body, declared_size=None): + with subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=0) as process: + try: + send(process, HELLO) + assert receive(process)["kind"] == "ready" + process.stdin.write(struct.pack(">I", len(body) if declared_size is None else declared_size) + body) + process.stdin.flush() + process.stdin.close() + assert process.wait(timeout=3) != 0 + assert process.stdout.read() == b"" + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=3) + + +def run_unresponsive_handler(command, final_message): + with subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=0) as process: + try: + send(process, HELLO) + assert receive(process)["kind"] == "ready" + send(process, {"kind": "call", "id": 1, "operation": "text.stubborn", "payload": {"text": "late", "delay_ms": 0}}) + time.sleep(0.1) + send(process, final_message) + if final_message["kind"] == "cancel": + send(process, {"kind": "call", "id": 2, "operation": "text.render", "payload": {"text": "new", "delay_ms": 0}}) + send(process, {"kind": "cancel", "id": 2}) + send(process, {"kind": "shutdown"}) + assert process.wait(timeout=3) != 0 + assert process.stdout.read() == b"" + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=3) + + +def runtime(path): + if path is None: + return None + executable = Path(path) + if not executable.is_absolute() or not executable.is_file(): + raise SystemExit(f"runtime path must be an existing absolute file: {path}") + return str(executable) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--python") + parser.add_argument("--node") + parser.add_argument("--go") + args = parser.parse_args() + with tempfile.TemporaryDirectory(prefix="dever-sdk-check-") as directory: + selected = { + "Python": [runtime(args.python), "-B", str(ROOT / "sdk/python/example.py")] if args.python else None, + "JavaScript": [runtime(args.node), str(ROOT / "sdk/javascript/example.mjs")] if args.node else None, + } + if args.go: + go = runtime(args.go) + binary = str(Path(directory) / "go-worker") + build_environment = { + "GOCACHE": str(Path(directory) / "go-cache"), + "GOPATH": str(Path(directory) / "go-path"), + "GOENV": "off", "GOTOOLCHAIN": "local", "GOPROXY": "off", "GOSUMDB": "off", + } + subprocess.run([go, "build", "-o", binary, "./example"], cwd=ROOT / "sdk/go", env=build_environment, check=True) + selected["Go"] = [binary] + else: + selected["Go"] = None + manifest = str(ROOT / "test/external-sdk/typed_contract.json") + typed = { + "Python": [runtime(args.python), "-B", str(ROOT / "test/external-sdk/typed_worker.py"), manifest] if args.python else None, + "JavaScript": [runtime(args.node), str(ROOT / "test/external-sdk/typed_worker.mjs"), manifest] if args.node else None, + } + if args.go: + typed_binary = str(Path(directory) / "go-typed-worker") + subprocess.run([go, "build", "-o", typed_binary, str(ROOT / "test/external-sdk/typed_worker.go")], cwd=ROOT / "sdk/go", env=build_environment, check=True) + typed["Go"] = [typed_binary, manifest] + else: + typed["Go"] = None + for language, command in selected.items(): + if command is None: + print(f"{language}: unavailable (no absolute runtime path supplied)") + continue + run_case(command, keep_input_open=language == "Python") + if language == "Python": + run_queued_shutdown(command) + run_rejection(command, json.dumps({**HELLO, "schema": "wrong"}).encode()) + run_rejection(command, (json.dumps(HELLO)[:-1] + ',"schema":"duplicate"}').encode()) + run_rejection(command, (json.dumps(HELLO)[:-1] + ',"extra":true}').encode()) + run_rejection(command, b"\xff") + call = {"kind": "call", "id": 1, "operation": "text.render", "payload": {"text": "x", "delay_ms": 0}} + run_protocol_rejection(command, json.dumps({**call, "id": 2}).encode()) + run_protocol_rejection(command, json.dumps({**call, "operation": "unknown"}).encode()) + run_protocol_rejection(command, (json.dumps(call)[:-2] + ',"text":"duplicate"}}').encode()) + run_protocol_rejection(command, (json.dumps(call)[:-1] + ',"extra":true}').encode()) + run_protocol_rejection(command, b'{"kind":"call","id":1,"operation":"text.render","payload":NaN}') + run_protocol_rejection(command, b'{"kind":"call","id":1,"operation":"text.render","payload":1e999}') + nested = 0 + for _ in range(64): + nested = [nested] + run_protocol_rejection(command, json.dumps({**call, "payload": nested}).encode()) + run_protocol_rejection(command, json.dumps({**call, "payload": [0] * 65_537}).encode()) + run_protocol_rejection(command, b"", declared_size=16 * 1024 * 1024 + 1) + run_unresponsive_handler(command, {"kind": "cancel", "id": 1}) + run_unresponsive_handler(command, {"kind": "shutdown"}) + run_typed_case(typed[language], keep_input_open=language == "Python") + print(f"{language}: handshake, health, result, business error, cancel, shutdown and rejection passed") + + +if __name__ == "__main__": + main() diff --git a/test/external-sdk/typed_contract.json b/test/external-sdk/typed_contract.json new file mode 100644 index 0000000..12c1e69 --- /dev/null +++ b/test/external-sdk/typed_contract.json @@ -0,0 +1,19 @@ +{ + "port": "example.Typed", + "adapter": "example.Managed", + "ecosystem": "exec", + "entry": "test/typed", + "schema": "typed-fixture-v1", + "libs": [], + "capabilities": [], + "operations": ["render"], + "setting": {"type": "record", "fields": {"prefix": {"type": "text"}}}, + "inputs": {"render": {"type": "record", "fields": { + "message": {"type": "record", "fields": { + "name": {"type": "text"}, "number": {"type": "int64"}, + "labels": {"type": "list", "value": {"type": "text"}} + }} + }}}, + "outputs": {"render": {"type": "record", "fields": {"text": {"type": "text"}}}}, + "errors": {"example.Rejected": {"type": "record", "fields": {"reason": {"type": "text"}}}} +} diff --git a/test/external-sdk/typed_worker.go b/test/external-sdk/typed_worker.go new file mode 100644 index 0000000..d929a73 --- /dev/null +++ b/test/external-sdk/typed_worker.go @@ -0,0 +1,41 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + + component "dever-component" +) + +func main() { + bytes, err := os.ReadFile(os.Args[1]) + if err != nil { + panic(err) + } + var manifest component.Manifest + if err = json.Unmarshal(bytes, &manifest); err != nil { + panic(err) + } + worker, err := component.NewFromManifest(manifest) + if err == nil { + err = worker.RegisterHandlers(map[string]func(context.Context, any, any) (any, error){ + "render": func(_ context.Context, payload any, setting any) (any, error) { + message := payload.(map[string]any)["message"].(map[string]any) + if message["name"] == "reject" { + return nil, component.BusinessError{Identity: "example.Rejected", Payload: map[string]any{"reason": "rejected"}} + } + prefix := setting.(map[string]any)["prefix"].(string) + return map[string]any{"text": prefix + message["name"].(string) + fmt.Sprint(message["number"])}, nil + }, + }) + } + if err == nil { + err = worker.Serve(os.Stdin, os.Stdout) + } + if err != nil { + fmt.Fprintln(os.Stderr, "typed worker:", err) + os.Exit(1) + } +} diff --git a/test/external-sdk/typed_worker.mjs b/test/external-sdk/typed_worker.mjs new file mode 100644 index 0000000..3ecef42 --- /dev/null +++ b/test/external-sdk/typed_worker.mjs @@ -0,0 +1,12 @@ +import { readFileSync } from 'node:fs'; +import { BusinessError, Worker, main } from '../../sdk/javascript/dever_component.mjs'; + +const manifest = JSON.parse(readFileSync(process.argv[2], 'utf8')); +const worker = Worker.fromManifest(manifest, { + render: async (payload, setting) => { + const message = payload.message; + if (message.name === 'reject') throw new BusinessError('example.Rejected', { reason: 'rejected' }); + return { text: setting.prefix + message.name + String(message.number) }; + }, +}); +await main(worker); diff --git a/test/external-sdk/typed_worker.py b/test/external-sdk/typed_worker.py new file mode 100644 index 0000000..b6ae181 --- /dev/null +++ b/test/external-sdk/typed_worker.py @@ -0,0 +1,17 @@ +import json +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "sdk/python")) +from dever_component import BusinessError, Worker, main + + +async def render(payload, setting): + message = payload["message"] + if message["name"] == "reject": + raise BusinessError("example.Rejected", {"reason": "rejected"}) + return {"text": setting["prefix"] + message["name"] + str(message["number"])} + + +manifest = json.loads(Path(sys.argv[1]).read_text()) +main(Worker.from_manifest(manifest, {"render": render})) diff --git a/test/go-managed/analyze/main.go b/test/go-managed/analyze/main.go new file mode 100644 index 0000000..096363a --- /dev/null +++ b/test/go-managed/analyze/main.go @@ -0,0 +1,89 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "go/build" + "os" + "path/filepath" + "sort" + "strings" +) + +type selectedPackage struct { + Name string `json:"name"` + Files []string `json:"files"` + Imports []string `json:"imports"` + Embed map[string][]string `json:"embed"` +} + +func main() { + if len(os.Args) != 4 { + fail(errors.New("usage: analyze <goos> <goarch> <directory>")) + } + ctx := build.Default + ctx.GOOS, ctx.GOARCH = os.Args[1], os.Args[2] + ctx.CgoEnabled = false + ctx.BuildTags = nil + pkg, err := ctx.ImportDir(os.Args[3], 0) + if err != nil { + fail(err) + } + if len(pkg.CgoFiles)+len(pkg.SFiles)+len(pkg.CFiles)+len(pkg.CXXFiles)+len(pkg.SwigFiles)+len(pkg.SwigCXXFiles) != 0 { + fail(errors.New("managed Go Worker does not support cgo, assembly, or native source")) + } + if len(pkg.GoFiles) == 0 { + fail(errors.New("no Go source selected for target")) + } + embed := make(map[string][]string, len(pkg.EmbedPatterns)) + for _, rawPattern := range pkg.EmbedPatterns { + includeHidden := strings.HasPrefix(rawPattern, "all:") + pattern := strings.TrimPrefix(rawPattern, "all:") + matches, err := filepath.Glob(filepath.Join(os.Args[3], filepath.FromSlash(pattern))) + if err != nil || len(matches) == 0 { + fail(fmt.Errorf("Go embed pattern %q has no matches", rawPattern)) + } + for _, match := range matches { + err = filepath.WalkDir(match, func(path string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.Type()&os.ModeSymlink != 0 { + return errors.New("Go embed contains a symbolic link") + } + if !includeHidden && path != match && (strings.HasPrefix(entry.Name(), ".") || strings.HasPrefix(entry.Name(), "_")) { + if entry.IsDir() { + return filepath.SkipDir + } + return nil + } + if entry.IsDir() { + return nil + } + relative, err := filepath.Rel(os.Args[3], path) + if err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return errors.New("Go embed escapes package directory") + } + embed[rawPattern] = append(embed[rawPattern], filepath.ToSlash(relative)) + return nil + }) + if err != nil { + fail(err) + } + } + if len(embed[rawPattern]) == 0 { + fail(fmt.Errorf("Go embed pattern %q has no embeddable files", rawPattern)) + } + sort.Strings(embed[rawPattern]) + } + result := selectedPackage{Name: pkg.Name, Files: pkg.GoFiles, Imports: pkg.Imports, Embed: embed} + if err := json.NewEncoder(os.Stdout).Encode(result); err != nil { + fail(err) + } +} + +func fail(err error) { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) +} diff --git a/test/go-managed/make_pack.go b/test/go-managed/make_pack.go new file mode 100644 index 0000000..6961b3c --- /dev/null +++ b/test/go-managed/make_pack.go @@ -0,0 +1,212 @@ +//go:build ignore + +// make_pack.go creates private native/cross fixtures. It is not a signed release maker. +package main + +import ( + "archive/tar" + "bufio" + "compress/gzip" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "sort" + "strings" +) + +func main() { + if len(os.Args) != 3 && len(os.Args) != 4 { + fatal(errors.New("usage: go run make_pack.go <absolute-go-binary> <output-runtime.pack> [linux-aarch64]")) + } + goos, goarch := runtime.GOOS, runtime.GOARCH + if len(os.Args) == 4 { + if os.Args[3] != "linux-aarch64" || runtime.GOOS != "linux" || runtime.GOARCH != "amd64" { + fatal(errors.New("cross fixture supports only Linux x86_64 to linux-aarch64")) + } + goarch = "arm64" + } + goBinary, err := filepath.Abs(os.Args[1]) + if err != nil { + fatal(err) + } + outputPack, err := filepath.Abs(os.Args[2]) + if err != nil { + fatal(err) + } + root, err := os.MkdirTemp("", "dever-go-pack-") + if err != nil { + fatal(err) + } + defer os.RemoveAll(root) + toolDir := filepath.Join(filepath.Dir(goBinary), "..", "pkg", "tool", runtime.GOOS+"_"+runtime.GOARCH) + goRoot := filepath.Clean(filepath.Join(filepath.Dir(goBinary), "..")) + analyzer := filepath.Join(root, "analyze") + build := exec.Command(goBinary, "build", "-p=1", "-o", analyzer, "./test/go-managed/analyze") + build.Env = []string{"GOROOT=" + goRoot, "GOTOOLCHAIN=local", "GOWORK=off", "GO111MODULE=off", "CGO_ENABLED=0", "GOCACHE=" + filepath.Join(filepath.Dir(outputPack), "author-cache"), "GOMODCACHE=" + filepath.Join(root, "modules")} + if output, err := build.CombinedOutput(); err != nil { + fatal(fmt.Errorf("build Go source selector: %w: %s", err, output)) + } + list := exec.Command(goBinary, "list", "-p=1", "-deps", "-export", "-f", "{{if .Export}}{{.ImportPath}} {{.Export}}{{end}}", "std") + list.Env = append(build.Env, "GOOS="+goos, "GOARCH="+goarch) + output, err := list.Output() + if err != nil { + fatal(fmt.Errorf("list fixed-target standard library: %w", err)) + } + stdlib := make(map[string]string) + scanner := bufio.NewScanner(strings.NewReader(string(output))) + for scanner.Scan() { + fields := strings.Fields(scanner.Text()) + if len(fields) == 2 { + stdlib[fields[0]] = fields[1] + } + } + if err := scanner.Err(); err != nil { + fatal(err) + } + if len(stdlib) == 0 || stdlib["runtime"] == "" { + fatal(errors.New("standard library export map is incomplete")) + } + files := map[string]string{ + "bin/compile": filepath.Join(toolDir, "compile"), + "bin/link": filepath.Join(toolDir, "link"), + "bin/analyze": analyzer, + } + if goarch != runtime.GOARCH { + tools := crossTools(goBinary, goRoot, root, goarch, build.Env) + files["bin/compile"], files["bin/link"] = tools[0], tools[1] + } + var imports []string + for path, archive := range stdlib { + files["stdlib/"+path+".a"] = archive + imports = append(imports, path) + } + sort.Strings(imports) + var importcfg strings.Builder + for _, path := range imports { + fmt.Fprintf(&importcfg, "packagefile %s=stdlib/%s.a\n", path, path) + } + version := exec.Command(goBinary, "version") + version.Env = []string{"GOROOT=" + goRoot, "GOTOOLCHAIN=local"} + versionText, err := version.Output() + if err != nil { + fatal(err) + } + deverOS, deverArch := runtime.GOOS, runtime.GOARCH + if deverOS == "darwin" { + deverOS = "macos" + } + if deverArch == "amd64" { + deverArch = "x86_64" + } else if deverArch == "arm64" { + deverArch = "aarch64" + } + target := deverOS + "-" + deverArch + if len(os.Args) == 4 { + target = os.Args[3] + } + manifest, err := json.Marshal(map[string]any{ + "format": "dever-worker-runtime-v1", "ecosystem": "go", "target": target, + "build": map[string]any{"host": deverOS + "-" + deverArch, "compiler": "bin/compile", "linker": "bin/link", "analyzer": "bin/analyze", "stdlib_importcfg": "stdlib/importcfg", "goos": goos, "goarch": goarch, "go_version": strings.TrimSpace(string(versionText))}, + }) + if err != nil { + fatal(err) + } + outputFile, err := os.Create(outputPack) + if err != nil { + fatal(err) + } + defer outputFile.Close() + gzipWriter := gzip.NewWriter(outputFile) + tarWriter := tar.NewWriter(gzipWriter) + writeBytes(tarWriter, "dever-runtime.json", manifest) + writeBytes(tarWriter, "stdlib/importcfg", []byte(importcfg.String())) + names := make([]string, 0, len(files)) + for name := range files { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + input, err := os.Open(files[name]) + if err != nil { + fatal(err) + } + info, err := input.Stat() + if err != nil { + fatal(err) + } + mode := int64(0644) + if strings.HasPrefix(name, "bin/") { + mode = 0755 + } + if err := tarWriter.WriteHeader(&tar.Header{Name: name, Size: info.Size(), Mode: mode, Typeflag: tar.TypeReg}); err != nil { + fatal(err) + } + if _, err := io.Copy(tarWriter, input); err != nil { + fatal(err) + } + input.Close() + } + if err := tarWriter.Close(); err != nil { + fatal(err) + } + if err := gzipWriter.Close(); err != nil { + fatal(err) + } +} + +// The product clears the environment. Bake the output architecture into private +// host tools through an overlay; never edit the author's installed Go SDK. +func crossTools(goBinary, goRoot, root, architecture string, environment []string) []string { + original := filepath.Join(goRoot, "src", "internal", "buildcfg", "zbootstrap.go") + contents, err := os.ReadFile(original) + if err != nil { + fatal(err) + } + const declaration = "const defaultGOARCH = runtime.GOARCH" + if strings.Count(string(contents), declaration) != 1 { + fatal(errors.New("Go SDK default architecture declaration differs")) + } + override := filepath.Join(root, "zbootstrap.go") + updated := strings.Replace(string(contents), declaration, "const defaultGOARCH = "+fmt.Sprintf("%q", architecture), 1) + if err := os.WriteFile(override, []byte(updated), 0600); err != nil { + fatal(err) + } + overlay := filepath.Join(root, "overlay.json") + encoded, err := json.Marshal(map[string]any{"Replace": map[string]string{original: override}}) + if err != nil { + fatal(err) + } + if err := os.WriteFile(overlay, encoded, 0600); err != nil { + fatal(err) + } + var outputs []string + for _, tool := range []string{"compile", "link"} { + output := filepath.Join(root, tool) + command := exec.Command(goBinary, "build", "-p=1", "-overlay", overlay, "-o", output, "cmd/"+tool) + command.Env = environment + if bytes, err := command.CombinedOutput(); err != nil { + fatal(fmt.Errorf("build cross %s: %w: %s", tool, err, bytes)) + } + outputs = append(outputs, output) + } + return outputs +} + +func writeBytes(writer *tar.Writer, name string, data []byte) { + if err := writer.WriteHeader(&tar.Header{Name: name, Size: int64(len(data)), Mode: 0644, Typeflag: tar.TypeReg}); err != nil { + fatal(err) + } + if _, err := writer.Write(data); err != nil { + fatal(err) + } +} + +func fatal(err error) { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) +} diff --git a/test/llvm_api_security_cases.rs b/test/llvm_api_security_cases.rs new file mode 100644 index 0000000..df76baf --- /dev/null +++ b/test/llvm_api_security_cases.rs @@ -0,0 +1,644 @@ +//! Real checked routes exercise the existing identity, RBAC and tenant owners. +use super::native::{Server, TRANSPORT, application_ir, settings}; +use dever_backend_bridge::{Target, emit_object}; +use dever_runtime::{ + api, auth, + bytes::Bytes, + config::Settings, + http::{Header, Request}, + task, wire, +}; +use serde_json::{Value, json}; + +pub(super) fn files() -> Vec<(&'static str, &'static str)> { + let mut files = TRANSPORT[3..].to_vec(); + files[1].0 = "sample/echo/api/test/probe.dever"; + files.extend([ + ("user/account/model.dever", r#"global type Account { email: Text(1, 254) unique } +seed { { email = "owner" } + { email = "reader" } } +"#), + ("platform/tenant/model.dever", r#"global type Tenant { name: Text(1, 64) unique } +seed { { name = "one" } + { name = "two" } } +"#), + ("platform/tenant/app.dever", r#"resolve(key: null) (id: model.id) { fail(dever.api.Error.Unauthorized) } +resolve(key: Text) (id: model.id) { + found = model.first({ where = name == key }) + id = required(found) +} +required(value: null) (id: model.id) { fail(dever.api.Error.Unauthorized) } +required(value: model.Tenant) (id: model.id) { id = value.id } +"#), + ("user/account/app.dever", r#"type Identity { + id: Text + user_id: model.id? + tenant_id: platform.tenant.model.id? +} +verify(claims: dever.auth.Claims) (identity: Identity) { + domain.active(claims.session == "active") + found = model.first({ where = email == claims.subject }) + identity = Identity { id = claims.subject + user_id = required(found) + tenant_id = platform.tenant.resolve(claims.tenant) } +} +required(value: null) (id: model.id) { fail(dever.api.Error.Unauthorized) } +required(value: model.Account) (id: model.id) { id = value.id } +"#), + ("user/account/domain.dever", "active(value: true) () {}\nactive(value: false) () { fail(dever.api.Error.Unauthorized) }"), + ("news/article/model.dever", r#"type Article { + private owner author_id: user.account.model.id = dever.auth.user_id() + name: Text(1, 32) unique create = domain.normalize(input) replace = domain.normalize(input) search = domain.normalize(input) + note: Text? + private created_by: Text create = dever.auth.id() + private updated_by: Text? replace = dever.auth.id() +} +"#), + ("news/article/domain.dever", "normalize(value: Text) (normalized: Text) pure { normalized = text.trim(value) }"), + ("news/article/app.dever", r#"type IdentityView { + id: Text + user_id: Int? + tenant_id: Int? + session: Text + site: Text + request_id: Text +} +identity() (view: IdentityView) { + dever.task.sleep(1) + view = IdentityView { id = dever.auth.id() + user_id = dever.auth.user_id() + tenant_id = dever.auth.tenant_id() + session = dever.auth.session() + site = dever.site.key() + request_id = dever.api.request_id() } +} +permissions() (values: List<dever.auth.Permission>) { values = dever.auth.permissions() } +role(id: Text, keys: List<Text>) (saved: Bool) { dever.auth.save_role(id, id, false, keys) + saved = true } +grant(user_id: Int, role_id: Text) (saved: Bool) { dever.auth.grant_role(user_id, role_id) + saved = true } +revoke(user_id: Int, role_id: Text) (saved: Bool) { dever.auth.revoke_role(user_id, role_id) + saved = true } +disable(role_id: Text) (saved: Bool) { dever.auth.disable_role(role_id) + saved = true } +rollback(id: model.id) (value: Text) { + changed = model.update(id, { name = "rolled-back" }) + dever.api.set_header("x-staged", "discarded") + fail(dever.api.Error.Forbidden) +} +cookies() (value: Text) { + dever.api.set_cookie("theme", "dark", dever.api.secure_cookie_options()) + dever.api.set_secret_cookie("copy", dever.auth.issue(dever.auth.id(), dever.auth.session(), null), dever.api.secure_cookie_options()) + value = "issued" +} +"#), + ("news/article/api/admin/manage.dever", "rest model\nget identity = app.identity\nget permissions = app.permissions\npost role = app.role\npost grant = app.grant\npost revoke = app.revoke\npost disable = app.disable\npost rollback = app.rollback\nget cookies = app.cookies"), + ("news/article/api/front/browse.dever", "get identity = app.identity"), + ]); + files +} + +pub(super) fn configuration() -> Value { + let mut value: Value = serde_json::from_str(&settings(8080)).unwrap(); + value["database"]["default"]["tenant_directory"] = "data/tenants".into(); + value["database"]["default"]["max_page_size"] = 4.into(); + value["tenant"] = json!({"database": "default", "max_pools": 2, "idle_timeout_ms": 1000}); + value["sites"]["test"]["path"] = "test".into(); + for site in ["admin", "front"] { + value["sites"][site] = json!({"path": site, "auth": "session", "hosts": ["localhost"], "origin": "https://localhost"}); + } + value +} + +pub(super) fn tenant_bootstrap() -> Vec<Vec<String>> { + [ + vec!["--dever-tenant-migrate", "1"], + vec!["--dever-tenant-migrate", "2"], + vec!["--dever-tenant-owner", "1", "admin", "1"], + vec!["--dever-tenant-owner", "2", "admin", "1"], + ] + .map(|args| args.into_iter().map(str::to_owned).collect()) + .into() +} + +/// Mint credentials through the existing approved Secret cookie sink, never reveal Secret. +pub(super) fn credential( + settings: &Settings, + site: &str, + subject: &str, + session: &str, + tenant: &str, +) -> String { + task::run_entry(async { + let request = Request { + method: "POST".into(), + target: "/fixture/login".into(), + headers: vec![ + Header { + name: "host".into(), + value: Bytes::from_text("localhost"), + }, + Header { + name: "origin".into(), + value: Bytes::from_text("https://localhost"), + }, + ], + body: Bytes::new(vec![]), + }; + let response = api::scoped_request(request, |request| async move { + let prepared = auth::prepare_with_settings(settings, &[site], &request, true) + .map_err(|error| error.to_string())?; + auth::scope(prepared, None, async { + auth::issue_cookie(subject, session, Some(tenant))?; + api::commit_response_metadata()?; + Ok(api::success(wire::Encoded::null())) + }) + .await + .map_err(|error| error.to_string())? + }) + .await?; + let cookie = response + .headers + .iter() + .find(|header| header.name == "set-cookie") + .ok_or("missing fixture credential")?; + let cookie = + std::str::from_utf8(cookie.value.values()).map_err(|error| error.to_string())?; + assert!(cookie.contains("Secure") && cookie.contains("HttpOnly")); + Ok(cookie.split(';').next().unwrap().to_owned()) + }) + .unwrap() +} + +pub(super) fn request( + server: &Server, + method: &str, + path: &str, + cookie: &str, + body: Value, + expected: u16, +) -> Value { + let headers = [ + ("cookie", cookie), + ("origin", "https://localhost"), + ("content-type", "application/json"), + ]; + let body = if method == "GET" || method == "DELETE" { + String::new() + } else { + body.to_string() + }; + let response = server.request(method, path, &headers, &body).unwrap(); + assert_eq!( + response.status, + expected, + "{method} {path}: {}\n{}", + response.body, + server.stderr() + ); + response.body["data"].clone() +} + +#[test] +fn trusted_identity_and_rest_objects_cover_all_six_targets() { + let ir = application_ir(&files(), &configuration().to_string(), false, None); + for target in Target::ALL { + assert!(!emit_object(&ir, target).unwrap().is_empty()); + } +} + +#[test] +#[ignore = "requires explicit runtime-api/sqlite archive; uses only owned loopback and two owned tenant files"] +fn private_rest_isolates_tenants_roles_identity_and_transactions() { + exercise_private_rest(configuration(), 64); +} + +fn exercise_private_rest(configuration: Value, iterations: usize) { + let commands = tenant_bootstrap(); + let mut server = Server::start_with(&files(), iterations, configuration, &commands); + let settings = Settings::load_project(server.directory.path()).unwrap(); + let owner_one = credential(&settings, "admin", "owner", "active", "one"); + let owner_two = credential(&settings, "admin", "owner", "active", "two"); + let reader = credential(&settings, "admin", "reader", "active", "one"); + let front = credential(&settings, "front", "owner", "active", "one"); + let unknown = credential(&settings, "admin", "unknown", "active", "one"); + let stale = credential(&settings, "admin", "owner", "stale", "one"); + let path = "/news/article/admin/manage"; + for cycle in 1..=iterations + 1 { + server.ready_at("/sample/echo/test/probe/increment?q=1"); + assert_eq!(server.request("GET", path, &[], "").unwrap().status, 401); + for cookie in [&unknown, &stale, &front] { + request(&server, "GET", path, cookie, Value::Null, 401); + } + request(&server, "GET", path, &reader, Value::Null, 403); + for invalid in ["?unknown=1", "?page=0", "?size=5", "?name="] { + request( + &server, + "GET", + &format!("{path}{invalid}"), + &owner_one, + Value::Null, + 400, + ); + } + request( + &server, + "POST", + path, + &owner_one, + json!({"name": null}), + 400, + ); + let keys = request( + &server, + "GET", + &format!("{path}/permissions"), + &owner_one, + Value::Null, + 200, + ); + assert!( + keys.as_array() + .unwrap() + .iter() + .all(|permission| permission["site"] != "test") + ); + request( + &server, + "POST", + &format!("{path}/role"), + &owner_one, + json!({"id": "reader", "keys": ["news.article.admin.read", "news.article.admin.identity"]}), + 200, + ); + request( + &server, + "POST", + &format!("{path}/grant"), + &owner_one, + json!({"user_id": 2, "role_id": "reader"}), + 200, + ); + request( + &server, + "GET", + &format!("{path}?size=4"), + &reader, + Value::Null, + 200, + ); + request( + &server, + "POST", + path, + &reader, + json!({"name": "blocked"}), + 403, + ); + request( + &server, + "POST", + &format!("{path}/role"), + &owner_one, + json!({"id": "writer", "keys": ["news.article.admin.create"]}), + 200, + ); + request( + &server, + "POST", + &format!("{path}/grant"), + &owner_one, + json!({"user_id": 2, "role_id": "writer"}), + 200, + ); + let own = request( + &server, + "POST", + path, + &reader, + json!({"name": " reader ", "note": null}), + 200, + ); + assert_eq!(own["name"], "reader"); + assert!(own.get("author_id").is_none() && own.get("created_by").is_none()); + let owner = request( + &server, + "POST", + path, + &owner_one, + json!({"name": " shared ", "note": "one"}), + 200, + ); + let second = request( + &server, + "POST", + path, + &owner_two, + json!({"name": "shared", "note": "two"}), + 200, + ); + assert_eq!(owner["name"], "shared"); + request( + &server, + "GET", + &format!("{path}?name=%20shared%20&size=4"), + &owner_one, + Value::Null, + 200, + ); + let selected = request( + &server, + "GET", + &format!("{path}?name=shared&size=4"), + &owner_two, + Value::Null, + 200, + ); + assert_eq!(selected["total"], 1); + assert_eq!(selected["items"][0]["note"], "two"); + let reader_page = request( + &server, + "GET", + &format!("{path}?size=4"), + &reader, + Value::Null, + 200, + ); + assert_eq!(reader_page["total"], 1); + request( + &server, + "GET", + &format!("{path}/{}", owner["id"]), + &reader, + Value::Null, + 404, + ); + request( + &server, + "GET", + &format!("{path}/bad-id"), + &reader, + Value::Null, + 404, + ); + let replaced = request( + &server, + "PUT", + &format!("{path}/{}", owner["id"]), + &owner_one, + json!({"name": " renamed ", "note": null}), + 200, + ); + assert_eq!(replaced["name"], "renamed"); + assert_eq!(replaced["note"], Value::Null); + request( + &server, + "PUT", + &format!("{path}/{}", owner["id"]), + &owner_one, + json!({"name": ""}), + 400, + ); + request( + &server, + "POST", + &format!("{path}/rollback"), + &owner_one, + json!({"id": owner["id"]}), + 403, + ); + let persisted = request( + &server, + "GET", + &format!("{path}?size=4"), + &owner_one, + Value::Null, + 200, + ); + assert_eq!(persisted["total"], 1); + assert_eq!(persisted["items"][0]["name"], "renamed"); + let identity = request( + &server, + "GET", + &format!("{path}/identity"), + &reader, + Value::Null, + 200, + ); + assert_eq!(identity["id"], "reader"); + assert_eq!(identity["user_id"], 2); + assert_eq!(identity["tenant_id"], 1); + assert_eq!(identity["site"], "admin"); + std::thread::scope(|scope| { + let server = &server; + let pending = [ + (&owner_one, "owner", 1, 1), + (&owner_two, "owner", 1, 2), + (&reader, "reader", 2, 1), + ] + .map(|(cookie, subject, user, tenant)| { + scope.spawn(move || { + let view = request( + server, + "GET", + &format!("{path}/identity"), + cookie, + Value::Null, + 200, + ); + assert_eq!(view["id"], subject); + assert_eq!(view["user_id"], user); + assert_eq!(view["tenant_id"], tenant); + assert_eq!(view["session"], "active"); + assert_eq!(view["site"], "admin"); + view["request_id"].as_str().unwrap().to_owned() + }) + }); + let ids = pending.map(|task| task.join().unwrap()); + assert_eq!( + ids.into_iter() + .collect::<std::collections::BTreeSet<_>>() + .len(), + 3 + ); + }); + let cookies = server + .request( + "GET", + &format!("{path}/cookies"), + &[("cookie", &owner_one), ("origin", "https://localhost")], + "", + ) + .unwrap(); + assert_eq!(cookies.status, 200); + let issued = cookies + .headers + .iter() + .filter(|(name, _)| name == "set-cookie") + .collect::<Vec<_>>(); + assert_eq!(issued.len(), 2); + assert!( + issued + .iter() + .all(|(_, value)| value.contains("Secure") && value.contains("HttpOnly")) + ); + assert!( + issued + .iter() + .any(|(_, value)| value.starts_with("theme=dark;")) + ); + assert!(issued.iter().any(|(_, value)| value.starts_with("copy="))); + request( + &server, + "GET", + "/news/article/front/browse/identity", + &front, + Value::Null, + 403, + ); + request( + &server, + "POST", + &format!("{path}/revoke"), + &owner_one, + json!({"user_id": 2, "role_id": "writer"}), + 200, + ); + request( + &server, + "POST", + path, + &reader, + json!({"name": "revoked"}), + 403, + ); + // Saving a disabled role deliberately never re-enables it. Revoke reusable + // membership between roots; test permanent role disable on the final root. + let (action, body) = if cycle == iterations + 1 { + ("disable", json!({"role_id": "reader"})) + } else { + ("revoke", json!({"user_id": 2, "role_id": "reader"})) + }; + request( + &server, + "POST", + &format!("{path}/{action}"), + &owner_one, + body, + 200, + ); + request( + &server, + "GET", + &format!("{path}?size=4"), + &reader, + Value::Null, + 403, + ); + for (cookie, value) in [(&owner_one, &owner), (&owner_two, &second)] { + request( + &server, + "DELETE", + &format!("{path}/{}", value["id"]), + cookie, + Value::Null, + 200, + ); + } + // Owner filtering is deliberate; remove the reader's row through a new exact role. + request( + &server, + "POST", + &format!("{path}/role"), + &owner_one, + json!({"id": "cleaner", "keys": ["news.article.admin.delete"]}), + 200, + ); + request( + &server, + "POST", + &format!("{path}/grant"), + &owner_one, + json!({"user_id": 2, "role_id": "cleaner"}), + 200, + ); + request( + &server, + "DELETE", + &format!("{path}/{}", own["id"]), + &reader, + Value::Null, + 200, + ); + request( + &server, + "POST", + &format!("{path}/revoke"), + &owner_one, + json!({"user_id": 2, "role_id": "cleaner"}), + 200, + ); + server.stop_cycle(cycle); + } + server.finish(); +} + +#[cfg(feature = "runtime-postgres")] +#[test] +#[ignore = "requires explicit PostgreSQL fixture in config/setting.json; owns control and two tenant databases"] +fn postgres_private_rest_uses_the_same_identity_permission_and_transaction_contract() { + super::postgres::run_in_isolated_databases("llvm_api", |fixture| { + let mut configuration = configuration(); + configuration["database"]["default"] = fixture.database_setting(); + configuration["database"]["default"]["max_page_size"] = 4.into(); + exercise_private_rest(configuration, 2); + Ok(()) + }); +} + +#[test] +#[ignore = "requires explicit runtime-api/sqlite archive; owns all listener, component and tenant state"] +fn disabled_component_cannot_be_bypassed_by_tenant_owner() { + let mut commands = tenant_bootstrap(); + for state in ["disable", "enable", "disable"] { + commands.push( + ["--dever-tenant-component", "1", state, "news"] + .map(str::to_owned) + .into(), + ); + } + let mut server = Server::start_with(&files(), 64, configuration(), &commands); + let settings = Settings::load_project(server.directory.path()).unwrap(); + let disabled = credential(&settings, "admin", "owner", "active", "one"); + let allowed = credential(&settings, "admin", "owner", "active", "two"); + for cycle in 1..=65 { + server.ready_at("/sample/echo/test/probe/increment?q=1"); + request( + &server, + "GET", + "/news/article/admin/manage?size=4", + &disabled, + Value::Null, + 403, + ); + request( + &server, + "POST", + "/news/article/admin/manage", + &disabled, + json!({"name": "blocked"}), + 403, + ); + let page = request( + &server, + "GET", + "/news/article/admin/manage?size=4", + &allowed, + Value::Null, + 200, + ); + assert_eq!(page["total"], 0); + server.stop_cycle(cycle); + } + server.finish(); +} diff --git a/test/llvm_api_source.rs b/test/llvm_api_source.rs new file mode 100644 index 0000000..354d463 --- /dev/null +++ b/test/llvm_api_source.rs @@ -0,0 +1,895 @@ +//! Checked-source HTTP applications linked to the explicit runtime-only archive. +#[cfg(all( + target_os = "linux", + target_arch = "x86_64", + feature = "runtime-sqlite" +))] +#[path = "llvm_job_security_cases.rs"] +mod jobs; +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/llvm_module.rs"] +mod llvm_module; +#[cfg(all( + target_os = "linux", + target_arch = "x86_64", + feature = "runtime-postgres" +))] +#[path = "dever-tests/tests/support/postgres.rs"] +pub mod postgres; +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/process.rs"] +mod process; +#[cfg(all( + target_os = "linux", + target_arch = "x86_64", + feature = "runtime-sqlite" +))] +#[path = "llvm_api_security_cases.rs"] +mod security; +#[cfg(all( + target_os = "linux", + target_arch = "x86_64", + feature = "runtime-sqlite" +))] +#[path = "dever-tests/tests/support/sqlite_inspect.rs"] +mod sqlite_inspect; +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/temp.rs"] +mod temp; + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +mod native { + use super::{llvm_module, process, temp}; + use dever_backend_bridge::{Target, emit_object}; + use dever_core::{hir::Program, source::SourceMap}; + use dever_runtime::config::Settings; + use serde_json::{Value, json}; + use std::fs; + use std::io::{Read, Write}; + use std::net::{TcpListener, TcpStream}; + use std::process::{Child, Command, Stdio}; + use std::time::{Duration, Instant}; + + pub(super) const TRANSPORT: &[(&str, &str)] = &[ + ( + "user/account/model.dever", + "global type Account { email: Text(1, 254) }", + ), + ( + "platform/tenant/model.dever", + "global type Tenant { name: Text(1, 64) }", + ), + ( + "user/account/app.dever", + r#"type Identity { + id: Text + user_id: user.account.model.id? + tenant_id: platform.tenant.model.id? +} +verify(claims: dever.auth.Claims) (identity: Identity) { + identity = Identity { id = claims.subject + user_id = null + tenant_id = null } +} +"#, + ), + ( + "sample/echo/app.dever", + r#"type Payload { + name: Text + count: Int + tags: List<Text?> + note: Text? +} +type Snapshot { + request_id: Text + method: Text + path: Text + header: Text? + cookie: Text? + peer: Text? +} +type Outcome { + error Rejected(message: Text) +} +increment(q: Int, note: Text?) (value: Int) { value = q + 1 } +echo(payload: Payload) (value: Payload) { value = payload } +inspect() (value: Snapshot) { + dever.api.set_header("x-response-probe", "committed") + dever.task.sleep(1) + value = Snapshot { + request_id = dever.api.request_id() + method = dever.api.method() + path = dever.api.path() + header = dever.api.header("x-probe") + cookie = dever.api.cookie("probe") + peer = dever.api.client_address() + } +} +invalid() (value: Text) { + dever.api.set_header("x-response-probe", "must-not-commit") + fail(dever.api.Error.Invalid) +} +unauthorized() (value: Text) { fail(dever.api.Error.Unauthorized) } +forbidden() (value: Text) { fail(dever.api.Error.Forbidden) } +missing() (value: Text) { fail(dever.api.Error.NotFound) } +conflict() (value: Text) { fail(dever.api.Error.Conflict) } +limited() (value: Text) { fail(dever.api.Error.TooManyRequests) } +fault() (value: Text) { fail(Outcome.Rejected("must-not-appear-in-http-or-log")) } +upload(file: Upload, caption: Text) (size: Int) { + size = dever.api.upload_size(file) + dever.api.close_upload(file) +} +"#, + ), + ( + "sample/echo/api.dever", + "public get increment = app.increment\npublic post echo = app.echo\npublic get inspect = app.inspect\npublic get invalid = app.invalid\npublic get unauthorized = app.unauthorized\npublic get forbidden = app.forbidden\npublic get missing = app.missing\npublic get conflict = app.conflict\npublic get limited = app.limited\npublic get fault = app.fault\npublic post upload = app.upload\ncmd echo = app.echo", + ), + ]; + + fn echo_files(markdown: bool) -> Vec<(&'static str, &'static str)> { + let mut files = TRANSPORT[..3].to_vec(); + if markdown { + files.extend([ + ( + "sample/echo/app.dever.md", + r#"# Echo application + +Owned JSON echo contract. + +- 包：`sample.echo.app` +- 公开类型： + - `Payload` +- 公开方法： + - `echo` +- 使用： + - `sample.echo.echo(payload)` + +## Payload + +Keep concrete input fields. + +- 类型：`Payload` +- 字段： + - `name: Text`：Name. + - `count: Int`：Count. + - `tags: List<Text?>`：Tags. + - `note: Text?`：Optional note. + +```dever +type Payload { name: Text + count: Int + tags: List<Text?> + note: Text? } +``` + +## Echo + +Return the checked input. + +- 函数：`echo` +- 输入： + - `payload: Payload`：Payload. +- 输出： + - `value: Payload`：Payload. + +```dever +echo(payload: Payload) (value: Payload) { value = payload } +``` +"#, + ), + ( + "sample/echo/api.dever.md", + r#"# Echo interfaces + +HTTP and command-line entry bindings. + +- 包：`sample.echo.api` +- 公开类型：无 +- 公开方法：无 +- 使用：无 + +## HTTP + +Anonymous typed JSON echo. + +- 声明：`post echo` + +```dever +public post echo = app.echo +``` + +## Command + +Command-line JSON echo. + +- 声明：`cmd echo` + +```dever +cmd echo = app.echo +``` +"#, + ), + ]); + } else { + files.extend([ + ("sample/echo/app.dever", "type Payload { name: Text\ncount: Int\ntags: List<Text?>\nnote: Text? }\necho(payload: Payload) (value: Payload) { value = payload }"), + ("sample/echo/api.dever", "public post echo = app.echo\ncmd echo = app.echo"), + ]); + } + files + } + + pub(super) fn settings(port: u16) -> String { + json!({ + "http": {"host": "127.0.0.1", "port": port}, + "runtime": {"mode": "api", "shutdown_ms": 2000}, + "log": {"level": "error"}, + "database": {"default": {"type": "sqlite", "path": "data/control.db", "max_connections": 2}}, + "auth": {"providers": {"session": {"verify": "user.account.verify", "jwtSecret": "0123456789abcdef0123456789abcdef", "cookie": "test_session"}}}, + "sites": {"test": {"path": "", "auth": "session", "hosts": ["localhost"], "origin": "https://localhost"}} + }) + .to_string() + } + + fn checked(files: &[(&str, &str)], setting: &str) -> (SourceMap, Program) { + let directory = temp::TemporaryDirectory::new(); + fs::create_dir(directory.path().join("config")).unwrap(); + fs::write(directory.path().join("config/setting.json"), setting).unwrap(); + let settings = Settings::load_project(directory.path()).unwrap(); + let mut sources = SourceMap::default(); + for (path, source) in files { + sources.add(*path, *source); + } + let program = + dever_core::check_with_settings(&sources, &settings).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }); + (sources, program) + } + + pub(super) fn application_ir( + files: &[(&str, &str)], + setting: &str, + marker: bool, + failure: Option<u32>, + ) -> String { + let (sources, program) = checked(files, setting); + let mut ir = dever_core::llvm::emit_application(&program, &sources).unwrap(); + if marker { + ir.push_str("\n@api_probe_cycle = private constant [19 x i8] c\"api-cycle-complete\\00\"\ndeclare i32 @puts(ptr)\ndeclare i32 @fflush(ptr)\n@stderr = external global ptr\ndeclare i64 @fwrite(ptr, i64, i64, ptr)\n"); + } + ir.push_str("\ndefine i32 @dever_test_run() {\nentry:\n %out = alloca i8\n %fault = alloca %dever.fault\n %status = call i32 @dever_application_entry(ptr %out, ptr %fault)\n"); + match failure { + Some(code) => ir.push_str(&llvm_module::fault_check(code, None)), + None => ir.push_str(" %passed = icmp eq i32 %status, 0"), + } + if marker { + // Owned fixture diagnostics only, while the failed root still owns its buffer. + ir.push_str("\n %diagnostic_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 4\n %diagnostic = load { ptr, i64 }, ptr %diagnostic_ptr\n %diagnostic_data = extractvalue { ptr, i64 } %diagnostic, 0\n %diagnostic_size = extractvalue { ptr, i64 } %diagnostic, 1\n %has_diagnostic = icmp ugt i64 %diagnostic_size, 0\n %failed_root = xor i1 %passed, true\n %print_diagnostic = and i1 %failed_root, %has_diagnostic\n br i1 %print_diagnostic, label %diagnose, label %release_root\ndiagnose:\n %diagnostic_stream = load ptr, ptr @stderr\n %diagnostic_written = call i64 @fwrite(ptr %diagnostic_data, i64 1, i64 %diagnostic_size, ptr %diagnostic_stream)\n br label %release_root\nrelease_root:\n"); + } + ir.push_str("\n call void @dever_outputs_release(ptr %out)\n call void @dever_fault_release(ptr %fault)\n"); + if marker { + ir.push_str(" %printed = call i32 @puts(ptr @api_probe_cycle)\n %flushed = call i32 @fflush(ptr null)\n"); + } + ir.push_str(" %exit = select i1 %passed, i32 0, i32 1\n ret i32 %exit\n}\n"); + ir + } + + #[test] + #[ignore = "requires explicit runtime-api archive; invalid descriptor must not bind a listener"] + fn runtime_rejects_missing_authentication_before_serving() { + let reservation = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let setting = settings(reservation.local_addr().unwrap().port()); + let ir = application_ir(TRANSPORT, &setting, false, Some(3)); + assert!(ir.contains("ptr @dever_api_auth_0")); + let ir = ir.replace("ptr @dever_api_auth_0", "ptr null"); + llvm_module::execute( + &ir, + "invalid HTTP metadata", + 64, + Duration::from_secs(15), + &[], + Some(&setting), + ); + } + + #[test] + #[ignore = "requires explicit runtime-api archive; occupied listener belongs to this test only"] + fn failed_listener_start_drains_application_resources() { + let reservation = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let setting = settings(reservation.local_addr().unwrap().port()); + let ir = application_ir(TRANSPORT, &setting, false, Some(3)); + llvm_module::execute( + &ir, + "occupied owned listener", + 64, + Duration::from_secs(15), + &[], + Some(&setting), + ); + } + + #[test] + #[ignore = "requires explicit runtime-api archive; reads only owned setting.json and captured stderr"] + fn log_only_cmd_uses_configured_owner_and_flushes_before_return() { + let files = [ + ( + "sample/log/app.dever", + "write() (okay: Bool) { dever.log.info(\"filtered-info\", {})\n dever.log.warn(\"configured-warning\", {\"component\" = \"probe\"})\n okay = true }", + ), + ("sample/log/api.dever", "cmd write = app.write"), + ]; + let mut setting: Value = serde_json::from_str(&settings(8080)).unwrap(); + setting["log"]["level"] = "warn".into(); + let setting = setting.to_string(); + let ir = application_ir(&files, &setting, false, None); + let output = llvm_module::execute( + &ir, + "log-only CMD configuration", + 64, + Duration::from_secs(15), + &["sample.log.write", "{}"], + Some(&setting), + ); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!(!stderr.contains("filtered-info")); + assert_eq!(stderr.matches("configured-warning").count(), 65); + assert!(stderr.contains("probe")); + } + + #[test] + fn http_and_cmd_sources_preserve_checker_and_markdown_contracts() { + let setting = settings(8080); + let (_, dever) = checked(&echo_files(false), &setting); + let (_, markdown) = checked(&echo_files(true), &setting); + assert_eq!(dever.api_snapshot(), markdown.api_snapshot()); + assert!(dever.api_snapshot().contains("function sample.echo.echo")); + } + + #[test] + fn http_application_emits_objects_for_all_six_targets() { + let ir = application_ir(TRANSPORT, &settings(8080), false, None); + for target in Target::ALL { + assert!(!emit_object(&ir, target).unwrap().is_empty()); + } + } + + #[test] + fn http_lowering_rejects_unbound_authentication_metadata() { + for declaration in ["get echo = app.echo", "public get echo = app.echo"] { + let mut sources = SourceMap::default(); + sources.add( + "sample/echo/app.dever", + "echo() (value: Text) { value = \"owned\" }", + ); + sources.add("sample/echo/api.dever", declaration); + let program = dever_core::check(&sources).unwrap(); + let error = dever_core::llvm::emit_application(&program, &sources).unwrap_err(); + assert!(error.contains("sample/echo/api.dever:1:1"), "{error}"); + assert!(error.contains("requires a configured site"), "{error}"); + } + } + + pub(super) struct Server { + pub(super) directory: temp::TemporaryDirectory, + child: Child, + port: u16, + } + + impl Server { + fn start(files: &[(&str, &str)], iterations: usize) -> Self { + Self::start_with( + files, + iterations, + serde_json::from_str(&settings(8080)).unwrap(), + &[], + ) + } + + pub(super) fn start_with( + files: &[(&str, &str)], + iterations: usize, + mut setting: Value, + commands: &[Vec<String>], + ) -> Self { + let reservation = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let port = reservation.local_addr().unwrap().port(); + setting["http"]["port"] = port.into(); + let setting = setting.to_string(); + let ir = application_ir(files, &setting, true, None); + let directory = temp::TemporaryDirectory::new(); + fs::create_dir(directory.path().join("config")).unwrap(); + fs::write(directory.path().join("config/setting.json"), setting).unwrap(); + if !commands.is_empty() { + let program = + llvm_module::link_in(&directory, &ir, "checked management command", 1); + for arguments in commands { + let stderr = directory.path().join("command-stderr"); + let status = process::status( + Command::new(&program) + .env_clear() + .args(arguments) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(fs::File::create(&stderr).unwrap()), + Duration::from_secs(10), + ) + .unwrap(); + assert!( + status.success(), + "management command failed {arguments:?}: {status}\n{}", + fs::read_to_string(stderr).unwrap() + ); + } + } + let program = + llvm_module::link_in(&directory, &ir, "checked HTTP application", iterations); + drop(reservation); + let child = Command::new(program) + .env_clear() + .stdin(Stdio::null()) + .stdout(fs::File::create(directory.path().join("stdout")).unwrap()) + .stderr(fs::File::create(directory.path().join("stderr")).unwrap()) + .spawn() + .unwrap(); + Self { + directory, + child, + port, + } + } + + pub(super) fn ready(&mut self) { + self.ready_at("/sample/echo/increment?q=1"); + } + + pub(super) fn ready_at(&mut self, path: &str) { + let deadline = Instant::now() + Duration::from_secs(10); + loop { + if let Ok(response) = self.request("GET", path, &[], "") { + assert_eq!(response.status, 200); + assert_eq!(response.body["data"], 2); + return; + } + if let Some(status) = self.child.try_wait().unwrap() { + panic!( + "HTTP application exited before readiness: {status}\n{}", + self.stderr() + ); + } + assert!( + Instant::now() < deadline, + "HTTP application did not become ready\n{}", + self.stderr() + ); + std::thread::sleep(Duration::from_millis(5)); + } + } + + pub(super) fn request( + &self, + method: &str, + path: &str, + headers: &[(&str, &str)], + body: &str, + ) -> std::io::Result<Reply> { + let mut connection = TcpStream::connect_timeout( + &std::net::SocketAddr::from(([127, 0, 0, 1], self.port)), + Duration::from_secs(1), + )?; + connection.set_read_timeout(Some(Duration::from_secs(5)))?; + connection.set_write_timeout(Some(Duration::from_secs(5)))?; + write!( + connection, + "{method} {path} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\nContent-Length: {}\r\n", + body.len() + )?; + for (name, value) in headers { + write!(connection, "{name}: {value}\r\n")?; + } + write!(connection, "\r\n{body}")?; + let mut raw = Vec::new(); + connection.take(1_048_577).read_to_end(&mut raw)?; + assert!( + raw.len() <= 1_048_576, + "fixture response exceeded its bound" + ); + if !raw.windows(4).any(|bytes| bytes == b"\r\n\r\n") { + return Err(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + format!( + "{method} {path}: no complete HTTP response\n{}", + self.stderr() + ), + )); + } + Ok(Reply::parse(&raw)) + } + + pub(super) fn stop_cycle(&mut self, completed: usize) { + assert!( + process::status( + Command::new("/usr/bin/kill") + .env_clear() + .arg("-TERM") + .arg(self.child.id().to_string()), + Duration::from_secs(2), + ) + .unwrap() + .success() + ); + let deadline = Instant::now() + Duration::from_secs(5); + loop { + let output = fs::read_to_string(self.directory.path().join("stdout")).unwrap(); + if output + .lines() + .filter(|line| *line == "api-cycle-complete") + .count() + == completed + { + return; + } + if let Some(status) = self.child.try_wait().unwrap() { + panic!( + "HTTP application exited before completing cycle {completed}: {status}\n{}", + self.stderr() + ); + } + assert!( + Instant::now() < deadline, + "HTTP cycle did not drain\n{}", + self.stderr() + ); + std::thread::sleep(Duration::from_millis(5)); + } + } + + pub(super) fn finish(&mut self) { + let status = process::wait(&mut self.child, Duration::from_secs(5)).unwrap(); + assert!( + status.success(), + "HTTP application failed allocation/fault oracle: {status}\n{}", + self.stderr() + ); + } + + pub(super) fn restart(&mut self, mode: &str) { + assert!( + self.child + .try_wait() + .unwrap() + .is_some_and(|status| status.success()) + ); + let path = self.directory.path().join("config/setting.json"); + let mut setting: Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + setting["runtime"]["mode"] = mode.into(); + fs::write(path, setting.to_string()).unwrap(); + self.child = Command::new(self.directory.path().join("managed-kernel")) + .env_clear() + .stdin(Stdio::null()) + .stdout(fs::File::create(self.directory.path().join("stdout")).unwrap()) + .stderr(fs::File::create(self.directory.path().join("stderr")).unwrap()) + .spawn() + .unwrap(); + } + + pub(super) fn stderr(&self) -> String { + fs::read_to_string(self.directory.path().join("stderr")).unwrap() + } + + fn disconnect_partial_upload(&self) { + let directory = self.directory.path().join("data/tmp"); + let mut connection = TcpStream::connect_timeout( + &std::net::SocketAddr::from(([127, 0, 0, 1], self.port)), + Duration::from_secs(1), + ) + .unwrap(); + connection + .set_write_timeout(Some(Duration::from_secs(1))) + .unwrap(); + write!(connection, "POST /media/image/store HTTP/1.1\r\nHost: localhost\r\nContent-Type: multipart/form-data; boundary=boundary\r\nContent-Length: 65536\r\n\r\n--boundary\r\nContent-Disposition: form-data; name=\"file\"; filename=\"partial.txt\"\r\n\r\n").unwrap(); + // 超过解码器的一块正文，避免请求头/正文的 TCP 合并决定是否写盘； + // 仍少于 Content-Length，关闭连接才会结束这次不完整上传。 + connection.write_all(&[b'x'; 32 * 1024]).unwrap(); + let deadline = Instant::now() + Duration::from_secs(3); + while !fs::read_dir(&directory) + .unwrap() + .any(|entry| entry.unwrap().metadata().unwrap().len() > 1024) + { + assert!( + Instant::now() < deadline, + "incomplete body never reached the upload owner" + ); + std::thread::sleep(Duration::from_millis(5)); + } + connection.shutdown(std::net::Shutdown::Both).unwrap(); + drop(connection); + while fs::read_dir(&directory).unwrap().count() != 0 { + assert!( + Instant::now() < deadline, + "disconnected upload retained its temporary file" + ); + std::thread::sleep(Duration::from_millis(5)); + } + } + } + + impl Drop for Server { + fn drop(&mut self) { + if !matches!(self.child.try_wait(), Ok(Some(_))) { + let _ = process::terminate(&mut self.child); + } + } + } + + pub(super) struct Reply { + pub(super) status: u16, + pub(super) headers: Vec<(String, String)>, + pub(super) body: Value, + } + + impl Reply { + fn parse(raw: &[u8]) -> Self { + let split = raw + .windows(4) + .position(|bytes| bytes == b"\r\n\r\n") + .expect("HTTP response headers"); + let header = std::str::from_utf8(&raw[..split]).unwrap(); + let mut lines = header.split("\r\n"); + let status = lines + .next() + .unwrap() + .split_whitespace() + .nth(1) + .unwrap() + .parse() + .unwrap(); + let headers = lines + .map(|line| { + let (name, value) = line.split_once(':').unwrap(); + (name.to_ascii_lowercase(), value.trim().to_owned()) + }) + .collect::<Vec<_>>(); + assert!( + !headers + .iter() + .any(|(name, value)| name == "transfer-encoding" && value != "identity"), + "fixture requires a bounded Content-Length response" + ); + let body = serde_json::from_slice(&raw[split + 4..]).unwrap(); + Self { + status, + headers, + body, + } + } + } + + #[test] + #[ignore = "requires explicit runtime-api archive; does not start a listener"] + fn mixed_http_application_dispatches_cmd_without_binding_http() { + let reservation = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let setting = settings(reservation.local_addr().unwrap().port()); + let payload = json!({"name": "owned", "count": 4, "tags": [], "note": null}); + let ir = application_ir(&echo_files(true), &setting, false, None); + let body = json!({"payload": payload}).to_string(); + let output = llvm_module::execute( + &ir, + "API plus CMD", + 64, + Duration::from_secs(15), + &["sample.echo.echo", &body], + Some(&setting), + ); + let lines = std::str::from_utf8(&output.stdout) + .unwrap() + .lines() + .collect::<Vec<_>>(); + assert_eq!(lines.len(), 65); + for line in lines { + assert_eq!( + serde_json::from_str::<Value>(line).unwrap()["data"], + payload + ); + } + let ir = application_ir(TRANSPORT, &setting, false, Some(3)); + llvm_module::execute( + &ir, + "API CMD rejects invalid input", + 64, + Duration::from_secs(15), + &["sample.echo.echo", "{}"], + Some(&setting), + ); + } + + #[test] + #[ignore = "requires explicit runtime-api archive; starts only test-owned loopback services"] + fn public_http_inputs_context_errors_and_repeated_roots_release_owners() { + let mut server = Server::start(TRANSPORT, 64); + let mut request_ids = std::collections::BTreeSet::new(); + for cycle in 1..=65 { + server.ready(); + let context = server + .request( + "GET", + "/sample/echo/inspect?ignored=1", + &[("x-probe", "owned"), ("cookie", "probe=ordinary")], + "", + ) + .unwrap(); + assert_eq!(context.status, 400); + let context = server + .request( + "GET", + "/sample/echo/inspect", + &[("x-probe", "owned"), ("cookie", "probe=ordinary")], + "", + ) + .unwrap(); + assert_eq!(context.status, 200); + let data = &context.body["data"]; + assert_eq!(data["method"], "GET"); + assert_eq!(data["path"], "/sample/echo/inspect"); + assert_eq!(data["header"], "owned"); + assert_eq!(data["cookie"], "ordinary"); + assert_eq!(data["peer"], "127.0.0.1"); + assert!(request_ids.insert(data["request_id"].as_str().unwrap().to_owned())); + assert!( + context + .headers + .contains(&("x-response-probe".into(), "committed".into())) + ); + for path in [ + "/sample/echo/increment", + "/sample/echo/increment?q=x", + "/sample/echo/increment?q=1&q=2", + "/sample/echo/increment?q=1&unknown=2", + ] { + assert_eq!(server.request("GET", path, &[], "").unwrap().status, 400); + } + let payload = json!({"name": "你好 🌍", "count": 9_007_199_254_740_993_i64, "tags": ["owned", null], "note": null}); + let body = json!({"payload": payload}).to_string(); + let response = server + .request( + "POST", + "/sample/echo/echo", + &[("content-type", "application/json")], + &body, + ) + .unwrap(); + assert_eq!(response.status, 200, "{}", response.body); + assert_eq!( + response.body, + json!({"code": 0, "message": "ok", "data": payload}) + ); + let multipart = "--boundary\r\nContent-Disposition: form-data; name=\"caption\"\r\n\r\nowned\r\n--boundary\r\nContent-Disposition: form-data; name=\"file\"; filename=\"owned.txt\"\r\nContent-Type: text/plain\r\n\r\n你好\r\n--boundary--\r\n"; + let response = server + .request( + "POST", + "/sample/echo/upload", + &[("content-type", "multipart/form-data; boundary=boundary")], + multipart, + ) + .unwrap(); + assert_eq!(response.status, 200); + assert_eq!(response.body["data"], 6); + assert_eq!( + fs::read_dir(server.directory.path().join("data/tmp")) + .unwrap() + .count(), + 0 + ); + for (action, status) in [ + ("invalid", 400), + ("unauthorized", 401), + ("forbidden", 403), + ("missing", 404), + ("conflict", 409), + ("limited", 429), + ("fault", 500), + ] { + let response = server + .request("GET", &format!("/sample/echo/{action}"), &[], "") + .unwrap(); + assert_eq!(response.status, status); + assert_eq!(response.body["code"], status); + assert_eq!(response.body["data"], Value::Null); + assert!( + !response + .headers + .iter() + .any(|(name, _)| name == "x-response-probe") + ); + assert!(!response.body.to_string().contains("must-not-appear")); + } + assert_eq!( + server + .request("DELETE", "/sample/echo/increment", &[], "") + .unwrap() + .status, + 405 + ); + assert_eq!( + server.request("GET", "/missing", &[], "").unwrap().status, + 404 + ); + server.stop_cycle(cycle); + } + server.finish(); + assert!(!server.stderr().contains("must-not-appear-in-http-or-log")); + } + + #[test] + #[ignore = "requires explicit runtime-api archive; owns listener, upload files and storage outputs"] + fn upload_adapter_store_and_failed_input_release_affine_owners() { + let mut files = TRANSPORT.to_vec(); + files.extend([ + ( + "media/image/app.dever", + "accept(file: Upload) (key: Uuid) { key = port.store(file) }", + ), + ( + "media/image/port.dever", + "store(file: Upload) (key: Uuid) fails dever.storage.PutResult", + ), + ( + "media/image/adapter.dever", + "port.store(file: Upload) (key: Uuid) { key = dever.storage.put(file) }", + ), + ("media/image/api.dever", "public post store = app.accept"), + ]); + let mut server = Server::start(&files, 64); + let multipart = "--boundary\r\nContent-Disposition: form-data; name=\"file\"; filename=\"owned.txt\"\r\nContent-Type: text/plain\r\n\r\n你好\r\n--boundary--\r\n"; + for cycle in 1..=65 { + server.ready(); + let response = server + .request( + "POST", + "/media/image/store", + &[("content-type", "multipart/form-data; boundary=boundary")], + multipart, + ) + .unwrap(); + assert_eq!(response.status, 200, "{}", response.body); + let key = response.body["data"].as_str().unwrap(); + assert_eq!( + dever_runtime::orm::Uuid::parse(key).unwrap().to_string(), + key + ); + let stored = server.directory.path().join("data/upload").join(key); + assert_eq!(fs::read(&stored).unwrap(), "你好".as_bytes()); + fs::remove_file(stored).unwrap(); + server.disconnect_partial_upload(); + // File was decoded before the missing caption was discovered. + assert_eq!( + server + .request( + "POST", + "/sample/echo/upload", + &[("content-type", "multipart/form-data; boundary=boundary")], + multipart + ) + .unwrap() + .status, + 400 + ); + assert_eq!( + fs::read_dir(server.directory.path().join("data/tmp")) + .unwrap() + .count(), + 0 + ); + server.stop_cycle(cycle); + } + server.finish(); + } +} diff --git a/test/llvm_application_source.rs b/test/llvm_application_source.rs new file mode 100644 index 0000000..10f9b4f --- /dev/null +++ b/test/llvm_application_source.rs @@ -0,0 +1,468 @@ +//! Generated application entries, not ordinary kernels, through the real archive. + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/llvm_module.rs"] +mod llvm_module; +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/process.rs"] +mod process; +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/temp.rs"] +mod temp; + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +mod native { + use super::llvm_module; + use dever_backend_bridge::{Target, emit_object}; + use dever_core::{check, hir::Program, source::SourceMap}; + use std::time::Duration; + + const ECHO: &[(&str, &str)] = &[ + ( + "sample/echo/app.dever", + r#"type Payload { + name: Text + count: Int + tags: List<Text?> + note: Text? +} +echo(payload: Payload) (value: Payload) { value = payload } +"#, + ), + ("sample/echo/api.dever", "cmd echo = app.echo"), + ]; + + const SCALARS: &[(&str, &str)] = &[ + ( + "sample/scalar/app.dever", + r#"type Scalars { + enabled: Bool + number: Float + money: Decimal + identity: Id + uuid: Uuid + day: Date + time: Time + instant: DateTime + duration: Duration + raw: Json +} +echo(payload: Scalars) (value: Scalars) { value = payload } +"#, + ), + ("sample/scalar/api.dever", "cmd echo = app.echo"), + ]; + + const ASYNC: &[(&str, &str)] = &[ + ( + "sample/echo/app.dever", + r#"type Payload { + name: Text + tags: List<Text?> +} +echo(payload: Payload) (value: Payload) { + dever.task.sleep(1) + value = payload +} +"#, + ), + ("sample/echo/api.dever", "cmd echo = app.echo"), + ]; + + const PORT: &[(&str, &str)] = &[ + ( + "notification/mail/app.dever", + r#"read() (value: Int) { value = port.read() } +"#, + ), + ( + "notification/mail/port.dever", + "read() (value: Int) fails dever.time.SleepResult", + ), + ( + "notification/mail/adapter/fixed.dever", + "port.read() (value: Int) { value = 7 }", + ), + ( + "notification/mail/adapter/configured.dever", + r#"setting { + token: Secret + value: Int +} +port.read() (value: Int) { + dever.task.sleep(1) + value = setting.value +} +"#, + ), + ("notification/mail/api.dever", "cmd read = app.read"), + ]; + + const FAILURES: &[(&str, &str)] = &[ + ( + "sample/action/app.dever", + r#"type Outcome { + Done(value: Text) + error Denied(message: Text) +} +load(allowed: true) (value: Text) { value = "accepted" } +load(allowed: false) (value: Text) { fail(Outcome.Denied("owned denial")) } +read(value: Outcome.Done(text)) (answer: Text) recover("return the CMD failure message") { answer = text } +read(value: Outcome.Denied(message)) (answer: Text) recover("return the CMD failure message") { answer = message } +capture(allowed: Bool) (answer: Text) { answer = read(result(load(allowed))) } +divide(divisor: Int) (answer: Int) { answer = 42 // divisor } +"#, + ), + ( + "sample/action/api.dever", + "cmd load = app.load\ncmd capture = app.capture\ncmd divide = app.divide", + ), + ]; + + fn checked(files: &[(&str, &str)]) -> (SourceMap, Program) { + let mut sources = SourceMap::default(); + for (path, text) in files { + sources.add(*path, *text); + } + let program = check(&sources).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }); + (sources, program) + } + + fn lower(files: &[(&str, &str)]) -> String { + let (sources, program) = checked(files); + dever_core::llvm::emit_application(&program, &sources).unwrap() + } + + fn execute( + files: &[(&str, &str)], + arguments: &[&str], + setting: Option<&str>, + fault_code: Option<u32>, + fault_message: Option<&str>, + ) -> std::process::Output { + let mut ir = lower(files); + let check = match fault_code { + None => " %passed = icmp eq i32 %status, 0".to_owned(), + Some(code) => llvm_module::fault_check(code, fault_message), + }; + ir.push_str(&format!( + "\ndefine i32 @dever_test_run() {{\nentry:\n %out = alloca i8\n %fault = alloca %dever.fault\n %status = call i32 @dever_application_entry(ptr %out, ptr %fault)\n{check}\n call void @dever_outputs_release(ptr %out)\n call void @dever_fault_release(ptr %fault)\n %exit = select i1 %passed, i32 0, i32 1\n ret i32 %exit\n}}\n" + )); + llvm_module::execute( + &ir, + "checked CMD application", + 64, + Duration::from_secs(10), + arguments, + setting, + ) + } + + fn success(files: &[(&str, &str)], name: &str, body: &str, setting: Option<&str>, data: &str) { + let output = execute(files, &[name, body], setting, None, None); + let expected = format!("{{\"code\":0,\"message\":\"ok\",\"data\":{data}}}\n"); + // One warmup plus 64 calls: every entry must reinitialize and release owners. + assert_eq!(output.stdout, expected.repeat(65).as_bytes()); + assert!(output.stderr.is_empty()); + } + + #[test] + fn application_fixtures_pass_the_existing_checker() { + for files in [ECHO, SCALARS, ASYNC, PORT, FAILURES] { + checked(files); + } + } + + #[test] + fn command_applications_emit_objects_for_all_six_targets() { + let ir = lower(ECHO); + for target in Target::ALL { + assert!(!emit_object(&ir, target).unwrap().is_empty()); + } + } + + #[test] + fn ordinary_kernel_is_not_an_application_entry() { + let (sources, program) = checked(ECHO); + assert!(dever_core::llvm::emit_kernel(&program, &sources, "sample.echo.echo").is_err()); + } + + #[test] + fn unsupported_application_metadata_is_rejected_at_its_source() { + let files = [ + ( + "sample/echo/app.dever", + "echo() (value: Text) { value = \"okay\" }", + ), + ( + "sample/echo/api.dever", + "public get echo = app.echo\ncmd echo = app.echo", + ), + ]; + let (sources, program) = checked(&files); + let error = dever_core::llvm::emit_application(&program, &sources).unwrap_err(); + assert!(error.contains("sample/echo/api.dever"), "{error}"); + } + + #[test] + #[ignore = "requires explicitly prepared runtime-only archive; no database or service"] + fn cmd_owned_record_list_unicode_and_missing_nullable() { + success( + ECHO, + "sample.echo.echo", + r#"{"payload":{"name":"你好 🌍","count":9223372036854775807,"tags":["first",null,"last"]}}"#, + None, + r#"{"name":"你好 🌍","count":9223372036854775807,"tags":["first",null,"last"],"note":null}"#, + ); + } + + #[test] + #[ignore = "requires explicitly prepared runtime-only archive; no database or service"] + fn cmd_scalar_codecs_preserve_existing_wire_formats() { + let body = r#"{"payload":{"enabled":true,"number":1.5,"money":"123.45","identity":"external-id","uuid":"01890f3e-7c00-7000-8000-000000000001","day":"2026-10-01","time":"12:34:56.789","instant":"2026-10-01T12:34:56.789Z","duration":-42,"raw":{"large":9999999999999999999999999999,"nested":[true,null,"value"]}}}"#; + success( + SCALARS, + "sample.scalar.echo", + body, + None, + &body[11..body.len() - 1], + ); + } + + #[test] + #[ignore = "requires explicitly prepared runtime-only archive; no database or service"] + fn cmd_owned_inputs_survive_the_initial_suspend() { + success( + ASYNC, + "sample.echo.echo", + r#"{"payload":{"name":"pending-owned","tags":["one",null,"three"]}}"#, + None, + r#"{"name":"pending-owned","tags":["one",null,"three"]}"#, + ); + } + + #[test] + #[ignore = "requires explicitly prepared runtime-only archive; no database or service"] + fn cmd_input_errors_release_partially_decoded_owners() { + for body in [ + r#"{"payload":{"name":"owned","count":"bad","tags":["first"],"note":null}}"#, + r#"{"payload":{"name":"owned","count":1,"tags":["first",false]}}"#, + r#"{"payload":{"name":"owned","count":1,"tags":[]},"unknown":1}"#, + r#"{"payload":{"name":"owned","name":"duplicate","count":1,"tags":[]}}"#, + r#"{"payload":{"name":"owned","tags":[]}}"#, + r#"{"payload":{"name":"owned","count":9223372036854775808,"tags":[]}}"#, + r#"[]"#, + ] { + let output = execute(ECHO, &["sample.echo.echo", body], None, Some(3), None); + assert!(output.stdout.is_empty()); + } + } + + #[test] + #[ignore = "requires explicitly prepared runtime-only archive; no database or service"] + fn cmd_argument_errors_do_not_run_app() { + for arguments in [ + &[][..], + &["sample.echo.echo"][..], + &["unknown", "{}"][..], + &["sample.echo.echo", "{}", "extra"][..], + ] { + let output = execute(ECHO, arguments, None, Some(3), None); + assert!(output.stdout.is_empty()); + } + } + + #[test] + #[ignore = "requires explicitly prepared runtime-only archive; no database or service"] + fn port_selects_only_the_configured_adapter_and_releases_settings() { + for (setting, value) in [ + (r#"{"adapter":{"notification.mail":{"use":"fixed"}}}"#, "7"), + ( + r#"{"adapter":{"notification.mail":{"use":"configured","setting":{"value":19,"token":"private-token"}}}}"#, + "19", + ), + ] { + success(PORT, "notification.mail.read", "{}", Some(setting), value); + } + } + + #[test] + #[ignore = "requires explicitly prepared runtime-only archive; no database or service"] + fn selected_adapter_invalid_setting_fails_closed() { + for setting in [ + r#"{"adapter":{"notification.mail":{"use":"unknown"}}}"#, + r#"{"adapter":{"notification.mail":{"use":"configured"}}}"#, + r#"{"adapter":{"notification.mail":{"use":"configured","setting":{"value":"private-token","token":"private-token"}}}}"#, + r#"{"adapter":{"notification.mail":{"use":"fixed","use":"configured"}}}"#, + r#"{"adapter":{"unknown.port":{"use":"fixed"}}}"#, + ] { + let output = execute( + PORT, + &["notification.mail.read", "{}"], + Some(setting), + Some(3), + None, + ); + assert!(output.stdout.is_empty()); + } + } + + #[test] + #[ignore = "requires explicitly prepared runtime-only archive; no database or service"] + fn cmd_keeps_typed_business_faults_and_explicit_recovery() { + let output = execute( + FAILURES, + &["sample.action.load", r#"{"allowed":false}"#], + None, + Some(4), + None, + ); + assert!(output.stdout.is_empty()); + success( + FAILURES, + "sample.action.capture", + r#"{"allowed":false}"#, + None, + r#""owned denial""#, + ); + success( + FAILURES, + "sample.action.capture", + r#"{"allowed":true}"#, + None, + r#""accepted""#, + ); + } + + #[test] + #[ignore = "requires explicitly prepared runtime-only archive; no database or service"] + fn cmd_numeric_fault_is_not_reclassified_as_input_or_business_error() { + let output = execute( + FAILURES, + &["sample.action.divide", r#"{"divisor":0}"#], + None, + Some(2), + None, + ); + assert!(output.stdout.is_empty()); + } + + #[test] + #[ignore = "requires explicitly prepared runtime-only archive; no database or service"] + fn cmd_preserves_native_input_diagnostics_and_validation_order() { + for (body, message) in [ + (r#"{}"#, "input 'payload' is required"), + (r#"[]"#, "invalid JSON object body: expected wire record"), + ( + r#"{"payload":{"name":"owned","count":1,"tags":[]},"unknown":false}"#, + "unknown input 'unknown'", + ), + ( + r#"{"payload":{"name":"owned","count":"bad","tags":[]},"unknown":false}"#, + "expected wire integer", + ), + ] { + let output = execute( + ECHO, + &["sample.echo.echo", body], + None, + Some(3), + Some(message), + ); + assert!(output.stdout.is_empty()); + } + } + + #[test] + #[ignore = "requires explicitly prepared runtime-only archive; no database or service"] + fn selected_setting_diagnostic_is_redacted_after_partial_secret_decode() { + let setting = r#"{"adapter":{"notification.mail":{"use":"configured","setting":{"value":"private-token","token":"private-token"}}}}"#; + let output = execute( + PORT, + &["notification.mail.read", "{}"], + Some(setting), + Some(3), + Some("expected wire integer"), + ); + assert!(output.stdout.is_empty()); + } + + #[test] + #[ignore = "requires explicitly prepared runtime-only archive; no database or service"] + fn cmd_top_level_nullable_missing_null_and_owned_present() { + let files = [ + ( + "sample/nullable/app.dever", + "echo(name: Text?) (value: Text?) { value = name }", + ), + ("sample/nullable/api.dever", "cmd echo = app.echo"), + ]; + for (body, data) in [ + ("{}", "null"), + (r#"{"name":null}"#, "null"), + (r#"{"name":"owned nullable"}"#, r#""owned nullable""#), + ] { + success(&files, "sample.nullable.echo", body, None, data); + } + } + + #[test] + #[ignore = "requires explicitly prepared runtime-only archive; no database or service"] + fn single_unconfigured_adapter_needs_no_setting_file() { + let files = [ + ( + "notification/mail/app.dever", + "read() (value: Int) { value = port.read() }", + ), + ( + "notification/mail/port.dever", + "read() (value: Int) fails dever.time.SleepResult", + ), + ( + "notification/mail/adapter.dever", + "port.read() (value: Int) { value = 7 }", + ), + ("notification/mail/api.dever", "cmd read = app.read"), + ]; + success(&files, "notification.mail.read", "{}", None, "7"); + } + + #[test] + #[ignore = "requires explicitly prepared runtime-only archive; no database or service"] + fn task_reads_selected_setting_within_the_application_scope() { + let mut files = PORT.to_vec(); + files[0].1 = "read() (value: Int) { pending = run(port.read())\n value = wait(pending) }"; + let setting = r#"{"adapter":{"notification.mail":{"use":"configured","setting":{"value":29,"token":"private-task-token"}}}}"#; + success(&files, "notification.mail.read", "{}", Some(setting), "29"); + } + + #[test] + #[ignore = "requires explicitly prepared runtime-only archive; no database or service"] + fn later_adapter_initialization_failure_releases_earlier_settings_and_gate() { + let mut files = PORT.to_vec(); + files.extend([ + ("notification/sms/app.dever", "read() (value: Int) { value = port.read() }"), + ("notification/sms/port.dever", "read() (value: Int) fails dever.time.SleepResult"), + ("notification/sms/adapter.dever", "setting { value: Int }\nport.read() (value: Int) { value = setting.value }"), + ("notification/send/app.dever", "read() (value: Int) { value = notification.mail.read() + notification.sms.read() }"), + ("notification/send/api.dever", "cmd read = app.read"), + ]); + let setting = r#"{"adapter":{"notification.mail":{"use":"configured","setting":{"value":19,"token":"private-earlier-token"}}}}"#; + let output = execute( + &files, + &["notification.send.read", "{}"], + Some(setting), + Some(3), + Some("selected Adapter requires setting"), + ); + assert!(output.stdout.is_empty()); + } +} diff --git a/test/llvm_async_native.rs b/test/llvm_async_native.rs new file mode 100644 index 0000000..288ea7a --- /dev/null +++ b/test/llvm_async_native.rs @@ -0,0 +1,280 @@ +//! Actual LLVM coroutine object linked with the runtime-only archive. + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/process.rs"] +mod process; +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/temp.rs"] +mod temp; + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[test] +#[ignore = "build runtime-abi-only staticlib in target/native-runtime-abi before running"] +fn switched_resume_coroutine_suspends_resumes_and_destroys() { + use dever_backend_bridge::{Target, emit_object}; + use std::path::Path; + use std::process::{Command, Stdio}; + use std::time::Duration; + + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let archive = root.join("target/native-runtime-abi/debug/libdever_backend_bridge.a"); + assert!( + archive.is_file(), + "missing explicit runtime ABI fixture archive" + ); + let directory = temp::TemporaryDirectory::new(); + let object = directory.path().join("coroutine.o"); + std::fs::write( + &object, + emit_object(COROUTINE, Target::LinuxX86_64).unwrap(), + ) + .unwrap(); + let executable = directory.path().join("coroutine"); + let compiler = Command::new("/usr/bin/cc") + .env_clear() + .args(["-std=c11", "-Wall", "-Wextra", "-Werror", "-O2"]) + .arg(root.join("test/native-runtime-abi/async-driver.c")) + .arg(object) + .arg(archive) + .args(["-ldl", "-lpthread", "-lm", "-o"]) + .arg(&executable) + .arg("-B/usr/bin/") + .output() + .unwrap(); + assert!( + compiler.status.success(), + "{}", + String::from_utf8_lossy(&compiler.stderr) + ); + let status = process::status( + Command::new(executable) + .env_clear() + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()), + Duration::from_secs(5), + ) + .unwrap(); + assert!(status.success(), "coroutine driver failed: {status}"); +} + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[test] +#[ignore = "build runtime-abi-only staticlib in target/native-runtime-abi before running"] +fn async_operations_preserve_scope_and_backpressure() { + use std::path::Path; + use std::process::{Command, Stdio}; + use std::time::Duration; + + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let archive = root.join("target/native-runtime-abi/debug/libdever_backend_bridge.a"); + assert!( + archive.is_file(), + "missing explicit runtime ABI fixture archive" + ); + let directory = temp::TemporaryDirectory::new(); + let executable = directory.path().join("async-operations"); + let compiler = Command::new("/usr/bin/cc") + .env_clear() + .args(["-std=c11", "-Wall", "-Wextra", "-Werror", "-O2"]) + .arg("-I") + .arg(root.join("crates/dever-backend-bridge/include")) + .arg(root.join("test/native-runtime-abi/async-operations-driver.c")) + .arg(archive) + .args(["-ldl", "-lpthread", "-lm", "-o"]) + .arg(&executable) + .arg("-B/usr/bin/") + .output() + .unwrap(); + assert!( + compiler.status.success(), + "{}", + String::from_utf8_lossy(&compiler.stderr) + ); + let status = process::status( + Command::new(executable) + .env_clear() + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()), + Duration::from_secs(5), + ) + .unwrap(); + assert!(status.success(), "async operation driver failed: {status}"); +} + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +const COROUTINE: &str = r#" +@progress = global i32 0 +declare ptr @malloc(i64) +declare void @free(ptr) +declare token @llvm.coro.id(i32, ptr, ptr, ptr) +declare i64 @llvm.coro.size.i64() +declare ptr @llvm.coro.begin(token, ptr) +declare i8 @llvm.coro.suspend(token, i1) +declare ptr @llvm.coro.free(token, ptr) +declare i1 @llvm.coro.end(ptr, i1, token) +declare void @llvm.coro.resume(ptr) +declare void @llvm.coro.destroy(ptr) +declare i1 @llvm.coro.done(ptr) +declare i64 @llvm.coro.align.i64() +declare ptr @dever_rt_v1_async_frame_alloc(i64, i64) +declare void @dever_rt_v1_async_frame_free(ptr, i64, i64) +declare void @dever_rt_v1_async_complete(ptr, i32) +declare i32 @dever_rt_v1_async_root(ptr, ptr, ptr, ptr, ptr) +declare ptr @dever_rt_v1_async_sleep(i64) +declare i32 @dever_rt_v1_async_op_poll(ptr, ptr, ptr, ptr, ptr, ptr) +declare void @dever_rt_v1_async_op_release(ptr) +@root_polls = global i32 0 + +@probe_type = private constant { i64, i64, ptr, ptr, i8 } { i64 8, i64 8, ptr @probe_move, ptr @probe_drop, i8 1 } +@probe_function = private constant { ptr, ptr, ptr, ptr, ptr, ptr, i8 } { + ptr @probe_type, ptr @probe_type, ptr @dever_coro_root_create, + ptr @dever_coro_root_resume, ptr @dever_coro_root_destroy, ptr @dever_coro_root_done, i8 1 +} + +define void @probe_move(ptr %source, ptr %destination) { +entry: + %value = load i64, ptr %source + store i64 %value, ptr %destination + store i64 0, ptr %source + ret void +} + +define void @probe_drop(ptr %value) { +entry: + ret void +} + +define ptr @dever_coro_probe_create() presplitcoroutine { +entry: + %id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null) + %size = call i64 @llvm.coro.size.i64() + %memory = call ptr @malloc(i64 %size) + %frame = call ptr @llvm.coro.begin(token %id, ptr %memory) + %initial = call i8 @llvm.coro.suspend(token none, i1 false) + switch i8 %initial, label %suspend [i8 0, label %body i8 1, label %cleanup] +body: + store i32 42, ptr @progress + %final = call i8 @llvm.coro.suspend(token none, i1 true) + switch i8 %final, label %suspend [i8 1, label %cleanup] +cleanup: + %allocation = call ptr @llvm.coro.free(token %id, ptr %frame) + call void @free(ptr %allocation) + br label %suspend +suspend: + call i1 @llvm.coro.end(ptr %frame, i1 false, token none) + ret ptr %frame +} + +define i32 @dever_coro_probe_progress() { +entry: + %value = load i32, ptr @progress + ret i32 %value +} + +define void @dever_coro_probe_resume(ptr %frame) { +entry: + call void @llvm.coro.resume(ptr %frame) + ret void +} + +define i1 @dever_coro_probe_done(ptr %frame) { +entry: + %done = call i1 @llvm.coro.done(ptr %frame) + ret i1 %done +} + +define void @dever_coro_probe_destroy(ptr %frame) { +entry: + call void @llvm.coro.destroy(ptr %frame) + ret void +} + +define ptr @dever_coro_root_create(ptr %state, ptr %input, ptr %output, ptr %fault) presplitcoroutine { +entry: + %id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null) + %size = call i64 @llvm.coro.size.i64() + %align = call i64 @llvm.coro.align.i64() + %memory = call ptr @dever_rt_v1_async_frame_alloc(i64 %size, i64 %align) + %frame = call ptr @llvm.coro.begin(token %id, ptr %memory) + %operation = alloca ptr + store ptr null, ptr %operation + %error = alloca { ptr, i64 } + %initial = call i8 @llvm.coro.suspend(token none, i1 false) + switch i8 %initial, label %suspend [i8 0, label %body i8 1, label %cleanup] +body: + %sleep = call ptr @dever_rt_v1_async_sleep(i64 1) + store ptr %sleep, ptr %operation + br label %poll_sleep +poll_sleep: + %polls = load i32, ptr @root_polls + %next_polls = add i32 %polls, 1 + store i32 %next_polls, ptr @root_polls + %active = load ptr, ptr %operation + %sleep_status = call i32 @dever_rt_v1_async_op_poll(ptr %state, ptr %active, ptr null, ptr null, ptr null, ptr %error) + %pending = icmp eq i32 %sleep_status, 0 + br i1 %pending, label %wait_sleep, label %finished_sleep +wait_sleep: + %waiting = call i8 @llvm.coro.suspend(token none, i1 false) + switch i8 %waiting, label %suspend [i8 0, label %poll_sleep i8 1, label %cleanup] +finished_sleep: + %completed_op = load ptr, ptr %operation + call void @dever_rt_v1_async_op_release(ptr %completed_op) + store ptr null, ptr %operation + store i64 42, ptr %output + call void @dever_rt_v1_async_complete(ptr %state, i32 1) + %final = call i8 @llvm.coro.suspend(token none, i1 true) + switch i8 %final, label %suspend [i8 1, label %cleanup] +cleanup: + %remaining = load ptr, ptr %operation + %has_op = icmp ne ptr %remaining, null + br i1 %has_op, label %release_op, label %free_frame +release_op: + call void @dever_rt_v1_async_op_release(ptr %remaining) + br label %free_frame +free_frame: + %allocation = call ptr @llvm.coro.free(token %id, ptr %frame) + call void @dever_rt_v1_async_frame_free(ptr %allocation, i64 %size, i64 %align) + br label %suspend +suspend: + call i1 @llvm.coro.end(ptr %frame, i1 false, token none) + ret ptr %frame +} + +define void @dever_coro_root_resume(ptr %frame) { +entry: + call void @llvm.coro.resume(ptr %frame) + ret void +} + +define i8 @dever_coro_root_done(ptr %frame) { +entry: + %done = call i1 @llvm.coro.done(ptr %frame) + %result = zext i1 %done to i8 + ret i8 %result +} + +define void @dever_coro_root_destroy(ptr %frame) { +entry: + call void @llvm.coro.destroy(ptr %frame) + ret void +} + +define i32 @dever_coro_root_run() { +entry: + %output = alloca i64 + %fault = alloca i64 + %error = alloca { ptr, i64 } + %status = call i32 @dever_rt_v1_async_root(ptr @probe_function, ptr null, ptr %output, ptr %fault, ptr %error) + %value = load i64, ptr %output + %value_ok = icmp eq i64 %value, 42 + %status_ok = icmp eq i32 %status, 1 + %polls = load i32, ptr @root_polls + %suspended = icmp uge i32 %polls, 2 + %result_ok = and i1 %value_ok, %status_ok + %passed = and i1 %result_ok, %suspended + %code = select i1 %passed, i32 0, i32 1 + ret i32 %code +} +"#; diff --git a/test/llvm_async_source.rs b/test/llvm_async_source.rs new file mode 100644 index 0000000..efd7ecf --- /dev/null +++ b/test/llvm_async_source.rs @@ -0,0 +1,639 @@ +//! Checked-source async acceptance against the real LLVM object and runtime archive. + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/llvm.rs"] +mod llvm; +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/llvm_managed.rs"] +mod llvm_managed; +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/process.rs"] +mod process; +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/temp.rs"] +mod temp; + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +mod native { + use super::{llvm, llvm_managed}; + use dever_core::{check, source::SourceMap}; + use std::fmt::Write; + + const TIMER: &str = r#"slept(value: dever.time.SleepResult.Done) (okay: Bool) recover("test checks timer result") { okay = true } +slept(value: dever.time.SleepResult.Failed(message)) (okay: Bool) recover("test checks timer result") { okay = false } +"#; + + const NAMED_VALUES: &str = r#"type Bundle { + label: Text + names: List<Text> + lookup: Map<Text, List<Text>> + sequence: List<Int> + first: Int + second: Int + third: Int + fourth: Int + fifth: Int + sixth: Int +} +delayed(label: Text, names: List<Text>, lookup: Map<Text, List<Text>>) (bundle: Bundle, ready: Bool) { + ready = slept(result(dever.task.sleep(1))) + bundle = Bundle { label = label + names = names + lookup = lookup + sequence = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + first = 11 + second = 12 + third = 13 + fourth = 14 + fifth = 15 + sixth = 16 } +} +public main() (answer: Bool) { + value = delayed("owned", ["first", "second"], { "key" = ["inside"] }) + answer = value.ready and value.bundle.label == "owned" and value.bundle.names == ["first", "second"] and get(value.bundle.lookup, "key") == optional_names(["inside"]) and value.bundle.sequence == [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] and value.bundle.first == 11 and value.bundle.second == 12 and value.bundle.third == 13 and value.bundle.fourth == 14 and value.bundle.fifth == 15 and value.bundle.sixth == 16 +} +optional_names(value: List<Text>) (answer: List<Text>?) { answer = value } +"#; + + const TASKS_AND_GROUPS: &str = r#"some_text(value: Text) (answer: Text?) { answer = value } +delayed(value: Text, names: List<Text>) (answer: Text) { + ready = slept(result(dever.task.sleep(1))) + answer = value + first_name(names) + suffix(ready) +} +first_name(names: List<Text>) (answer: Text) { answer = read_name(first(names)) } +read_name(value: Text) (answer: Text) { answer = value } +read_name(value: null) (answer: Text) { answer = "missing" } +suffix(ready: true) (answer: Text) { answer = "!" } +suffix(ready: false) (answer: Text) { answer = "bad" } +send_label(channel: Channel<Text>, label: Text) () { send(channel, label) } +public main() (answer: Bool) { + task = run(delayed("A", ["B"])) + result = wait(task) + channel = channel(Text, 2) + workers = group(1) + run(workers, send_label(channel, "first")) + run(workers, send_label(channel, "second")) + wait(workers) + first_value = receive(channel) + second_value = receive(channel) + close(channel) + answer = result == "AB!" and first_value == some_text("first") and second_value == some_text("second") +} +"#; + + const NULLABLE_CHANNEL: &str = r#"some_int(value: Int) (answer: Int?) { answer = value } +send_later(channel: Channel<Int?>) () { send(channel, some_int(7)) } +public main() (answer: Bool) { + channel = channel(Int?, 1) + send(channel, null) + sender = run(send_later(channel)) + ready = slept(result(dever.task.sleep(1))) + first_value = receive(channel) + wait(sender) + close(channel) + second_value = receive(channel) + end = receive(channel) + answer = ready and first_value == null and second_value == some_int(7) and end == null +} +"#; + + const TASK_BUSINESS_FAILURE: &str = r#"type Detail { + message: Text + history: List<Text> +} +type Outcome { + Done(value: Text) + error Failed(detail: Detail) +} +fail_job() (answer: Text) { fail(Outcome.Failed(Detail { message = "task failed" + history = ["before", "task"] })) } +wrapper() (answer: Text) { + task = run(fail_job()) + answer = wait(task) +} +inspect(value: Outcome.Done(text)) (answer: Bool) recover("test expects task failure") { answer = false } +inspect(value: Outcome.Failed(detail)) (answer: Bool) recover("test expects task failure") { answer = detail.message == "task failed" and detail.history == ["before", "task"] } +public main() (answer: Bool) { answer = inspect(result(wrapper())) } +"#; + + const CANCEL_AND_NESTED: &str = r#"type Outcome { + Done + error Failed(message: Text) +} +wait_long(value: Text) () { ready = slept(result(dever.task.sleep(1000))) } +parent() () { + child = run(wait_long("descendant")) + fail(Outcome.Failed("parent exit")) +} +inspect(value: Outcome.Done) (answer: Bool) recover("test expects parent failure") { answer = false } +inspect(value: Outcome.Failed(message)) (answer: Bool) recover("test expects parent failure") { answer = message == "parent exit" } +public main() (answer: Bool) { + task = run(wait_long("task")) + ready = slept(result(dever.task.sleep(1))) + stop(task) + workers = group(1) + run(workers, wait_long("group")) + stop(workers) + answer = ready and inspect(result(parent())) +} +"#; + + const BLOCKING_AND_PARALLEL: &str = r#"double(value: Int) (answer: Int) pure { answer = value * 2 } +pause() (ready: Bool) { ready = slept(result(dever.time.sleep(1))) } +public main() (answer: Bool) { + number = parallel(double(21)) + ready = blocking(pause()) + answer = number == 42 and ready +} +"#; + + const CAPTURE_THEN_REFAIL: &str = r#"type Outcome { + Done(value: Text) + error Failed(message: Text, history: List<Text>) +} +origin() (answer: Text) { + ready = slept(result(dever.task.sleep(1))) + fail(Outcome.Failed("origin", ["owned", "before"])) +} +rethrow(value: Outcome.Done(text)) (answer: Text) { answer = text } +rethrow(value: Outcome.Failed(message, history)) (answer: Text) { + fail(Outcome.Failed(message + " again", append(history, "after"))) +} +wrapper() (answer: Text) { answer = rethrow(result(origin())) } +inspect(value: Outcome.Done(text)) (answer: Bool) recover("test expects repeated failure") { answer = false } +inspect(value: Outcome.Failed(message, history)) (answer: Bool) recover("test expects repeated failure") { answer = message == "origin again" and history == ["owned", "before", "after"] } +public main() (answer: Bool) { answer = inspect(result(wrapper())) } +"#; + + const UNCAPTURED_RUNTIME_FAULT: &str = r#"divide(value: Int, denominator: Int) (answer: Int) pure { answer = value // denominator } +worker() (answer: Int) { + ready = slept(result(dever.task.sleep(1))) + answer = guard(parallel(divide(42, 0))) +} +type Outcome { + Done(value: Int) + error Failed(message: Text) +} +guard(value: 0) (answer: Int) { fail(Outcome.Failed("zero")) } +guard(value: other) (answer: Int) { answer = value } +inspect(value: Outcome.Done(number)) (answer: Bool) recover("test checks capture") { answer = number == 42 } +inspect(value: Outcome.Failed(message)) (answer: Bool) recover("test checks capture") { answer = false } +public main() (answer: Bool) { answer = inspect(result(worker())) } +"#; + + const BLOCKING_RUNTIME_FAULT: &str = r#"pause_divide(denominator: Int) (answer: Int) { + ready = slept(result(dever.time.sleep(1))) + answer = 42 // denominator +} +public main() (answer: Int) { answer = blocking(pause_divide(0)) } +"#; + + const DEFAULT_FAILURE_TRACE: &str = r#"type Outcome { + Done + error Failed(message: Text, history: List<Text>) +} +origin() () { + ready = slept(result(dever.task.sleep(1))) + fail(Outcome.Failed("origin", ["owned", "history"])) +} +middle() () { origin() } +public main() () { middle() } +"#; + + const ASYNC_STATIC_TRAVERSAL: &str = r#"label(value: Text, prefix: Text) (answer: Text) { answer = prefix + value } +collect(value: Text, state: List<Text>) (next: List<Text>) { next = append(state, value) } +add(value: Int, state: Int) (next: Int) { next = state + value } +public main() (answer: Bool) { + ready = slept(result(dever.task.sleep(1))) + labels = each(label, ["one", "two"], "item=") + collected = reduce(collect, labels, ["seed"]) + bytes = dever.bytes.from_text("AB") + total = reduce(add, bytes, 0) + answer = ready and labels == ["item=one", "item=two"] and collected == ["seed", "item=one", "item=two"] and total == 131 +} +"#; + + const SYNC_TASK_AND_GROUP: &str = r#"type Bundle { + label: Text + names: List<Text> + lookup: Map<Text, List<Text>> +} +make(label: Text, names: List<Text>, lookup: Map<Text, List<Text>>) (text: Text, items: List<Text>, bundle: Bundle) { + text = label + "!" + items = append(names, "last") + bundle = Bundle { label = label + names = names + lookup = lookup } +} +send_label(channel: Channel<Text>, label: Text) () { send(channel, label) } +some_text(value: Text) (answer: Text?) { answer = value } +public main() (answer: Bool) { + channel = channel(Text, 1) + workers = group(1) + run(workers, send_label(channel, "group")) + wait(workers) + received = receive(channel) + close(channel) + task = run(make("sync", ["first"], { "key" = ["inside"] })) + value = wait(task) + answer = value.text == "sync!" and value.items == ["first", "last"] and value.bundle.label == "sync" and value.bundle.names == ["first"] and get(value.bundle.lookup, "key") == optional_names(["inside"]) and received == some_text("group") +} +optional_names(value: List<Text>) (answer: List<Text>?) { answer = value } +"#; + + const OWNED_BLOCKING_AND_PARALLEL: &str = r#"type Bundle { + label: Text + names: List<Text> + lookup: Map<Text, List<Text>> +} +transform(label: Text, names: List<Text>, lookup: Map<Text, List<Text>>) (bundle: Bundle, names_again: List<Text>) pure { + bundle = Bundle { label = label + "!" + names = append(names, "tail") + lookup = lookup } + names_again = names +} +blocking_transform(label: Text, names: List<Text>, lookup: Map<Text, List<Text>>) (bundle: Bundle, names_again: List<Text>) { + ready = slept(result(dever.time.sleep(1))) + value = transform(label, names, lookup) + bundle = value.bundle + names_again = value.names_again +} +optional_names(value: List<Text>) (answer: List<Text>?) { answer = value } +check_blocking() (answer: Bool) { + value = blocking(blocking_transform("block", ["a"], { "key" = ["b"] })) + answer = value.bundle.label == "block!" and value.bundle.names == ["a", "tail"] and value.names_again == ["a"] and get(value.bundle.lookup, "key") == optional_names(["b"]) +} +check_parallel() (answer: Bool) { + value = parallel(transform("parallel", ["x"], { "key" = ["y"] })) + answer = value.bundle.label == "parallel!" and value.bundle.names == ["x", "tail"] and value.names_again == ["x"] and get(value.bundle.lookup, "key") == optional_names(["y"]) +} +public main() (answer: Bool) { + ready = slept(result(dever.task.sleep(1))) + background = check_blocking() + compute = check_parallel() + answer = ready and background and compute +} +"#; + + const BLOCKING_AND_PARALLEL_BUSINESS_FAILURE: &str = r#"type Detail { + message: Text + history: List<Text> +} +type Outcome { + Done(value: Text) + error Failed(detail: Detail) +} +fail_owned(label: Text, history: List<Text>) (answer: Text) pure { + fail(Outcome.Failed(Detail { message = label + history = append(history, "failed") })) +} +fail_owned_blocking(label: Text, history: List<Text>) (answer: Text) { + ready = slept(result(dever.time.sleep(1))) + answer = fail_owned(label, history) +} +blocking_job() (answer: Text) { answer = blocking(fail_owned_blocking("blocking", ["before"])) } +parallel_job() (answer: Text) { answer = parallel(fail_owned("parallel", ["before"])) } +inspect(value: Outcome.Done(text), expected: Text) (answer: Bool) recover("test expects business failure") { answer = false } +inspect(value: Outcome.Failed(detail), expected: Text) (answer: Bool) recover("test expects business failure") { answer = detail.message == expected and detail.history == ["before", "failed"] } +public main() (answer: Bool) { + ready = slept(result(dever.task.sleep(1))) + answer = ready and inspect(result(blocking_job()), "blocking") and inspect(result(parallel_job()), "parallel") +} +"#; + + const NEGATIVE_TASK_SLEEP: &str = r#"inspect(value: dever.time.SleepResult.Done) (answer: Bool) recover("test checks negative timer") { answer = false } +inspect(value: dever.time.SleepResult.Failed(message)) (answer: Bool) recover("test checks negative timer") { answer = message == "sleep duration must not be negative" } +public main() (answer: Bool) { answer = inspect(result(dever.task.sleep(-1))) } +"#; + + const SYNC_CHANNEL_ONLY: &str = r#"public main() (answer: Bool) { + channel = channel(Int, 1) + alias = channel + answer = true +} +"#; + + fn source(body: &str) -> String { + format!("{TIMER}\n{body}") + } + + fn checked(source: &str) { + let mut sources = SourceMap::default(); + sources.add("main.dever", source); + check(&sources).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }); + } + + #[test] + fn async_sources_pass_the_existing_checker() { + for body in [ + NAMED_VALUES, + TASKS_AND_GROUPS, + NULLABLE_CHANNEL, + TASK_BUSINESS_FAILURE, + CANCEL_AND_NESTED, + BLOCKING_AND_PARALLEL, + CAPTURE_THEN_REFAIL, + UNCAPTURED_RUNTIME_FAULT, + BLOCKING_RUNTIME_FAULT, + DEFAULT_FAILURE_TRACE, + ASYNC_STATIC_TRAVERSAL, + SYNC_TASK_AND_GROUP, + OWNED_BLOCKING_AND_PARALLEL, + BLOCKING_AND_PARALLEL_BUSINESS_FAILURE, + NEGATIVE_TASK_SLEEP, + SYNC_CHANNEL_ONLY, + ] { + checked(&source(body)); + } + } + + #[test] + fn task_spawn_rejects_nontransferable_sync_stream_at_source() { + let mut sources = SourceMap::default(); + sources.add( + "main.dever", + "consume(stream: Stream<dever.io.ReadEvent>) () { close(stream) }\npublic main() () { file = dever.io.open(\"owned-test-file\")\nstream = dever.io.chunks(file, 2)\ntask = run(consume(stream))\nstop(task) }", + ); + let errors = check(&sources).expect_err("Task must reject a synchronous Stream argument"); + assert!(errors.iter().any(|error| { + let rendered = error.render(&sources); + rendered.contains("main.dever:4:") + && rendered.contains("task inputs and outputs must be transferable between threads") + })); + } + + #[test] + fn suspending_source_rejects_inline_file_io_at_source() { + let mut sources = SourceMap::default(); + sources.add( + "main.dever", + "slept(value: dever.time.SleepResult.Done) () recover(\"test timer\") {}\nslept(value: dever.time.SleepResult.Failed(message)) () recover(\"test timer\") {}\npublic main() (file: dever.system.File) {\nslept(result(dever.task.sleep(1)))\nfile = dever.io.open(\"owned-test-file\")\n}", + ); + let errors = + check(&sources).expect_err("inline file I/O must fail in a suspending function"); + assert!(errors.iter().any(|error| { + let rendered = error.render(&sources); + rendered.contains("main.dever:5:") + && rendered.contains("a blocking or parallel operation cannot run inline inside a suspending function") + })); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn named_async_call_preserves_nested_managed_output() { + llvm_managed::boolean(&source(NAMED_VALUES)); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn cloned_channel_fields_survive_their_source_scope() { + llvm_managed::boolean( + "type Pair { first: Channel<Int>\n second: Channel<Int> }\n\ + shared() (pair: Pair) { pipe = channel(Int, 1)\n\ + pair = Pair { first = pipe\n second = pipe } }\n\ + some_int(value: Int) (answer: Int?) { answer = value }\n\ + public main() (okay: Bool) { task = run(shared())\n pair = wait(task)\n\ + send(pair.first, 7)\n received = receive(pair.second)\n\ + close(pair.first)\n close(pair.second)\n okay = received == some_int(7) }", + ); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn task_wait_and_capacity_one_group_preserve_order() { + llvm_managed::boolean(&source(TASKS_AND_GROUPS)); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn nullable_channel_backpressure_close_and_drain() { + llvm_managed::boolean(&source(NULLABLE_CHANNEL)); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn task_business_failure_preserves_owned_payload() { + llvm_managed::boolean(&source(TASK_BUSINESS_FAILURE)); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn stop_task_group_and_nested_descendants_release_frames() { + llvm_managed::boolean(&source(CANCEL_AND_NESTED)); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn blocking_and_parallel_return_checked_values() { + llvm_managed::boolean(&source(BLOCKING_AND_PARALLEL)); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn capture_then_refail_preserves_owned_history() { + llvm_managed::boolean(&source(CAPTURE_THEN_REFAIL)); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn runtime_fault_is_not_captured_as_business_failure() { + numeric_division_fault( + &source(UNCAPTURED_RUNTIME_FAULT), + "value // denominator", + "{ i1 }", + ); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn blocking_sync_call_propagates_typed_runtime_fault() { + numeric_division_fault( + &source(BLOCKING_RUNTIME_FAULT), + "42 // denominator", + "{ i64 }", + ); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn blocking_abi_fault_keeps_original_buffer_and_is_not_captured() { + llvm_managed::runtime_fault( + &source( + r#"identity(value: Text) (answer: Text) pure { answer = value } +build() (answer: Map<Text, List<Text>>) { + ready = slept(result(dever.time.sleep(1))) + key = identity("same") + answer = { key = ["first"] + identity(key) = ["second"] } +} +type Outcome { + Done(value: Map<Text, List<Text>>) + error Failed(message: Text) +} +require_rows(count: 0) () { fail(Outcome.Failed("empty")) } +require_rows(count: other) () {} +worker() (answer: Map<Text, List<Text>>) { + ready = slept(result(dever.task.sleep(1))) + answer = blocking(build()) + require_rows(length(answer)) +} +inspect(value: Outcome.Done(entries)) (answer: Bool) recover("test expects an ABI fault") { answer = false } +inspect(value: Outcome.Failed(message)) (answer: Bool) recover("test expects an ABI fault") { answer = false } +public main() (answer: Bool) { answer = inspect(result(worker())) } +"#, + ), + "duplicate Map key", + ); + } + + fn numeric_division_fault(source: &str, expression: &str, output_type: &str) { + let origin = source.find(expression).unwrap(); + llvm_managed::execute( + source, + output_type, + &format!( + " %failed = icmp eq i32 %status, 1 + %code_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 0 + %code = load i32, ptr %code_ptr + %division = icmp eq i32 %code, 2 + %type_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 5 + %type = load i32, ptr %type_ptr + %not_business = icmp eq i32 %type, 0 + %origin_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 1 + %origin_id = load i32, ptr %origin_ptr + %origin_span = getelementptr {{ i64, i64, i64 }}, ptr @dever_source_spans, i32 %origin_id, i32 1 + %origin_start = load i64, ptr %origin_span + %at_expression = icmp eq i64 %origin_start, {origin} + %depth_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 2 + %depth = load i32, ptr %depth_ptr + %called = icmp uge i32 %depth, 1 + %fault_ok = and i1 %failed, %division + %kind_ok = and i1 %not_business, %at_expression + %trace_ok = and i1 %kind_ok, %called + %passed = and i1 %fault_ok, %trace_ok" + ), + ); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn async_static_handlers_preserve_traversal_state() { + llvm_managed::boolean(&source(ASYNC_STATIC_TRAVERSAL)); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn synchronous_task_multi_output_and_group_unit() { + llvm_managed::boolean(&source(SYNC_TASK_AND_GROUP)); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn blocking_and_parallel_preserve_owned_inputs_and_outputs() { + llvm_managed::boolean(&source(OWNED_BLOCKING_AND_PARALLEL)); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn blocking_and_parallel_preserve_typed_business_failure() { + llvm_managed::boolean(&source(BLOCKING_AND_PARALLEL_BUSINESS_FAILURE)); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn negative_task_sleep_keeps_runtime_message() { + llvm_managed::boolean(&source(NEGATIVE_TASK_SLEEP)); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn synchronous_channel_only_root_releases_aliases() { + llvm_managed::boolean(SYNC_CHANNEL_ONLY); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn default_failure_retains_origin_callers_and_owned_payload() { + let source = source(DEFAULT_FAILURE_TRACE); + let origin = source.find("fail(Outcome.Failed(").unwrap(); + let middle = source.rfind("origin()").unwrap(); + let root = source.rfind("middle()").unwrap(); + let mut checks = String::from( + " %failed = icmp eq i32 %status, 1 + %code_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 0 + %code = load i32, ptr %code_ptr + %business = icmp eq i32 %code, 4 + %depth_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 2 + %depth = load i32, ptr %depth_ptr + %frames = icmp eq i32 %depth, 2 + %type_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 5 + %type = load i32, ptr %type_ptr + %typed = icmp ne i32 %type, 0 + %payload = getelementptr %dever.fault, ptr %fault, i32 0, i32 7 + %message_ptr = getelementptr i64, ptr %payload, i32 1 + %message = load ptr, ptr %message_ptr + %history_ptr = getelementptr i64, ptr %payload, i32 2 + %history = load ptr, ptr %history_ptr + %message_owned = icmp ne ptr %message, null + %history_owned = icmp ne ptr %history, null + %owned = and i1 %message_owned, %history_owned + %base = and i1 %failed, %business + %base2 = and i1 %frames, %typed + %base3 = and i1 %base, %base2 + %base4 = and i1 %base3, %owned +", + ); + for (index, (field, expected)) in [(1, origin), (3, middle), (3, root)] + .into_iter() + .enumerate() + { + let pointer = if index == 0 { + format!( + " %id_ptr{index} = getelementptr %dever.fault, ptr %fault, i32 0, i32 {field}\n" + ) + } else { + format!( + " %id_ptr{index} = getelementptr %dever.fault, ptr %fault, i32 0, i32 {field}, i32 {}\n", + index - 1 + ) + }; + checks.push_str(&pointer); + write!( + checks, + " %id{index} = load i32, ptr %id_ptr{index} + %span_ptr{index} = getelementptr {{ i64, i64, i64 }}, ptr @dever_source_spans, i32 %id{index} + %span{index} = load {{ i64, i64, i64 }}, ptr %span_ptr{index} + %start{index} = extractvalue {{ i64, i64, i64 }} %span{index}, 1 + %at{index} = icmp eq i64 %start{index}, {expected} +" + ) + .unwrap(); + } + checks.push_str( + " %trace01 = and i1 %at0, %at1 + %trace = and i1 %trace01, %at2 + %passed = and i1 %base4, %trace", + ); + llvm_managed::execute(&source, "{}", &checks); + } + + #[test] + fn async_source_emits_objects_for_all_targets() { + let ir = llvm::lower(&source(NAMED_VALUES)); + for target in dever_backend_bridge::Target::ALL { + assert!( + !dever_backend_bridge::emit_object(&ir, target) + .unwrap() + .is_empty() + ); + } + } +} diff --git a/test/llvm_database_source.rs b/test/llvm_database_source.rs new file mode 100644 index 0000000..2337cce --- /dev/null +++ b/test/llvm_database_source.rs @@ -0,0 +1,967 @@ +//! Checked Model/transaction source through LLVM and the real database ABI. + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/llvm_module.rs"] +mod llvm_module; +#[cfg(all( + target_os = "linux", + target_arch = "x86_64", + feature = "runtime-postgres" +))] +#[path = "dever-tests/tests/support/postgres.rs"] +pub mod postgres; +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/process.rs"] +mod process; +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/temp.rs"] +mod temp; + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +mod native { + use super::llvm_module; + use dever_backend_bridge::{Target, emit_object}; + use dever_core::{check, hir::Program, source::SourceMap}; + use std::time::Duration; + + const SETTING: &str = r#"{"database":{"default":{"type":"sqlite","path":"data/db/owned.db","max_connections":1,"max_page_size":10}}}"#; + const ITEM: &str = r#"type Item { + name: Text(1, 64) unique + amount: Int default 1 index + note: Text? +} +"#; + + struct Fixture<'a> { + model: &'a str, + app: &'a str, + body: &'a str, + } + + const CRUD: Fixture<'static> = Fixture { + model: ITEM, + app: r#"some_text(value: Text) (answer: Text?) { answer = value } +amount(value: null) (answer: Int) { answer = -1 } +amount(value: model.Item) (answer: Int) { answer = value.amount } +exercise() (okay: Bool) { + saved = model.create({ name = "你好 🌍" }) + changed = model.update(saved.id, { amount = 7 + note = "owned" }) + loaded = model.get(saved.id) + first = model.first({ where = name == "你好 🌍" }) + absent = model.first({ where = name == "missing" }) + page = model.list({ order = id.asc + size = 2 }) + present = model.exists({ where = name == "你好 🌍" }) + removed = model.delete(saved.id) + remaining = model.count() + okay = loaded.id == saved.id and loaded.name == "你好 🌍" and loaded.amount == 7 and loaded.note == some_text("owned") and amount(first) == 7 and amount(absent) == -1 and changed == 1 and page.total == 1 and length(page.items) == 1 and present and removed == 1 and remaining == 0 +} +"#, + body: "{}", + }; + + const BULK: Fixture<'static> = Fixture { + model: ITEM, + app: r#"exercise() (okay: Bool) { + inserted = model.create_many([{ name = "one" + amount = 2 }, { name = "two" + amount = 3 }]) + updated = model.upsert({ name = "one" }, { amount = 99 }, { amount = 4 }) + created = model.upsert({ name = "three" }, { amount = 5 }, { amount = 99 }) + changed = model.update({ where = amount >= 3 }, { note = "bulk" }) + count = model.count({ where = not contains(name, "missing") and in(amount, [3, 4, 5]) and between(amount, 3, 5) and note != null }) + removed = model.delete({ where = amount >= 3 }) + okay = inserted == 2 and updated.amount == 4 and created.amount == 5 and changed == 3 and count == 3 and removed == 3 and model.count() == 0 +} +"#, + body: "{}", + }; + + const NULL_EQUALITY: Fixture<'static> = Fixture { + model: ITEM, + app: r#"some_text(value: Text) (answer: Text?) { answer = value } +missing_text() (answer: Text?) { answer = null } +exercise() (okay: Bool) { + absent = model.create({ name = "absent" }) + present = model.create({ name = "present" + note = "value" }) + missing = missing_text() + value = some_text("value") + nulls = model.count({ where = note == missing }) + values = model.count({ where = note == value }) + not_nulls = model.count({ where = note != missing }) + removed = model.delete({ where = name == "absent" or name == "present" }) + okay = nulls == 1 and values == 1 and not_nulls == 1 and absent.note == null and present.note == value and removed == 2 +} +"#, + body: "{}", + }; + + const CURSOR: Fixture<'static> = Fixture { + model: ITEM, + app: r#"names(value: model.Item) (name: Text) { name = value.name } +cursor_after(items: List<model.Item>) (value: model.Item?) { value = first(items) } +exercise() (okay: Bool) { + inserted = model.create_many([{ name = "one" + amount = 1 }, { name = "two" + amount = 1 }, { name = "three" + amount = 2 }]) + first = model.cursor({ order = [amount.asc, id.asc] + size = 2 }) + second = model.cursor({ after = first.next + order = [amount.asc, id.asc] + size = 2 }) + end = model.cursor({ after = cursor_after(second.items) + order = [amount.asc, id.asc] + size = 2 }) + page = model.list({ order = [amount.asc, id.asc] + page = 2 + size = 2 }) + removed = model.delete({ where = amount >= 1 }) + okay = inserted == 3 and each(names, first.items) == ["one", "two"] and first.has_more and each(names, second.items) == ["three"] and second.next == null and not second.has_more and length(end.items) == 0 and end.next == null and each(names, page.items) == ["three"] and page.total == 3 and removed == 3 +} +"#, + body: "{}", + }; + + const SCALARS: Fixture<'static> = Fixture { + model: r#"type State { + Active + Disabled +} +type Item { + name: Text(1, 64) unique + enabled: Bool + amount: Int + fraction: Float + price: Decimal(10, 2) + payload: Bytes(8) + external_id: Uuid + generated_id: Uuid generated unique + instant: DateTime + day: Date + clock: Time + elapsed: Duration + document: Json + state: State default State.Active + private memo: Text? +} +"#, + app: r#"exercise(uuid: Uuid, instant: DateTime, day: Date, clock: Time, elapsed: Duration, document: Json) (okay: Bool) { + payload = dever.bytes.from_ints([0, 128, 255]) + saved = model.create({ name = "scalar" + enabled = true + amount = 9223372036854775807 + fraction = 1.5 + price = 123.45 + payload = payload + external_id = uuid + instant = instant + day = day + clock = clock + elapsed = elapsed + document = document + memo = "private" }) + loaded = model.get(saved.id) + removed = model.delete(saved.id) + okay = loaded.enabled and loaded.amount == 9223372036854775807 and loaded.fraction == 1.5 and loaded.price == 123.45 and loaded.payload == payload and loaded.external_id == uuid and loaded.generated_id == saved.generated_id and loaded.instant == instant and loaded.day == day and loaded.clock == clock and loaded.elapsed == elapsed and loaded.document == document and loaded.state == model.State.Active and loaded.memo != null and removed == 1 +} +"#, + body: r#"{"uuid":"01890f3e-7c00-7000-8000-000000000001","instant":"2026-10-01T12:34:56.789Z","day":"2026-10-01","clock":"12:34:56.789","elapsed":-42,"document":{"large":9999999999999999999999999999,"nested":[true,null]}}"#, + }; + + const TRANSACTION: Fixture<'static> = Fixture { + model: ITEM, + app: r#"transaction insert() (saved: model.Item) { saved = model.create({ name = "nested" }) } +forward() (saved: model.Item) { saved = insert() } +transaction edit() (saved: model.Item) { + inserted = forward() + changed = model.update(inserted.id, { amount = 7 }) + saved = model.get(inserted.id) +} +exercise() (okay: Bool) { + saved = edit() + loaded = model.get(saved.id) + removed = model.delete(saved.id) + okay = saved.amount == 7 and loaded.amount == 7 and removed == 1 and model.count() == 0 +} +"#, + body: "{}", + }; + + const STREAM: Fixture<'static> = Fixture { + model: r#"type Item { + name: Text(1, 64) unique + amount: Int default 1 +} +seed { { name = "one" + amount = 2 } + { name = "two" + amount = 3 } } +"#, + app: r#"add(value: model.Item, previous: Int) (next: Int) { next = previous + value.amount } +one(value: model.Item, previous: Int) (next: Int, stop: Bool) { next = previous + value.amount + stop = true } +exercise() (okay: Bool) { + full = model.stream({ order = id.asc + size = 1 }) + total = reduce(add, full, 0) + partial = model.stream({ order = id.asc + size = 1 }) + alias = partial + first = reduce_until(one, partial, 0) + remaining = reduce(add, alias, 0) + unopened = model.stream({ size = 1 }) + close(unopened) + okay = total == 5 and first == 2 and remaining == 0 and model.count() == 2 +} +"#, + body: "{}", + }; + + const SQL: Fixture<'static> = Fixture { + model: r#"type Summary { amount: Int } +type Item { + name: Text(1, 64) unique + amount: Int default 1 +} +sql by_name(name: Text) (item: Item?) { + sqlite = "SELECT id, created_at, name, amount FROM item WHERE name = ?1" + postgres = "SELECT id, created_at, name, amount FROM item WHERE name = $1" +} +sql all_items(amount: Int) (items: List<Item>) { + sqlite = "SELECT id, created_at, name, amount FROM item WHERE amount >= ?1 ORDER BY id" + postgres = "SELECT id, created_at, name, amount FROM item WHERE amount >= $1::bigint ORDER BY id" +} +sql summary() (summary: Summary) { + sqlite = "SELECT SUM(amount) FROM item" + postgres = "SELECT SUM(amount)::bigint FROM item" +} +"#, + app: r#"amount(value: null) (answer: Int) { answer = -1 } +amount(value: model.Item) (answer: Int) { answer = value.amount } +transaction populate() (okay: Bool) { + inserted = model.create_many([{ name = "one" + amount = 2 }, { name = "two" + amount = 3 }]) + found = model.by_name("one") + absent = model.by_name("missing") + rows = model.all_items(2) + summary = model.summary() + okay = inserted == 2 and amount(found) == 2 and amount(absent) == -1 and length(rows) == 2 and summary.amount == 5 +} +exercise() (okay: Bool) { + valid = populate() + removed = model.delete({ where = amount >= 2 }) + okay = valid and removed == 2 and model.count() == 0 +} +"#, + body: "{}", + }; + + const ERRORS: &str = r#"type ReadResult { + Done(value: model.Item) + error Failed(error: dever.database.Error) +} +category(error: dever.database.Error.Pool(message)) (kind: Int) recover("assert database failure identity") { kind = 1 } +category(error: dever.database.Error.PoolExhausted(message)) (kind: Int) recover("assert database failure identity") { kind = 2 } +category(error: dever.database.Error.Connection(message)) (kind: Int) recover("assert database failure identity") { kind = 3 } +category(error: dever.database.Error.Timeout(message)) (kind: Int) recover("assert database failure identity") { kind = 4 } +category(error: dever.database.Error.Cancelled(message)) (kind: Int) recover("assert database failure identity") { kind = 5 } +category(error: dever.database.Error.Database(message)) (kind: Int) recover("assert database failure identity") { kind = 6 } +category(error: dever.database.Error.Constraint(message)) (kind: Int) recover("assert database failure identity") { kind = 7 } +category(error: dever.database.Error.NotFound(message)) (kind: Int) recover("assert database failure identity") { kind = 8 } +category(error: dever.database.Error.InvalidData(message)) (kind: Int) recover("assert database failure identity") { kind = 9 } +category(error: dever.database.Error.Migration(message)) (kind: Int) recover("assert database failure identity") { kind = 10 } +captured(value: ReadResult.Done(item)) (kind: Int) recover("assert database failure identity") { kind = 0 } +captured(value: ReadResult.Failed(error)) (kind: Int) recover("assert database failure identity") { kind = category(error) } +"#; + + fn captured_errors() -> Fixture<'static> { + Fixture { + model: ITEM, + app: r#"transaction missing() (value: model.Item) { + saved = model.create({ name = "temporary" }) + removed = model.delete(saved.id) + value = model.get(saved.id) +} +transaction duplicate() (value: model.Item) { + saved = model.create({ name = "duplicate" }) + value = model.create({ name = "duplicate" }) +} +exercise() (okay: Bool) { + missing = captured(result(missing())) + duplicate = captured(result(duplicate())) + retained = model.count() + removed = model.delete({ where = name == "duplicate" }) + okay = missing == 8 and duplicate == 7 and retained == 1 and removed == 1 and model.count() == 0 +} +"#, + body: "{}", + } + } + + const NUMERIC_ROLLBACK: Fixture<'static> = Fixture { + model: ITEM, + app: r#"transaction fail_numeric(divisor: Int) (okay: Bool) { + saved = model.create({ name = "numeric rollback" }) + okay = 42 // divisor == 42 +} +exercise(divisor: Int) (okay: Bool) { okay = fail_numeric(divisor) } +"#, + body: r#"{"divisor":0}"#, + }; + + const CAPTURED_NOT_FOUND: Fixture<'static> = Fixture { + model: ITEM, + app: r#"transaction missing() (value: model.Item) { + saved = model.create({ name = "temporary" }) + removed = model.delete(saved.id) + value = model.get(saved.id) +} +exercise() (okay: Bool) { + missing = captured(result(missing())) + okay = missing == 8 and model.count() == 0 +} +"#, + body: "{}", + }; + + const CONSTRAINT_ROLLBACK: Fixture<'static> = Fixture { + model: ITEM, + app: r#"exercise() (okay: Bool) { + first = model.create({ name = "duplicate" }) + second = model.create({ name = "duplicate" }) + okay = true +} +inspect() (okay: Bool) { okay = model.count() == 0 } +"#, + body: "{}", + }; + + const ENCODING_ROLLBACK: Fixture<'static> = Fixture { + model: ITEM, + app: r#"exercise(denominator: Float) (value: Float) { + saved = model.create({ name = "encoding rollback" }) + value = 1.0 / denominator +} +"#, + body: r#"{"denominator":0}"#, + }; + + const TASK_ERROR: Fixture<'static> = Fixture { + model: ITEM, + app: r#"missing(id: model.id) (value: model.Item) { value = model.get(id) } +task_missing(id: model.id) (value: model.Item) { + pending = run(missing(id)) + value = wait(pending) +} +exercise(id: model.id) (okay: Bool) { + missing = captured(result(task_missing(id))) + okay = missing == 8 and model.count() == 0 +} +"#, + body: r#"{"id":9223372036854775807}"#, + }; + + const HANDLER_TRANSACTION: Fixture<'static> = Fixture { + model: ITEM, + app: r#"update(item: model.Item) (count: Int) { count = model.update(item.id, { amount = 7 }) } +transaction change_all() (count: Int) { + inserted = model.create_many([{ name = "one" }, { name = "two" }]) + page = model.list({ order = id.asc + size = 10 }) + count = sum(update, page.items) +} +exercise() (okay: Bool) { + changed = change_all() + matching = model.count({ where = amount == 7 }) + removed = model.delete({ where = amount == 7 }) + okay = changed == 2 and matching == 2 and removed == 2 and model.count() == 0 +} +"#, + body: "{}", + }; + + const SEED: Fixture<'static> = Fixture { + model: r#"type Item { + name: Text(1, 64) unique + amount: Int default 1 +} +seed { { name = "seed" + amount = 2 } } +"#, + app: r#"exercise() (okay: Bool) { + seeded = model.count({ where = name == "seed" and amount == 2 }) + created = model.create({ name = "temporary" }) + removed = model.delete(created.id) + okay = seeded == 1 and created.amount == 1 and removed == 1 and model.count() == 1 +} +"#, + body: "{}", + }; + + const EMPTY_BULK: Fixture<'static> = Fixture { + model: "type Item {}", + app: r#"exercise() (okay: Bool) { + created = model.create_many([{}, {}]) + count = model.count() + removed = model.delete({}) + okay = created == 2 and count == 2 and removed == 2 +} +"#, + body: "{}", + }; + + const MIGRATION_INITIAL: Fixture<'static> = Fixture { + model: r#"type Item { + name: Text(1, 64) unique + amount: Int default 1 + legacy: Text? +} +seed { { name = "too-long-name" + legacy = "old" } } +"#, + app: r#"exercise() (okay: Bool) { okay = model.count({ where = name == "too-long-name" and amount == 1 }) == 1 } +"#, + body: "{}", + }; + + const MIGRATION_UPGRADED: Fixture<'static> = Fixture { + model: r#"type Item { + display_name: Text(1, 8) from name unique + amount: Int default 2 +} +migrate normalize_names { + before { + sqlite = "UPDATE item SET name = ?1 WHERE name = ?2" + postgres = "UPDATE item SET name = $1 WHERE name = $2" + parameters = ["short", "too-long-name"] + } + drop legacy + after { + sqlite = "UPDATE item SET display_name = ?1 WHERE display_name = ?2" + postgres = "UPDATE item SET display_name = $1 WHERE display_name = $2" + parameters = ["final", "short"] + } +} +seed { { display_name = "second" } } +"#, + app: r#"exercise() (okay: Bool) { + original = model.count({ where = display_name == "final" and amount == 1 }) + seeded = model.count({ where = display_name == "second" and amount == 2 }) + created = model.create({ display_name = "temp" }) + removed = model.delete(created.id) + okay = original == 1 and seeded == 1 and created.amount == 2 and removed == 1 and model.count() == 2 +} +"#, + body: "{}", + }; + + const CANCEL_STREAM: Fixture<'static> = Fixture { + model: r#"type Item { amount: Int unique } +seed { { amount = 1 } + { amount = 2 } + { amount = 3 } + { amount = 4 } } +"#, + app: r#"pause(item: model.Item, ready: Channel<Int>) (next: Channel<Int>) { + send(ready, item.amount) + dever.task.sleep(1000) + next = ready +} +scan(ready: Channel<Int>) (okay: Bool) { + rows = model.stream({ order = id.asc + size = 1 }) + signal = reduce(pause, rows, ready) + okay = false +} +fallback() (okay: Bool) { okay = model.count() == 4 } +exercise() (okay: Bool) { + ready = channel(Int, 1) + pending = run(scan(ready)) + started = receive(ready) + released = timeout(pending, 1, fallback) + close(ready) + okay = started != null and released and model.count() == 4 +} +"#, + body: "{}", + }; + + fn relations() -> SourceMap { + let mut sources = SourceMap::default(); + sources.add( + "store/account/model/organization.dever", + "type Organization { name: Text(1, 64) }\nrelation users = store.account.model.user.organization_id", + ); + sources.add( + "store/account/model/user.dever", + "type User { name: Text(1, 64)\n organization_id: store.account.model.organization.id?\n manager_id: store.account.model.user.id? }\nrelation posts = store.account.model.post.author_id\nrelation other_posts = store.account.model.post.author_id\nrelation unloaded_posts = store.account.model.post.author_id", + ); + sources.add( + "store/account/model/post.dever", + "type Post { author_id: store.account.model.user.id\n title: Text(1, 64) }", + ); + sources.add( + "store/account/app.dever", + r#"organization_name(value: Related.Unloaded<model.organization.Organization?>()) (name: Text) { name = "unloaded" } +organization_name(value: Related.Loaded<model.organization.Organization?>(organization)) (name: Text) { name = nullable_organization(organization) } +nullable_organization(value: null) (name: Text) { name = "none" } +nullable_organization(value: model.organization.Organization) (name: Text) { name = value.name } +manager_name(value: Related.Unloaded<model.user.User?>()) (name: Text) { name = "unloaded" } +manager_name(value: Related.Loaded<model.user.User?>(manager)) (name: Text) { name = nullable_user(manager) } +nullable_user(value: null) (name: Text) { name = "none" } +nullable_user(value: model.user.User) (name: Text) { name = value.name } +post_count(value: Related.Unloaded<List<model.post.Post>>()) (count: Int) { count = -1 } +post_count(value: Related.Loaded<List<model.post.Post>>(posts)) (count: Int) { count = length(posts) } +is_primary(value: model.user.User) (yes: Bool) { yes = value.name == "First" } +is_secondary(value: model.user.User) (yes: Bool) { yes = value.name == "Second" } +transaction loaded() (page: model.user.Page) { page = model.user.list({ order = id.asc + size = 10 + with = [organization, manager, posts, other_posts] }) } +exercise() (okay: Bool) { + organization = model.organization.create({ name = "Acme" }) + primary = model.user.create({ name = "First" + organization_id = organization.id }) + secondary = model.user.create({ name = "Second" + manager_id = primary.id }) + inserted = model.post.create_many([{ author_id = primary.id + title = "One" }, { author_id = primary.id + title = "Two" }]) + unloaded = model.user.get(primary.id) + page = loaded() + first_loaded = find(is_primary, page.items) + second_loaded = find(is_secondary, page.items) + valid = validate(first_loaded, second_loaded) + deleted_posts = model.post.delete({ where = author_id == primary.id }) + deleted_second = model.user.delete(secondary.id) + deleted_first = model.user.delete(primary.id) + deleted_org = model.organization.delete(organization.id) + okay = inserted == 2 and organization_name(unloaded.organization) == "unloaded" and page.total == 2 and valid and deleted_posts == 2 and deleted_second == 1 and deleted_first == 1 and deleted_org == 1 +} +validate(first: null, second: model.user.User?) (okay: Bool) { okay = false } +validate(first: model.user.User, second: null) (okay: Bool) { okay = false } +validate(first: model.user.User, second: model.user.User) (okay: Bool) { okay = organization_name(first.organization) == "Acme" and organization_name(second.organization) == "none" and manager_name(second.manager) == "First" and post_count(first.posts) == 2 and post_count(first.other_posts) == 2 and post_count(second.posts) == 0 and post_count(first.unloaded_posts) == -1 } +"#, + ); + sources.add("store/account/api.dever", "cmd exercise = app.exercise"); + sources + } + + fn fixture_sources(fixture: &Fixture<'_>) -> SourceMap { + fixture_sources_with_api(fixture, "cmd exercise = app.exercise") + } + + fn fixture_sources_with_api(fixture: &Fixture<'_>, api: &str) -> SourceMap { + let mut sources = SourceMap::default(); + sources.add("store/item/model.dever", fixture.model); + sources.add("store/item/app.dever", fixture.app); + sources.add("store/item/api.dever", api); + sources + } + + fn checked(sources: &SourceMap) -> Program { + check(sources).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(sources)) + .collect::<String>() + ) + }) + } + + fn entry_ir(sources: &SourceMap, fault: Option<u32>) -> String { + let program = checked(sources); + let mut ir = dever_core::llvm::emit_application(&program, sources).unwrap(); + let checks = fault.map_or_else( + || " %passed = icmp eq i32 %status, 0".to_owned(), + |code| llvm_module::fault_check(code, None), + ); + ir.push_str(&format!( + "\ndefine i32 @dever_test_run() {{\nentry:\n %out = alloca i8\n %fault = alloca %dever.fault\n %status = call i32 @dever_application_entry(ptr %out, ptr %fault)\n{checks}\n call void @dever_outputs_release(ptr %out)\n call void @dever_fault_release(ptr %fault)\n %exit = select i1 %passed, i32 0, i32 1\n ret i32 %exit\n}}\n" + )); + ir + } + + fn assert_output(output: std::process::Output, fault: bool) { + if fault { + assert!(output.stdout.is_empty()); + } else { + assert_eq!( + String::from_utf8(output.stdout).unwrap(), + "{\"code\":0,\"message\":\"ok\",\"data\":true}\n".repeat(65) + ); + } + assert!(output.stderr.is_empty()); + } + + fn execute( + sources: &SourceMap, + command: &str, + body: &str, + setting: &str, + fault: Option<u32>, + timeout: Duration, + ) { + let ir = entry_ir(sources, fault); + let output = llvm_module::execute( + &ir, + "checked database application", + 64, + timeout, + &[command, body], + Some(setting), + ); + assert_output(output, fault.is_some()); + } + + fn sqlite(fixture: &Fixture<'_>) { + execute( + &fixture_sources(fixture), + "store.item.exercise", + fixture.body, + SETTING, + None, + Duration::from_secs(20), + ); + } + + #[test] + fn database_fixtures_pass_the_existing_checker() { + for fixture in [ + &CRUD, + &BULK, + &NULL_EQUALITY, + &CURSOR, + &SCALARS, + &TRANSACTION, + &STREAM, + &SQL, + &NUMERIC_ROLLBACK, + &ENCODING_ROLLBACK, + &HANDLER_TRANSACTION, + &SEED, + &EMPTY_BULK, + &MIGRATION_INITIAL, + &MIGRATION_UPGRADED, + &CANCEL_STREAM, + ] { + checked(&fixture_sources(fixture)); + } + let fixture = captured_errors(); + let app = format!("{ERRORS}{}", fixture.app); + checked(&fixture_sources(&Fixture { + app: &app, + ..fixture + })); + let app = format!("{ERRORS}{}", TASK_ERROR.app); + checked(&fixture_sources(&Fixture { + app: &app, + ..TASK_ERROR + })); + let app = format!("{ERRORS}{}", CAPTURED_NOT_FOUND.app); + checked(&fixture_sources(&Fixture { + app: &app, + ..CAPTURED_NOT_FOUND + })); + checked(&fixture_sources_with_api( + &CONSTRAINT_ROLLBACK, + "cmd exercise = app.exercise\ncmd inspect = app.inspect", + )); + checked(&relations()); + } + + #[test] + fn model_application_emits_objects_for_all_six_targets() { + let sources = fixture_sources(&CRUD); + let ir = dever_core::llvm::emit_application(&checked(&sources), &sources).unwrap(); + for target in Target::ALL { + assert!(!emit_object(&ir, target).unwrap().is_empty()); + } + } + + #[test] + #[ignore = "requires explicitly prepared database runtime archive; owned SQLite only"] + fn sqlite_crud_defaults_owned_rows_and_repeat_entry() { + sqlite(&CRUD); + } + + #[test] + #[ignore = "requires explicitly prepared database runtime archive; owned SQLite only"] + fn sqlite_bulk_upsert_and_static_predicates() { + sqlite(&BULK); + } + + #[test] + #[ignore = "requires explicitly prepared database runtime archive; owned SQLite only"] + fn sqlite_nullable_equality_keeps_present_and_absent_binds() { + sqlite(&NULL_EQUALITY); + } + + #[test] + #[ignore = "requires explicitly prepared database runtime archive; owned SQLite only"] + fn sqlite_cursor_ties_nullable_next_and_offset_page() { + sqlite(&CURSOR); + } + + #[test] + #[ignore = "requires explicitly prepared database runtime archive; owned SQLite only"] + fn sqlite_persistent_scalars_choices_and_private_fields_round_trip() { + sqlite(&SCALARS); + } + + #[test] + #[ignore = "requires explicitly prepared database runtime archive; owned SQLite only"] + fn sqlite_nested_transactions_share_the_one_connection() { + sqlite(&TRANSACTION); + } + + #[test] + #[ignore = "requires explicitly prepared database runtime archive; owned SQLite only"] + fn sqlite_row_stream_full_early_stop_and_close_release_connections() { + sqlite(&STREAM); + } + + #[test] + #[ignore = "requires explicitly prepared database runtime archive; owned SQLite only"] + fn sqlite_static_sql_typed_optional_list_and_record_results() { + sqlite(&SQL); + } + + #[test] + #[ignore = "requires explicitly prepared database runtime archive; owned SQLite only"] + fn sqlite_captured_nested_database_faults_keep_exact_categories() { + let fixture = captured_errors(); + let app = format!("{ERRORS}{}", fixture.app); + sqlite(&Fixture { + app: &app, + ..fixture + }); + } + + #[test] + #[ignore = "requires explicitly prepared database runtime archive; owned SQLite only"] + fn sqlite_task_wait_keeps_database_failure_identity() { + let app = format!("{ERRORS}{}", TASK_ERROR.app); + sqlite(&Fixture { + app: &app, + ..TASK_ERROR + }); + } + + #[test] + #[ignore = "requires explicitly prepared database runtime archive; owned SQLite only"] + fn sqlite_numeric_fault_rolls_back_before_the_next_entry() { + execute( + &fixture_sources(&NUMERIC_ROLLBACK), + "store.item.exercise", + NUMERIC_ROLLBACK.body, + SETTING, + Some(2), + Duration::from_secs(20), + ); + } + + #[test] + #[ignore = "requires explicitly prepared database runtime archive; owned SQLite only"] + fn sqlite_command_encoding_failure_rolls_back_before_retry() { + execute( + &fixture_sources(&ENCODING_ROLLBACK), + "store.item.exercise", + ENCODING_ROLLBACK.body, + SETTING, + Some(3), + Duration::from_secs(20), + ); + } + + #[test] + #[ignore = "requires explicitly prepared database runtime archive; owned SQLite only"] + fn sqlite_uncaught_constraint_rolls_back_before_the_next_command() { + constraint_sequence(SETTING, Duration::from_secs(20)); + } + + #[test] + #[ignore = "requires explicitly prepared database runtime archive; owned SQLite only"] + fn sqlite_sequential_database_handler_inherits_transaction_context() { + sqlite(&HANDLER_TRANSACTION); + } + + #[test] + #[ignore = "requires explicitly prepared database runtime archive; owned SQLite only"] + fn sqlite_seed_history_is_reused_without_overwriting_business_rows() { + sqlite(&SEED); + } + + #[test] + #[ignore = "requires explicitly prepared database runtime archive; owned SQLite only"] + fn sqlite_empty_create_many_retains_valid_generated_defaults() { + sqlite(&EMPTY_BULK); + } + + #[test] + #[ignore = "requires explicitly prepared database runtime archive; owned SQLite only"] + fn sqlite_related_cycles_join_batch_and_unloaded_values() { + execute( + &relations(), + "store.account.exercise", + "{}", + SETTING, + None, + Duration::from_secs(20), + ); + } + + #[test] + #[ignore = "requires explicitly prepared database runtime archive; owned SQLite only"] + fn sqlite_schema_upgrade_preserves_rows_and_replays_migration_and_seed_once() { + migration_sequence(SETTING, Duration::from_secs(20)); + } + + fn migration_sequence(setting: &str, timeout: Duration) { + let directory = configured_directory(setting); + for fixture in [&MIGRATION_INITIAL, &MIGRATION_UPGRADED] { + let ir = entry_ir(&fixture_sources(fixture), None); + assert_output( + llvm_module::execute_in( + &directory, + &ir, + "successive checked Model revisions", + 64, + timeout, + &["store.item.exercise", "{}"], + ), + false, + ); + } + } + + fn configured_directory(setting: &str) -> super::temp::TemporaryDirectory { + let directory = super::temp::TemporaryDirectory::new(); + std::fs::create_dir(directory.path().join("config")).unwrap(); + std::fs::write(directory.path().join("config/setting.json"), setting).unwrap(); + directory + } + + fn constraint_sequence(setting: &str, timeout: Duration) { + let directory = configured_directory(setting); + let sources = fixture_sources_with_api( + &CONSTRAINT_ROLLBACK, + "cmd exercise = app.exercise\ncmd inspect = app.inspect", + ); + for (command, fault) in [ + ("store.item.exercise", Some(4)), + ("store.item.inspect", None), + ] { + let ir = entry_ir(&sources, fault); + assert_output( + llvm_module::execute_in( + &directory, + &ir, + "constraint failure then persisted rollback inspection", + 64, + timeout, + &[command, "{}"], + ), + fault.is_some(), + ); + } + } + + #[test] + #[ignore = "requires explicitly prepared database runtime archive; owned SQLite only"] + fn sqlite_cancelled_row_stream_drains_before_single_connection_fallback() { + sqlite(&CANCEL_STREAM); + } + + #[cfg(feature = "runtime-postgres")] + fn postgres_case(sources: &SourceMap, command: &str, body: &str, fault: Option<u32>) { + super::postgres::run_in_isolated_schema("llvm_database", |database| async move { + execute( + sources, + command, + body, + &database.settings_json(), + fault, + Duration::from_secs(60), + ); + Ok(()) + }); + } + + #[cfg(feature = "runtime-postgres")] + #[test] + #[ignore = "requires configured isolated database.postgres_test and an explicit database runtime archive"] + fn postgres_checked_models_transactions_streams_and_schema_upgrade() { + for fixture in [ + &CRUD, + &BULK, + &NULL_EQUALITY, + &CURSOR, + &SCALARS, + &TRANSACTION, + &STREAM, + &SQL, + &HANDLER_TRANSACTION, + &SEED, + &EMPTY_BULK, + &CANCEL_STREAM, + ] { + postgres_case( + &fixture_sources(fixture), + "store.item.exercise", + fixture.body, + None, + ); + } + for fixture in [CAPTURED_NOT_FOUND, TASK_ERROR] { + let app = format!("{ERRORS}{}", fixture.app); + postgres_case( + &fixture_sources(&Fixture { + app: &app, + ..fixture + }), + "store.item.exercise", + fixture.body, + None, + ); + } + postgres_case( + &fixture_sources(&NUMERIC_ROLLBACK), + "store.item.exercise", + NUMERIC_ROLLBACK.body, + Some(2), + ); + postgres_case(&relations(), "store.account.exercise", "{}", None); + super::postgres::run_in_isolated_schema("llvm_database", |database| async move { + migration_sequence(&database.settings_json(), Duration::from_secs(60)); + Ok(()) + }); + } + + #[cfg(feature = "runtime-postgres")] + #[test] + #[ignore = "requires configured isolated database.postgres_test and an explicit database runtime archive"] + fn postgres_command_encoding_failure_rolls_back_before_retry() { + postgres_case( + &fixture_sources(&ENCODING_ROLLBACK), + "store.item.exercise", + ENCODING_ROLLBACK.body, + Some(3), + ); + } + + #[cfg(feature = "runtime-postgres")] + #[test] + #[ignore = "requires configured isolated database.postgres_test and an explicit database runtime archive"] + fn postgres_uncaught_constraint_rolls_back_before_the_next_command() { + super::postgres::run_in_isolated_schema("llvm_database", |database| async move { + constraint_sequence(&database.settings_json(), Duration::from_secs(60)); + Ok(()) + }); + } +} diff --git a/test/llvm_external_source.rs b/test/llvm_external_source.rs new file mode 100644 index 0000000..f55137d --- /dev/null +++ b/test/llvm_external_source.rs @@ -0,0 +1,609 @@ +//! Typed Worker calls and verified executable resources through the LLVM backend. +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/llvm_module.rs"] +mod llvm_module; +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/process.rs"] +mod process; +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/sandbox.rs"] +mod sandbox; +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/temp.rs"] +mod temp; + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +mod native { + use super::{llvm_module, temp}; + use dever_backend_bridge::{BinaryResource, Target, emit_object, emit_object_with_resources}; + use dever_core::{hir::Program, native::EmbeddedResource, source::SourceMap}; + use serde_json::json; + use sha2::{Digest, Sha256}; + use std::fs; + use std::path::Path; + use std::time::Duration; + + const READ_APP: &str = r#"type Outcome { + Done(value: Int) + error Rejected(reason: Text) +} +read() (value: Int) { value = port.read() } +message(value: Outcome.Done(number)) (text: Text) recover("inspect Worker outcome") { text = "accepted" } +message(value: Outcome.Rejected(reason)) (text: Text) recover("inspect Worker outcome") { text = reason } +capture() (text: Text) { text = message(result(read())) } +"#; + const READ_PORT: &str = "read() (value: Int) fails app.Outcome"; + const READ_API: &str = "cmd read = app.read\ncmd capture = app.capture"; + const SETTING: &str = "setting {\n mode: Text\n identity: Text?\n marker: Text?\n}\n"; + + fn checked(ecosystem: &str, app: &str, port: &str, api: &str) -> (SourceMap, Program) { + let mut sources = SourceMap::default(); + sources.add("notification/delivery/app.dever", app); + sources.add("notification/delivery/port.dever", port); + sources.add("notification/delivery/api.dever", api); + sources.add( + "notification/delivery/adapter.dever", + format!("{SETTING}external {ecosystem} \"worker\" {{ allow network\nallow file }}"), + ); + let program = dever_core::check(&sources).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }); + (sources, program) + } + + fn read(ecosystem: &str) -> (SourceMap, Program) { + checked(ecosystem, READ_APP, READ_PORT, READ_API) + } + + fn settings(program: &Program, mode: &str) -> String { + let contracts = program.external_worker_contracts(); + let identity = contracts[0].errors.keys().next().unwrap(); + json!({"adapter":{"notification.delivery":{"setting":{"mode":mode,"identity":identity,"marker":"/data/lifecycle/worker"},"files":{"lifecycle":{"path":"lifecycle","write":true}}}}}) + .to_string() + } + + fn driver(mut ir: String, fault: Option<(u32, Option<&str>)>) -> String { + let check = match fault { + Some((code, message)) => llvm_module::fault_check(code, message), + None => " %passed = icmp eq i32 %status, 0".into(), + }; + ir.push_str(&format!("\ndefine i32 @dever_test_run() {{\nentry:\n %out = alloca i8\n %fault = alloca %dever.fault\n %status = call i32 @dever_application_entry(ptr %out, ptr %fault)\n{check}\n call void @dever_outputs_release(ptr %out)\n call void @dever_fault_release(ptr %fault)\n %exit = select i1 %passed, i32 0, i32 1\n ret i32 %exit\n}}\n")); + ir + } + + fn fixture() -> Vec<u8> { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + fs::read(root.join("target/native-release-inputs/component-fixture")) + .expect("prepare the immutable component-fixture in target/native-release-inputs") + } + + fn resource(path: impl Into<String>, bytes: Vec<u8>, executable: bool) -> EmbeddedResource { + EmbeddedResource { + path: path.into(), + sha256: format!("{:x}", Sha256::digest(&bytes)), + bytes, + executable, + } + } + + fn resources(program: &Program) -> Vec<EmbeddedResource> { + let contract = &program.external_worker_contracts()[0]; + let launch = json!({ + "format":"dever-worker-launch-v1", "ecosystem":contract.ecosystem, + "entry":contract.entry, "executable":"workers/fixture/worker", + "arguments":[], "working_directory":"workers/fixture" + }); + let mut resources = + super::sandbox::assets(&Path::new(env!("CARGO_MANIFEST_DIR")).join("../..")) + .into_iter() + .map(|asset| resource(asset.path, asset.bytes, asset.executable)) + .collect::<Vec<_>>(); + resources.extend([ + resource( + format!("{}.dever-worker.json", contract.entry), + serde_json::to_vec(&launch).unwrap(), + false, + ), + resource("workers/fixture/worker", fixture(), true), + ]); + resources + } + + fn stage(directory: &temp::TemporaryDirectory, setting: &str) { + fs::create_dir_all(directory.path().join("config")).unwrap(); + fs::create_dir_all(directory.path().join("lifecycle")).unwrap(); + fs::write(directory.path().join("config/setting.json"), setting).unwrap(); + } + + fn run( + source: &(SourceMap, Program), + mode: &str, + command: &str, + body: &str, + fault: Option<(u32, Option<&str>)>, + ) -> std::process::Output { + let (_, program) = source; + let directory = temp::TemporaryDirectory::new(); + stage(&directory, &settings(program, mode)); + let resources = resources(program); + let output = execute( + source, + &directory, + &resources, + &[command, body], + fault, + Duration::from_secs(60), + ); + if fault.is_none() { + assert!( + directory.path().join("lifecycle/worker.shutdown").is_file(), + "Worker must complete orderly shutdown" + ); + } + output + } + + fn execute( + source: &(SourceMap, Program), + directory: &temp::TemporaryDirectory, + resources: &[EmbeddedResource], + arguments: &[&str], + fault: Option<(u32, Option<&str>)>, + timeout: Duration, + ) -> std::process::Output { + let (sources, program) = source; + let module = + dever_core::llvm::emit_application_with_resources(program, sources, resources).unwrap(); + let ir = driver(module.ir, fault); + let attachments = module + .resources + .iter() + .map(|resource| BinaryResource { + symbol: &resource.symbol, + bytes: resource.bytes, + }) + .collect::<Vec<_>>(); + let executable = llvm_module::link_with_resources( + directory, + &ir, + "external Worker", + 64, + "managed-driver.c", + &attachments, + ); + llvm_module::execute_program(&executable, "external Worker", timeout, arguments) + } + + fn command_source() -> (SourceMap, Program) { + let mut sources = SourceMap::default(); + sources.add( + "tools/command/adapter.dever", + "external command \"bin/tool\" {}", + ); + sources.add("tools/command/port.dever", "execute(args: List<Text>, stdin: Bytes?) (output: dever.process.Output) fails dever.process.Error"); + sources.add("tools/command/api.dever", "cmd read = app.read"); + sources.add( + "tools/command/app.dever", + r#"type Summary { + code: Int + binary: Bool + stderr: Bool + eof: Bool +} +read() (summary: Summary) { + input = dever.bytes.from_ints([0, 255, 128, 10]) + echoed = port.execute(["stdin"], input) + empty = port.execute(["stdin"], null) + failed = port.execute(["exit", "7"], null) + summary = Summary { + code = failed.code + binary = echoed.stdout == input + stderr = echoed.stderr == input + eof = dever.bytes.length(empty.stdout) == 0 + } +} +"#, + ); + let program = dever_core::check(&sources).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }); + (sources, program) + } + + #[test] + fn command_apps_emit_typed_bytes_objects_for_all_targets() { + let (sources, program) = command_source(); + let ir = dever_core::llvm::emit_application(&program, &sources).unwrap(); + for target in Target::ALL { + assert!(!emit_object(&ir, target).unwrap().is_empty()); + } + } + + #[test] + #[ignore = "requires fresh runtime-external archive, command-fixture and sandbox assets"] + fn command_binary_io_and_exit_status_use_real_typed_llvm_calls() { + let source = command_source(); + let contract = &source.1.external_command_contracts()[0]; + let workspace = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let mut resources = super::sandbox::assets(&workspace) + .into_iter() + .map(|asset| resource(asset.path, asset.bytes, asset.executable)) + .collect::<Vec<_>>(); + resources.push(resource( + "commands/fixture/tool", + fs::read(workspace.join("target/native-release-inputs/command-fixture")) + .expect("prepare immutable command-fixture in target/native-release-inputs"), + true, + )); + resources.push(resource(format!("{}.dever-command.json", contract.entry), serde_json::to_vec(&json!({ + "format":"dever-command-launch-v1", "ecosystem":"command", "entry":contract.entry, + "executable":"commands/fixture/tool", "arguments":[], "working_directory":"commands/fixture" + })).unwrap(), false)); + let directory = temp::TemporaryDirectory::new(); + stage(&directory, "{}"); + let output = execute( + &source, + &directory, + &resources, + &["tools.command.read", "{}"], + None, + // Each of 65 roots starts three isolated processes and verifies + // their resources, unlike the persistent single-call Worker probe. + Duration::from_secs(180), + ); + result( + &output, + json!({"code":7, "binary":true, "stderr":true, "eof":true}), + ); + } + + fn result(output: &std::process::Output, expected: serde_json::Value) { + let rows: Vec<serde_json::Value> = String::from_utf8_lossy(&output.stdout) + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect(); + assert_eq!( + rows, + vec![json!({"code":0,"message":"ok","data":expected}); 65] + ); + } + + #[test] + fn external_apps_emit_objects_for_all_targets_and_kernels_stay_closed() { + let (sources, program) = read("exec"); + assert!( + dever_core::llvm::emit_kernel(&program, &sources, "notification.delivery.read") + .is_err() + ); + let ir = dever_core::llvm::emit_application(&program, &sources).unwrap(); + for target in Target::ALL { + assert!(!emit_object(&ir, target).unwrap().is_empty()); + } + } + + #[test] + fn large_embedded_resources_keep_ir_small_and_share_payloads() { + let (sources, program) = read("exec"); + let bytes = vec![0xA5; 9 * 1024 * 1024]; + let resources = [ + resource("shared/large", bytes.clone(), false), + resource("shared/alias", bytes, true), + ]; + let module = + dever_core::llvm::emit_application_with_resources(&program, &sources, &resources) + .unwrap(); + assert!(module.ir.len() < 1024 * 1024); + assert_eq!(module.resources.len(), 1); + assert_eq!(module.resources[0].bytes, resources[0].bytes); + assert_eq!( + module + .ir + .matches("= external constant [9437184 x i8]") + .count(), + 1 + ); + assert_eq!( + module + .ir + .matches("ptr @dever_external_bytes_0, i64 9437184") + .count(), + 2 + ); + } + + #[test] + fn embedded_metadata_rejects_bad_hashes_before_linking() { + let (sources, program) = read("exec"); + let mut entry = resource( + "module/notification/delivery/worker", + b"fixture".to_vec(), + true, + ); + let resources = [ + entry.clone(), + resource("shared/same-bytes", entry.bytes.clone(), false), + resource("shared/empty", Vec::new(), false), + ]; + let valid = + dever_core::llvm::emit_application_with_resources(&program, &sources, &resources) + .unwrap(); + assert_eq!( + valid.resources.len(), + 2, + "identical content has one initializer" + ); + let attachments = valid + .resources + .iter() + .map(|resource| BinaryResource { + symbol: &resource.symbol, + bytes: resource.bytes, + }) + .collect::<Vec<_>>(); + for target in Target::ALL { + assert!( + !emit_object_with_resources(&valid.ir, target, &attachments) + .unwrap() + .is_empty() + ); + } + entry.sha256 = "0".repeat(64); + assert!( + dever_core::llvm::emit_application_with_resources(&program, &sources, &[entry]) + .is_err() + ); + } + + #[test] + #[ignore = "requires explicit runtime-external archive and existing fixture Worker"] + fn worker_success_and_declared_failure_survive_repeated_roots() { + let source = read("exec"); + result( + &run(&source, "normal", "notification.delivery.read", "{}", None), + json!(7), + ); + result( + &run( + &source, + "declared_error", + "notification.delivery.capture", + "{}", + None, + ), + json!("fixture"), + ); + } + + #[test] + #[ignore = "requires explicit runtime-external archive and existing fixture Worker"] + fn typed_nested_values_and_multiple_outputs_round_trip() { + let source = checked( + "exec", + r#"type Outcome { error Rejected(reason: Text) } +type Payload { name: Text + tags: List<Text?> + count: Int + note: Text? } +echo(value: Payload) (answer: Payload) { answer = port.echo(value) } +"#, + "echo(value: app.Payload) (value: app.Payload) fails app.Outcome", + "cmd echo = app.echo", + ); + let value = json!({"name":"你好 🌍","tags":["first",null,"last"],"count":9223372036854775807_i64,"note":null}); + result( + &run( + &source, + "normal", + "notification.delivery.echo", + &json!({"value":value}).to_string(), + None, + ), + value, + ); + let source = checked( + "exec", + "type Outcome { error Rejected(reason: Text) }\ntype Pair { count: Int\n label: Text }\necho(input_count: Int, input_label: Text) (answer: Pair) { reply = port.echo(input_count, input_label)\n answer = Pair { count = reply.count\n label = reply.label } }", + "echo(count: Int, label: Text) (count: Int, label: Text) fails app.Outcome", + "cmd echo = app.echo", + ); + result( + &run( + &source, + "normal", + "notification.delivery.echo", + r#"{"input_count":42,"input_label":"owned"}"#, + None, + ), + json!({"count":42,"label":"owned"}), + ); + let source = checked( + "exec", + "type Outcome { error Rejected(reason: Text) }\necho() (done: Bool) { port.echo()\n done = true }", + "echo() () fails app.Outcome", + "cmd echo = app.echo", + ); + result( + &run(&source, "normal", "notification.delivery.echo", "{}", None), + json!(true), + ); + } + + #[test] + #[ignore = "requires explicit runtime-external archive and existing fixture Worker"] + fn malformed_output_and_business_error_payloads_are_runtime_faults() { + let source = read("exec"); + for mode in [ + "missing_output", + "extra_output", + "wrong_output", + "invalid_error", + "missing_error", + ] { + let output = run( + &source, + mode, + "notification.delivery.capture", + "{}", + Some((3, None)), + ); + assert!( + output.stdout.is_empty(), + "{mode} must not become a recoverable business error" + ); + } + let source = checked( + "exec", + "type Outcome { error Rejected(reason: Text) }\nfail() (value: Int) { value = port.fail() }", + "fail() (value: Int) fails app.Outcome", + "cmd fail = app.fail", + ); + let output = run( + &source, + "normal", + "notification.delivery.fail", + "{}", + Some(( + 3, + Some("external Adapter returned an undeclared business error"), + )), + ); + assert!(output.stdout.is_empty()); + let source = checked( + "exec", + "type Outcome { error Rejected(reason: Text) }\nread() (value: Int?) { value = port.read() }", + "read() (value: Int?) fails app.Outcome", + "cmd read = app.read", + ); + assert!( + run( + &source, + "missing_output", + "notification.delivery.read", + "{}", + Some((3, None)) + ) + .stdout + .is_empty() + ); + } + + #[test] + #[ignore = "requires explicit runtime-external archive and existing fixture Worker"] + fn handshake_and_setting_failures_do_not_leak_a_worker() { + let source = read("exec"); + for mode in ["duplicate_ready", "capability_mismatch"] { + assert!( + run( + &source, + mode, + "notification.delivery.read", + "{}", + Some((3, None)) + ) + .stdout + .is_empty() + ); + } + let (sources, program) = source; + let ir = driver( + dever_core::llvm::emit_application(&program, &sources).unwrap(), + Some((3, Some("expected wire string"))), + ); + // No worker exists: typed Setting rejection must happen before process launch. + assert!( + llvm_module::execute( + &ir, + "invalid external Setting", + 64, + Duration::from_secs(15), + &["notification.delivery.read", "{}"], + Some(r#"{"adapter":{"notification.delivery":{"setting":{"mode":7}}}}"#) + ) + .stdout + .is_empty() + ); + } + + #[test] + #[ignore = "requires explicit runtime-external archive and existing fixture Worker"] + fn cancelling_a_call_preserves_the_shared_worker() { + let source = checked( + "exec", + r#"type Outcome { error Rejected(reason: Text) } +pending() () { port.wait() } +read() (value: Int) { + task = run(pending()) + dever.task.sleep(5) + stop(task) + value = port.read() +} +"#, + "wait() () fails app.Outcome\nread() (value: Int) fails app.Outcome", + "cmd read = app.read", + ); + result( + &run(&source, "normal", "notification.delivery.read", "{}", None), + json!(7), + ); + } + + #[test] + #[ignore = "requires explicit runtime-external archive and existing fixture Worker"] + fn independent_executables_use_verified_embedded_exec_and_managed_launches() { + for ecosystem in ["exec", "pip", "npm", "go"] { + let (sources, program) = read(ecosystem); + let resources = resources(&program); + let module = + dever_core::llvm::emit_application_with_resources(&program, &sources, &resources) + .unwrap(); + let ir = driver(module.ir, None); + let attachments = module + .resources + .iter() + .map(|resource| BinaryResource { + symbol: &resource.symbol, + bytes: resource.bytes, + }) + .collect::<Vec<_>>(); + let build = temp::TemporaryDirectory::new(); + let executable = llvm_module::link_with_resources( + &build, + &ir, + ecosystem, + 64, + "managed-driver.c", + &attachments, + ); + let directory = temp::TemporaryDirectory::new(); + let relocated = directory.path().join("program"); + fs::copy(executable, &relocated).unwrap(); + drop(build); + stage(&directory, &settings(&program, "embedded")); + result( + &llvm_module::execute_program( + &relocated, + ecosystem, + Duration::from_secs(120), + &["notification.delivery.read", "{}"], + ), + json!(7), + ); + assert!( + !directory.path().join("module").exists(), + "only embedded Worker resources may be used" + ); + } + } +} diff --git a/test/llvm_http_system_source.rs b/test/llvm_http_system_source.rs new file mode 100644 index 0000000..c1a5811 --- /dev/null +++ b/test/llvm_http_system_source.rs @@ -0,0 +1,702 @@ +//! Checked-source protocol/system acceptance through the actual LLVM runtime archive. + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/llvm.rs"] +mod llvm; +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/llvm_managed.rs"] +mod llvm_managed; +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/process.rs"] +mod process; +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/temp.rs"] +mod temp; + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +mod native { + use super::{llvm, llvm_managed}; + use dever_backend_bridge::{Target, emit_object}; + use dever_core::{check, source::SourceMap}; + use std::io::{Read, Write}; + use std::net::{TcpListener, TcpStream}; + use std::time::Duration; + + const COMMON: &str = r#"some_h2(value: dever.http.Http2Limits) (answer: dever.http.Http2Limits?) { answer = value } +some_text(value: Text) (answer: Text?) { answer = value } +some_int(value: Int) (answer: Int?) { answer = value } +status(valid: true) (code: Int) { code = 207 } +status(valid: false) (code: Int) { code = 500 } +named(header: dever.http.Header, name: Text) (matches: Bool) { matches = header.name == name } +header_bytes(header: dever.http.Header, previous: List<Bytes>) (values: List<Bytes>) { values = append(previous, header.value) } +collect(event: dever.io.ReadEvent.Chunk(bytes), previous: Bytes) (next: Bytes) { next = dever.bytes.concat(previous, bytes) } +collect(event: dever.io.ReadEvent.Failed(message), previous: Bytes) (next: Bytes) { fail(dever.io.ReadEvent.Failed(message)) } +body(values: AsyncStream<dever.io.ReadEvent>) (bytes: Bytes) { bytes = reduce(collect, values, dever.bytes.from_text("")) } +request() (value: dever.http.Request) { + value = dever.http.Request { method = "POST" + target = "/echo?raw=1" + headers = [dever.http.header("Host", "fixture.test"), dever.http.header("X-Check", "one"), dever.http.header("X-Check", "two")] + body = dever.bytes.from_text("payload") } +} +route(request: dever.http.Request) (response: dever.http.Response) { + values = reduce(header_bytes, filter(named, request.headers, "x-check"), []) + valid = request.method == "POST" and request.target == "/echo?raw=1" and values == [dever.bytes.from_text("one"), dever.bytes.from_text("two")] + response = dever.http.Response { status = status(valid) + headers = [dever.http.header("X-Echo", "first"), dever.http.header("X-Echo", "second")] + body = request.body } +} +good(response: dever.http.Response) (valid: Bool) { + values = reduce(header_bytes, filter(named, response.headers, "x-echo"), []) + valid = response.status == 207 and response.body == dever.bytes.from_text("payload") and values == [dever.bytes.from_text("first"), dever.bytes.from_text("second")] +} +"#; + + const HTTP_SEND: &str = r#"public main() (answer: Bool) { + listener = dever.net.listen("127.0.0.1", 0) + port = dever.net.port(listener) + server = run(dever.http.serve(route, listener, dever.http.default_limits())) + response = dever.http.send("127.0.0.1", port, request(), dever.http.default_limits()) + answer = good(response) + stop(server) + dever.net.close_listener(listener) +} +"#; + + const HTTP_POOL: &str = r#"public main() (answer: Bool) { + listener = dever.net.listen("127.0.0.1", 0) + port = dever.net.port(listener) + limits = dever.http.default_limits() + limits.http2 = HTTP2 + server = run(SERVE) + client = dever.http.client(ORIGIN + int.to_text(port), TRUST, limits, dever.http.default_pool_limits()) + first = dever.http.request(client, request()) + first_good = good(first) + second = dever.http.open(client, request()) + second_status = second.status + second_body = body(second.body) + input = channel(dever.io.ReadEvent, 2) + send(input, dever.io.ReadEvent.Chunk(dever.bytes.from_text("pay"))) + send(input, dever.io.ReadEvent.Chunk(dever.bytes.from_text("load"))) + close(input) + third_request = dever.http.StreamRequest { method = "POST" + target = "/echo?raw=1" + headers = request().headers + body = stream(input) } + third = dever.http.open_stream(client, third_request) + third_status = third.status + third_body = body(third.body) + dever.http.close_client(client) + stop(server) + dever.net.close_listener(listener) + answer = first_good and second_status == 207 and second_body == dever.bytes.from_text("payload") and third_status == 207 and third_body == dever.bytes.from_text("payload") +} +"#; + + const LIVE: &str = r#"live(request: dever.http.Request, reply: dever.system.HttpReply) () { + LIVE_BODY +} +public main() (answer: Bool) { + listener = dever.net.listen("127.0.0.1", 0) + port = dever.net.port(listener) + server = run(dever.http.serve_live(live, listener, dever.http.default_limits(), dever.http.default_live_limits())) + response = dever.http.send("127.0.0.1", port, request(), dever.http.default_limits()) + answer = LIVE_CHECK + stop(server) + dever.net.close_listener(listener) +} +"#; + + const HANDLER_FAILURE: &str = r#"type Failure { error Broken(message: Text, history: List<Text>) } +fault_route(request: dever.http.Request) (response: dever.http.Response) { + response = pick_route(request.target, request) +} +pick_route(target: "/fail", request: dever.http.Request) (response: dever.http.Response) { + dever.task.sleep(1) + fail(Failure.Broken("owned handler fault", ["original", "payload"])) +} +pick_route(target: other, request: dever.http.Request) (response: dever.http.Response) { response = route(request) } +public main() (answer: Bool) { + listener = dever.net.listen("127.0.0.1", 0) + port = dever.net.port(listener) + server = run(dever.http.serve(fault_route, listener, dever.http.default_limits())) + failed_request = request() + failed_request.target = "/fail" + failed = dever.http.send("127.0.0.1", port, failed_request, dever.http.default_limits()) + failed_status = failed.status + response = dever.http.send("127.0.0.1", port, request(), dever.http.default_limits()) + answer = failed_status == 500 and good(response) + stop(server) + dever.net.close_listener(listener) +} +"#; + + const CONTEXT: &str = r#"type Context { label: Text + tags: List<Text> } +first_text(value: Text) (text: Text) { text = value } +first_text(value: null) (text: Text) { text = "missing" } +context_route(request: dever.http.Request, context: Context) (response: dever.http.Response) { + dever.task.sleep(1) + response = dever.http.Response { status = 200 + headers = [] + body = dever.bytes.from_text(context.label + first_text(first(context.tags))) } +} +serve_context(listener: dever.system.Listener, context: Context) () { + dever.system.http_serve(context_route, listener, dever.http.default_limits(), context) +} +public main() (answer: Bool) { + listener = dever.net.listen("127.0.0.1", 0) + port = dever.net.port(listener) + context = Context { label = "initial" + tags = ["-owned"] } + server = run(serve_context(listener, context)) + context.label = "changed" + response = dever.http.send("127.0.0.1", port, request(), dever.http.default_limits()) + answer = response.status == 200 and response.body == dever.bytes.from_text("initial-owned") and context.label == "changed" + stop(server) + dever.net.close_listener(listener) +} +"#; + + const ERRORS: &str = r#"bad_date(value: dever.time.DateResult.Read(date)) (valid: Bool) recover("fixture expects invalid calendar") { valid = false } +bad_date(value: dever.time.DateResult.Failed(message)) (valid: Bool) recover("fixture expects invalid calendar") { valid = message == "invalid calendar Date" } +bad_time(value: dever.time.TimeResult.Read(time)) (valid: Bool) recover("fixture expects invalid clock") { valid = false } +bad_time(value: dever.time.TimeResult.Failed(message)) (valid: Bool) recover("fixture expects invalid clock") { valid = message == "invalid clock Time" } +bad_tls(value: dever.tls.ClientResult.Ready(config)) (valid: Bool) recover("fixture expects invalid CA") { valid = false } +bad_tls(value: dever.tls.ClientResult.Failed(message)) (valid: Bool) recover("fixture expects invalid CA") { valid = message == "TLS PEM contains no certificates" } +bad_token(value: dever.crypto.TokenResult.Ready(secret)) (valid: Bool) recover("fixture expects invalid token length") { valid = false } +bad_token(value: dever.crypto.TokenResult.Failed(message)) (valid: Bool) recover("fixture expects invalid token length") { valid = message == "token length must be between 16 and 1024 bytes" } +public main() (answer: Bool) { + answer = bad_date(result(dever.time.parse_date("2024-02-30"))) and bad_time(result(dever.time.parse_time("25:00:00"))) and bad_tls(result(dever.tls.client(dever.bytes.from_text("")))) and bad_token(result(dever.crypto.token(1))) +} +"#; + + const WEBSOCKET: &str = r#"message(state: dever.websocket.ReceiveState.Read(value)) (message: dever.websocket.Message) { message = value } +message(state: dever.websocket.ReceiveState.End) (message: dever.websocket.Message) { message = dever.websocket.Message.Text("unexpected end") } +echo(request: dever.http.Request, reply: dever.system.HttpReply) () { + socket = dever.websocket.accept(reply, dever.websocket.default_limits()) + first = message(dever.websocket.receive(socket)) + dever.websocket.send(socket, first) + second = message(dever.websocket.receive(socket)) + dever.websocket.send(socket, second) + dever.websocket.close(socket, 1000, "done") +} +public main() (answer: Bool) { + listener = dever.net.listen("127.0.0.1", 0) + port = dever.net.port(listener) + server = run(WS_SERVE) + socket = WS_CONNECT + dever.websocket.send(socket, dever.websocket.Message.Text("你好")) + first = message(dever.websocket.receive(socket)) + first_ok = first == dever.websocket.Message.Text("你好") + dever.websocket.send(socket, dever.websocket.Message.Binary(dever.bytes.from_ints([0, 255, 1]))) + second = message(dever.websocket.receive(socket)) + second_ok = second == dever.websocket.Message.Binary(dever.bytes.from_ints([0, 255, 1])) + close_state = dever.websocket.receive(socket) + ended = is_end(close_state) + stop(server) + dever.net.close_listener(listener) + answer = first_ok and second_ok and ended +} +is_end(state: dever.websocket.ReceiveState.End) (yes: Bool) { yes = true } +is_end(state: dever.websocket.ReceiveState.Read(value)) (yes: Bool) { yes = false } +"#; + + const WEBSOCKET_STREAM: &str = r#"push_messages(request: dever.http.Request, reply: dever.system.HttpReply) () { + socket = dever.websocket.accept(reply, dever.websocket.default_limits()) + dever.websocket.send(socket, dever.websocket.Message.Text("stream")) + dever.websocket.send(socket, dever.websocket.Message.Binary(dever.bytes.from_ints([0, 255]))) + dever.websocket.send(socket, dever.websocket.Message.Ping(dever.bytes.from_ints([4]))) + dever.websocket.send(socket, dever.websocket.Message.Pong(dever.bytes.from_ints([5]))) + dever.websocket.close(socket, 1000, "done") +} +collect_message(event: dever.websocket.MessageEvent.Read(message), previous: List<dever.websocket.Message>) (next: List<dever.websocket.Message>) { next = append(previous, message) } +collect_message(event: dever.websocket.MessageEvent.Failed(message), previous: List<dever.websocket.Message>) (next: List<dever.websocket.Message>) { fail(dever.websocket.MessageEvent.Failed(message)) } +public main() (answer: Bool) { + listener = dever.net.listen("127.0.0.1", 0) + port = dever.net.port(listener) + server = run(dever.http.serve_live(push_messages, listener, dever.http.default_limits(), dever.http.default_live_limits())) + socket = dever.websocket.connect("127.0.0.1", port, "/events", dever.websocket.default_limits()) + values = dever.websocket.messages(socket) + alias = values + messages = reduce(collect_message, values, []) + remaining = reduce(collect_message, alias, []) + stop(server) + dever.net.close_listener(listener) + answer = messages == [dever.websocket.Message.Text("stream"), dever.websocket.Message.Binary(dever.bytes.from_ints([0, 255])), dever.websocket.Message.Ping(dever.bytes.from_ints([4])), dever.websocket.Message.Pong(dever.bytes.from_ints([5]))] and length(remaining) == 0 +} +"#; + + const TIME: &str = r#"public main() (answer: Bool) { + value = dever.time.parse_datetime("2024-02-29T01:02:03.004+01:00") + date = dever.time.parse_date("2024-02-29") + time = dever.time.parse_time("01:02:03.004") + delta = dever.time.duration(1000) + later = dever.time.add(value, delta) + previous = dever.time.subtract(later, delta) + difference = dever.time.difference(later, value) + now = dever.time.now() + clock = dever.time.unix_millis() + first = dever.time.monotonic_nanos() + second = dever.time.monotonic_nanos() + answer = dever.time.format_datetime(value) == "2024-02-29T00:02:03.004Z" and dever.time.format_date(date) == "2024-02-29" and dever.time.format_time(time) == "01:02:03.004" and previous == value and difference == delta and text.ends_with(dever.time.format_datetime(now), "Z") and clock > 0 and second >= first +} +"#; + + const UUID: &str = r#"type Identifier { value: Uuid } +to_text(value: null) (text: Text) { text = "none" } +to_text(value: Uuid) (text: Text) { text = dever.system.uuid_to_text(value) } +public main() (answer: Bool) { + value = dever.system.uuid_parse("550E8400-E29B-41D4-A716-446655440000") + missing = dever.system.uuid_parse("invalid") + identifier = Identifier { value = "550e8400-e29b-41d4-a716-446655440000" } + expected = Identifier { value = "550E8400-E29B-41D4-A716-446655440000" } + answer = to_text(value) == "550e8400-e29b-41d4-a716-446655440000" and to_text(missing) == "none" and identifier.value == expected.value +} +"#; + + const CRYPTO: &str = r#"public main() (answer: Bool) { + key = dever.crypto.token(16) + alias = key + bytes = dever.bytes.from_text("abc") + digest = dever.crypto.sha256(bytes) + first = dever.crypto.hmac_sha256(key, bytes) + second = dever.crypto.hmac_sha256(alias, bytes) + expected = dever.bytes.from_ints([186,120,22,191,143,1,207,234,65,65,64,222,93,174,34,35,176,3,97,163,150,23,122,156,180,16,255,97,242,0,21,173]) + answer = digest == expected and dever.bytes.length(first) == 32 and dever.crypto.constant_time_eq(first, second) and not dever.crypto.constant_time_eq(first, dever.bytes.from_text("different")) +} +"#; + + const PASSWORD: &str = r#"public main() (answer: Bool) { + key = dever.crypto.token(16) + wrong = dever.crypto.token(16) + encoded = blocking(dever.crypto.password_hash(key)) + verified = blocking(dever.crypto.password_verify(key, encoded)) + rejected = blocking(dever.crypto.password_verify(wrong, encoded)) + answer = verified and not rejected and text.starts_with(encoded, "$argon2id$v=19$") +} +"#; + + const OUTPUT: &str = r#"public main() (answer: Bool) { + dever.io.println("原生输出") + fields = {"component" = "fixture" + "quoted" = "line\n\"value\""} + dever.log.debug("ready", fields) + dever.log.info("ready", fields) + dever.log.warn("ready", fields) + dever.log.error("ready", fields) + answer = true +} +"#; + + fn source(body: &str) -> String { + format!("{COMMON}\n{body}") + } + + fn pool(http2: bool, tls: bool) -> String { + let serve = if tls { + "dever.http.serve_tls(route, listener, limits, dever.tls.server(CERT, KEY))" + } else { + "dever.http.serve(route, listener, limits)" + }; + certificates( + &HTTP_POOL + .replace( + "HTTP2", + if http2 { + "some_h2(dever.http.default_http2_limits())" + } else { + "null" + }, + ) + .replace("SERVE", serve) + .replace( + "ORIGIN", + if tls { + "\"https://localhost:\"" + } else { + "\"http://127.0.0.1:\"" + }, + ) + .replace( + "TRUST", + if tls { + "dever.tls.client(ROOT)" + } else { + "dever.tls.system()" + }, + ), + ) + } + + fn certificates(body: &str) -> String { + body.replace( + "CERT", + &format!( + "dever.bytes.from_text({:?})", + include_str!("dever-tests/fixtures/tls/server.pem") + ), + ) + .replace( + "KEY", + &format!( + "dever.bytes.from_text({:?})", + include_str!("dever-tests/fixtures/tls/server-key.pem") + ), + ) + .replace( + "ROOT", + &format!( + "dever.bytes.from_text({:?})", + include_str!("dever-tests/fixtures/tls/root.pem") + ), + ) + } + + fn live(body: &str, check: &str) -> String { + LIVE.replace("LIVE_BODY", body).replace("LIVE_CHECK", check) + } + + fn websocket(tls: bool, url: bool) -> String { + let serve = if tls { + "dever.http.serve_live_tls(echo, listener, dever.http.default_limits(), dever.http.default_live_limits(), dever.tls.server(CERT, KEY))" + } else { + "dever.http.serve_live(echo, listener, dever.http.default_limits(), dever.http.default_live_limits())" + }; + let connect = if tls { + "dever.websocket.open(\"wss://localhost:\" + int.to_text(port) + \"/echo\", dever.tls.client(ROOT), dever.websocket.default_limits())" + } else if url { + "dever.websocket.open(\"ws://127.0.0.1:\" + int.to_text(port) + \"/echo\", dever.tls.system(), dever.websocket.default_limits())" + } else { + "dever.websocket.connect(\"127.0.0.1\", port, \"/echo\", dever.websocket.default_limits())" + }; + certificates( + &WEBSOCKET + .replace("WS_SERVE", serve) + .replace("WS_CONNECT", connect), + ) + } + + fn cases() -> Vec<(&'static str, String)> { + vec![ + ("http_send", HTTP_SEND.into()), + ("http_pool", pool(false, false)), + ("http2_pool", pool(true, false)), + ("tls_pool", pool(false, true)), + ("tls_http2_pool", pool(true, true)), + ( + "live_respond", + live( + "dever.http.respond(reply, route(request))", + "good(response)", + ), + ), + ( + "live_chunks", + live( + "dever.http.start(reply, 207, [])\n dever.http.write(reply, dever.bytes.from_text(\"pay\"))\n dever.task.sleep(1)\n dever.http.write(reply, dever.bytes.from_text(\"load\"))\n dever.http.finish(reply)", + "response.status == 207 and response.body == dever.bytes.from_text(\"payload\")", + ), + ), + ( + "sse", + live( + "dever.sse.start(reply, [])\n dever.sse.send(reply, dever.sse.Event { event = \"update\"\n data = \"first\\nsecond\\n\"\n id = some_text(\"42\")\n retry_ms = some_int(123) })\n dever.http.finish(reply)", + "response.status == 200 and response.body == dever.bytes.from_text(\"event: update\\nid: 42\\nretry: 123\\ndata: first\\ndata: second\\ndata: \\n\\n\")", + ), + ), + ("handler_failure", HANDLER_FAILURE.into()), + ("context", CONTEXT.into()), + ("ws_connect", websocket(false, false)), + ("ws_open", websocket(false, true)), + ("wss_open", websocket(true, true)), + ("ws_messages", WEBSOCKET_STREAM.into()), + ("time", TIME.into()), + ("uuid", UUID.into()), + ("crypto", CRYPTO.into()), + ("password", PASSWORD.into()), + ("output", OUTPUT.into()), + ("errors", ERRORS.into()), + ( + "arguments", + "public main() (answer: Bool) { answer = length(dever.process.arguments()) == 0 }" + .into(), + ), + ] + } + + #[test] + fn protocol_system_sources_pass_the_existing_checker() { + for (name, body) in cases() { + let mut sources = SourceMap::default(); + sources.add("main.dever", source(&body)); + check(&sources).unwrap_or_else(|errors| { + panic!( + "{name}: {}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }); + } + } + + #[test] + fn protocol_system_sources_emit_objects_for_six_targets() { + for (name, body) in cases() { + let ir = llvm::lower(&source(&body)); + for target in [ + Target::LinuxX86_64, + Target::LinuxAarch64, + Target::MacosX86_64, + Target::MacosAarch64, + Target::WindowsX86_64, + Target::WindowsAarch64, + ] { + let object = emit_object(&ir, target) + .unwrap_or_else(|error| panic!("{name}/{target:?}: {error}")); + assert!(!object.is_empty(), "{name}/{target:?}"); + } + } + } + + fn run_case(name: &str) { + let body = cases() + .into_iter() + .find(|(candidate, _)| *candidate == name) + .unwrap() + .1; + llvm_managed::boolean(&source(&body)); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive and owned loopback"] + fn single_http_request_preserves_headers_body_and_status() { + run_case("http_send"); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive and owned loopback"] + fn pooled_http_requests_and_stream_upload_download() { + run_case("http_pool"); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive and owned loopback"] + fn pooled_http2_requests_and_stream_upload_download() { + run_case("http2_pool"); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive and owned loopback"] + fn tls_verifies_local_certificate_and_streams_http() { + run_case("tls_pool"); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive and owned loopback"] + fn tls_negotiates_http2_and_streams_upload_download() { + run_case("tls_http2_pool"); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive and owned loopback"] + fn live_reply_respond_preserves_response() { + run_case("live_respond"); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive and owned loopback"] + fn live_reply_start_write_finish_preserves_chunks() { + run_case("live_chunks"); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive and owned loopback"] + fn sse_preserves_nullable_metadata_and_multiline_data() { + run_case("sse"); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive and owned loopback"] + fn owned_handler_failure_is_a_500_then_service_continues() { + run_case("handler_failure"); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive and owned loopback"] + fn static_suspending_handler_clones_owned_context_before_initial_suspend() { + run_case("context"); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive and owned loopback"] + fn websocket_connect_echoes_text_binary_and_natural_end() { + run_case("ws_connect"); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive and owned loopback"] + fn websocket_url_open_echoes_text_binary_and_natural_end() { + run_case("ws_open"); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive and owned loopback"] + fn secure_websocket_uses_local_tls_trust() { + run_case("wss_open"); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive and owned loopback"] + fn websocket_message_stream_keeps_all_typed_variants_and_closes_aliases() { + run_case("ws_messages"); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn calendar_duration_and_clocks_keep_runtime_semantics() { + run_case("time"); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn uuid_parse_null_and_canonical_text() { + run_case("uuid"); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn crypto_secret_alias_sha256_hmac_and_comparison() { + run_case("crypto"); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn recoverable_system_errors_keep_the_original_messages() { + run_case("errors"); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive and bounded crypto workers"] + fn password_hash_verify_uses_the_existing_blocking_boundary() { + llvm_managed::boolean_bounded(&source(PASSWORD), 2, Duration::from_secs(45), &[]); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn process_arguments_exclude_executable_and_keep_unicode() { + llvm_managed::boolean_bounded( + "public main() (answer: Bool) { answer = dever.process.arguments() == [\"first\", \"中文\"] }", + 64, + Duration::from_secs(5), + &["first", "中文"], + ); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn stdout_and_structured_logs_are_flushed_before_native_return() { + let output = + llvm_managed::boolean_bounded(&source(OUTPUT), 64, Duration::from_secs(5), &[]); + assert_eq!( + String::from_utf8(output.stdout).unwrap(), + "原生输出\n".repeat(65) + ); + let record = |level: &str| { + format!( + "{{\"level\":\"{level}\",\"message\":\"ready\",\"fields\":{{\"component\":\"fixture\",\"quoted\":\"line\\n\\\"value\\\"\"}}}}\n" + ) + }; + let expected = ["debug", "info", "warn", "error"] + .map(record) + .concat() + .repeat(65); + assert_eq!(String::from_utf8(output.stderr).unwrap(), expected); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive and independent owned loopback peer"] + fn http_client_matches_independent_chunked_wire_and_repeated_headers() { + let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + listener.set_nonblocking(true).unwrap(); + let port = listener.local_addr().unwrap().port(); + let peer = std::thread::spawn(move || { + let deadline = std::time::Instant::now() + Duration::from_secs(20); + for _ in 0..65 { + let mut socket = accept_before(&listener, deadline); + socket + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + socket + .set_write_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let expected = + b"POST /echo HTTP/1.1\r\nhost: example.test\r\ncontent-length: 1\r\n\r\nx"; + let mut request = vec![0; expected.len()]; + socket.read_exact(&mut request).unwrap(); + assert_eq!(request, expected); + for chunk in [ + b"HTTP/1.1 201 Created\r\nTransfer-Encoding: chunked\r\nSet-Cookie: a=1\r\n" + .as_slice(), + b"Set-Cookie: b=2\r\n\r\n1\r\na\r\n", + b"2\r\nbc\r\n0\r\n\r\n", + ] { + socket.write_all(chunk).unwrap(); + } + } + }); + let source = source(&format!( + r#"public main() (answer: Bool) {{ + request = dever.http.Request {{ method = "POST" + target = "/echo" + headers = [dever.http.header("Host", "example.test")] + body = dever.bytes.from_text("x") }} + response = dever.http.send("127.0.0.1", {port}, request, dever.http.default_limits()) + values = reduce(header_bytes, filter(named, response.headers, "set-cookie"), []) + answer = response.status == 201 and response.body == dever.bytes.from_text("abc") and values == [dever.bytes.from_text("a=1"), dever.bytes.from_text("b=2")] +}} +"# + )); + // Always join our peer, including when source lowering or execution fails. + let execution = std::panic::catch_unwind(|| llvm_managed::boolean(&source)); + let peer_result = peer.join(); + if let Err(failure) = execution { + std::panic::resume_unwind(failure); + } + peer_result.unwrap(); + } + + fn accept_before(listener: &TcpListener, deadline: std::time::Instant) -> TcpStream { + loop { + match listener.accept() { + Ok((socket, _)) => return socket, + Err(error) + if error.kind() == std::io::ErrorKind::WouldBlock + && std::time::Instant::now() < deadline => + { + std::thread::sleep(Duration::from_millis(2)) + } + Err(error) => panic!("owned HTTP peer accept failed: {error}"), + } + } + } + + #[test] + #[ignore = "requires explicit runtime-abi archive and owned loopback listener"] + fn invalid_server_limits_are_an_uncaptured_original_runtime_fault() { + llvm_managed::runtime_fault( + &source( + r#"public main() (answer: Bool) { + listener = dever.net.listen("127.0.0.1", 0) + limits = dever.http.default_limits() + limits.header_bytes = 1 + dever.http.serve(route, listener, limits) + answer = false +} +"#, + ), + "HTTP header_bytes must be at least 8192", + ); + } +} diff --git a/test/llvm_job_security_cases.rs b/test/llvm_job_security_cases.rs new file mode 100644 index 0000000..8e7c19f --- /dev/null +++ b/test/llvm_job_security_cases.rs @@ -0,0 +1,252 @@ +//! HTTP enqueues trusted references; the later worker must recheck current grants. +use super::{ + native::{Server, application_ir}, + process, security, sqlite_inspect, +}; +use dever_backend_bridge::{Target, emit_object}; +use dever_runtime::{config::Settings, orm::Value as SqlValue}; +use serde_json::{Value, json}; +use std::fs; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +fn files() -> Vec<(String, String)> { + let mut files = security::files() + .into_iter() + .map(|(path, text)| (path.to_owned(), text.to_owned())) + .collect::<Vec<_>>(); + files.push(( + "news/article/model/delivery.dever".into(), + "type Delivery { name: Text }".into(), + )); + files + .iter_mut() + .find(|(path, _)| path == "news/article/app.dever") + .unwrap() + .1 + .push_str( + r#" +type Delivery { name: Text } +submit(name: Text) (id: Id) { + id = dever.job.enqueue(job.publish, Delivery { name = name }, name) +} +deliver(input: Delivery) () { + saved = model.delivery.create({ name = input.name }) +} +"#, + ); + files + .iter_mut() + .find(|(path, _)| path == "news/article/api/admin/manage.dever") + .unwrap() + .1 + .push_str("\npost submit = app.submit"); + files.push(("news/article/job.dever".into(), "database default\njob publish(input: app.Delivery) () retry(2) timeout(1000) { app.deliver(input) }".into())); + files +} + +fn configuration() -> Value { + let mut value = security::configuration(); + value["database"]["default"]["max_connections"] = 3.into(); + value["job"] = json!({"workers": 2, "poll_ms": 10, "lease_ms": 3000, "retry_base_ms": 20, "retry_max_ms": 100}); + value +} + +#[test] +fn job_application_auth_and_tenant_roots_emit_all_six_targets() { + let files = files(); + let files = files + .iter() + .map(|(path, source)| (path.as_str(), source.as_str())) + .collect::<Vec<_>>(); + let ir = application_ir(&files, &configuration().to_string(), false, None); + for target in Target::ALL { + assert!(!emit_object(&ir, target).unwrap().is_empty()); + } +} + +fn queue_settled(server: &Server, tenant: i64) { + let path = server + .directory + .path() + .join(format!("data/tenants/tenant_{tenant}.db")); + let deadline = Instant::now() + Duration::from_secs(10); + loop { + let rows = sqlite_inspect::query( + &path, + "SELECT COUNT(*) FROM _dever_jobs WHERE state IN ('pending','running')", + ); + if rows == [vec![SqlValue::Int(0)]] { + return; + } + assert!( + Instant::now() < deadline, + "tenant {tenant} jobs did not settle: {rows:?}\n{}", + server.stderr() + ); + std::thread::sleep(Duration::from_millis(10)); + } +} + +#[test] +#[ignore = "requires explicit runtime archive; owned loopback API/worker and two tenant SQLite databases"] +fn queued_users_recheck_roles_sessions_catalog_and_tenant_components() { + let owned_files = files(); + let files = owned_files + .iter() + .map(|(path, source)| (path.as_str(), source.as_str())) + .collect::<Vec<_>>(); + let mut server = Server::start_with(&files, 0, configuration(), &security::tenant_bootstrap()); + server.ready_at("/sample/echo/test/probe/increment?q=1"); + let settings = Settings::load_project(server.directory.path()).unwrap(); + let owner_one = security::credential(&settings, "admin", "owner", "active", "one"); + let owner_two = security::credential(&settings, "admin", "owner", "active", "two"); + let reader = security::credential(&settings, "admin", "reader", "active", "one"); + let manage = "/news/article/admin/manage"; + security::request( + &server, + "POST", + &format!("{manage}/role"), + &owner_one, + json!({"id": "publisher", "keys": ["news.article.admin.submit"]}), + 200, + ); + security::request( + &server, + "POST", + &format!("{manage}/grant"), + &owner_one, + json!({"user_id": 2, "role_id": "publisher"}), + 200, + ); + for (name, cookie) in [ + ("allowed-one", &owner_one), + ("allowed-two", &owner_two), + ("revoked", &reader), + ("stale-session", &owner_one), + ("removed-permission", &owner_one), + ("missing-subject", &owner_one), + ("wrong-tenant", &owner_one), + ] { + security::request( + &server, + "POST", + &format!("{manage}/submit"), + cookie, + json!({"name": name}), + 200, + ); + } + security::request( + &server, + "POST", + &format!("{manage}/revoke"), + &owner_one, + json!({"user_id": 2, "role_id": "publisher"}), + 200, + ); + server.stop_cycle(1); + server.finish(); + + let first = server.directory.path().join("data/tenants/tenant_1.db"); + let second = server.directory.path().join("data/tenants/tenant_2.db"); + assert_eq!( + sqlite_inspect::query( + &first, + "SELECT DISTINCT auth_permission,execution_kind,state FROM _dever_jobs" + ), + [vec![ + SqlValue::Text("news.article.admin.submit".into()), + SqlValue::Text("user".into()), + SqlValue::Text("pending".into()) + ]] + ); + // Keep stored references structurally valid so dispatch must reverify them. + // Corrupt storage ownership is rejected earlier by the shared queue store. + for sql in [ + "UPDATE _dever_jobs SET auth_session='stale' WHERE dedupe_key='business:stale-session'", + "UPDATE _dever_jobs SET auth_permission='news.article.admin.removed' WHERE dedupe_key='business:removed-permission'", + "UPDATE _dever_jobs SET auth_subject='deleted-account' WHERE dedupe_key='business:missing-subject'", + "UPDATE _dever_jobs SET auth_tenant='two' WHERE dedupe_key='business:wrong-tenant'", + ] { + sqlite_inspect::execute(&first, sql); + } + + server.restart("all"); + server.ready_at("/sample/echo/test/probe/increment?q=1"); + queue_settled(&server, 1); + queue_settled(&server, 2); + assert_eq!( + sqlite_inspect::query(&first, "SELECT name FROM delivery"), + [vec![SqlValue::Text("allowed-one".into())]] + ); + assert_eq!( + sqlite_inspect::query(&second, "SELECT name FROM delivery"), + [vec![SqlValue::Text("allowed-two".into())]] + ); + let blocked = sqlite_inspect::query( + &first, + "SELECT dedupe_key,state,attempt,error FROM _dever_jobs WHERE dedupe_key!='business:allowed-one' ORDER BY dedupe_key", + ); + assert_eq!(blocked.len(), 5); + assert!( + blocked.iter().all(|row| row[1..] + == [ + SqlValue::Text("blocked".into()), + SqlValue::Int(1), + SqlValue::Text("identity_rejected".into()) + ]), + "{blocked:?}" + ); + server.stop_cycle(1); + server.finish(); + + // A fresh API-only root enqueues while enabled. The same stored user must + // be rejected after the platform disables that tenant's component. + server.restart("api"); + server.ready_at("/sample/echo/test/probe/increment?q=1"); + security::request( + &server, + "POST", + &format!("{manage}/submit"), + &owner_one, + json!({"name": "disabled"}), + 200, + ); + server.stop_cycle(1); + server.finish(); + let status = process::status( + Command::new(server.directory.path().join("managed-kernel")) + .env_clear() + .args(["--dever-tenant-component", "1", "disable", "news"]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()), + Duration::from_secs(10), + ) + .unwrap(); + assert!(status.success()); + server.restart("worker"); + queue_settled(&server, 1); + server.stop_cycle(1); + server.finish(); + assert_eq!( + sqlite_inspect::query( + &first, + "SELECT state,error FROM _dever_jobs WHERE dedupe_key='business:disabled'" + ), + [vec![ + SqlValue::Text("blocked".into()), + SqlValue::Text("component_disabled".into()) + ]] + ); + assert_eq!( + sqlite_inspect::query(&first, "SELECT COUNT(*) FROM delivery"), + [vec![SqlValue::Int(1)]] + ); + assert!( + !fs::read_to_string(server.directory.path().join("stderr")) + .unwrap() + .contains("panicked") + ); +} diff --git a/test/llvm_job_test_source.rs b/test/llvm_job_test_source.rs new file mode 100644 index 0000000..4b238fe --- /dev/null +++ b/test/llvm_job_test_source.rs @@ -0,0 +1,621 @@ +//! Existing Job and application Test contracts through the LLVM runtime archive. +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/llvm_module.rs"] +mod llvm_module; +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/process.rs"] +mod process; +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/sqlite_inspect.rs"] +mod sqlite_inspect; +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/temp.rs"] +mod temp; + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +mod native { + use super::{llvm_module, process, sqlite_inspect, temp}; + use dever_backend_bridge::{Target, emit_object}; + use dever_core::{hir::Program, source::SourceMap}; + use std::fs; + use std::path::{Path, PathBuf}; + use std::process::{Command, Output, Stdio}; + use std::time::Duration; + + const SQLITE: &str = r#"{"database":{"default":{"type":"sqlite","path":"data/db/case.db","max_connections":2}}}"#; + const PURE_MODULE: &[(&str, &str)] = &[ + ("main.dever", "main() () {}"), + ( + "unrelated/account/model.dever", + "type Account { email: Text }", + ), + ]; + const PURE_TESTS: &[(&str, &str)] = &[ + ( + "user/account/success.dever", + r#"type Pair { left: Int + right: Text } +type Number { Present(value: Int) + Missing } +success() () { + assert(true) + assert_eq(length("@dever_entry %dever.fault"), 25) + assert_eq(Pair { left = 1 + right = "@dever_entry %dever.fault" }, Pair { left = 1 + right = "@dever_entry %dever.fault" }) + assert_eq(Number.Present(1), Number.Present(1)) + assert_eq(first([1]), first([1])) + assert_eq([1, 2], [1, 2]) + assert_eq({ "one" = 1 }, { "one" = 1 }) + dever.test.advance_clock(1000) + assert_eq(dever.test.drain_jobs(5), 0) +}"#, + ), + ( + "user/account/failure.dever", + "failure() () {\n assert_eq(1, 2)\n}", + ), + ( + "user/account/boolean.dever", + "boolean() () {\n assert(false)\n}", + ), + ( + "user/account/business.dever", + "type Outcome { error Denied(message: Text) }\nbusiness() () { fail(Outcome.Denied(\"owned failure payload\")) }", + ), + ]; + + const JOB_MODULE: &[(&str, &str)] = &[ + ("main.dever", "main() () {}"), + ( + "content/article/model.dever", + "type Article { title: Text }", + ), + ( + "content/article/app.dever", + r#"type Publish { title: Text + article_id: model.id + tags: List<Text?> } +type DeliveryError { error Failed(message: Text) } +transaction submit(title: Text) (id: Id) { + article = model.create({ title = title }) + id = dever.job.enqueue(job.publish, Publish { title = title + article_id = article.id + tags = ["source", null] }, title) +} +transaction later(title: Text) (id: Id) { + article = model.create({ title = title }) + id = dever.job.enqueue_at(job.publish, Publish { title = title + article_id = article.id + tags = ["source", null] }, title, dever.time.parse_datetime("1970-01-01T00:00:01Z")) +} +transaction abort() () { + id = submit("rolled-back") + fail(DeliveryError.Failed("abort the whole transaction")) +} +apply(input: Publish) () { + original = model.get(input.article_id) + marker = model.create({ title = "started" }) + article = model.create({ title = port.title(original.title) }) +} +count(title: Text) (total: Int) { total = model.count({ where = title == title }) } +arm() (id: Id) { id = dever.job.enqueue(job.cleanup, "manual") } +clean() () { article = model.create({ title = "cron" }) } +"#, + ), + ( + "content/article/job.dever", + "database default\njob publish(input: app.Publish) () retry(2) timeout(1000) { app.apply(input) }\njob cleanup() () retry(2) timeout(1000) { app.clean() }\nschedule cleanup = \"* * * * *\"", + ), + ( + "content/article/port.dever", + "title(value: Text) (title: Text) fails app.DeliveryError", + ), + ( + "content/article/adapter.dever", + "external exec \"worker/must-not-start\" {}", + ), + ]; + + const JOB_TESTS: &[(&str, &str)] = &[ + ( + "content/article/empty.dever", + "empty() () { dever.test.advance_clock(1000)\n assert_eq(dever.test.drain_jobs(5), 0) }", + ), + ( + "content/article/first.dever", + r#"port.title(value: Text) (title: Text) { title = "first" } +first() () { + assert_eq(app.count("first"), 0) + first = app.submit("same-key") + duplicate = app.submit("same-key") + assert_eq(first, duplicate) + assert_eq(dever.test.drain_jobs(5), 1) + assert_eq(app.count("first"), 1) + assert_eq(app.count("started"), 1) + dever.test.advance_clock(1000) + assert_eq(dever.test.drain_jobs(5), 0) +}"#, + ), + ( + "content/article/second.dever", + r#"port.title(value: Text) (title: Text) { title = "second" } +second() () { + assert_eq(app.count("first"), 0) + id = app.submit("same-key") + assert_eq(dever.test.drain_jobs(5), 1) + assert_eq(app.count("second"), 1) +}"#, + ), + ( + "content/article/delayed.dever", + r#"port.title(value: Text) (title: Text) { title = "delivered" } +delayed() () { + id = app.later("later") + assert_eq(dever.test.drain_jobs(5), 0) + dever.test.advance_clock(999) + assert_eq(dever.test.drain_jobs(5), 0) + dever.test.advance_clock(1) + assert_eq(dever.test.drain_jobs(5), 1) + assert_eq(app.count("delivered"), 1) +}"#, + ), + ( + "content/article/retry.dever", + r#"port.title(value: Text) (title: Text) { fail(app.DeliveryError.Failed("retry")) } +retry() () { + id = app.submit("retry-key") + assert_eq(dever.test.drain_jobs(5), 1) + assert_eq(app.count("started"), 0) + assert_eq(dever.test.drain_jobs(5), 0) + dever.test.advance_clock(1000) + assert_eq(dever.test.drain_jobs(5), 1) + assert_eq(app.count("started"), 0) + dever.test.advance_clock(10000) + assert_eq(dever.test.drain_jobs(5), 0) +}"#, + ), + ( + "content/article/scheduled.dever", + r#"scheduled() () { + id = app.arm() + assert_eq(dever.test.drain_jobs(5), 1) + assert_eq(app.count("cron"), 1) + dever.test.advance_clock(60000) + assert_eq(dever.test.drain_jobs(5), 1) + assert_eq(app.count("cron"), 2) + assert_eq(dever.test.drain_jobs(5), 0) +}"#, + ), + ( + "content/article/abort.dever", + "port.title(value: Text) (title: Text) { title = value }\nabort() () { app.abort() }", + ), + ]; + + fn sources( + module: &[(&str, &str)], + tests: &[(&str, &str)], + ) -> (temp::TemporaryDirectory, SourceMap, Program) { + let directory = temp::TemporaryDirectory::new(); + for (root, files) in [("module", module), ("test", tests)] { + fs::create_dir(directory.path().join(root)).unwrap(); + for (path, text) in files { + let path = directory.path().join(root).join(path); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, text).unwrap(); + } + } + let sources = SourceMap::load_project( + &directory.path().join("module"), + &directory.path().join("test"), + ) + .unwrap(); + let program = dever_core::check(&sources).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }); + (directory, sources, program) + } + + struct Suite { + _directory: temp::TemporaryDirectory, + executable: PathBuf, + cases: Vec<(String, bool)>, + } + + impl Suite { + fn compile(module: &[(&str, &str)], tests: &[(&str, &str)], iterations: usize) -> Self { + let (directory, sources, program) = sources(module, tests); + let ir = dever_core::llvm::emit_test_suite(&program, &sources).unwrap(); + let executable = llvm_module::link_with_driver( + &directory, + &ir, + "one compiled application test suite", + iterations, + "test-suite-driver.c", + ); + let cases = program + .tests() + .iter() + .map(|case| (case.name().to_owned(), case.uses_database())) + .collect(); + Self { + _directory: directory, + executable, + cases, + } + } + + fn stage(&self, setting: Option<&str>) -> temp::TemporaryDirectory { + let directory = temp::TemporaryDirectory::new(); + fs::copy(&self.executable, directory.path().join("suite")).unwrap(); + if let Some(setting) = setting { + fs::create_dir(directory.path().join("config")).unwrap(); + fs::write(directory.path().join("config/setting.json"), setting).unwrap(); + } + directory + } + + fn run(&self, directory: &Path, name: &str, expected_status: i32) -> Output { + let index = self + .cases + .iter() + .position(|(case, _)| case == name) + .expect("checked test identity"); + let stdout = directory.join("stdout"); + let stderr = directory.join("stderr"); + let status = process::status( + Command::new(directory.join("suite")) + .env_clear() + .args([index.to_string(), expected_status.to_string()]) + .stdin(Stdio::null()) + .stdout(fs::File::create(&stdout).unwrap()) + .stderr(fs::File::create(&stderr).unwrap()), + Duration::from_secs(20), + ) + .unwrap(); + let output = Output { + status, + stdout: fs::read(stdout).unwrap(), + stderr: fs::read(stderr).unwrap(), + }; + assert!( + output.status.success(), + "{name}: {status}\n{}", + String::from_utf8_lossy(&output.stderr) + ); + output + } + } + + #[test] + fn single_suite_keeps_sorted_cases_and_emits_all_six_targets() { + let (_directory, sources, program) = sources(PURE_MODULE, PURE_TESTS); + assert_eq!( + program + .tests() + .iter() + .map(|case| case.name()) + .collect::<Vec<_>>(), + [ + "user/account/boolean", + "user/account/business", + "user/account/failure", + "user/account/success" + ] + ); + assert!(program.tests().iter().all(|case| !case.uses_database())); + let ir = dever_core::llvm::emit_test_suite(&program, &sources).unwrap(); + for target in Target::ALL { + assert!(!emit_object(&ir, target).unwrap().is_empty()); + } + } + + #[test] + #[ignore = "requires explicit runtime archive; bounded native ownership checks"] + fn pure_suite_reports_source_failures_without_loading_deployment_settings() { + let suite = Suite::compile(PURE_MODULE, PURE_TESTS, 64); + for (case, message) in [ + ("success", None), + ( + "failure", + Some("assert_eq failed: actual = 1, expected = 2"), + ), + ("boolean", Some("assertion failed")), + ("business", Some("owned failure payload")), + ] { + // A malformed deployment setting must be completely irrelevant here. + let directory = suite.stage(Some("not deployment JSON")); + let output = suite.run( + directory.path(), + &format!("user/account/{case}"), + i32::from(message.is_some()), + ); + let stderr = String::from_utf8_lossy(&output.stderr); + if let Some(message) = message { + assert_eq!(stderr.matches(message).count(), 65, "{stderr}"); + assert!( + stderr.contains(&format!("test/user/account/{case}.dever:2:")), + "{stderr}" + ); + } else { + assert!(stderr.is_empty(), "{stderr}"); + assert!(output.stdout.is_empty()); + } + assert!(!directory.path().join("data").exists()); + } + } + + #[test] + fn job_cases_preserve_checked_effects_without_reaching_real_libs() { + let (_directory, sources, program) = sources(JOB_MODULE, JOB_TESTS); + assert_eq!( + program + .tests() + .iter() + .filter(|case| case.uses_database()) + .count(), + 6 + ); + let ir = dever_core::llvm::emit_test_suite(&program, &sources).unwrap(); + for target in Target::ALL { + assert!(!emit_object(&ir, target).unwrap().is_empty()); + } + } + + fn rows(directory: &Path, sql: &'static str) -> Vec<Vec<dever_runtime::orm::Value>> { + sqlite_inspect::query(&directory.join("data/db/case.db"), sql) + } + + #[test] + #[ignore = "requires explicit runtime archive; isolated SQLite and no production Worker"] + fn jobs_use_case_fakes_clock_retry_cron_and_atomic_transactions() { + use dever_runtime::orm::Value; + let suite = Suite::compile(JOB_MODULE, JOB_TESTS, 0); + for (case, database) in &suite.cases { + let directory = suite.stage(database.then_some(SQLITE)); + let failed = case.ends_with("/abort"); + let output = suite.run(directory.path(), case, i32::from(failed)); + if !database { + assert!(!directory.path().join("data").exists()); + continue; + } + if failed { + assert!( + String::from_utf8_lossy(&output.stderr).contains("abort the whole transaction") + ); + assert_eq!( + rows(directory.path(), "SELECT COUNT(*) FROM _dever_jobs"), + vec![vec![Value::Int(0)]] + ); + assert_eq!( + rows(directory.path(), "SELECT COUNT(*) FROM article"), + vec![vec![Value::Int(0)]] + ); + } else if case.ends_with("/retry") { + assert_eq!( + rows( + directory.path(), + "SELECT state,attempt,error FROM _dever_jobs" + ), + vec![vec![ + Value::Text("dead".into()), + Value::Int(2), + Value::Text("handler_failed".into()) + ]] + ); + } else { + assert!( + rows( + directory.path(), + "SELECT id FROM _dever_jobs WHERE state!='succeeded'" + ) + .is_empty() + ); + } + } + } + + #[test] + #[ignore = "requires explicit runtime archive; case-local Port effects and no external Worker"] + fn suite_keeps_blocking_and_async_fakes_separate() { + let module = [ + ("main.dever", "main() () {}"), + ( + "notification/mail/app.dever", + "read() (value: Int) { value = port.read() }", + ), + ( + "notification/mail/port.dever", + "read() (value: Int) fails dever.time.SleepResult", + ), + ( + "notification/mail/adapter.dever", + "external exec \"worker/must-not-start\" {}", + ), + ]; + let tests = [ + ( + "notification/mail/blocking.dever", + "blocking() () { dever.time.sleep(1)\n assert_eq(app.read(), 11) }\nport.read() (value: Int) { dever.time.sleep(1)\n value = 11 }", + ), + ( + "notification/mail/suspending.dever", + "suspending() () { assert_eq(app.read(), 23) }\nport.read() (value: Int) { dever.task.sleep(1)\n value = 23 }", + ), + ]; + let suite = Suite::compile(&module, &tests, 64); + for (case, database) in &suite.cases { + assert!(!database); + let directory = suite.stage(Some("must not load production config")); + assert!(suite.run(directory.path(), case, 0).stderr.is_empty()); + } + } + + #[test] + #[ignore = "requires explicit runtime archive; inspect only this suite's persisted queue"] + fn job_dispatch_blocks_incompatible_schema_payload_and_policy() { + use dever_runtime::orm::Value; + let tests = [ + ( + "content/article/prepare.dever", + "prepare() () { id = app.submit(\"queued\") }\nport.title(value: Text) (title: Text) { title = value }", + ), + ( + "content/article/drain.dever", + "drain() () { id = app.submit(\"queued\")\n assert_eq(dever.test.drain_jobs(5), 1)\n assert_eq(app.count(\"started\"), 0) }\nport.title(value: Text) (title: Text) { title = value }", + ), + ]; + let suite = Suite::compile(JOB_MODULE, &tests, 0); + for (mutation, expected) in [ + ( + "UPDATE _dever_jobs SET schema='old-version'", + "schema_mismatch", + ), + ("UPDATE _dever_jobs SET payload='{}'", "invalid_payload"), + ( + "UPDATE _dever_jobs SET timeout_ms=3600001", + "invalid_policy", + ), + ] { + let directory = suite.stage(Some(SQLITE)); + suite.run(directory.path(), "content/article/prepare", 0); + sqlite_inspect::execute(&directory.path().join("data/db/case.db"), mutation); + suite.run(directory.path(), "content/article/drain", 0); + assert_eq!( + rows( + directory.path(), + "SELECT state,error,attempt FROM _dever_jobs" + ), + [vec![ + Value::Text("blocked".into()), + Value::Text(expected.into()), + Value::Int(1) + ]] + ); + } + } + + #[test] + #[ignore = "requires explicit runtime archive; CMD enters System scope without running worker"] + fn command_enqueues_system_jobs_and_releases_repeated_application_roots() { + let mut module = JOB_MODULE.to_vec(); + module + .iter_mut() + .find(|(path, _)| *path == "content/article/adapter.dever") + .unwrap() + .1 = "port.title(value: Text) (title: Text) { title = value }"; + module.push(("content/article/api.dever", "cmd submit = app.submit")); + let (_directory, sources, program) = sources(&module, &[]); + let mut ir = dever_core::llvm::emit_application(&program, &sources).unwrap(); + ir.push_str("\ndefine i32 @dever_test_run() {\nentry:\n %out = alloca i8\n %fault = alloca %dever.fault\n %status = call i32 @dever_application_entry(ptr %out, ptr %fault)\n call void @dever_outputs_release(ptr %out)\n call void @dever_fault_release(ptr %fault)\n ret i32 %status\n}\n"); + let setting = r#"{"database":{"default":{"type":"sqlite","path":"data/db/case.db","max_connections":1}}}"#; + let output = llvm_module::execute( + &ir, + "System CMD enqueue", + 64, + Duration::from_secs(20), + &["content.article.submit", r#"{"title":"queued"}"#], + Some(setting), + ); + let lines = String::from_utf8(output.stdout).unwrap(); + let replies = lines + .lines() + .map(|line| serde_json::from_str::<serde_json::Value>(line).unwrap()) + .collect::<Vec<_>>(); + assert_eq!(replies.len(), 65); + assert!( + replies + .iter() + .all(|reply| reply["code"] == 0 && reply["data"] == replies[0]["data"]) + ); + } + + #[test] + #[ignore = "requires explicit runtime archive; invalid worker policy must fail before schema I/O"] + fn invalid_worker_policy_fails_before_creating_database() { + let module = [ + ("content/article/app.dever", "clean() () {}"), + ( + "content/article/job.dever", + "database default\njob cleanup() () retry(2) timeout(1000) { app.clean() }", + ), + ]; + let (_source_dir, sources, program) = sources(&module, &[]); + let mut ir = dever_core::llvm::emit_application(&program, &sources).unwrap(); + let check = llvm_module::fault_check(3, None); + ir.push_str(&format!("\ndefine i32 @dever_test_run() {{\nentry:\n %out = alloca i8\n %fault = alloca %dever.fault\n %status = call i32 @dever_application_entry(ptr %out, ptr %fault)\n{check}\n call void @dever_outputs_release(ptr %out)\n call void @dever_fault_release(ptr %fault)\n %exit = select i1 %passed, i32 0, i32 1\n ret i32 %exit\n}}\n")); + let directory = temp::TemporaryDirectory::new(); + fs::create_dir(directory.path().join("config")).unwrap(); + fs::write(directory.path().join("config/setting.json"), r#"{"database":{"default":{"type":"sqlite","path":"data/db/case.db","max_connections":1}},"runtime":{"mode":"worker","shutdown_ms":1000},"job":{"workers":1,"poll_ms":10,"lease_ms":1000,"retry_base_ms":10,"retry_max_ms":100}}"#).unwrap(); + llvm_module::execute_in( + &directory, + &ir, + "worker lease preflight", + 64, + Duration::from_secs(10), + &[], + ); + assert!(!directory.path().join("data/db/case.db").exists()); + } + + #[test] + #[ignore = "requires explicit runtime archive; bounded task cancellation inside owned SQLite transactions"] + fn timed_out_job_drains_children_and_rolls_back_before_retry() { + use dever_runtime::orm::Value; + let module = [ + ("main.dever", "main() () {}"), + ( + "content/article/model.dever", + "type Article { title: Text }", + ), + ( + "content/article/app.dever", + "submit() (id: Id) { id = dever.job.enqueue(job.publish, \"timeout\") }\napply() () { saved = model.create({ title = \"rollback\" })\n port.pause() }\ncount() (total: Int) { total = model.count() }", + ), + ( + "content/article/port.dever", + "pause() () fails dever.time.SleepResult", + ), + ("content/article/adapter.dever", "port.pause() () {}"), + ( + "content/article/job.dever", + "database default\njob publish() () retry(2) timeout(10) { app.apply() }", + ), + ]; + let tests = [( + "content/article/timeout.dever", + r#"port.pause() () { dever.task.sleep(5000) } +timeout() () { + id = app.submit() + assert_eq(dever.test.drain_jobs(5), 1) + assert_eq(app.count(), 0) + dever.test.advance_clock(1000) + assert_eq(dever.test.drain_jobs(5), 1) + assert_eq(app.count(), 0) + dever.test.advance_clock(10000) + assert_eq(dever.test.drain_jobs(5), 0) +}"#, + )]; + let suite = Suite::compile(&module, &tests, 0); + let directory = suite.stage(Some(SQLITE)); + suite.run(directory.path(), "content/article/timeout", 0); + assert_eq!( + rows( + directory.path(), + "SELECT state,error,attempt FROM _dever_jobs" + ), + [vec![ + Value::Text("dead".into()), + Value::Text("handler_timeout".into()), + Value::Int(2) + ]] + ); + } +} diff --git a/test/llvm_lowering.rs b/test/llvm_lowering.rs new file mode 100644 index 0000000..82d7e2d --- /dev/null +++ b/test/llvm_lowering.rs @@ -0,0 +1,166 @@ +use dever_core::source::SourceMap; + +#[path = "dever-tests/tests/support/llvm.rs"] +mod llvm; +use llvm::lower; + +#[test] +fn typed_clauses_records_choices_and_handlers_lower_to_direct_functions() { + let ir = lower( + r#"type Pair { left: Int +right: Int } +type Selection { Found(value: Int) +Missing } +increment(value: Int) (answer: Int) { answer = value + 1 } +apply(route: handler(value: Int) (answer: Int), value: Int) (answer: Int) { answer = route(value) } +choose(value: Selection.Found(payload)) (answer: Int) { answer = payload } +choose(value: Selection.Missing) (answer: Int) { answer = -1 } +optional(value: Int) (answer: Int?) { answer = value } +public main() (sum: Int, selected: Int, maybe: Int?, ready: Bool) { + original = Pair { left = 4 +right = 5 } + changed = original + changed.left = apply(increment, original.left) + sum = changed.left + original.left + selected = choose(Selection.Found(changed.right)) + maybe = optional(7) + ready = false and (increment(9223372036854775807) == 0) or true +} +"#, + ); + assert!(ir.contains("define i32 @dever_entry(ptr %out, ptr %fault)")); + assert!(ir.contains("store { i64, i64, { i1, i64 }, i1 } zeroinitializer, ptr %out")); + assert!(ir.contains("%dever.fault = type { i32, i32, i32, [")); + assert!(ir.contains("@dever_fault_capacity = constant i32")); + assert!(ir.contains("@dever_source_spans = constant")); + assert!(ir.contains("@llvm.sadd.with.overflow.i64")); + assert!(ir.contains("phi i1")); + assert!(ir.contains("getelementptr %dever.fault, ptr %fault, i32 0, i32 3")); + assert!(ir.contains("%T")); + assert!(!ir.contains("dever_runtime")); +} + +#[test] +fn stdout_writes_lower_through_the_typed_system_abi() { + let ir = lower("public main() () { dever.io.println(\"hello\") }\n"); + assert!(ir.contains("call i32 @dever_rt_v1_stdout_write(")); +} + +#[test] +fn unsupported_reachable_effect_reports_original_source_location() { + let mut sources = SourceMap::default(); + sources.add( + "user/account/app.dever", + "identity() (value: Text) { value = dever.auth.id() }\n", + ); + let program = dever_core::check(&sources).unwrap(); + let error = + dever_core::llvm::emit_kernel(&program, &sources, "user.account.identity").unwrap_err(); + let (_, location) = error.split_once(".dever:").expect("Dever source path"); + let mut position = location.split(':'); + assert!(position.next().unwrap().parse::<usize>().is_ok(), "{error}"); + assert!(position.next().unwrap().parse::<usize>().is_ok(), "{error}"); + assert!(error.contains("does not support"), "{error}"); +} + +#[test] +fn dynamic_integer_faults_have_origin_and_caller_frame_storage() { + let ir = lower( + r#"divide(left: Int, right: Int) (answer: Int) { answer = left // right } +outer(value: Int) (answer: Int) { answer = divide(value, 0) } +public main() (answer: Int) { answer = outer(7) } +"#, + ); + assert!(ir.contains("@dever_fault_capacity = constant i32 4")); + assert!(ir.contains("store i32 2, ptr")); + assert!(ir.matches("i32 0, i32 3, i32 %v").count() >= 2); + assert!(ir.contains("@dever_source_span_count = constant i32 3")); +} + +#[test] +fn temporary_nominal_values_are_emitted_even_without_nominal_local_slots() { + let ir = lower( + r#"type Pair { left: Int +right: Int } +public main() (equal: Bool) { + equal = Pair { left = 1 +right = 2 } == Pair { left = 1 +right = 2 } +} +"#, + ); + assert!(ir.contains("%T0 = type { i64, i64 }")); + assert!(ir.contains("extractvalue %T0")); +} + +#[test] +fn nullable_collection_rows_keep_their_full_storage_before_flattening() { + let ir = lower( + r#"present(value: Int?) (answer: Bool) { answer = value != null } +first_optional(values: List<Text?>) (answer: Text?) { answer = first(values) } +lookup(values: Map<Text, Text?>) (answer: Text?) { answer = get(values, "key") } +find_optional(values: List<Int?>) (answer: Int?) { answer = find(present, values) } +public main() (first_value: Text?, map_value: Text?, found: Int?) { + first_value = first_optional([null, "next"]) + map_value = lookup({ "key" = null }) + found = find_optional([null, 1]) +} +"#, + ); + assert!(ir.contains("@dever_rt_v1_list_first_take")); + assert!(ir.contains("@dever_rt_v1_map_get_take")); + assert!(ir.contains("@dever_rt_v1_list_cursor_next")); + assert!(ir.matches("store { i1, ptr } %v").count() >= 2); + assert!(ir.contains("store { i1, i64 } %v")); +} + +#[test] +fn business_failures_and_result_capture_keep_typed_payloads() { + let ir = lower( + r#"public type Failure { error Missing(message: Text) +error Conflict(message: Text) } +public type Outcome { Done(value: Int) +error Failed(error: Failure) } +public type Empty { Done +error Missing(message: Text) } +public type Pair { Done(left: Int, right: Int) +error Missing(message: Text) } +load(value: 0) (answer: Int) { fail(Failure.Missing("absent")) } +load(value: 1) (answer: Int) { fail(Failure.Conflict("duplicate")) } +load(value: other) (answer: Int) { answer = value } +stop() () { fail(Empty.Missing("stopped")) } +two(value: 0) (left: Int, right: Int) { fail(Pair.Missing("missing")) } +two(value: other) (left: Int, right: Int) { left = 1 +right = 2 } +public main() (outcome: Outcome, empty: Empty, pair: Pair) { + outcome = result(load(0)) + empty = result(stop()) + pair = result(two(1)) +} +"#, + ); + assert!(ir.contains("i32, i32, [")); + assert!(ir.contains("@dever_fault_release")); + assert!(ir.contains("drop_failure_")); + assert!(ir.contains("capture_match")); + assert!(ir.contains("capture_propagate")); + assert!(ir.contains("store %dever.fault zeroinitializer, ptr %fault")); +} + +#[test] +fn bytes_results_distinguish_business_errors_from_abi_faults() { + let ir = lower( + r#"public main() (decoded: dever.bytes.DecodeResult, sliced: dever.bytes.BufferResult, from_ints: dever.bytes.BufferResult) { + bytes = dever.system.bytes_from_text("abc") + decoded = dever.system.bytes_to_text(bytes) + sliced = dever.system.bytes_slice(bytes, 0, 2) + from_ints = dever.system.bytes_from_ints([65, 66]) +} +"#, + ); + assert!(ir.contains("@dever_rt_v1_bytes_to_text_handle")); + assert!(ir.contains("@dever_rt_v1_bytes_slice")); + assert!(ir.contains("@dever_rt_v1_bytes_from_ints")); + assert!(ir.matches("icmp eq i32 %v").count() >= 6); + assert!(ir.contains("@dever_rt_v1_buffer_free")); +} diff --git a/test/llvm_managed_native.rs b/test/llvm_managed_native.rs new file mode 100644 index 0000000..606fce8 --- /dev/null +++ b/test/llvm_managed_native.rs @@ -0,0 +1,266 @@ +//! Bounded native acceptance for the typed, managed-value LLVM kernel subset. +//! These checks use an explicit test archive, never a host-language fallback. + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/llvm.rs"] +mod llvm; +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/llvm_managed.rs"] +mod llvm_managed; +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/process.rs"] +mod process; +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/temp.rs"] +mod temp; + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +mod native { + use super::{llvm, llvm_managed}; + use dever_backend_bridge::{Target, emit_object}; + use llvm_managed::{boolean, execute, runtime_fault}; + + #[test] + fn managed_nested_values_emit_real_objects_on_all_six_targets() { + let ir = llvm::lower( + r#"type Bundle { + ready: Bool + label: Text + bytes: Bytes + costs: List<Decimal> + lookup: Map<Text, List<Decimal>> +} +type Envelope { Ready(value: Bundle) +Missing } +optional(value: Bundle) (answer: Envelope?) { answer = Envelope.Ready(value) } +public main() (answer: Bool) { + costs = [1.25, 2.5] + bundle = Bundle { ready = true +label = "typed" +bytes = dever.bytes.from_text("bytes") +costs = costs +lookup = { "costs" = costs } } + answer = optional(bundle) == optional(bundle) +} +"#, + ); + for target in Target::ALL { + assert!(emit_object(&ir, target).unwrap().len() > 128); + } + } + + #[test] + #[ignore = "build runtime-abi-only staticlib in target/native-runtime-abi before running"] + fn unicode_text_nullable_results_and_aliases_execute_and_release() { + boolean( + r#"identity(value: Text) (answer: Text) { answer = value } +public main() (answer: Bool) { + original = " A中🙂 " + alias = identity(original) + normalized = text.lower(text.trim(alias)) + replaced = text.replace(normalized, "a", "b") + parts = text.split("中🙂", "") + answer = original == " A中🙂 " and alias == original and normalized == "a中🙂" and replaced == "b中🙂" and text.upper(normalized) == "A中🙂" and text.at(normalized, 1) == maybe_text("中") and text.slice(normalized, 1, 3) == maybe_text("中🙂") and text.slice(normalized, 3, 1) == null and text.index_of(normalized, "🙂") == maybe_int(2) and text.index_of(normalized, "z") == null and text.contains(normalized, "中") and text.starts_with(normalized, "a") and text.ends_with(normalized, "🙂") and text.codepoint("🙂") == maybe_int(128578) and text.codepoint("ab") == null and text.from_codepoint(128578) == maybe_text("🙂") and text.from_codepoint(-1) == null and parts == ["中", "🙂"] and length(normalized) == 3 and "a" + "中" == "a中" +} +"#, + ); + } + + #[test] + #[ignore = "build runtime-abi-only staticlib in target/native-runtime-abi before running"] + fn decimal_numbers_parsing_rendering_and_ieee_intrinsics_execute() { + boolean( + r#"public main() (answer: Bool) { + ratio = 7.0 / decimal.from_int(2) + rounded = decimal.round(2.345, 2) + nan = 0.0e0 / 0.0e0 + infinity = 1.0e0 / 0.0e0 + answer = ratio == 3.5 and ratio > 3.0 and ratio <= 3.5 and -ratio == -3.5 and 1.5 + 2.0 == 3.5 and 5.5 - 2.0 == 3.5 and 1.75 * 2.0 == 3.5 and rounded == 2.34 and decimal.parse("3.5") == maybe_decimal(ratio) and decimal.parse("invalid") == null and decimal.to_text(ratio) == "3.5" and int.parse("42") == maybe_int(42) and int.parse("invalid") == null and int.to_text(-42) == "-42" and float.parse("3.5") == maybe_float(3.5e0) and float.parse("invalid") == null and float.to_text(3.5e0) == "3.5" and float.from_int(2) == 2.0e0 and math.sqrt(4.0e0) == 2.0e0 and math.sin(0.0e0) == 0.0e0 and math.cos(0.0e0) == 1.0e0 and math.log(1.0e0) == 0.0e0 and math.pow(2.0e0, 3.0e0) == 8.0e0 and math.is_nan(nan) and math.is_infinite(infinity) and math.is_finite(3.5e0) and not (nan == nan) +} +"#, + ); + } + + #[test] + #[ignore = "build runtime-abi-only staticlib in target/native-runtime-abi before running"] + fn nested_records_choices_nullable_and_field_overwrites_keep_value_semantics() { + boolean( + r#"type Profile { + name: Text + aliases: List<Text> +} +type Selection { + Found(value: Profile) + Missing +} +read(value: Selection.Found(profile)) (answer: Profile) { answer = profile } +read(value: Selection.Missing) (answer: Profile) { answer = Profile { name = "missing" +aliases = [] } } +read(value: null) (answer: Profile) { answer = Profile { name = "absent" +aliases = [] } } +optional(value: Selection) (answer: Selection?) { answer = value } +public main() (answer: Bool) { + original = Profile { name = "first" +aliases = ["a", "b"] } + changed = original + changed.name = "second" + changed.aliases = append(changed.aliases, changed.name) + wrapped = optional(Selection.Found(changed)) + selected = read(wrapped) + answer = original.name == "first" and original.aliases == ["a", "b"] and changed.name == "second" and selected.aliases == ["a", "b", "second"] and read(null).name == "absent" and wrapped != optional(Selection.Missing) +} +"#, + ); + } + + #[test] + #[ignore = "build runtime-abi-only staticlib in target/native-runtime-abi before running"] + fn list_map_cow_order_and_supported_map_keys_execute() { + boolean( + r#"type Category { First +Second } +key(value: MapEntry<Text, List<Text>>) (answer: Text) { answer = value.key } +first_name(values: List<Text>) (answer: Text?) { answer = first(values) } +public main() (answer: Bool) { + names = ["a", "b"] + extended = append(names, "c") + original = { "first" = names +"second" = extended } + changed = put(original, "first", ["new"]) + removed = remove(changed, "second") + reordered = { "second" = extended +"first" = names } + flags = { true = "yes" +false = "no" } + counts = { 7 = "seven" +9 = "nine" } + categories = { Category.First = names +Category.Second = extended } + identifier = dever.id.from_text("stable") + identities = { identifier = names } + nan = 0.0e0 / 0.0e0 + unordered = [nan] + unordered_map = { "value" = nan } + answer = names == ["a", "b"] and extended == ["a", "b", "c"] and get(original, "first") == maybe_list(names) and get(changed, "first") == maybe_list(["new"]) and get(removed, "second") == null and length(removed) == 1 and original != reordered and each(key, entries(original)) == ["first", "second"] and get(flags, false) == maybe_text("no") and get(counts, 7) == maybe_text("seven") and get(categories, Category.Second) == maybe_list(extended) and first(extended) == maybe_text("a") and first_name([]) == null and get(identities, dever.id.from_text("stable")) == maybe_list(names) and dever.id.to_text(identifier) == "stable" and unordered != unordered and unordered_map != unordered_map +} +"#, + ); + } + + #[test] + #[ignore = "build runtime-abi-only staticlib in target/native-runtime-abi before running"] + fn static_handler_traversals_and_bytes_execute_without_interpreting_values() { + boolean( + r#"add(value: Int, total: Int) (next: Int) { next = total + value } +until(value: Int, total: Int) (next: Int, stop: Bool) { next = total + value +stop = value == 2 } +increment(value: Int) (answer: Int) { answer = value + 1 } +even(value: Int) (answer: Bool) { answer = value % 2 == 0 } +label(value: Text, prefix: Text) (answer: Text) { answer = prefix + value } +forward(route: handler(value: Int) (answer: Int), values: List<Int>) (answer: List<Int>) { answer = each(route, values) } +total(values: List<Int>) (answer: Int) { answer = sum(values) } +public main() (answer: Bool) { + bytes = dever.bytes.from_text("ABC") + joined = dever.bytes.concat(bytes, dever.bytes.from_text("D")) + labels = each(label, ["a", "b"], "prefix=") + answer = reduce(add, [1, 2, 3], 0) == 6 and reduce(add, [], 7) == 7 and reduce_until(until, [1, 2, 100], 0) == 3 and sum([1, 2, 3]) == 6 and sum(increment, [1, 2, 3]) == 9 and total([]) == 0 and filter(even, [1, 2, 3, 4]) == [2, 4] and find(even, [1, 2, 3]) == maybe_int(2) and find(even, [1, 3]) == null and labels == ["prefix=a", "prefix=b"] and forward(increment, [1, 2]) == [2, 3] and reduce(add, bytes, 0) == 198 and each(increment, bytes) == [66, 67, 68] and dever.bytes.length(bytes) == 3 and dever.bytes.length(joined) == 4 and dever.bytes.at(joined, 3) == maybe_int(68) and dever.bytes.at(joined, -1) == null and bytes == dever.bytes.from_text("ABC") +} +"#, + ); + } + + #[test] + #[ignore = "build runtime-abi-only staticlib in target/native-runtime-abi before running"] + fn nullable_elements_flatten_and_unit_handlers_and_owned_reduce_state_release() { + boolean( + r#"present(value: Int) (answer: Bool) { answer = true } +present(value: null) (answer: Bool) { answer = false } +nonempty(value: Text) (answer: Bool) { answer = true } +nonempty(value: null) (answer: Bool) { answer = false } +first_optional(values: List<Text?>) (answer: Text?) { answer = first(values) } +ignore(value: Text) () {} +collect(value: Text, state: List<Text>) (next: List<Text>) { next = append(state, value) } +until_name(value: Text, state: List<Text>) (next: List<Text>, stop: Bool) { + next = append(state, value) + stop = value == "b" +} +public main() (answer: Bool) { + each(ignore, ["discarded", "also-discarded"]) + collected = reduce(collect, ["a", "b"], []) + stopped = reduce_until(until_name, ["a", "b", "unvisited"], []) + labels = { "present" = maybe_text("value") +"absent" = null } + answer = first_optional([null, "later"]) == null and first_optional(["first", null]) == maybe_text("first") and find(present, [null, 1, 2]) == maybe_int(1) and find(nonempty, [null, "found"]) == maybe_text("found") and get(labels, "present") == maybe_text("value") and get(labels, "absent") == null and get(labels, "missing") == null and collected == ["a", "b"] and stopped == ["a", "b"] and sum([1.25, 2.5]) == 3.75 and sum([1.0e16, 1.0e0, -1.0e16]) == 0.0e0 +} +"#, + ); + } + + #[test] + #[ignore = "build runtime-abi-only staticlib in target/native-runtime-abi before running"] + fn traversal_failure_cleans_managed_arguments_results_and_error_paths() { + execute( + r#"fault(value: Text) (answer: Text) { + prepared = [value, value + "alias"] + zero = length(prepared) - 2 + answer = int.to_text(1 // zero) +} +public main() (answer: Bool) { + mapped = each(fault, ["a", "b"]) + answer = length(mapped) == 2 +} +"#, + "{ i1 }", + " %failed = icmp eq i32 %status, 1 + %code_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 0 + %code = load i32, ptr %code_ptr + %division = icmp eq i32 %code, 2 + %depth_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 2 + %depth = load i32, ptr %depth_ptr + %called = icmp uge i32 %depth, 1 + %correct = and i1 %failed, %division + %passed = and i1 %correct, %called", + ); + } + + #[test] + #[ignore = "build runtime-abi-only staticlib in target/native-runtime-abi before running"] + fn dynamic_map_duplicate_key_preserves_runtime_diagnostic_and_releases_it() { + runtime_fault( + r#"identity(value: Text) (answer: Text) { answer = value } +public main() (answer: Bool) { + key = identity("same") + values = { key = ["first"] +identity(key) = ["second"] } + answer = length(values) == 1 +} +"#, + "duplicate Map key", + ); + } + + #[test] + #[ignore = "build runtime-abi-only staticlib in target/native-runtime-abi before running"] + fn root_owned_outputs_are_transferred_and_released_by_the_caller() { + execute( + r#"public main() (label: Text, bytes: Bytes, names: List<Text>) { + label = "root-owned" + bytes = dever.bytes.from_text(label) + names = [label, label] +} +"#, + "{ ptr, ptr, ptr }", + " %okay = icmp eq i32 %status, 0 + %values = load { ptr, ptr, ptr }, ptr %out + %label = extractvalue { ptr, ptr, ptr } %values, 0 + %bytes = extractvalue { ptr, ptr, ptr } %values, 1 + %names = extractvalue { ptr, ptr, ptr } %values, 2 + %label_ok = icmp ne ptr %label, null + %bytes_ok = icmp ne ptr %bytes, null + %names_ok = icmp ne ptr %names, null + %first_ok = and i1 %label_ok, %bytes_ok + %second_ok = and i1 %first_ok, %names_ok + %passed = and i1 %okay, %second_ok", + ); + } +} diff --git a/test/llvm_native.rs b/test/llvm_native.rs new file mode 100644 index 0000000..497f909 --- /dev/null +++ b/test/llvm_native.rs @@ -0,0 +1,308 @@ +use dever_backend_bridge::{LinkKind, Target, emit_object, link}; +use std::fs; + +#[path = "dever-tests/tests/support/llvm.rs"] +mod llvm; +use llvm::lower; + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/process.rs"] +mod process; +#[path = "dever-tests/tests/support/temp.rs"] +mod temp; + +const AGGREGATE_SOURCE: &str = r#"type Pair { + enabled: Bool + amount: Float + count: Int +} +type Payload { + Value(value: Pair) + Empty +} +type Envelope { + Present(value: Payload) + Missing +} +increment(value: Int) (answer: Int) { answer = value + 1 } +apply(route: handler(value: Int) (answer: Int), value: Int) (answer: Int) { answer = route(value) } +score(value: true) (answer: Int) { answer = 1 } +score(value: false) (answer: Int) { answer = 0 } +read(value: Payload.Value(payload)) (answer: Int) { answer = payload.count + score(payload.enabled and payload.amount == 3.5e0) } +read(value: Payload.Empty) (answer: Int) { answer = 0 } +unwrap(value: Envelope.Present(payload)) (answer: Int) { answer = read(payload) } +unwrap(value: Envelope.Missing) (answer: Int) { answer = -1 } +optional(value: Int) (answer: Int?) { answer = value } +absent() (answer: Int?) { answer = null } +public main() (answer: Int) { + pair = Pair { + count = apply(increment, 40) + enabled = true + amount = 3.5e0 + } + first = Payload.Value(pair) + second = Payload.Value(pair) + equal = first == second and first != Payload.Empty and optional(1) == optional(1) and absent() == absent() and optional(1) != absent() + answer = unwrap(Envelope.Present(first)) + score(equal) +} +"#; + +#[test] +fn checked_dever_functions_emit_and_link_on_all_six_targets() { + let source = r#"double(value: Int) (answer: Int) { answer = value * 2 } +public main() (answer: Int) { answer = double(21) } +"#; + let directory = temp::TemporaryDirectory::new(); + let ir = lower(source); + for (index, target) in Target::ALL.into_iter().enumerate() { + let object = directory.path().join(format!("{index}.o")); + fs::write(&object, emit_object(&ir, target).unwrap()).unwrap(); + let output = directory.path().join(format!("{index}.image")); + link(target, LinkKind::Shared, &[object], &output, None).unwrap(); + assert!(fs::metadata(output).unwrap().len() > 128); + } +} + +#[test] +fn checked_aggregate_functions_emit_real_objects_on_all_six_targets() { + let ir = lower(AGGREGATE_SOURCE); + for target in Target::ALL { + assert!(emit_object(&ir, target).unwrap().len() > 128); + } +} + +#[test] +fn windows_aggregate_link_requires_target_runtime_support() { + // LLVM emits the real Windows stack-probe and floating-point CRT references. + // A signed target runtime pack must provide them; no fixture stubs replace it. + let directory = temp::TemporaryDirectory::new(); + let object = directory.path().join("aggregate.obj"); + fs::write( + &object, + emit_object(&lower(AGGREGATE_SOURCE), Target::WindowsX86_64).unwrap(), + ) + .unwrap(); + let output = directory.path().join("aggregate.dll"); + let error = link( + Target::WindowsX86_64, + LinkKind::Shared, + &[object], + &output, + None, + ) + .unwrap_err(); + assert!(error.contains("__chkstk"), "{error}"); + assert!(error.contains("_fltused"), "{error}"); + assert!(!output.exists()); +} + +// The driver is freestanding and test-owned. Foreign targets above are linked, +// not executed; the executable checks below only apply to this native host. +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +mod native { + use super::*; + use std::process::{Command, Stdio}; + use std::time::Duration; + + fn execute(source: &str, output_type: &str, checks: &str) { + let mut ir = lower(source); + ir.push_str(&format!( + "\ndefine void @_start() noreturn {{\nentry:\n %out = alloca {output_type}\n %fault = alloca %dever.fault\n %status = call i32 @dever_entry(ptr %out, ptr %fault)\n{checks}\n %exit = select i1 %passed, i64 0, i64 1\n call void asm sideeffect \"syscall\", \"{{rax}},{{rdi}},~{{rcx}},~{{r11}},~{{memory}}\"(i64 60, i64 %exit)\n unreachable\n}}\n" + )); + let directory = temp::TemporaryDirectory::new(); + let object = directory.path().join("entry.o"); + fs::write(&object, emit_object(&ir, Target::LinuxX86_64).unwrap()).unwrap(); + let output = directory.path().join("program"); + link( + Target::LinuxX86_64, + LinkKind::Executable, + &[object], + &output, + Some("_start"), + ) + .unwrap(); + let status = process::status( + Command::new(output) + .env_clear() + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()), + Duration::from_secs(5), + ) + .unwrap(); + assert!(status.success(), "native kernel failed: {status}\n{source}"); + } + + fn integer(source: &str, expected: i64) { + execute( + source, + "{ i64 }", + &format!( + " %okay = icmp eq i32 %status, 0\n br i1 %okay, label %success, label %failed\nsuccess:\n %actual = load i64, ptr %out\n %correct = icmp eq i64 %actual, {expected}\n br label %done\nfailed:\n br label %done\ndone:\n %passed = phi i1 [ %correct, %success ], [ false, %failed ]" + ), + ); + } + + #[test] + fn clauses_nullable_choices_and_record_values_execute_without_rustc() { + integer( + r#"type Selection { + Found(value: Int) + Missing +} +type Pair { + first: Int + second: Int +} +choose(value: Selection.Found(payload)) (answer: Int) { answer = payload } +choose(value: Selection.Missing) (answer: Int) { answer = -1 } +choose(value: null) (answer: Int) { answer = -2 } +public main() (answer: Int) { + original = Pair { + second = 2 + first = 1 + } + changed = original + changed.first = 99 + answer = original.first + changed.first + choose(Selection.Found(7)) + choose(Selection.Missing) + choose(null) +} +"#, + 104, + ); + } + + #[test] + fn short_circuit_does_not_execute_a_faulting_right_operand() { + integer( + r#"truth(value: Bool) (answer: Bool) { answer = value } +probe(left: Int, right: Int) (answer: Bool) { answer = left // right > 0 } +score(value: true) (answer: Int) { answer = 42 } +score(value: false) (answer: Int) { answer = 0 } +public main() (answer: Int) { answer = score(truth(false) and probe(1, 0) or truth(true)) } +"#, + 42, + ); + } + + #[test] + fn static_handlers_and_nested_aligned_choice_payloads_execute() { + integer(AGGREGATE_SOURCE, 43); + } + + #[test] + fn multiple_outputs_preserve_order_and_ieee_float_semantics() { + execute( + r#"divide(left: Float, right: Float) (answer: Float) { answer = left / right } +public main() (first: Int, second: Float, equal: Bool) { + first = 7 + second = divide(6.0e0, 2.0e0) + nan = divide(0.0e0, 0.0e0) + equal = nan == nan +} +"#, + "{ i64, double, i1 }", + " %okay = icmp eq i32 %status, 0 + br i1 %okay, label %success, label %failed +success: + %first_ptr = getelementptr { i64, double, i1 }, ptr %out, i32 0, i32 0 + %second_ptr = getelementptr { i64, double, i1 }, ptr %out, i32 0, i32 1 + %equal_ptr = getelementptr { i64, double, i1 }, ptr %out, i32 0, i32 2 + %first = load i64, ptr %first_ptr + %second = load double, ptr %second_ptr + %equal = load i1, ptr %equal_ptr + %first_ok = icmp eq i64 %first, 7 + %second_ok = fcmp oeq double %second, 3.0 + %nan_ok = xor i1 %equal, true + %values_ok = and i1 %first_ok, %second_ok + %correct = and i1 %values_ok, %nan_ok + br label %done +failed: + br label %done +done: + %passed = phi i1 [ %correct, %success ], [ false, %failed ]", + ); + } + + #[test] + fn integer_overflow_and_division_faults_return_checked_statuses() { + for operation in [ + "left + right", + "left - right", + "left * right", + "left // right", + "left % right", + ] { + let (left, right, fault) = match operation { + "left + right" => (i64::MAX, 1, 1), + "left - right" => (i64::MIN, 1, 1), + "left * right" => (i64::MAX, 2, 1), + _ => (1, 0, 2), + }; + let source = format!( + "calculate(left: Int, right: Int) (answer: Int) {{ answer = {operation} }}\npublic main() (answer: Int) {{ answer = calculate({left}, {right}) }}" + ); + execute( + &source, + "{ i64 }", + &format!( + " %failed = icmp eq i32 %status, 1\n %code_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 0\n %code = load i32, ptr %code_ptr\n %correct = icmp eq i32 %code, {fault}\n %passed = and i1 %failed, %correct" + ), + ); + } + for (operation, parameters, arguments) in [ + ("-left", "left: Int", i64::MIN.to_string()), + ( + "left // right", + "left: Int, right: Int", + format!("{}, -1", i64::MIN), + ), + ( + "left % right", + "left: Int, right: Int", + format!("{}, -1", i64::MIN), + ), + ] { + let source = format!( + "calculate({parameters}) (answer: Int) {{ answer = {operation} }}\npublic main() (answer: Int) {{ answer = calculate({arguments}) }}" + ); + execute( + &source, + "{ i64 }", + " %failed = icmp eq i32 %status, 1\n %code_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 0\n %code = load i32, ptr %code_ptr\n %correct = icmp eq i32 %code, 1\n %passed = and i1 %failed, %correct", + ); + } + } + + #[test] + fn numeric_faults_retain_origin_and_every_caller_source_span() { + let source = "divide(left: Int, right: Int) (answer: Int) { answer = left // right }\nwrap(left: Int, right: Int) (answer: Int) { answer = divide(left, right) }\npublic main() (answer: Int) { answer = wrap(1, 0) }"; + let origin = source.find("left // right").unwrap(); + let inner_call = source.find("divide(left, right)").unwrap(); + let outer_call = source.find("wrap(1, 0)").unwrap(); + execute(source, "{ i64 }", &format!( + " %failed = icmp eq i32 %status, 1 + %origin_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 1 + %origin_index = load i32, ptr %origin_ptr + %origin_span = getelementptr {{ i64, i64, i64 }}, ptr @dever_source_spans, i32 %origin_index, i32 1 + %origin = load i64, ptr %origin_span + %origin_ok = icmp eq i64 %origin, {origin} + %depth_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 2 + %depth = load i32, ptr %depth_ptr + %depth_ok = icmp eq i32 %depth, 2 + %frame0_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 3, i32 0 + %frame1_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 3, i32 1 + %frame0 = load i32, ptr %frame0_ptr + %frame1 = load i32, ptr %frame1_ptr + %span0 = getelementptr {{ i64, i64, i64 }}, ptr @dever_source_spans, i32 %frame0, i32 1 + %span1 = getelementptr {{ i64, i64, i64 }}, ptr @dever_source_spans, i32 %frame1, i32 1 + %start0 = load i64, ptr %span0 + %start1 = load i64, ptr %span1 + %inner_ok = icmp eq i64 %start0, {inner_call} + %outer_ok = icmp eq i64 %start1, {outer_call} + %chain_ok = and i1 %inner_ok, %outer_ok + %location_ok = and i1 %origin_ok, %depth_ok + %fault_ok = and i1 %failed, %location_ok + %passed = and i1 %fault_ok, %chain_ok" + )); + } +} diff --git a/test/llvm_process_entry_source.rs b/test/llvm_process_entry_source.rs new file mode 100644 index 0000000..8d212f7 --- /dev/null +++ b/test/llvm_process_entry_source.rs @@ -0,0 +1,107 @@ +//! Checked process-entry lowering; real execution belongs to the CLI regression. +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/temp.rs"] +mod temp; + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +mod native { + use super::temp; + use dever_backend_bridge::{Target, emit_object}; + use dever_core::{hir::Program, llvm, source::SourceMap}; + use std::fs; + + const APPLICATION: &[(&str, &str)] = &[ + ( + "sample/echo/app.dever", + r#"type Outcome { error Denied(message: Text, labels: List<Text?>) } +echo(value: Text) (answer: Text) { + dever.task.sleep(1) + answer = value +} +reject() (answer: Text) { fail(Outcome.Denied("denied 你好", ["first", null])) } +divide(divisor: Int) (answer: Int) { answer = 42 // divisor } +"#, + ), + ( + "sample/echo/api.dever", + "cmd echo = app.echo\ncmd reject = app.reject\ncmd divide = app.divide", + ), + ]; + const TESTS: &[(&str, &str)] = &[ + ( + "sample/echo/pass.dever", + "pass() () { assert_eq(app.echo(\"@dever_entry %dever.fault\"), \"@dever_entry %dever.fault\") }", + ), + ( + "sample/echo/assertion.dever", + "assertion() () {\n assert_eq([\"actual\"], [\"expected\"])\n}", + ), + ( + "sample/echo/reject.dever", + "reject() () { value = app.reject() }", + ), + ]; + + fn checked(tests: Option<&[(&str, &str)]>) -> (temp::TemporaryDirectory, SourceMap, Program) { + let directory = temp::TemporaryDirectory::new(); + for (root, files) in [("module", APPLICATION), ("test", tests.unwrap_or(&[]))] { + fs::create_dir(directory.path().join(root)).unwrap(); + for (name, contents) in files { + let path = directory.path().join(root).join(name); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, contents).unwrap(); + } + } + let sources = SourceMap::load_project( + &directory.path().join("module"), + &directory.path().join("test"), + ) + .unwrap(); + let program = dever_core::check(&sources).unwrap_or_else(|errors| { + panic!( + "{}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }); + (directory, sources, program) + } + + #[test] + fn executable_entries_emit_objects_for_all_six_targets() { + let (_directory, sources, program) = checked(Some(TESTS)); + for ir in [ + llvm::emit_executable(&program, &sources).unwrap(), + llvm::emit_test_executable(&program, &sources).unwrap(), + ] { + for target in Target::ALL { + assert!(!emit_object(&ir, target).unwrap().is_empty()); + } + } + } + + #[test] + fn zero_cases_keep_the_existing_compile_boundary() { + let (_directory, sources, program) = checked(None); + assert!(program.tests().is_empty()); + assert_eq!( + llvm::emit_test_executable(&program, &sources).unwrap_err(), + "cannot compile an empty test suite" + ); + } + + #[test] + fn callable_entries_do_not_acquire_a_process_main() { + let (_directory, sources, program) = checked(Some(TESTS)); + for ir in [ + llvm::emit_application(&program, &sources).unwrap(), + llvm::emit_test_suite(&program, &sources).unwrap(), + ] { + assert!(!ir.contains("define i32 @main(")); + assert!(!ir.contains("@dever_rt_v1_process_test_index(")); + assert!(!ir.contains("@dever_rt_v1_process_init(")); + } + } +} diff --git a/test/llvm_resource_native.rs b/test/llvm_resource_native.rs new file mode 100644 index 0000000..082d268 --- /dev/null +++ b/test/llvm_resource_native.rs @@ -0,0 +1,305 @@ +//! Checked-source resource acceptance using owned temporary files only. + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/llvm.rs"] +mod llvm; +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/llvm_managed.rs"] +mod llvm_managed; +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/process.rs"] +mod process; +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/temp.rs"] +mod temp; + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +mod native { + use super::{llvm, llvm_managed, process, temp}; + use std::fs; + use std::path::Path; + use std::process::{Command, Stdio}; + use std::time::Duration; + + const HELPERS: &str = r#"read_text(value: dever.io.ReadState.Read(bytes)) (answer: Text) { answer = dever.bytes.to_text(bytes) } +read_text(value: dever.io.ReadState.End) (answer: Text) { answer = "EOF" } +closed(value: dever.io.CloseResult.Closed, wanted: Bool) (answer: Bool) recover("test explicitly checks close results") { answer = wanted } +closed(value: dever.io.CloseResult.Failed(message), wanted: Bool) (answer: Bool) recover("test explicitly checks close results") { answer = not wanted and message == "resource is closed" } +read_failed(value: dever.io.ReadResult.Done(state), expected: Text) (answer: Bool) recover("test checks failure identity and message") { answer = false } +read_failed(value: dever.io.ReadResult.Failed(message), expected: Text) (answer: Bool) recover("test checks failure identity and message") { answer = message == expected } +take_one(value: dever.io.ReadEvent.Chunk(bytes), previous: Text) (answer: Text, stop: Bool) recover("test records exactly one stream event") { answer = dever.bytes.to_text(bytes) +stop = true } +take_one(value: dever.io.ReadEvent.Failed(message), previous: Text) (answer: Text, stop: Bool) recover("test records exactly one stream event") { answer = message +stop = true } +next(stream: Stream<dever.io.ReadEvent>) (answer: Text) recover("test records exactly one stream event") { answer = reduce_until(take_one, stream, "EOF") } +count(value: dever.io.ReadEvent.Chunk(bytes), total: Int) (answer: Int) recover("test observes stream terminal errors") { answer = total + dever.bytes.length(bytes) } +count(value: dever.io.ReadEvent.Failed(message), total: Int) (answer: Int) recover("test observes stream terminal errors") { answer = -1 } +finish(value: dever.io.ReadEvent.Chunk(bytes), total: Int) (answer: Int, stop: Bool) recover("test observes stream terminal errors") { answer = total + dever.bytes.length(bytes) +stop = true } +finish(value: dever.io.ReadEvent.Failed(message), total: Int) (answer: Int, stop: Bool) recover("test observes stream terminal errors") { answer = -1 +stop = true } +"#; + + fn source(body: &str, path: &Path) -> String { + format!( + "{HELPERS}\n{}", + body.replace("FIXTURE_PATH", &format!("{:?}", path.to_str().unwrap())) + ) + } + + fn fixture() -> (temp::TemporaryDirectory, std::path::PathBuf) { + let directory = temp::TemporaryDirectory::new(); + let path = directory.path().join("owned.txt"); + fs::write(&path, "abcdef").unwrap(); + (directory, path) + } + + #[test] + fn file_and_stream_lowering_uses_runtime_owners_and_original_choices() { + let ir = llvm::lower(&source( + r#"public main() (answer: Bool) { +file = dever.io.open(FIXTURE_PATH) +stream = dever.io.chunks(file, 2) +first_chunk = next(stream) +remaining = reduce(count, stream, 0) +close(stream) +answer = first_chunk == "ab" and remaining == 4 +}"#, + Path::new("owned-test-file"), + )); + assert!(ir.contains("@dever_rt_v1_file_open")); + assert!(ir.contains("@dever_rt_v1_file_chunks")); + assert!(ir.contains("@dever_rt_v1_stream_pull")); + assert!(ir.contains("@dever_rt_v1_stream_release")); + assert!(ir.contains("@dever_read_Chunk_")); + assert!(ir.contains("@dever_read_Failed_")); + assert!(!ir.contains("dever_runtime")); + for target in dever_backend_bridge::Target::ALL { + assert!( + !dever_backend_bridge::emit_object(&ir, target) + .unwrap() + .is_empty() + ); + } + } + + #[test] + #[ignore = "requires the explicitly prepared runtime-abi archive"] + fn read_eof_and_alias_close_preserve_business_error_identity() { + let (_directory, path) = fixture(); + llvm_managed::boolean(&source( + r#"public main() (answer: Bool) { +file = dever.io.open(FIXTURE_PATH) +alias = file +first_read = read_text(dever.io.read(file, 2)) +remaining = read_text(dever.io.read(alias, 8)) +eof = read_text(dever.io.read(file, 2)) +bad_limit = read_failed(result(dever.io.read(file, 0)), "read size must be a positive Int") +first_close = closed(result(dever.io.close(file)), true) +second_close = closed(result(dever.io.close(alias)), false) +closed_read = read_failed(result(dever.io.read(alias, 1)), "resource is closed") +answer = first_read == "ab" and remaining == "cdef" and eof == "EOF" and bad_limit and first_close and second_close and closed_read +}"#, + &path, + )); + } + + #[test] + #[ignore = "requires the explicitly prepared runtime-abi archive"] + fn stream_aliases_share_progress_and_early_close() { + let (_directory, path) = fixture(); + llvm_managed::boolean(&source( + r#"public main() (answer: Bool) { +file = dever.io.open(FIXTURE_PATH) +stream = dever.io.chunks(file, 2) +alias = stream +first_chunk = next(stream) +second_chunk = next(alias) +close(stream) +close(alias) +ended = next(alias) +file_still_open = read_text(dever.io.read(file, 2)) +answer = first_chunk == "ab" and second_chunk == "cd" and ended == "EOF" and file_still_open == "ef" +}"#, + &path, + )); + } + + #[test] + #[ignore = "requires the explicitly prepared runtime-abi archive"] + fn terminal_read_failure_is_one_event_then_end() { + let (_directory, path) = fixture(); + llvm_managed::boolean(&source( + r#"public main() (answer: Bool) { +file = dever.io.open(FIXTURE_PATH) +stream = dever.io.chunks(file, 2) +dever.io.close(file) +failed = next(stream) +ended = next(stream) +answer = failed == "resource is closed" and ended == "EOF" +}"#, + &path, + )); + } + + #[test] + #[ignore = "requires the explicitly prepared runtime-abi archive"] + fn reduce_until_retains_the_shared_unvisited_cursor() { + let (_directory, path) = fixture(); + llvm_managed::boolean(&source( + r#"public main() (answer: Bool) { +file = dever.io.open(FIXTURE_PATH) +stream = dever.io.chunks(file, 2) +alias = stream +total = reduce_until(finish, stream, 0) +next_chunk = next(alias) +remaining = reduce(count, alias, 0) +answer = total == 2 and next_chunk == "cd" and remaining == 2 +}"#, + &path, + )); + } + + #[test] + #[ignore = "requires the explicitly prepared runtime-abi archive"] + fn owned_file_output_and_stream_release_do_not_leak() { + let (_directory, path) = fixture(); + llvm_managed::execute( + &source( + r#"public main() (file: dever.system.File, answer: Bool) { +file = dever.io.open(FIXTURE_PATH) +stream = dever.io.chunks(file, 2) +answer = next(stream) == "ab" +}"#, + &path, + ), + "{ ptr, i1 }", + " %okay = icmp eq i32 %status, 0\n %row = load { ptr, i1 }, ptr %out\n %file = extractvalue { ptr, i1 } %row, 0\n %answer = extractvalue { ptr, i1 } %row, 1\n %present = icmp ne ptr %file, null\n %valid = and i1 %okay, %present\n %passed = and i1 %valid, %answer", + ); + } + + #[test] + #[ignore = "requires the explicitly prepared runtime-abi archive"] + fn create_new_and_write_never_overwrite_existing_content() { + let directory = temp::TemporaryDirectory::new(); + let path = directory.path().join("exclusive.txt"); + assert!(!path.exists()); + llvm_managed::boolean(&source( + r#"created(value: dever.io.OpenResult.Opened(file)) (answer: Bool) recover("test checks create_new on repeated calls") { +dever.io.write(file, dever.bytes.from_text("written")) +dever.io.close(file) +answer = not_created(result(dever.io.create(FIXTURE_PATH))) +} +created(value: dever.io.OpenResult.Failed(message)) (answer: Bool) recover("test checks create_new on repeated calls") { answer = dever.system.text_contains(message, "File exists") } +not_created(value: dever.io.OpenResult.Opened(file)) (answer: Bool) recover("test rejects a second successful creation") { +dever.io.write(file, dever.bytes.from_text("unexpected overwrite")) +dever.io.close(file) +answer = false +} +not_created(value: dever.io.OpenResult.Failed(message)) (answer: Bool) recover("test rejects a second successful creation") { answer = dever.system.text_contains(message, "File exists") } +public main() (answer: Bool) { +created_once = created(result(dever.io.create(FIXTURE_PATH))) +file = dever.io.open(FIXTURE_PATH) +answer = created_once and read_text(dever.io.read(file, 64)) == "written" +}"#, + &path, + )); + assert_eq!(fs::read(&path).unwrap(), b"written"); + } + + #[test] + #[ignore = "requires the explicitly prepared runtime-abi archive"] + fn runtime_fault_unwinds_owned_file_and_partly_read_stream() { + let (_directory, path) = fixture(); + llvm_managed::runtime_fault( + &source( + r#"identity(value: Text) (answer: Text) { answer = value } +duplicate(file: dever.system.File) (answer: Int) { +stream = dever.io.chunks(file, 2) +observed = next(stream) +key = identity("same") +keys = { key = 1 +identity(key) = 2 } +answer = length(keys) + length(observed) +} +public main() (answer: Int) { +file = dever.io.open(FIXTURE_PATH) +answer = duplicate(file) +}"#, + &path, + ), + "duplicate Map key", + ); + } + + #[test] + #[ignore = "requires the explicitly prepared runtime-abi archive"] + fn each_propagates_owned_business_failure_and_releases_its_cursor() { + let (_directory, path) = fixture(); + llvm_managed::boolean(&source( + r#"type TraversalResult { +Done +error Failed(message: Text, history: List<Text>) +} +stream_value(value: dever.io.ReadStreamResult.Streaming(stream)) (answer: Stream<dever.io.ReadEvent>) { answer = stream } +stream_value(value: dever.io.ReadStreamResult.Failed(message)) (answer: Stream<dever.io.ReadEvent>) { fail(TraversalResult.Failed(message, ["terminal error"])) } +close_value(value: dever.io.CloseResult.Closed) () {} +close_value(value: dever.io.CloseResult.Failed(message)) () { fail(TraversalResult.Failed(message, ["terminal error"])) } +consume(value: dever.io.ReadEvent.Chunk(bytes), file: dever.system.File) () { close_value(result(dever.io.close(file))) } +consume(value: dever.io.ReadEvent.Failed(message), file: dever.system.File) () { fail(TraversalResult.Failed(message, ["terminal error"])) } +drain(file: dever.system.File) () { +stream = stream_value(result(dever.io.chunks(file, 2))) +each(consume, stream, file) +} +observed(value: TraversalResult.Done) (answer: Bool) recover("test requires the owned terminal failure") { answer = false } +observed(value: TraversalResult.Failed(message, history)) (answer: Bool) recover("test requires the owned terminal failure") { answer = message == "resource is closed" and history == ["terminal error"] } +public main() (answer: Bool) { answer = observed(result(drain(dever.io.open(FIXTURE_PATH)))) } +"#, + &path, + )); + } + + #[test] + #[ignore = "requires the explicitly prepared runtime-abi archive"] + fn resource_c_abi_rejects_invalid_outputs_before_io_or_cursor_progress() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let archive = root.join("target/native-runtime-abi/debug/libdever_backend_bridge.a"); + assert!( + archive.is_file(), + "missing explicit runtime ABI fixture archive" + ); + let directory = temp::TemporaryDirectory::new(); + let program = directory.path().join("resource-abi"); + let mut compiler = Command::new("/usr/bin/cc"); + compiler + .env_clear() + .args(["-std=c11", "-Wall", "-Wextra", "-Werror", "-O2", "-I"]) + .arg(root.join("crates/dever-backend-bridge/include")) + .arg(root.join("test/native-runtime-abi/resource-driver.c")) + .arg(archive) + .args(["-ldl", "-lpthread", "-lm", "-o"]) + .arg(&program) + .arg("-B/usr/bin/") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()); + assert!( + process::status(&mut compiler, Duration::from_secs(30)) + .unwrap() + .success() + ); + assert!( + process::status( + Command::new(program) + .arg(directory.path().join("exclusive.txt")) + .env_clear() + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()), + Duration::from_secs(5) + ) + .unwrap() + .success() + ); + } +} diff --git a/test/llvm_result_native.rs b/test/llvm_result_native.rs new file mode 100644 index 0000000..0abec77 --- /dev/null +++ b/test/llvm_result_native.rs @@ -0,0 +1,388 @@ +//! Real checked-source, LLVM-object and runtime-archive result acceptance. + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/llvm.rs"] +mod llvm; +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/llvm_managed.rs"] +mod llvm_managed; +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/process.rs"] +mod process; +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/temp.rs"] +mod temp; + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +mod native { + use super::{llvm, llvm_managed}; + + const SINGLE: &str = r#"type Outcome { + Done(value: Text) + error Failed(message: Text) +} +step(value: 0) (answer: Text) { fail(Outcome.Failed("specific failure")) } +step(value: 1) (answer: Text) { fail(Outcome.Failed("second failure")) } +step(value: other) (answer: Text) { answer = "successful value" } +inspect(value: Outcome.Done(text), expected: Text, wanted_success: Bool) (answer: Bool) recover("test handles failures") { answer = wanted_success and text == expected } +inspect(value: Outcome.Failed(message), expected: Text, wanted_success: Bool) (answer: Bool) recover("test handles failures") { answer = not wanted_success and message == expected } +public main() (answer: Bool) { + first = inspect(result(step(0)), "specific failure", false) + second = inspect(result(step(7)), "successful value", true) + third = inspect(result(step(1)), "second failure", false) + answer = first and second and third +} +"#; + + const ARITY: &str = r#"type EmptyResult { + Done + error Failed(message: Text) +} +type PairResult { + Many(label: Text, names: List<Text>) + error Failed(message: Text) +} +empty(bad: true) () { fail(EmptyResult.Failed("empty failed")) } +empty(bad: false) () {} +pair(bad: true) (label: Text, names: List<Text>) { fail(PairResult.Failed("pair failed")) } +pair(bad: false) (label: Text, names: List<Text>) { label = "pair label" +names = ["one", "two"] } +read_empty(value: EmptyResult.Done, wanted_success: Bool) (answer: Bool) recover("test handles failures") { answer = wanted_success } +read_empty(value: EmptyResult.Failed(message), wanted_success: Bool) (answer: Bool) recover("test handles failures") { answer = not wanted_success and message == "empty failed" } +read_pair(value: PairResult.Many(label, names), wanted_success: Bool) (answer: Bool) recover("test handles failures") { answer = wanted_success and label == "pair label" and names == ["one", "two"] } +read_pair(value: PairResult.Failed(message), wanted_success: Bool) (answer: Bool) recover("test handles failures") { answer = not wanted_success and message == "pair failed" } +public main() (answer: Bool) { + answer = read_empty(result(empty(false)), true) and read_empty(result(empty(true)), false) and read_pair(result(pair(false)), true) and read_pair(result(pair(true)), false) +} +"#; + + const WRAPPED: &str = r#"type Failure { + error Missing(message: Text) + error Conflict(message: Text) +} +type Outcome { + Done(value: Int) + error Failed(error: Failure) +} +load(value: 0) (answer: Int) { fail(Failure.Missing("missing")) } +load(value: 1) (answer: Int) { fail(Failure.Conflict("conflict")) } +load(value: other) (answer: Int) { answer = value } +rethrow(value: Outcome.Done(number)) (answer: Int) { answer = number } +rethrow(value: Outcome.Failed(error)) (answer: Int) { fail(Outcome.Failed(error)) } +inspect(value: Outcome.Done(number), wanted: Text) (answer: Bool) recover("test handles failures") { answer = wanted == "done" and number == 7 } +inspect(value: Outcome.Failed(error), wanted: Text) (answer: Bool) recover("test handles failures") { answer = inspect_error(error, wanted) } +inspect_error(value: Failure.Missing(message), wanted: Text) (answer: Bool) recover("test handles failures") { answer = wanted == "missing" and message == "missing" } +inspect_error(value: Failure.Conflict(message), wanted: Text) (answer: Bool) recover("test handles failures") { answer = wanted == "conflict" and message == "conflict" } +public main() (answer: Bool) { + answer = inspect(result(load(0)), "missing") and inspect(result(load(1)), "conflict") and inspect(result(rethrow(result(load(0)))), "missing") and inspect(result(load(7)), "done") +} +"#; + + const HANDLERS: &str = r#"type Detail { + label: Text + history: List<Text> +} +type Outcome { + Done(value: List<Text>) + error Failed(detail: Detail) +} +type State { + prefix: Text + names: List<Text> +} +map_name(value: Text, prefix: Text) (answer: Text) { + label = prefix + value + fail_if(value, label) + answer = label +} +fail_if(value: "stop", label: Text) () { + fail(Outcome.Failed(Detail { label = label +history = ["before", label] })) +} +fail_if(value: other, label: Text) () {} +collect(value: Text, state: State) (next: State) { + label = state.prefix + value + fail_if(value, label) + next = State { prefix = state.prefix +names = append(state.names, label) } +} +mapped(bad: true) (value: List<Text>) { value = each(map_name, ["first", "stop", "unvisited"], "ctx=") } +mapped(bad: false) (value: List<Text>) { value = each(map_name, ["first", "last"], "ctx=") } +reduced(bad: true) (value: List<Text>) { value = reduce(collect, ["first", "stop", "unvisited"], State { prefix = "ctx=" +names = ["seed"] }).names } +reduced(bad: false) (value: List<Text>) { value = reduce(collect, ["first", "last"], State { prefix = "ctx=" +names = ["seed"] }).names } +inspect(value: Outcome.Done(items), expected: List<Text>) (answer: Bool) recover("test handles failures") { answer = items == expected } +inspect(value: Outcome.Failed(detail), expected: List<Text>) (answer: Bool) recover("test handles failures") { answer = detail.label == "ctx=stop" and detail.history == ["before", "ctx=stop"] and length(expected) == 0 } +public main() (answer: Bool) { + first = inspect(result(mapped(true)), []) + second = inspect(result(mapped(false)), ["ctx=first", "ctx=last"]) + third = inspect(result(reduced(true)), []) + fourth = inspect(result(reduced(false)), ["seed", "ctx=first", "ctx=last"]) + fifth = inspect(result(mapped(true)), []) + answer = first and second and third and fourth and fifth +} +"#; + + const ROOT_FAILURE: &str = r#"type Detail { + names: List<Text> + lookup: Map<Text, List<Text>> +} +type Outcome { + error Other(message: Text) + error Failed(detail: Detail) +} +origin() () { + names = ["owned", "payload"] + detail = Detail { names = names +lookup = { "key" = names } } + fail(Outcome.Failed(detail)) +} +middle() () { origin() } +public main() () { middle() } +"#; + + const BYTES_OK: &str = r#"decoded(value: dever.bytes.DecodeResult.Decoded(text)) (answer: Text) recover("test handles decode failures") { answer = text } +decoded(value: dever.bytes.DecodeResult.Failed(message)) (answer: Text) recover("test handles decode failures") { answer = message } +ready(value: dever.bytes.BufferResult.Ready(bytes)) (answer: Bytes) recover("test handles byte failures") { answer = bytes } +ready(value: dever.bytes.BufferResult.Failed(message)) (answer: Bytes) recover("test handles byte failures") { answer = dever.bytes.from_text(message) } +public main() (answer: Bool) { + original = dever.bytes.from_text("中🙂") + alias = original + numeric = ready(result(dever.bytes.from_ints([228, 184, 173, 240, 159, 153, 130]))) + trimmed = ready(result(dever.bytes.slice(numeric, 3, 7))) + empty = ready(result(dever.bytes.from_ints([]))) + answer = decoded(result(dever.bytes.to_text(numeric))) == "中🙂" and decoded(result(dever.bytes.to_text(trimmed))) == "🙂" and decoded(result(dever.bytes.to_text(empty))) == "" and dever.bytes.length(empty) == 0 and original == alias and original == numeric and dever.bytes.length(trimmed) == 4 and dever.bytes.to_text(original) == "中🙂" +} +"#; + + const BYTES_BAD: &str = r#"buffer_error(value: dever.bytes.BufferResult.Ready(bytes)) (answer: Text) recover("test handles byte failures") { answer = "unexpected success" } +buffer_error(value: dever.bytes.BufferResult.Failed(message)) (answer: Text) recover("test handles byte failures") { answer = message } +decode_error(value: dever.bytes.DecodeResult.Decoded(text)) (answer: Text) recover("test handles decode failures") { answer = "unexpected success" } +decode_error(value: dever.bytes.DecodeResult.Failed(message)) (answer: Text) recover("test handles decode failures") { answer = message } +public main() (answer: Bool) { + bytes = dever.bytes.from_text("abc") + invalid = dever.bytes.from_text("x") + answer = buffer_error(result(dever.bytes.from_ints([-1]))) == "byte value must be between 0 and 255" and buffer_error(result(dever.bytes.from_ints([256]))) == "byte value must be between 0 and 255" and buffer_error(result(dever.bytes.slice(bytes, -1, 2))) == "byte range is out of bounds" and buffer_error(result(dever.bytes.slice(bytes, 0, 4))) == "byte range is out of bounds" and buffer_error(result(dever.bytes.slice(bytes, 2, 1))) == "byte range is out of bounds" and decode_error(result(dever.bytes.to_text(dever.bytes.from_ints([255])))) == "invalid utf-8 sequence of 1 bytes from index 0" and invalid == dever.bytes.from_text("x") +} +"#; + + const NESTED_OWNED: &str = r#"type Detail { + names: List<Text> + lookup: Map<Text, List<Text>> +} +type Outcome { + Done(value: Text) + error Failed(detail: Detail) +} +fail_owned() (answer: Text) { + names = ["first", "second"] + fail(Outcome.Failed(Detail { names = names +lookup = { "key" = names } })) +} +inspect(value: Outcome.Done(text)) (answer: Bool) recover("test handles failures") { answer = text == "none" } +inspect(value: Outcome.Failed(detail)) (answer: Bool) recover("test handles failures") { + alias = detail + alias.names = append(alias.names, "third") + alias.lookup = put(alias.lookup, "key", alias.names) + answer = detail.names == ["first", "second"] and get(detail.lookup, "key") == optional_names(["first", "second"]) and alias.names == ["first", "second", "third"] and get(alias.lookup, "key") == optional_names(["first", "second", "third"]) +} +optional_names(value: List<Text>) (answer: List<Text>?) { answer = value } +public main() (answer: Bool) { answer = inspect(result(fail_owned())) } +"#; + + const ROOT_SUCCESS: &str = r#"type Outcome { + Done(value: Text) + error Failed(message: Text) +} +step(bad: true) (value: Text) { fail(Outcome.Failed("unexpected")) } +step(bad: false) (value: Text) { value = "root-owned" } +unwrap(value: Outcome.Done(text)) (answer: Text) recover("test handles failures") { answer = text } +unwrap(value: Outcome.Failed(message)) (answer: Text) recover("test handles failures") { answer = message } +public main() (answer: Text) { answer = unwrap(result(step(false))) } +"#; + + #[test] + fn result_fixtures_pass_the_real_checker() { + for source in [ + SINGLE, + ARITY, + WRAPPED, + HANDLERS, + ROOT_FAILURE, + BYTES_OK, + BYTES_BAD, + NESTED_OWNED, + ROOT_SUCCESS, + ] { + llvm::lower(source); + } + } + + #[test] + #[ignore = "requires explicit target/native-runtime-abi staticlib"] + fn direct_failure_and_single_success_capture() { + llvm_managed::boolean(SINGLE); + } + + #[test] + #[ignore = "requires explicit target/native-runtime-abi staticlib"] + fn zero_and_multiple_outputs_capture_in_order() { + llvm_managed::boolean(ARITY); + } + + #[test] + #[ignore = "requires explicit target/native-runtime-abi staticlib"] + fn wrapped_and_explicit_outer_failures_keep_nominal_identity() { + llvm_managed::boolean(WRAPPED); + } + + #[test] + #[ignore = "requires explicit target/native-runtime-abi staticlib"] + fn static_handler_failures_release_cursors_context_and_state() { + llvm_managed::boolean(HANDLERS); + } + + #[test] + #[ignore = "requires explicit target/native-runtime-abi staticlib"] + fn root_failure_retains_business_code_owned_payload_and_call_chain() { + let origin = ROOT_FAILURE.find("fail(Outcome.Failed(detail))").unwrap(); + let middle_call = ROOT_FAILURE.rfind("origin()").unwrap(); + let root_call = ROOT_FAILURE.rfind("middle()").unwrap(); + llvm_managed::execute( + ROOT_FAILURE, + "{}", + &format!( + " %failed = icmp eq i32 %status, 1 + %code_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 0 + %code = load i32, ptr %code_ptr + %business = icmp eq i32 %code, 4 + %depth_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 2 + %depth = load i32, ptr %depth_ptr + %frames = icmp eq i32 %depth, 2 + %type_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 5 + %type = load i32, ptr %type_ptr + %typed = icmp ne i32 %type, 0 + %variant_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 6 + %variant = load i32, ptr %variant_ptr + %correct_variant = icmp eq i32 %variant, 1 + %origin_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 1 + %origin_id = load i32, ptr %origin_ptr + %origin_span = getelementptr {{ i64, i64, i64 }}, ptr @dever_source_spans, i32 %origin_id + %origin_bounds = load {{ i64, i64, i64 }}, ptr %origin_span + %origin_start = extractvalue {{ i64, i64, i64 }} %origin_bounds, 1 + %origin_end = extractvalue {{ i64, i64, i64 }} %origin_bounds, 2 + %origin_start_ok = icmp eq i64 %origin_start, {origin} + %origin_end_ok = icmp eq i64 %origin_end, {} + %origin_ok = and i1 %origin_start_ok, %origin_end_ok + %frame0_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 3, i32 0 + %frame0_id = load i32, ptr %frame0_ptr + %frame0_span = getelementptr {{ i64, i64, i64 }}, ptr @dever_source_spans, i32 %frame0_id + %frame0_bounds = load {{ i64, i64, i64 }}, ptr %frame0_span + %frame0_start = extractvalue {{ i64, i64, i64 }} %frame0_bounds, 1 + %frame0_end = extractvalue {{ i64, i64, i64 }} %frame0_bounds, 2 + %frame0_start_ok = icmp eq i64 %frame0_start, {middle_call} + %frame0_end_ok = icmp eq i64 %frame0_end, {} + %frame0_ok = and i1 %frame0_start_ok, %frame0_end_ok + %frame1_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 3, i32 1 + %frame1_id = load i32, ptr %frame1_ptr + %frame1_span = getelementptr {{ i64, i64, i64 }}, ptr @dever_source_spans, i32 %frame1_id + %frame1_bounds = load {{ i64, i64, i64 }}, ptr %frame1_span + %frame1_start = extractvalue {{ i64, i64, i64 }} %frame1_bounds, 1 + %frame1_end = extractvalue {{ i64, i64, i64 }} %frame1_bounds, 2 + %frame1_start_ok = icmp eq i64 %frame1_start, {root_call} + %frame1_end_ok = icmp eq i64 %frame1_end, {} + %frame1_ok = and i1 %frame1_start_ok, %frame1_end_ok + %payload = getelementptr %dever.fault, ptr %fault, i32 0, i32 7 + %names_ptr = getelementptr i64, ptr %payload, i32 1 + %names = load ptr, ptr %names_ptr + %lookup_ptr = getelementptr i64, ptr %payload, i32 2 + %lookup = load ptr, ptr %lookup_ptr + %names_ok = icmp ne ptr %names, null + %lookup_ok = icmp ne ptr %lookup, null + %owners_ok = and i1 %names_ok, %lookup_ok + %first = and i1 %failed, %business + %second = and i1 %frames, %typed + %third = and i1 %first, %second + %fourth = and i1 %third, %correct_variant + %fifth = and i1 %origin_ok, %frame0_ok + %sixth = and i1 %fifth, %frame1_ok + %seventh = and i1 %fourth, %sixth + %passed = and i1 %seventh, %owners_ok", + origin + "fail(Outcome.Failed(detail))".len(), + middle_call + "origin()".len(), + root_call + "middle()".len() + ), + ); + } + + #[test] + #[ignore = "requires explicit target/native-runtime-abi staticlib"] + fn nested_owned_failure_payload_keeps_cow_aliases_independent() { + llvm_managed::boolean(NESTED_OWNED); + } + + #[test] + #[ignore = "requires explicit target/native-runtime-abi staticlib"] + fn captured_success_transfers_owned_root_output() { + llvm_managed::execute( + ROOT_SUCCESS, + "{ ptr }", + " %okay = icmp eq i32 %status, 0 + %answer = load ptr, ptr %out + %owned = icmp ne ptr %answer, null + %passed = and i1 %okay, %owned", + ); + } + + #[test] + #[ignore = "requires explicit target/native-runtime-abi staticlib"] + fn capture_does_not_consume_dynamic_numeric_faults() { + llvm_managed::execute( + r#"type Outcome { Done(value: Int) +error Failed(message: Text) } +divide(value: -1) (answer: Int) { fail(Outcome.Failed("business")) } +divide(value: other) (answer: Int) { answer = 9 // value } +inspect(value: Outcome.Done(number)) (answer: Bool) recover("test handles failures") { answer = number == 0 } +inspect(value: Outcome.Failed(message)) (answer: Bool) recover("test handles failures") { answer = message == "none" } +public main() (answer: Bool) { denominator = length(["one"]) - 1 +answer = inspect(result(divide(denominator))) }"#, + "{ i1 }", + " %failed = icmp eq i32 %status, 1 + %code_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 0 + %code = load i32, ptr %code_ptr + %division = icmp eq i32 %code, 2 + %passed = and i1 %failed, %division", + ); + } + + #[test] + #[ignore = "requires explicit target/native-runtime-abi staticlib"] + fn capture_does_not_consume_original_runtime_abi_faults() { + llvm_managed::runtime_fault( + r#"type Outcome { Done(value: Map<Text, List<Text>>) +error Failed(message: Text) } +identity(value: Text) (answer: Text) { answer = value } +build(bad: true) (answer: Map<Text, List<Text>>) { fail(Outcome.Failed("business")) } +build(bad: false) (answer: Map<Text, List<Text>>) { key = identity("same") +answer = { key = ["first"] +identity(key) = ["second"] } } +inspect(value: Outcome.Done(entries)) (answer: Bool) recover("test handles failures") { answer = length(entries) == 0 } +inspect(value: Outcome.Failed(message)) (answer: Bool) recover("test handles failures") { answer = message == "none" } +public main() (answer: Bool) { answer = inspect(result(build(false))) }"#, + "duplicate Map key", + ); + } + + #[test] + #[ignore = "requires explicit target/native-runtime-abi staticlib"] + fn bytes_results_decode_unicode_and_keep_aliases() { + llvm_managed::boolean(BYTES_OK); + } + + #[test] + #[ignore = "requires explicit target/native-runtime-abi staticlib"] + fn bytes_failures_keep_original_messages() { + llvm_managed::boolean(BYTES_BAD); + } +} diff --git a/test/llvm_stream_network_source.rs b/test/llvm_stream_network_source.rs new file mode 100644 index 0000000..f71b53a --- /dev/null +++ b/test/llvm_stream_network_source.rs @@ -0,0 +1,596 @@ +//! Checked-source stream/network/task composition, not hand-written ABI approximations. + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/llvm.rs"] +mod llvm; +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/llvm_managed.rs"] +mod llvm_managed; +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/process.rs"] +mod process; +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[path = "dever-tests/tests/support/temp.rs"] +mod temp; + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +mod native { + use super::{llvm, llvm_managed}; + use dever_backend_bridge::{Target, emit_object}; + use dever_core::{check, source::SourceMap}; + + const STREAMS: &str = r#"collect(value: Text?, previous: List<Text?>) (next: List<Text?>) { next = append(previous, value) } +add(value: Int, previous: Int) (next: Int) { next = previous + value } +one(value: Text, previous: List<Text>) (next: List<Text>, stop: Bool) { next = append(previous, value) + stop = true } +collect_text(value: Text, previous: List<Text>) (next: List<Text>) { next = append(previous, value) } +public main() (answer: Bool) { + nullable = stream([some_text("first"), null, some_text("last")]) + collected = reduce(collect, nullable, []) + bytes = stream(dever.bytes.from_text("AB")) + total = reduce(add, bytes, 0) + values = stream(["one", "two"]) + alias = values + first = reduce_until(one, values, []) + remaining = reduce(collect_text, alias, []) + close(alias) + close(values) + answer = collected == [some_text("first"), null, some_text("last")] and total == 131 and first == ["one"] and length(remaining) == 0 +} +"#; + + const CHANNEL_STREAM: &str = r#"send_values(channel: Channel<Text?>) () { + send(channel, null) + dever.task.sleep(1) + send(channel, some_text("pending")) + close(channel) +} +collect(value: Text?, previous: List<Text?>) (next: List<Text?>) { + dever.task.sleep(1) + next = append(previous, value) +} +public main() (answer: Bool) { + channel = channel(Text?, 1) + sender = run(send_values(channel)) + values = stream(channel) + alias = values + collected = reduce(collect, values, []) + wait(sender) + remaining = reduce(collect, alias, []) + answer = collected == [null, some_text("pending")] and length(remaining) == 0 +} +"#; + + const TICKS: &str = r#"tick(value: Int, count: Int) (next: Int, stop: Bool) { next = count + 1 + stop = next == 2 } +add(value: Int, previous: Int) (next: Int) { next = previous + value } +public main() (answer: Bool) { + values = dever.task.ticks(1) + alias = values + count = reduce_until(tick, values, 0) + rest = reduce(add, alias, 0) + close(values) + answer = count == 2 and rest == 0 +} +"#; + + const PARALLEL_STREAM: &str = r#"collect(value: Int, output: Channel<Int>) () { dever.task.sleep(1) + send(output, value) } +add(value: Int, previous: Int) (next: Int) { next = previous + value } +public main() (answer: Bool) { + output = channel(Int, 8) + parallel_each(collect, stream([1, 2, 3]), 2, output) + parallel_each(collect, dever.bytes.from_text("AB"), 2, output) + parallel_each(collect, [4, 5], 2, output) + close(output) + total = reduce(add, stream(output), 0) + answer = total == 146 +} +"#; + + const PARALLEL_FAILURE: &str = r#"type Outcome { Done + error Failed(message: Text, history: List<Text>) } +visit(value: 2) () { fail(Outcome.Failed("parallel original", ["owned", "worker"])) } +visit(value: other) () { ready = pause(1) } +apply() () { parallel_each(visit, stream([1, 2, 3]), 2) } +inspect(value: Outcome.Done) (answer: Bool) recover("test requires parallel failure") { answer = false } +inspect(value: Outcome.Failed(message, history)) (answer: Bool) recover("test requires parallel failure") { answer = message == "parallel original" and history == ["owned", "worker"] } +public main() (answer: Bool) { answer = inspect(result(apply())) } +"#; + + const IDLE_PARALLEL_FAILURE: &str = r#"type Outcome { Done + error Failed(message: Text, history: List<Text>) } +visit(value: Int) () { fail(Outcome.Failed("idle producer", ["owned", "worker"])) } +apply(values: AsyncStream<Int>) () { parallel_each(visit, values, 2) } +add(value: Int, previous: Int) (next: Int) { next = previous + value } +inspect(value: Outcome.Done) (answer: Bool) recover("test requires failure without another producer item") { answer = false } +inspect(value: Outcome.Failed(message, history)) (answer: Bool) recover("test requires failure without another producer item") { answer = message == "idle producer" and history == ["owned", "worker"] } +public main() (answer: Bool) { + input = channel(Int, 1) + send(input, 2) + values = stream(input) + alias = values + failed = inspect(result(apply(values))) + remaining = reduce(add, alias, 0) + send(input, 3) + independent = receive(input) + close(input) + answer = failed and remaining == 0 and independent == some_int(3) +} +"#; + + const SYNC_PARALLEL: &str = r#"type Outcome { Done + error Failed(message: Text, history: List<Text>) } +visit(value: 0) () { fail(Outcome.Failed("sync original", ["owned", "worker"])) } +visit(value: other) () {} +good() () { parallel_each(visit, [1, 2, 3], 2) + parallel_each(visit, dever.bytes.from_text("AB"), 2) } +bad() () { parallel_each(visit, [1, 0, 3], 2) } +inspect(value: Outcome.Done, expected: Bool) (answer: Bool) recover("test checks the expected branch") { answer = expected } +inspect(value: Outcome.Failed(message, history), expected: Bool) (answer: Bool) recover("test checks the expected branch") { answer = not expected and message == "sync original" and history == ["owned", "worker"] } +public main() (answer: Bool) { answer = inspect(result(good()), true) and inspect(result(bad()), false) } +"#; + + const ASYNC_SYNC_PARALLEL: &str = r#"type Outcome { Done + error Failed(message: Text, history: List<Text>) } +visit(value: 0) () { fail(Outcome.Failed("sync original", ["owned", "worker"])) } +visit(value: other) () {} +good() () { ready = pause(1) + parallel_each(visit, [1, 2, 3], 2) + parallel_each(visit, dever.bytes.from_text("AB"), 2) + parallel_each(visit, stream([1, 2]), 2) } +bad() () { ready = pause(1) + parallel_each(visit, stream([1, 0, 3]), 2) } +inspect(value: Outcome.Done, expected: Bool) (answer: Bool) recover("test checks the expected branch") { answer = expected } +inspect(value: Outcome.Failed(message, history), expected: Bool) (answer: Bool) recover("test checks the expected branch") { answer = not expected and message == "sync original" and history == ["owned", "worker"] } +public main() (answer: Bool) { answer = inspect(result(good()), true) and inspect(result(bad()), false) } +"#; + + const SYNC_FILE_PARALLEL: &str = r#"type Outcome { Done + error Failed(message: Text, history: List<Text>) } +visit(value: dever.io.ReadEvent.Chunk(bytes)) () { fail(Outcome.Failed("stream original", ["owned", "worker"])) } +visit(value: dever.io.ReadEvent.Failed(message)) () { fail(Outcome.Failed(message, ["wrong", "read failure"])) } +apply(values: Stream<dever.io.ReadEvent>) () { parallel_each(visit, values, 2) } +inspect(value: Outcome.Done) (answer: Bool) recover("test requires a worker fault") { answer = false } +inspect(value: Outcome.Failed(message, history)) (answer: Bool) recover("test requires a worker fault") { answer = message == "stream original" and history == ["owned", "worker"] } +public main() (answer: Bool) { + file = dever.io.open(FIXTURE_PATH) + values = dever.io.chunks(file, 1) + answer = inspect(result(apply(values))) + close(values) + dever.io.close(file) +} +"#; + + const UNIT_COMPOSITION: &str = r#"finish() () { ready = pause(1) } +pending() () { ready = pause(1000) } +fallback() () { ready = pause(1) } +public main() (answer: Bool) { + slow = run(pending()) + timeout(slow, 1, fallback) + quick = run(finish()) + timeout(quick, 500, fallback) + first = run(pending()) + second = run(finish()) + race(first, second) + answer = true +} +"#; + + const TIMEOUT_VALUES: &str = r#"type Bundle { label: Text + names: List<Text> + lookup: Map<Text, List<Text>> } +pair(millis: Int) (bundle: Bundle, names: List<Text>) { + dever.task.sleep(millis) + bundle = Bundle { label = "completed" + names = ["inside"] + lookup = { "key" = ["value"] } } + names = ["result"] +} +fallback() (bundle: Bundle, names: List<Text>) { + dever.task.sleep(1) + bundle = Bundle { label = "fallback" + names = ["fallback"] + lookup = { "key" = ["timeout"] } } + names = ["fallback result"] +} +nullable() (value: Text?) { value = null } +wrong_nullable() (value: Text?) { value = "wrong fallback" } +public main() (answer: Bool) { + slow = run(pair(1000)) + timed = timeout(slow, 1, fallback) + timed_ok = timed.bundle.label == "fallback" and timed.bundle.names == ["fallback"] and get(timed.bundle.lookup, "key") == some_names(["timeout"]) and timed.names == ["fallback result"] + quick = run(pair(1)) + ready = timeout(quick, 500, fallback) + ready_ok = ready.bundle.label == "completed" and ready.names == ["result"] + optional = run(nullable()) + nothing = timeout(optional, 500, wrong_nullable) + answer = timed_ok and ready_ok and nothing == null +} +some_names(value: List<Text>) (answer: List<Text>?) { answer = value } +"#; + + const RACE_VALUES: &str = r#"pair(label: Text, millis: Int) (text: Text, history: List<Text>) { + dever.task.sleep(millis) + text = label + history = [label, "owned"] +} +public main() (answer: Bool) { + slow = run(pair("slow", 1000)) + fast = run(pair("fast", 1)) + third = run(pair("other", 1000)) + winner = race(slow, fast, third) + answer = winner.text == "fast" and winner.history == ["fast", "owned"] +} +"#; + + const COMPOSITION_FAILURE: &str = r#"type Outcome { Done(value: Text) + error Failed(message: Text, history: List<Text>) } +broken() (value: Text) { ready = pause(1) + fail(Outcome.Failed("original", ["owned", "failure"])) } +slow() (value: Text) { ready = pause(1000) + value = "wrong success" } +fallback() (value: Text) { value = "wrong fallback" } +timed() (value: Text) { job = run(broken()) + value = timeout(job, 500, fallback) } +compete() (value: Text) { failing = run(broken()) + pending = run(slow()) + value = race(failing, pending) } +inspect(value: Outcome.Done(text)) (answer: Bool) recover("test requires the original failure") { answer = false } +inspect(value: Outcome.Failed(message, history)) (answer: Bool) recover("test requires the original failure") { answer = message == "original" and history == ["owned", "failure"] } +public main() (answer: Bool) { answer = inspect(result(timed())) and inspect(result(compete())) } +"#; + + const TCP_ROUNDTRIP: &str = r#"collect_body(listener: dever.system.Listener) (body: Bytes) { + socket = dever.net.accept(listener) + dever.net.timeout(socket, 500) + values = dever.net.chunks(socket, 2) + body = reduce(collect, values, dever.bytes.from_text("")) + dever.net.close(socket) +} +collect(value: dever.io.ReadEvent.Chunk(bytes), previous: Bytes) (next: Bytes) { next = dever.bytes.concat(previous, bytes) } +collect(value: dever.io.ReadEvent.Failed(message), previous: Bytes) (next: Bytes) { fail(dever.io.ReadResult.Failed(message)) } +public main() (answer: Bool) { + listener = dever.net.listen("127.0.0.1", 0) + worker = run(collect_body(listener)) + socket = dever.net.connect("127.0.0.1", dever.net.port(listener)) + dever.net.timeout(socket, 500) + dever.net.write(socket, dever.bytes.from_text("ping")) + dever.net.close(socket) + body = wait(worker) + dever.net.close_listener(listener) + answer = body == dever.bytes.from_text("ping") +} +"#; + + // Exactly one byte is written and one byte read; this does not assume TCP chunk boundaries. + const TCP_DUPLEX: &str = r#"bytes(value: dever.io.ReadState.Read(bytes)) (answer: Bytes) { answer = bytes } +bytes(value: dever.io.ReadState.End) (answer: Bytes) { answer = dever.bytes.from_text("wrong EOF") } +ended(value: dever.io.ReadState.Read(bytes)) (answer: Bool) { answer = false } +ended(value: dever.io.ReadState.End) (answer: Bool) { answer = true } +closed(value: dever.io.CloseResult.Closed) (answer: Bool) recover("test requires shared close failure") { answer = false } +closed(value: dever.io.CloseResult.Failed(message)) (answer: Bool) recover("test requires shared close failure") { answer = message == "resource is closed" } +echo(listener: dever.system.Listener) (answer: Bool) { + socket = dever.net.accept(listener) + dever.net.timeout(socket, 500) + request = bytes(dever.net.read(socket, 1)) + dever.net.write(socket, request) + dever.net.close(socket) + answer = request == dever.bytes.from_text("Q") +} +public main() (answer: Bool) { + listener = dever.net.listen("127.0.0.1", 0) + worker = run(echo(listener)) + socket = dever.net.connect_timeout("127.0.0.1", dever.net.port(listener), 500) + dever.net.timeout(socket, 500) + dever.net.write(socket, dever.bytes.from_text("Q")) + reply = bytes(dever.net.read(socket, 1)) + end = ended(dever.net.read(socket, 1)) + okay = wait(worker) + alias = socket + dever.net.close(alias) + shared_close = closed(result(dever.net.close(socket))) + dever.net.close_listener(listener) + answer = okay and reply == dever.bytes.from_text("Q") and end and shared_close +} +"#; + + const NETWORK_ERRORS: &str = r#"bad_connect(value: dever.net.ConnectResult.Connected(socket)) (answer: Bool) recover("test rejects nonnumeric timed connect") { answer = false } +bad_connect(value: dever.net.ConnectResult.Failed(message)) (answer: Bool) recover("test rejects nonnumeric timed connect") { answer = message == "timed connect requires a numeric IP address" } +bad_listen(value: dever.net.ListenResult.Listening(listener)) (answer: Bool) recover("test rejects invalid listen port") { answer = false } +bad_listen(value: dever.net.ListenResult.Failed(message)) (answer: Bool) recover("test rejects invalid listen port") { answer = message == "port must be between 0 and 65535" } +closed_port(value: dever.net.PortResult.Bound(port)) (answer: Bool) recover("test checks shared listener closure") { answer = false } +closed_port(value: dever.net.PortResult.Failed(message)) (answer: Bool) recover("test checks shared listener closure") { answer = message == "resource is closed" } +public main() (answer: Bool) { + connect = bad_connect(result(dever.net.connect_timeout("localhost", 0, 1))) + listen = bad_listen(result(dever.net.listen("127.0.0.1", -1))) + listener = dever.net.listen("127.0.0.1", 0) + alias = listener + dever.net.close_listener(alias) + port = closed_port(result(dever.net.port(listener))) + answer = connect and listen and port +} +"#; + + const TERMINAL_STREAM: &str = r#"count_read(value: dever.io.ReadEvent.Chunk(bytes), previous: Int) (next: Int) recover("test requires one terminal error") { next = -100 } +count_read(value: dever.io.ReadEvent.Failed(message), previous: Int) (next: Int) recover("test requires one terminal error") { next = previous + matching_closed(message) } +matching_closed(message: "resource is closed") (answer: Int) { answer = 1 } +matching_closed(message: other) (answer: Int) { answer = -100 } +count_connection(value: dever.net.ConnectResult.Connected(socket), previous: Int) (next: Int) recover("test requires one terminal error") { next = -100 } +count_connection(value: dever.net.ConnectResult.Failed(message), previous: Int) (next: Int) recover("test requires one terminal error") { next = previous + matching_closed(message) } +public main() (answer: Bool) { + listener = dever.net.listen("127.0.0.1", 0) + connecting = run(dever.net.connect_timeout("127.0.0.1", dever.net.port(listener), 500)) + socket = dever.net.accept(listener) + peer = wait(connecting) + reads = dever.net.chunks(socket, 1) + read_alias = reads + dever.net.close(socket) + failed_reads = reduce(count_read, reads, 0) + end_reads = reduce(count_read, read_alias, 0) + incoming = dever.net.connections(listener) + incoming_alias = incoming + dever.net.close_listener(listener) + failed_accept = reduce(count_connection, incoming, 0) + end_accept = reduce(count_connection, incoming_alias, 0) + dever.net.close(peer) + answer = failed_reads == 1 and end_reads == 0 and failed_accept == 1 and end_accept == 0 +} +"#; + + const TIMEOUT_DRAINS_SOCKET: &str = r#"ended(value: dever.io.ReadState.Read(bytes)) (answer: Bool) { answer = false } +ended(value: dever.io.ReadState.End) (answer: Bool) { answer = true } +pending(socket: dever.system.Socket) (answer: Bool) { answer = ended(dever.net.read(socket, 1)) } +parent(listener: dever.system.Listener, ready: Channel<Int>) (answer: Bool) { + socket = dever.net.accept(listener) + child = run(pending(socket)) + send(ready, 1) + answer = wait(child) +} +fallback() (answer: Bool) { answer = true } +public main() (answer: Bool) { + listener = dever.net.listen("127.0.0.1", 0) + ready = channel(Int, 1) + worker = run(parent(listener, ready)) + peer = dever.net.connect_timeout("127.0.0.1", dever.net.port(listener), 500) + dever.net.timeout(peer, 500) + started = receive(ready) + timed = timeout(worker, 1, fallback) + eof = ended(dever.net.read(peer, 1)) + close(ready) + dever.net.close(peer) + dever.net.close_listener(listener) + answer = started == some_int(1) and timed and eof +} +"#; + + fn source(body: &str) -> String { + let body = body.replace("FIXTURE_PATH", "\"owned-fixture-file\""); + format!( + "some_text(value: Text) (answer: Text?) {{ answer = value }}\nsome_int(value: Int) (answer: Int?) {{ answer = value }}\ntimer_status(value: dever.time.SleepResult.Done) (okay: Bool) recover(\"fixture checks timer\") {{ okay = true }}\ntimer_status(value: dever.time.SleepResult.Failed(message)) (okay: Bool) recover(\"fixture checks timer\") {{ okay = false }}\npause(millis: Int) (okay: Bool) {{ okay = timer_status(result(dever.task.sleep(millis))) }}\n{body}" + ) + } + + const CASES: &[(&str, &str)] = &[ + ("streams", STREAMS), + ("channel_stream", CHANNEL_STREAM), + ("ticks", TICKS), + ("parallel_stream", PARALLEL_STREAM), + ("parallel_failure", PARALLEL_FAILURE), + ("idle_parallel_failure", IDLE_PARALLEL_FAILURE), + ("sync_parallel", SYNC_PARALLEL), + ("async_sync_parallel", ASYNC_SYNC_PARALLEL), + ("sync_file_parallel", SYNC_FILE_PARALLEL), + ("unit_composition", UNIT_COMPOSITION), + ("timeout_values", TIMEOUT_VALUES), + ("race_values", RACE_VALUES), + ("composition_failure", COMPOSITION_FAILURE), + ("tcp_roundtrip", TCP_ROUNDTRIP), + ("tcp_duplex", TCP_DUPLEX), + ("network_errors", NETWORK_ERRORS), + ("terminal_stream", TERMINAL_STREAM), + ("timeout_drains_socket", TIMEOUT_DRAINS_SOCKET), + ]; + + #[test] + fn stream_network_sources_pass_the_existing_checker() { + for (name, body) in CASES { + let mut sources = SourceMap::default(); + sources.add("main.dever", source(body)); + check(&sources).unwrap_or_else(|errors| { + panic!( + "{name}: {}", + errors + .iter() + .map(|error| error.render(&sources)) + .collect::<String>() + ) + }); + } + } + + #[test] + fn stream_network_sources_emit_objects_for_six_targets() { + for (name, body) in CASES { + let ir = llvm::lower(&source(body)); + for target in [ + Target::LinuxX86_64, + Target::LinuxAarch64, + Target::MacosX86_64, + Target::MacosAarch64, + Target::WindowsX86_64, + Target::WindowsAarch64, + ] { + let object = emit_object(&ir, target) + .unwrap_or_else(|error| panic!("{name}/{target:?}: {error}")); + assert!(!object.is_empty(), "{name}/{target:?}"); + } + } + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn nullable_memory_streams_early_stop_and_alias_close() { + llvm_managed::boolean(&source(STREAMS)); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn channel_stream_backpressure_with_suspending_handler() { + llvm_managed::boolean(&source(CHANNEL_STREAM)); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn ticks_suspend_then_early_stop_closes_aliases() { + llvm_managed::boolean(&source(TICKS)); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn parallel_stream_and_memory_workers_preserve_all_values() { + llvm_managed::boolean(&source(PARALLEL_STREAM)); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn parallel_stream_preserves_typed_owned_worker_failure() { + llvm_managed::boolean(&source(PARALLEL_FAILURE)); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn parallel_failure_interrupts_idle_producer_and_preserves_channel_alias() { + llvm_managed::boolean(&source(IDLE_PARALLEL_FAILURE)); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn synchronous_parallel_memory_dispatch_preserves_failure_and_ownership() { + llvm_managed::boolean(&source(SYNC_PARALLEL)); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn async_parallel_dispatch_supports_non_suspending_handlers() { + llvm_managed::boolean(&source(ASYNC_SYNC_PARALLEL)); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive and owned temporary file"] + fn synchronous_file_stream_parallel_failure_releases_workers_and_rows() { + let directory = super::temp::TemporaryDirectory::new(); + let path = directory.path().join("parallel-source"); + std::fs::write(&path, b"abc").unwrap(); + llvm_managed::boolean(&source( + &SYNC_FILE_PARALLEL.replace("FIXTURE_PATH", &format!("{:?}", path.to_str().unwrap())), + )); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn timeout_preserves_owned_multi_output_and_some_null() { + llvm_managed::boolean(&source(TIMEOUT_VALUES)); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn race_preserves_owned_multi_output_and_drains_losers() { + llvm_managed::boolean(&source(RACE_VALUES)); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn zero_output_timeout_and_race_preserve_presence_and_cleanup() { + llvm_managed::boolean(&source(UNIT_COMPOSITION)); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn timeout_and_race_propagate_business_failure_without_fallback() { + llvm_managed::boolean(&source(COMPOSITION_FAILURE)); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive and owned loopback sockets"] + fn tcp_read_write_eof_and_shared_resource_close() { + llvm_managed::boolean(&source(TCP_DUPLEX)); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive and owned loopback sockets"] + fn tcp_chunks_collects_exact_bytes_across_short_reads_and_eof() { + llvm_managed::boolean(&source(TCP_ROUNDTRIP)); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive and owned loopback sockets"] + fn invalid_network_inputs_preserve_original_failure_messages() { + llvm_managed::boolean(&source(NETWORK_ERRORS)); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive and owned loopback sockets"] + fn chunks_and_connections_emit_one_terminal_failure_before_end() { + llvm_managed::boolean(&source(TERMINAL_STREAM)); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive and owned loopback sockets"] + fn timeout_drains_pending_descendant_socket_before_fallback_returns() { + llvm_managed::boolean(&source(TIMEOUT_DRAINS_SOCKET)); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn zero_ticks_is_an_original_uncaptured_runtime_fault() { + llvm_managed::runtime_fault( + "public main() (values: AsyncStream<Int>) { values = dever.task.ticks(0) }", + "duration must be positive", + ); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn zero_task_timeout_is_an_original_uncaptured_runtime_fault() { + llvm_managed::runtime_fault( + "pending() (answer: Bool) { dever.task.sleep(1000)\nanswer = false }\nfallback() (answer: Bool) { answer = false }\npublic main() (answer: Bool) { job = run(pending())\nanswer = timeout(job, 0, fallback) }", + "duration must be positive", + ); + } + + #[test] + #[ignore = "requires explicit runtime-abi archive"] + fn timeout_and_race_keep_numeric_fault_kind_and_origin() { + for body in [ + "job = run(broken(0))\nanswer = timeout(job, 500, fallback)", + "job = run(broken(0))\nremaining = run(pending())\nanswer = race(job, remaining)", + ] { + let source = format!( + "broken(denominator: Int) (answer: Int) {{ dever.task.sleep(1)\nanswer = 42 // denominator }}\npending() (answer: Int) {{ dever.task.sleep(1000)\nanswer = 0 }}\nfallback() (answer: Int) {{ answer = 9 }}\npublic main() (answer: Int) {{ {body} }}" + ); + let origin = source.find("42 // denominator").unwrap(); + llvm_managed::execute( + &source, + "{ i64 }", + &format!( + " %failed = icmp eq i32 %status, 1 + %code_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 0 + %code = load i32, ptr %code_ptr + %numeric = icmp eq i32 %code, 2 + %origin_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 1 + %origin_id = load i32, ptr %origin_ptr + %span = getelementptr {{ i64, i64, i64 }}, ptr @dever_source_spans, i32 %origin_id, i32 1 + %start = load i64, ptr %span + %original = icmp eq i64 %start, {origin} + %depth_ptr = getelementptr %dever.fault, ptr %fault, i32 0, i32 2 + %depth = load i32, ptr %depth_ptr + %called = icmp uge i32 %depth, 1 + %kind_ok = and i1 %failed, %numeric + %source_ok = and i1 %original, %called + %passed = and i1 %kind_ok, %source_ok" + ), + ); + } + } +} diff --git a/test/native-runtime-abi/async-driver.c b/test/native-runtime-abi/async-driver.c new file mode 100644 index 0000000..c767811 --- /dev/null +++ b/test/native-runtime-abi/async-driver.c @@ -0,0 +1,23 @@ +#include <stdint.h> + +extern uint32_t dever_rt_v1_version(void); +extern void *dever_coro_probe_create(void); +extern int32_t dever_coro_probe_progress(void); +extern void dever_coro_probe_resume(void *frame); +extern uint8_t dever_coro_probe_done(void *frame); +extern void dever_coro_probe_destroy(void *frame); +extern int32_t dever_coro_root_run(void); + +int main(void) { + if (dever_rt_v1_version() != 1) return 1; + void *frame = dever_coro_probe_create(); + if (!frame || dever_coro_probe_progress() != 0 || dever_coro_probe_done(frame)) return 2; + dever_coro_probe_resume(frame); + if (dever_coro_probe_progress() != 42 || !dever_coro_probe_done(frame)) return 3; + dever_coro_probe_destroy(frame); + frame = dever_coro_probe_create(); + if (!frame || dever_coro_probe_done(frame)) return 4; + dever_coro_probe_destroy(frame); + if (dever_coro_root_run() != 0) return 5; + return 0; +} diff --git a/test/native-runtime-abi/async-operations-driver.c b/test/native-runtime-abi/async-operations-driver.c new file mode 100644 index 0000000..1c20505 --- /dev/null +++ b/test/native-runtime-abi/async-operations-driver.c @@ -0,0 +1,470 @@ +#define _POSIX_C_SOURCE 200809L +#include "runtime.h" + +#include <stdint.h> +#include <stdatomic.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> + +static _Atomic int unit_destroy_count; +static _Atomic int blocking_done_count; +static _Atomic int group_sync_done_count; + +typedef struct { + DeverRtPollState *poll; + void *output; + void *fault; + DeverRtAsyncOp *operation; + DeverRtAsyncOp *blocked_send; + DeverRtAsyncTask *task; + DeverRtAsyncGroup *group; + DeverRtAsyncChannel *channel; + uint32_t phase; + uint8_t done; + uint8_t pending; + uint8_t group_pending; +} Frame; + +static void move_int(void *source, void *destination) { + *(int64_t *)destination = *(int64_t *)source; + *(int64_t *)source = 0; +} + +static void drop_int(void *value) { (void)value; } +static void move_unit(void *source, void *destination) { (void)source; (void)destination; } +static void drop_unit(void *value) { (void)value; } + +typedef struct { + uint8_t present; + int64_t value; +} OptionalInt; + +static void move_optional(void *source, void *destination) { + OptionalInt *old = source; + OptionalInt *new_value = destination; + new_value->present = old->present; + new_value->value = old->value; + old->present = 0; + old->value = 0; +} + +static const DeverRtOwnedType integer = { 8, 8, move_int, drop_int, 1 }; +static const DeverRtOwnedType unit = { 0, 1, move_unit, drop_unit, 1 }; +static const DeverRtOwnedType optional_integer = { + sizeof(OptionalInt), _Alignof(OptionalInt), move_optional, drop_int, 1 +}; + +static void *create(DeverRtPollState *poll, const void *input, void *output, void *fault) { + (void)input; + Frame *frame = calloc(1, sizeof(*frame)); + if (frame) { + frame->poll = poll; + frame->output = output; + frame->fault = fault; + } + return frame; +} + +static void destroy(void *pointer) { + Frame *frame = pointer; + if (frame->operation) dever_rt_v1_async_op_release(frame->operation); + if (frame->blocked_send) dever_rt_v1_async_op_release(frame->blocked_send); + if (frame->task) dever_rt_v1_async_task_release(frame->task); + if (frame->group) dever_rt_v1_async_group_release(frame->group); + if (frame->channel) dever_rt_v1_async_channel_release(frame->channel); + free(frame); +} + +static void destroy_unit(void *pointer) { + atomic_fetch_add(&unit_destroy_count, 1); + destroy(pointer); +} + +static uint8_t done(void *pointer) { return ((Frame *)pointer)->done; } + +static void complete(Frame *frame, uint32_t status) { + frame->done = 1; + dever_rt_v1_async_complete(frame->poll, status); +} + +static void resume_child(void *pointer, uint8_t has_value, int64_t delay) { + Frame *frame = pointer; + DeverRtBuffer error = { 0 }; + if (!frame->operation) frame->operation = dever_rt_v1_async_sleep(delay); + uint32_t status = dever_rt_v1_async_op_poll(frame->poll, frame->operation, + NULL, NULL, NULL, &error); + if (status == DEVER_RT_ASYNC_PENDING) return; + if (status != DEVER_RT_ASYNC_READY_OK) abort(); + dever_rt_v1_async_op_release(frame->operation); + frame->operation = NULL; + if (has_value) *(int64_t *)frame->output = 7; + complete(frame, DEVER_RT_ASYNC_READY_OK); +} + +static void value_resume(void *pointer) { resume_child(pointer, 1, 2); } +static void unit_resume(void *pointer) { resume_child(pointer, 0, 1000); } +static void fault_resume(void *pointer) { + Frame *frame = pointer; + *(int64_t *)frame->fault = 99; + complete(frame, DEVER_RT_ASYNC_READY_FAULT); +} + +static const DeverRtAsyncFunction value_child = { + &integer, &integer, create, value_resume, destroy, done, 1 +}; +static const DeverRtAsyncFunction unit_child = { + &unit, &integer, create, unit_resume, destroy_unit, done, 1 +}; +static const DeverRtAsyncFunction fault_child = { + &unit, &integer, create, fault_resume, destroy, done, 1 +}; + +static uint32_t twice(void *input, void *output, void *fault) { + (void)fault; + struct timespec delay = { 0, 80000000 }; + nanosleep(&delay, NULL); + *(int64_t *)output = *(int64_t *)input * 2; + atomic_fetch_add(&blocking_done_count, 1); + return 0; +} + +static const DeverRtSyncFunction synchronous_twice = { + &integer, &integer, &integer, twice, 1 +}; + +static uint32_t group_sync(void *input, void *output, void *fault) { + (void)output; + (void)fault; + if (*(int64_t *)input != 21) abort(); + atomic_fetch_add(&group_sync_done_count, 1); + return 0; +} + +static const DeverRtSyncFunction synchronous_unit = { + &integer, &unit, &integer, group_sync, 1 +}; + +static uint32_t poll(Frame *frame, DeverRtAsyncOp *operation, void *output, + uint8_t *present, void *fault) { + DeverRtBuffer error = { 0 }; + uint32_t status = dever_rt_v1_async_op_poll(frame->poll, operation, + output, present, fault, &error); + if (status == DEVER_RT_ASYNC_PENDING) frame->pending = 1; + if (error.ptr) dever_rt_v1_buffer_free(error.ptr, error.len); + return status; +} + +static void root_resume(void *pointer) { + Frame *frame = pointer; + DeverRtBuffer error = { 0 }; + uint32_t status; + int64_t item; + int64_t fault_value; + OptionalInt optional; + uint8_t present; + for (;;) { + switch (frame->phase) { + case 0: + frame->operation = dever_rt_v1_async_task_run(&value_child, NULL, &error); + if (!frame->operation) abort(); + frame->phase = 1; + break; + case 1: + status = poll(frame, frame->operation, &frame->task, NULL, NULL); + if (status == DEVER_RT_ASYNC_PENDING) return; + if (status != DEVER_RT_ASYNC_READY_OK || !frame->task) abort(); + dever_rt_v1_async_op_release(frame->operation); + DeverRtAsyncTask *task_alias = dever_rt_v1_async_task_retain(frame->task); + if (!task_alias) abort(); + frame->operation = dever_rt_v1_async_task_wait(frame->task); + frame->task = NULL; + dever_rt_v1_async_task_release(task_alias); + frame->phase = 2; + break; + case 2: + status = poll(frame, frame->operation, &item, NULL, frame->fault); + if (status == DEVER_RT_ASYNC_PENDING) return; + if (status != DEVER_RT_ASYNC_READY_OK || item != 7 || !frame->pending) abort(); + dever_rt_v1_async_op_release(frame->operation); + frame->operation = NULL; + frame->operation = dever_rt_v1_async_task_run(&unit_child, NULL, &error); + if (!frame->operation) abort(); + frame->phase = 12; + break; + case 12: + status = poll(frame, frame->operation, &frame->task, NULL, NULL); + if (status == DEVER_RT_ASYNC_PENDING) return; + if (status != DEVER_RT_ASYNC_READY_OK || !frame->task) abort(); + dever_rt_v1_async_op_release(frame->operation); + frame->operation = dever_rt_v1_async_task_stop(frame->task); + frame->task = NULL; + frame->phase = 13; + break; + case 13: + status = poll(frame, frame->operation, NULL, NULL, frame->fault); + if (status == DEVER_RT_ASYNC_PENDING) return; + if (status != DEVER_RT_ASYNC_READY_OK || atomic_load(&unit_destroy_count) < 1) abort(); + dever_rt_v1_async_op_release(frame->operation); + frame->operation = dever_rt_v1_async_task_run(&fault_child, NULL, &error); + if (!frame->operation) abort(); + frame->phase = 14; + break; + case 14: + status = poll(frame, frame->operation, &frame->task, NULL, NULL); + if (status == DEVER_RT_ASYNC_PENDING) return; + if (status != DEVER_RT_ASYNC_READY_OK || !frame->task) abort(); + dever_rt_v1_async_op_release(frame->operation); + frame->operation = dever_rt_v1_async_task_wait(frame->task); + frame->task = NULL; + frame->phase = 15; + break; + case 15: + status = poll(frame, frame->operation, NULL, NULL, &fault_value); + if (status == DEVER_RT_ASYNC_PENDING) return; + if (status != DEVER_RT_ASYNC_READY_FAULT || fault_value != 99) abort(); + dever_rt_v1_async_op_release(frame->operation); + frame->operation = NULL; + if (dever_rt_v1_async_group_new(1, &frame->group, &error) != DEVER_RT_ASYNC_PENDING) abort(); + frame->operation = dever_rt_v1_async_group_run(frame->group, &unit_child, NULL, &error); + if (!frame->operation) abort(); + frame->phase = 3; + break; + case 3: + status = poll(frame, frame->operation, NULL, NULL, frame->fault); + if (status == DEVER_RT_ASYNC_PENDING) return; + if (status != DEVER_RT_ASYNC_READY_OK) abort(); + dever_rt_v1_async_op_release(frame->operation); + frame->operation = dever_rt_v1_async_group_run(frame->group, &unit_child, NULL, &error); + if (!frame->operation) abort(); + frame->phase = 4; + break; + case 4: + status = poll(frame, frame->operation, NULL, NULL, frame->fault); + if (status != DEVER_RT_ASYNC_PENDING) abort(); + frame->group_pending = 1; + dever_rt_v1_async_op_release(frame->operation); + DeverRtAsyncGroup *group_alias = dever_rt_v1_async_group_retain(frame->group); + if (!group_alias) abort(); + frame->operation = dever_rt_v1_async_group_stop(frame->group); + frame->group = NULL; + dever_rt_v1_async_group_release(group_alias); + frame->phase = 5; + break; + case 5: + status = poll(frame, frame->operation, NULL, NULL, frame->fault); + if (status == DEVER_RT_ASYNC_PENDING) return; + if (status != DEVER_RT_ASYNC_READY_OK || atomic_load(&unit_destroy_count) < 2) abort(); + dever_rt_v1_async_op_release(frame->operation); + frame->operation = NULL; + if (dever_rt_v1_async_group_new(1, &frame->group, &error) != DEVER_RT_ASYNC_PENDING) abort(); + frame->operation = dever_rt_v1_async_group_run(frame->group, &fault_child, NULL, &error); + if (!frame->operation) abort(); + frame->phase = 18; + break; + case 18: + status = poll(frame, frame->operation, NULL, NULL, &fault_value); + if (status == DEVER_RT_ASYNC_PENDING) return; + if (status != DEVER_RT_ASYNC_READY_OK) abort(); + dever_rt_v1_async_op_release(frame->operation); + frame->operation = dever_rt_v1_async_group_wait(frame->group); + frame->group = NULL; + frame->phase = 19; + break; + case 19: + status = poll(frame, frame->operation, NULL, NULL, &fault_value); + if (status == DEVER_RT_ASYNC_PENDING) return; + if (status != DEVER_RT_ASYNC_READY_FAULT || fault_value != 99) abort(); + dever_rt_v1_async_op_release(frame->operation); + frame->operation = NULL; + if (dever_rt_v1_async_channel_new(1, &frame->channel, &error) != DEVER_RT_ASYNC_PENDING) abort(); + item = 11; + frame->operation = dever_rt_v1_async_channel_send(frame->channel, &integer, &item, &error); + if (!frame->operation || item != 0) abort(); + frame->phase = 6; + break; + case 6: + status = poll(frame, frame->operation, NULL, NULL, frame->fault); + if (status != DEVER_RT_ASYNC_READY_OK) abort(); + dever_rt_v1_async_op_release(frame->operation); + item = 12; + frame->blocked_send = dever_rt_v1_async_channel_send(frame->channel, &integer, &item, &error); + if (!frame->blocked_send || item != 0) abort(); + if (poll(frame, frame->blocked_send, NULL, NULL, frame->fault) != DEVER_RT_ASYNC_PENDING) abort(); + frame->operation = dever_rt_v1_async_channel_receive(frame->channel); + frame->phase = 7; + break; + case 7: + status = poll(frame, frame->operation, &item, &present, frame->fault); + if (status != DEVER_RT_ASYNC_READY_OK || !present || item != 11) abort(); + dever_rt_v1_async_op_release(frame->operation); + frame->operation = NULL; + status = poll(frame, frame->blocked_send, NULL, NULL, frame->fault); + frame->phase = 11; + if (status == DEVER_RT_ASYNC_PENDING) return; + if (status != DEVER_RT_ASYNC_READY_OK) abort(); + dever_rt_v1_async_op_release(frame->blocked_send); + frame->blocked_send = NULL; + frame->operation = dever_rt_v1_async_channel_close(frame->channel); + frame->phase = 8; + break; + case 11: + status = poll(frame, frame->blocked_send, NULL, NULL, frame->fault); + if (status == DEVER_RT_ASYNC_PENDING) return; + if (status != DEVER_RT_ASYNC_READY_OK) abort(); + dever_rt_v1_async_op_release(frame->blocked_send); + frame->blocked_send = NULL; + frame->operation = dever_rt_v1_async_channel_close(frame->channel); + frame->phase = 8; + break; + case 8: + status = poll(frame, frame->operation, NULL, NULL, frame->fault); + if (status != DEVER_RT_ASYNC_READY_OK) abort(); + dever_rt_v1_async_op_release(frame->operation); + frame->operation = dever_rt_v1_async_channel_receive(frame->channel); + frame->phase = 9; + break; + case 9: + status = poll(frame, frame->operation, &item, &present, frame->fault); + if (status != DEVER_RT_ASYNC_READY_OK || !present || item != 12) abort(); + dever_rt_v1_async_op_release(frame->operation); + frame->operation = dever_rt_v1_async_channel_receive(frame->channel); + frame->phase = 10; + break; + case 10: + status = poll(frame, frame->operation, &item, &present, frame->fault); + if (status != DEVER_RT_ASYNC_READY_OK || present) abort(); + if (!frame->group_pending) abort(); + dever_rt_v1_async_op_release(frame->operation); + dever_rt_v1_async_channel_release(frame->channel); + frame->channel = NULL; + if (dever_rt_v1_async_channel_new(1, &frame->channel, &error) != DEVER_RT_ASYNC_PENDING) abort(); + optional.present = 0; + optional.value = 0; + frame->operation = dever_rt_v1_async_channel_send(frame->channel, &optional_integer, &optional, &error); + if (!frame->operation) abort(); + frame->phase = 24; + break; + case 24: + status = poll(frame, frame->operation, NULL, NULL, frame->fault); + if (status != DEVER_RT_ASYNC_READY_OK) abort(); + dever_rt_v1_async_op_release(frame->operation); + frame->operation = dever_rt_v1_async_channel_close(frame->channel); + frame->phase = 25; + break; + case 25: + status = poll(frame, frame->operation, NULL, NULL, frame->fault); + if (status != DEVER_RT_ASYNC_READY_OK) abort(); + dever_rt_v1_async_op_release(frame->operation); + frame->operation = dever_rt_v1_async_channel_receive(frame->channel); + frame->phase = 26; + break; + case 26: + status = poll(frame, frame->operation, &optional, &present, frame->fault); + if (status != DEVER_RT_ASYNC_READY_OK || !present || optional.present) abort(); + dever_rt_v1_async_op_release(frame->operation); + frame->operation = dever_rt_v1_async_channel_receive(frame->channel); + frame->phase = 27; + break; + case 27: + status = poll(frame, frame->operation, &optional, &present, frame->fault); + if (status != DEVER_RT_ASYNC_READY_OK || present) abort(); + dever_rt_v1_async_op_release(frame->operation); + dever_rt_v1_async_channel_release(frame->channel); + frame->channel = NULL; + item = 21; + frame->operation = dever_rt_v1_async_parallel(&synchronous_twice, &item, &error); + if (!frame->operation || item != 0) abort(); + frame->phase = 16; + break; + case 16: + status = poll(frame, frame->operation, &item, NULL, frame->fault); + if (status == DEVER_RT_ASYNC_PENDING) return; + if (status != DEVER_RT_ASYNC_READY_OK || item != 42) abort(); + dever_rt_v1_async_op_release(frame->operation); + item = 21; + frame->operation = dever_rt_v1_async_task_run_sync(&synchronous_twice, &item, &error); + if (!frame->operation || item != 0) abort(); + frame->phase = 20; + break; + case 20: + status = poll(frame, frame->operation, &frame->task, NULL, frame->fault); + if (status == DEVER_RT_ASYNC_PENDING) return; + if (status != DEVER_RT_ASYNC_READY_OK || !frame->task) abort(); + dever_rt_v1_async_op_release(frame->operation); + frame->operation = dever_rt_v1_async_task_wait(frame->task); + frame->task = NULL; + frame->phase = 21; + break; + case 21: + status = poll(frame, frame->operation, &item, NULL, frame->fault); + if (status == DEVER_RT_ASYNC_PENDING) return; + if (status != DEVER_RT_ASYNC_READY_OK || item != 42) abort(); + dever_rt_v1_async_op_release(frame->operation); + if (dever_rt_v1_async_group_new(1, &frame->group, &error) != DEVER_RT_ASYNC_PENDING) abort(); + item = 21; + frame->operation = dever_rt_v1_async_group_run_sync(frame->group, &synchronous_unit, &item, &error); + if (!frame->operation || item != 0) abort(); + frame->phase = 22; + break; + case 22: + status = poll(frame, frame->operation, NULL, NULL, frame->fault); + if (status == DEVER_RT_ASYNC_PENDING) return; + if (status != DEVER_RT_ASYNC_READY_OK) abort(); + dever_rt_v1_async_op_release(frame->operation); + frame->operation = dever_rt_v1_async_group_wait(frame->group); + frame->group = NULL; + frame->phase = 23; + break; + case 23: + status = poll(frame, frame->operation, NULL, NULL, frame->fault); + if (status == DEVER_RT_ASYNC_PENDING) return; + if (status != DEVER_RT_ASYNC_READY_OK || atomic_load(&group_sync_done_count) != 1) abort(); + dever_rt_v1_async_op_release(frame->operation); + item = 21; + frame->operation = dever_rt_v1_async_blocking(&synchronous_twice, &item, &error); + if (!frame->operation || item != 0) abort(); + frame->phase = 17; + break; + case 17: + status = poll(frame, frame->operation, &item, NULL, frame->fault); + if (status != DEVER_RT_ASYNC_PENDING) abort(); + dever_rt_v1_async_op_release(frame->operation); + frame->operation = NULL; + *(int64_t *)frame->output = 42; + complete(frame, DEVER_RT_ASYNC_READY_OK); + return; + default: + abort(); + } + } +} + +static const DeverRtAsyncFunction root = { + &integer, &integer, create, root_resume, destroy, done, 0 +}; + +int main(void) { + int64_t output = 0; + int64_t fault = 0; + DeverRtBuffer error = { 0 }; + if (dever_rt_v1_async_op_poll(NULL, NULL, NULL, NULL, NULL, &error) + != DEVER_RT_ASYNC_INVALID_INPUT || !error.ptr) return 4; + dever_rt_v1_buffer_free(error.ptr, error.len); + error = (DeverRtBuffer){ 0 }; + uint8_t unit_output = 7; + if (dever_rt_v1_time_sleep(0, &unit_output, &error) != DEVER_RT_OK + || unit_output != 0 || error.ptr) return 2; + unit_output = 7; + if (dever_rt_v1_time_sleep(-1, &unit_output, &error) != DEVER_RT_RUNTIME_ERROR + || unit_output != 7 || !error.ptr + || error.len != sizeof("sleep duration must not be negative") - 1 + || memcmp(error.ptr, "sleep duration must not be negative", error.len)) return 3; + dever_rt_v1_buffer_free(error.ptr, error.len); + error = (DeverRtBuffer){ 0 }; + uint32_t status = dever_rt_v1_async_root(&root, NULL, &output, &fault, &error); + if (error.ptr) dever_rt_v1_buffer_free(error.ptr, error.len); + return status == DEVER_RT_ASYNC_READY_OK && output == 42 + && atomic_load(&blocking_done_count) == 3 + && atomic_load(&group_sync_done_count) == 1 ? 0 : 1; +} diff --git a/test/native-runtime-abi/database-driver.c b/test/native-runtime-abi/database-driver.c new file mode 100644 index 0000000..e6b8385 --- /dev/null +++ b/test/native-runtime-abi/database-driver.c @@ -0,0 +1,245 @@ +/* Database ABI ownership without settings, pools or database connections. + * The existing managed-driver supplies warm 1 + 64 and the strict ledger. */ +#include "runtime.h" + +#include <stdint.h> +#include <string.h> + +typedef struct { + uint32_t kind; + void *message; + void *cause; +} Fault; + +typedef struct { + uint8_t present; + void *text; +} OptionalText; + +static unsigned packs; +static unsigned moves; +static unsigned drops; + +static void release_buffer(DeverRtBuffer *buffer) { + dever_rt_v1_buffer_free(buffer->ptr, buffer->len); + *buffer = (DeverRtBuffer){0}; +} + +static int buffer_is(const DeverRtBuffer *buffer, const char *expected) { + size_t length = strlen(expected); + return buffer->ptr && buffer->len == length && + memcmp(buffer->ptr, expected, length) == 0; +} + +static int text_is(const void *text, const char *expected) { + DeverRtBuffer buffer = {0}; + uint32_t status = dever_rt_v1_text_export(text, &buffer); + int matches = status == 0 && buffer_is(&buffer, expected); + release_buffer(&buffer); + return matches; +} + +static int new_text(const char *text, void **out) { + DeverRtBuffer buffer = {0}; + uint32_t status = dever_rt_v1_text_new( + (const uint8_t *)text, strlen(text), out, &buffer); + release_buffer(&buffer); + return status == 0; +} + +static void move_fault(void *source, void *destination) { + *(Fault *)destination = *(Fault *)source; + *(Fault *)source = (Fault){0}; + ++moves; +} + +static void drop_fault(void *value) { + Fault *fault = value; + dever_rt_v1_text_release(fault->message); + dever_rt_v1_text_release(fault->cause); + *fault = (Fault){0}; + ++drops; +} + +static void pack_fault(uint32_t kind, const void *message, void *out) { + *(Fault *)out = (Fault){kind, dever_rt_v1_text_retain(message), NULL}; + ++packs; +} + +static void append_cause(void *value, const void *message) { + dever_rt_v1_db_cause_append(&((Fault *)value)->cause, message); +} + +static const DeverRtOwnedType fault_type = { + sizeof(Fault), _Alignof(Fault), move_fault, drop_fault, 1 +}; +static const DeverRtDbError errors = {&fault_type, pack_fault, append_cause}; + +static void clone_optional(const void *source, void *destination) { + const OptionalText *from = source; + *(OptionalText *)destination = (OptionalText){ + from->present, from->present ? dever_rt_v1_text_retain(from->text) : NULL + }; +} + +static void drop_optional(void *value) { + OptionalText *text = value; + if (text->present) dever_rt_v1_text_release(text->text); + *text = (OptionalText){0}; +} + +static uint8_t equal_optional(const void *left, const void *right) { + const OptionalText *a = left; + const OptionalText *b = right; + return a->present == b->present && + (!a->present || dever_rt_v1_text_equal(a->text, b->text)); +} + +static const DeverRtType optional_type = { + sizeof(OptionalText), _Alignof(OptionalText), + clone_optional, drop_optional, equal_optional, NULL +}; + +/* Preserve cleanup on an assertion failure so the first diagnostic identifies + * the broken contract, not a second error caused by the test's own resources. */ +#define CHECK(condition) do { if (!(condition)) { result = __LINE__; goto cleanup; } } while (0) + +static int error_kinds(void) { + int result = 0; + DeverRtBuffer buffer = {0}; + Fault fault = {0}; + void *message = NULL; + unsigned before_packs = packs; + unsigned before_moves = moves; + unsigned before_drops = drops; + for (uint32_t kind = 0; kind < 10; ++kind) { + CHECK(new_text("original database diagnostic: \xe6\x95\xb0\xe6\x8d\xae", &message)); + CHECK(dever_rt_v1_db_error(kind, message, &errors, &fault, &buffer) == 1); + CHECK(buffer.ptr == NULL && buffer.len == 0 && fault.kind == kind); + dever_rt_v1_text_release(message); + message = NULL; + /* pack's borrowed handle is retainable after the bridge returns. */ + CHECK(text_is(fault.message, "original database diagnostic: \xe6\x95\xb0\xe6\x8d\xae")); + CHECK(new_text("rollback failed", &message)); + append_cause(&fault, message); + append_cause(&fault, message); + dever_rt_v1_text_release(message); + message = NULL; + CHECK(fault.kind == kind && text_is(fault.cause, + "caused by: rollback failed; caused by: rollback failed")); + drop_fault(&fault); + } + CHECK(packs == before_packs + 10 && moves == before_moves + 10 && + drops == before_drops + 10); +cleanup: + dever_rt_v1_text_release(message); + if (fault.message || fault.cause) drop_fault(&fault); + release_buffer(&buffer); + return result; +} + +static int related_owners(void) { + int result = 0; + DeverRtBuffer buffer = {0}; + OptionalText source = {1, NULL}, extracted = {0}, absent = {0}; + void *related = NULL, *alias = NULL, *loaded_null = NULL, *second_null = NULL; + CHECK(new_text("related-owned", &source.text)); + CHECK(dever_rt_v1_db_related_new(&optional_type, &source, &related, &buffer) == 0); + alias = dever_rt_v1_db_related_retain(related); + drop_optional(&source); + dever_rt_v1_db_related_release(related); + related = NULL; + CHECK(dever_rt_v1_db_related_get(alias, &extracted, &buffer) == 0); + dever_rt_v1_db_related_release(alias); + alias = NULL; + CHECK(extracted.present == 1 && text_is(extracted.text, "related-owned")); + CHECK(dever_rt_v1_db_related_new(&optional_type, &absent, &loaded_null, &buffer) == 0); + CHECK(dever_rt_v1_db_related_new(&optional_type, &absent, &second_null, &buffer) == 0); + CHECK(loaded_null != NULL && dever_rt_v1_db_related_equal(loaded_null, second_null) == 1); + CHECK(dever_rt_v1_db_related_equal(NULL, loaded_null) == 0); + CHECK(dever_rt_v1_db_related_equal(NULL, NULL) == 1); + drop_optional(&extracted); + CHECK(dever_rt_v1_db_related_get(loaded_null, &extracted, &buffer) == 0); + CHECK(extracted.present == 0 && extracted.text == NULL); +cleanup: + drop_optional(&source); + drop_optional(&extracted); + dever_rt_v1_db_related_release(related); + dever_rt_v1_db_related_release(alias); + dever_rt_v1_db_related_release(loaded_null); + dever_rt_v1_db_related_release(second_null); + release_buffer(&buffer); + return result; +} + +static int invalid_destinations(void) { + int result = 0; + DeverRtBuffer buffer = {0}; + Fault fault = {37, NULL, NULL}; + int64_t output = 41; + unsigned before = packs; + CHECK(dever_rt_v1_db_cursor_size(0, 7, NULL, &errors, &fault, &buffer) == 2); + CHECK(buffer_is(&buffer, "invalid ABI output pointer") && packs == before && fault.kind == 37); + release_buffer(&buffer); + _Alignas(int64_t) unsigned char storage[sizeof(int64_t) + 1]; + memset(storage, 53, sizeof(storage)); + CHECK(dever_rt_v1_db_stream_capacity(0, 7, (int64_t *)(storage + 1), &errors, &fault, &buffer) == 2); + CHECK(packs == before && fault.kind == 37); + for (size_t i = 0; i < sizeof(storage); ++i) CHECK(storage[i] == 53); + release_buffer(&buffer); + CHECK(dever_rt_v1_db_cursor_size(0, 7, &output, &errors, NULL, &buffer) == 2); + CHECK(output == 41 && packs == before && buffer_is(&buffer, "invalid database fault output")); + release_buffer(&buffer); + CHECK(dever_rt_v1_db_cursor_size(0, 7, &output, &errors, &fault, &buffer) == 1); + CHECK(output == 41 && fault.kind == 8 && text_is(fault.message, "cursor size must be between 1 and 7")); + drop_fault(&fault); + CHECK(dever_rt_v1_db_stream_capacity(0, 7, &output, &errors, &fault, &buffer) == 1); + CHECK(output == 41 && fault.kind == 8 && text_is(fault.message, "stream buffer size must be between 1 and 7")); + drop_fault(&fault); + CHECK(dever_rt_v1_db_cursor_size(3, 7, &output, &errors, &fault, &buffer) == 0 && output == 3); + CHECK(packs == before + 2 && buffer.ptr == NULL && buffer.len == 0); +cleanup: + if (fault.message || fault.cause) drop_fault(&fault); + release_buffer(&buffer); + return result; +} + +static int invalid_constructors(void) { + int result = 0; + DeverRtBuffer buffer = {0}; + Fault primary = {6, NULL, NULL}; + DeverRtAsyncOp *operation = NULL; + void *session = (void *)(uintptr_t)47; + unsigned before_moves = moves, before_drops = drops; + CHECK(new_text("primary-owned", &primary.message)); + operation = dever_rt_v1_db_rollback_take(NULL, &primary, &fault_type, &errors, &buffer); + CHECK(operation == NULL && buffer_is(&buffer, "invalid ABI handle")); + CHECK(moves == before_moves && drops == before_drops && primary.kind == 6 && text_is(primary.message, "primary-owned")); + release_buffer(&buffer); + + const DeverRtDbBinding bad_flag = {NULL, {(const uint8_t *)"fixture", 7}, 2}; + CHECK(dever_rt_v1_db_session_new(&bad_flag, 1, NULL, 0, NULL, &buffer) == 2); + CHECK(buffer_is(&buffer, "invalid ABI output pointer")); + release_buffer(&buffer); + /* Metadata rejects before looking for config/setting.json or opening pools. */ + CHECK(dever_rt_v1_db_session_new(&bad_flag, 1, NULL, 0, &session, &buffer) == 2); + CHECK(session == (void *)(uintptr_t)47 && buffer_is(&buffer, "invalid database ABI boolean")); + release_buffer(&buffer); + _Alignas(DeverRtDbBinding) unsigned char unaligned[sizeof(DeverRtDbBinding) + 1]; + CHECK(dever_rt_v1_db_session_new((const DeverRtDbBinding *)(unaligned + 1), 1, NULL, 0, &session, &buffer) == 2); + CHECK(session == (void *)(uintptr_t)47 && buffer_is(&buffer, "invalid database ABI array")); +cleanup: + dever_rt_v1_async_op_release(operation); + drop_fault(&primary); + release_buffer(&buffer); + return result; +} + +int32_t dever_test_run(void) { + packs = moves = drops = 0; + int result = error_kinds(); + if (result == 0) result = related_owners(); + if (result == 0) result = invalid_destinations(); + if (result == 0) result = invalid_constructors(); + return result; +} diff --git a/test/native-runtime-abi/driver-fixture.c b/test/native-runtime-abi/driver-fixture.c new file mode 100644 index 0000000..8b203fa --- /dev/null +++ b/test/native-runtime-abi/driver-fixture.c @@ -0,0 +1,42 @@ +#include "../../crates/dever-backend-bridge/include/runtime.h" +#include <pthread.h> +#include <stdlib.h> + +static int initialized; +static size_t calls; +static pthread_key_t cache_key; +static pthread_once_t cache_once = PTHREAD_ONCE_INIT; + +#if DEVER_LEDGER_SCENARIO == 1 +static void *volatile leaked; +#endif + +static void create_cache_key(void) { + if (pthread_key_create(&cache_key, free) != 0) abort(); +} + +uint32_t dever_rt_v1_process_init(void) { + initialized = 1; + return DEVER_RT_OK; +} + +int dever_test_run(void) { + if (!initialized) return 9; + ++calls; + /* Deterministically initialize TLS after warmup, as a later full queue can. */ + if (calls > 1) { + if (pthread_once(&cache_once, create_cache_key) != 0) abort(); + if (pthread_getspecific(cache_key) == NULL) { + void *cache = malloc(48); + if (cache == NULL || pthread_setspecific(cache_key, cache) != 0) abort(); + } + } +#if DEVER_LEDGER_SCENARIO == 1 + /* Keep the allocation observable to the optimizer and deliberately leak it. */ + leaked = malloc(1); + if (leaked == NULL) abort(); +#elif DEVER_LEDGER_SCENARIO == 2 + if (calls > 1) return 7; +#endif + return 0; +} diff --git a/test/native-runtime-abi/driver.c b/test/native-runtime-abi/driver.c new file mode 100644 index 0000000..4aaaa25 --- /dev/null +++ b/test/native-runtime-abi/driver.c @@ -0,0 +1,251 @@ +#include "runtime.h" +#include <limits.h> +#include <math.h> +#include <signal.h> +#include <stddef.h> +#include <stdio.h> +#include <string.h> + +#ifdef DEVER_LLVM_ABI +extern int32_t dever_abi_probe(void); +#endif + +/* This harness consumes only the canonical ABI header, never Rust layouts. */ +static int expect_buffer(DeverRtBuffer *buffer, const char *expected) { + const size_t len = strlen(expected); + const int okay = buffer->len == len && + (len == 0 || (buffer->ptr != NULL && memcmp(buffer->ptr, expected, len) == 0)); + dever_rt_v1_buffer_free(buffer->ptr, buffer->len); + buffer->ptr = NULL; + buffer->len = 0; + return okay; +} + +#define EXPECT(condition) do { \ + if (!(condition)) { fprintf(stderr, "ABI assertion failed at line %d: %s\n", __LINE__, #condition); return 1; } \ +} while (0) + +typedef struct { + void *label; + int64_t count; +} FixtureRecord; + +static void text_clone(const void *source, void *destination) { + *(void **)destination = dever_rt_v1_text_retain(*(void *const *)source); +} + +static void text_drop(void *value) { + dever_rt_v1_text_release(*(void **)value); +} + +static uint8_t text_equal(const void *left, const void *right) { + return dever_rt_v1_text_equal(*(void *const *)left, *(void *const *)right); +} + +static uint64_t text_hash(const void *value) { + return dever_rt_v1_text_hash(*(void *const *)value); +} + +static void record_clone(const void *source, void *destination) { + const FixtureRecord *record = source; + *(FixtureRecord *)destination = (FixtureRecord){ + dever_rt_v1_text_retain(record->label), record->count + }; +} + +static void record_drop(void *value) { + dever_rt_v1_text_release(((FixtureRecord *)value)->label); +} + +static uint8_t record_equal(const void *left, const void *right) { + const FixtureRecord *first = left; + const FixtureRecord *second = right; + return first->count == second->count && + dever_rt_v1_text_equal(first->label, second->label); +} + +static const DeverRtType TEXT_TYPE = { + sizeof(void *), _Alignof(void *), text_clone, text_drop, text_equal, text_hash +}; +static const DeverRtType RECORD_TYPE = { + sizeof(FixtureRecord), _Alignof(FixtureRecord), + record_clone, record_drop, record_equal, NULL +}; + +static int managed_values(void) { + DeverRtBuffer error = {NULL, 0}; + void *label = NULL; + void *key = NULL; + void *second_key = NULL; + EXPECT(dever_rt_v1_text_new((const uint8_t *)"label", 5, &label, &error) == DEVER_RT_OK); + EXPECT(dever_rt_v1_text_new((const uint8_t *)"first", 5, &key, &error) == DEVER_RT_OK); + EXPECT(dever_rt_v1_text_new((const uint8_t *)"second", 6, &second_key, &error) == DEVER_RT_OK); + /* These source views are borrowed; only the ABI's clones own row fields. */ + const FixtureRecord source = {label, 42}; + const void *rows[] = {&source, &source}; + void *list = NULL; + void *changed = NULL; + int64_t size = 0; + EXPECT(dever_rt_v1_list_new(&RECORD_TYPE, rows, 2, &list, &error) == DEVER_RT_OK); + EXPECT(dever_rt_v1_list_append_take(dever_rt_v1_list_retain(list), &source, &changed, &error) == DEVER_RT_OK); + EXPECT(dever_rt_v1_list_length(list, &size, &error) == DEVER_RT_OK && size == 2); + EXPECT(dever_rt_v1_list_length(changed, &size, &error) == DEVER_RT_OK && size == 3); + FixtureRecord extracted; + uint8_t present = 0; + EXPECT(dever_rt_v1_list_first_take(dever_rt_v1_list_retain(list), &extracted, &present, &error) == DEVER_RT_OK && present); + EXPECT(record_equal(&source, &extracted)); + record_drop(&extracted); + void *cursor = NULL; + EXPECT(dever_rt_v1_list_cursor_take(changed, &cursor, &error) == DEVER_RT_OK); + EXPECT(dever_rt_v1_list_cursor_next(cursor, &extracted, &present, &error) == DEVER_RT_OK && present); + EXPECT(record_equal(&source, &extracted)); + record_drop(&extracted); + /* Early cursor release must destroy the unvisited rows too. */ + dever_rt_v1_list_cursor_release(cursor); + dever_rt_v1_list_release(list); + + const void *keys[] = {&key, &second_key}; + void *map = NULL; + EXPECT(dever_rt_v1_map_new(&TEXT_TYPE, &RECORD_TYPE, keys, rows, 2, &map, &error) == DEVER_RT_OK); + EXPECT(dever_rt_v1_map_get_take(dever_rt_v1_map_retain(map), &key, &extracted, &present, &error) == DEVER_RT_OK && present); + EXPECT(record_equal(&source, &extracted)); + record_drop(&extracted); + const FixtureRecord replacement = {label, 99}; + EXPECT(dever_rt_v1_map_put_take(dever_rt_v1_map_retain(map), &key, &replacement, &changed, &error) == DEVER_RT_OK); + EXPECT(!dever_rt_v1_map_equal(map, changed)); + EXPECT(dever_rt_v1_map_get_take(changed, &key, &extracted, &present, &error) == DEVER_RT_OK && present); + EXPECT(extracted.count == 99); + record_drop(&extracted); + EXPECT(dever_rt_v1_map_cursor_take(map, &cursor, &error) == DEVER_RT_OK); + void *extracted_key = NULL; + EXPECT(dever_rt_v1_map_cursor_next(cursor, &extracted_key, &extracted, &present, &error) == DEVER_RT_OK && present); + EXPECT(dever_rt_v1_text_equal(key, extracted_key)); + text_drop(&extracted_key); + record_drop(&extracted); + dever_rt_v1_map_cursor_release(cursor); + const void *duplicate_keys[] = {&key, &key}; + EXPECT(dever_rt_v1_map_new(&TEXT_TYPE, &RECORD_TYPE, duplicate_keys, rows, 2, &map, &error) == DEVER_RT_RUNTIME_ERROR); + EXPECT(expect_buffer(&error, "duplicate Map key")); + + void *text_list = NULL; + void *separator = NULL; + EXPECT(dever_rt_v1_text_new((const uint8_t *)"a", 1, &separator, &error) == DEVER_RT_OK); + EXPECT(dever_rt_v1_text_split(label, separator, &TEXT_TYPE, &text_list, &error) == DEVER_RT_OK); + EXPECT(dever_rt_v1_list_length(text_list, &size, &error) == DEVER_RT_OK && size == 2); + dever_rt_v1_list_release(text_list); + dever_rt_v1_text_release(separator); + void *bytes = NULL; + void *slice = NULL; + void *decoded = NULL; + EXPECT(dever_rt_v1_bytes_from_text(label, &bytes, &error) == DEVER_RT_OK); + EXPECT(dever_rt_v1_bytes_slice(bytes, 1, 4, &slice, &error) == DEVER_RT_OK); + EXPECT(dever_rt_v1_bytes_to_text_handle(slice, &decoded, &error) == DEVER_RT_OK); + EXPECT(dever_rt_v1_text_export(decoded, &error) == DEVER_RT_OK); + EXPECT(expect_buffer(&error, "abe")); + dever_rt_v1_text_release(decoded); + dever_rt_v1_bytes_release(slice); + dever_rt_v1_bytes_release(bytes); + dever_rt_v1_text_release(label); + dever_rt_v1_text_release(key); + dever_rt_v1_text_release(second_key); + return 0; +} + +int main(void) { +#ifdef SIGPIPE + EXPECT(signal(SIGPIPE, SIG_DFL) != SIG_ERR); +#endif + EXPECT(dever_rt_v1_process_init() == DEVER_RT_OK); +#ifdef SIGPIPE + EXPECT(raise(SIGPIPE) == 0); +#endif + const char *arguments[] = {"suite", "1", NULL}; + uint64_t selected = 99; + EXPECT(dever_rt_v1_process_test_index(2, arguments, 2, &selected) == DEVER_RT_OK); + EXPECT(selected == 1); + const char *invalid_indices[] = {"", "-1", "+1", " 1", "1x", "2", "18446744073709551616"}; + for (size_t index = 0; index < sizeof(invalid_indices) / sizeof(invalid_indices[0]); index++) { + arguments[1] = invalid_indices[index]; + selected = 99; + EXPECT(dever_rt_v1_process_test_index(2, arguments, 2, &selected) == DEVER_RT_INVALID_INPUT); + EXPECT(selected == 99); + } + EXPECT(dever_rt_v1_process_test_index(1, arguments, 2, &selected) == DEVER_RT_INVALID_INPUT); + EXPECT(dever_rt_v1_process_test_index(2, NULL, 2, &selected) == DEVER_RT_INVALID_INPUT); + EXPECT(dever_rt_v1_process_stderr(NULL, 0) == DEVER_RT_OK); + EXPECT(dever_rt_v1_process_stderr(NULL, 1) == DEVER_RT_INVALID_INPUT); + EXPECT(dever_rt_v1_process_error(NULL) == DEVER_RT_INVALID_INPUT); + DeverRtBuffer buffer = {NULL, 0}; + int64_t integer = 0; + double floating = 0.0; + DeverRtDecimal seven, two, zero, decimal; + EXPECT(dever_rt_v1_version() == 1); + EXPECT(dever_rt_v1_int_add(40, 2, &integer, &buffer) == DEVER_RT_OK && integer == 42); + EXPECT(buffer.ptr == NULL && buffer.len == 0); + EXPECT(dever_rt_v1_int_sub(8, 3, &integer, &buffer) == DEVER_RT_OK && integer == 5); + EXPECT(dever_rt_v1_int_mul(7, 6, &integer, &buffer) == DEVER_RT_OK && integer == 42); + EXPECT(dever_rt_v1_int_div(-7, 3, &integer, &buffer) == DEVER_RT_OK && integer == -2); + EXPECT(dever_rt_v1_int_rem(-7, 3, &integer, &buffer) == DEVER_RT_OK && integer == -1); + EXPECT(dever_rt_v1_int_neg(-42, &integer, &buffer) == DEVER_RT_OK && integer == 42); + EXPECT(dever_rt_v1_int_add(INT64_MAX, 1, &integer, &buffer) == DEVER_RT_RUNTIME_ERROR); + EXPECT(expect_buffer(&buffer, "Int overflow")); + EXPECT(dever_rt_v1_int_div(1, 0, &integer, &buffer) == DEVER_RT_RUNTIME_ERROR); + EXPECT(expect_buffer(&buffer, "Int division by zero")); + EXPECT(dever_rt_v1_int_neg(INT64_MIN, &integer, &buffer) == DEVER_RT_RUNTIME_ERROR); + EXPECT(expect_buffer(&buffer, "Int overflow")); + EXPECT(dever_rt_v1_int_add(1, 2, NULL, &buffer) == DEVER_RT_INVALID_INPUT); + EXPECT(buffer.ptr != NULL && buffer.len != 0); + dever_rt_v1_buffer_free(buffer.ptr, buffer.len); + buffer = (DeverRtBuffer){NULL, 0}; + EXPECT(dever_rt_v1_int_add(1, 2, &integer, NULL) == DEVER_RT_INVALID_INPUT); + + EXPECT(dever_rt_v1_float_div(0.0, 0.0, &floating, &buffer) == DEVER_RT_OK && isnan(floating)); + EXPECT(dever_rt_v1_float_div(1.0, 0.0, &floating, &buffer) == DEVER_RT_OK && isinf(floating)); + EXPECT(dever_rt_v1_float_add(1.5, 2.5, &floating, &buffer) == DEVER_RT_OK && floating == 4.0); + EXPECT(dever_rt_v1_float_rem(7.0, 3.0, &floating, &buffer) == DEVER_RT_OK && floating == 1.0); + EXPECT(dever_rt_v1_float_neg(0.0, &floating, &buffer) == DEVER_RT_OK && signbit(floating)); + EXPECT(dever_rt_v1_float_to_text(3.0, &buffer) == DEVER_RT_OK); + EXPECT(expect_buffer(&buffer, "3")); + + EXPECT(dever_rt_v1_decimal_from_int(7, &seven, &buffer) == DEVER_RT_OK); + EXPECT(dever_rt_v1_decimal_from_int(2, &two, &buffer) == DEVER_RT_OK); + EXPECT(dever_rt_v1_decimal_from_int(0, &zero, &buffer) == DEVER_RT_OK); + EXPECT(dever_rt_v1_decimal_div(seven.low, seven.high, two.low, two.high, &decimal, &buffer) == DEVER_RT_OK); + EXPECT(dever_rt_v1_decimal_to_text(decimal.low, decimal.high, &buffer) == DEVER_RT_OK); + EXPECT(expect_buffer(&buffer, "3.5")); + EXPECT(dever_rt_v1_decimal_round(decimal.low, decimal.high, 0, &decimal, &buffer) == DEVER_RT_OK); + EXPECT(dever_rt_v1_decimal_to_text(decimal.low, decimal.high, &buffer) == DEVER_RT_OK); + EXPECT(expect_buffer(&buffer, "4")); + EXPECT(dever_rt_v1_decimal_div(seven.low, seven.high, zero.low, zero.high, &decimal, &buffer) == DEVER_RT_RUNTIME_ERROR); + EXPECT(expect_buffer(&buffer, "Decimal division by zero")); + EXPECT(dever_rt_v1_int_to_text(INT64_MIN, &buffer) == DEVER_RT_OK); + EXPECT(expect_buffer(&buffer, "-9223372036854775808")); + + const uint8_t text[] = {0x61, 0xe4, 0xb8, 0xad}; + EXPECT(dever_rt_v1_text_copy_utf8(text, sizeof(text), &buffer) == DEVER_RT_OK); + EXPECT(expect_buffer(&buffer, "a\xe4\xb8\xad")); + const uint8_t bytes[] = {0, 255, 42}; + EXPECT(dever_rt_v1_bytes_copy(bytes, sizeof(bytes), &buffer) == DEVER_RT_OK); + EXPECT(buffer.len == sizeof(bytes) && memcmp(buffer.ptr, bytes, sizeof(bytes)) == 0); + dever_rt_v1_buffer_free(buffer.ptr, buffer.len); + buffer = (DeverRtBuffer){NULL, 0}; + EXPECT(dever_rt_v1_bytes_to_text(text, sizeof(text), &buffer) == DEVER_RT_OK); + EXPECT(expect_buffer(&buffer, "a\xe4\xb8\xad")); + const uint8_t invalid[] = {255}; + EXPECT(dever_rt_v1_bytes_to_text(invalid, sizeof(invalid), &buffer) == DEVER_RT_RUNTIME_ERROR); + EXPECT(buffer.ptr != NULL && buffer.len != 0); + dever_rt_v1_buffer_free(buffer.ptr, buffer.len); + buffer = (DeverRtBuffer){NULL, 0}; + EXPECT(dever_rt_v1_text_copy_utf8(NULL, 0, &buffer) == DEVER_RT_OK); + EXPECT(buffer.ptr == NULL && buffer.len == 0); + EXPECT(dever_rt_v1_bytes_copy(NULL, 1, &buffer) == DEVER_RT_INVALID_INPUT); + EXPECT(buffer.ptr != NULL && buffer.len != 0); + dever_rt_v1_buffer_free(buffer.ptr, buffer.len); + dever_rt_v1_buffer_free(NULL, 0); +#ifdef DEVER_LLVM_ABI + EXPECT(dever_abi_probe() == 0); +#endif + EXPECT(managed_values() == 0); + puts("native runtime ABI verified"); + return 0; +} diff --git a/test/native-runtime-abi/log-driver.c b/test/native-runtime-abi/log-driver.c new file mode 100644 index 0000000..96bbd98 --- /dev/null +++ b/test/native-runtime-abi/log-driver.c @@ -0,0 +1,8 @@ +#include "runtime.h" + +/* Link with managed-driver.c: the first flush initializes the logger; every + * later flush must finish without outstanding per-call or deferred owners. */ +int dever_test_run(void) { + dever_rt_v1_log_flush(); + return 0; +} diff --git a/test/native-runtime-abi/managed-driver.c b/test/native-runtime-abi/managed-driver.c new file mode 100644 index 0000000..659f31a --- /dev/null +++ b/test/native-runtime-abi/managed-driver.c @@ -0,0 +1,116 @@ +#include "../../crates/dever-backend-bridge/include/runtime.h" +#include <stddef.h> +#include <stdio.h> +#include <stdlib.h> +#include <stdatomic.h> +#include <pthread.h> +#include <string.h> + +extern int dever_test_run(void); + +#ifndef DEVER_TEST_ITERATIONS +#define DEVER_TEST_ITERATIONS 64 +#endif + +/* Link-time wrappers observe allocations in this fixture's static runtime. + * This is a bounded ownership regression, not a general memory profiler. */ +static _Atomic long long live_allocations; + +void *__real_malloc(size_t size); +void *__real_calloc(size_t count, size_t size); +void *__real_realloc(void *pointer, size_t size); +void __real_free(void *pointer); +int __real_posix_memalign(void **pointer, size_t alignment, size_t size); +char *__real_realpath(const char *path, char *resolved); + +/* glibc allocates this returned owner inside its DSO, bypassing --wrap=malloc. + * Rust canonicalize frees it through the executable's wrapped free. */ +char *__wrap_realpath(const char *path, char *resolved) { + char *pointer = __real_realpath(path, resolved); + atomic_fetch_add(&live_allocations, pointer != NULL && resolved == NULL); + return pointer; +} + +void *__wrap_malloc(size_t size) { + void *pointer = __real_malloc(size); + atomic_fetch_add(&live_allocations, pointer != NULL); + return pointer; +} + +void *__wrap_calloc(size_t count, size_t size) { + void *pointer = __real_calloc(count, size); + atomic_fetch_add(&live_allocations, pointer != NULL); + return pointer; +} + +void *__wrap_realloc(void *pointer, size_t size) { + int had_pointer = pointer != NULL; + void *resized = __real_realloc(pointer, size); + if (!had_pointer && resized != NULL) { + atomic_fetch_add(&live_allocations, 1); + } else if (had_pointer && resized == NULL && size == 0) { + atomic_fetch_sub(&live_allocations, 1); + } + return resized; +} + +void __wrap_free(void *pointer) { + atomic_fetch_sub(&live_allocations, pointer != NULL); + __real_free(pointer); +} + +int __wrap_posix_memalign(void **pointer, size_t alignment, size_t size) { + int status = __real_posix_memalign(pointer, alignment, size); + atomic_fetch_add(&live_allocations, status == 0 && *pointer != NULL); + return status; +} + +/* Standard-library queue caches live until the host thread exits. Join before + * measuring so lazy TLS initialization cannot masquerade as a per-call leak. */ +static void *invoke_kernel(void *result) { + *(int *)result = dever_test_run(); + return NULL; +} + +static int run_kernel(void) { + pthread_t thread; + int result = 0; + int status = pthread_create(&thread, NULL, invoke_kernel, &result); + if (status == 0) { + status = pthread_join(thread, NULL); + } + if (status != 0) { + fprintf(stderr, "managed kernel thread failed: %s\n", strerror(status)); + exit(EXIT_FAILURE); + } + return result; +} + +int main(void) { + /* This C process owns signal policy just like the generated process main. */ + if (dever_rt_v1_process_init() != DEVER_RT_OK) { + fprintf(stderr, "could not initialize the managed kernel process\n"); + return 1; + } + /* Permit process-wide lazy initialization, then require no per-call leak. */ + int result = run_kernel(); + if (result != 0) { + fprintf(stderr, "managed kernel returned an unexpected result: %d\n", result); + return 1; + } + long long baseline = atomic_load(&live_allocations); + for (size_t remaining = DEVER_TEST_ITERATIONS; remaining != 0; --remaining) { + result = run_kernel(); + if (result != 0) { + fprintf(stderr, "managed kernel changed its result on repeated execution: %d\n", result); + return 1; + } + long long current = atomic_load(&live_allocations); + if (current != baseline) { + fprintf(stderr, "managed kernel allocation imbalance: %lld\n", + current - baseline); + return 1; + } + } + return 0; +} diff --git a/test/native-runtime-abi/protocol-driver.c b/test/native-runtime-abi/protocol-driver.c new file mode 100644 index 0000000..5a7e528 --- /dev/null +++ b/test/native-runtime-abi/protocol-driver.c @@ -0,0 +1,183 @@ +#include "runtime.h" + +#include <stdint.h> +#include <stdlib.h> +#include <string.h> + +/* These deliberately invalid domain inputs fail before opening a connection. + * Retrying the same operation after invalid output slots proves poll did not + * advance the future or mark it completed on the rejected call. */ +static unsigned frames_created; +static unsigned frames_destroyed; + +static void release_buffer(DeverRtBuffer *buffer) { + dever_rt_v1_buffer_free(buffer->ptr, buffer->len); + buffer->ptr = NULL; + buffer->len = 0; +} + +static int message_is(const DeverRtBuffer *buffer, const char *expected) { + size_t length = strlen(expected); + return buffer->len == length && buffer->ptr && + memcmp(buffer->ptr, expected, length) == 0; +} + +static int text(const char *value, void **out) { + DeverRtBuffer error = {0}; + uint32_t status = dever_rt_v1_text_new((const uint8_t *)value, + (uint64_t)strlen(value), out, &error); + release_buffer(&error); + return status == DEVER_RT_OK; +} + +static int probe_slots(DeverRtPollState *state, DeverRtAsyncOp *operation, + const char *runtime_message) { + if (!operation) return 1; + DeverRtBuffer error = {0}; + uint8_t present = 37; + int64_t fault = 41; + void *output = (void *)(uintptr_t)47; + uint32_t status = dever_rt_v1_async_op_poll(state, operation, NULL, &present, + &fault, &error); + int valid = status == DEVER_RT_ASYNC_INVALID_INPUT && + message_is(&error, "invalid protocol handle output") && + present == 37 && fault == 41; + release_buffer(&error); + if (!valid) return 2; + + _Alignas(void *) unsigned char unaligned[sizeof(void *) + 1]; + memset(unaligned, 53, sizeof(unaligned)); + status = dever_rt_v1_async_op_poll(state, operation, unaligned + 1, + &present, &fault, &error); + valid = status == DEVER_RT_ASYNC_INVALID_INPUT && + message_is(&error, "invalid protocol handle output") && + present == 37 && fault == 41; + for (size_t index = 0; index < sizeof(unaligned); ++index) { + valid = valid && unaligned[index] == 53; + } + release_buffer(&error); + if (!valid) return 3; + + status = dever_rt_v1_async_op_poll(state, operation, &output, &present, + &fault, &error); + valid = status == DEVER_RT_ASYNC_READY_RUNTIME_ERROR && + message_is(&error, runtime_message) && + output == (void *)(uintptr_t)47 && present == 37 && fault == 41; + release_buffer(&error); + if (!valid) return 4; + + status = dever_rt_v1_async_op_poll(state, operation, &output, &present, + &fault, &error); + valid = status == DEVER_RT_ASYNC_INVALID_INPUT && + message_is(&error, "async operation is not polling"); + release_buffer(&error); + return valid ? 0 : 5; +} + +static int probe_http(DeverRtPollState *state) { + void *host = NULL, *method = NULL, *target = NULL, *empty = NULL; + void *headers = NULL, *body = NULL, *request = NULL; + DeverRtAsyncOp *operation = NULL; + DeverRtBuffer error = {0}; + int result = 10; + if (!text("127.0.0.1", &host) || !text("GET", &method) || + !text("/", &target) || !text("", &empty)) goto cleanup; + if (dever_rt_v1_http_headers_new(&headers, &error) != DEVER_RT_OK) goto cleanup; + if (dever_rt_v1_bytes_from_text(empty, &body, &error) != DEVER_RT_OK) goto cleanup; + DeverRtHttpRequestFields fields = {method, target, headers, body}; + if (dever_rt_v1_http_request_new(&fields, &request, &error) != DEVER_RT_OK) goto cleanup; + DeverRtHttpLimits limits = {1, 0, 100, 1, NULL}; + operation = dever_rt_v1_async_http_send(host, 0, request, &limits); + /* Constructor ownership survives releasing every borrowed input. */ + dever_rt_v1_http_request_release(request); + request = NULL; + dever_rt_v1_text_release(host); + host = NULL; + result = probe_slots(state, operation, "HTTP header_bytes must be at least 8192"); +cleanup: + dever_rt_v1_async_op_release(operation); + dever_rt_v1_http_request_release(request); + dever_rt_v1_http_headers_release(headers); + dever_rt_v1_bytes_release(body); + dever_rt_v1_text_release(host); + dever_rt_v1_text_release(method); + dever_rt_v1_text_release(target); + dever_rt_v1_text_release(empty); + release_buffer(&error); + return result; +} + +static int probe_websocket_open(DeverRtPollState *state) { + void *url = NULL, *tls = NULL; + DeverRtAsyncOp *operation = NULL; + DeverRtBuffer error = {0}; + int result = 20; + if (!text("http://fixture.invalid/", &url)) goto cleanup; + if (dever_rt_v1_tls_system(&tls, &error) != DEVER_RT_OK) goto cleanup; + DeverRtWsLimits limits = {128, 100, 100}; + operation = dever_rt_v1_async_ws_open(url, tls, &limits); + dever_rt_v1_text_release(url); + url = NULL; + dever_rt_v1_client_tls_release(tls); + tls = NULL; + result = probe_slots(state, operation, "WebSocket URL must use ws or wss"); +cleanup: + dever_rt_v1_async_op_release(operation); + dever_rt_v1_text_release(url); + dever_rt_v1_client_tls_release(tls); + release_buffer(&error); + return result; +} + +typedef struct { + DeverRtPollState *state; + int64_t *output; + uint8_t completed; +} ProbeFrame; + +static void move_int(void *source, void *destination) { + *(int64_t *)destination = *(int64_t *)source; + *(int64_t *)source = 0; +} + +static void drop_int(void *value) { *(int64_t *)value = 0; } + +static void *create(DeverRtPollState *state, const void *input, void *output, void *fault) { + (void)input; + (void)fault; + ProbeFrame *frame = calloc(1, sizeof(*frame)); + if (!frame) return NULL; + frame->state = state; + frame->output = output; + ++frames_created; + return frame; +} + +static void resume(void *raw) { + ProbeFrame *frame = raw; + int result = probe_http(frame->state); + if (result == 0) result = probe_websocket_open(frame->state); + *frame->output = result; + frame->completed = 1; + dever_rt_v1_async_complete(frame->state, DEVER_RT_ASYNC_READY_OK); +} + +static void destroy(void *raw) { ++frames_destroyed; free(raw); } +static uint8_t done(void *raw) { return ((ProbeFrame *)raw)->completed; } + +int main(void) { + static const DeverRtOwnedType integer = { + sizeof(int64_t), _Alignof(int64_t), move_int, drop_int, 1 + }; + static const DeverRtAsyncFunction function = { + &integer, &integer, create, resume, destroy, done, 1 + }; + int64_t output = -1, fault = -1; + DeverRtBuffer error = {0}; + uint32_t status = dever_rt_v1_async_root(&function, NULL, &output, &fault, &error); + int valid = status == DEVER_RT_ASYNC_READY_OK && output == 0 && + fault == -1 && error.ptr == NULL && error.len == 0 && + frames_created == 1 && frames_destroyed == 1; + release_buffer(&error); + return valid ? 0 : 1; +} diff --git a/test/native-runtime-abi/resource-driver.c b/test/native-runtime-abi/resource-driver.c new file mode 100644 index 0000000..0bf3ace --- /dev/null +++ b/test/native-runtime-abi/resource-driver.c @@ -0,0 +1,204 @@ +#include "runtime.h" +#include <stdint.h> +#include <stdio.h> +#include <string.h> + +#define EXPECT(condition) do { \ + if (!(condition)) { \ + fprintf(stderr, "resource ABI assertion failed at line %d: %s\n", __LINE__, #condition); \ + return 1; \ + } \ +} while (0) + +typedef struct { + uint64_t kind; + void *value; +} ReadEvent; + +static unsigned event_retains; +static unsigned event_drops; + +static void event_clone(const void *source, void *destination) { + const ReadEvent *event = source; + ReadEvent *copy = destination; + copy->kind = event->kind; + copy->value = event->kind == 1 + ? dever_rt_v1_bytes_retain(event->value) + : dever_rt_v1_text_retain(event->value); + ++event_retains; +} + +static void event_drop(void *value) { + ReadEvent *event = value; + if (event->kind == 1) { + dever_rt_v1_bytes_release(event->value); + } else { + dever_rt_v1_text_release(event->value); + } + ++event_drops; +} + +static uint8_t event_equal(const void *left, const void *right) { + const ReadEvent *first = left; + const ReadEvent *second = right; + if (first->kind != second->kind) return 0; + return first->kind == 1 + ? dever_rt_v1_bytes_equal(first->value, second->value) + : dever_rt_v1_text_equal(first->value, second->value); +} + +static void event_chunk(const void *bytes, void *out) { + *(ReadEvent *)out = (ReadEvent){1, dever_rt_v1_bytes_retain(bytes)}; + ++event_retains; +} + +static void event_failed(const void *text, void *out) { + *(ReadEvent *)out = (ReadEvent){2, dever_rt_v1_text_retain(text)}; + ++event_retains; +} + +static const DeverRtType EVENT_TYPE = { + sizeof(ReadEvent), _Alignof(ReadEvent), event_clone, event_drop, event_equal, NULL +}; +static const DeverRtReadEvent READ_EVENT = { + &EVENT_TYPE, event_chunk, event_failed +}; + +static int expect_buffer(DeverRtBuffer *buffer, const char *expected) { + size_t length = strlen(expected); + int equal = buffer->len == length && + (length == 0 || (buffer->ptr && memcmp(buffer->ptr, expected, length) == 0)); + dever_rt_v1_buffer_free(buffer->ptr, buffer->len); + *buffer = (DeverRtBuffer){NULL, 0}; + return equal; +} + +static int expect_nonempty(DeverRtBuffer *buffer) { + int nonempty = buffer->ptr != NULL && buffer->len != 0; + dever_rt_v1_buffer_free(buffer->ptr, buffer->len); + *buffer = (DeverRtBuffer){NULL, 0}; + return nonempty; +} + +static int expect_event_bytes(ReadEvent *event, const char *expected) { + DeverRtBuffer bytes = {NULL, 0}; + if (event->kind != 1 || dever_rt_v1_bytes_export(event->value, &bytes) != DEVER_RT_OK) { + return 0; + } + return expect_buffer(&bytes, expected); +} + +int main(int argc, char **argv) { + EXPECT(argc == 2); + DeverRtBuffer error = {NULL, 0}; + void *path = NULL; + void *file = NULL; + void *bytes = NULL; + void *stream = NULL; + uint8_t done = 0; + uint8_t present = 0; + EXPECT(dever_rt_v1_file_retain(NULL) == NULL); + dever_rt_v1_file_release(NULL); + EXPECT(dever_rt_v1_stream_retain(NULL) == NULL); + dever_rt_v1_stream_release(NULL); + EXPECT(dever_rt_v1_text_new((const uint8_t *)argv[1], strlen(argv[1]), &path, &error) == DEVER_RT_OK); + char missing_path[4096]; + int missing_length = snprintf(missing_path, sizeof(missing_path), "%s.missing", argv[1]); + EXPECT(missing_length > 0 && (size_t)missing_length < sizeof(missing_path)); + void *missing = NULL; + EXPECT(dever_rt_v1_text_new((const uint8_t *)missing_path, (uint64_t)missing_length, &missing, &error) == DEVER_RT_OK); + EXPECT(dever_rt_v1_file_open(missing, &file, &error) == DEVER_RT_RUNTIME_ERROR); + EXPECT(expect_nonempty(&error)); + dever_rt_v1_text_release(missing); + + /* Invalid output must not create the file. */ + EXPECT(dever_rt_v1_file_create(path, NULL, &error) == DEVER_RT_INVALID_INPUT); + EXPECT(expect_buffer(&error, "invalid ABI output pointer")); + EXPECT(dever_rt_v1_file_create(path, &file, &error) == DEVER_RT_OK); + EXPECT(dever_rt_v1_file_create(path, &bytes, &error) == DEVER_RT_RUNTIME_ERROR); + EXPECT(expect_nonempty(&error)); + EXPECT(dever_rt_v1_bytes_new((const uint8_t *)"abcd", 4, &bytes, &error) == DEVER_RT_OK); + EXPECT(dever_rt_v1_file_write(file, bytes, &done, &error) == DEVER_RT_OK && done == 1); + void *alias = dever_rt_v1_file_retain(file); + EXPECT(dever_rt_v1_file_close(file, &done, &error) == DEVER_RT_OK && done == 1); + EXPECT(dever_rt_v1_file_write(alias, bytes, &done, &error) == DEVER_RT_RUNTIME_ERROR); + EXPECT(expect_buffer(&error, "resource is closed")); + EXPECT(dever_rt_v1_file_close(alias, &done, &error) == DEVER_RT_RUNTIME_ERROR); + EXPECT(expect_buffer(&error, "resource is closed")); + dever_rt_v1_file_release(alias); + dever_rt_v1_file_release(file); + dever_rt_v1_bytes_release(bytes); + + EXPECT(dever_rt_v1_file_open(path, &file, &error) == DEVER_RT_OK); + EXPECT(dever_rt_v1_file_read(file, 0, &bytes, &present, &error) == DEVER_RT_RUNTIME_ERROR); + EXPECT(expect_buffer(&error, "read size must be a positive Int")); + EXPECT(dever_rt_v1_file_read(file, 4, NULL, &present, &error) == DEVER_RT_INVALID_INPUT); + EXPECT(expect_buffer(&error, "invalid ABI output pointer")); + EXPECT(dever_rt_v1_file_read(file, 4, &bytes, &present, &error) == DEVER_RT_OK && present); + EXPECT(dever_rt_v1_bytes_export(bytes, &error) == DEVER_RT_OK); + EXPECT(expect_buffer(&error, "abcd")); + dever_rt_v1_bytes_release(bytes); + bytes = NULL; + EXPECT(dever_rt_v1_file_read(file, 4, &bytes, &present, &error) == DEVER_RT_OK && !present); + EXPECT(bytes == NULL); + EXPECT(dever_rt_v1_file_close(file, &done, &error) == DEVER_RT_OK); + EXPECT(dever_rt_v1_file_chunks(file, 2, &READ_EVENT, &stream, &error) == DEVER_RT_RUNTIME_ERROR); + EXPECT(expect_buffer(&error, "resource is closed")); + dever_rt_v1_file_release(file); + + EXPECT(dever_rt_v1_file_open(path, &file, &error) == DEVER_RT_OK); + DeverRtReadEvent invalid_event = {&EVENT_TYPE, event_chunk, NULL}; + EXPECT(dever_rt_v1_file_chunks(file, 2, &invalid_event, &stream, &error) == DEVER_RT_INVALID_INPUT); + EXPECT(expect_buffer(&error, "incomplete ABI read event descriptor")); + EXPECT(dever_rt_v1_file_chunks(file, 0, &READ_EVENT, &stream, &error) == DEVER_RT_RUNTIME_ERROR); + EXPECT(expect_buffer(&error, "read size must be a positive Int")); + EXPECT(dever_rt_v1_file_chunks(file, 2, &READ_EVENT, &stream, &error) == DEVER_RT_OK); + alias = dever_rt_v1_stream_retain(stream); + ReadEvent event; + EXPECT(dever_rt_v1_stream_pull(stream, NULL, &present, &error) == DEVER_RT_INVALID_INPUT); + EXPECT(expect_buffer(&error, "invalid ABI output pointer")); + EXPECT(dever_rt_v1_stream_pull(alias, &event, &present, &error) == DEVER_RT_OK && present); + EXPECT(expect_event_bytes(&event, "ab")); + event_drop(&event); + EXPECT(dever_rt_v1_stream_pull(stream, &event, &present, &error) == DEVER_RT_OK && present); + EXPECT(expect_event_bytes(&event, "cd")); + event_drop(&event); + EXPECT(dever_rt_v1_stream_pull(alias, &event, &present, &error) == DEVER_RT_OK && !present); + EXPECT(dever_rt_v1_stream_close(stream, &done, &error) == DEVER_RT_OK && done == 1); + EXPECT(dever_rt_v1_stream_close(alias, &done, &error) == DEVER_RT_OK && done == 1); + dever_rt_v1_stream_release(alias); + dever_rt_v1_stream_release(stream); + EXPECT(dever_rt_v1_file_close(file, &done, &error) == DEVER_RT_OK); + dever_rt_v1_file_release(file); + + EXPECT(dever_rt_v1_file_open(path, &file, &error) == DEVER_RT_OK); + EXPECT(dever_rt_v1_file_chunks(file, 1, &READ_EVENT, &stream, &error) == DEVER_RT_OK); + EXPECT(dever_rt_v1_stream_pull(stream, &event, &present, &error) == DEVER_RT_OK && present); + EXPECT(expect_event_bytes(&event, "a")); + event_drop(&event); + EXPECT(dever_rt_v1_stream_close(stream, &done, &error) == DEVER_RT_OK); + EXPECT(dever_rt_v1_stream_pull(stream, &event, &present, &error) == DEVER_RT_OK && !present); + dever_rt_v1_stream_release(stream); + EXPECT(dever_rt_v1_file_close(file, &done, &error) == DEVER_RT_OK); + dever_rt_v1_file_release(file); + + EXPECT(dever_rt_v1_file_open(path, &file, &error) == DEVER_RT_OK); + EXPECT(dever_rt_v1_file_chunks(file, 2, &READ_EVENT, &stream, &error) == DEVER_RT_OK); + EXPECT(dever_rt_v1_file_close(file, &done, &error) == DEVER_RT_OK); + EXPECT(dever_rt_v1_stream_pull(stream, &event, &present, &error) == DEVER_RT_OK && present); + EXPECT(event.kind == 2); + EXPECT(dever_rt_v1_text_export(event.value, &error) == DEVER_RT_OK); + EXPECT(expect_buffer(&error, "resource is closed")); + event_drop(&event); + EXPECT(dever_rt_v1_stream_pull(stream, &event, &present, &error) == DEVER_RT_OK && !present); + dever_rt_v1_stream_release(stream); + dever_rt_v1_file_release(file); + + EXPECT(dever_rt_v1_file_chunks(NULL, 2, &READ_EVENT, &stream, &error) == DEVER_RT_INVALID_INPUT); + EXPECT(expect_buffer(&error, "invalid ABI handle")); + EXPECT(dever_rt_v1_file_open(NULL, &file, &error) == DEVER_RT_INVALID_INPUT); + EXPECT(expect_buffer(&error, "invalid ABI handle")); + dever_rt_v1_text_release(path); + EXPECT(event_retains == event_drops); + return 0; +} diff --git a/test/native-runtime-abi/test-suite-driver.c b/test/native-runtime-abi/test-suite-driver.c new file mode 100644 index 0000000..f0aa1d5 --- /dev/null +++ b/test/native-runtime-abi/test-suite-driver.c @@ -0,0 +1,64 @@ +#include "../../crates/dever-backend-bridge/include/runtime.h" +#include <errno.h> + +/* Keep the same allocation oracle as application/kernel probes. Only case + * selection and the suite's indexed fault ownership differ. */ +#define main managed_main +#include "managed-driver.c" +#undef main + +extern int32_t dever_test_entry(int64_t index, void *out, void *fault); +extern uint64_t dever_test_fault_size(int64_t index); +extern void dever_test_fault_release(int64_t index, void *fault); +extern int32_t dever_test_fault_render(int64_t index, const void *fault, + void **out, DeverRtBuffer *error); +extern int64_t dever_test_count(void); + +static int64_t case_index; +static int32_t expected_status; + +int dever_test_run(void) { + uint64_t size = dever_test_fault_size(case_index); + if (size == 0 || size > 1024 * 1024) return 2; + void *fault = calloc(1, (size_t)size); + if (fault == NULL) return 3; + uint8_t unit = 0; + int32_t status = dever_test_entry(case_index, &unit, fault); + int failed = status != expected_status; + if (status != 0) { + void *rendered = NULL; + DeverRtBuffer error = {0}; + int32_t render_status = dever_test_fault_render(case_index, fault, &rendered, &error); + if (render_status == 0) { + DeverRtBuffer text = {0}; + if (dever_rt_v1_text_export(rendered, &text) != 0) { + failed = 1; + } else { + fwrite(text.ptr, 1, (size_t)text.len, stderr); + fputc('\n', stderr); + } + dever_rt_v1_buffer_free(text.ptr, text.len); + dever_rt_v1_text_release(rendered); + } else { + failed = 1; + fwrite(error.ptr, 1, (size_t)error.len, stderr); + } + dever_rt_v1_buffer_free(error.ptr, error.len); + } + dever_test_fault_release(case_index, fault); + free(fault); + return failed; +} + +int main(int argc, char **argv) { + if (argc != 3) return 4; + char *end = NULL; + errno = 0; + long long index = strtoll(argv[1], &end, 10); + if (errno != 0 || end == argv[1] || *end != '\0' || index < 0 || index >= dever_test_count()) return 5; + case_index = (int64_t)index; + if (argv[2][0] == '0' && argv[2][1] == '\0') expected_status = 0; + else if (argv[2][0] == '1' && argv[2][1] == '\0') expected_status = 1; + else return 6; + return managed_main(); +} diff --git a/test/native-runtime-abi/wire-driver.c b/test/native-runtime-abi/wire-driver.c new file mode 100644 index 0000000..eba3cd8 --- /dev/null +++ b/test/native-runtime-abi/wire-driver.c @@ -0,0 +1,232 @@ +/* Canonical wire callbacks; repeated by the existing atomic allocation ledger. */ +#include "runtime.h" +#include <stdint.h> +#include <stdio.h> +#include <string.h> + +typedef struct { void *title; int64_t count; } Row; +static const DeverRtWireName fields[] = { + { (const uint8_t *)"title", 5 }, + { (const uint8_t *)"count", 5 } +}; +static uint32_t calls; + +static void row_clone(const void *source, void *target) { + const Row *from = source; + Row *to = target; + to->title = dever_rt_v1_text_retain(from->title); + to->count = from->count; +} + +static void row_drop(void *value) { + Row *row = value; + dever_rt_v1_text_release(row->title); + row->title = NULL; +} + +static uint8_t row_equal(const void *left, const void *right) { + const Row *a = left; + const Row *b = right; + return a->count == b->count && dever_rt_v1_text_equal(a->title, b->title); +} + +static const DeverRtType row_type = { + sizeof(Row), _Alignof(Row), row_clone, row_drop, row_equal, NULL +}; + +static void error_release(DeverRtBuffer *error) { + dever_rt_v1_buffer_free(error->ptr, error->len); + *error = (DeverRtBuffer){0}; +} + +static uint32_t row_decode(const void *node, void *output, DeverRtBuffer *error) { + ++calls; + Row *row = output; + *row = (Row){0}; + uint32_t status = dever_rt_v1_wire_node_fields(node, fields, 2, error); + const void *child = NULL; + uint8_t present = 0; + if (status == 0) status = dever_rt_v1_wire_node_field(node, &fields[0], 1, &child, &present, error); + if (status == 0) status = dever_rt_v1_wire_node_text(child, &row->title, error); + if (status == 0) status = dever_rt_v1_wire_node_field(node, &fields[1], 1, &child, &present, error); + if (status == 0) status = dever_rt_v1_wire_node_int(child, &row->count, error); + if (status != 0) row_drop(row); + return status; +} + +static uint32_t row_encode(const void *value, void *encoder, DeverRtBuffer *error) { + ++calls; + const Row *row = value; + uint32_t status = dever_rt_v1_wire_encoder_begin_object(encoder, error); + if (status == 0) status = dever_rt_v1_wire_encoder_key(encoder, &fields[0], error); + if (status == 0) status = dever_rt_v1_wire_encoder_text(encoder, row->title, error); + if (status == 0) status = dever_rt_v1_wire_encoder_key(encoder, &fields[1], error); + if (status == 0) status = dever_rt_v1_wire_encoder_int(encoder, row->count, error); + if (status == 0) status = dever_rt_v1_wire_encoder_end(encoder, error); + return status; +} + +static uint32_t bad_success_decode(const void *node, void *output, DeverRtBuffer *error) { + uint32_t status = row_decode(node, output, error); + if (status != 0) return status; + int64_t ignored; + (void)dever_rt_v1_int_div(1, 0, &ignored, error); + /* A successful codec must not leave an error Buffer alongside its owner. */ + return 0; +} + +/* Invalid destinations must not inspect a Node or advance a scoped encoder. */ +static uint32_t destination_probe(const void *node, void *output, DeverRtBuffer *error) { + const void *child = (const void *)(uintptr_t)17; + uint8_t present = 91; + if (dever_rt_v1_wire_node_field(NULL, &fields[0], 1, NULL, &present, error) != 2 || present != 91) return 2; + error_release(error); + if (dever_rt_v1_wire_node_field(NULL, &fields[0], 1, &child, NULL, error) != 2 || child != (const void *)(uintptr_t)17) return 2; + error_release(error); + unsigned char unaligned[sizeof(int64_t) + _Alignof(int64_t)]; + uintptr_t base = (uintptr_t)unaligned; + size_t offset = (_Alignof(int64_t) - base % _Alignof(int64_t)) % _Alignof(int64_t); + int64_t *bad = (int64_t *)(unaligned + offset + 1); + if (dever_rt_v1_wire_node_int(NULL, bad, error) != 2) return 2; + error_release(error); + /* Correcting the destination still sees the original parsed object. */ + return row_decode(node, output, error); +} + +static uint32_t command_decode(void *inputs, void *output, DeverRtBuffer *error) { + const DeverRtWireName field = { (const uint8_t *)"record", 6 }; + void *raw = NULL; + uint8_t present = 91; + *(Row *)output = (Row){0}; + if (dever_rt_v1_cmd_input_field(inputs, &field, 1, NULL, &present, error) != 2 || present != 91) return 2; + error_release(error); + uint32_t status = dever_rt_v1_cmd_input_field(inputs, &field, 1, &raw, &present, error); + if (status == 0) status = dever_rt_v1_wire_decode(raw, &row_type, row_decode, output, error); + dever_rt_v1_text_release(raw); + return status; +} + +static int text_equals(void *text, const char *expected) { + DeverRtBuffer bytes = {0}; + uint32_t status = dever_rt_v1_text_export(text, &bytes); + int equal = status == 0 && bytes.len == strlen(expected) && memcmp(bytes.ptr, expected, (size_t)bytes.len) == 0; + error_release(&bytes); + return equal; +} + +static void bytes_clone(const void *source, void *target) { + *(void **)target = dever_rt_v1_bytes_retain(*(void *const *)source); +} + +static void bytes_drop(void *value) { + dever_rt_v1_bytes_release(*(void **)value); + *(void **)value = NULL; +} + +static uint8_t bytes_equal(const void *left, const void *right) { + return dever_rt_v1_bytes_equal(*(void *const *)left, *(void *const *)right); +} + +static const DeverRtType bytes_type = { + sizeof(void *), _Alignof(void *), bytes_clone, bytes_drop, bytes_equal, NULL +}; + +static uint32_t bytes_decode(const void *node, void *output, DeverRtBuffer *error) { + if (dever_rt_v1_wire_node_bytes(NULL, NULL, error) != 2) return 2; + error_release(error); + return dever_rt_v1_wire_node_bytes(node, output, error); +} + +static uint32_t bytes_encode(const void *value, void *encoder, DeverRtBuffer *error) { + return dever_rt_v1_wire_encoder_bytes(encoder, *(void *const *)value, error); +} + +static int bytes_round_trip(void) { + DeverRtBuffer error = {0}; + void *text = NULL, *bytes = NULL, *encoded = NULL; + const char *json = "\"AP+ACg==\""; + const uint8_t expected[] = {0, 255, 128, 10}; + if (dever_rt_v1_text_new((const uint8_t *)json, strlen(json), &text, &error) != 0) return 20; + if (dever_rt_v1_wire_decode(text, &bytes_type, bytes_decode, &bytes, &error) != 0) return 21; + DeverRtBuffer actual = {0}; + if (dever_rt_v1_bytes_export(bytes, &actual) != 0 || actual.len != sizeof(expected) || memcmp(actual.ptr, expected, sizeof(expected)) != 0) return 22; + error_release(&actual); + if (dever_rt_v1_wire_encode(&bytes, &bytes_type, bytes_encode, &encoded, &error) != 0 || !text_equals(encoded, json)) return 23; + dever_rt_v1_text_release(encoded); + dever_rt_v1_text_release(text); + bytes_drop(&bytes); + const char *invalid[] = {"null", "[0,255]", "\"/w\"", "\"/x==\"", "\"_w==\""}; + for (size_t i = 0; i < sizeof(invalid) / sizeof(invalid[0]); ++i) { + if (dever_rt_v1_text_new((const uint8_t *)invalid[i], strlen(invalid[i]), &text, &error) != 0) return 24; + if (dever_rt_v1_wire_decode(text, &bytes_type, bytes_decode, &bytes, &error) != 1 || bytes != NULL || error.len == 0) return 25; + error_release(&error); + dever_rt_v1_text_release(text); + } + return 0; +} + +int32_t dever_test_run(void) { + int bytes_status = bytes_round_trip(); + if (bytes_status != 0) return bytes_status; + DeverRtBuffer error = {0}; + void *text = NULL; + Row row = {0}; + void *encoded = NULL; + const char *json = "{\"title\":\"owned\",\"count\":9223372036854775807}"; + if (dever_rt_v1_text_new((const uint8_t *)json, strlen(json), &text, &error) != 0) return 1; + calls = 0; + if (dever_rt_v1_wire_decode(text, &row_type, row_decode, NULL, &error) != 2 || calls != 0) return 2; + error_release(&error); + if (dever_rt_v1_wire_decode(text, &row_type, NULL, &row, &error) != 2 || calls != 0) return 3; + error_release(&error); + if (dever_rt_v1_wire_decode(text, &row_type, destination_probe, &row, &error) != 0 || row.count != INT64_MAX || !text_equals(row.title, "owned")) return 4; + calls = 0; + if (dever_rt_v1_wire_encode(&row, &row_type, row_encode, NULL, &error) != 2 || calls != 0) return 5; + error_release(&error); + if (dever_rt_v1_wire_encode(&row, &row_type, row_encode, &encoded, &error) != 0 || calls != 1 || !text_equals(encoded, json)) return 6; + dever_rt_v1_text_release(encoded); + row_drop(&row); + if (dever_rt_v1_wire_decode(text, &row_type, bad_success_decode, &row, &error) != 2 || row.title != NULL) return 9; + error_release(&error); + dever_rt_v1_text_release(text); + + const char *invalid[] = { + "{\"title\":\"partially-owned\",\"count\":\"wrong\"}", + "{\"title\":\"partially-owned\"}", + "{\"title\":\"partially-owned\",\"count\":9223372036854775808}", + "{\"title\":\"one\",\"title\":\"duplicate\",\"count\":1}", + "{\"title\":\"owned\",\"count\":1,\"extra\":false}", + "[]", + "{broken" + }; + for (size_t i = 0; i < sizeof(invalid) / sizeof(invalid[0]); ++i) { + if (dever_rt_v1_text_new((const uint8_t *)invalid[i], strlen(invalid[i]), &text, &error) != 0) return 7; + row = (Row){0}; + if (dever_rt_v1_wire_decode(text, &row_type, row_decode, &row, &error) != 1 || row.title != NULL || error.len == 0) return 8; + error_release(&error); + dever_rt_v1_text_release(text); + } + + const char *command = "{\"record\":{\"title\":\"command-owned\",\"count\":42}}"; + if (dever_rt_v1_text_new((const uint8_t *)command, strlen(command), &text, &error) != 0) return 10; + if (dever_rt_v1_cmd_decode(text, &row_type, command_decode, &row, &error) != 0 || row.count != 42 || !text_equals(row.title, "command-owned")) return 11; + row_drop(&row); + dever_rt_v1_text_release(text); + + const char *bad_commands[] = { + "{\"record\":{\"title\":\"complete-owned\",\"count\":42},\"unknown\":false}", + "{\"record\":{\"title\":\"partial-owned\",\"count\":\"bad\"},\"unknown\":false}", + "{}" + }; + const char *messages[] = { + "unknown input 'unknown'", "expected wire integer", "input 'record' is required" + }; + for (size_t i = 0; i < sizeof(bad_commands) / sizeof(bad_commands[0]); ++i) { + if (dever_rt_v1_text_new((const uint8_t *)bad_commands[i], strlen(bad_commands[i]), &text, &error) != 0) return 12; + row = (Row){0}; + if (dever_rt_v1_cmd_decode(text, &row_type, command_decode, &row, &error) != 1 || row.title != NULL || error.len != strlen(messages[i]) || memcmp(error.ptr, messages[i], (size_t)error.len) != 0) return 13; + error_release(&error); + dever_rt_v1_text_release(text); + } + return 0; +} diff --git a/test/native_runtime_abi.rs b/test/native_runtime_abi.rs new file mode 100644 index 0000000..c1c8fd8 --- /dev/null +++ b/test/native_runtime_abi.rs @@ -0,0 +1,89 @@ +use dever_backend_bridge::{ABI_INVALID_INPUT, ABI_OK, ABI_RUNTIME_ERROR, AbiDecimal}; +use dever_runtime::abi; +use dever_runtime::bytes::Bytes; +use dever_runtime::collections::{List, Map}; +use dever_runtime::number::{self, DecimalValue}; +use dever_runtime::text; + +#[test] +fn abi_decimal_words_preserve_finite_values_and_exact_arithmetic() { + assert_eq!(std::mem::size_of::<AbiDecimal>(), 16); + for value in [ + DecimalValue::ZERO, + DecimalValue::from_int(i64::MIN), + DecimalValue::maximum(), + DecimalValue::minimum(), + ] { + assert_eq!(AbiDecimal::encode(value).decode().unwrap(), value); + } + let left = AbiDecimal::encode(DecimalValue::parse("1.25").unwrap()); + let right = AbiDecimal::encode(DecimalValue::parse("0.75").unwrap()); + let sum = left + .decode() + .unwrap() + .checked_add(right.decode().unwrap()) + .unwrap(); + assert_eq!(abi::decimal_to_text(AbiDecimal::encode(sum)).unwrap(), "2"); + assert_eq!( + DecimalValue::from_int(1).checked_div(DecimalValue::ZERO), + Err("Decimal division by zero") + ); +} + +#[test] +fn abi_facade_preserves_numeric_text_and_error_contracts() { + assert_eq!((ABI_OK, ABI_RUNTIME_ERROR, ABI_INVALID_INPUT), (0, 1, 2)); + assert_eq!(number::int_add(i64::MAX, 1), Err("Int overflow")); + assert_eq!(number::int_div(1, 0), Err("Int division by zero")); + assert_eq!(abi::int_to_text(i64::MIN), i64::MIN.to_string()); + assert_eq!(abi::float_to_text(f64::INFINITY), "inf"); + assert_eq!(abi::float_to_text(f64::NAN), "NaN"); + assert_eq!( + abi::text_from_utf8("中文".as_bytes()).unwrap(), + "中文".as_bytes() + ); + assert!(abi::text_from_utf8(&[0xff]).is_err()); +} + +#[test] +fn managed_values_reuse_unicode_bytes_and_decimal_semantics() { + assert_eq!(text::slice("a中b", 1, 2), Some("中".into())); + assert_eq!(text::codepoint("中"), Some('中' as i64)); + assert_eq!(text::index_of("a中b", "b"), Some(2)); + assert_eq!(text::split("a,,b", ",").values(), &["a", "", "b"]); + assert_eq!(Bytes::from_ints(&[0, 255]).unwrap().values(), &[0, 255]); + assert_eq!( + Bytes::from_ints(&[256]), + Err("byte value must be between 0 and 255") + ); + assert_eq!( + Bytes::new(vec![1, 2, 3]).slice(1, 3).unwrap().values(), + &[2, 3] + ); + assert_eq!(number::parse_decimal("1.00"), number::parse_decimal("1.0"),); +} + +#[test] +fn managed_collections_preserve_cow_order_and_duplicate_key_errors() { + let original = List::new(vec![String::from("one")]); + let shared = original.clone(); + let appended = original.append(String::from("two")); + assert_eq!(shared.values(), &["one"]); + assert_eq!(appended.into_values().collect::<Vec<_>>(), ["one", "two"]); + + let original = Map::new(vec![(String::from("a"), 1), (String::from("b"), 2)]).unwrap(); + let shared = original.clone(); + let changed = original + .put(String::from("a"), 3) + .remove(&String::from("b")); + assert_eq!( + shared + .into_entries() + .into_values() + .map(|entry| (entry.key, entry.value)) + .collect::<Vec<_>>(), + [("a".into(), 1), ("b".into(), 2)] + ); + assert_eq!(changed.into_get(&String::from("a")), Some(3)); + assert_eq!(Map::new(vec![(1, 1), (1, 2)]), Err("duplicate Map key")); +} diff --git a/test/native_runtime_link.rs b/test/native_runtime_link.rs new file mode 100644 index 0000000..56ecd49 --- /dev/null +++ b/test/native_runtime_link.rs @@ -0,0 +1,208 @@ +//! Developer-only C ABI acceptance; normal execution never invokes these tools. +use std::path::Path; +use std::process::{Command, Stdio}; +use std::time::Duration; + +#[path = "dever-tests/tests/support/process.rs"] +mod process; +#[path = "dever-tests/tests/support/temp.rs"] +mod temp; + +// Requires an explicitly produced runtime-abi-only archive. It is kept separate +// from the default tests because Cargo must not recursively build from a test. +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[test] +#[ignore = "build runtime-abi-only staticlib in target/native-runtime-abi before running"] +fn canonical_c_header_links_and_executes_the_real_runtime_archive() { + run_driver("driver.c", None, false); +} + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +fn run_driver(source: &str, llvm_object: Option<&Path>, allocation_ledger: bool) { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let archive = root.join("target/native-runtime-abi/debug/libdever_backend_bridge.a"); + assert!( + archive.is_file(), + "missing explicit runtime ABI fixture archive" + ); + let directory = temp::TemporaryDirectory::new(); + let program = directory.path().join("abi-acceptance"); + let mut command = Command::new("/usr/bin/cc"); + command + .env_clear() + .args(["-std=c11", "-Wall", "-Wextra", "-Werror", "-O2", "-I"]) + .arg(root.join("crates/dever-backend-bridge/include")) + .arg(root.join("test/native-runtime-abi").join(source)); + if let Some(object) = llvm_object { + command.arg("-DDEVER_LLVM_ABI").arg(object); + } + if allocation_ledger { + command.arg(root.join("test/native-runtime-abi/managed-driver.c")); + command.args([ + "-Wl,--wrap=malloc", + "-Wl,--wrap=calloc", + "-Wl,--wrap=realloc", + "-Wl,--wrap=free", + "-Wl,--wrap=posix_memalign", + "-Wl,--wrap=realpath", + ]); + } + let compile = command + .arg(archive) + .args(["-ldl", "-lpthread", "-lm", "-o"]) + .arg(&program) + // GCC's internal assembler/linker are fixture authoring tools, not a + // runtime PATH fallback. Their search directories are explicit. + .arg("-B/usr/bin/") + .output() + .unwrap(); + assert!( + compile.status.success(), + "{}", + String::from_utf8_lossy(&compile.stderr) + ); + let status = process::status( + Command::new(program) + .env_clear() + .stdin(Stdio::null()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()), + Duration::from_secs(5), + ) + .unwrap(); + assert!(status.success(), "C ABI driver failed: {status}"); +} + +#[cfg(all(feature = "embedded", target_os = "linux", target_arch = "x86_64"))] +#[test] +#[ignore = "build runtime-abi-only staticlib in target/native-runtime-abi before running"] +fn llvm_object_calls_the_versioned_runtime_abi_without_rust_layouts() { + use dever_backend_bridge::{Target, emit_object}; + let directory = temp::TemporaryDirectory::new(); + let object = directory.path().join("probe.o"); + std::fs::write( + &object, + emit_object(ABI_PROBE, Target::LinuxX86_64).unwrap(), + ) + .unwrap(); + run_driver("driver.c", Some(&object), false); +} + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[test] +#[ignore = "build runtime-abi-only staticlib in target/native-runtime-abi before running"] +fn protocol_poll_rejects_invalid_output_before_advancing_the_operation() { + run_driver("protocol-driver.c", None, false); +} + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[test] +#[ignore = "build runtime-abi-only staticlib in target/native-runtime-abi before running"] +fn logger_flush_is_a_stable_native_allocation_barrier() { + run_driver("log-driver.c", None, true); +} + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +#[test] +#[ignore = "build runtime-abi-only staticlib in target/native-runtime-abi before running"] +fn wire_codecs_validate_slots_before_callback_and_release_partial_values() { + run_driver("wire-driver.c", None, true); +} + +#[cfg(all( + any(feature = "runtime-sqlite", feature = "runtime-postgres"), + target_os = "linux", + target_arch = "x86_64" +))] +#[test] +#[ignore = "build a runtime-sqlite or runtime-postgres staticlib in target/native-runtime-abi before running"] +fn database_abi_preserves_typed_owners_and_rejects_invalid_slots_without_io() { + run_driver("database-driver.c", None, true); +} + +#[cfg(all(feature = "embedded", target_os = "linux", target_arch = "x86_64"))] +const ABI_PROBE: &str = r#" +%abi.buffer = type { ptr, i64 } +%abi.decimal = type { i64, i64 } +declare i32 @dever_rt_v1_version() +declare i32 @dever_rt_v1_int_add(i64, i64, ptr, ptr) +declare i32 @dever_rt_v1_decimal_from_int(i64, ptr, ptr) +declare i32 @dever_rt_v1_decimal_div(i64, i64, i64, i64, ptr, ptr) +declare i32 @dever_rt_v1_decimal_to_text(i64, i64, ptr) +declare void @dever_rt_v1_buffer_free(ptr, i64) +define i32 @dever_abi_probe() { +entry: + %buffer = alloca %abi.buffer + %integer = alloca i64 + %seven = alloca %abi.decimal + %two = alloca %abi.decimal + %decimal = alloca %abi.decimal + %version = call i32 @dever_rt_v1_version() + %version_ok = icmp eq i32 %version, 1 + %add = call i32 @dever_rt_v1_int_add(i64 40, i64 2, ptr %integer, ptr %buffer) + %add_ok = icmp eq i32 %add, 0 + %first_ok = and i1 %version_ok, %add_ok + br i1 %first_ok, label %added, label %failed +added: + %value = load i64, ptr %integer + %value_ok = icmp eq i64 %value, 42 + %overflow = call i32 @dever_rt_v1_int_add(i64 9223372036854775807, i64 1, ptr %integer, ptr %buffer) + %overflow_ok = icmp eq i32 %overflow, 1 + %error = load %abi.buffer, ptr %buffer + %error_ptr = extractvalue %abi.buffer %error, 0 + %error_len = extractvalue %abi.buffer %error, 1 + %message_ok = icmp eq i64 %error_len, 12 + call void @dever_rt_v1_buffer_free(ptr %error_ptr, i64 %error_len) + %error_ok = and i1 %overflow_ok, %message_ok + %numeric_ok = and i1 %value_ok, %error_ok + br i1 %numeric_ok, label %decimal_input, label %failed +decimal_input: + %from_seven = call i32 @dever_rt_v1_decimal_from_int(i64 7, ptr %seven, ptr %buffer) + %from_two = call i32 @dever_rt_v1_decimal_from_int(i64 2, ptr %two, ptr %buffer) + %seven_ok = icmp eq i32 %from_seven, 0 + %two_ok = icmp eq i32 %from_two, 0 + %inputs_ok = and i1 %seven_ok, %two_ok + br i1 %inputs_ok, label %divide, label %failed +divide: + %seven_value = load %abi.decimal, ptr %seven + %two_value = load %abi.decimal, ptr %two + %seven_low = extractvalue %abi.decimal %seven_value, 0 + %seven_high = extractvalue %abi.decimal %seven_value, 1 + %two_low = extractvalue %abi.decimal %two_value, 0 + %two_high = extractvalue %abi.decimal %two_value, 1 + %divided = call i32 @dever_rt_v1_decimal_div(i64 %seven_low, i64 %seven_high, i64 %two_low, i64 %two_high, ptr %decimal, ptr %buffer) + %divide_ok = icmp eq i32 %divided, 0 + br i1 %divide_ok, label %render, label %failed +render: + %result = load %abi.decimal, ptr %decimal + %low = extractvalue %abi.decimal %result, 0 + %high = extractvalue %abi.decimal %result, 1 + %rendered = call i32 @dever_rt_v1_decimal_to_text(i64 %low, i64 %high, ptr %buffer) + %render_ok = icmp eq i32 %rendered, 0 + %text = load %abi.buffer, ptr %buffer + %text_ptr = extractvalue %abi.buffer %text, 0 + %text_len = extractvalue %abi.buffer %text, 1 + %len_ok = icmp eq i64 %text_len, 3 + %text_ok = and i1 %render_ok, %len_ok + br i1 %text_ok, label %read_text, label %release_failed +read_text: + %byte0 = load i8, ptr %text_ptr + %ptr1 = getelementptr i8, ptr %text_ptr, i64 1 + %ptr2 = getelementptr i8, ptr %text_ptr, i64 2 + %byte1 = load i8, ptr %ptr1 + %byte2 = load i8, ptr %ptr2 + %byte0_ok = icmp eq i8 %byte0, 51 + %byte1_ok = icmp eq i8 %byte1, 46 + %byte2_ok = icmp eq i8 %byte2, 53 + %prefix_ok = and i1 %byte0_ok, %byte1_ok + %passed = and i1 %prefix_ok, %byte2_ok + call void @dever_rt_v1_buffer_free(ptr %text_ptr, i64 %text_len) + %status = select i1 %passed, i32 0, i32 1 + ret i32 %status +release_failed: + call void @dever_rt_v1_buffer_free(ptr %text_ptr, i64 %text_len) + br label %failed +failed: + ret i32 1 +} +"#; diff --git a/test/onboarding/test_installer.py b/test/onboarding/test_installer.py new file mode 100644 index 0000000..37f80a3 --- /dev/null +++ b/test/onboarding/test_installer.py @@ -0,0 +1,259 @@ +"""Offline install/asset contracts; bootstrap is mocked, never installed globally.""" + +import argparse +import contextlib +import gzip +import hashlib +import importlib.util +import io +import json +from pathlib import Path +import subprocess +import tarfile +import tempfile +import unittest +from unittest import mock + + +ROOT = Path(__file__).resolve().parents[2] + + +def load_module(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +installer = load_module("dever_installer", ROOT / "skills/scripts/install.py") +assets = load_module("dever_release_assets", ROOT / "sdk/release-assets.py") + + +class InstallerTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix="dever-installer-test-") + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.release = self.directory("release") + self.scratch = self.directory("scratch") + self.platform = installer.platform_name() + self.payloads = { + "dever-core": b"fixture core, never executed\n", + "bootstrap/dever": b"fixture launcher, never executed\n", + "skills/dever-language/SKILL.md": b"fixture skill\n", + } + self.catalog = {} + for name, content in self.payloads.items(): + path = self.release / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + self.catalog[name] = { + "path": name, "bytes": len(content), + "sha256": hashlib.sha256(content).hexdigest(), + } + manifest = { + "format": "dever-release-v1", "platform": self.platform, + "version": "1.2.3", "artifacts": list(self.catalog.values()), + } + (self.release / "manifest.json").write_text(json.dumps(manifest)) + self.private = self.root / "test-key.pem" + self.openssl("genpkey", "-algorithm", "ED25519", "-out", str(self.private)) + public_der = self.openssl("pkey", "-in", str(self.private), "-pubout", "-outform", "DER") + self.public = self.root / "test-key.pub" + self.public.write_text(public_der[-32:].hex()) + self.sign_manifest() + + def sign_manifest(self): + signature = self.root / "signature.bin" + self.openssl("pkeyutl", "-sign", "-inkey", str(self.private), "-rawin", + "-in", str(self.release / "manifest.json"), "-out", str(signature)) + (self.release / "manifest.sig").write_text(signature.read_bytes().hex()) + + def directory(self, name): + path = self.root / name + path.mkdir() + return path + + def openssl(self, *arguments): + return subprocess.run(["/usr/bin/openssl", *arguments], env={}, + capture_output=True, check=True, timeout=15).stdout + + def authenticate(self, key=None): + return installer.authenticate(self.release, key or self.public, self.scratch, + self.platform, "1.2.3") + + def archive(self, entries, trailer=None): + stream = io.BytesIO() + with tarfile.open(fileobj=stream, mode="w", format=tarfile.USTAR_FORMAT) as archive: + for name, content, kind in entries: + member = tarfile.TarInfo(name) + member.type = kind + member.size = len(content) if kind == tarfile.REGTYPE else 0 + if kind == tarfile.SYMTYPE: + member.linkname = "../../outside" + archive.addfile(member, io.BytesIO(content) if member.size else None) + contents = stream.getvalue() + if trailer is not None: + contents += trailer + return gzip.compress(contents) + + def extract(self, compressed, destination): + with mock.patch.object(installer, "download", return_value=io.BytesIO(compressed)): + installer.payload(destination, None, "1.2.3", self.platform, self.catalog) + + def test_signature_accepts_independent_key_and_rejects_wrong_key_or_changed_manifest(self): + version, catalog, key = self.authenticate() + self.assertEqual(version, "1.2.3") + self.assertEqual(catalog, self.catalog) + self.assertEqual(len(key), 32) + wrong = self.root / "wrong.pub" + wrong.write_text(bytes(32).hex()) + with self.assertRaisesRegex(ValueError, "签名校验失败"): + self.authenticate(wrong) + with (self.release / "manifest.json").open("a") as stream: + stream.write(" ") + with self.assertRaisesRegex(ValueError, "签名校验失败"): + self.authenticate() + + def test_asset_author_round_trip_is_deterministic_and_contains_only_signed_payload(self): + first = self.root / "first" + second = self.root / "second" + (self.release / "unlisted").write_text("not published") + names = assets.create(self.release, first) + self.assertEqual(names, assets.create(self.release, second)) + self.assertEqual(len(names), 3) + for name in names: + self.assertEqual((first / name).read_bytes(), (second / name).read_bytes()) + extracted = self.directory("extracted") + self.extract((first / f"dever-{self.platform}.tar.gz").read_bytes(), extracted) + actual = {str(path.relative_to(extracted)): path.read_bytes() + for path in extracted.rglob("*") if path.is_file()} + self.assertEqual(actual, self.payloads) + self.assertEqual((extracted / "bootstrap/dever").stat().st_mode & 0o777, 0o755) + with self.assertRaisesRegex(ValueError, "already exists"): + assets.create(self.release, first) + + def test_signed_release_without_matching_skill_is_rejected(self): + path = self.release / "manifest.json" + manifest = json.loads(path.read_text()) + manifest["artifacts"] = [entry for entry in manifest["artifacts"] + if not entry["path"].startswith("skills/")] + path.write_text(json.dumps(manifest)) + self.sign_manifest() + with self.assertRaisesRegex(ValueError, "配套 skill"): + self.authenticate() + + def test_publish_preserves_concurrent_targets_and_owned_staging_on_failure(self): + staging = self.directory("staging") + (staging / "asset").write_bytes(b"owned release bytes") + empty = self.directory("empty") + populated = self.directory("populated") + (populated / "existing").write_bytes(b"existing user bytes") + file = self.root / "file" + file.write_bytes(b"existing file bytes") + symlink = self.root / "symlink" + symlink.symlink_to(self.root / "missing") + for destination in (empty, populated, file, symlink): + with self.subTest(destination=destination.name), self.assertRaises(OSError): + assets.publish(staging, destination) + self.assertEqual((staging / "asset").read_bytes(), b"owned release bytes") + self.assertEqual(list(empty.iterdir()), []) + self.assertEqual((populated / "existing").read_bytes(), b"existing user bytes") + self.assertEqual(file.read_bytes(), b"existing file bytes") + self.assertEqual(symlink.readlink(), self.root / "missing") + destination = self.root / "published" + assets.publish(staging, destination) + self.assertFalse(staging.exists()) + self.assertEqual((destination / "asset").read_bytes(), b"owned release bytes") + + def test_create_cleans_only_its_staging_when_target_appears_during_preparation(self): + actual_publish = assets.publish + for populated in (False, True): + output = self.root / ("race-populated" if populated else "race-empty") + + def publish(prepared, destination): + destination.mkdir() + if populated: + (destination / "user-file").write_bytes(b"concurrent user bytes") + actual_publish(prepared, destination) + + with mock.patch.object(assets, "publish", side_effect=publish), \ + self.assertRaises(OSError): + assets.create(self.release, output) + self.assertTrue(output.is_dir()) + if populated: + self.assertEqual((output / "user-file").read_bytes(), b"concurrent user bytes") + else: + self.assertEqual(list(output.iterdir()), []) + self.assertEqual(list(self.root.glob(".dever-release-assets-*")), []) + + def test_archive_rejects_unsigned_duplicate_symlink_wrong_size_missing_and_bad_tail(self): + normal = [(name, content, tarfile.REGTYPE) for name, content in self.payloads.items()] + cases = { + "unsigned": normal + [("outside", b"extra", tarfile.REGTYPE)], + "duplicate": normal + [normal[0]], + "symlink": [(normal[0][0], b"", tarfile.SYMTYPE)] + normal[1:], + "size": [(normal[0][0], b"short", tarfile.REGTYPE)] + normal[1:], + "missing": normal[:-1], + } + for name, entries in cases.items(): + with self.subTest(name=name), self.assertRaises(ValueError): + self.extract(self.archive(entries), self.directory(name)) + with self.assertRaisesRegex(ValueError, "尾部"): + self.extract(self.archive(normal, b"hidden"), self.directory("tail")) + corrupted = [(normal[0][0], b"x" * len(normal[0][1]), tarfile.REGTYPE)] + normal[1:] + with self.assertRaisesRegex(ValueError, "摘要"): + self.extract(self.archive(corrupted), self.directory("digest")) + + def test_local_payload_rejects_symlink_and_source_directory(self): + copied = self.directory("copied") + installer.payload(copied, self.release, "1.2.3", self.platform, self.catalog) + self.assertEqual((copied / "dever-core").read_bytes(), self.payloads["dever-core"]) + core = self.release / "dever-core" + core.unlink() + core.symlink_to(self.public) + with self.assertRaisesRegex(ValueError, "符号链接"): + installer.payload(self.directory("symlink"), self.release, "1.2.3", self.platform, self.catalog) + with self.assertRaisesRegex(ValueError, "symlinks"): + assets.create(self.release, self.root / "invalid-assets") + core.unlink() + core.mkdir() + with self.assertRaisesRegex(ValueError, "类型"): + installer.payload(self.directory("directory"), self.release, "1.2.3", self.platform, self.catalog) + + def test_image_install_authenticates_before_mock_bootstrap_and_cleans_owned_staging(self): + image = self.directory("image") + arguments = argparse.Namespace(version="1.2.3", system_root=image, release=self.release, + trusted_key=self.public, no_start=False) + actual_run = subprocess.run + calls = [] + + def run(command, **kwargs): + if command[0] == "/usr/bin/openssl": + return actual_run(command, **kwargs) + self.assertEqual(command[1], "--dever-install") + scratch = Path(command[2]) + setting = json.loads((scratch / "config/setting.json").read_text()) + self.assertEqual(set(setting), {"release", "system_root", "trusted_key", "activate_service"}) + self.assertEqual(setting["system_root"], str(image)) + self.assertFalse(setting["activate_service"]) + self.assertEqual(kwargs["env"], {}) + self.assertFalse(Path(setting["trusted_key"]).is_relative_to(Path(setting["release"]))) + calls.append(command) + return subprocess.CompletedProcess(command, 0) + + with mock.patch.object(installer.os, "geteuid", return_value=0), \ + mock.patch.object(installer.subprocess, "run", side_effect=run), \ + contextlib.redirect_stdout(io.StringIO()): + installer.install(arguments) + self.assertEqual(len(calls), 1) + self.assertEqual(list(image.iterdir()), []) + (self.release / "dever-core").write_bytes(b"corrupt") + with self.assertRaises(ValueError): + installer.install(arguments) + self.assertEqual(len(calls), 1) + self.assertEqual(list(image.iterdir()), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/performance/README.md b/test/performance/README.md new file mode 100644 index 0000000..4a7fda2 --- /dev/null +++ b/test/performance/README.md @@ -0,0 +1,349 @@ +# 独立二进制性能基准 + +这里测常驻内存、异步调度、HTTP/1.1、HTTP/2 和 TCP/WebSocket/SSE 长连接；不改运行时。构建单独执行，运行阶段只启动已编译产物。全部程序绑定自己创建的 loopback 随机端口，不接触已有服务，不进入默认 `cargo test`。 + +## 构建与短时验证 + +在仓库根目录运行，使用已经安装的 Rust 工具链与锁定的离线依赖： + +```bash +cargo build --offline -p dever-cli --bin dever +cargo build --offline --locked --release --target-dir target/native-runtime -p dever-tests --example network_bench --example live_bench --example async_bench +python3 test/performance/run.py build --output target/performance/build-current --workers 1 --connections 512 +python3 -m unittest discover -s test/performance -p test_runner.py -v +python3 test/performance/test_peer_settings.py -v +python3 test/performance/test_network_bench.py -v +python3 test/performance/test_live_bench.py -v +python3 test/performance/run.py run --artifacts target/performance/build-current --output target/performance/smoke-current --suite all --duration 0.5 --warmup 0.2 --rates 100 --paths /plain --repeats 1 +``` + +只复验当前 Dever/Markdown 双源码 CMS 和 base/SQLite/PostgreSQL/both 四个 profile 时，可使用 +`python3 test/performance/run.py build --selection cms-profiles --output target/performance/cms-build-current`。 +该选择不构建无关的 HTTP、长连接、ORM 场景和 peer；生成的产物仍可用 +`run --suite cms` 分别测量 `cms`（Dever）和 `cms-md`（Markdown）的真实登录、文章创建/替换、发布和前台查询；旧 `READY|3` 计数 fixture 已删除。仅做四 profile 构建可选 `--selection profiles`。PostgreSQL profile 使用不连接数据库的编译专用声明，不需要真实 URL；真实 PostgreSQL ORM/压力测试仍必须显式配置根 `config/setting.json` 的 `database.postgres_test`。 + +仅复验 runtime/HTTP/TLS/HTTP2/TCP/WebSocket/SSE 时使用 `--selection protocols`，不构建 ORM/CMS 或重复四 profile；三种 Rust peer 可用 `--network-bench`、`--live-bench`、`--async-bench` 指定明确准备的路径。 + +输出目录必须是新路径，保留先前报告。当前 Dever 产物统一经公开 `dever build` 使用 LLVM O2、LLD 和明确准备的 runtime archive；构建日志及二进制 SHA-256、大小、耗时保存在 `manifest.json`。runtime/http/live 每组构建一个包含该组 CMD 的应用，各测量通过不同命令启动，大小不能解读为仅一个函数的程序。`profile-report.json` 仍使用同一份最小 CMD 源码生成 base、SQLite、PostgreSQL 和 both 产物，记录 bridge 的锁定依赖。作者必须准备匹配源码的四个 archive；编译器自身的 debug/release 不决定 archive 优化级别。Rust 对照程序由上面的 Cargo release 命令另行准备并记录二进制摘要。Dever 使用生产默认调度，`--workers` 和 `task_capacity` 只配置 Rust 对照，不声称两者调度配置相同；可用专属 CPU affinity 统一核数。运行器要求 `application_configuration=llvm-cmd-v1`，拒绝旧 Rust 后端产物冒充当前 LLVM 测量。测量不调用 Cargo。 + +双源码 CMS 也有独立入口，不构建网络 peer 和无关场景： + +```text +python3 -B test/performance/cms.py build --output target/performance/cms-publish-build +python3 -B test/performance/cms.py run --manifest target/performance/cms-publish-build/manifest.json --output target/performance/cms-publish-128 --cgroup-parent /sys/fs/cgroup --memory-mib 128 +python3 -B test/performance/cms.py run --manifest target/performance/cms-publish-build/manifest.json --output target/performance/cms-publish-64 --cgroup-parent /sys/fs/cgroup --memory-mib 64 +python3 -B -m unittest discover -s test/performance -p test_cms.py -v +``` + +运行使用仅含二进制和私有 `config/setting.json` 的副本，不保留 module。先在自有 SQLite 平台库 bootstrap、迁移租户库并给 admin/front 分别建立 Owner，再启动随机 loopback 端口。服务启动与登录/发布/查询在专属 cgroup 内，bootstrap 和部署命令不在该预算内。保留 Cookie/Origin、真实身份和租户验证，不伪造登录身份。逐篇创建、替换、发布并核对全部前台结果；重复发布必须返回 409，两套源码结果必须一致。默认 16 篇顺序业务基线报告启动时间、每种请求样本数/中位/最大延迟、RSS/PSS 与 cgroup peak/OOM，不称为并发饱和吞吐或 p99。测试手动传送服务返回的 Secure Cookie，只验证 owned HTTP 协议，不代表浏览器 HTTPS 验收。日志阈值固定 error，出现服务错误、超时、OOM 或清理失败均失败。 + +追加 `--concurrency 8 --read-seconds 10` 会在发布后运行八个独立连接的已登录前台列表查询，每次都核对完整业务结果。报告另列 `concurrent_reads` 的实际成功 QPS、p50/p95/p99、测量期资源趋势和断连 FD 回落；这是闭环读负载，不能解释为写入吞吐或预设请求到达率。并发数最多64、时长最多3600秒，同一服务进程内支持30分钟持续读取；客户端使用固定2048桶的对数延迟直方图，分位数报告包含请求及结果核对的桶上界，避免随请求数保存无限样本。首次失败停止其他客户端，所有连接按原请求期限关闭；不记录 Cookie 或密码。完整Linux门及64/128MiB长测入口见 [test/ci/README.md](../ci/README.md)。 + +真实 PostgreSQL 的 Owner HTTP/双物理租户验收另由共享 Rust fixture 管理数据库生命周期： + +```text +cargo test --offline --locked -p dever-tests --features api,postgres --test postgres_api postgres_cms_http_isolates_two_tenant_databases -- --ignored --exact --nocapture +``` + +根 `database.postgres_test` 必须指向专属测试实例，URL 数据库名包含 `{case}`，账号允许创建/删除测试库；默认不运行、不探测外部服务。fixture 使用随机控制库及租户前缀，只清理本轮拥有的库，失败也清理，清理失败不能记录 passed。`cms.py postgres-case` 始终先复制传入项目到自有临时目录，端口/密钥/配置/产物仅写副本，输出目录保留日志和不含凭据的结果。Rust 目标额外注入仅在 `test/` 的普通成员 CMD，并显式启用 `--rbac-fixture`：覆盖 Owner、两个租户各两篇发布、相同 slug 独立数据、跨站 Cookie/跨租户登录拒绝，以及非 Owner 多角色、跨站同名角色和原会话撤销立即失效。单独调用不启用该测试 fixture 的工具仍仅证明 Owner 合同。权限键分别读取实际站点目录，GET/DELETE 参数统一走 query，生产 CMS 和 Owner 保留角色规则不改。 + +runtime/HTTP/live 夹具已使用当前 App→Port→Adapter 结构，CMD 根有真实 Bool 输出；live 的 typed Adapter setting 在每个产物的私有配置中独立绑定。ORM seed 是测量前的 CMD 部署步骤，count/cancel 是普通 API，取消测试使用单连接池并核对计数和连接归还，不能把 handler 超时冒充客户端取消。 + +2026-10-02 LLVM 基线使用 `protocols-llvm-10-02`、`cms-llvm-10-02` 及 `profiles-llvm-10-02` 的准确产物摘要，私有作者编译器来源记录在 `target/closure-current-compiler/provenance.json`。四个最小 CMD profile 大小分别为 base 2,083,880、SQLite 4,139,976、PostgreSQL 2,265,224、both 4,238,568 字节。Dever/Markdown CMS 分别 8,323,880 / 8,326,120 字节。 + +| 当前测量 | 条件 | 结果与报告目录（均位于 target/performance） | +| --- | --- | --- | +| 双 CMS 发布 | 64/128 MiB、每组顺序16篇 | 零错误/OOM，启动44.6–51.9ms，RSS峰值28.20–28.33MiB；`cms-llvm-{64,128}-10-02` | +| 双 CMS 登录后列表读取 | 64/128 MiB、8并发、10秒、不绑核 | 1210.6–1322.2 QPS，p99 11.48–13.59ms，RSS峰值30.81–31.04MiB，零错误/OOM；`cms-read-llvm-{64,128}-10-02` | +| HTTP/2 明文及 TLS 稳态 | 64/128 MiB、单核、4连接×16流、目标10k、60秒 | 成功/计划99.718–99.794%，9971.6–9979.2 QPS，p99 2.912–3.424ms，RSS峰值4.88–5.72MiB，零错误/OOM/断开后FD差；`http2-steady-llvm-{64,128}-10-02` | +| HTTP/2 连接恢复 | 64 MiB、单核、两种传输各100轮 | 合计19,887个完整成功请求，200次断开后FD差均0，零错误/OOM；`http2-recovery-llvm-64-10-02-v2` | +| TCP/WS/SSE 连接恢复 | 64/128 MiB、单核、32/128连接、各10轮 | 12个case、120次断开后FD差均0，完整消息校验通过，零错误/OOM；`live-llvm-{64,128}-10-02` | + +这些是本机 loopback 与热文件缓存的有限测量。CMS使用闭环客户端，HTTP/2使用固定速率且保留少量未派发计数，两者不能直接比较。15k TLS短时校准未达到99%计划成功门，所以当前一分钟基线选择10k，不把下文历史20k/饱和结果当作本次结果。首次恢复因运行器重复复制peer耗尽磁盘的失败目录保留，最终恢复证据仅用v2。历史 `cms-publish-*-09-30-*`、`orm-runtime-64-09-30-v1` 与 `live-tcp-64-09-30-v1` 不代表本次重跑。 + +共享编译的本机基线记录于 `target/closure-shared-compile-measurement.json`：私有debug核心、热OS缓存下，首次请求7.318秒、命中5.564秒；两个同步客户端提交同一份新源码分别7.463/7.838秒且产物一致。时间包含签名输入复核及IPC，不能当纯编译耗时或正式优化版CLI成绩。每个产物4,239,368字节，两条缓存合计8,478,736字节，校验无损坏；跨真实UID共享另有独立定向验收。 + +产物目录含二进制、测试证书、构建配置，可整体复制到兼容的 Linux 机器；运行器仅依赖 Python 3.11+ 标准库和 `/proc`。TLS 证书来自 `test/dever-tests/fixtures/tls/`，仅用于测试，执行时校验证书链和 localhost 名称。测试证书过期后需更新 fixture，不得关闭验证。 + +两个独立 peer 测试从仓库 `config/setting.json` 的 `performance` 读取 `network_peer`、`live_peer`，例如： + +```json +{"performance":{"network_peer":"target/performance/build-current/network_bench","live_peer":"target/performance/build-current/live_bench"}} +``` + +把该 `performance` 对象合并进已有设置，保留原有数据库配置。基准进程配置不读取环境变量。运行器在同文件系统硬链接已验证的只读 peer 二进制，仅跨文件系统时复制；每轮仍在其旁边以 `0600` 写入独立 `config/setting.json` 的 `benchmark` 对象。测量期间不得修改输入二进制。字段为 `workers`、`connections`、`task_capacity`、`timeout_ms`、`lifetime_ms`、HTTP/2 参数和 TLS 证书路径。CLI 负载参数仍显式传递。缺少配置、非法类型或越界值会报错；构建器通过不启动网络的 `check-config` 检查 peer 的 JSON 配置协议。旧协议的 manifest 不再接受，应重新构建产物，旧报告仍保留历史意义。 + +## 场景与统计口径 + +| 组 | 场景 | 计时/验证 | +| --- | --- | --- | +| memory | 同步空闲、异步空闲、256 个等待 Channel 的任务、HTTP/HTTPS 监听 | READY 后采样 RSS/PSS、VmHWM、线程/FD；parked 用取消场景的等待阶段，不另建实现 | +| async | 16384 次 run/wait、16384 次有界 Channel 发送与消费、取消并排空 256 个已启动任务 | 源码内单调时钟；计算结果/数量必须正确，打印在计时外；前两项每进程 10 轮、去掉首轮，取消保留完整排空耗时 | +| http | HTTP/HTTPS × Dever/Rust-runtime/原生 Hyper × 三种响应 | 固定速率、有限并发；状态码、Content-Type 和完整正文必须匹配；连接复用与新建连接可分别运行 | +| http2 | h2c prior knowledge/TLS ALPN × Dever/Rust-runtime/原生 Hyper × 物理连接/流拓扑 | 计时前建立固定物理连接；校验握手数、状态码、Content-Type、完整正文、吞吐、延迟和资源 | +| http2-recovery | 同一服务进程内重复建立、使用、关闭 HTTP/2 连接 | 每轮校验请求总账和握手数，断开后 FD 必须回到 READY 基线，保留 RSS 序列 | +| live | TCP/WebSocket/SSE × 连接档位 × 多轮连接/保持/断开 | TCP/WS 校验带轮次和连接标识的回显；SSE 校验初始事件与心跳；对比各轮断开后的 FD/RSS | +| orm | SQLite/PostgreSQL 的 idle、CRUD、list/cursor/stream、pool saturation、transaction cancel、HTTP+JSON+DB | 每个 case 使用独立 bundle 和数据库；校验精确结果，再报告数据库操作延迟、吞吐、资源和连接回收 | +| cms | `examples/cms/dever` 和 `examples/cms/md` 的 Model、Seed、用户注册和发布事务 | 相同入口、单独 SQLite bundle，业务初始化与事务完成后分别采样 RSS/PSS/VmHWM 和可用时的 cgroup peak | + +## ORM 基准 + +所有 Dever 产物在 `build` 阶段通过公开 `dever` 项目路径生成；无数据库声明的 runtime/http/live 使用 base archive。`run` 阶段只复制和执行已有二进制。每个数据库 case 都创建独立 bundle；SQLite 使用该 bundle 自己的 `data/db/orm.db`，重复轮次不会复用数据库状态。 + +构建器把测试源码暂存到 `module/benchmark/<domain>/app.dever`，共享能力位于 `benchmark/bench`、`benchmark/config`，由 `api.dever` 声明 CMD。fixture 不生成 main/package/exposes，也不向 `dever build` 传任意 entry。 + +CMS 从原始双源码项目构建，通过真实登录/租户/发布 HTTP 流程测量，不注入新的业务入口。普通 `build` 只构建 Dever CMS，`cms-profiles` 构建双源码。下方历史 `READY|3` 报告不代表当前实现。 + +2026-09-18 的当前源码专项报告保存在 `target/performance/cms-dual-build-09-18-v2/{manifest.json,profile-report.json}` 和 `target/performance/cms-dual-rss-09-18/report.json`。base/SQLite/PostgreSQL/both 原生程序分别为 411,352 / 2,558,200 / 2,708,112 / 4,672,888 字节。CMS Dever/Markdown 产物分别为 3,151,352 / 3,151,416 字节；各运行 3 次，READY 全部为 3，完成时间中位数分别为 19.87 / 19.69 ms，空闲 RSS 中位数同为 5.37 MiB，PSS 分别为 3.28 / 3.33 MiB，VmHWM 中位数同为 5.37 MiB。采用本机热缓存、独立 SQLite bundle、每次 READY 后 2 秒采样；无吞吐负载，不能把启动时间或空闲内存解释为业务吞吐/峰值容量。当前 cgroup 父目录不可写，未施加 64/128 MiB 硬限额，也未记录 cgroup peak。 + +默认只构建和运行 SQLite。真实 PostgreSQL 统一读取仓库根 `config/setting.json` 的 `database.postgres_test`，不接受环境变量或命令行连接覆盖；该本地文件已被 Git 忽略。其 `url` 必须在数据库名中包含唯一 `{case}`，例如 `postgres://bench:secret@127.0.0.1/dever_{case}`，查询参数或用户名中的占位符不算隔离。运行器按“输出目录短哈希 + driver + entry + repeat”展开 case，因此不同报告不会复用数据库；构建器和运行器以 create-new 和 `0600` 权限原子写入 bundle 的 `config/setting.json`，构建结束会删除含连接信息的临时配置，URL 不写入 manifest 或 report。运行器不会创建或清理 PostgreSQL 数据库，因此模板必须指向预先准备的专用测试数据库，不能指向已有业务库。 + +```json +{ + "database": { + "default": { + "type": "sqlite", + "path": "data/db/test.db" + }, + "postgres_test": { + "type": "postgres", + "url": "postgres://bench:secret@127.0.0.1/dever_{case}", + "tls": "disabled", + "min_connections": 0, + "max_connections": 4, + "max_page_size": 100, + "wait_timeout_ms": 5000, + "io_timeout_ms": 30000 + } + } +} +``` + +```bash +python3 test/performance/run.py run \ + --artifacts target/performance/build-current \ + --output target/performance/orm-smoke \ + --suite orm --duration 1 --warmup 0.2 --repeats 1 + +python3 test/performance/run.py run \ + --artifacts target/performance/build-current \ + --output target/performance/orm-postgres-smoke \ + --suite orm --duration 1 --warmup 0.2 --repeats 1 +``` + +ORM 报告包含 binary size、RSS/PSS/VmHWM、cgroup current/peak/OOM、线程/FD、吞吐和 p50/p95/p99。当前生产 runtime 没有低成本的 pool wait、SQL 数和数据库连接数观测点，报告将这些字段明确标为 `unavailable`，同时记录配置的 pool capacity 和进程 FD，不用推算值冒充真实指标。transaction cancel 先验证单连接 pool 可用，再由长事务占满 pool、在响应完成前断开客户端，并要求连接在 HTTP handler timeout 前归还、计数仍为 0；这样不会把 handler timeout 回滚误报为客户端取消，也不会绕过语言对 transaction 内并发任务的静态限制。 + +run/wait 同时跑 Dever 与 Rust-runtime 的 root/worker 两种位置。root 在 `run_entry_with` 入口执行；worker 在一个计时外创建的结构化外层任务内执行。二者内部都用原有 `task::run/wait`、Bytes 输入和 checked Int 加法，保留子任务额度、取消和清理；没有以裸 Tokio task 代替语言任务。这个实验不修改默认调度。Channel/取消暂时仍是 Dever 单方基线。`/proc` 上下文切换计数汇总采样时仍存活的线程，线程退出会使累计值减少；总量分析用独立 `perf stat`，不能将采样差值当作精确任务计数。 + +live 用 `--connection-tiers 32,128,512 --cycles 3 --duration 2` 配置，每个档位独立启动服务，每轮保持约 2 秒，再关闭全部客户端并等待 300 ms。Rust 对端发布零基轮次的 `connecting/connected/closing/disconnected` 四阶段；Python 只用 connected/disconnected 保持窗口比较资源，记录最后一次断开样本和相对基线的 FD/RSS 差值。采样前后阶段变化的过渡样本不归属任何阶段，也不写入阶段报告。正文不匹配、阶段缺失、SSE 保持期无心跳或未回收全部客户端都失败；RSS 保留可能来自分配器缓存，不能单凭不回落断言泄漏。默认只测 32 条，档位不能超过构建时的 connections；`--duration` 为 0.3–60 秒，采样间隔最多 100 ms。live 的 SSE/WS 是明文 HTTP/1 场景，不包含 WSS 或 HTTP/2。 + +HTTP/2 用 `物理连接数 x 每连接并发流数` 描述拓扑。当前固定每连接 16 流、单流接收窗口 65535 字节、连接接收窗口 262144 字节、frame/send-buffer 上限 16384 字节。客户端在计时前完成精确数量的 h2c 或 TLS+h2 握手，每个连接的 `SendRequest` clone 由其 16 个请求槽共享；测量期间不重试、不替换连接、不自动扩容。`request_slots` 必须等于两个维度的乘积，`successful_handshakes` 必须等于物理连接数。断开后等待 300 ms，再以 READY 基线核对 FD。 + +三个路径为 `/plain`（`hello`）、`/json`（预制 `{"ok":true}`）、`/bytes`（65536 个 `x`）。JSON 路径测 JSON 响应传输，不代表动态 JSON 解析/序列化性能。`/bytes` 跨越 HTTP/2 默认单流接收窗口，用于验证流控下的大正文传输。大正文在服务启动时准备，各服务均无业务日志。 + +网络记录 `scheduled/sent/completed/succeeded/errors/dropped`；`dropped_capacity` 表示无可用请求槽，`dispatch_expired` 表示负载发生器未能在窗口内发出。`latency_ms` 从**计划发送时间**算到成功完成，包含排队/调度；`request_latency_ms` 单独测实际发出到成功完成，`dispatch_lag_ms` 记录调度延迟。不能用两个 p99 相减估算请求耗时。没有成功样本时成功延迟为 null，失败不会稀释成功分位数。Tokio 定时器会让高频请求成批到期，这个发生器用于可复现的有界负载和相对比较，不适合独立证明微秒级服务延迟或极限吞吐。检查负载端 CPU、lag、丢弃，再解释服务结果。 + +延迟使用有界直方图：1 ms 内按微秒分桶，之后每个二倍区间分 64 桶，分位数取桶上界。每轮保留原始计数和资源采样，不把多个 p99 再平均成整体 p99。异步的 `p95_ns_per_operation_sample` 是批次平均耗时的分位数，**不是逐任务 p95**。`ready_seconds` 是父进程启动到观察 READY 的耗时，包含创建进程和最多约 5 ms 的观察误差。 + +## 小机器运行预算 + +建议先测 1 核/128 MiB，再试 64 MiB 和 2 核/256 MiB。这里是应用预算，不包含操作系统自身内存。`--server-cpu` 固定服务端 CPU，`--client-cpus` 指定逗号分隔的客户端 CPU 集合；`--client-workers` 指定 HTTP/1、HTTP/2 和 live 负载的 worker 数。客户端 CPU 不能包含服务端 CPU；不指定客户端集合时继承当前允许的 CPU。`--cpu-quota` 是计算份额上限，不能代替绑核。 + +```bash +python3 test/performance/run.py run \ + --artifacts target/performance/build-current \ + --output target/performance/budget-128m-1 \ + --suite all --duration 10 --warmup 2 --repeats 3 \ + --rates 100,1000 --paths /plain,/json,/bytes \ + --concurrency 16 --connection-mode both \ + --http2-connections 1,4 --recovery-connections 1 \ + --connection-tiers 32,128,512 --cycles 3 \ + --server-cpu 0 --client-cpus 1 --client-workers 1 \ + --cgroup-parent /sys/fs/cgroup/system.slice \ + --memory-mib 128 --cpu-quota 1 +``` + +上例 cgroup 路径只适用于有相应权限且父组已启用 cpu/memory 控制器的机器。工具为每个被测进程创建新的 `dever-perf-*` 子组，仅把自己的子进程放进去；不会修改父组、其他服务或全局 swap。指定内存限额时把该子组 swap 限额设为零。没有权限会明确失败，不回退成“无限额通过”。负载端和 Python 采样进程留在预算之外。 + +`memory.current/peak` 包含该组被计费的用户内存、内核/网络资源等；不能与进程 RSS 混用。报告同时保留两者、OOM 事件和 CPU 节流统计。进程由 Python 子进程 exec 启动，cgroup 峰值也可能包含 exec 前短暂的 Python 启动页，空闲基线应看 READY 后的 current/RSS/PSS。50 ms 默认采样可能漏掉短暂峰值，VmHWM 和 cgroup peak 补充整个生命周期峰值。 + +构建和指纹读取会预热文件缓存，已缓存的可执行文件/共享库页可能已计费到父组或其他组，因此本工具的 cgroup current 可以小于进程 RSS。它不是冷启动总 RAM 成本；三者应交叉观察。本阶段测热文件缓存下的新进程，不操作系统 `drop_caches`，也不据此宣称整台机器最低内存。 + +运行器在结束/异常/中断时终止并回收自己创建的进程，删除自己的空 cgroup。memory/http/http2/live 服务最终由运行器结束进程，**不把这个过程当作语言优雅退出验证**；http2-recovery 和 live 单独验证连接多轮断开后的回收。小时级 soak、故障网络和真实小机器矩阵需另行执行。 + +每轮 Rust 负载程序使用同一已验摘要的不可变二进制硬链接，各自保留独立 `config/setting.json` 和日志；跨文件系统才复制。测量期间不得改写输入二进制。这样 100 轮连接恢复不会累积 100 份相同程序，也不通过符号链接让配置解析回到源程序目录。 + +## 对照边界与报告 + +- Rust-runtime 服务使用与 Dever 相同的 HTTP/TLS/task runtime，帮助定位源代码桥接开销。 +- 原生 Hyper 服务匹配本测试正常 GET 的响应、连接数、头/正文上限和 TLS 配置；没有复刻 runtime 的完整 WriteTimeout、程序故障传播和结构化 Scope。不能由这个对照推断慢读/慢上传、异常报文的能力等价。 +- 每轮倒转实现顺序，减轻固定执行顺序造成的偏置。构建配置、CPU 型号、CPU 亲和性、父 cgroup、二进制指纹、原始日志、资源采样、负载端占用写入报告。 +- `cases.jsonl` 在每个场景完成后追加；`report.json` 即使中途失败也保留已完成结果。`status=complete` 表示测量完整，不表示没有请求错误或已经达到性能目标。先查看 errors/dropped/OOM，再查看延迟和内存。 +- 不预设绝对 QPS 或最低内存。已有计算热路径的 1.15 对照门槛仍在 `test/dever-tests/tests/performance.rs`，不推广成 HTTP 或整个语言的性能承诺。 + +## 首轮结果：2026-09-12 + +环境为 Linux x86_64、Intel Xeon Processor (Skylake, IBRS) 虚拟 CPU。服务绑定 CPU 0，负载端绑定 CPU 1；服务 runtime 1 worker、2 blocking slots、272 task slots、32 connections，负载端 16 个请求槽。主测每场景预热 0.5 秒、测量 2 秒，重复 3 个独立进程，目标速率 1000 请求/秒。不是持续压力或吞吐上限测量。 + +| Dever 空闲场景 | 三轮 RSS 中位值 | +| --- | ---: | +| 同步程序 | 1.90 MiB | +| 异步运行时 | 2.55 MiB | +| 256 个已启动、等待 Channel 的任务 | 3.38 MiB | +| HTTP 监听 | 3.45 MiB | +| HTTPS 监听 | 4.37 MiB | + +以上为 128 MiB 专属 cgroup 下的 READY 后采样。另用 64 MiB 限额重复 3 轮空闲场景，全部完成、无 OOM。READY 的一次输出会启动异步程序的 blocking worker，异步场景观察到 3 个线程；这是该 fixture 的基线，不是“完全不打印”的最小运行时。 + +异步三轮中位值：完整 `run/wait` 一轮约 15.90 µs，Channel 生产与消费平均每项约 0.89 µs；停止并排空 256 个已启动等待任务约 0.61–0.67 ms。前两项每进程校验 10×16384 项，去掉首批预热；包含结构化任务语义，不能把 run/wait 数字称为纯任务分配耗时。尚未提供这几组异步操作的同语义 Rust 对照。 + +HTTP/HTTPS 三实现、三路径共 54 组正式测量，108000 个计划请求中 107962 个发出且成功，0 请求错误。38 个未发出：32 个超过发送窗口、6 个无空闲并发槽；预热另有 43 个未发出，原始记录保留。各组成功速率约 999 请求/秒，受目标速率限制，不能宣称最大吞吐为 1000。 + +| Dever 路径 | HTTP 每轮实际请求 p99 范围 | HTTPS 每轮实际请求 p99 范围 | +| --- | ---: | ---: | +| /plain | 0.243–1.248 ms | 0.580–0.909 ms | +| /json | 0.280–0.342 ms | 0.897–2.752 ms | +| /bytes | 0.392–0.500 ms | 0.839–2.400 ms | + +表中是 dispatched→completed，每列列出三个独立 p99 的范围，没有把它们合并成全局 p99；计划发送延迟和负载端 lag 另在 JSON 中。短时虚拟机抖动明显，这些结果不用于宣称哪种实现更快。Dever 在这组负载的 RSS 中位值约 HTTP 3.9 MiB、HTTPS 4.9–5.1 MiB。 + +另以每次新建连接、100 请求/秒测试 HTTP/HTTPS 全路径三实现，共 18 组、900 个正式请求，全部成功且无丢弃。主测 78 组（15 内存、9 异步、54 网络）、新连接 18 组、64 MiB 内存 15 组均无 OOM。 + +原始结果及对应产物保存在仓库 `target/performance/`： + +- `baseline-128m/report.json`：最终三轮基线，对应 `build-2/manifest.json`。 +- `connections-new-128m/report.json`：新连接模式，对应 build-2。 +- `memory-64m/report.json`：64 MiB 空闲基线，对应 build-1；该组使用的 Dever 二进制与 build-2 一致。 +- `smoke-1/report.json`：早期连通检查，保留当时指纹，不用于最终网络比较。 + +以上首轮结果保留当时的测试范围；后续测量见下节。 + +## 第二轮：高负载与调度定位 + +`http-load-p2-final/report.json` 复用 build-2 产物，1 worker、32 服务连接和 32 客户端请求槽、服务 1 CPU/128 MiB。仅 `/plain`，预热 0.5 秒、测量 2 秒；HTTP/HTTPS、三个实现、5000/15000/30000 请求每秒，均重复三轮，共 54 组。 + +正式窗口共计划 1800000 个请求，1500830 个实际发出且成功，错误 0。未发出 299170 个：容量丢弃 298476、发送窗口过期 694；所有预算组无 OOM。下表是三个独立进程的成功 QPS 中位值，不是最大吞吐。 + +| 目标请求/秒 | Dever HTTP | Rust-runtime HTTP | Hyper HTTP | Dever HTTPS | Rust-runtime HTTPS | Hyper HTTPS | +| ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 5000 | 4993 | 4994 | 4993 | 4970 | 4972 | 4970 | +| 15000 | 14904 | 14897 | 14922 | 14839 | 14756 | 14726 | +| 30000 | 21866 | 22187 | 22324 | 21217 | 21320 | 22236 | + +最高输入档位 Dever 服务 CPU 中位值为 HTTP 0.53 核、HTTPS 0.62 核，客户端为 0.77/0.86 核；Dever RSS 约 4.12/5.32 MiB。32 请求槽与定时器成批发送已经影响结果，不能据此断言服务端上限是每秒两万。提高并发槽、使用独立负载机器应作为下一次吞吐上限测量的条件。 + +`http-load-p2/report.json` 保留初次中断：Linux 退出时 `/proc` 内存字段先于 waitpid 消失，导致采样工具失败。修复后才完整重跑；不能把初次中断算成语言服务失败，也不将两次样本混合。 + +`async-p2/report.json` 使用 build-p2、1 worker/1 CPU/128 MiB、1040 task slots、每批 16384 次操作，重复 3 个进程；每进程十批、去掉首批。共 18 组，校验全部通过且无 OOM。下表为进程内中位数再取三进程中位数： + +| run/wait 位置 | Dever | 同 runtime、同优化参数的 Rust | +| --- | ---: | ---: | +| 入口 root | 15.50 µs/次 | 15.59 µs/次 | +| 外层结构化任务 worker | 2.003 µs/次 | 1.984 µs/次 | + +这个串行微任务场景中，调度位置使 Dever 平均单次耗时降低约 87%，约 7.7 倍速度；不推广为所有业务程序加速。两种语言路径数值接近。源码路径显示 `run_entry_with` 用入口线程 block_on，run body 和 supervisor 在 Tokio worker 执行；单核下逐项等待造成频繁 OS 线程切换。`profile-p2/final-runtime-*.stat` 是独立 perf stat 的整体软件计数，对应 build-p2/async_bench；未加 cgroup、固定 CPU 0。虚拟机不支持 cycles/instructions，未报告硬件 IPC 或采样火焰图。 + +同组 Channel 约 0.912 µs/项；256 个任务停止并排空三次共 0.688/0.916/1.045 ms。这里只新增对照和证据，生产入口调度尚未优化。后续优化入口调度时需确认入口 future 的 Send 条件，根任务不应挤占子任务配额，并验证入口故障、子任务取消和 blocking 清理；不应直接照搬实验外层 run 或删除结构化任务语义。 + +## 第二轮：长连接容量与回收 + +`live-p2-final/report.json` 使用 build-p2-final、1 worker/1 CPU/128 MiB、512 connection slots、1040 task slots；32/128/512 条连接各重复三进程，每进程三轮连接/保持 2 秒/断开。TCP/WS 每约 100 ms 校验一次小消息回显，SSE 校验初始事件和 100 ms 心跳。 + +27 组共建立并关闭 18144 条连接，校验 377061 条消息，无协议错误和 OOM。81 次断开后的服务 FD 均回到启动基线。下面是第三轮保持窗口 RSS 的三个独立进程中位值，包含分配器在此前两轮保留的内存： + +| 同时连接数 | TCP RSS | WebSocket RSS | SSE RSS | +| ---: | ---: | ---: | ---: | +| 32 | 3.99 MiB | 5.11 MiB | 5.03 MiB | +| 128 | 4.90 MiB | 9.44 MiB | 9.16 MiB | +| 512 | 8.70 MiB | 17.00 MiB | 24.97 MiB | + +512 条连接保持时服务 CPU 约 TCP 0.15–0.16 核、WS 0.18–0.20 核、SSE 0.12–0.14 核。这里测的是持续小消息和心跳连接，不是大消息吞吐或完全不读的客户端。 + +初次 `live-p2/report.json` 也完成了全部消息校验，但有一条下一轮的采样被误归属上轮 disconnected，产生假的 FD +15。最终报告使用采样前后阶段一致性判断，完整重跑；资源回收结论只引用 final 报告。 + +随后 `live-cycles-p2/report.json` 对每种协议保持 512 条连接，十轮重连、每轮保持 3 秒，单场景约 34 秒。共 15360 次连接、471207 条校验消息，错误/OOM 为零，30 次 FD 回落均为零差值。TCP 断开后的 RSS 最后稳定在约 8.6–8.8 MiB，WS 约 17 MiB；SSE 前三轮 21.33/24.48/25.12 MiB，后七轮为 25.24–25.36 MiB，未观察到按轮持续增长。这是短时重复回收证据，不是小时级泄漏或生产网络稳定性证明。 + +复现本轮长连接两组命令（输出目录另取新名称）： + +```bash +python3 test/performance/run.py run --artifacts target/performance/build-p2-final --output target/performance/live-repeat --suite live --duration 2 --connection-tiers 32,128,512 --cycles 3 --repeats 3 --server-cpu 0 --client-cpus 1 --client-workers 1 --cgroup-parent /sys/fs/cgroup/system.slice --memory-mib 128 --cpu-quota 1 +python3 test/performance/run.py run --artifacts target/performance/build-p2-final --output target/performance/live-more-cycles --suite live --duration 3 --connection-tiers 512 --cycles 10 --repeats 1 --server-cpu 0 --client-cpus 1 --client-workers 1 --cgroup-parent /sys/fs/cgroup/system.slice --memory-mib 128 --cpu-quota 1 +``` + +本轮 12 个运行器检查、4 个独立 TCP/SSE 对端检查通过，11 个 Dever 入口及两个新 Rust example 离线构建通过。Rustfmt/Clippy 未安装，未运行；Dever 格式、Python 解析、源码空白与最终范围检查通过。生产编译器/运行时没有修改，未运行全量测试。 + +## 第三轮：HTTP/2 吞吐、流控与连接回收 + +2026-09-13 使用 `build-http2-final` 产物完成正式矩阵。环境仍为 Linux x86_64、Intel Xeon Processor (Skylake, IBRS) 虚拟 CPU；服务绑定 CPU 0，负载端绑定 CPU 1。服务使用 1 worker、128 MiB cgroup、1 CPU quota、swap 0；负载端不计入服务预算。每个 case 预热 0.5 秒、测量 2 秒、启动独立服务进程，三实现与两种传输的执行顺序逐轮倒转。 + +`/plain` 共 108 个 case、3600000 个计划请求，其中 2729242 个发出且完整成功，错误 0。870758 个未发出请求中，869368 个受请求槽限制，1390 个超过调度窗口；全部计数守恒。108 次正式负载和 108 次预热的握手数均与物理连接数一致，断开后的 FD 均回到 READY 基线，无 OOM。下表是 30000 请求/秒输入档位的三轮中位数；p99 是实际发出到完整响应的请求延迟。 + +| 实现 | 传输 | 1x16 成功 QPS | 4x16 成功 QPS | 4x16 请求 p99 | 4x16 服务 RSS | +| --- | --- | ---: | ---: | ---: | ---: | +| Dever | h2c | 10578 | 29575 | 2.208 ms | 4.60 MiB | +| Rust-runtime | h2c | 10804 | 28996 | 3.392 ms | 5.00 MiB | +| Hyper | h2c | 11956 | 29108 | 1.984 ms | 4.59 MiB | +| Dever | TLS+h2 | 9957 | 29814 | 1.840 ms | 5.59 MiB | +| Rust-runtime | TLS+h2 | 10122 | 28681 | 2.880 ms | 6.07 MiB | +| Hyper | TLS+h2 | 11555 | 28719 | 3.200 ms | 5.68 MiB | + +Dever 4x16 在 5000、15000、30000 三档的 h2c 成功 QPS 中位值分别为 4995、14983、29575，TLS+h2 分别为 4994、14977、29814。1x16 在高输入下只有约 10k–10.6k 成功 QPS，而 4x16 接近 30k，说明这次单连接结果受固定 16 个请求槽明显约束；不能把任一数字解释成服务端极限。最高档 4x16 的 Dever 服务 CPU 中位值为 h2c 0.89 核、TLS+h2 0.93 核，负载端也约 0.9 核，进一步限制了极限吞吐结论。三种实现的短时差异不具备跨机器性能承诺意义。 + +`/bytes` 共 108 个 case、115200 个计划请求，115180 个完整成功、错误 0、丢弃 20、无 OOM，握手和 FD 检查全部通过。三种实现、两种传输、两种拓扑在 100/500/1000 请求每秒档均维持目标速率；这相当于单 case 最高约 62.5 MiB/s 的响应正文。Dever 的 RSS 中位值为 h2c 4.04–4.23 MiB、TLS+h2 5.05–5.30 MiB。短 case 的 p99 有明显虚拟机抖动，原始三轮数据比单个汇总值更适合定位异常。 + +`http2-recovery-128m` 对三种实现、h2c/TLS 各在同一服务进程中运行 10 轮 1x16 连接周期，共 60 轮、30000 个计划请求；29984 个完整成功、错误 0、丢弃 16。每轮握手数正确，60 次断开后的 FD 差值均为 0，无 OOM。六组最后一轮相对第一轮的断开后 RSS 变化为 8–100 KiB，短时序列未观察到按轮持续增长；这不是小时级泄漏证明。 + +另在 64 MiB、1 CPU、swap 0 下运行 Dever 1x16、5000 请求每秒的 h2c/TLS 代表场景。20000 个计划请求中 19991 个完整成功、错误 0、丢弃 9、无 OOM；h2c/TLS 成功 QPS 为 4997/4996，RSS 为 4.04/5.17 MiB，PSS 为 2.00/3.10 MiB。cgroup peak 只有 1.30/1.35 MiB，是热文件页可能已计费给其它组所致，不能代替进程 RSS/PSS 或冷启动总内存。 + +正式结果和构建清单位于： + +- `build-http2-final/manifest.json`：16 个二进制的指纹、大小、构建配置和固定 HTTP/2 limits。 +- `http2-plain-128m/report.json`：`/plain` 128 MiB 正式矩阵。 +- `http2-bytes-128m/report.json`：`/bytes` 128 MiB 正式矩阵。 +- `http2-recovery-128m/report.json`：128 MiB 十轮连接恢复矩阵。 +- `http2-active-64m/report.json`：64 MiB Dever 活跃代表场景。 + +历史 `build-http2-final` 产物已不在当前工作树；以下是当时使用的命令记录，精确复现历史二进制须先恢复匹配的旧产物。本机当前可直接运行的命令见下方第四轮，复现时必须换新输出目录： + +```bash +python3 test/performance/run.py run --artifacts target/performance/build-http2-final --output target/performance/http2-plain-repeat --suite http2 --duration 2 --warmup 0.5 --rates 5000,15000,30000 --paths /plain --http2-connections 1,4 --repeats 3 --server-cpu 0 --client-cpus 1 --client-workers 1 --cgroup-parent /sys/fs/cgroup/system.slice --memory-mib 128 --cpu-quota 1 +python3 test/performance/run.py run --artifacts target/performance/build-http2-final --output target/performance/http2-bytes-repeat --suite http2 --duration 2 --warmup 0.5 --rates 100,500,1000 --paths /bytes --http2-connections 1,4 --repeats 3 --server-cpu 0 --client-cpus 1 --client-workers 1 --cgroup-parent /sys/fs/cgroup/system.slice --memory-mib 128 --cpu-quota 1 +python3 test/performance/run.py run --artifacts target/performance/build-http2-final --output target/performance/http2-recovery-repeat --suite http2-recovery --duration 0.5 --warmup 0.1 --cycles 10 --recovery-rate 1000 --recovery-path /plain --recovery-connections 1 --repeats 1 --server-cpu 0 --client-cpus 1 --client-workers 1 --cgroup-parent /sys/fs/cgroup/system.slice --memory-mib 128 --cpu-quota 1 +``` + +N5 的接口与约束见 [LANGUAGE.md](../../LANGUAGE.md)。这轮只建立 loopback、热文件缓存、短时固定速率下的相对证据，不包含跨主机网络、故障注入、小时级 soak 或多平台矩阵。HTTP/2 不保证所有负载都比 HTTP/1.1 快；普通 WebSocket 继续走 HTTP/1，RFC 8441 扩展 CONNECT 尚未实现。 + +## 第四轮：多核负载端、HTTP/2 饱和与稳态 + +2026-09-18 使用 `typed-orm-build-v2/manifest.json` 中校验过指纹的现存 HTTP/2 二进制和测试证书；原 `build-http2-final` 不在当前工作树，未重建产物。服务端绑定 CPU 0、1 worker、1 CPU quota、无 swap；负载端为 CPU 1–4、4 workers，位于服务 cgroup 外。机器允许 CPU 0–4。服务预算为 128 MiB，稳态另测 64 MiB。所有场景使用 `/plain`、16 条物理连接、每连接 16 流，只有自有 loopback 端口；这是同一虚拟机的热缓存观测，不是跨机器或生产容量。 + +128 MiB 扫描覆盖 h2c/TLS+h2 的 30k/45k/60k/90k 输入，每档预热 1 秒、计时 5 秒、2 个重复。16/16 case 计数守恒，请求错误、OOM 和断开后的 FD 差均为 0；每次握手数为 16。下表为成功 QPS 的两轮中位值： + +| 传输 | 30k | 45k | 60k | 90k | +| --- | ---: | ---: | ---: | ---: | +| h2c | 29,899 | 36,690 | 36,027 | 39,486 | +| TLS+h2 | 29,770 | 38,205 | 37,410 | 38,136 | + +45k 以上服务 CPU 约 0.96–1.00 核，客户端约 1.43–1.97 核；未发送请求主要因 256 个请求槽满，调度窗口过期数量相对很少。因此观测到当前拓扑约 36k–40k req/s 的吞吐平台，不能把它归结为纯服务端极限或把输入 90k 当成实际吞吐。28k 的首轮 60 秒成功率为 h2c 98.68%、TLS+h2 97.91%，未达 99% 验收门槛；24k 的 10 秒校准 TLS+h2 为 99.18%，余量仍小。20k 的 10 秒校准分别为 99.96% 和 99.92%，所以正式稳态统一采用 20k，低于较低观测平台的 80%。这些探索报告保留，不算正式稳态通过。 + +正式每个 case 预热 1 秒、测量 60 秒、计划 1,200,000 次请求。成功率是成功请求除以计划请求，容量丢弃仍在原始报告中，不称为零丢弃： + +| 服务预算 | 传输 | 成功/计划 | 成功 QPS | 请求 p99 | 服务 RSS 高水位 | OOM/错误/FD 差 | +| --- | --- | ---: | ---: | ---: | ---: | --- | +| 128 MiB | h2c | 1,197,220/1,200,000 (99.77%) | 19,953 | 5.376 ms | 6.78 MiB | 0/0/0 | +| 128 MiB | TLS+h2 | 1,199,036/1,200,000 (99.92%) | 19,983 | 4.800 ms | 7.83 MiB | 0/0/0 | +| 64 MiB | h2c | 1,199,221/1,200,000 (99.94%) | 19,986 | 4.672 ms | 6.43 MiB | 0/0/0 | +| 64 MiB | TLS+h2 | 1,197,545/1,200,000 (99.80%) | 19,958 | 5.184 ms | 7.90 MiB | 0/0/0 | + +128 MiB 下在各自的同一服务进程上完成 h2c/TLS+h2 各 100 轮 16x16 恢复；每轮 0.1 秒、1000 req/s，随后观察断开 0.3 秒。两组各计划 10,000 次、成功 9,944 次，其余各 56 次为调度窗口过期，请求错误和 OOM 为 0。每轮至少成功 95（h2c）/98（TLS+h2）次，握手始终 16，200 次断开后的 FD 差全部为 0。断开后 RSS 从首轮到末轮为 h2c 4,845,568→5,357,568 B，TLS+h2 5,959,680→6,647,808 B；保留完整逐轮序列，不将分钟级增长或回落解释成小时级泄漏结论。邻接的 HTTP/HTTPS 各 30/30 成功，TCP/WS/SSE 各 16/16 连接完成关闭，零错误。 + +复现命令（输出目录必须换新名称；运行前确认 CPU 0–4 和 cgroup 权限，不接触现有服务）： + +```bash +python3 test/performance/run.py run --artifacts target/performance/typed-orm-build-v2 --output target/performance/http2-scan-next --suite http2 --duration 5 --warmup 1 --rates 30000,45000,60000,90000 --paths /plain --http2-connections 16 --repeats 2 --implementations dever --server-cpu 0 --client-cpus 1,2,3,4 --client-workers 4 --cgroup-parent /sys/fs/cgroup/system.slice --memory-mib 128 --cpu-quota 1 +python3 test/performance/run.py run --artifacts target/performance/typed-orm-build-v2 --output target/performance/http2-steady-128-next --suite http2 --duration 60 --warmup 1 --rates 20000 --paths /plain --http2-connections 16 --repeats 1 --implementations dever --server-cpu 0 --client-cpus 1,2,3,4 --client-workers 4 --cgroup-parent /sys/fs/cgroup/system.slice --memory-mib 128 --cpu-quota 1 +python3 test/performance/run.py run --artifacts target/performance/typed-orm-build-v2 --output target/performance/http2-steady-64-next --suite http2 --duration 60 --warmup 1 --rates 20000 --paths /plain --http2-connections 16 --repeats 1 --implementations dever --server-cpu 0 --client-cpus 1,2,3,4 --client-workers 4 --cgroup-parent /sys/fs/cgroup/system.slice --memory-mib 64 --cpu-quota 1 +python3 test/performance/run.py run --artifacts target/performance/typed-orm-build-v2 --output target/performance/http2-recovery-next --suite http2-recovery --duration 0.1 --cycles 100 --recovery-rate 1000 --recovery-connections 16 --repeats 1 --implementations dever --server-cpu 0 --client-cpus 1,2,3,4 --client-workers 4 --cgroup-parent /sys/fs/cgroup/system.slice --memory-mib 128 --cpu-quota 1 +``` + +本轮原始结果位于 `target/performance/http2-ceiling-{scan,steady-128m-20k,steady-64m-20k,recovery}-09-18/`；28k 首轮及 24k/20k 校准在相邻命名目录。吞吐平台、20k 持续负载和 100 轮恢复是不同问题，不能互相替代；服务与负载仍共享虚拟机和热缓存，结果不外推到真实网络、小物理机或小时级运行。 diff --git a/test/performance/build.py b/test/performance/build.py new file mode 100644 index 0000000..f03a46c --- /dev/null +++ b/test/performance/build.py @@ -0,0 +1,466 @@ +"""构建可搬运的基准产物；构建耗时与运行测量分开。""" + +from copy import deepcopy +import hashlib +import json +from pathlib import Path +import secrets +import shutil +import subprocess +import tempfile +import time +from urllib.parse import urlsplit + +from settings import PEER_CONFIGURATION, stage_peer, write_private_json + +ROOT = Path(__file__).resolve().parents[2] +FIXTURES = Path(__file__).parent / "fixtures" +TLS = ROOT / "test/dever-tests/fixtures/tls" +SETTING_PATH = ROOT / "config/setting.json" +POSTGRES_TEST_CONNECTION = "postgres_test" +APPLICATION_CONFIGURATION = "llvm-cmd-v1" +ENTRIES = { + "runtime": ("sync_idle", "async_idle", "tasks", "tasks_on_worker", "channels", "cancel"), + "http": ("http", "https", "http2", "https2"), + "live": ("tcp", "ws", "sse"), +} +ORM_ENTRIES = ("orm_idle", "orm_crud", "orm_list", "orm_cursor", "orm_stream", + "orm_pool", "orm_http") +RUNTIME_PROFILES = { + "base": ("runtime-api", "runtime-external"), + "sqlite": ("runtime-api", "runtime-external", "runtime-sqlite"), + "postgres": ("runtime-api", "runtime-external", "runtime-postgres"), + "both": ("runtime-api", "runtime-external", "runtime-sqlite", "runtime-postgres"), +} +BENCHMARK_COMMAND = "benchmark.command.run" +CONFIG_FUNCTIONS = { + "runtime": ("lifetime", "idle_sample_ms", "pending", "channel_capacity", "input", "pending_input"), + "http": ("lifetime", "limits", "http2_limits", "large_body", "certificate", "key"), + "live": (), + "orm": ("lifetime", "orm_input", "orm_rows", "orm_concurrency", "orm_cancel_work"), +} + + +def fingerprint(path): + with path.open("rb") as binary: + digest = hashlib.file_digest(binary, "sha256").hexdigest() + return {"sha256": digest, "bytes": path.stat().st_size} + + +def postgres_test_connection(document): + try: + connection = document["database"][POSTGRES_TEST_CONNECTION] + except (KeyError, TypeError) as error: + raise ValueError( + f"config/setting.json must define database.{POSTGRES_TEST_CONNECTION}" + ) from error + if not isinstance(connection, dict): + raise ValueError(f"database.{POSTGRES_TEST_CONNECTION} must be an object") + if connection.get("type") != "postgres": + raise ValueError(f"database.{POSTGRES_TEST_CONNECTION} must use type 'postgres'") + max_connections = connection.get("max_connections") + if type(max_connections) is not int or not 1 <= max_connections <= 256: + raise ValueError( + f"database.{POSTGRES_TEST_CONNECTION}.max_connections must be an explicit 1..256 integer" + ) + return connection + + +def postgres_case_url(document, case_name): + template = postgres_test_connection(document).get("url") + if not isinstance(template, str): + raise ValueError(f"database.{POSTGRES_TEST_CONNECTION}.url must be Text") + parsed = urlsplit(template) + database = parsed.path.removeprefix("/") + if (template.count("{case}") != 1 or "{case}" not in database + or not database or "/" in database): + raise ValueError( + f"database.{POSTGRES_TEST_CONNECTION}.url must contain one {{case}} placeholder " + "in its database name" + ) + if parsed.scheme not in ("postgres", "postgresql"): + raise ValueError( + f"database.{POSTGRES_TEST_CONNECTION}.url must use postgres:// or postgresql://" + ) + return template.replace("{case}", case_name) + + +def postgres_database_case(output, case_name): + run = hashlib.sha256(str(output.resolve()).encode()).hexdigest()[:12] + return f"{run}_{case_name.replace('-', '_')}" + + +def load_postgres_setting(path=SETTING_PATH): + if not path.is_file(): + return None + try: + document = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError) as error: + raise ValueError(f"cannot load '{path}': {error}") from error + if not isinstance(document, dict): + raise ValueError(f"'{path}' must contain a JSON object") + connections = document.get("database") + if connections is None: + return None + if not isinstance(connections, dict): + raise ValueError(f"'{path}' database must be an object") + if POSTGRES_TEST_CONNECTION not in connections: + return None + postgres_case_url(document, "validation") + return document + + +def configuration(args, postgres_setting=None): + settings = { + "workers": args.workers, + "connections": args.connections, + "pending": args.pending, + "iterations": args.iterations, + "channel_capacity": 64, + "timeout_ms": 2000, + "lifetime_ms": 3_600_000, + "idle_sample_ms": 3000, + "task_capacity": max(args.pending + 16, 2 * args.connections + 16), + "max_blocking_threads": 2, + "http2_streams": args.http2_streams, + "http2_stream_window_bytes": args.http2_stream_window_bytes, + "http2_connection_window_bytes": args.http2_connection_window_bytes, + "orm_iterations": args.orm_iterations, + "orm_rows": args.orm_rows, + "orm_concurrency": args.orm_concurrency, + "orm_cancel_work": args.orm_cancel_work, + "orm_sqlite_connections": args.orm_sqlite_connections, + "orm_postgres_connections": ( + postgres_test_connection(postgres_setting)["max_connections"] + if postgres_setting is not None else None + ), + } + if settings["task_capacity"] > 65536: + raise ValueError("task capacity exceeds the language limit") + return settings + + +def config_source(settings, fixture): + names = CONFIG_FUNCTIONS[fixture] + functions = { + "lifetime": ("Int", str(settings["lifetime_ms"])), + "idle_sample_ms": ("Int", str(settings["idle_sample_ms"])), + "pending": ("Int", str(settings["pending"])), + "channel_capacity": ("Int", str(settings["channel_capacity"])), + "input": ("Bytes", f'dever.bytes.from_text({json.dumps("x" * settings["iterations"])})'), + "pending_input": ("Bytes", f'dever.bytes.from_text({json.dumps("x" * settings["pending"])})'), + "orm_input": ("Bytes", f'dever.bytes.from_text({json.dumps("x" * settings["orm_iterations"])})'), + "orm_rows": ("Bytes", f'dever.bytes.from_text({json.dumps("r" * settings["orm_rows"])})'), + "orm_concurrency": ("Int", str(settings["orm_concurrency"])), + "orm_cancel_work": ("Int", str(settings["orm_cancel_work"])), + "limits": ("dever.http.Limits", "dever.http.Limits {\nheader_bytes = 8192\n" + f'body_bytes = 65536\ntimeout_ms = {settings["timeout_ms"]}\n' + f'connections = {settings["connections"]}\nhttp2 = null\n}}'), + "http2_limits": ("dever.http.Limits", "dever.http.Limits {\nheader_bytes = 8192\n" + f'body_bytes = 65536\ntimeout_ms = {settings["timeout_ms"]}\n' + f'connections = {settings["connections"]}\nhttp2 = dever.http.Http2Limits {{\n' + f'streams = {settings["http2_streams"]}\n' + f'stream_window_bytes = {settings["http2_stream_window_bytes"]}\n' + f'connection_window_bytes = {settings["http2_connection_window_bytes"]}\n}}\n}}'), + } + if "certificate" in names: + functions["certificate"] = ("Bytes", f'dever.bytes.from_text({json.dumps((TLS / "server.pem").read_text())})') + if "key" in names: + functions["key"] = ("Bytes", f'dever.bytes.from_text({json.dumps((TLS / "server-key.pem").read_text())})') + source = "" + for name in names: + if name == "large_body": + continue + kind, expression = functions[name] + source += f"\n{name}() (result: {kind}) {{\nresult = {expression}\n}}\n" + if "large_body" in names: + source += '\ndouble(value: Int, text: Text) (result: Text) { result = text + text }\n' + source += '\nlarge_body() (result: Bytes) {\nresult = dever.bytes.from_text(reduce(double, ' + source += '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], "x"))\n}\n' + return source + + +def stage_application_command(module, capability): + command = module / "benchmark/command" + command.mkdir(parents=True, exist_ok=True) + (command / "app.dever").write_text( + f"run() (result: Bool) {{\n result = {capability}()\n}}\n" + ) + (command / "api.dever").write_text("cmd run = app.run\n") + + +def stage_benchmark_support(module, settings, fixture): + if fixture in ("runtime", "orm"): + bench = module / "benchmark/bench/app.dever" + bench.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(FIXTURES / "bench.dever", bench) + if CONFIG_FUNCTIONS[fixture]: + config = module / "benchmark/config/app.dever" + config.parent.mkdir(parents=True, exist_ok=True) + config.write_text(config_source(settings, fixture)) + + +def database_setting(driver, settings, postgres_setting=None, case_name=None): + if driver == "sqlite": + database = { + "type": "sqlite", + "path": "data/db/orm.db", + "max_connections": settings["orm_sqlite_connections"], + "max_page_size": max(32, min(settings["orm_rows"], 10_000)), + } + elif driver == "postgres": + if postgres_setting is None or case_name is None: + raise ValueError("PostgreSQL requires config/setting.json") + database = deepcopy(postgres_test_connection(postgres_setting)) + database["url"] = postgres_case_url(postgres_setting, case_name) + else: + raise ValueError(f"unknown database driver: {driver}") + document = ({name: deepcopy(postgres_setting[name]) + for name in ("http", "log") if name in postgres_setting} + if driver == "postgres" else {}) + document.setdefault("http", {}) + document.setdefault("log", {}) + document["database"] = {"default": database} + document["auth"] = {"providers": {"benchmark": { + "verify": "benchmark.item.verify", + "jwtSecret": secrets.token_urlsafe(48), + "ttlSeconds": 3600, + }}} + document["sites"] = {"benchmark": {"path": "", "auth": "benchmark"}} + return document + + +def profile_database_setting(profile): + if profile == "base": + return {} + sqlite = { + "type": "sqlite", + "path": "data/db/profile.db", + "max_connections": 1, + "max_page_size": 100, + } + if profile == "sqlite": + return {"database": {"default": sqlite}} + if profile not in ("postgres", "both"): + raise ValueError(f"unknown runtime profile: {profile}") + postgres = { + "type": "postgres", + "url": "postgres://build-only.invalid/dever_profile", + "tls": "disabled", + "max_connections": 1, + } + if profile == "postgres": + return {"database": {"default": postgres}} + return {"database": {"default": sqlite, "postgres_profile": postgres}} + + +def dependency_packages(text): + return sorted({line.removesuffix(" (*)") for line in text.splitlines() if line}) + + +def runtime_dependencies(cargo, features): + command = [cargo, "tree", "--offline", "--locked", "-p", "dever-backend-bridge", + "--no-default-features", "--prefix", "none"] + if features: + command.extend(("--features", ",".join(features))) + return dependency_packages(subprocess.check_output(command, cwd=ROOT, text=True)) + + +def prepare_native_project(destination, fixture, settings): + module = destination / "module" + domain = module / "benchmark" / fixture + domain.mkdir(parents=True) + shutil.copyfile(FIXTURES / fixture / "app.dever", domain / "app.dever") + for role in ("port", "adapter", "api"): + shutil.copyfile(FIXTURES / fixture / f"{role}.dever", + domain / f"{role}.dever") + stage_benchmark_support(module, settings, fixture) + config = destination / "config" + config.mkdir() + write_private_json(config / "setting.json", + native_runtime_setting(settings) if fixture == "live" else {}) + + +def prepare_project(destination, fixture, settings, driver, postgres_setting=None): + module = destination / "module" + domain = module / "benchmark/item" + domain.mkdir(parents=True) + shutil.copyfile(FIXTURES / fixture / "app.dever", domain / "app.dever") + shutil.copyfile(FIXTURES / fixture / "app/model/item.dever", domain / "model.dever") + if fixture == "orm": + shutil.copyfile(FIXTURES / fixture / "app/api.dever", domain / "api.dever") + stage_benchmark_support(module, settings, fixture) + stage_application_command(module, "benchmark.item.orm_idle") + config = destination / "config" + config.mkdir() + write_private_json( + config / "setting.json", + database_setting(driver, settings, postgres_setting, "build_only"), + ) + (destination / "data").mkdir() + + +def prepare_profile_project(destination, setting): + shutil.copytree(FIXTURES / "profile", destination / "module") + config = destination / "config" + config.mkdir() + write_private_json(config / "setting.json", setting) + (destination / "data").mkdir() + + +def prepare_cms_project(destination, source): + shutil.copytree(ROOT / "examples/cms" / source, destination) + + +def native_runtime_setting(settings): + return {"adapter": {"benchmark.live": {"setting": { + "limits": { + "header_bytes": 8192, + "body_bytes": 65536, + "timeout_ms": settings["timeout_ms"], + "connections": settings["connections"], + "http2": None, + }, + "lifetime_ms": settings["lifetime_ms"], + }}}} + + +def record_binary(manifest, name, binary, started, **metadata): + manifest["binaries"][name] = { + **fingerprint(binary), + "build_seconds": time.monotonic() - started, + **metadata, + } + + +def build_profile_report(args, manifest, log): + report = {"source": "profile/benchmark/command/app.dever", "profiles": {}} + for profile, features in RUNTIME_PROFILES.items(): + item = {"runtime_features": list(features), + "dependencies": runtime_dependencies(args.cargo, features)} + print(f"build profile-{profile}", flush=True) + binary_name = f"profile-{profile}" + binary = args.output.resolve() / binary_name + started = time.monotonic() + project = args.output.resolve() / "source" / f"profile-{profile}" + setting = profile_database_setting(profile) + prepare_profile_project(project, setting) + subprocess.run( + [args.compiler.resolve(), "build", project, "--output", binary], + stdout=log, + stderr=log, + check=True, + ) + record_binary(manifest, binary_name, binary, started, + profile=profile, fixture="profile") + item.update({"status": "built", "binary": fingerprint(binary)}) + report["profiles"][profile] = item + path = args.output.resolve() / "profile-report.json" + path.write_text(json.dumps(report, indent=2) + "\n") + manifest["files"]["profile-report.json"] = fingerprint(path) + + +def build(args): + postgres_setting = load_postgres_setting() if args.selection == "all" else None + settings = configuration(args, postgres_setting) + destination = args.output.resolve() + destination.mkdir(parents=True, exist_ok=False) + compiler = args.compiler.resolve() + cargo = shutil.which(str(args.cargo)) + if cargo is None: + raise ValueError(f"cannot find Cargo command: {args.cargo}") + args.cargo = Path(cargo) + has_protocols = args.selection in ("all", "protocols") + peers = ({name: getattr(args, name).resolve() + for name in ("network_bench", "live_bench", "async_bench")} + if has_protocols else {}) + required_tools = (compiler, *peers.values()) + for executable in required_tools: + if not executable.is_file(): + raise ValueError(f"build the executable first: {executable}") + # 在昂贵构建之前拒绝旧环境变量协议产物；这里只读配置，不启动网络。 + for name, peer in peers.items(): + if name == "async_bench": + continue # The finite Rust reference takes explicit positional arguments. + with tempfile.TemporaryDirectory(prefix="dever-peer-config-") as temporary: + executable = stage_peer(peer, Path(temporary), {"workers": 2}) + result = subprocess.run([executable, "check-config"], capture_output=True, + text=True, check=True, timeout=5) + if json.loads(result.stdout) != {"workers": 2}: + raise ValueError(f"benchmark peer does not implement {PEER_CONFIGURATION}: {peer}") + manifest = { + "peer_configuration": PEER_CONFIGURATION, + "application_configuration": APPLICATION_CONFIGURATION, + "selection": args.selection, + "settings": settings, + "cargo": subprocess.check_output([cargo, "-V"], text=True), + "compiler": fingerprint(compiler), + "optimization": "LLVM O2 application objects, LLD static link, selected prebuilt runtime archive", + "runtime_scheduling": "Dever uses production runtime defaults; workers/task_capacity configure Rust references only", + "binaries": {}, + "files": {}, + } + with (destination / "build.log").open("w") as log: + for fixture, entries in (ENTRIES.items() if has_protocols else ()): + source = destination / "source" / fixture + prepare_native_project(source, fixture, settings) + print(f"build {fixture}", flush=True) + started = time.monotonic() + executable = source / "app" + subprocess.run([compiler, "build", source, "--output", executable], + stdout=log, stderr=log, check=True) + for entry in entries: + binary = destination / entry + shutil.copy2(executable, binary) + record_binary(manifest, entry, binary, started, profile="base", fixture=fixture, + command=f"benchmark.{fixture}.{entry}", shared_application=fixture) + drivers = () + if args.selection == "all": + drivers = ("sqlite", "postgres") if postgres_setting is not None else ("sqlite",) + for driver in drivers: + project = destination / "source" / f"orm-{driver}" + try: + prepare_project(project, "orm", settings, driver, postgres_setting) + subprocess.run([compiler, "fmt", project], stdout=log, stderr=log, check=True) + for entry in ORM_ENTRIES: + name = f"{driver}-{entry.removeprefix('orm_')}" + print(f"build {name}", flush=True) + binary = destination / name + started = time.monotonic() + command_entry = "benchmark.item.orm_seed" if entry == "orm_http" else f"benchmark.item.{entry}" + stage_application_command(project / "module", command_entry) + subprocess.run([compiler, "build", project, "--output", binary], + stdout=log, stderr=log, check=True) + record_binary(manifest, name, binary, started, profile=driver, fixture="orm", + entry=entry) + finally: + if driver == "postgres": + (project / "config/setting.json").unlink(missing_ok=True) + cms_sources = {"all": ("dever",), "cms-profiles": ("dever", "md"), + "profiles": (), "protocols": ()}[args.selection] + for source in cms_sources: + cms_project = destination / "source" / f"cms-{source}" + prepare_cms_project(cms_project, source) + config_name = f"source/cms-{source}/config/setting.json" + manifest["files"][config_name] = fingerprint(cms_project / "config/setting.json") + binary_name = "cms" if source == "dever" else "cms-md" + print(f"build cms-{source}", flush=True) + cms_binary = destination / binary_name + started = time.monotonic() + subprocess.run([compiler, "build", cms_project, "--output", cms_binary], + stdout=log, stderr=log, check=True) + record_binary(manifest, binary_name, cms_binary, started, + profile="sqlite", fixture="cms", source=source) + if args.selection != "protocols": + build_profile_report(args, manifest, log) + for name, peer in peers.items(): + shutil.copy2(peer, destination / name) + manifest["binaries"][name] = fingerprint(destination / name) + if has_protocols: + shutil.copytree(TLS, destination / "tls") + for path in sorted((destination / "tls").rglob("*")): + if path.is_file(): + name = path.relative_to(destination).as_posix() + manifest["files"][name] = fingerprint(path) + (destination / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + print(f"artifacts: {destination}", flush=True) diff --git a/test/performance/cms.py b/test/performance/cms.py new file mode 100644 index 0000000..37e2356 --- /dev/null +++ b/test/performance/cms.py @@ -0,0 +1,675 @@ +"""显式验收双源码 CMS 的真实身份、租户、发布链及专属内存预算。""" + +import argparse +from collections import Counter +from concurrent.futures import ThreadPoolExecutor +from copy import deepcopy +import http.client +from http.cookies import SimpleCookie +import json +import math +import os +from pathlib import Path +import secrets +import shutil +import signal +import socket +import subprocess +import tempfile +from threading import Event +import time +from urllib.parse import urlencode + +from build import ROOT, fingerprint +from process import Budget, Process, observe, summarize, resource_trend +from settings import write_private_json + +FORMATS = ("dever", "md") +ARTICLE_PATH = "/news/article/admin/manage" +ARTICLE_LIST_PATH = f"{ARTICLE_PATH}/list?size=100" +REVISION_PATH = "/content/revision/admin/manage/create" +SESSION_PATH = "/user/account/{site}/session/login" +PUBLICATIONS_PATH = "/news/article/front/browse/list?size=100" + + +def require(condition, message): + if not condition: + raise RuntimeError(message) + + +def execute(command, directory, log_name, *, envelope=False, timeout=180): + child = subprocess.Popen([str(value) for value in command], cwd=directory, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, start_new_session=True) + failure = None + try: + stdout, stderr = child.communicate(timeout=timeout) + except BaseException as error: + failure = error + try: + signal_group(child.pid, signal.SIGTERM) + stdout, stderr = child.communicate(timeout=3) + except subprocess.TimeoutExpired: + signal_group(child.pid, signal.SIGKILL) + stdout, stderr = child.communicate(timeout=3) + (directory / f"{log_name}.stdout.log").write_bytes(stdout) + (directory / f"{log_name}.stderr.log").write_bytes(stderr) + if failure is not None: + raise failure + require(child.returncode == 0, f"{log_name} exited {child.returncode}: {directory}") + if envelope: + return decode_response(stdout, 200) + + +def signal_group(pid, signal_number): + try: + os.killpg(pid, signal_number) + except ProcessLookupError: + pass + + +def decode_response(body, status): + document = json.loads(body) + require(isinstance(document, dict) and set(document) == {"code", "message", "data"}, + f"unexpected response envelope (HTTP {status})") + require(status == 200 and document["code"] == 0, + f"request failed: HTTP {status}, code={document['code']}, {document['message']}") + return document["data"] + + +def copy_cms_project(source, destination): + destination.mkdir() + for name in ("config", "module", "test"): + shutil.copytree(source / name, destination / name) + + +def build_cases(compiler, output): + output.mkdir(parents=True, exist_ok=False) + cases = {} + for source_format in FORMATS: + source = ROOT / "examples/cms" / source_format + project = output / source_format + copy_cms_project(source, project) + started = time.monotonic() + execute([compiler, "check", project], project, "check") + execute([compiler, "build", project, "--output", project / "cms-app"], project, "build", timeout=900) + executable = project / "cms-app" + cases[source_format] = { + "executable": str(executable), "config": str(project / "config/setting.json"), + "fingerprint": fingerprint(executable), + "build_seconds": time.monotonic() - started, + } + manifest = {"format": "dever-cms-publish-v1", "cases": cases} + write_private_json(output / "manifest.json", manifest) + return manifest + + +def available_port(): + with socket.socket() as reservation: + reservation.bind(("127.0.0.1", 0)) + return reservation.getsockname()[1] + + +def isolated_settings(settings, port): + settings = deepcopy(settings) + settings["runtime"]["mode"] = "api" + settings["http"] = {"host": "127.0.0.1", "port": port} + settings["auth"]["providers"]["session"]["jwtSecret"] = secrets.token_hex(32) + for site in settings["sites"].values(): + site["origin"] = f"http://127.0.0.1:{port}" + database = settings["database"]["default"] + if database["type"] == "sqlite": + database["path"] = "data/db/platform.db" + database["tenant_directory"] = "data/tenants" + elif database["type"] == "postgres": + require(bool(database.get("url")) and bool(database.get("tenant_database_prefix")), + "PostgreSQL CMS acceptance requires an explicit isolated control database and tenant prefix") + else: + raise RuntimeError("CMS acceptance supports only SQLite and PostgreSQL") + settings["log"]["level"] = "error" + return settings + + +class CmsClient: + def __init__(self, port): + self.connection = http.client.HTTPConnection("127.0.0.1", port, timeout=10) + self.origin = f"http://127.0.0.1:{port}" + self.cookies = {} + self.timings = {} + self.record_timings = True + + def request(self, method, path, payload=None, *, site=None, expected=200): + headers = {"Content-Type": "application/json", "Origin": self.origin} + if site in self.cookies: + headers["Cookie"] = self.cookies[site] + body = None + if payload is not None: + if method in ("GET", "DELETE"): + query = {name: value if isinstance(value, str) else json.dumps(value) + for name, value in payload.items()} + path += ("&" if "?" in path else "?") + urlencode(query) + else: + body = json.dumps(payload) + started = time.monotonic() + self.connection.request(method, path, body, headers) + response = self.connection.getresponse() + body = response.read() + route = path.split("?")[0] + label = f"{method} {route}" + (f" expected {expected}" if expected != 200 else "") + if self.record_timings: + self.timings.setdefault(label, []).append( + (time.monotonic() - started) * 1000) + require(response.status == expected, + f"{method} {path}: HTTP {response.status}, expected {expected}: {body!r}") + if expected != 200: + document = json.loads(body) + require(document.get("code") != 0, "rejected request returned success") + return None + data = decode_response(body, response.status) + if path.endswith("/login"): + cookie = SimpleCookie() + for name, value in response.getheaders(): + if name.lower() == "set-cookie": + cookie.load(value) + require(bool(cookie), "login did not return a session cookie") + self.cookies[site] = "; ".join(f"{name}={value.value}" for name, value in cookie.items()) + return data + + def close(self): + self.connection.close() + + +def wait_ready(process, client): + deadline = time.monotonic() + 20 + while time.monotonic() < deadline: + process.sample() + require(process.child.poll() is None, f"CMS exited before readiness: {process.directory}") + try: + client.request("GET", ARTICLE_LIST_PATH, expected=401) + return time.monotonic() - process.started + except (ConnectionError, http.client.HTTPException, OSError): + client.close() + time.sleep(0.02) + raise TimeoutError(f"CMS readiness timeout: {process.directory}") + + +def login_client(client, identity, password, tenant_key): + for site in ("admin", "front"): + data = client.request("POST", SESSION_PATH.format(site=site), { + "credential_id": identity["credential_id"], "password": password, "tenant": tenant_key, + }, site=site) + require(data["account"]["id"] == identity["user_id"] + and data["tenant_id"] == identity["tenant_id"] and data["site"] == site, + "login changed the trusted identity or tenant/site") + + +def publish_workload(client, process, identity, password, articles, *, + tenant_key="bench", title_prefix="Article"): + login_client(client, identity, password, tenant_key) + titles = [] + ids = [] + for index in range(articles): + slug, title = f"bench-{index}", f"{title_prefix} {index}" + created = client.request("POST", f"{ARTICLE_PATH}/create", { + "slug": f" {slug.upper()} ", "title": f" {title} ", "body": "Draft body", + }, site="admin") + require(created["slug"] == slug and created["title"] == title, + "article creation did not normalize input") + require("author_id" not in created and "status" not in created, + "article creation leaked private owner or publication state") + changed = client.request("POST", REVISION_PATH, { + "selected_article_id": created["id"], "expected_version": created["version"], + "title": f" {title} published ", "body": "Published body", + }, site="admin") + expected_title = f"{title} published" + require(changed["title"] == expected_title + and changed["article_id"] == created["id"] + and changed["version"] == created["version"] + 1, + "revision did not preserve normalized content and advance the article version") + published = client.request("POST", f"{ARTICLE_PATH}/publish", { + "article_id": created["id"], + }, site="admin") + require(published["slug"] == slug and published["title"] == expected_title + and published["body"] == "Published body", + "publication did not reflect the stored article") + titles.append(expected_title) + ids.append(created["id"]) + process.sample() + listing = client.request("GET", PUBLICATIONS_PATH, site="front") + require(listing["total"] == articles + and sorted(value["title"] for value in listing["items"]) == sorted(titles), + "front query did not return every published article exactly once") + client.request("POST", f"{ARTICLE_PATH}/publish", {"article_id": ids[0]}, + site="admin", expected=409) + return {"articles": articles, "titles": sorted(titles), "duplicate_publish_status": 409} + + +def provision_identity(executable, directory, tenant_key): + password = secrets.token_urlsafe(24) + identity = execute([executable, "user.account.bootstrap", json.dumps({ + "tenant_key": tenant_key, "tenant_name": f"Benchmark {tenant_key}", + "email": f"{tenant_key}@example.invalid", "display_name": "Benchmark", + "password": password, + })], directory, f"bootstrap-{tenant_key}", envelope=True) + execute([executable, "--dever-tenant-migrate", identity["tenant_id"]], + directory, f"migrate-{tenant_key}") + for site in ("admin", "front"): + execute([executable, "--dever-tenant-owner", identity["tenant_id"], site, + identity["user_id"]], directory, f"owner-{tenant_key}-{site}") + return identity, password + + +def latency_summary(timings): + result = {} + for action, samples in timings.items(): + values = sorted(samples) + result[action] = {"samples": len(values), "p50_ms": values[len(values) // 2], + "max_ms": values[-1]} + return result + + +def catalog_key(catalog, component, domain, site, action, method): + matches = [entry["key"] for entry in catalog + if (entry["component"], entry["domain"], entry["site"], + entry["action"], entry["method"]) == (component, domain, site, action, method)] + require(len(matches) == 1 and bool(matches[0]), + f"permission catalog must uniquely identify {component}.{domain}.{site}.{action}") + return matches[0] + + +def run_postgres_rbac(executable, directory, port, owner, identity): + catalog = owner.request("GET", "/user/authorization/admin/manage/permissions", site="admin") + front_catalog = owner.request("GET", "/user/authorization/front/manage/permissions", site="front") + permissions = { + "admin_read": catalog_key(catalog, "news", "article", "admin", "list", "GET"), + "admin_permissions": catalog_key(catalog, "user", "authorization", "admin", "permissions", "GET"), + "front_list": catalog_key(front_catalog, "news", "article", "front", "list", "GET"), + } + require(len(set(permissions.values())) == 3, "RBAC acceptance reused a permission key") + password = secrets.token_urlsafe(24) + member = execute([executable, "user.account.fixture_member", json.dumps({ + "selected_tenant_id": identity["tenant_id"], "email": f"member-{secrets.token_hex(8)}@example.invalid", + "display_name": "Acceptance member", "password": password, + })], directory, "rbac-member", envelope=True) + require(isinstance(member["user_id"], int) and not isinstance(member["user_id"], bool) + and member["user_id"] > 0 and member["user_id"] != identity["user_id"] + and member["tenant_id"] == identity["tenant_id"], + "RBAC acceptance did not create a distinct member in the owner's tenant") + probes = (("admin", ARTICLE_LIST_PATH), + ("admin", "/user/authorization/admin/manage/permissions"), + ("front", PUBLICATIONS_PATH)) + client = CmsClient(port) + try: + login_client(client, member, password, "tenant_a") + for site, path in probes: + client.request("GET", path, site=site, expected=403) + roles = (("admin", "fixture-reader", "admin_read"), + ("admin", "fixture-catalog", "admin_permissions"), + ("front", "fixture-reader", "front_list")) + for site, role_id, permission in roles: + base = f"/user/authorization/{site}/manage" + owner.request("POST", f"{base}/save", { + "id": role_id, "name": role_id, "all_permissions": False, + "permission_keys": [permissions[permission]], + }, site=site) + owner.request("POST", f"{base}/grant", { + "user_id": member["user_id"], "role_id": role_id, + }, site=site) + for site, path in probes: + client.request("GET", path, site=site) + owner.request("DELETE", "/user/authorization/admin/manage/revoke", { + "user_id": member["user_id"], "role_id": "fixture-reader", + }, site="admin") + client.request("GET", ARTICLE_LIST_PATH, site="admin", expected=403) + client.request("GET", probes[1][1], site="admin") + client.request("GET", PUBLICATIONS_PATH, site="front") + for site, role_id, path in (("admin", "fixture-catalog", probes[1][1]), + ("front", "fixture-reader", PUBLICATIONS_PATH)): + owner.request("DELETE", f"/user/authorization/{site}/manage/revoke", { + "user_id": member["user_id"], "role_id": role_id, + }, site=site) + client.request("GET", path, site=site, expected=403) + finally: + client.close() + return {"member_user_id": member["user_id"], "permissions": permissions, + "before_grant_denied": True, "both_admin_roles_allowed": True, + "revoked_role_denied": True, "second_role_still_allowed": True, + "front_same_named_role_still_allowed": True} + + +class ReadLatency: + """固定内存的对数直方图；分位数报告桶上界，最大相邻桶比例约1.011。""" + + def __init__(self): + self.buckets = [0] * 2048 + self.count = 0 + + def add(self, milliseconds): + index = math.ceil(math.log2(1 + milliseconds) * 64) + require(0 <= index < len(self.buckets), "CMS latency exceeds histogram bounds") + self.buckets[index] += 1 + self.count += 1 + + def merge(self, other): + self.buckets = [left + right for left, right in zip(self.buckets, other.buckets)] + self.count += other.count + + def percentile(self, fraction): + threshold = math.ceil(self.count * fraction) + count = 0 + for index, samples in enumerate(self.buckets): + count += samples + if count >= threshold: + return 2 ** (index / 64) - 1 + raise ValueError("empty latency histogram") + + +def concurrent_reads(port, cookies, expected, process, *, concurrency, seconds, interval): + """Each closed-loop client keeps its own connection and verifies every reply.""" + stop = Event() + started = time.monotonic() + deadline = started + seconds + first = len(process.samples) + + def read_publications(): + client = CmsClient(port) + client.cookies = cookies.copy() + client.record_timings = False + latency = ReadLatency() + try: + while not stop.is_set() and time.monotonic() < deadline: + request_started = time.monotonic() + listing = client.request("GET", PUBLICATIONS_PATH, site="front") + require(listing == expected, "concurrent CMS read changed publication results") + latency.add((time.monotonic() - request_started) * 1000) + return latency + except BaseException: + stop.set() + raise + finally: + client.close() + + with ThreadPoolExecutor(max_workers=concurrency) as clients: + futures = [clients.submit(read_publications) for _ in range(concurrency)] + try: + while any(not future.done() for future in futures): + process.check() + observe([process], interval, interval) + elapsed = time.monotonic() - started + latency = ReadLatency() + for future in futures: + latency.merge(future.result()) + finally: + stop.set() + require(latency.count > 0, "concurrent CMS read completed without a successful request") + require(elapsed >= seconds, "concurrent CMS read ended before the measurement window") + process.sample() + return {"mode": "closed_loop_authenticated_read", "concurrency": concurrency, + "requested_seconds": seconds, "elapsed_seconds": elapsed, + "succeeded": latency.count, "errors": 0, "result_checked_on_every_request": True, + "actual_success_qps": latency.count / elapsed, + "latency_measurement": "request and reply validation; logarithmic bucket upper bound, 64 buckets per octave", + "request_latency_ms": {name: latency.percentile(fraction) + for name, fraction in (("p50", .5), ("p95", .95), ("p99", .99))}, + "resources": summarize(process.samples[first:]), + "trend": resource_trend(process.samples[first:]) if len(process.samples[first:]) >= 4 else None} + + +def sqlite_pool_descriptor_limits(settings, directory): + database = settings["database"]["default"] + require(database["type"] == "sqlite", "CMS read benchmark requires isolated SQLite") + databases = [directory / database["path"]] + databases.extend((directory / database["tenant_directory"]).glob("*.db")) + # 每个连接可持有主文件与WAL；同进程各连接共用一个SHM句柄。 + return {str(path.resolve()) + suffix: limit for path in databases + for suffix, limit in (("", database["max_connections"]), + ("-wal", database["max_connections"]), ("-shm", 1))} + + +def validate_read_descriptor_recovery(load): + before = Counter(load["descriptors_before"].values()) + after = Counter(load["descriptors_after"].values()) + for path, limit in load["pooled_descriptor_limits"].items(): + require(after[path] <= limit, f"CMS SQLite descriptor count exceeds pool limit: {path}") + before.pop(path, None) + after.pop(path, None) + require(before == after, + f"CMS read descriptors did not recover: added={dict(after - before)}, missing={dict(before - after)}") + + +def run_case(case, directory, *, articles, cgroup_parent, memory_mib, + idle_seconds=0.2, interval=0.05, server_cpu=None, cpu_quota=None, + concurrency=0, read_seconds=5): + executable = Path(case["executable"]) + require(fingerprint(executable) == case["fingerprint"], "CMS artifact fingerprint changed") + directory.mkdir(parents=True, exist_ok=False) + target = directory / "cms-app" + shutil.copy2(executable, target) + (directory / "config").mkdir() + (directory / "data").mkdir() + port = available_port() + settings = isolated_settings(json.loads(Path(case["config"]).read_text()), port) + write_private_json(directory / "config/setting.json", settings) + identity, password = provision_identity(target, directory, "bench") + budget = Budget(cgroup_parent, memory_mib, cpu_quota) + client = CmsClient(port) + failure = None + result = {} + try: + with Process([target], directory / "server", budget=budget, + cpus=(server_cpu,) if server_cpu is not None else None) as process: + result["raw"] = process.samples + ready_seconds = wait_ready(process, client) + started = time.monotonic() + contract = publish_workload(client, process, identity, password, articles) + elapsed = time.monotonic() - started + result.update(contract=contract, ready_seconds=ready_seconds, + workload_seconds=elapsed, published_articles_per_second=articles / elapsed, + latency=latency_summary(client.timings)) + if concurrency: + expected = client.request("GET", PUBLICATIONS_PATH, site="front") + # 控制连接不参与长读负载，先关闭，避免其空闲超时改变基线。 + client.close() + observe([process], 0.3, interval) + descriptors_before = process.descriptors() + pooled_limits = sqlite_pool_descriptor_limits(settings, directory) + read_load = {"descriptors_before": descriptors_before, + "pooled_descriptor_limits": pooled_limits} + result["concurrent_reads"] = read_load + read_load.update(concurrent_reads(port, client.cookies, expected, process, + concurrency=concurrency, seconds=read_seconds, interval=interval)) + observe([process], 0.3, interval) + descriptors_after = process.descriptors() + read_load.update(descriptors_after=descriptors_after, + fd_delta_after_disconnect=len(descriptors_after) - len(descriptors_before)) + validate_read_descriptor_recovery(read_load) + observe([process], idle_seconds, interval) + process.sample() + process.check() + require(process.child.poll() is None, "CMS exited during the publish workload") + resources = summarize(process.samples) + result.update(resources=resources, budget_before_stop=budget.sample()) + except BaseException as error: + failure = error + finally: + client.close() + try: + budget.close() + except BaseException as error: + if failure is None: + failure = error + else: + failure.add_note(f"cannot close owned CMS budget: {error}") + result["cgroup"] = budget.final + if failure is None and budget.final is not None and ( + budget.final["events"]["oom"] or budget.final["events"]["oom_kill"]): + failure = RuntimeError("CMS exceeded its memory budget") + result["status"] = "failed" if failure else "passed" + if failure is not None: + result["error"] = str(failure) + write_private_json(directory / "report.json", result) + if failure is not None: + raise failure + return result + + +def run_postgres_case(project, output, compiler, *, articles, rbac_fixture=False): + # 构建、临时密钥和端口只写测试副本，调用者的项目配置始终保持不变。 + with tempfile.TemporaryDirectory(prefix="dever-cms-postgres-") as temporary: + staged = Path(temporary) / "project" + copy_cms_project(project, staged) + return run_staged_postgres_case(staged, output, compiler, articles=articles, + rbac_fixture=rbac_fixture) + + +def run_staged_postgres_case(project, output, compiler, *, articles, rbac_fixture): + require(1 <= articles <= 100, "PostgreSQL CMS acceptance requires 1..100 articles") + output.mkdir(parents=True, exist_ok=False) + report = {"status": "running", "source": "dever", "tenants": {}, + "scope": "PostgreSQL HTTP owner flow and physical database tenant isolation; not non-owner RBAC revocation"} + if rbac_fixture: + report["scope"] = "PostgreSQL HTTP owner flow, physical database tenant isolation and non-owner multi-role/site grant-revoke" + clients = {} + try: + setting_path = project / "config/setting.json" + initial = json.loads(setting_path.read_text()) + require(initial["database"]["default"]["type"] == "postgres", + "postgres-case requires a PostgreSQL project configuration") + port = available_port() + write_private_json(setting_path, isolated_settings(initial, port), replace=True) + execute([compiler, "check", project], output, "check") + executable = project / "cms-app" + execute([compiler, "build", project, "--output", executable], output, "build", timeout=900) + report["binary"] = fingerprint(executable) + identities = {key: provision_identity(executable, output, key) + for key in ("tenant_a", "tenant_b")} + require(identities["tenant_a"][0]["tenant_id"] != identities["tenant_b"][0]["tenant_id"], + "PostgreSQL acceptance reused a tenant identity") + clients = {key: CmsClient(port) for key in identities} + with Process([executable], output / "server") as process: + report["ready_seconds"] = wait_ready(process, clients["tenant_a"]) + for key, (identity, password) in identities.items(): + contract = publish_workload(clients[key], process, identity, password, articles, + tenant_key=key, title_prefix=key) + permissions = clients[key].request( + "GET", "/user/authorization/admin/manage/permissions", site="admin") + require(bool(permissions), "CMS permission catalog is empty") + report["tenants"][key] = {"tenant_id": identity["tenant_id"], + "user_id": identity["user_id"], "contract": contract} + for key, client in clients.items(): + listing = client.request("GET", PUBLICATIONS_PATH, site="front") + contract = report["tenants"][key]["contract"] + require(listing["total"] == articles + and sorted(value["title"] for value in listing["items"]) == contract["titles"], + "PostgreSQL tenant query leaked another tenant's articles") + require(sorted(value["slug"] for value in listing["items"]) + == sorted(f"bench-{index}" for index in range(articles)), + "PostgreSQL tenants did not preserve the same slug namespace") + report["tenants"][key]["slugs"] = sorted(value["slug"] for value in listing["items"]) + identity, password = identities["tenant_a"] + clients["tenant_a"].request("POST", SESSION_PATH.format(site="admin"), { + "credential_id": identity["credential_id"], "password": password, "tenant": "tenant_b", + }, site="admin", expected=401) + front_cookie = clients["tenant_a"].cookies["front"] + clients["tenant_a"].cookies["front"] = clients["tenant_a"].cookies["admin"] + clients["tenant_a"].request("GET", PUBLICATIONS_PATH, site="front", expected=401) + clients["tenant_a"].cookies["front"] = front_cookie + if rbac_fixture: + report["rbac"] = run_postgres_rbac(executable, output, port, + clients["tenant_a"], identity) + process.sample() + process.check() + require(process.child.poll() is None, "PostgreSQL CMS exited during tenant acceptance") + report["resources"] = summarize(process.samples) + report["cross_site_cookie_rejected"] = True + report["cross_tenant_login_rejected"] = True + report["status"] = "passed" + except BaseException as error: + report["status"] = "failed" + report["error"] = str(error) + raise + finally: + for client in clients.values(): + client.close() + write_private_json(output / "report.json", report) + return report + + +def run_cases(manifest, output, *, articles, cgroup_parent, memory_mib, + concurrency=0, read_seconds=5): + require(manifest["format"] == "dever-cms-publish-v1", "unexpected CMS manifest format") + require(1 <= articles <= 100, "articles must be 1..100") + require(0 <= concurrency <= 64, "read concurrency must be 0..64") + require(math.isfinite(read_seconds) and 0.1 <= read_seconds <= 3600, + "read seconds must be finite and within 0.1..3600") + require((memory_mib is None) == (cgroup_parent is None), + "memory budget and cgroup parent must be provided together") + output.mkdir(parents=True, exist_ok=False) + report = {"format": manifest["format"], "memory_mib": memory_mib, + "scope": "owned loopback HTTP/1, sequential publish, isolated SQLite; not a saturation result", + "cases": {}} + if concurrency: + report["scope"] = "owned loopback HTTP/1, sequential publish and closed-loop authenticated reads, isolated SQLite" + try: + for source_format in FORMATS: + report["cases"][source_format] = run_case(manifest["cases"][source_format], + output / source_format, articles=articles, + cgroup_parent=cgroup_parent, memory_mib=memory_mib, + concurrency=concurrency, read_seconds=read_seconds) + require(report["cases"]["dever"]["contract"] == report["cases"]["md"]["contract"], + "Dever and Markdown CMS changed the publish contract") + report["status"] = "passed" + except BaseException as error: + report["status"] = "failed" + report["error"] = str(error) + raise + finally: + for source_format in FORMATS: + case_report = output / source_format / "report.json" + if source_format not in report["cases"] and case_report.is_file(): + report["cases"][source_format] = json.loads(case_report.read_text()) + write_private_json(output / "report.json", report) + return report + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + actions = parser.add_subparsers(dest="action", required=True) + build = actions.add_parser("build") + build.add_argument("--compiler", type=Path, default=ROOT / "target/debug/dever") + build.add_argument("--output", type=Path, required=True) + run = actions.add_parser("run") + run.add_argument("--manifest", type=Path, required=True) + run.add_argument("--output", type=Path, required=True) + run.add_argument("--articles", type=int, default=16) + run.add_argument("--cgroup-parent", type=Path) + run.add_argument("--memory-mib", type=int, choices=(64, 128)) + run.add_argument("--concurrency", type=int, default=0, + help="0 disables load; otherwise 1..64 independent authenticated read clients") + run.add_argument("--read-seconds", type=float, default=5) + postgres = actions.add_parser("postgres-case") + postgres.add_argument("--project", type=Path, required=True) + postgres.add_argument("--output", type=Path, required=True) + postgres.add_argument("--compiler", type=Path, default=ROOT / "target/debug/dever") + postgres.add_argument("--articles", type=int, default=2) + postgres.add_argument("--rbac-fixture", action="store_true", + help="run test-only member CMD provided by the Rust PostgreSQL fixture") + args = parser.parse_args() + if args.action == "build": + result = build_cases(args.compiler.resolve(strict=True), args.output.resolve()) + elif args.action == "postgres-case": + report = run_postgres_case(args.project.resolve(strict=True), args.output.resolve(), + args.compiler.resolve(strict=True), articles=args.articles, + rbac_fixture=args.rbac_fixture) + result = {"status": report["status"], "report": str(args.output.resolve() / "report.json")} + else: + result = run_cases(json.loads(args.manifest.read_text()), args.output.resolve(), + articles=args.articles, cgroup_parent=args.cgroup_parent, + memory_mib=args.memory_mib, concurrency=args.concurrency, + read_seconds=args.read_seconds) + result = {"status": result["status"], "memory_mib": result["memory_mib"], + "report": str(args.output / "report.json")} + print(json.dumps(result, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/test/performance/fixtures/bench.dever b/test/performance/fixtures/bench.dever new file mode 100644 index 0000000..2515e3f --- /dev/null +++ b/test/performance/fixtures/bench.dever @@ -0,0 +1,21 @@ +ready(port: Int) () { + dever.io.println("READY|" + int.to_text(port)) +} + +pause(millis: Int) () { + dever.task.sleep(millis) +} + +report( + begin: Int, + finish: Int, + name: Text, + count: Int, + checksum: Int +) () recover("基准失败输出由运行器检查") { + dever.io.println( + "SAMPLE|" + name + "|" + int.to_text(count) + "|" + int.to_text(finish - begin) + "|" + int.to_text( + checksum + ) + ) +} diff --git a/test/performance/fixtures/http/adapter.dever b/test/performance/fixtures/http/adapter.dever new file mode 100644 index 0000000..a7102b1 --- /dev/null +++ b/test/performance/fixtures/http/adapter.dever @@ -0,0 +1,136 @@ +type Content { + body: Bytes + kind: Text +} + +listening(result: dever.net.ListenResult.Listening(listener)) ( + value: dever.system.Listener +) { + value = listener +} + +listening(result: dever.net.ListenResult.Failed(message)) (value: dever.system.Listener) { + fail(app.WorkerFailure.Failed(message)) +} + +bound(result: dever.net.PortResult.Bound(port)) (value: Int) { + value = port +} + +bound(result: dever.net.PortResult.Failed(message)) (value: Int) { + fail(app.WorkerFailure.Failed(message)) +} + +paused(result: dever.time.SleepResult.Done) () {} + +paused(result: dever.time.SleepResult.Failed(message)) () { + fail(app.WorkerFailure.Failed(message)) +} + +closed(result: dever.io.CloseResult.Closed) () {} + +closed(result: dever.io.CloseResult.Failed(message)) () { + fail(app.WorkerFailure.Failed(message)) +} + +server_tls(result: dever.tls.ServerResult.Ready(tls)) (value: dever.system.ServerTls) { + value = tls +} + +server_tls(result: dever.tls.ServerResult.Failed(message)) ( + value: dever.system.ServerTls +) { + fail(app.WorkerFailure.Failed(message)) +} + +response_value(content: Content) (result: dever.http.Response) { + result = dever.http.Response { + status = 200 + headers = [dever.http.header("Content-Type", content.kind)] + body = content.body + } +} + +route(request: dever.http.Request, large: Bytes) (response: dever.http.Response) { + response = response_value(select(request.target, large)) +} + +select(path: "/plain", large: Bytes) (result: Content) { + result = Content { + body = dever.bytes.from_text("hello") + kind = "text/plain" + } +} + +select(path: "/json", large: Bytes) (result: Content) { + result = Content { + body = dever.bytes.from_text("{\"ok\":true}") + kind = "application/json" + } +} + +select(path: "/bytes", large: Bytes) (result: Content) { + result = Content { + body = large + kind = "application/octet-stream" + } +} + +select(path: other, large: Bytes) (result: Content) { + result = Content { + body = dever.bytes.from_text("unknown benchmark path") + kind = "text/plain" + } +} + +serve( + listener: dever.system.Listener, + tls: null, + limits: dever.http.Limits, + large: Bytes +) () { + dever.system.http_serve(route, listener, limits, large) +} + +serve( + listener: dever.system.Listener, + tls: dever.system.ServerTls, + limits: dever.http.Limits, + large: Bytes +) () { + dever.system.http_serve_tls(route, listener, limits, tls, large) +} + +worker( + tls: dever.system.ServerTls?, + limits: dever.http.Limits, + large: Bytes, + lifetime: Int +) () { + listener = listening(result(dever.net.listen("127.0.0.1", 0))) + port = bound(result(dever.net.port(listener))) + server = run(serve(listener, tls, limits, large)) + blocking(dever.io.println("READY|" + int.to_text(port))) + paused(result(dever.task.sleep(lifetime))) + closed(result(dever.net.close_listener(listener))) + stop(server) +} + +port.plain(limits: dever.http.Limits, large: Bytes, lifetime: Int) () { + worker(null, limits, large, lifetime) +} + +port.secure( + limits: dever.http.Limits, + large: Bytes, + lifetime: Int, + certificate: Bytes, + key: Bytes +) () { + worker( + server_tls(result(dever.tls.server(certificate, key))), + limits, + large, + lifetime + ) +} diff --git a/test/performance/fixtures/http/api.dever b/test/performance/fixtures/http/api.dever new file mode 100644 index 0000000..290f714 --- /dev/null +++ b/test/performance/fixtures/http/api.dever @@ -0,0 +1,7 @@ +cmd http = app.http + +cmd https = app.https + +cmd http2 = app.http2 + +cmd https2 = app.https2 diff --git a/test/performance/fixtures/http/app.dever b/test/performance/fixtures/http/app.dever new file mode 100644 index 0000000..57b0352 --- /dev/null +++ b/test/performance/fixtures/http/app.dever @@ -0,0 +1,35 @@ +type WorkerFailure { + error Failed(message: Text) +} + +http() (result: Bool) { + port.plain(config.limits(), config.large_body(), config.lifetime()) + result = true +} + +https() (result: Bool) { + port.secure( + config.limits(), + config.large_body(), + config.lifetime(), + config.certificate(), + config.key() + ) + result = true +} + +http2() (result: Bool) { + port.plain(config.http2_limits(), config.large_body(), config.lifetime()) + result = true +} + +https2() (result: Bool) { + port.secure( + config.http2_limits(), + config.large_body(), + config.lifetime(), + config.certificate(), + config.key() + ) + result = true +} diff --git a/test/performance/fixtures/http/port.dever b/test/performance/fixtures/http/port.dever new file mode 100644 index 0000000..3dc5c83 --- /dev/null +++ b/test/performance/fixtures/http/port.dever @@ -0,0 +1,9 @@ +plain(limits: dever.http.Limits, large: Bytes, lifetime: Int) () fails app.WorkerFailure + +secure( + limits: dever.http.Limits, + large: Bytes, + lifetime: Int, + certificate: Bytes, + key: Bytes +) () fails app.WorkerFailure diff --git a/test/performance/fixtures/live/adapter.dever b/test/performance/fixtures/live/adapter.dever new file mode 100644 index 0000000..9759427 --- /dev/null +++ b/test/performance/fixtures/live/adapter.dever @@ -0,0 +1,213 @@ +setting { + limits: dever.http.Limits + lifetime_ms: Int +} + +type Service { + Tcp + Live +} + +listening(result: dever.net.ListenResult.Listening(listener)) ( + value: dever.system.Listener +) { + value = listener +} + +listening(result: dever.net.ListenResult.Failed(message)) (value: dever.system.Listener) { + fail(app.WorkerFailure.Failed(message)) +} + +bound(result: dever.net.PortResult.Bound(port)) (value: Int) { + value = port +} + +bound(result: dever.net.PortResult.Failed(message)) (value: Int) { + fail(app.WorkerFailure.Failed(message)) +} + +written(result: dever.io.WriteResult.Done) () {} + +written(result: dever.io.WriteResult.Failed(message)) () { + fail(app.WorkerFailure.Failed(message)) +} + +closed(result: dever.io.CloseResult.Closed) () {} + +closed(result: dever.io.CloseResult.Failed(message)) () { + fail(app.WorkerFailure.Failed(message)) +} + +paused(result: dever.time.SleepResult.Done) () {} + +paused(result: dever.time.SleepResult.Failed(message)) () { + fail(app.WorkerFailure.Failed(message)) +} + +tcp_chunk(event: dever.io.ReadEvent.Chunk(bytes), socket: dever.system.Socket) () recover("TCP 连接错误由运行器检查") { + written(result(dever.net.write(socket, bytes))) +} + +tcp_chunk(event: dever.io.ReadEvent.Failed(message), socket: dever.system.Socket) () recover("TCP 连接错误由运行器检查") { + blocking(dever.io.println("ERROR|" + message)) +} + +tcp_stream( + result: dever.net.ReadStreamResult.Streaming(stream), + socket: dever.system.Socket +) () { + each(tcp_chunk, stream, socket) +} + +tcp_stream( + result: dever.net.ReadStreamResult.Failed(message), + socket: dever.system.Socket +) () { + fail(app.WorkerFailure.Failed(message)) +} + +tcp_connection( + event: dever.net.ConnectResult.Connected(socket), + limits: dever.http.Limits +) () recover("TCP 连接错误由运行器检查") { + written(result(dever.net.timeout(socket, limits.timeout_ms))) + tcp_stream(result(dever.net.chunks(socket, 4096)), socket) + closed(result(dever.net.close(socket))) +} + +tcp_connection( + event: dever.net.ConnectResult.Failed(message), + limits: dever.http.Limits +) () recover("TCP 连接错误由运行器检查") { + blocking(dever.io.println("ERROR|" + message)) +} + +tcp_server(listener: dever.system.Listener, limits: dever.http.Limits) () { + stream = dever.net.connections(listener) + parallel_each(tcp_connection, stream, limits.connections, limits) +} + +websocket_message( + event: dever.websocket.MessageEvent.Read(message), + socket: dever.system.WebSocket +) () recover("WebSocket 连接错误由运行器检查") { + websocket_echo(message, socket) +} + +websocket_message( + event: dever.websocket.MessageEvent.Failed(message), + socket: dever.system.WebSocket +) () recover("WebSocket 连接错误由运行器检查") { + blocking(dever.io.println("ERROR|" + message)) +} + +websocket_echo( + message: dever.websocket.Message.Text(value), + socket: dever.system.WebSocket +) () recover("WebSocket 连接错误由运行器检查") { + written(result(dever.websocket.send(socket, dever.websocket.Message.Text(value)))) +} + +websocket_echo( + message: dever.websocket.Message.Binary(value), + socket: dever.system.WebSocket +) () recover("WebSocket 连接错误由运行器检查") { + written(result(dever.websocket.send(socket, dever.websocket.Message.Binary(value)))) +} + +websocket_echo(message: other, socket: dever.system.WebSocket) () recover("WebSocket 连接错误由运行器检查") {} + +websocket_limits() (result: dever.websocket.Limits) { + result = dever.websocket.Limits { + message_bytes = 4096 + idle_ms = setting.lifetime_ms + write_ms = setting.limits.timeout_ms + } +} + +websocket_connection(result: dever.websocket.ConnectResult.Connected(socket)) () { + each(websocket_message, dever.websocket.messages(socket), socket) +} + +websocket_connection(result: dever.websocket.ConnectResult.Failed(message)) () { + fail(app.WorkerFailure.Failed(message)) +} + +route(request: dever.http.Request, reply: dever.system.HttpReply) () { + live_path(request.target, reply) +} + +live_path(target: "/ws", reply: dever.system.HttpReply) () recover("实时连接错误由运行器检查") { + websocket_connection(result(dever.websocket.accept(reply, websocket_limits()))) +} + +live_path(target: "/events", reply: dever.system.HttpReply) () recover("实时连接错误由运行器检查") { + written(result(dever.sse.start(reply, []))) + written( + result( + dever.sse.send( + reply, + dever.sse.Event { + event = "live" + data = "ready" + id = "1" + retry_ms = null + } + ) + ) + ) + paused(result(dever.task.sleep(setting.lifetime_ms))) +} + +live_path(target: other, reply: dever.system.HttpReply) () recover("实时连接错误由运行器检查") { + written( + result( + dever.http.respond( + reply, + dever.http.Response { + status = 404 + headers = [] + body = dever.bytes.from_text("unknown live benchmark path") + } + ) + ) + ) +} + +live_limits() (result: dever.http.LiveLimits) { + result = dever.http.LiveLimits { + chunk_bytes = 4096 + idle_ms = setting.lifetime_ms + heartbeat_ms = 100 + } +} + +live_server(listener: dever.system.Listener) () { + dever.system.http_serve_live(route, listener, setting.limits, live_limits()) +} + +serve(service: Service.Tcp, listener: dever.system.Listener) () { + tcp_server(listener, setting.limits) +} + +serve(service: Service.Live, listener: dever.system.Listener) () { + live_server(listener) +} + +worker(service: Service) () { + listener = listening(result(dever.net.listen("127.0.0.1", 0))) + port = bound(result(dever.net.port(listener))) + server = run(serve(service, listener)) + blocking(dever.io.println("READY|" + int.to_text(port))) + paused(result(dever.task.sleep(setting.lifetime_ms))) + stop(server) + closed(result(dever.net.close_listener(listener))) +} + +port.tcp() () { + worker(Service.Tcp) +} + +port.live() () { + worker(Service.Live) +} diff --git a/test/performance/fixtures/live/api.dever b/test/performance/fixtures/live/api.dever new file mode 100644 index 0000000..3e0d02e --- /dev/null +++ b/test/performance/fixtures/live/api.dever @@ -0,0 +1,5 @@ +cmd tcp = app.tcp + +cmd ws = app.ws + +cmd sse = app.sse diff --git a/test/performance/fixtures/live/app.dever b/test/performance/fixtures/live/app.dever new file mode 100644 index 0000000..96c6d2e --- /dev/null +++ b/test/performance/fixtures/live/app.dever @@ -0,0 +1,18 @@ +type WorkerFailure { + error Failed(message: Text) +} + +tcp() (result: Bool) { + port.tcp() + result = true +} + +ws() (result: Bool) { + port.live() + result = true +} + +sse() (result: Bool) { + port.live() + result = true +} diff --git a/test/performance/fixtures/live/port.dever b/test/performance/fixtures/live/port.dever new file mode 100644 index 0000000..a1918c2 --- /dev/null +++ b/test/performance/fixtures/live/port.dever @@ -0,0 +1,3 @@ +tcp() () fails app.WorkerFailure + +live() () fails app.WorkerFailure diff --git a/test/performance/fixtures/orm/app.dever b/test/performance/fixtures/orm/app.dever new file mode 100644 index 0000000..2c7f47b --- /dev/null +++ b/test/performance/fixtures/orm/app.dever @@ -0,0 +1,144 @@ +type Identity { + id: Text + user_id: model.id? + tenant_id: model.id? +} + +verify(claims: dever.auth.Claims) (identity: Identity) { + identity = Identity { + id = claims.subject + user_id = null + tenant_id = null + } +} + +ready() () { + blocking(bench.ready(0)) +} + +populate_one(value: Int) () { + created = model.create({ + value = value + label = "fixture" + }) +} + +populate() () { + each(populate_one, config.orm_rows()) +} + +crud_once(value: Int) () { + begin = dever.time.monotonic_nanos() + created = model.create({ + value = value + label = "created" + }) + changed = model.update(created.id, { label = "updated" }) + removed = model.delete(created.id) + end = dever.time.monotonic_nanos() + blocking(bench.report(begin, end, "crud", 1, changed + removed)) +} + +list_once(value: Int) () { + begin = dever.time.monotonic_nanos() + page = model.list({ + order = id.asc + page = 1 + size = 32 + }) + end = dever.time.monotonic_nanos() + blocking(bench.report(begin, end, "list", 1, page.total + length(page.items))) +} + +cursor_once(value: Int) () { + begin = dever.time.monotonic_nanos() + page = model.cursor({ + order = id.asc + size = 32 + }) + end = dever.time.monotonic_nanos() + blocking(bench.report(begin, end, "cursor", 1, length(page.items))) +} + +count_item(value: model.Item, total: Int) (next: Int) { + next = total + 1 +} + +stream_once(value: Int) () { + begin = dever.time.monotonic_nanos() + rows = model.stream({ + order = id.asc + size = 16 + }) + total = reduce(count_item, rows, 0) + end = dever.time.monotonic_nanos() + blocking(bench.report(begin, end, "stream", 1, total)) +} + +pool_once(value: Int) () { + begin = dever.time.monotonic_nanos() + total = model.count() + end = dever.time.monotonic_nanos() + blocking(bench.report(begin, end, "pool", 1, total)) +} + +transaction fill_cancelled_transaction() () { + created = model.create({ + value = 1 + label = "cancelled" + }) + delayed = model.delay(config.orm_cancel_work()) +} + +orm_idle() (result: Bool) { + total = model.count() + blocking(bench.ready(total)) + bench.pause(config.lifetime()) + result = true +} + +orm_crud() (result: Bool) { + ready() + each(crud_once, config.orm_input()) + result = true +} + +orm_list() (result: Bool) { + populate() + ready() + each(list_once, config.orm_input()) + result = true +} + +orm_cursor() (result: Bool) { + populate() + ready() + each(cursor_once, config.orm_input()) + result = true +} + +orm_stream() (result: Bool) { + ready() + each(stream_once, config.orm_input()) + result = true +} + +orm_pool() (result: Bool) { + ready() + parallel_each(pool_once, config.orm_input(), config.orm_concurrency()) + result = true +} + +orm_seed() (result: Bool) { + populate() + result = true +} + +count() (total: Int) { + total = model.count() +} + +cancel() (result: Bool) { + fill_cancelled_transaction() + result = true +} diff --git a/test/performance/fixtures/orm/app/api.dever b/test/performance/fixtures/orm/app/api.dever new file mode 100644 index 0000000..43d956f --- /dev/null +++ b/test/performance/fixtures/orm/app/api.dever @@ -0,0 +1,9 @@ +public get count = app.count +public post cancel = app.cancel +cmd idle = app.orm_idle +cmd crud = app.orm_crud +cmd list = app.orm_list +cmd cursor = app.orm_cursor +cmd stream = app.orm_stream +cmd pool = app.orm_pool +cmd seed = app.orm_seed diff --git a/test/performance/fixtures/orm/app/model/item.dever b/test/performance/fixtures/orm/app/model/item.dever new file mode 100644 index 0000000..7756679 --- /dev/null +++ b/test/performance/fixtures/orm/app/model/item.dever @@ -0,0 +1,13 @@ +type DelayResult { + total: Int +} + +global type Item { + value: Int index + label: Text(1, 32) +} + +sql delay(work: Int) (result: DelayResult) { + sqlite = "WITH RECURSIVE counter(value) AS (SELECT 1 UNION ALL SELECT value + 1 FROM counter WHERE value < ?1) SELECT SUM(value) AS total FROM counter" + postgres = "SELECT SUM(value)::bigint AS total FROM generate_series(1::bigint, $1::bigint) AS value" +} diff --git a/test/performance/fixtures/profile/benchmark/command/api.dever b/test/performance/fixtures/profile/benchmark/command/api.dever new file mode 100644 index 0000000..356e42c --- /dev/null +++ b/test/performance/fixtures/profile/benchmark/command/api.dever @@ -0,0 +1 @@ +cmd run = app.run diff --git a/test/performance/fixtures/profile/benchmark/command/app.dever b/test/performance/fixtures/profile/benchmark/command/app.dever new file mode 100644 index 0000000..dfcdfae --- /dev/null +++ b/test/performance/fixtures/profile/benchmark/command/app.dever @@ -0,0 +1,4 @@ +run() (result: Bool) { + dever.io.println("profile") + result = true +} diff --git a/test/performance/fixtures/runtime/adapter.dever b/test/performance/fixtures/runtime/adapter.dever new file mode 100644 index 0000000..f01821c --- /dev/null +++ b/test/performance/fixtures/runtime/adapter.dever @@ -0,0 +1,7 @@ +port.blocking_pause(millis: Int) () { + dever.time.sleep(millis) +} + +port.report_error(message: Text) () { + dever.io.println("ERROR|" + message) +} diff --git a/test/performance/fixtures/runtime/api.dever b/test/performance/fixtures/runtime/api.dever new file mode 100644 index 0000000..75826c1 --- /dev/null +++ b/test/performance/fixtures/runtime/api.dever @@ -0,0 +1,11 @@ +cmd sync_idle = app.sync_idle + +cmd async_idle = app.async_idle + +cmd tasks = app.tasks + +cmd tasks_on_worker = app.tasks_on_worker + +cmd channels = app.channels + +cmd cancel = app.cancel diff --git a/test/performance/fixtures/runtime/app.dever b/test/performance/fixtures/runtime/app.dever new file mode 100644 index 0000000..ece5f7a --- /dev/null +++ b/test/performance/fixtures/runtime/app.dever @@ -0,0 +1,132 @@ +type Parking { + ready: Channel<Int> + gate: Channel<Int> +} + +sync_idle() (result: Bool) { + bench.ready(0) + port.blocking_pause(config.lifetime()) + result = true +} + +idle() () { + blocking(bench.ready(0)) + bench.pause(config.lifetime()) +} + +async_idle() (result: Bool) { + idle() + result = true +} + +identity(value: Int) (result: Int) { + result = value +} + +task_sum(value: Int, state: Int) (result: Int) { + job = run(identity(value)) + result = state + wait(job) +} + +sum(value: Int, state: Int) (result: Int) { + result = state + value +} + +task_sample(index: Int, input: Bytes) () { + begin = dever.time.monotonic_nanos() + checksum = reduce(task_sum, input, 0) + end = dever.time.monotonic_nanos() + blocking(bench.report(begin, end, "tasks", length(input), checksum)) +} + +send_byte(value: Int, output: Channel<Int>) () { + send(output, value) +} + +produce(input: Bytes, output: Channel<Int>) () { + each(send_byte, input, output) + close(output) +} + +channel_sample(index: Int, input: Bytes) () { + output = channel(Int, config.channel_capacity()) + begin = dever.time.monotonic_nanos() + producer = run(produce(input, output)) + checksum = reduce(sum, stream(output), 0) + wait(producer) + end = dever.time.monotonic_nanos() + blocking(bench.report(begin, end, "channels", length(input), checksum)) +} + +wait_one(value: Int, parking: Parking) () { + send(parking.ready, value) + signaled(receive(parking.gate)) +} + +signaled(value: Int) () {} + +signaled(value: null) () { + blocking(port.report_error("channel closed before expected signal")) +} + +hold(parking: Parking) () { + parallel_each(wait_one, config.pending_input(), config.pending(), parking) +} + +started(value: Int, ready: Channel<Int>) () { + signaled(receive(ready)) +} + +cancel_worker() () { + parking = Parking { + ready = channel(Int, config.pending()) + gate = channel(Int, 1) + } + job = run(hold(parking)) + each(started, config.pending_input(), parking.ready) + blocking(bench.ready(config.pending())) + bench.pause(config.idle_sample_ms()) + begin = dever.time.monotonic_nanos() + stop(job) + end = dever.time.monotonic_nanos() + close(parking.ready) + close(parking.gate) + blocking(bench.report(begin, end, "cancel", config.pending(), config.pending())) +} + +task_worker() () { + input = config.input() + blocking(bench.ready(0)) + each(task_sample, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], input) +} + +channel_worker() () { + input = config.input() + blocking(bench.ready(0)) + each(channel_sample, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], input) +} + +tasks() (result: Bool) { + task_worker() + result = true +} + +nested_task_worker() () { + job = run(task_worker()) + wait(job) +} + +tasks_on_worker() (result: Bool) { + nested_task_worker() + result = true +} + +channels() (result: Bool) { + channel_worker() + result = true +} + +cancel() (result: Bool) { + cancel_worker() + result = true +} diff --git a/test/performance/fixtures/runtime/port.dever b/test/performance/fixtures/runtime/port.dever new file mode 100644 index 0000000..822e81f --- /dev/null +++ b/test/performance/fixtures/runtime/port.dever @@ -0,0 +1,3 @@ +blocking_pause(millis: Int) () fails dever.time.SleepResult + +report_error(message: Text) () fails dever.io.WriteResult diff --git a/test/performance/live.py b/test/performance/live.py new file mode 100644 index 0000000..16be689 --- /dev/null +++ b/test/performance/live.py @@ -0,0 +1,97 @@ +"""长连接对端的阶段校验与采样；进程预算由统一运行器管理。""" + +import json +import time + +from process import Process, summarize + +PHASES = ("connecting", "connected", "closing", "disconnected") +SETTLE_SECONDS = 0.3 +PROCESS_MARGIN_SECONDS = 10 + + +def phase_sequence(cycles): + return [(cycle, phase) for cycle in range(cycles) for phase in PHASES] + + +def lifecycle_seconds(cycles, hold_seconds, timeout_ms): + timeout_seconds = timeout_ms / 1000 + # Each cycle has independent connect, hold-overrun and close deadlines. + return cycles * (hold_seconds + 3 * timeout_seconds + SETTLE_SECONDS) + PROCESS_MARGIN_SECONDS + + +def stable_phase(before, after): + return before[-1] if before and before == after else None + + +def parse_progress(text, cycles): + phases = [] + result = None + # stdout is sampled while its writer is alive; only consume complete lines. + for line in text.splitlines(keepends=True): + if not line.endswith("\n"): + continue + if result is not None: + raise ValueError("live benchmark output continued after its final result") + if line.startswith("PHASE|"): + _, cycle, phase = line.strip().split("|") + phases.append((int(cycle), phase)) + elif line.startswith("{"): + result = json.loads(line) + else: + raise ValueError(f"unexpected live benchmark output: {line.strip()}") + if phases != phase_sequence(cycles)[:len(phases)]: + raise ValueError("live benchmark phases are missing, duplicated or out of order") + return phases, result + + +def validate_result(result, protocol, connections, cycles): + counts = ("attempted_connections", "connected", "disconnected", "messages", "errors") + if result is None or any(type(result[key]) is not int or result[key] < 0 for key in counts): + raise ValueError("missing or invalid live counters") + expected = connections * cycles + if (result["protocol"] != protocol or result["connections"] != connections or result["cycles"] != cycles + or any(result[key] != expected for key in counts[:3]) + or result["messages"] < expected or result["errors"] != 0 or result["elapsed_ms"] <= 0): + raise ValueError("live benchmark did not validate and close all requested connections") + + +def load_live(args, process, port, protocol, connections, timeout_ms): + scheme, path = {"tcp": ("tcp", "/"), "ws": ("ws", "/ws"), "sse": ("http", "/events")}[protocol] + hold_ms = round(args.duration * 1000) + command = [args.artifacts / "live_bench", protocol, f"{scheme}://127.0.0.1:{port}{path}", + connections, args.cycles, hold_ms, timeout_ms] + phases = {phase: [] for phase in phase_sequence(args.cycles)} + with Process(command, process.directory / "load", cpus=args.client_cpus, + peer_settings={"workers": args.client_workers}) as client: + deadline = time.monotonic() + lifecycle_seconds(args.cycles, args.duration, timeout_ms) + while client.child.poll() is None: + if time.monotonic() >= deadline: + raise TimeoutError("long connection client exceeded its bounded lifecycle") + progress_before, _ = parse_progress(client.text(), args.cycles) + first = len(process.samples) + process.sample() + process.check() + if process.child.poll() is not None: + raise RuntimeError("server exited during long connection load") + progress_after, _ = parse_progress(client.text(), args.cycles) + if phase := stable_phase(progress_before, progress_after): + phases[phase].extend(process.samples[first:]) + client.sample() + time.sleep(args.interval) + client.check() + progress, result = parse_progress(client.text(), args.cycles) + if progress != phase_sequence(args.cycles): + raise ValueError("long connection client ended before all phases") + validate_result(result, protocol, connections, args.cycles) + # Connection setup/close can finish between polls. Only the two explicit + # hold windows require observations; retain setup samples when present. + observations = [] + for (cycle, phase), samples in phases.items(): + if phase in ("connected", "disconnected") and len(samples) < 2: + raise ValueError(f"sampling interval missed {cycle}/{phase} window") + observations.append({"cycle": cycle, "phase": phase, + "resources": summarize(samples) if samples else None, + "last": samples[-1] if samples else None, "raw": samples}) + return {"result": result, "phases": observations, + "client_resources": summarize(client.samples), "client_raw": client.samples} diff --git a/test/performance/process.py b/test/performance/process.py new file mode 100644 index 0000000..0051cbd --- /dev/null +++ b/test/performance/process.py @@ -0,0 +1,231 @@ +"""Linux 进程与可选专属 cgroup 的生命周期和采样。""" + +import os +from pathlib import Path +import signal +import statistics +import subprocess +import time +import uuid + +from settings import stage_peer + + +def key_values(path): + return {fields[0].rstrip(":"): fields[1] for line in path.read_text().splitlines() + if len(fields := line.split()) >= 2} + + +def process_sample(pid): + root = Path("/proc") / str(pid) + status = key_values(root / "status") + if "VmRSS" not in status: + raise ProcessLookupError(f"process {pid} exited during sampling") + memory = key_values(root / "smaps_rollup") + # comm 字段可含空格和括号；从最后一个右括号之后解析固定 stat 字段。 + stat = (root / "stat").read_text().rsplit(")", 1)[1].split() + switches = {"voluntary_ctxt_switches": 0, "nonvoluntary_ctxt_switches": 0} + for thread in (root / "task").iterdir(): + try: + thread_status = key_values(thread / "status") + except FileNotFoundError: + continue # A completed blocking thread can disappear during a sample. + for name in switches: + switches[name] += int(thread_status[name]) + return { + "time": time.monotonic(), + "rss_bytes": int(status["VmRSS"]) * 1024, + "rss_high_water_bytes": int(status["VmHWM"]) * 1024, + "pss_bytes": int(memory["Pss"]) * 1024, + "threads": int(status["Threads"]), + "fds": len(list((root / "fd").iterdir())), + "cpu_seconds": (int(stat[11]) + int(stat[12])) / os.sysconf("SC_CLK_TCK"), + **switches, + } + + +class Budget: + def __init__(self, parent, memory_mib, cpu_quota): + self.path = None + self.final = None + if parent is None: + if memory_mib is not None or cpu_quota is not None: + raise ValueError("memory/cpu quotas require --cgroup-parent") + return + self.path = parent.resolve() / ("dever-perf-" + uuid.uuid4().hex) + self.path.mkdir() + try: + if memory_mib is not None: + (self.path / "memory.max").write_text(str(memory_mib * 1024 * 1024)) + (self.path / "memory.swap.max").write_text("0") + if cpu_quota is not None: + (self.path / "cpu.max").write_text(f"{round(cpu_quota * 100000)} 100000") + except BaseException: + self.path.rmdir() + raise + + def sample(self): + if self.path is None: + return None + return { + "current_bytes": int((self.path / "memory.current").read_text()), + "peak_bytes": int((self.path / "memory.peak").read_text()), + "events": {k: int(v) for k, v in key_values(self.path / "memory.events").items()}, + "cpu": {k: int(v) for k, v in key_values(self.path / "cpu.stat").items()}, + "max": (self.path / "memory.max").read_text().strip(), + "cpu_max": (self.path / "cpu.max").read_text().strip(), + } + + def close(self): + if self.path is not None: + try: + self.final = self.sample() + except OSError as error: + try: + self.path.rmdir() + except OSError as cleanup_error: + error.add_note(f"cannot remove owned cgroup {self.path}: {cleanup_error}") + raise + else: + self.path.rmdir() + + +class Process: + def __init__(self, command, directory, *, peer_settings=None, cpus=None, budget=None): + self.command = [str(arg) for arg in command] + self.directory = directory + self.budget = budget + self.samples = [] + self.directory.mkdir(parents=True, exist_ok=False) + if peer_settings is not None: + self.command[0] = str(stage_peer(command[0], directory, peer_settings)) + self.stdout = (directory / "stdout.log").open("w") + self.stderr = (directory / "stderr.log").open("w") + self.started = time.monotonic() + + def prepare_child(): + # 运行器单线程；只对即将 exec 的自有子进程设置 CPU/cgroup。 + if cpus is not None: + os.sched_setaffinity(0, cpus) + if budget is not None and budget.path is not None: + (budget.path / "cgroup.procs").write_text(str(os.getpid())) + + try: + self.child = subprocess.Popen(self.command, stdout=self.stdout, stderr=self.stderr, + preexec_fn=prepare_child, start_new_session=True) + except BaseException: + self.stdout.close() + self.stderr.close() + raise + + def text(self): + return (self.directory / "stdout.log").read_text() + + def descriptors(self): + descriptors = {} + for path in (Path("/proc") / str(self.child.pid) / "fd").iterdir(): + try: + descriptors[path.name] = os.readlink(path) + except FileNotFoundError: + continue # A descriptor may close while /proc is being read. + return descriptors + + def sample(self): + if self.child.poll() is not None: + return + try: + sample = process_sample(self.child.pid) + except (FileNotFoundError, ProcessLookupError): + # Linux can release /proc memory fields before waitpid reports exit. + # This observation is unavailable; check() still owns exit status. + return + sample["elapsed_seconds"] = sample.pop("time") - self.started + if self.budget is not None: + sample["cgroup"] = self.budget.sample() + self.samples.append(sample) + + def ready(self, timeout=10): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + self.sample() + for line in self.text().splitlines(): + if line.startswith("ERROR|"): + raise RuntimeError(line) + if line.startswith("READY|"): + self.ready_seconds = time.monotonic() - self.started + return int(line.split("|")[1]) + if self.child.poll() is not None: + raise RuntimeError(f"process exited before readiness: {self.directory}") + time.sleep(0.005) + raise TimeoutError(f"readiness timeout: {self.directory}") + + def check(self): + code = self.child.poll() + if code is not None and code != 0: + raise RuntimeError(f"process exited {code}: {self.directory}") + if "ERROR|" in self.text() or (self.directory / "stderr.log").stat().st_size: + raise RuntimeError(f"benchmark reported an error: {self.directory}") + + def close(self): + try: + if self.child.poll() is None: + try: + os.killpg(self.child.pid, signal.SIGTERM) + except ProcessLookupError: + # 子进程可能恰好在 poll 与 signal 之间自然退出。 + pass + try: + self.child.wait(timeout=3) + except subprocess.TimeoutExpired: + os.killpg(self.child.pid, signal.SIGKILL) + self.child.wait(timeout=3) + finally: + self.stdout.close() + self.stderr.close() + + def __enter__(self): + return self + + def __exit__(self, *_): + self.close() + + +def observe(processes, seconds, interval): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + for process in processes: + process.sample() + process.check() + time.sleep(min(interval, max(0, deadline - time.monotonic()))) + + +def summarize(samples): + if not samples: + raise ValueError("process ended before any resource sample") + result = {"samples": len(samples)} + for metric in ("rss_bytes", "pss_bytes", "threads", "fds"): + values = sorted(sample[metric] for sample in samples) + result[metric + "_median"] = values[len(values) // 2] + result[metric + "_max"] = max(values) + result["rss_high_water_bytes"] = max(sample["rss_high_water_bytes"] for sample in samples) + elapsed = samples[-1]["elapsed_seconds"] - samples[0]["elapsed_seconds"] + cpu = samples[-1]["cpu_seconds"] - samples[0]["cpu_seconds"] + result["cpu_seconds_sampled"] = cpu + result["cpu_cores_sampled"] = cpu / elapsed if elapsed > 0 else None + for name in ("voluntary_ctxt_switches", "nonvoluntary_ctxt_switches"): + result[name + "_sampled"] = samples[-1][name] - samples[0][name] + return result + + +def resource_trend(samples): + """报告四段中位数与后半程增长，避免将启动分配误记为长期泄漏。""" + if len(samples) < 4: + raise ValueError("resource trend requires at least four observations") + windows = [samples[len(samples) * index // 4:len(samples) * (index + 1) // 4] + for index in range(4)] + medians = [statistics.median(sample["rss_bytes"] for sample in window) for window in windows] + elapsed = samples[-1]["elapsed_seconds"] - samples[0]["elapsed_seconds"] + return {"observed_seconds": elapsed, "rss_quarter_medians_bytes": medians, + "rss_late_growth_bytes": medians[-1] - medians[2], + "rss_late_growth_bytes_per_second": (medians[-1] - medians[2]) / (elapsed / 4) if elapsed > 0 else None, + "fds_first": samples[0]["fds"], "fds_last": samples[-1]["fds"]} diff --git a/test/performance/run.py b/test/performance/run.py new file mode 100644 index 0000000..ddb246b --- /dev/null +++ b/test/performance/run.py @@ -0,0 +1,1244 @@ +#!/usr/bin/env python3 +"""显式构建/运行独立二进制基准；不属于 cargo test 默认目标。""" + +import argparse +from contextlib import contextmanager +import http.client +from itertools import product +import json +import math +import os +from pathlib import Path +import platform +import select +import shutil +import socket +import statistics +import subprocess +import time + +from build import (APPLICATION_CONFIGURATION, BENCHMARK_COMMAND, ENTRIES, ROOT, build, database_setting, fingerprint, load_postgres_setting, + native_runtime_setting, postgres_case_url, postgres_database_case, + write_private_json) +from live import lifecycle_seconds, load_live +from process import Budget, Process, observe, summarize +from settings import PEER_CONFIGURATION + +HTTP_IMPLEMENTATIONS = ("dever", "runtime", "hyper") +HTTP2_DISCONNECT_SECONDS = 0.3 +HTTP_BODY_BYTES = {"/plain": 5, "/json": 11, "/bytes": 65_536} +ORM_ENTRIES = ("idle", "crud", "list", "cursor", "stream", "pool", "http") +ORM_COUNT_PATH = "/benchmark/item/count" +ORM_CANCEL_PATH = "/benchmark/item/cancel" + + +def application_command(executable): + return [executable, BENCHMARK_COMMAND, "{}"] + + +def fixture_command(executable, entry): + fixture = next(name for name, entries in ENTRIES.items() if entry in entries) + return [executable, f"benchmark.{fixture}.{entry}", "{}"] + + +def positive(value): + number = float(value) + if not math.isfinite(number) or number <= 0: + raise argparse.ArgumentTypeError("must be finite and positive") + return number + + +def positive_int(value): + number = int(value) + if number <= 0: + raise argparse.ArgumentTypeError("must be positive") + return number + + +def csv_positive_ints(value, name): + parts = value.split(",") + if any(not part for part in parts): + raise ValueError(f"{name} must contain unique positive integers") + values = tuple(positive_int(part) for part in parts) + if len(values) != len(set(values)): + raise ValueError(f"{name} must contain unique positive integers") + return values + + +def parse_client_cpus(value): + parts = value.split(",") + if any(not part for part in parts): + raise ValueError("client-cpus must contain unique nonnegative integers") + try: + cpus = tuple(int(part) for part in parts) + except ValueError as error: + raise ValueError("client-cpus must contain unique nonnegative integers") from error + if any(cpu < 0 for cpu in cpus) or len(cpus) != len(set(cpus)): + raise ValueError("client-cpus must contain unique nonnegative integers") + return cpus + + +def validate_cpu_placement(server_cpu, client_cpus, allowed): + if server_cpu is not None and server_cpu not in allowed: + raise ValueError(f"CPU {server_cpu} is outside allowed CPUs {sorted(allowed)}") + if client_cpus is not None: + for cpu in client_cpus: + if cpu not in allowed: + raise ValueError(f"CPU {cpu} is outside allowed CPUs {sorted(allowed)}") + if server_cpu in client_cpus: + raise ValueError("server and client must use different CPUs") + + +def parse_implementations(value): + implementations = tuple(value.split(",")) + if (any(not part for part in implementations) + or len(implementations) != len(set(implementations)) + or any(part not in HTTP_IMPLEMENTATIONS for part in implementations)): + raise ValueError("implementations must contain unique dever, runtime or hyper values") + return implementations + + +def implementation_order(implementations, repeat): + return implementations if repeat % 2 == 0 else tuple(reversed(implementations)) + + +def parser(): + cli = argparse.ArgumentParser(description=__doc__) + commands = cli.add_subparsers(dest="action", required=True) + prepare = commands.add_parser("build", help="build release native fixtures; no measurements") + prepare.add_argument("--output", type=Path, required=True) + prepare.add_argument("--selection", choices=("all", "cms-profiles", "profiles", "protocols"), default="all", + help="build all fixtures, CMS/profiles, profiles, or runtime/network protocols") + prepare.add_argument("--compiler", type=Path, default=ROOT / "target/debug/dever") + prepare.add_argument("--cargo", default="cargo") + for name in ("network_bench", "live_bench", "async_bench"): + prepare.add_argument("--" + name.replace("_", "-"), type=Path, + default=ROOT / "target/native-runtime/release/examples" / name) + prepare.add_argument("--workers", type=positive_int, default=1) + prepare.add_argument("--connections", type=positive_int, default=32) + prepare.add_argument("--http2-streams", type=positive_int, default=16) + prepare.add_argument("--http2-stream-window-bytes", type=positive_int, default=65_535) + prepare.add_argument("--http2-connection-window-bytes", type=positive_int, default=262_144) + prepare.add_argument("--pending", type=positive_int, default=256) + prepare.add_argument("--iterations", type=positive_int, default=16384) + prepare.add_argument("--orm-iterations", type=positive_int, default=128) + prepare.add_argument("--orm-rows", type=positive_int, default=128) + prepare.add_argument("--orm-concurrency", type=positive_int, default=8) + prepare.add_argument("--orm-cancel-work", type=positive_int, default=1_000_000) + prepare.add_argument("--orm-sqlite-connections", type=positive_int, default=1) + measure = commands.add_parser("run", help="measure already-built binaries only") + measure.add_argument("--artifacts", type=Path, required=True) + measure.add_argument("--output", type=Path, required=True) + measure.add_argument("--suite", choices=("all", "memory", "async", "http", "live", "http2", "http2-recovery", "orm", "cms"), default="all") + measure.add_argument("--duration", type=positive, default=10) + measure.add_argument("--warmup", type=positive, default=2) + measure.add_argument("--idle-seconds", type=positive, default=1) + measure.add_argument("--interval", type=positive, default=0.05) + measure.add_argument("--repeats", type=positive_int, default=3) + measure.add_argument("--rates", default="100,1000") + measure.add_argument("--paths", default="/plain,/json,/bytes") + measure.add_argument("--concurrency", type=positive_int, default=16) + measure.add_argument("--connection-mode", choices=("reuse", "new", "both"), default="reuse") + measure.add_argument("--implementations", default=",".join(HTTP_IMPLEMENTATIONS)) + measure.add_argument("--http2-connections", default="1,4") + measure.add_argument("--recovery-connections", type=positive_int, default=1) + measure.add_argument("--recovery-rate", type=positive_int, default=1_000) + measure.add_argument("--recovery-path", choices=("/plain", "/json", "/bytes"), default="/plain") + measure.add_argument("--connection-tiers", default="32") + measure.add_argument("--cycles", type=positive_int, default=3) + measure.add_argument("--server-cpu", type=int) + measure.add_argument("--client-cpus", type=parse_client_cpus) + measure.add_argument("--client-workers", type=positive_int, default=1) + measure.add_argument("--cgroup-parent", type=Path) + measure.add_argument("--memory-mib", type=positive_int) + measure.add_argument("--cpu-quota", type=positive) + return cli + + +@contextmanager +def server(args, command, directory, peer_settings): + budget = Budget(args.cgroup_parent, args.memory_mib, args.cpu_quota) + failure = None + try: + network_settings = peer_settings if Path(command[0]).name == "network_bench" else None + with Process(command, directory, peer_settings=network_settings, + cpus=(args.server_cpu,) if args.server_cpu is not None else None, + budget=budget) as process: + yield process, budget + except BaseException as error: + failure = error + raise + finally: + try: + budget.close() + require_no_oom(budget.final) + except BaseException as cleanup_error: + if failure is None: + raise + failure.add_note(f"cannot close benchmark budget: {cleanup_error}") + + +def percentile(values, fraction): + ordered = sorted(values) + return ordered[max(0, math.ceil(len(ordered) * fraction) - 1)] + + +def orm_measurements(text, expected_name, expected_samples, expected_checksum, elapsed_seconds): + samples = [] + for line in text.splitlines(): + if not line.startswith("SAMPLE|"): + continue + fields = line.split("|") + if len(fields) != 5: + raise ValueError(f"invalid ORM sample: {line}") + _, name, operations, nanos, checksum = fields + sample = {"name": name, "operations": int(operations), "nanos": int(nanos), + "checksum": int(checksum)} + if (name != expected_name or sample["operations"] <= 0 or sample["nanos"] <= 0 + or sample["checksum"] != expected_checksum + or len(samples) >= expected_samples): + raise ValueError(f"invalid ORM sample: {line}") + samples.append(sample) + if len(samples) != expected_samples: + raise ValueError( + f"ORM {expected_name} returned {len(samples)} samples, expected {expected_samples}" + ) + latencies = [sample["nanos"] / 1_000_000 for sample in samples] + operations = sum(sample["operations"] for sample in samples) + if elapsed_seconds <= 0: + raise ValueError("ORM measurement duration must be positive") + return { + "samples": len(samples), + "operations": operations, + "elapsed_seconds": elapsed_seconds, + "throughput_operations_per_second": operations / elapsed_seconds, + "latency_ms": { + "p50": percentile(latencies, 0.50), + "p95": percentile(latencies, 0.95), + "p99": percentile(latencies, 0.99), + }, + "raw": samples, + } + + +def expected_orm_checksum(entry, settings): + rows = settings["orm_rows"] + return { + "crud": 2, + "list": rows + min(32, rows), + "cursor": min(32, rows), + "stream": rows, + "pool": rows, + }[entry] + + +def async_measurements(text, settings): + samples = [] + for line in text.splitlines(): + if not line.startswith("SAMPLE|"): + continue + _, name, count, nanos, checksum = line.split("|") + count, nanos, checksum = int(count), int(nanos), int(checksum) + expected_count = settings["pending"] if name == "cancel" else settings["iterations"] + expected_checksum = count if name == "cancel" else count * ord("x") + if name not in ("tasks", "channels", "cancel") or count != expected_count or checksum != expected_checksum or nanos <= 0: + raise ValueError(f"invalid benchmark result: {line}") + samples.append({"name": name, "operations": count, "nanos": nanos, "checksum": checksum}) + if not samples: + raise ValueError("missing timing samples") + expected_samples = 1 if samples[0]["name"] == "cancel" else 10 + if len(samples) != expected_samples or len({sample["name"] for sample in samples}) != 1: + raise ValueError("incomplete or mixed timing samples") + measured = samples if expected_samples == 1 else samples[1:] + per_operation = [sample["nanos"] / sample["operations"] for sample in measured] + return {"raw": samples, "warmup_samples": expected_samples != 1, + "median_ns_per_operation": statistics.median(per_operation), + "p95_ns_per_operation_sample": percentile(per_operation, 0.95)} + + +def write_case(output, case): + with (output / "cases.jsonl").open("a") as record: + record.write(json.dumps(case, ensure_ascii=False) + "\n") + return case + + +def validate_load(result): + counters = ("scheduled", "sent", "completed", "succeeded", "errors", "dropped", + "dropped_capacity", "dispatch_expired", "latency_samples", "request_latency_samples", "dispatch_lag_samples") + if any(type(result[key]) is not int or result[key] < 0 for key in counters): + raise ValueError("load counts must be nonnegative integers") + if (result["scheduled"] != result["sent"] + result["dropped"] + or result["sent"] != result["completed"] + or result["completed"] != result["succeeded"] + result["errors"] + or result["dropped"] != result["dropped_capacity"] + result["dispatch_expired"] + or result["latency_samples"] != result["succeeded"] + or result["request_latency_samples"] != result["succeeded"] + or result["dispatch_lag_samples"] > result["completed"]): + raise ValueError("load counts do not account for all scheduled requests") + for metric, count in (("latency_ms", "latency_samples"), ("request_latency_ms", "request_latency_samples"), ("dispatch_lag_ms", "dispatch_lag_samples")): + values = result[metric] + if result[count] == 0: + if values is not None: + raise ValueError("empty histogram must be null") + elif (values is None or values.get("sample_count") != result[count] + or not 0 <= values["p50"] <= values["p95"] <= values["p99"]): + raise ValueError("invalid latency percentiles") + if result["elapsed_seconds"] <= 0 or not math.isfinite(result["actual_success_qps"]): + raise ValueError("invalid load duration/throughput") + if result["http_version"] not in ("h1", "h2"): + raise ValueError("invalid HTTP version in load result") + for name in ("physical_connections", "streams_per_connection", "request_slots"): + if type(result[name]) is not int or result[name] <= 0: + raise ValueError("invalid connection shape in load result") + if result["request_slots"] != result["physical_connections"] * result["streams_per_connection"]: + raise ValueError("load result connection shape is inconsistent") + if type(result["successful_handshakes"]) is not int or result["successful_handshakes"] < 0: + raise ValueError("invalid handshake count in load result") + if result["http_version"] == "h2" and result["successful_handshakes"] != result["physical_connections"]: + raise ValueError("HTTP/2 load did not establish its configured physical connections") + + +def require_no_request_errors(result, stage): + if result["errors"]: + raise RuntimeError(f"{stage} completed with {result['errors']} request errors") + + +def require_successful_requests(result, stage): + if result["succeeded"] == 0: + raise RuntimeError(f"{stage} completed without a successful request") + + +def require_no_oom(budget): + if budget is None: + return + events = budget["events"] + if any(events[name] for name in ("oom", "oom_kill", "oom_group_kill")): + raise RuntimeError(f"benchmark cgroup recorded OOM events: {events}") + + +def memory_cases(args, settings, peer_settings): + cases = [] + for repeat in range(args.repeats): + for entry in ("sync_idle", "async_idle", "parked", "http", "https"): + name = f"memory-{entry}-{repeat}" + print(name, flush=True) + binary = "cancel" if entry == "parked" else entry + command = fixture_command(args.artifacts / binary, binary) + with server(args, command, args.output / name, peer_settings) as (process, budget): + ready = process.ready() + # 采样窗口从 READY 后开始，避免把启动页错误当作空闲基线。 + first = len(process.samples) + observe([process], args.idle_seconds, args.interval) + if process.child.poll() is not None: + raise RuntimeError(f"idle fixture ended before sampling completed: {entry}") + case = {"suite": "memory", "entry": entry, "repeat": repeat, + "ready_seconds": process.ready_seconds, "ready_value": ready, + "resources": summarize(process.samples[first:]), "raw": process.samples, + "budget_before_stop": budget.sample()} + case["budget_after_stop"] = budget.final + cases.append(write_case(args.output, case)) + return cases + + +def async_cases(args, settings, peer_settings): + cases = [] + workloads = [("dever", "root", "tasks", fixture_command(args.artifacts / "tasks", "tasks")), + ("dever", "worker", "tasks", fixture_command(args.artifacts / "tasks_on_worker", "tasks_on_worker"))] + for mode in ("root", "worker"): + workloads.append(("runtime", mode, "tasks", [args.artifacts / "async_bench", mode, + settings["workers"], settings["task_capacity"], settings["iterations"]])) + workloads.extend(("dever", "root", entry, fixture_command(args.artifacts / entry, entry)) + for entry in ("channels", "cancel")) + for repeat in range(args.repeats): + for implementation, mode, entry, command in workloads[::1 if repeat % 2 == 0 else -1]: + name = f"async-{implementation}-{mode}-{entry}-{repeat}" + print(name, flush=True) + with server(args, command, args.output / name, peer_settings) as (process, budget): + process.ready() + deadline = time.monotonic() + 60 + while process.child.poll() is None: + if time.monotonic() >= deadline: + raise TimeoutError(f"async case exceeded 60 seconds: {entry}") + process.sample() + time.sleep(args.interval) + process.check() + case = {"suite": "async", "entry": entry, "repeat": repeat, + "implementation": implementation, "mode": mode, + "timing": async_measurements(process.text(), settings), + "resources": summarize(process.samples), "raw": process.samples, + "budget_before_stop": budget.sample()} + case["budget_after_stop"] = budget.final + cases.append(write_case(args.output, case)) + return cases + + +def load(args, process, directory, http_version, url, path, rate, duration, + physical_connections, streams_per_connection, reuse, timeout_ms, *, orm_count=None): + command = [args.artifacts / "network_bench", "load", http_version, url, path, rate, duration, + physical_connections, streams_per_connection, timeout_ms, str(reuse).lower(), + args.artifacts / "tls/root.pem"] + client_settings = {"workers": args.client_workers, + "http2_stream_window_bytes": args.http2_stream_window_bytes, + "http2_connection_window_bytes": args.http2_connection_window_bytes} + if orm_count is not None: + client_settings["orm_count"] = orm_count + with Process(command, directory, cpus=args.client_cpus, peer_settings=client_settings) as client: + deadline = time.monotonic() + duration + timeout_ms / 1000 + 10 + while client.child.poll() is None: + if time.monotonic() >= deadline: + raise TimeoutError(f"load client exceeded deadline: {directory}") + process.sample() + process.check() + if process.child.poll() is not None: + raise RuntimeError("server exited during load") + client.sample() + time.sleep(args.interval) + client.check() + result = json.loads(client.text()) + validate_load(result) + expected_shape = (http_version, physical_connections, streams_per_connection) + actual_shape = (result["http_version"], result["physical_connections"], + result["streams_per_connection"]) + if actual_shape != expected_shape: + raise ValueError(f"load result shape {actual_shape} does not match {expected_shape}") + result["client_resources"] = summarize(client.samples) + result["client_raw"] = client.samples + return result + + +def http_cases(args, settings, peer_settings): + cases = [] + modes = [True, False] if args.connection_mode == "both" else [args.connection_mode == "reuse"] + for repeat in range(args.repeats): + # 各轮倒转顺序，降低热机/后台负载变化对某个实现的固定偏置。 + implementations = implementation_order(args.implementations, repeat) + workloads = product(("http", "https"), args.paths.split(","), args.rates.split(","), modes, implementations) + for protocol, path, rate, reuse, implementation in workloads: + name = f"{implementation}-{protocol}-{path[1:]}-{rate}-{reuse}-{repeat}" + print(name, flush=True) + command = fixture_command(args.artifacts / protocol, protocol) if implementation == "dever" else [args.artifacts / "network_bench", f"{implementation}-{protocol}"] + with server(args, command, args.output / name, peer_settings) as (process, budget): + port = process.ready() + url = f"{protocol}://localhost:{port}" + warmup = load(args, process, process.directory / "warmup", "h1", url, path, + rate, args.warmup, args.concurrency, 1, reuse, settings["timeout_ms"]) + require_no_request_errors(warmup, "HTTP warmup") + require_successful_requests(warmup, "HTTP warmup") + first = len(process.samples) + result = load(args, process, process.directory / "load", "h1", url, path, + rate, args.duration, args.concurrency, 1, reuse, settings["timeout_ms"]) + require_no_request_errors(result, "HTTP measurement") + require_successful_requests(result, "HTTP measurement") + case = {"suite": "http", "implementation": implementation, + "protocol": protocol, "path": path, "rate": float(rate), + "reuse": reuse, "repeat": repeat, "warmup": warmup, "load": result, + "resources": summarize(process.samples[first:]), + "raw": process.samples, "budget_before_stop": budget.sample()} + case["budget_after_stop"] = budget.final + cases.append(write_case(args.output, case)) + return cases + + +def http2_server_command(args, implementation, transport): + if implementation == "dever": + entry = "http2" if transport == "http" else "https2" + return fixture_command(args.artifacts / entry, entry) + mode = "http2" if transport == "http" else "https2" + return [args.artifacts / "network_bench", f"{implementation}-{mode}"] + + +def observe_disconnected(args, process, baseline_fds): + first = len(process.samples) + observe([process], HTTP2_DISCONNECT_SECONDS, args.interval) + samples = process.samples[first:] + resources = summarize(samples) + last = samples[-1] + fd_delta = last["fds"] - baseline_fds + if fd_delta != 0: + raise RuntimeError(f"HTTP/2 connections were not released: FD delta {fd_delta}") + return {"last": last, "resources": resources, "fd_delta": fd_delta} + + +def http2_cases(args, settings, peer_settings): + cases = [] + streams = settings["http2_streams"] + for repeat in range(args.repeats): + implementations = implementation_order(args.implementations, repeat) + workloads = product(("http", "https"), args.paths.split(","), args.rates.split(","), + args.http2_connections, implementations) + for transport, path, rate, connections, implementation in workloads: + name = (f"{implementation}-h2-{transport}-{path[1:]}-{rate}-" + f"{connections}x{streams}-{repeat}") + print(name, flush=True) + command = http2_server_command(args, implementation, transport) + with server(args, command, args.output / name, peer_settings) as (process, budget): + port = process.ready() + url = f"{transport}://localhost:{port}" + baseline_first = len(process.samples) + observe([process], HTTP2_DISCONNECT_SECONDS, args.interval) + baseline_samples = process.samples[baseline_first:] + baseline = summarize(baseline_samples) + baseline_fds = baseline_samples[-1]["fds"] + warmup = load(args, process, process.directory / "warmup", "h2", url, path, + rate, args.warmup, connections, streams, True, settings["timeout_ms"]) + warmup_disconnected = observe_disconnected(args, process, baseline_fds) + first = len(process.samples) + result = load(args, process, process.directory / "load", "h2", url, path, + rate, args.duration, connections, streams, True, settings["timeout_ms"]) + load_end = len(process.samples) + disconnected = observe_disconnected(args, process, baseline_fds) + case = {"suite": "http2", "implementation": implementation, + "http_version": "h2", "transport": transport, "path": path, + "body_bytes": HTTP_BODY_BYTES[path], + "rate": float(rate), "physical_connections": connections, + "streams_per_connection": streams, + "request_slots": connections * streams, + "stream_window_bytes": settings["http2_stream_window_bytes"], + "connection_window_bytes": settings["http2_connection_window_bytes"], + "frame_bytes": 16_384, "repeat": repeat, + "baseline": baseline, "warmup": warmup, + "warmup_disconnected": warmup_disconnected, + "load": result, "resources": summarize(process.samples[first:load_end]), + "disconnected": disconnected, "raw": process.samples, + "budget_before_stop": budget.sample()} + case["budget_after_stop"] = budget.final + cases.append(write_case(args.output, case)) + require_no_request_errors(warmup, "HTTP/2 warmup") + require_no_request_errors(result, "HTTP/2 load") + require_no_oom(case["budget_after_stop"]) + return cases + + +def http2_recovery_cases(args, settings, peer_settings): + cases = [] + streams = settings["http2_streams"] + connections = args.recovery_connections + for repeat in range(args.repeats): + for transport, implementation in product(("http", "https"), + implementation_order(args.implementations, repeat)): + name = f"{implementation}-h2-recovery-{transport}-{connections}x{streams}-{repeat}" + print(name, flush=True) + command = http2_server_command(args, implementation, transport) + with server(args, command, args.output / name, peer_settings) as (process, budget): + port = process.ready() + url = f"{transport}://localhost:{port}" + baseline_first = len(process.samples) + observe([process], HTTP2_DISCONNECT_SECONDS, args.interval) + baseline_samples = process.samples[baseline_first:] + baseline = summarize(baseline_samples) + baseline_fds = baseline_samples[-1]["fds"] + first = len(process.samples) + cycles = [] + for cycle in range(args.cycles): + result = load(args, process, process.directory / f"cycle-{cycle}", "h2", url, + args.recovery_path, args.recovery_rate, args.duration, + connections, streams, True, settings["timeout_ms"]) + if result["succeeded"] == 0: + raise RuntimeError(f"HTTP/2 recovery cycle {cycle} did not complete cleanly") + require_no_request_errors(result, f"HTTP/2 recovery cycle {cycle}") + disconnected = observe_disconnected(args, process, baseline_fds) + cycles.append({"cycle": cycle, "load": result, "disconnected": disconnected}) + case = {"suite": "http2-recovery", "implementation": implementation, + "http_version": "h2", "transport": transport, + "path": args.recovery_path, "rate": args.recovery_rate, + "physical_connections": connections, + "streams_per_connection": streams, + "request_slots": connections * streams, + "stream_window_bytes": settings["http2_stream_window_bytes"], + "connection_window_bytes": settings["http2_connection_window_bytes"], + "frame_bytes": 16_384, "cycles": cycles, + "repeat": repeat, "baseline": baseline, + "resources": summarize(process.samples[first:]), + "raw": process.samples, "budget_before_stop": budget.sample()} + case["budget_after_stop"] = budget.final + cases.append(write_case(args.output, case)) + require_no_oom(case["budget_after_stop"]) + return cases + + +def http2_summary(cases): + groups = {} + for case in cases: + if case["suite"] != "http2": + continue + key = (case["implementation"], case["transport"], case["path"], case["rate"], + case["physical_connections"], case["streams_per_connection"]) + groups.setdefault(key, []).append(case) + summary = [] + for key, repeated in sorted(groups.items()): + implementation, transport, path, rate, connections, streams = key + p99_values = [case["load"]["request_latency_ms"]["p99"] for case in repeated + if case["load"]["request_latency_ms"] is not None] + summary.append({ + "implementation": implementation, + "transport": transport, + "path": path, + "rate": rate, + "physical_connections": connections, + "streams_per_connection": streams, + "request_slots": connections * streams, + "repeats": len(repeated), + "actual_success_qps_median": statistics.median( + case["load"]["actual_success_qps"] for case in repeated), + "request_latency_p99_ms_median": statistics.median(p99_values) if p99_values else None, + "dropped_median": statistics.median(case["load"]["dropped"] for case in repeated), + "server_rss_bytes_median": statistics.median( + case["resources"]["rss_bytes_median"] for case in repeated), + "server_pss_bytes_median": statistics.median( + case["resources"]["pss_bytes_median"] for case in repeated), + "server_rss_high_water_bytes_median": statistics.median( + case["resources"]["rss_high_water_bytes"] for case in repeated), + }) + return summary + + +def stage_case_binary(args, case_name, binary_name): + bundle = args.output / "bundles" / case_name + bundle.mkdir(parents=True, exist_ok=False) + executable = bundle / "app" + shutil.copy2(args.artifacts / binary_name, executable) + return executable + + +def stage_live_bundle(args, settings, case_name, protocol): + executable = stage_case_binary(args, case_name, protocol) + config = executable.parent / "config" + config.mkdir() + write_private_json(config / "setting.json", native_runtime_setting(settings)) + return executable + + +def live_cases(args, settings, peer_settings): + cases = [] + for repeat, protocol, connections in product(range(args.repeats), ("tcp", "ws", "sse"), + map(int, args.connection_tiers.split(","))): + name = f"live-{protocol}-{connections}-{repeat}" + print(name, flush=True) + executable = stage_live_bundle(args, settings, name, protocol) + with server(args, fixture_command(executable, protocol), args.output / name, peer_settings) as (process, budget): + port = process.ready() + first = len(process.samples) + observe([process], 0.3, args.interval) + baseline = summarize(process.samples[first:]) + measured = load_live(args, process, port, protocol, connections, settings["timeout_ms"]) + disconnected = [phase["last"] for phase in measured["phases"] if phase["phase"] == "disconnected"] + fd_deltas = [sample["fds"] - baseline["fds_median"] for sample in disconnected] + case = {"suite": "live", "protocol": protocol, "connections": connections, + "repeat": repeat, "baseline": baseline, **measured, + "fd_delta_after_cycles": fd_deltas, + "rss_delta_after_cycles": [sample["rss_bytes"] - baseline["rss_bytes_median"] for sample in disconnected], + "budget_before_stop": budget.sample()} + case["budget_after_stop"] = budget.final + cases.append(write_case(args.output, case)) + if any(delta != 0 for delta in fd_deltas): + raise RuntimeError(f"live {protocol} did not release file descriptors: {fd_deltas}") + return cases + + +def report_arguments(args): + values = {key: str(value) if isinstance(value, Path) else value + for key, value in vars(args).items() if not key.startswith("_")} + return values + + +def required_binaries(args): + suites = { + "memory": {"sync_idle", "async_idle", "cancel", "http", "https"}, + "async": {"tasks", "tasks_on_worker", "channels", "cancel", "async_bench"}, + "http": {"http", "https", "network_bench"}, + "live": {"tcp", "ws", "sse", "live_bench"}, + "http2": {"http2", "https2", "network_bench"}, + "http2-recovery": {"http2", "https2", "network_bench"}, + "orm": {"network_bench", *(f"sqlite-{entry}" for entry in ORM_ENTRIES)}, + "cms": {"cms"}, + } + if args._postgres_setting is not None: + suites["orm"].update(f"postgres-{entry}" for entry in ORM_ENTRIES) + if "md" in args._cms_sources: + suites["cms"].add("cms-md") + selected = suites if args.suite == "all" else {args.suite: suites[args.suite]} + return set().union(*selected.values()) + + +def verify_artifacts(directory, manifest, required): + if manifest.get("peer_configuration") != PEER_CONFIGURATION: + raise ValueError("benchmark artifacts use an obsolete configuration contract; rebuild them") + if manifest.get("application_configuration") != APPLICATION_CONFIGURATION: + raise ValueError("benchmark artifacts use an obsolete application contract; rebuild with the current LLVM compiler") + binaries = manifest["binaries"] + missing = sorted(required - binaries.keys()) + if missing: + raise ValueError(f"manifest is missing required binaries: {', '.join(missing)}") + for group in (binaries, manifest["files"]): + for name, recorded in group.items(): + actual = fingerprint(directory / name) + if actual != {"sha256": recorded["sha256"], "bytes": recorded["bytes"]}: + raise ValueError(f"artifact changed since build: {name}") + + +def stage_orm_bundle(args, settings, driver, entry, repeat, *, binary_entry=None, + pool_capacity=None): + case_name = f"orm-{driver}-{entry}-{repeat}" + binary_name = f"{driver}-{binary_entry or entry}" + executable = stage_case_binary(args, case_name, binary_name) + bundle = executable.parent + if driver == "postgres": + setting = database_setting( + driver, + settings, + args._postgres_setting, + postgres_database_case(args.output, case_name), + ) + else: + setting = database_setting(driver, settings) + if pool_capacity is not None: + setting["database"]["default"]["max_connections"] = pool_capacity + if entry in ("http", "cancel"): + with socket.socket() as reservation: + reservation.bind(("127.0.0.1", 0)) + port = reservation.getsockname()[1] + setting["http"] = {"host": "127.0.0.1", "port": port} + config = bundle / "config" + config.mkdir() + setting_path = config / "setting.json" + write_private_json(setting_path, setting) + return case_name, executable, args.output / case_name, binary_name + + +def orm_observability(settings, driver, pool_capacity=None): + capacity = pool_capacity or settings[f"orm_{driver}_connections"] + return { + "configured_pool_capacity": capacity, + "pool_wait": {"status": "unavailable", + "reason": "runtime does not expose pool acquisition timing"}, + "sql_count": {"status": "unavailable", + "reason": "runtime does not expose executed statement counters"}, + "database_connections": { + "status": "unavailable", + "reason": "process FD count is reported, but database connection identity is not observable", + }, + } + + +def finite_orm_case(args, settings, peer_settings, driver, entry, repeat): + name, executable, directory, binary_name = stage_orm_bundle( + args, settings, driver, entry, repeat + ) + print(name, flush=True) + if entry in ("stream", "pool"): + seed_orm_bundle(executable) + with server(args, application_command(executable), directory, peer_settings) as (process, budget): + ready = process.ready() + first = max(0, len(process.samples) - 1) + measured_at = time.monotonic() + deadline = measured_at + max(30, args.duration + args.warmup + 20) + while process.child.poll() is None: + if time.monotonic() >= deadline: + raise TimeoutError(f"ORM case exceeded deadline: {name}") + process.sample() + process.check() + time.sleep(args.interval) + elapsed = time.monotonic() - measured_at + process.check() + expected_samples = 1 if entry == "cancel" else settings["orm_iterations"] + measured = orm_measurements( + process.text(), entry, expected_samples, + expected_orm_checksum(entry, settings), elapsed, + ) + samples = process.samples[first:] + if not samples: + raise ValueError(f"ORM case ended before a resource sample: {name}") + case = { + "suite": "orm", + "driver": driver, + "entry": entry, + "repeat": repeat, + "ready_seconds": process.ready_seconds, + "ready_value": ready, + "binary": {"name": binary_name, **fingerprint(executable)}, + "measurement": measured, + "resources": summarize(samples), + "raw": process.samples, + "observability": orm_observability(settings, driver), + "budget_before_stop": budget.sample(), + } + case["budget_after_stop"] = budget.final + require_no_oom(case["budget_after_stop"]) + return write_case(args.output, case) + + +def idle_orm_case(args, settings, peer_settings, driver, repeat): + name, executable, directory, binary_name = stage_orm_bundle( + args, settings, driver, "idle", repeat + ) + print(name, flush=True) + with server(args, application_command(executable), directory, peer_settings) as (process, budget): + ready = process.ready() + first = len(process.samples) + observe([process], args.idle_seconds, args.interval) + samples = process.samples[first:] + case = { + "suite": "orm", + "driver": driver, + "entry": "idle", + "repeat": repeat, + "ready_seconds": process.ready_seconds, + "ready_value": ready, + "binary": {"name": binary_name, **fingerprint(executable)}, + "resources": summarize(samples), + "raw": process.samples, + "observability": orm_observability(settings, driver), + "budget_before_stop": budget.sample(), + } + case["budget_after_stop"] = budget.final + require_no_oom(case["budget_after_stop"]) + return write_case(args.output, case) + + +def http_orm_case(args, settings, peer_settings, driver, repeat): + name, executable, directory, binary_name = stage_orm_bundle( + args, settings, driver, "http", repeat + ) + print(name, flush=True) + rate = positive_int(args.rates.split(",")[0]) + seed_orm_bundle(executable) + expected_count = settings["orm_rows"] + port = orm_http_port(executable) + with server(args, [executable], directory, peer_settings) as (process, budget): + wait_orm_ready(process, port, expected_count) + url = f"http://127.0.0.1:{port}" + warmup = load(args, process, process.directory / "warmup", "h1", url, ORM_COUNT_PATH, + rate, args.warmup, args.concurrency, 1, True, settings["timeout_ms"], + orm_count=expected_count) + first = len(process.samples) + result = load(args, process, process.directory / "load", "h1", url, ORM_COUNT_PATH, + rate, args.duration, args.concurrency, 1, True, settings["timeout_ms"], + orm_count=expected_count) + case = { + "suite": "orm", + "driver": driver, + "entry": "http", + "repeat": repeat, + "rate": rate, + "warmup": warmup, + "load": result, + "binary": {"name": binary_name, **fingerprint(executable)}, + "resources": summarize(process.samples[first:]), + "raw": process.samples, + "observability": orm_observability(settings, driver), + "budget_before_stop": budget.sample(), + } + case["budget_after_stop"] = budget.final + require_no_request_errors(warmup, "ORM HTTP warmup") + require_no_request_errors(result, "ORM HTTP load") + require_successful_requests(warmup, "ORM HTTP warmup") + require_successful_requests(result, "ORM HTTP load") + require_no_oom(case["budget_after_stop"]) + return write_case(args.output, case) + + +def orm_http_port(executable): + return json.loads((executable.parent / "config/setting.json").read_text())["http"]["port"] + + +def seed_orm_bundle(executable): + result = subprocess.run([executable, "benchmark.item.seed", "{}"], cwd=executable.parent, + capture_output=True, text=True, timeout=30) + if result.returncode != 0: + raise RuntimeError(f"ORM seed failed: {result.stderr or result.stdout}") + + +def http_data(port, path, timeout): + connection = http.client.HTTPConnection("127.0.0.1", port, timeout=timeout) + try: + connection.request("GET", path, headers={"Connection": "close"}) + response = connection.getresponse() + body = response.read().decode("utf-8") + if response.status != 200: + raise RuntimeError(f"ORM probe {path} returned HTTP {response.status}") + if response.getheader("Content-Type") != "application/json; charset=utf-8": + raise ValueError(f"ORM probe {path} returned a non-JSON response") + envelope = json.loads(body) + if (type(envelope) is not dict or set(envelope) != {"code", "message", "data"} + or type(envelope["code"]) is not int or envelope["code"] != 0 + or envelope["message"] != "ok"): + raise ValueError(f"ORM probe {path} returned an invalid API envelope") + return envelope["data"] + finally: + connection.close() + + +def orm_count(port, timeout): + count = http_data(port, ORM_COUNT_PATH, timeout) + if type(count) is not int or count < 0: + raise ValueError("ORM count API returned a nonnegative-integer violation") + return count + + +def wait_orm_ready(process, port, expected_count): + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + process.sample() + process.check() + if process.child.poll() is not None: + raise RuntimeError(f"ORM HTTP service exited before readiness: {process.directory}") + try: + count = orm_count(port, 0.2) + except (ConnectionError, TimeoutError, socket.timeout): + time.sleep(0.01) + continue + if count != expected_count: + raise RuntimeError(f"ORM HTTP service started with {count} rows, expected {expected_count}") + process.ready_seconds = time.monotonic() - process.started + return + raise TimeoutError(f"ORM HTTP readiness timeout: {process.directory}") + + +def transaction_cancel_measurement(remaining, elapsed_seconds): + if remaining != 0: + raise RuntimeError(f"cancelled transaction committed {remaining} row(s)") + if elapsed_seconds <= 0: + raise ValueError("transaction cancellation duration must be positive") + latency = elapsed_seconds * 1000 + return { + "samples": 1, + "operations": 1, + "elapsed_seconds": elapsed_seconds, + "throughput_operations_per_second": 1 / elapsed_seconds, + "latency_ms": {"p50": latency, "p95": latency, "p99": latency}, + "raw": [{"name": "cancel", "operations": 1, + "nanos": round(elapsed_seconds * 1_000_000_000), + "checksum": remaining}], + } + + +def cancel_orm_case(args, settings, peer_settings, driver, repeat): + name, executable, directory, binary_name = stage_orm_bundle( + args, settings, driver, "cancel", repeat, binary_entry="http", pool_capacity=1 + ) + print(name, flush=True) + port = orm_http_port(executable) + with server(args, [executable], directory, peer_settings) as (process, budget): + wait_orm_ready(process, port, 0) + baseline_count = orm_count(port, 0.2) + if baseline_count != 0: + raise RuntimeError(f"cancel fixture started with {baseline_count} row(s)") + first = len(process.samples) + cancel = socket.create_connection(("127.0.0.1", port), timeout=settings["timeout_ms"] / 1000) + try: + cancel.sendall( + b"POST /benchmark/item/cancel HTTP/1.1\r\nHost: localhost\r\n" + b"Content-Type: application/json\r\nContent-Length: 2\r\n" + b"Connection: close\r\n\r\n{}" + ) + blocked = False + handler_timeout = settings["timeout_ms"] / 1000 + probe_deadline = time.monotonic() + handler_timeout * 0.5 + probe_attempts = 0 + for probe_attempts in range(1, 65): + if time.monotonic() >= probe_deadline: + break + process.sample() + process.check() + if select.select([cancel], [], [], 0)[0]: + raise RuntimeError("transaction cancel request completed before pool saturation") + try: + orm_count(port, 0.05) + except (TimeoutError, socket.timeout): + blocked = True + break + if not blocked: + raise TimeoutError("transaction did not occupy the only pool connection") + cancelled_at = time.monotonic() + finally: + try: + cancel.shutdown(socket.SHUT_RDWR) + except OSError: + pass + cancel.close() + + recovery_deadline = cancelled_at + handler_timeout * 0.75 + remaining = None + recovery_attempts = 0 + for recovery_attempts in range(1, 51): + if time.monotonic() >= recovery_deadline: + break + process.sample() + process.check() + try: + remaining = orm_count(port, 0.2) + break + except (TimeoutError, socket.timeout): + continue + if remaining is None: + raise TimeoutError("database connection did not return after request cancellation") + elapsed = time.monotonic() - cancelled_at + samples = process.samples[first:] + case = { + "suite": "orm", + "driver": driver, + "entry": "cancel", + "repeat": repeat, + "binary": {"name": binary_name, **fingerprint(executable)}, + "measurement": transaction_cancel_measurement(remaining, elapsed), + "resources": summarize(samples), + "raw": process.samples, + "observability": orm_observability(settings, driver, pool_capacity=1), + "baseline_count": baseline_count, + "pool_saturation_observed": blocked, + "client_disconnected_before_completion": True, + "connection_recovered_before_handler_timeout": True, + "probe_attempts": probe_attempts, + "recovery_attempts": recovery_attempts, + "budget_before_stop": budget.sample(), + } + case["budget_after_stop"] = budget.final + require_no_oom(case["budget_after_stop"]) + return write_case(args.output, case) + + +def orm_cases(args, settings, peer_settings): + drivers = ["sqlite"] + if args._postgres_setting is not None: + drivers.append("postgres") + cases = [] + for repeat in range(args.repeats): + for driver in drivers: + cases.append(idle_orm_case(args, settings, peer_settings, driver, repeat)) + for entry in ("crud", "list", "cursor", "stream", "pool"): + cases.append(finite_orm_case(args, settings, peer_settings, driver, entry, repeat)) + cases.append(cancel_orm_case(args, settings, peer_settings, driver, repeat)) + cases.append(http_orm_case(args, settings, peer_settings, driver, repeat)) + return cases + + +def orm_summary(cases): + groups = {} + for case in cases: + if case["suite"] != "orm": + continue + groups.setdefault((case["driver"], case["entry"]), []).append(case) + summary = [] + for (driver, entry), repeated in sorted(groups.items()): + item = { + "driver": driver, + "entry": entry, + "repeats": len(repeated), + "binary_bytes": repeated[0]["binary"]["bytes"], + "rss_bytes_median": statistics.median( + case["resources"]["rss_bytes_median"] for case in repeated), + "pss_bytes_median": statistics.median( + case["resources"]["pss_bytes_median"] for case in repeated), + "rss_high_water_bytes_median": statistics.median( + case["resources"]["rss_high_water_bytes"] for case in repeated), + } + measurements = [case["measurement"] for case in repeated if "measurement" in case] + loads = [case["load"] for case in repeated if "load" in case] + if measurements: + item.update({ + "throughput_operations_per_second_median": statistics.median( + value["throughput_operations_per_second"] for value in measurements), + "latency_p50_ms_median": statistics.median( + value["latency_ms"]["p50"] for value in measurements), + "latency_p95_ms_median": statistics.median( + value["latency_ms"]["p95"] for value in measurements), + "latency_p99_ms_median": statistics.median( + value["latency_ms"]["p99"] for value in measurements), + }) + elif loads: + item.update({ + "throughput_operations_per_second_median": statistics.median( + value["actual_success_qps"] for value in loads), + "latency_p50_ms_median": statistics.median( + value["request_latency_ms"]["p50"] for value in loads), + "latency_p95_ms_median": statistics.median( + value["request_latency_ms"]["p95"] for value in loads), + "latency_p99_ms_median": statistics.median( + value["request_latency_ms"]["p99"] for value in loads), + }) + summary.append(item) + return summary + + +def cms_cases(args, _settings, _peer_settings): + from cms import run_case + + cases = [] + for repeat in range(args.repeats): + for source in args._cms_sources: + name = f"cms-{source}-{repeat}" + binary_name = "cms" if source == "dever" else "cms-md" + executable = args.artifacts / binary_name + identity = { + "executable": str(executable), + "config": str(args.artifacts / "source" / f"cms-{source}" / "config/setting.json"), + "fingerprint": fingerprint(executable), + } + print(name, flush=True) + result = run_case( + identity, args.output / name, articles=16, + cgroup_parent=args.cgroup_parent, memory_mib=args.memory_mib, + idle_seconds=args.idle_seconds, interval=args.interval, + server_cpu=args.server_cpu, cpu_quota=args.cpu_quota, + ) + require_no_oom(result["cgroup"]) + case = {"suite": "cms", "source": source, "repeat": repeat, + "workload_mode": "sequential_publish", + "binary": {"name": binary_name, **identity["fingerprint"]}, **result} + cases.append(write_case(args.output, case)) + if len({json.dumps(case["contract"], sort_keys=True) for case in cases}) != 1: + raise RuntimeError("Dever and Markdown CMS changed the publish contract") + return cases + + +def cms_summary(cases): + repeated = [case for case in cases if case["suite"] == "cms"] + if not repeated: + return None + return {source: cms_source_summary([case for case in repeated + if case["source"] == source]) + for source in sorted({case["source"] for case in repeated})} + + +def cms_source_summary(repeated): + peaks = [case["cgroup"]["peak_bytes"] for case in repeated + if case["cgroup"] is not None] + return { + "repeats": len(repeated), + "workload_mode": "sequential_publish", + "binary_bytes": repeated[0]["binary"]["bytes"], + "articles": repeated[0]["contract"]["articles"], + "ready_seconds_median": statistics.median( + case["ready_seconds"] for case in repeated), + "workload_seconds_median": statistics.median( + case["workload_seconds"] for case in repeated), + "published_articles_per_second_median": statistics.median( + case["published_articles_per_second"] for case in repeated), + "rss_bytes_median": statistics.median( + case["resources"]["rss_bytes_median"] for case in repeated), + "pss_bytes_median": statistics.median( + case["resources"]["pss_bytes_median"] for case in repeated), + "rss_high_water_bytes_median": statistics.median( + case["resources"]["rss_high_water_bytes"] for case in repeated), + "cgroup_peak_bytes_median": statistics.median(peaks) if peaks else None, + } + + +def measure(args): + args.artifacts = args.artifacts.resolve() + args.output = args.output.resolve() + args._postgres_setting = ( + load_postgres_setting() if args.suite in ("all", "orm") else None + ) + manifest = json.loads((args.artifacts / "manifest.json").read_text()) + args._cms_sources = ("dever", "md") if "cms-md" in manifest["binaries"] else ("dever",) + verify_artifacts(args.artifacts, manifest, required_binaries(args)) + settings = manifest["settings"] + args.http2_stream_window_bytes = settings["http2_stream_window_bytes"] + args.http2_connection_window_bytes = settings["http2_connection_window_bytes"] + args.implementations = parse_implementations(args.implementations) + args.http2_connections = csv_positive_ints(args.http2_connections, "http2-connections") + if args.suite in ("all", "http") and args.concurrency > settings["connections"]: + raise ValueError("client concurrency must not exceed compiled server connections") + if args.idle_seconds >= settings["idle_sample_ms"] / 1000: + raise ValueError("idle window must be shorter than the parked fixture's 3-second hold") + if any(path not in ("/plain", "/json", "/bytes") for path in args.paths.split(",")): + raise ValueError("paths must be /plain, /json or /bytes") + for rate in args.rates.split(","): + positive_int(rate) + if args.duration + args.warmup + 20 >= settings["lifetime_ms"] / 1000: + raise ValueError("measurement exceeds fixture lifetime") + if args.suite in ("all", "http2", "http2-recovery"): + streams = settings["http2_streams"] + physical_connections = (*args.http2_connections, args.recovery_connections) + if max(physical_connections) > settings["connections"]: + raise ValueError("HTTP/2 physical connections exceed compiled server capacity") + if max(physical_connections) * streams > 65_536: + raise ValueError("HTTP/2 request slots exceed the language limit") + if max(physical_connections) * (streams + 1) > settings["task_capacity"]: + raise ValueError("HTTP/2 topology exceeds compiled task capacity") + if (args.suite in ("all", "http2-recovery") + and args.cycles * (args.duration + HTTP2_DISCONNECT_SECONDS) + 20 + >= settings["lifetime_ms"] / 1000): + raise ValueError("HTTP/2 recovery cycles exceed fixture lifetime") + if args.suite in ("all", "live"): + tiers = [positive_int(value) for value in args.connection_tiers.split(",")] + if max(tiers) > min(settings["connections"], 4096) or args.cycles > 100: + raise ValueError("long connection workload exceeds configured capacity") + if not 0.3 <= args.duration <= 60 or args.interval > 0.1: + raise ValueError("live duration must be 0.3..60 seconds and interval <= 0.1 seconds") + if lifecycle_seconds(args.cycles, args.duration, settings["timeout_ms"]) >= settings["lifetime_ms"] / 1000: + raise ValueError("long connection cycles exceed fixture lifetime") + if args.suite in ("all", "orm"): + if settings["orm_concurrency"] > settings["orm_iterations"]: + raise ValueError("ORM concurrency must not exceed ORM iterations") + allowed = os.sched_getaffinity(0) + validate_cpu_placement(args.server_cpu, args.client_cpus, allowed) + peer_settings = {name: settings[name] for name in ( + "workers", "connections", "task_capacity", "timeout_ms", "lifetime_ms", + "http2_streams", "http2_stream_window_bytes", "http2_connection_window_bytes")} + peer_settings.update({"cert_path": str(args.artifacts / "tls/server.pem"), + "key_path": str(args.artifacts / "tls/server-key.pem")}) + args.output.mkdir(parents=True, exist_ok=False) + report = {"status": "running", "manifest": manifest, + "environment": {"platform": platform.platform(), "cpu_count": os.cpu_count(), + "client_workers": args.client_workers, + "client_cpus": list(args.client_cpus) if args.client_cpus is not None else sorted(allowed), + "allowed_cpus": sorted(allowed), "kernel": platform.release(), + "cpuinfo": Path("/proc/cpuinfo").read_text(), + "parent_cgroup": Path("/proc/self/cgroup").read_text()}, + "arguments": report_arguments(args), + "cases": []} + failure = None + try: + for suite, execute in (("memory", memory_cases), ("async", async_cases), + ("http", http_cases), ("live", live_cases), + ("http2", http2_cases), + ("http2-recovery", http2_recovery_cases), + ("orm", orm_cases), ("cms", cms_cases)): + if args.suite in ("all", suite): + report["cases"].extend(execute(args, settings, peer_settings)) + report["status"] = "complete" + except BaseException as error: + failure = error + report["status"] = "failed" + report["error"] = str(error) + raise + finally: + try: + # 即使中途失败，也保留已完成 case 与原始日志。 + records = args.output / "cases.jsonl" + report["cases"] = [json.loads(line) for line in records.read_text().splitlines()] if records.exists() else [] + report["summary"] = {"http2": http2_summary(report["cases"]), + "orm": orm_summary(report["cases"]), + "cms": cms_summary(report["cases"])} + (args.output / "report.json").write_text( + json.dumps(report, ensure_ascii=False, indent=2) + "\n" + ) + except BaseException as cleanup_error: + if failure is None: + raise + failure.add_note(f"cannot persist benchmark report: {cleanup_error}") + print(f"report: {args.output / 'report.json'}", flush=True) + + +def main(): + args = parser().parse_args() + if args.action == "build": + if (args.workers > 64 or args.connections > 32760 or args.pending > 65520 + or args.iterations > 1_048_576 or args.http2_streams > 65_536 + or args.connections * args.http2_streams > 65_536 + or args.http2_stream_window_bytes > 2_147_483_647 + or not 65_535 <= args.http2_connection_window_bytes <= 2_147_483_647 + or args.orm_iterations > 4096 or args.orm_rows > 10_000 + or args.orm_concurrency > 256 or args.orm_cancel_work > 10_000_000 + or args.orm_sqlite_connections > 64 + or args.orm_concurrency > args.orm_iterations): + raise ValueError("configuration exceeds bounded fixture limits") + build(args) + else: + measure(args) + + +if __name__ == "__main__": + main() diff --git a/test/performance/settings.py b/test/performance/settings.py new file mode 100644 index 0000000..3d857b4 --- /dev/null +++ b/test/performance/settings.py @@ -0,0 +1,59 @@ +"""基准配置与隔离 peer 布置；配置只来自 config/setting.json。""" + +import errno +import json +import os +from pathlib import Path +import shutil +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +PEER_CONFIGURATION = "setting-json-v1" + + +def write_private_json(path, value, *, replace=False): + path = Path(path) + temporary = None + if replace: + descriptor, temporary = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}-") + else: + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + with os.fdopen(descriptor, "w") as output: + os.fchmod(output.fileno(), 0o600) + json.dump(value, output, indent=2) + output.write("\n") + if temporary is not None: + os.replace(temporary, path) + finally: + if temporary is not None: + Path(temporary).unlink(missing_ok=True) + + +def stage_peer(executable, directory, settings): + """独立配置共用不可变程序；测量期间不得重写输入二进制。""" + executable = Path(executable).resolve(strict=True) + destination = directory / executable.name + (directory / "config").mkdir() + write_private_json(directory / "config/setting.json", {"benchmark": settings}) + try: + os.link(executable, destination) + except OSError as error: + if error.errno != errno.EXDEV: + raise + # 配置目录可以位于另一文件系统；这种情况下硬链接不可用。 + with destination.open("xb") as output, executable.open("rb") as source: + shutil.copyfileobj(source, output) + shutil.copymode(executable, destination) + return destination.resolve() + + +def configured_peer(name, setting_path=ROOT / "config/setting.json"): + document = json.loads(setting_path.read_text()) + section = document.get("performance") if isinstance(document, dict) else None + value = section.get(name) if isinstance(section, dict) else None + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"config/setting.json must define performance.{name} as a binary path") + # 仓库设置中的相对路径以项目根目录为基准。 + path = Path(value) + return (setting_path.parent.parent / path).resolve(strict=True) diff --git a/test/performance/test_cms.py b/test/performance/test_cms.py new file mode 100644 index 0000000..bf6a182 --- /dev/null +++ b/test/performance/test_cms.py @@ -0,0 +1,255 @@ +import json +from pathlib import Path +import subprocess +import sys +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import Mock, patch + +from cms import (CmsClient, ReadLatency, concurrent_reads, decode_response, execute, isolated_settings, latency_summary, + publish_workload, run_postgres_case, run_postgres_rbac) +from settings import write_private_json + + +class CmsMeasurementTests(unittest.TestCase): + def test_long_read_histogram_is_bounded_and_reports_upper_bounds(self): + latency = ReadLatency() + for _ in range(10000): + latency.add(3) + other = ReadLatency() + other.add(100) + latency.merge(other) + self.assertEqual(len(latency.buckets), 2048) + self.assertEqual(latency.count, 10001) + self.assertGreaterEqual(latency.percentile(.99), 3) + self.assertLess(latency.percentile(.99), 3.05) + client = CmsClient(12345) + client.record_timings = False + client.connection = SimpleNamespace(request=Mock(), getresponse=Mock(return_value=SimpleNamespace( + status=200, read=Mock(return_value=b'{"code":0,"message":"ok","data":1}'))), close=Mock()) + try: + self.assertEqual(client.request("GET", "/test"), 1) + self.assertEqual(client.timings, {}) + finally: + client.close() + + def test_concurrent_read_failure_closes_clients_and_cannot_be_counted_as_success(self): + expected = {"total": 1, "items": [{"title": "published"}]} + for reply in ({"total": 0, "items": []}, RuntimeError("HTTP 403")): + with self.subTest(reply=reply): + opened = [] + + def client(_port): + request = Mock(side_effect=reply) if isinstance(reply, Exception) else Mock(return_value=reply) + instance = SimpleNamespace(cookies={}, timings={}, request=request, close=Mock()) + opened.append(instance) + return instance + + process = SimpleNamespace(samples=[], check=Mock()) + with patch("cms.CmsClient", side_effect=client), patch("cms.observe"): + with self.assertRaises(RuntimeError): + concurrent_reads(12345, {"front": "test-only-cookie"}, expected, process, + concurrency=2, seconds=.1, interval=.01) + self.assertEqual(len(opened), 2) + for instance in opened: + instance.close.assert_called_once() + + def test_delete_inputs_use_query_instead_of_json_body(self): + client = CmsClient(12345) + response = SimpleNamespace(status=200, read=Mock(return_value=b'{"code":0,"message":"ok","data":true}')) + connection = SimpleNamespace(request=Mock(), getresponse=Mock(return_value=response), close=Mock()) + client.connection = connection + try: + self.assertIs(client.request("DELETE", "/revoke", {"user_id": 3, "role_id": "fixture reader"}), True) + sent = connection.request.call_args.args + self.assertEqual(sent[:3], ("DELETE", "/revoke?user_id=3&role_id=fixture+reader", None)) + finally: + client.close() + + def test_postgres_rbac_checks_catalog_roles_and_same_session_revocation(self): + catalog = [ + {"component": "news", "domain": "article", "site": "admin", "action": "list", + "method": "GET", "key": "news.article.admin.list"}, + {"component": "user", "domain": "authorization", "site": "admin", "action": "permissions", + "method": "GET", "key": "user.authorization.admin.permissions"}, + {"component": "news", "domain": "article", "site": "front", "action": "list", + "method": "GET", "key": "news.article.front.list"}, + ] + identity = {"user_id": 1, "tenant_id": 2} + member = {"user_id": 3, "tenant_id": 2, "credential_id": 4} + login = [{"account": {"id": 3}, "tenant_id": 2, "site": site} + for site in ("admin", "front")] + for failure in (False, True): + with self.subTest(failure=failure): + owner = SimpleNamespace(request=Mock(side_effect=[ + [entry for entry in catalog if entry["site"] == "admin"], + [entry for entry in catalog if entry["site"] == "front"], + ] + [True] * 9)) + responses = login + ([RuntimeError("missing denial")] if failure else [None] * 11) + client = SimpleNamespace(request=Mock(side_effect=responses), close=Mock()) + with patch("cms.execute", return_value=member), patch("cms.CmsClient", return_value=client): + if failure: + with self.assertRaisesRegex(RuntimeError, "missing denial"): + run_postgres_rbac(Path("owned-app"), Path("owned-output"), 12345, owner, identity) + else: + result = run_postgres_rbac(Path("owned-app"), Path("owned-output"), 12345, owner, identity) + self.assertEqual(result["member_user_id"], 3) + self.assertEqual(set(result["permissions"]), {"admin_read", "admin_permissions", "front_list"}) + self.assertNotIn("password", result) + calls = client.request.call_args_list[2:] + self.assertEqual([call.kwargs.get("expected", 200) for call in calls], + [403, 403, 403, 200, 200, 200, 403, 200, 200, 403, 403]) + saves = [call for call in owner.request.call_args_list if call.args[1].endswith("/save")] + self.assertEqual(owner.request.call_args_list[1].kwargs["site"], "front") + self.assertEqual(saves[0].args[2]["id"], saves[2].args[2]["id"]) + self.assertNotEqual(saves[0].kwargs["site"], saves[2].kwargs["site"]) + client.close.assert_called_once() + + def test_postgres_case_keeps_source_configuration_on_build_failure(self): + with tempfile.TemporaryDirectory(prefix="dever-cms-source-") as temporary: + directory = Path(temporary) + source = directory / "project" + source.mkdir() + for name in ("config", "module", "test"): + (source / name).mkdir() + original = { + "runtime": {"mode": "all"}, + "database": {"default": { + "type": "postgres", "url": "postgres://fixture@127.0.0.1:12345/control", + "tenant_database_prefix": "dever_test_owned", + }}, + "auth": {"providers": {"session": {"jwtSecret": "original"}}}, + "sites": {}, "log": {"level": "info"}, + } + setting = source / "config/setting.json" + write_private_json(setting, original) + before = setting.read_bytes() + staged_paths = [] + + def fail_check(command, *_args, **_kwargs): + staged = command[2] + staged_paths.append(staged) + self.assertNotEqual(staged, source) + changed = json.loads((staged / "config/setting.json").read_text()) + self.assertEqual(changed["runtime"]["mode"], "api") + self.assertNotEqual(changed["auth"]["providers"]["session"]["jwtSecret"], "original") + raise RuntimeError("owned check failed") + + output = directory / "result" + with patch("cms.execute", side_effect=fail_check): + with self.assertRaisesRegex(RuntimeError, "owned check failed"): + run_postgres_case(source, output, directory / "compiler", articles=2) + self.assertEqual(setting.read_bytes(), before) + self.assertFalse((source / "cms-app").exists()) + self.assertTrue(staged_paths) + self.assertFalse(staged_paths[0].exists()) + self.assertEqual(json.loads((output / "report.json").read_text())["status"], "failed") + + def test_private_config_replacement_is_atomic_and_keeps_exclusive_creation(self): + with tempfile.TemporaryDirectory(prefix="dever-cms-setting-") as temporary: + directory = Path(temporary) + setting = directory / "setting.json" + write_private_json(setting, {"original": True}) + with self.assertRaises(FileExistsError): + write_private_json(setting, {"overwrite": True}) + with self.assertRaises(TypeError): + write_private_json(setting, {"invalid": object()}, replace=True) + self.assertEqual(json.loads(setting.read_text()), {"original": True}) + self.assertEqual(list(directory.iterdir()), [setting]) + write_private_json(setting, {"postgres": True}, replace=True) + self.assertEqual(json.loads(setting.read_text()), {"postgres": True}) + self.assertEqual(setting.stat().st_mode & 0o777, 0o600) + self.assertEqual(list(directory.iterdir()), [setting]) + + def test_envelope_failure_cannot_count_as_publication(self): + self.assertEqual(decode_response(b'{"code":0,"message":"ok","data":{"id":1}}', 200), + {"id": 1}) + for document, status in (({"code": 0, "message": "ok", "data": {}}, 500), + ({"code": 409, "message": "conflict", "data": None}, 200), + ({"id": 1}, 200)): + with self.assertRaises(RuntimeError): + decode_response(json.dumps(document).encode(), status) + + def test_isolation_preserves_site_contract_without_mutating_input(self): + source = { + "runtime": {"mode": "all"}, "http": {"port": 8080}, + "database": {"default": {"type": "sqlite", "path": "old.db"}}, + "auth": {"providers": {"session": {"jwtSecret": "example", "verify": "user.account.verify"}}}, + "sites": {"admin": {"path": "admin", "auth": "session", "origin": "https://example.invalid"}}, + "log": {"level": "info"}, + } + staged = isolated_settings(source, 12345) + self.assertEqual(source["runtime"]["mode"], "all") + self.assertEqual(source["database"]["default"]["path"], "old.db") + self.assertEqual(staged["runtime"]["mode"], "api") + self.assertEqual(staged["sites"]["admin"]["path"], "admin") + self.assertEqual(staged["sites"]["admin"]["origin"], "http://127.0.0.1:12345") + self.assertNotEqual(staged["auth"]["providers"]["session"]["jwtSecret"], "example") + self.assertEqual(staged["auth"]["providers"]["session"]["verify"], "user.account.verify") + self.assertEqual(source["log"]["level"], "info") + self.assertEqual(staged["log"]["level"], "error") + source["database"]["default"]["type"] = "postgres" + with self.assertRaises(RuntimeError): + isolated_settings(source, 12345) + + def test_postgres_isolation_preserves_explicit_control_database_and_prefix(self): + source = { + "runtime": {"mode": "all"}, + "database": {"default": { + "type": "postgres", "url": "postgres://fixture@127.0.0.1:12345/control", + "tenant_database_prefix": "dever_test_owned", "tls": "disabled", + }}, + "auth": {"providers": {"session": {"jwtSecret": "example"}}}, + "sites": {"admin": {"origin": "https://example.invalid"}}, + "log": {"level": "info"}, + } + staged = isolated_settings(source, 23456) + self.assertEqual(staged["database"], source["database"]) + self.assertEqual(staged["http"], {"host": "127.0.0.1", "port": 23456}) + self.assertNotIn("path", staged["database"]["default"]) + source["database"]["default"].pop("tenant_database_prefix") + with self.assertRaises(RuntimeError): + isolated_settings(source, 23456) + + def test_publish_workload_uses_tenant_and_title_parameters(self): + identity = {"credential_id": 11, "user_id": 3, "tenant_id": 2} + publication = {"slug": "bench-0", "title": "Tenant A 0 published", + "body": "Published body"} + client = SimpleNamespace(request=Mock(side_effect=[ + {"account": {"id": 3}, "tenant_id": 2, "site": "admin"}, + {"account": {"id": 3}, "tenant_id": 2, "site": "front"}, + {"id": 1, "slug": "bench-0", "title": "Tenant A 0", "version": 1}, + {"id": 7, "article_id": 1, "title": "Tenant A 0 published", "version": 2}, + publication, {"total": 1, "items": [publication]}, None, + ])) + process = SimpleNamespace(sample=Mock()) + contract = publish_workload(client, process, identity, "fixture-password", 1, + tenant_key="tenant_a", title_prefix="Tenant A") + self.assertEqual(contract, {"articles": 1, "titles": ["Tenant A 0 published"], + "duplicate_publish_status": 409}) + for call in client.request.call_args_list[:2]: + self.assertEqual(call.args[2]["tenant"], "tenant_a") + revision = client.request.call_args_list[3] + self.assertEqual(revision.args[:2], ("POST", "/content/revision/admin/manage/create")) + self.assertEqual(revision.args[2]["expected_version"], 1) + self.assertEqual(revision.args[2]["selected_article_id"], 1) + self.assertEqual(process.sample.call_count, 1) + + def test_small_workload_reports_samples_without_implying_p99(self): + self.assertEqual(latency_summary({"publish": [4, 1, 3]}), + {"publish": {"samples": 3, "p50_ms": 3, "max_ms": 4}}) + + def test_command_timeout_retains_output_and_reaps_its_owned_process(self): + with tempfile.TemporaryDirectory(prefix="dever-cms-timeout-") as temporary: + directory = Path(temporary) + command = [sys.executable, "-u", "-c", + "import time,sys; print('starting'); print('diagnostic',file=sys.stderr); time.sleep(10)"] + with self.assertRaises(subprocess.TimeoutExpired): + execute(command, directory, "timeout", timeout=0.2) + self.assertEqual((directory / "timeout.stdout.log").read_text(), "starting\n") + self.assertEqual((directory / "timeout.stderr.log").read_text(), "diagnostic\n") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/performance/test_live_bench.py b/test/performance/test_live_bench.py new file mode 100644 index 0000000..4905e39 --- /dev/null +++ b/test/performance/test_live_bench.py @@ -0,0 +1,122 @@ +"""用独立 Python 对端检查 Rust 长连接客户端的正文与生命周期判定。""" + +from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +from pathlib import Path +import socketserver +import subprocess +import tempfile +import threading +import time +import unittest + +from live import parse_progress, phase_sequence, validate_result +from settings import configured_peer, stage_peer + + +@contextmanager +def serving(server): + with server: + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield server.server_address[1] + finally: + server.shutdown() + thread.join(timeout=2) + if thread.is_alive(): + raise RuntimeError("owned test listener did not stop") + + +class Echo(socketserver.BaseRequestHandler): + def handle(self): + self.request.settimeout(2) + while payload := self.request.recv(4096): + response = b"!" * len(payload) if self.server.wrong_body else payload + self.request.sendall(response) + if self.server.wrong_body: + return + + +class InvalidEvent(BaseHTTPRequestHandler): + def log_message(self, *_): + pass + + def do_GET(self): + body = b"event: live\nid: 1\ndata: wrong\n\n" + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + try: + self.wfile.write(body) + except (BrokenPipeError, ConnectionResetError): + # The client aborts sibling sessions after the first deliberate mismatch. + return + + +class SilentEvent(BaseHTTPRequestHandler): + def log_message(self, *_): + pass + + def do_GET(self): + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.end_headers() + self.wfile.write(b"event: live\nid: 1\ndata: ready\n\n") + self.wfile.flush() + time.sleep(1) + + +class LiveClientTests(unittest.TestCase): + def setUp(self): + directory = tempfile.TemporaryDirectory(prefix="dever-live-client-") + self.addCleanup(directory.cleanup) + self.executable = stage_peer(configured_peer("live_peer"), Path(directory.name), {"workers": 1}) + + def execute(self, protocol, port, path): + scheme = "http" if protocol == "sse" else protocol + return subprocess.run([self.executable, protocol, f"{scheme}://127.0.0.1:{port}{path}", "2", "2", "300", "1000"], + capture_output=True, text=True, timeout=8) + + def test_tcp_reconnect_and_exact_echo(self): + listener = socketserver.ThreadingTCPServer(("127.0.0.1", 0), Echo) + listener.wrong_body = False + with serving(listener) as port: + result = self.execute("tcp", port, "/") + self.assertEqual(result.returncode, 0, result.stderr) + phases, counts = parse_progress(result.stdout, 2) + self.assertEqual(phases, phase_sequence(2)) + validate_result(counts, "tcp", 2, 2) + + def test_tcp_wrong_body_cannot_be_counted_as_held(self): + listener = socketserver.ThreadingTCPServer(("127.0.0.1", 0), Echo) + listener.wrong_body = True + with serving(listener) as port: + result = self.execute("tcp", port, "/") + self.assert_failure(result) + + def test_sse_wrong_event_cannot_pass_connection_readiness(self): + with serving(ThreadingHTTPServer(("127.0.0.1", 0), InvalidEvent)) as port: + result = self.execute("sse", port, "/events") + self.assert_failure(result) + + def test_sse_silent_hold_cannot_pass_heartbeat_validation(self): + with serving(ThreadingHTTPServer(("127.0.0.1", 0), SilentEvent)) as port: + result = self.execute("sse", port, "/events") + self.assertNotEqual(result.returncode, 0) + counts = json.loads(result.stdout.splitlines()[-1]) + self.assertGreater(counts["errors"], 0) + self.assertIn("PHASE|0|connected\n", result.stdout) + self.assertNotIn("PHASE|0|closing\n", result.stdout) + + def assert_failure(self, result): + self.assertNotEqual(result.returncode, 0) + counts = json.loads(result.stdout.splitlines()[-1]) + self.assertGreater(counts["errors"], 0) + self.assertNotIn("PHASE|0|connected\n", result.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/performance/test_network_bench.py b/test/performance/test_network_bench.py new file mode 100644 index 0000000..51fef79 --- /dev/null +++ b/test/performance/test_network_bench.py @@ -0,0 +1,120 @@ +"""用独立的 Python HTTP 对端验证负载计数；只占用自有随机回环端口。""" + +from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, HTTPServer +import json +from pathlib import Path +import subprocess +import tempfile +import threading +import unittest + +from process import Process +from run import validate_load +from settings import configured_peer, stage_peer + + +ROOT = Path(__file__).resolve().parents[2] +TLS = ROOT / "test/dever-tests/fixtures/tls" + + +@contextmanager +def peer(body): + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_GET(self): + self.send_response(200) + self.send_header("Content-Type", "text/plain") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *_): + pass + + server = HTTPServer(("127.0.0.1", 0), Handler) + worker = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.01}) + worker.start() + try: + yield server.server_port + finally: + server.shutdown() + server.server_close() + worker.join(timeout=2) + if worker.is_alive(): + raise RuntimeError("owned HTTP peer did not stop") + + +@contextmanager +def benchmark_service(executable, mode): + settings = {"workers": 1, "connections": 4, "task_capacity": 128, + "timeout_ms": 1000, "lifetime_ms": 5000, "http2_streams": 16, + "http2_stream_window_bytes": 65535, "http2_connection_window_bytes": 262144, + "cert_path": str(TLS / "server.pem"), "key_path": str(TLS / "server-key.pem")} + with tempfile.TemporaryDirectory(prefix="dever-h2-bench-") as directory: + with Process([executable, mode], Path(directory) / "server", peer_settings=settings) as process: + yield process.ready() + process.check() + + +class NetworkMeasurementTests(unittest.TestCase): + def setUp(self): + directory = tempfile.TemporaryDirectory(prefix="dever-network-client-") + self.addCleanup(directory.cleanup) + self.executable = stage_peer(configured_peer("network_peer"), Path(directory.name), {"workers": 1}) + + def load(self, port): + executable = self.executable + output = subprocess.run([str(executable), "load", "h1", f"http://localhost:{port}", + "/plain", "20", "0.2", "1", "1", "500", "true", "-"], + capture_output=True, text=True, timeout=5, check=True) + self.assertEqual(output.stderr, "") + result = json.loads(output.stdout) + validate_load(result) + self.assertEqual(result["scheduled"], 4) + self.assertGreaterEqual(result["elapsed_seconds"], 0.2) + return result + + def test_success_requires_exact_body_and_full_window(self): + with peer(b"hello") as port: + result = self.load(port) + self.assertEqual(result["succeeded"], 4) + self.assertEqual(result["errors"], 0) + self.assertEqual(result["dropped"], 0) + self.assertEqual(result["latency_samples"], 4) + self.assertEqual(result["http_version"], "h1") + self.assertEqual(result["physical_connections"], 1) + self.assertEqual(result["streams_per_connection"], 1) + self.assertEqual(result["request_slots"], 1) + + def test_wrong_body_is_an_error_with_no_success_latency(self): + with peer(b"wrong") as port: + result = self.load(port) + self.assertEqual(result["succeeded"], 0) + self.assertEqual(result["errors"], 4) + self.assertEqual(result["latency_samples"], 0) + self.assertIsNone(result["latency_ms"]) + + def test_http2_uses_one_physical_connection_for_sixteen_request_slots(self): + executable = self.executable + for mode, transport in (("runtime-http2", "http"), ("hyper-http2", "http"), + ("runtime-https2", "https"), ("hyper-https2", "https")): + with self.subTest(mode=mode), benchmark_service(executable, mode) as port: + output = subprocess.run( + [str(executable), "load", "h2", f"{transport}://localhost:{port}", + "/bytes", "200", "0.1", "1", "16", "1000", "true", str(TLS / "root.pem")], + capture_output=True, text=True, timeout=5, check=True) + self.assertEqual(output.stderr, "") + result = json.loads(output.stdout) + validate_load(result) + self.assertEqual(result["errors"], 0) + self.assertGreater(result["succeeded"], 0) + self.assertEqual(result["physical_connections"], 1) + self.assertEqual(result["streams_per_connection"], 16) + self.assertEqual(result["request_slots"], 16) + self.assertEqual(result["successful_handshakes"], 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/performance/test_peer_settings.py b/test/performance/test_peer_settings.py new file mode 100644 index 0000000..33150a7 --- /dev/null +++ b/test/performance/test_peer_settings.py @@ -0,0 +1,47 @@ +"""只执行 peer 的配置检查入口，不创建网络或连接数据库。""" + +import json +from pathlib import Path +import subprocess +import tempfile +import unittest + +from settings import configured_peer, stage_peer + + +class PeerSettingsTests(unittest.TestCase): + def check(self, peer, document): + with tempfile.TemporaryDirectory(prefix="dever-peer-config-test-") as temporary: + root = Path(temporary) + executable = stage_peer(configured_peer(peer), root, {}) + setting = root / "config/setting.json" + if document is None: + setting.unlink() + else: + setting.write_text(json.dumps(document)) + # 不在 executable 目录启动，验证读取位置不受工作目录影响。 + return subprocess.run([executable, "check-config"], cwd=root.parent, + capture_output=True, text=True, timeout=5) + + def test_both_peers_use_the_external_integer_setting(self): + for peer in ("network_peer", "live_peer"): + for workers in (1, 4, 64): + with self.subTest(peer=peer, workers=workers): + output = self.check(peer, {"benchmark": {"workers": workers}}) + self.assertEqual(output.returncode, 0, output.stderr) + self.assertEqual(json.loads(output.stdout), {"workers": workers}) + + def test_both_peers_reject_missing_malformed_and_out_of_range_configuration(self): + invalid = [None, {}, [], {"benchmark": []}, {"benchmark": {"unknown": 1}}] + invalid.extend({"benchmark": {"workers": value}} for value in (0, 65, -1, 1.5, True, "4", None)) + for peer in ("network_peer", "live_peer"): + for document in invalid: + with self.subTest(peer=peer, document=document): + output = self.check(peer, document) + self.assertNotEqual(output.returncode, 0) + self.assertEqual(output.stdout, "") + self.assertTrue(output.stderr.strip()) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/performance/test_runner.py b/test/performance/test_runner.py new file mode 100644 index 0000000..f64b27e --- /dev/null +++ b/test/performance/test_runner.py @@ -0,0 +1,803 @@ +"""基准运行器的最小正确性检查，不启动网络压测或编译器。""" + +import argparse +import errno +import json +import os +from pathlib import Path +from types import SimpleNamespace +import sys +import tempfile +import unittest +from unittest.mock import MagicMock, Mock, patch + +import cms +from process import Budget, Process, observe, summarize +from build import (APPLICATION_CONFIGURATION, BENCHMARK_COMMAND, ENTRIES, configuration, dependency_packages, fingerprint, + load_postgres_setting, postgres_case_url, postgres_database_case, + native_runtime_setting, prepare_cms_project, prepare_native_project, + prepare_profile_project, prepare_project, + profile_database_setting, stage_application_command) +from run import (HTTP_BODY_BYTES, application_command, fixture_command, async_measurements, csv_positive_ints, http2_summary, + cms_cases, cms_summary, expected_orm_checksum, orm_measurements, orm_summary, + orm_count, parse_client_cpus, parse_implementations, parser, percentile, + positive, report_arguments, required_binaries, require_no_oom, require_no_request_errors, + require_successful_requests, server, stage_orm_bundle, transaction_cancel_measurement, + validate_cpu_placement, validate_load) +from run import stage_live_bundle, verify_artifacts +from live import lifecycle_seconds, parse_progress, phase_sequence, stable_phase, validate_result +from settings import PEER_CONFIGURATION, configured_peer, stage_peer + + +class MeasurementTests(unittest.TestCase): + def test_application_command_matches_generated_declaration(self): + self.assertEqual(application_command(Path("app")), + [Path("app"), BENCHMARK_COMMAND, "{}"]) + with tempfile.TemporaryDirectory(prefix="dever-perf-command-") as temporary: + module = Path(temporary) / "module" + stage_application_command(module, "benchmark.item.orm_list") + self.assertEqual((module / "benchmark/command/api.dever").read_text(), + "cmd run = app.run\n") + self.assertIn("benchmark.item.orm_list()", + (module / "benchmark/command/app.dever").read_text()) + self.assertFalse((module / "main.dever").exists()) + + def test_generated_projects_use_stock_cms_and_cmd_for_orm_profiles(self): + settings = configuration(parser().parse_args(["build", "--output", "unused"])) + with tempfile.TemporaryDirectory(prefix="dever-perf-project-") as temporary: + root = Path(temporary) + for source in ("dever", "md"): + project = root / f"cms-{source}" + prepare_cms_project(project, source) + self.assertFalse((project / "module/main.dever").exists()) + self.assertFalse((project / "module/benchmark").exists()) + self.assertEqual((project / "config/setting.json").read_bytes(), + (Path(__file__).parents[2] / "examples/cms" / source + / "config/setting.json").read_bytes()) + self.assertTrue((project / "test/news/article").is_dir()) + orm = root / "orm" + prepare_project(orm, "orm", settings, "sqlite") + self.assertFalse((orm / "module/main.dever").exists()) + self.assertTrue((orm / "module/benchmark/command/api.dever").is_file()) + self.assertTrue((orm / "module/benchmark/item/api.dever").is_file()) + for profile in ("base", "sqlite", "postgres", "both"): + project = root / f"profile-{profile}" + prepare_profile_project(project, profile_database_setting(profile)) + self.assertFalse((project / "module/main.dever").exists()) + self.assertTrue((project / "module/benchmark/command/api.dever").is_file()) + + def test_profile_free_generation_copies_roles_and_only_used_helpers(self): + settings = configuration(parser().parse_args(["build", "--output", "unused"])) + expected_config = { + "runtime": ("lifetime", "idle_sample_ms", "pending", "channel_capacity", + "input", "pending_input"), + "http": ("lifetime", "limits", "http2_limits", "large_body", + "certificate", "key"), + "live": (), + } + with tempfile.TemporaryDirectory(prefix="dever-perf-sources-") as temporary: + for fixture, functions in expected_config.items(): + destination = Path(temporary) / fixture + prepare_native_project(destination, fixture, settings) + domain = destination / "module/benchmark" / fixture + self.assertTrue(all((domain / f"{role}.dever").is_file() + for role in ("app", "port", "adapter", "api"))) + config = destination / "module/benchmark/config/app.dever" + self.assertEqual(config.is_file(), bool(functions)) + if functions: + source = config.read_text() + self.assertTrue(all(f"\n{name}()" in source for name in functions)) + self.assertNotIn("orm_input()", source) + self.assertEqual((destination / "module/benchmark/bench/app.dever").is_file(), + fixture == "runtime") + setting = json.loads((destination / "config/setting.json").read_text()) + self.assertEqual(setting, native_runtime_setting(settings) if fixture == "live" else {}) + declarations = (domain / "api.dever").read_text() + for entry in ENTRIES[fixture]: + self.assertIn(f"cmd {entry} = app.{entry}", declarations) + self.assertEqual(fixture_command(Path("app"), entry), + [Path("app"), f"benchmark.{fixture}.{entry}", "{}"]) + live = native_runtime_setting(settings)["adapter"]["benchmark.live"]["setting"] + self.assertEqual(live["lifetime_ms"], settings["lifetime_ms"]) + self.assertEqual(live["limits"]["timeout_ms"], settings["timeout_ms"]) + self.assertNotIn("timeout_ms", live) + + def test_live_adapter_setting_is_private_to_each_executable(self): + settings = {"lifetime_ms": 4000, "timeout_ms": 2000, "connections": 8} + with tempfile.TemporaryDirectory(prefix="dever-live-bundle-") as temporary: + root = Path(temporary) + artifacts = root / "artifacts" + artifacts.mkdir() + for protocol in ("tcp", "ws"): + (artifacts / protocol).write_bytes(protocol.encode()) + output = root / "output" + output.mkdir() + args = SimpleNamespace(artifacts=artifacts, output=output) + for protocol in ("tcp", "ws"): + executable = stage_live_bundle(args, settings, f"live-{protocol}-0", protocol) + self.assertEqual(executable.read_bytes(), protocol.encode()) + setting = executable.parent / "config/setting.json" + self.assertEqual(json.loads(setting.read_text()), native_runtime_setting(settings)) + self.assertEqual(setting.stat().st_mode & 0o777, 0o600) + self.assertFalse((artifacts / "config").exists()) + + def test_cms_cases_use_stock_config_and_publish_result(self): + with tempfile.TemporaryDirectory(prefix="dever-perf-cms-case-") as temporary: + root = Path(temporary) + artifacts = root / "artifacts" + output = root / "output" + artifacts.mkdir() + output.mkdir() + for source, name in (("dever", "cms"), ("md", "cms-md")): + (artifacts / name).write_bytes(source.encode()) + config = artifacts / "source" / f"cms-{source}" / "config" + config.mkdir(parents=True) + (config / "setting.json").write_text("{}") + args = SimpleNamespace(artifacts=artifacts, output=output, repeats=1, + _cms_sources=("dever", "md"), cgroup_parent=None, + memory_mib=None, idle_seconds=0.1, interval=0.01, + server_cpu=None, cpu_quota=None) + result = {"contract": {"articles": 16, "titles": [f"Article {i}" for i in range(16)], + "duplicate_publish_status": 409}, + "ready_seconds": 0.2, "workload_seconds": 0.5, + "published_articles_per_second": 32, + "latency": {}, "resources": {"rss_bytes_median": 1, + "pss_bytes_median": 1, + "rss_high_water_bytes": 2}, + "raw": [], "budget_before_stop": None, "cgroup": None} + with patch("cms.run_case", return_value=result) as run_case: + cases = cms_cases(args, {}, {}) + self.assertEqual(len(cases), 2) + self.assertEqual(run_case.call_count, 2) + for call, source in zip(run_case.call_args_list, ("dever", "md")): + identity = call.args[0] + self.assertEqual(identity["config"], str( + artifacts / "source" / f"cms-{source}" / "config/setting.json")) + self.assertEqual(identity["fingerprint"], fingerprint(Path(identity["executable"]))) + self.assertEqual(call.kwargs["articles"], 16) + self.assertEqual(cms_summary(cases)["dever"]["articles"], 16) + self.assertTrue(all(case["workload_mode"] == "sequential_publish" for case in cases)) + self.assertTrue(all("ready_value" not in case for case in cases)) + + def test_focused_cms_build_selection_keeps_the_full_build_as_default(self): + self.assertEqual(parser().parse_args(["build", "--output", "unused"]).selection, "all") + self.assertEqual(parser().parse_args( + ["build", "--output", "unused", "--selection", "cms-profiles"] + ).selection, "cms-profiles") + self.assertEqual(parser().parse_args( + ["build", "--output", "unused", "--selection", "profiles"] + ).selection, "profiles") + + def test_cms_descriptor_recovery_bounds_pool_and_preserves_other_resources(self): + before = {"0": "/dev/null", "3": "socket:[listener]", "4": "/fixture/platform.db"} + after = {**before, "5": "/fixture/platform.db", "6": "/fixture/platform.db-wal"} + load = {"descriptors_before": before, "descriptors_after": after, + "pooled_descriptor_limits": {"/fixture/platform.db": 4, + "/fixture/platform.db-wal": 4, + "/fixture/platform.db-shm": 1}} + cms.validate_read_descriptor_recovery(load) + invalid = [ + {**after, "9": "socket:[request]"}, + {**after, "9": "socket:[listener]"}, + {**after, "3": "socket:[replacement]"}, + {**after, "0": "/unexpected/file"}, + {**after, **{str(index): "/fixture/platform.db" for index in range(10, 15)}}, + {**after, **{str(index): "/fixture/platform.db-wal" for index in range(10, 15)}}, + {**after, "10": "/fixture/platform.db-shm", "11": "/fixture/platform.db-shm"}, + ] + for descriptors in invalid: + with self.subTest(descriptors=descriptors), self.assertRaises(RuntimeError): + cms.validate_read_descriptor_recovery({**load, "descriptors_after": descriptors}) + + def test_cms_failed_recovery_keeps_case_and_aggregate_evidence(self): + with tempfile.TemporaryDirectory(prefix="dever-cms-failed-report-") as temporary: + root = Path(temporary) + executable = root / "app" + executable.write_bytes(b"fixture") + config = root / "setting.json" + config.write_text("{}") + identity = {"executable": str(executable), "fingerprint": fingerprint(executable), + "config": str(config)} + manifest = {"format": "dever-cms-publish-v1", "cases": {"dever": identity, "md": identity}} + client = Mock(cookies={}, timings={}) + client.request.return_value = [] + snapshots = iter(({"3": "socket:[listener]"}, + {"3": "socket:[listener]", "4": "socket:[leaked]"})) + + def descriptors(): + self.assertTrue(client.close.called, "control client must close before the baseline") + return next(snapshots) + + process = Mock(samples=[{"fds": 1}], descriptors=descriptors) + context = MagicMock() + context.__enter__.return_value = process + context.__exit__.return_value = False + budget = Mock(final={"events": {"oom": 0, "oom_kill": 0}}) + settings = {"database": {"default": {"type": "sqlite", "path": "data/platform.db", + "tenant_directory": "data/tenants", "max_connections": 4}}} + load = {"elapsed_seconds": 75, "succeeded": 100, "errors": 0} + with (patch("cms.isolated_settings", return_value=settings), + patch("cms.provision_identity", return_value=("actor", "password")), + patch("cms.CmsClient", return_value=client), patch("cms.Budget", return_value=budget), + patch("cms.Process", return_value=context), patch("cms.wait_ready", return_value=0.1), + patch("cms.publish_workload", return_value={"articles": 1}), patch("cms.observe"), + patch("cms.concurrent_reads", return_value=load) as reads): + with self.assertRaisesRegex(RuntimeError, "read descriptors did not recover"): + cms.run_cases(manifest, root / "output", articles=1, cgroup_parent=None, + memory_mib=None, concurrency=8, read_seconds=75) + snapshots = iter(({"3": "socket:[listener]"},)) + reads.side_effect = RuntimeError("read fixture failed") + with self.assertRaisesRegex(RuntimeError, "read fixture failed"): + cms.run_cases(manifest, root / "read-failure", articles=1, cgroup_parent=None, + memory_mib=None, concurrency=8, read_seconds=75) + case = json.loads((root / "output/dever/report.json").read_text()) + report = json.loads((root / "output/report.json").read_text()) + self.assertEqual(case["status"], "failed") + self.assertEqual(case["raw"], [{"fds": 1}]) + self.assertEqual(case["concurrent_reads"]["succeeded"], 100) + self.assertEqual(case["concurrent_reads"]["descriptors_after"]["4"], "socket:[leaked]") + self.assertEqual(case["cgroup"], budget.final) + self.assertEqual(report["cases"], {"dever": case}) + self.assertEqual(report["status"], "failed") + early = json.loads((root / "read-failure/report.json").read_text())["cases"]["dever"] + self.assertEqual(early["contract"], {"articles": 1}) + self.assertEqual(early["concurrent_reads"]["descriptors_before"], {"3": "socket:[listener]"}) + self.assertNotIn("succeeded", early["concurrent_reads"]) + self.assertEqual(context.__exit__.call_count, 2) + self.assertEqual(budget.close.call_count, 2) + + def test_cms_requires_every_declared_source_binary(self): + args = SimpleNamespace(suite="cms", _postgres_setting=None, + _cms_sources=("dever", "md")) + self.assertEqual(required_binaries(args), {"cms", "cms-md"}) + args._cms_sources = ("dever",) + self.assertEqual(required_binaries(args), {"cms"}) + + def test_peer_staging_preserves_binary_and_isolates_settings(self): + with tempfile.TemporaryDirectory(prefix="dever-peer-setting-") as temporary: + root = Path(temporary) + source = root / "peer" + source.write_bytes(b"owned executable") + source.chmod(0o700) + for workers in (1, 4): + directory = root / str(workers) + directory.mkdir() + executable = stage_peer(source, directory, {"workers": workers}) + self.assertEqual(executable.read_bytes(), source.read_bytes()) + self.assertTrue(executable.samefile(source)) + self.assertEqual(executable.stat().st_mode & 0o777, 0o700) + setting = directory / "config/setting.json" + self.assertEqual(json.loads(setting.read_text()), {"benchmark": {"workers": workers}}) + self.assertEqual(setting.stat().st_mode & 0o777, 0o600) + with self.assertRaises(FileExistsError): + stage_peer(source, directory, {"workers": 8}) + self.assertFalse((root / "config").exists()) + + def test_peer_staging_copies_only_across_filesystems(self): + with tempfile.TemporaryDirectory(prefix="dever-peer-filesystem-") as temporary: + root = Path(temporary) + source = root / "peer" + source.write_bytes(b"owned executable") + source.chmod(0o700) + directory = root / "cross-filesystem" + directory.mkdir() + with patch("settings.os.link", side_effect=OSError(errno.EXDEV, "different device")): + executable = stage_peer(source, directory, {}) + self.assertEqual(executable.read_bytes(), source.read_bytes()) + self.assertEqual(executable.stat().st_mode & 0o777, 0o700) + self.assertFalse(executable.samefile(source)) + denied = root / "denied" + denied.mkdir() + with patch("settings.os.link", side_effect=PermissionError(errno.EACCES, "denied")): + with self.assertRaises(PermissionError): + stage_peer(source, denied, {}) + self.assertFalse((denied / "peer").exists()) + + def test_peer_paths_come_from_project_setting(self): + with tempfile.TemporaryDirectory(prefix="dever-peer-path-") as temporary: + root = Path(temporary) + (root / "config").mkdir() + (root / "peer").write_bytes(b"binary") + setting = root / "config/setting.json" + setting.write_text(json.dumps({"performance": {"network_peer": "peer"}})) + self.assertEqual(configured_peer("network_peer", setting), root / "peer") + for value in ({}, {"performance": []}, {"performance": {"network_peer": 123}}): + setting.write_text(json.dumps(value)) + with self.assertRaisesRegex(ValueError, "performance.network_peer"): + configured_peer("network_peer", setting) + + def test_process_stages_peer_before_starting_child(self): + with tempfile.TemporaryDirectory(prefix="dever-peer-process-") as temporary: + root = Path(temporary) + command = [sys.executable, "-c", ( + "import json, pathlib, sys; " + "print((pathlib.Path(sys.executable).parent / 'config/setting.json').read_text())" + )] + with Process(command, root / "child", peer_settings={"workers": 3}) as process: + self.assertEqual(process.child.wait(timeout=5), 0) + process.check() + self.assertEqual(json.loads(process.text()), {"benchmark": {"workers": 3}}) + + @staticmethod + def postgres_setting(): + return { + "database": { + "postgres_test": { + "type": "postgres", + "url": "postgres://user:secret@localhost/dever_{case}", + "tls": "disabled", + "max_connections": 4, + } + } + } + + def test_orm_samples_are_bounded_and_report_percentiles(self): + result = orm_measurements( + "SAMPLE|crud|1|1000000|2\nSAMPLE|crud|1|3000000|2\n", + "crud", 2, 2, 0.5, + ) + self.assertEqual(result["operations"], 2) + self.assertEqual(result["throughput_operations_per_second"], 4) + self.assertEqual(result["latency_ms"], {"p50": 1, "p95": 3, "p99": 3}) + with self.assertRaises(ValueError): + orm_measurements("SAMPLE|crud|1|1|2\nSAMPLE|crud|1|1|2\n", "crud", 1, 2, 1) + with self.assertRaises(ValueError): + orm_measurements("SAMPLE|list|1|1|2\n", "crud", 1, 2, 1) + with self.assertRaises(ValueError): + orm_measurements("SAMPLE|crud|1|1|1\n", "crud", 1, 2, 1) + + def test_orm_checksums_cover_empty_cancel_and_bounded_pages(self): + settings = {"orm_rows": 20} + self.assertEqual(expected_orm_checksum("crud", settings), 2) + self.assertEqual(expected_orm_checksum("list", settings), 40) + self.assertEqual(expected_orm_checksum("cursor", settings), 20) + self.assertEqual(expected_orm_checksum("stream", settings), 20) + self.assertEqual(expected_orm_checksum("pool", settings), 20) + result = transaction_cancel_measurement(0, 0.01) + self.assertEqual(result["raw"][0]["checksum"], 0) + with self.assertRaises(RuntimeError): + transaction_cancel_measurement(1, 0.01) + + def test_postgres_cases_require_explicit_isolation_template(self): + setting = self.postgres_setting() + self.assertEqual( + postgres_case_url(setting, "orm_postgres_crud_0"), + "postgres://user:secret@localhost/dever_orm_postgres_crud_0", + ) + for value in ("postgres://localhost/shared", "sqlite:///{case}", + "postgres://localhost/shared?application_name={case}"): + setting["database"]["postgres_test"]["url"] = value + with self.assertRaises(ValueError): + postgres_case_url(setting, "case") + + def test_postgres_setting_is_loaded_from_the_project_config(self): + with tempfile.TemporaryDirectory(prefix="dever-setting-") as directory: + path = Path(directory) / "setting.json" + path.write_text(json.dumps(self.postgres_setting())) + self.assertEqual(load_postgres_setting(path), self.postgres_setting()) + path.write_text(json.dumps({ + "database": { + "default": { + "type": "sqlite", + "path": "data/db/dever.db", + "max_connections": 1, + "max_page_size": 100, + } + } + })) + self.assertIsNone(load_postgres_setting(path)) + path.write_text(json.dumps({ + "database": {"postgres_test": {"type": "sqlite"}} + })) + with self.assertRaises(ValueError): + load_postgres_setting(path) + self.assertIsNone(load_postgres_setting(path.with_name("missing.json"))) + + def test_profile_settings_cover_each_database_capability(self): + base = profile_database_setting("base") + sqlite = profile_database_setting("sqlite") + postgres = profile_database_setting("postgres") + both = profile_database_setting("both") + self.assertEqual(base, {}) + self.assertEqual(tuple(sqlite["database"]), ("default",)) + self.assertEqual(sqlite["database"]["default"]["type"], "sqlite") + self.assertEqual(postgres["database"]["default"]["type"], "postgres") + self.assertEqual(postgres["database"]["default"]["url"], + "postgres://build-only.invalid/dever_profile") + self.assertEqual(tuple(both["database"]), ("default", "postgres_profile")) + self.assertEqual(both["database"]["postgres_profile"]["type"], "postgres") + self.assertNotIn("postgres_test", both["database"]) + + def test_postgres_cancel_fixture_binds_int_as_bigint(self): + fixture = (Path(__file__).parent / "fixtures/orm/app/model/item.dever").read_text() + + self.assertIn("generate_series(1::bigint, $1::bigint)", fixture) + + def test_dependency_report_is_unique_and_stable(self): + self.assertEqual( + dependency_packages("z v1\na v1\nz v1 (*)\n"), + ["a v1", "z v1"], + ) + + def test_report_omits_private_postgres_configuration(self): + arguments = report_arguments(SimpleNamespace( + _postgres_setting=self.postgres_setting(), + artifacts=Path("artifacts"), + client_workers=4, + client_cpus=(1, 2, 3, 4), + )) + self.assertNotIn("_postgres_setting", arguments) + self.assertEqual(arguments["artifacts"], "artifacts") + self.assertEqual(json.loads(json.dumps(arguments))["client_cpus"], [1, 2, 3, 4]) + self.assertEqual(arguments["client_workers"], 4) + + def test_orm_bundle_uses_a_fresh_external_setting_per_case(self): + settings = {"orm_sqlite_connections": 1, "orm_postgres_connections": 4, + "orm_rows": 128} + with tempfile.TemporaryDirectory(prefix="dever-orm-bundle-") as directory: + root = Path(directory) + artifacts = root / "artifacts" + artifacts.mkdir() + (artifacts / "sqlite-crud").write_bytes(b"binary") + output = root / "output" + output.mkdir() + args = SimpleNamespace(output=output, artifacts=artifacts, _postgres_setting=None) + first = stage_orm_bundle(args, settings, "sqlite", "crud", 0) + second = stage_orm_bundle(args, settings, "sqlite", "crud", 1) + self.assertNotEqual(first[1].parent, second[1].parent) + first_setting = json.loads((first[1].parent / "config/setting.json").read_text()) + second_setting = json.loads((second[1].parent / "config/setting.json").read_text()) + self.assertEqual(first_setting["database"], second_setting["database"]) + self.assertNotEqual(first_setting["auth"], second_setting["auth"]) + self.assertEqual(first_setting["database"]["default"]["path"], "data/db/orm.db") + self.assertEqual((first[1].parent / "config/setting.json").stat().st_mode & 0o777, + 0o600) + + def test_orm_http_bundles_use_isolated_ports_and_private_auth(self): + settings = {"orm_sqlite_connections": 1, "orm_rows": 17} + with tempfile.TemporaryDirectory(prefix="dever-orm-http-bundle-") as directory: + root = Path(directory) + artifacts = root / "artifacts" + artifacts.mkdir() + (artifacts / "sqlite-http").write_bytes(b"binary") + output = root / "output" + output.mkdir() + args = SimpleNamespace(output=output, artifacts=artifacts, _postgres_setting=None) + bundles = [stage_orm_bundle(args, settings, "sqlite", entry, 0, + binary_entry="http" if entry == "cancel" else None) + for entry in ("http", "cancel")] + documents = [json.loads((bundle[1].parent / "config/setting.json").read_text()) + for bundle in bundles] + self.assertTrue(all(0 < document["http"]["port"] < 65536 + for document in documents)) + self.assertTrue(all(document["http"]["host"] == "127.0.0.1" + for document in documents)) + self.assertNotEqual(documents[0]["auth"], documents[1]["auth"]) + self.assertTrue(all(document["sites"]["benchmark"]["path"] == "" + for document in documents)) + self.assertTrue(all((bundle[1].parent / "config/setting.json").stat().st_mode & 0o777 + == 0o600 for bundle in bundles)) + + def test_orm_count_requires_the_current_api_envelope(self): + with patch("run.http.client.HTTPConnection") as connection: + response = connection.return_value.getresponse.return_value + response.status = 200 + response.getheader.return_value = "application/json; charset=utf-8" + response.read.return_value = b'{"code":0,"message":"ok","data":17}' + self.assertEqual(orm_count(12345, 0.2), 17) + connection.return_value.request.assert_called_with( + "GET", "/benchmark/item/count", headers={"Connection": "close"} + ) + for invalid in (b'{"ok":true}', b'{"code":0,"message":"ok","data":true}', + b'{"code":0,"message":"ok","data":-1}'): + response.read.return_value = invalid + with self.assertRaises(ValueError): + orm_count(12345, 0.2) + + def test_postgres_bundle_uses_the_named_project_setting(self): + settings = {"orm_sqlite_connections": 1, "orm_postgres_connections": 4, + "orm_rows": 128} + with tempfile.TemporaryDirectory(prefix="dever-postgres-bundle-") as directory: + root = Path(directory) + artifacts = root / "artifacts" + artifacts.mkdir() + (artifacts / "postgres-crud").write_bytes(b"binary") + output = root / "output" + output.mkdir() + args = SimpleNamespace( + output=output, + artifacts=artifacts, + _postgres_setting=self.postgres_setting(), + ) + _, executable, _, _ = stage_orm_bundle(args, settings, "postgres", "crud", 2) + setting_path = executable.parent / "config/setting.json" + setting = json.loads(setting_path.read_text()) + self.assertEqual(tuple(setting["database"]), ("default",)) + case = postgres_database_case(output, "orm-postgres-crud-2") + self.assertEqual( + setting["database"]["default"]["url"], + f"postgres://user:secret@localhost/dever_{case}", + ) + self.assertEqual(setting_path.stat().st_mode & 0o777, 0o600) + + def test_postgres_database_names_isolate_separate_reports(self): + first = postgres_database_case(Path("first"), "orm-postgres-crud-0") + second = postgres_database_case(Path("second"), "orm-postgres-crud-0") + self.assertNotEqual(first, second) + self.assertTrue(first.endswith("_orm_postgres_crud_0")) + + def test_artifact_verification_rejects_missing_and_changed_files(self): + with tempfile.TemporaryDirectory(prefix="dever-artifacts-") as directory: + root = Path(directory) + binary = root / "app" + certificate = root / "tls/root.pem" + certificate.parent.mkdir() + binary.write_bytes(b"binary") + certificate.write_bytes(b"certificate") + manifest = { + "peer_configuration": PEER_CONFIGURATION, + "application_configuration": APPLICATION_CONFIGURATION, + "binaries": {"app": fingerprint(binary)}, + "files": {"tls/root.pem": fingerprint(certificate)}, + } + verify_artifacts(root, manifest, {"app"}) + with self.assertRaisesRegex(ValueError, "obsolete configuration contract"): + verify_artifacts(root, {**manifest, "peer_configuration": "old"}, {"app"}) + with self.assertRaisesRegex(ValueError, "obsolete application contract"): + verify_artifacts(root, {**manifest, "application_configuration": "rust-native"}, {"app"}) + with self.assertRaises(ValueError): + verify_artifacts(root, manifest, {"app", "missing"}) + certificate.write_bytes(b"changed") + with self.assertRaises(ValueError): + verify_artifacts(root, manifest, {"app"}) + + def test_server_cleanup_does_not_mask_primary_failure(self): + args = SimpleNamespace(cgroup_parent=None, memory_mib=None, cpu_quota=None, + server_cpu=None) + with tempfile.TemporaryDirectory(prefix="dever-server-cleanup-") as directory: + with patch("run.Budget") as budget_type, patch("run.Process"): + budget_type.return_value.close.side_effect = OSError("cleanup") + with self.assertRaisesRegex(ValueError, "primary") as caught: + with server(args, ["unused"], Path(directory) / "case", {}): + raise ValueError("primary") + self.assertTrue(any("cannot close benchmark budget" in note + for note in caught.exception.__notes__)) + + def test_orm_summary_keeps_driver_and_operation_separate(self): + def case(driver, entry, latency): + return {"suite": "orm", "driver": driver, "entry": entry, + "binary": {"bytes": 10}, + "resources": {"rss_bytes_median": 20, "pss_bytes_median": 15, + "rss_high_water_bytes": 25}, + "measurement": {"throughput_operations_per_second": 100, + "latency_ms": {"p50": latency, "p95": latency + 1, + "p99": latency + 2}}} + + summary = orm_summary([case("sqlite", "crud", 1), case("postgres", "crud", 2)]) + self.assertEqual([(value["driver"], value["entry"]) for value in summary], + [("postgres", "crud"), ("sqlite", "crud")]) + self.assertEqual(summary[1]["latency_p99_ms_median"], 3) + + def test_cms_summary_reports_business_baseline(self): + cases = [{ + "suite": "cms", + "source": "dever", + "binary": {"bytes": 30}, + "ready_seconds": 0.5, + "contract": {"articles": 16}, + "workload_seconds": 0.25, + "published_articles_per_second": 64, + "resources": {"rss_bytes_median": 20, "pss_bytes_median": 15, + "rss_high_water_bytes": 25}, + "cgroup": {"peak_bytes": 40}, + }] + self.assertEqual( + cms_summary(cases), + {"dever": {"repeats": 1, "workload_mode": "sequential_publish", + "binary_bytes": 30, "articles": 16, + "ready_seconds_median": 0.5, "workload_seconds_median": 0.25, + "published_articles_per_second_median": 64, + "rss_bytes_median": 20, + "pss_bytes_median": 15, "rss_high_water_bytes_median": 25, + "cgroup_peak_bytes_median": 40}}, + ) + + def test_cms_summary_separates_source_variants(self): + cases = [ + {"suite": "cms", "source": source, "binary": {"bytes": size}, + "ready_seconds": 0.5, "contract": {"articles": 16}, + "workload_seconds": 0.25, "published_articles_per_second": 64, + "resources": {"rss_bytes_median": 20, "pss_bytes_median": 15, + "rss_high_water_bytes": 25}, "cgroup": None} + for source, size in (("dever", 30), ("md", 31)) + ] + summary = cms_summary(cases) + self.assertEqual(summary["dever"]["binary_bytes"], 30) + self.assertEqual(summary["md"]["binary_bytes"], 31) + + def test_live_lifecycle_includes_all_three_phase_deadlines(self): + self.assertAlmostEqual(lifecycle_seconds(2, 0.3, 2000), 23.2) + + def test_live_phase_order_and_partial_writes(self): + text = "PHASE|0|connecting\nPHASE|0|connected\nPHASE|0|clos" + self.assertEqual(parse_progress(text, 1), (phase_sequence(1)[:2], None)) + with self.assertRaises(ValueError): + parse_progress("PHASE|0|connected\n", 1) + with self.assertRaises(ValueError): + parse_progress("PHASE|0|connecting\nPHASE|0|connecting\n", 1) + + def test_live_sample_is_only_attributed_to_a_stable_phase(self): + connecting = phase_sequence(1)[:1] + connected = phase_sequence(1)[:2] + self.assertEqual(stable_phase(connected, connected), (0, "connected")) + self.assertIsNone(stable_phase(connecting, connected)) + self.assertIsNone(stable_phase([], connecting)) + + def test_live_incomplete_cleanup_cannot_pass(self): + result = {"protocol": "tcp", "connections": 4, "cycles": 3, + "attempted_connections": 12, "connected": 12, "disconnected": 12, + "messages": 24, "errors": 0, "elapsed_ms": 3000} + validate_result(result, "tcp", 4, 3) + for key, value in (("disconnected", 11), ("errors", 1), ("messages", 0)): + with self.assertRaises(ValueError): + validate_result({**result, key: value}, "tcp", 4, 3) + + def test_missing_cgroup_controller_still_removes_owned_empty_group(self): + with tempfile.TemporaryDirectory(prefix="dever-perf-test-") as directory: + budget = Budget(Path(directory), None, None) + owned = budget.path + with self.assertRaises(FileNotFoundError): + budget.close() + self.assertFalse(owned.exists()) + + def test_dropped_and_failed_requests_cannot_disappear_from_report(self): + result = {"scheduled": 10, "sent": 4, "completed": 4, "succeeded": 0, + "errors": 4, "dropped": 6, "dropped_capacity": 5, "dispatch_expired": 1, + "latency_samples": 0, "request_latency_samples": 0, "dispatch_lag_samples": 0, + "latency_ms": None, "request_latency_ms": None, "dispatch_lag_ms": None, + "elapsed_seconds": 1, "actual_success_qps": 0, "http_version": "h2", + "physical_connections": 1, "streams_per_connection": 16, + "request_slots": 16, "successful_handshakes": 1} + validate_load(result) + with self.assertRaises(ValueError): + validate_load({**result, "errors": 3}) + with self.assertRaises(ValueError): + validate_load({**result, "latency_ms": {"p50": 0, "p95": 0, "p99": 0}}) + with self.assertRaises(ValueError): + validate_load({**result, "dispatch_lag_samples": 1, + "dispatch_lag_ms": {"sample_count": 2, "p50": 0, + "p95": 0, "p99": 0}}) + with self.assertRaises(ValueError): + validate_load({**result, "request_slots": 15}) + with self.assertRaises(ValueError): + validate_load({**result, "successful_handshakes": 2}) + + def test_http2_run_fails_on_request_errors_or_oom(self): + require_no_request_errors({"errors": 0}, "load") + with self.assertRaises(RuntimeError): + require_no_request_errors({"errors": 1}, "load") + require_no_oom(None) + require_no_oom({"events": {"oom": 0, "oom_kill": 0, "oom_group_kill": 0}}) + with self.assertRaises(RuntimeError): + require_no_oom({"events": {"oom": 1, "oom_kill": 0, "oom_group_kill": 0}}) + + def test_orm_http_requires_at_least_one_success_without_a_qps_threshold(self): + require_successful_requests({"succeeded": 1}, "ORM HTTP load") + with self.assertRaises(RuntimeError): + require_successful_requests({"succeeded": 0}, "ORM HTTP load") + + def test_http2_summary_keeps_topologies_separate(self): + def case(connections, repeat, qps, p99, rss): + return {"suite": "http2", "implementation": "dever", "transport": "http", + "path": "/plain", "rate": 5000.0, "physical_connections": connections, + "streams_per_connection": 16, "repeat": repeat, + "load": {"actual_success_qps": qps, "dropped": 0, + "request_latency_ms": {"p99": p99}}, + "resources": {"rss_bytes_median": rss, "pss_bytes_median": rss - 10, + "rss_high_water_bytes": rss + 10}} + + summary = http2_summary([case(1, 0, 90, 2, 100), case(1, 1, 110, 4, 120), + case(4, 0, 200, 1, 140)]) + self.assertEqual(len(summary), 2) + self.assertEqual(summary[0]["physical_connections"], 1) + self.assertEqual(summary[0]["actual_success_qps_median"], 100) + self.assertEqual(summary[0]["request_latency_p99_ms_median"], 3) + self.assertEqual(summary[0]["server_rss_bytes_median"], 110) + self.assertEqual(summary[1]["physical_connections"], 4) + + def test_warmup_and_checksum_are_checked_before_aggregation(self): + lines = ["SAMPLE|tasks|4|400000|480"] + lines += [f"SAMPLE|tasks|4|{nanos}|480" for nanos in range(40, 76, 4)] + result = async_measurements("\n".join(lines), {"iterations": 4, "pending": 2}) + self.assertEqual(result["median_ns_per_operation"], 14) + self.assertEqual(len(result["raw"]), 10) + with self.assertRaises(ValueError): + async_measurements("\n".join(lines).replace("|480", "|479"), {"iterations": 4}) + with self.assertRaises(ValueError): + async_measurements("\n".join(lines[:-1]), {"iterations": 4}) + + def test_cancel_reports_full_group_drain_and_has_no_fake_warmup(self): + result = async_measurements("SAMPLE|cancel|32|32000|32", {"pending": 32}) + self.assertFalse(result["warmup_samples"]) + self.assertEqual(result["median_ns_per_operation"], 1000) + + def test_percentile_uses_nearest_rank(self): + self.assertEqual(percentile([9, 2, 5, 1], 0.5), 2) + self.assertEqual(percentile([9, 2, 5, 1], 0.99), 9) + with self.assertRaises(argparse.ArgumentTypeError): + positive("nan") + + def test_http2_matrix_lists_reject_empty_duplicate_and_unknown_values(self): + self.assertEqual(csv_positive_ints("1,4", "connections"), (1, 4)) + self.assertEqual(parse_implementations("dever,hyper"), ("dever", "hyper")) + for value in ("", "1,", "1,1"): + with self.assertRaises(ValueError): + csv_positive_ints(value, "connections") + for value in ("", "dever,", "dever,dever", "dever,other"): + with self.assertRaises(ValueError): + parse_implementations(value) + + def test_client_cpu_configuration_rejects_invalid_sets_and_placement(self): + self.assertEqual(parse_client_cpus("0,1,2"), (0, 1, 2)) + for value in ("", "1,", "1,,2", "1,1", "-1", "other"): + with self.subTest(value=value), self.assertRaises(ValueError): + parse_client_cpus(value) + args = parser().parse_args(["run", "--artifacts", "artifacts", "--output", "output", + "--client-workers", "4", "--client-cpus", "1,2,3,4"]) + self.assertEqual((args.client_workers, args.client_cpus), (4, (1, 2, 3, 4))) + with self.assertRaises(ValueError): + validate_cpu_placement(0, (1, 5), {0, 1, 2, 3, 4}) + with self.assertRaises(ValueError): + validate_cpu_placement(0, (0, 1), {0, 1, 2, 3, 4}) + with self.assertRaises(ValueError): + validate_cpu_placement(-1, (1,), {0, 1}) + validate_cpu_placement(0, (1, 2), {0, 1, 2}) + + def test_process_affinity_is_applied_only_to_owned_child(self): + allowed = os.sched_getaffinity(0) + cpus = tuple(sorted(allowed)[:2]) + with tempfile.TemporaryDirectory(prefix="dever-perf-test-") as directory: + command = [sys.executable, "-c", + "import os; print(sorted(os.sched_getaffinity(0)), flush=True)"] + with Process(command, Path(directory) / "affinity", cpus=cpus) as process: + self.assertEqual(process.child.wait(timeout=5), 0) + process.check() + self.assertEqual(json.loads(process.text()), list(cpus)) + self.assertEqual(os.sched_getaffinity(0), allowed) + + def test_http_body_size_metadata_matches_fixtures(self): + self.assertEqual(HTTP_BODY_BYTES, {"/plain": 5, "/json": 11, "/bytes": 65_536}) + + def test_live_process_sampling_and_owned_cleanup(self): + with tempfile.TemporaryDirectory(prefix="dever-perf-test-") as directory: + command = [sys.executable, "-c", "import time; print('READY|0', flush=True); time.sleep(10)"] + with Process(command, Path(directory) / "process") as process: + self.assertEqual(process.ready(), 0) + observe([process], 0.04, 0.01) + result = summarize(process.samples) + self.assertGreater(result["rss_bytes_median"], 0) + self.assertGreater(result["pss_bytes_median"], 0) + self.assertGreaterEqual(result["threads_median"], 1) + self.assertGreaterEqual(result["fds_median"], 3) + self.assertIsNotNone(process.child.poll()) + + def test_early_exit_does_not_become_successful_readiness(self): + with tempfile.TemporaryDirectory(prefix="dever-perf-test-") as directory: + with Process([sys.executable, "-c", "raise SystemExit(7)"], Path(directory) / "failed") as process: + with self.assertRaises(RuntimeError): + process.ready() + + def test_memory_teardown_can_precede_waitpid_exit(self): + with tempfile.TemporaryDirectory(prefix="dever-perf-test-") as directory: + command = [sys.executable, "-c", "import time; print('READY|0', flush=True); time.sleep(10)"] + with Process(command, Path(directory) / "teardown") as process: + process.ready() + count = len(process.samples) + with patch("process.process_sample", side_effect=ProcessLookupError): + process.sample() + self.assertEqual(len(process.samples), count) + self.assertIsNone(process.child.poll()) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/runtime_entry_cleanup.rs b/test/runtime_entry_cleanup.rs new file mode 100644 index 0000000..e396c7c --- /dev/null +++ b/test/runtime_entry_cleanup.rs @@ -0,0 +1,134 @@ +//! Application resources close after child drain, before the runtime is gone. + +use dever_runtime::task::{self, RuntimeConfig}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +struct DropWitness(Arc<AtomicBool>); + +impl Drop for DropWitness { + fn drop(&mut self) { + self.0.store(true, Ordering::Release); + } +} + +fn config() -> RuntimeConfig { + RuntimeConfig { + worker_threads: 1, + max_blocking_threads: 2, + task_capacity: 4, + } +} + +#[test] +fn cleanup_runs_after_pending_child_release_inside_the_same_runtime() { + let dropped = Arc::new(AtomicBool::new(false)); + let witness = DropWitness(Arc::clone(&dropped)); + let result = task::run_entry_with_typed_cleanup( + config(), + async move { + let child = task::run(async move { + let _witness = witness; + std::future::pending::<Result<(), String>>().await + }) + .await?; + tokio::task::yield_now().await; + // Return the handle: the child is still live until root Scope drain. + Ok::<_, String>(child) + }, + |result| async { + assert!(dropped.load(Ordering::Acquire)); + assert!(tokio::runtime::Handle::try_current().is_ok()); + tokio::time::sleep(Duration::from_millis(1)).await; + result + }, + ); + assert!(result.is_ok()); + drop(result); +} + +#[test] +fn entry_and_cleanup_keep_borrowed_non_send_state_across_suspension() { + let phase = std::rc::Rc::new(std::cell::Cell::new(0)); + let label = String::from("borrowed result"); + let result = task::run_entry_with_typed_cleanup( + config(), + async { + let phase = std::rc::Rc::clone(&phase); + tokio::task::yield_now().await; + phase.set(1); + Ok::<_, String>(label.as_str()) + }, + |result| async { + let phase = std::rc::Rc::clone(&phase); + tokio::task::yield_now().await; + assert_eq!(phase.get(), 1); + phase.set(2); + result + }, + ); + assert_eq!(result.unwrap(), label); + assert_eq!(phase.get(), 2); +} + +#[derive(Debug, Eq, PartialEq)] +enum Failure { + Business { message: String }, + Runtime(String), +} + +impl From<String> for Failure { + fn from(message: String) -> Self { + Self::Runtime(message) + } +} + +#[test] +fn cleanup_receives_and_preserves_the_original_typed_failure() { + let expected = Failure::Business { + message: "original owned payload".into(), + }; + let result = task::run_entry_with_typed_cleanup( + config(), + async { Err::<(), _>(expected) }, + |result| async { + assert_eq!( + result, + Err(Failure::Business { + message: "original owned payload".into(), + }) + ); + tokio::time::sleep(Duration::from_millis(1)).await; + result + }, + ); + assert_eq!( + result, + Err(Failure::Business { + message: "original owned payload".into(), + }) + ); +} + +#[test] +fn rejected_nested_entry_does_not_run_body_or_cleanup() { + let body_started = AtomicBool::new(false); + let cleanup_started = AtomicBool::new(false); + let result = task::run_entry(async { + task::run_entry_with_typed_cleanup( + config(), + async { + body_started.store(true, Ordering::Relaxed); + Ok::<(), String>(()) + }, + |result| async { + cleanup_started.store(true, Ordering::Relaxed); + result + }, + ) + }); + assert!(result.unwrap_err().contains("inside another runtime")); + assert!(!body_started.load(Ordering::Relaxed)); + assert!(!cleanup_started.load(Ordering::Relaxed)); +} diff --git a/test/runtime_logging.rs b/test/runtime_logging.rs new file mode 100644 index 0000000..d5ac4ac --- /dev/null +++ b/test/runtime_logging.rs @@ -0,0 +1,170 @@ +use super::*; +use std::sync::mpsc; +use std::time::Duration; + +fn record(level: Level, message: impl Into<String>) -> Record { + Record { + level, + message: message.into(), + fields: Vec::new(), + dropped: 0, + } +} + +fn next_record(logger: &Logger) -> Record { + match logger.receive() { + Some(Command::Record(record)) => record, + _ => panic!("expected a queued record"), + } +} + +#[test] +fn bounded_queue_keeps_fifo_and_carries_dropped_count_to_next_record() { + let logger = Logger::new(); + for index in 0..QUEUE_CAPACITY { + logger.submit(record(Level::Info, index.to_string())); + } + logger.submit(record(Level::Debug, "discard debug")); + logger.submit(record(Level::Info, "discard info")); + assert_eq!(logger.queue().commands.len(), QUEUE_CAPACITY); + assert_eq!(logger.dropped.load(Ordering::Relaxed), 2); + assert_eq!(next_record(&logger).message, "0"); + logger.submit(record(Level::Info, "after overflow")); + for index in 1..QUEUE_CAPACITY { + assert_eq!(next_record(&logger).message, index.to_string()); + } + let recovered = next_record(&logger); + assert_eq!(recovered.message, "after overflow"); + assert_eq!(recovered.dropped, 2); + assert_eq!(logger.dropped.load(Ordering::Relaxed), 0); + let mut output = Vec::new(); + write_record(&mut output, &recovered).unwrap(); + assert_eq!( + String::from_utf8(output).unwrap(), + "{\"level\":\"info\",\"message\":\"after overflow\",\"dropped_events\":2,\"fields\":{}}\n" + ); +} + +#[test] +fn warn_and_error_wait_for_capacity_without_being_dropped() { + for level in [Level::Warn, Level::Error] { + let logger = Arc::new(Logger::new()); + for index in 0..QUEUE_CAPACITY { + logger.submit(record(Level::Info, index.to_string())); + } + let writer = Arc::clone(&logger); + let (started, starting) = mpsc::channel(); + let (finished, finishing) = mpsc::channel(); + let producer = std::thread::spawn(move || { + started.send(()).unwrap(); + writer.submit(record(level, "must survive")); + finished.send(()).unwrap(); + }); + starting.recv_timeout(Duration::from_secs(1)).unwrap(); + assert!(matches!( + finishing.recv_timeout(Duration::from_millis(20)), + Err(mpsc::RecvTimeoutError::Timeout) + )); + assert_eq!(next_record(&logger).message, "0"); + finishing.recv_timeout(Duration::from_secs(1)).unwrap(); + producer.join().unwrap(); + for index in 1..QUEUE_CAPACITY { + assert_eq!(next_record(&logger).message, index.to_string()); + } + let important = next_record(&logger); + assert_eq!(important.message, "must survive"); + assert_eq!(important.level, level); + assert_eq!(important.dropped, 0); + } +} + +#[derive(Clone, Default)] +struct CapturedOutput { + bytes: Arc<Mutex<Vec<u8>>>, + flushes: Arc<AtomicU64>, +} + +impl Write for CapturedOutput { + fn write(&mut self, bytes: &[u8]) -> io::Result<usize> { + self.bytes.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + self.flushes.fetch_add(1, Ordering::Relaxed); + Ok(()) + } +} + +#[test] +fn concurrent_flushes_acknowledge_each_callers_prior_record() { + let logger = Arc::new(Logger::new()); + let output = CapturedOutput::default(); + let writer = Arc::clone(&logger); + let mut captured = output.clone(); + let writing = std::thread::spawn(move || writer.write(&mut captured)); + std::thread::scope(|scope| { + for caller in 0..8 { + let logger = &logger; + let output = &output; + scope.spawn(move || { + for sequence in 0..8 { + let message = format!("caller-{caller}-record-{sequence}"); + logger.submit(record(Level::Warn, message.clone())); + logger.flush(); + let bytes = output.bytes.lock().unwrap(); + let text = std::str::from_utf8(&bytes).unwrap(); + assert!(text.contains(&format!("\"message\":\"{message}\""))); + } + }); + } + }); + assert_eq!(output.flushes.load(Ordering::Relaxed), 64); + logger.close(); + writing.join().unwrap(); +} + +struct PanickingOutput; + +impl Write for PanickingOutput { + fn write(&mut self, _: &[u8]) -> io::Result<usize> { + panic!("fixture writer stopped") + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +#[test] +fn writer_failure_releases_queue_and_wakes_producers_and_flushers() { + let logger = Arc::new(Logger::new()); + for index in 0..QUEUE_CAPACITY { + logger.submit(record(Level::Info, index.to_string())); + } + let (finished, finishing) = mpsc::channel(); + let mut waiting = Vec::new(); + for index in 0..3 { + let logger = Arc::clone(&logger); + let finished = finished.clone(); + waiting.push(std::thread::spawn(move || { + if index == 0 { + logger.submit(record(Level::Warn, "writer closed")); + } else { + logger.flush(); + } + finished.send(()).unwrap(); + })); + } + let writer = Arc::clone(&logger); + let writing = std::thread::spawn(move || writer.write(&mut PanickingOutput)); + assert!(writing.join().is_err()); + for _ in 0..3 { + finishing.recv_timeout(Duration::from_secs(1)).unwrap(); + } + for thread in waiting { + thread.join().unwrap(); + } + let queue = logger.queue(); + assert!(queue.closed); + assert!(queue.commands.is_empty()); +}